From af1096522c0bd5ec03ce4cbeb4183aa582cf3b04 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 1 Nov 2024 15:19:02 +0800 Subject: [PATCH 01/19] set default compress --- source/common/src/tcol.c | 16 ++++++++++++++++ tests/script/tsim/compress/compress2.sim | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/source/common/src/tcol.c b/source/common/src/tcol.c index 84027c25b6..07a76e5785 100644 --- a/source/common/src/tcol.c +++ b/source/common/src/tcol.c @@ -81,26 +81,42 @@ const char* getDefaultEncodeStr(uint8_t type) { return columnEncodeStr(getDefaul uint16_t getDefaultCompress(uint8_t type) { switch (type) { case TSDB_DATA_TYPE_NULL: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_BOOL: + return TSDB_COLVAL_COMPRESS_ZSTD; case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: + return TSDB_COLVAL_COMPRESS_ZLIB; case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY + return TSDB_COLVAL_COMPRESS_ZSTD; case TSDB_DATA_TYPE_TIMESTAMP: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_NCHAR: + return TSDB_COLVAL_COMPRESS_ZSTD; case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_USMALLINT: + return TSDB_COLVAL_COMPRESS_ZLIB; case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_JSON: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_VARBINARY: + return TSDB_COLVAL_COMPRESS_ZSTD; case TSDB_DATA_TYPE_DECIMAL: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_BLOB: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_MEDIUMBLOB: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_GEOMETRY: + return TSDB_COLVAL_COMPRESS_LZ4; case TSDB_DATA_TYPE_MAX: return TSDB_COLVAL_COMPRESS_LZ4; default: diff --git a/tests/script/tsim/compress/compress2.sim b/tests/script/tsim/compress/compress2.sim index 0af6f87de4..179317dfbb 100644 --- a/tests/script/tsim/compress/compress2.sim +++ b/tests/script/tsim/compress/compress2.sim @@ -38,7 +38,7 @@ sql alter table $tb modify column b level 'm' sql_error alter table $tb modify column b level 'l' # already exist -sql_error alter table $tb modify column b compress 'lz4' +sql alter table $tb modify column b compress 'lz4' sql alter table $tb modify column b compress 'xz' sql alter table $tb modify column b compress 'zstd' sql_error alter table $tb modify column b compress 'tsz' @@ -147,7 +147,7 @@ sql alter table $stb modify column b level 'm' sql_error alter table $stb modify column b level 'l' # already exist sql desc $stb -sql_error alter table $stb modify column b compress 'lz4' +sql alter table $stb modify column b compress 'lz4' sql alter table $stb modify column b compress 'xz' sql alter table $stb modify column b compress 'zstd' sql_error alter table $stb modify column b compress 'tsz' From 1c940b23353d330cc59ca2f92b7964fa442aec77 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 14 Nov 2024 19:46:21 +0800 Subject: [PATCH 02/19] fix test case --- tests/army/storage/compressBasic.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/army/storage/compressBasic.py b/tests/army/storage/compressBasic.py index f24c4dd288..260461d0e4 100644 --- a/tests/army/storage/compressBasic.py +++ b/tests/army/storage/compressBasic.py @@ -137,7 +137,18 @@ class TDTestCase(TBase): defEncodes = [ "delta-i","delta-i","simple8b","simple8b","simple8b","simple8b","simple8b","simple8b", "simple8b","simple8b","delta-d","delta-d","bit-packing", "disabled","disabled","disabled","disabled"] - + compressDefaultDict = {}; + compressDefaultDict["BOOL"] = "zstd" + compressDefaultDict["TINYINT"] = "zlib" + compressDefaultDict["SMALLINT"] = "zlib" + compressDefaultDict["INT"] = "lz4" + compressDefaultDict["BIGINT"] = "lz4" + compressDefaultDict["FLOAT"] = "lz4" + compressDefaultDict["DOUBLE"] = "lz4" + compressDefaultDict["VARCHAR"] = "lz4" + compressDefaultDict["TIMESTAMP"] = "zstd" + compressDefaultDict["NCHAR"] = "zstd" + count = tdSql.getRows() for i in range(count): node = tdSql.getData(i, 3) @@ -145,7 +156,7 @@ class TDTestCase(TBase): break # check tdSql.checkData(i, 4, defEncodes[i]) - tdSql.checkData(i, 5, self.defCompress) + tdSql.checkData(i, 5, compressDefaultDict[tdSql.getData(i, 2)]) tdSql.checkData(i, 6, self.defLevel) # geometry encode is disabled From 72917aca9c5751f9e2442e03589c391555ff9277 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 14 Nov 2024 04:52:15 -0800 Subject: [PATCH 03/19] update case --- tests/army/storage/compressBasic.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/tests/army/storage/compressBasic.py b/tests/army/storage/compressBasic.py index 260461d0e4..5882c9ffbd 100644 --- a/tests/army/storage/compressBasic.py +++ b/tests/army/storage/compressBasic.py @@ -33,7 +33,7 @@ class TDTestCase(TBase): "compressMsgSize" : "100", } # compress - compresses = ["lz4","tsz","zlib","zstd","disabled","xz"] + compresses = ["lz4","zlib","zstd","disabled","xz"] # level levels = ["high","medium","low"] @@ -145,9 +145,16 @@ class TDTestCase(TBase): compressDefaultDict["BIGINT"] = "lz4" compressDefaultDict["FLOAT"] = "lz4" compressDefaultDict["DOUBLE"] = "lz4" - compressDefaultDict["VARCHAR"] = "lz4" - compressDefaultDict["TIMESTAMP"] = "zstd" + compressDefaultDict["VARCHAR"] = "zstd" + compressDefaultDict["TIMESTAMP"] = "lz4" compressDefaultDict["NCHAR"] = "zstd" + compressDefaultDict["TINYINT UNSIGNED"] = "zlib" + compressDefaultDict["SMALLINT UNSIGNED"] = "zlib" + compressDefaultDict["INT UNSIGNED"] = "lz4" + compressDefaultDict["BIGINT UNSIGNED"] = "lz4" + compressDefaultDict["NCHAR"] = "zstd" + compressDefaultDict["BLOB"] = "lz4" + compressDefaultDict["VARBINARY"] = "zstd" count = tdSql.getRows() for i in range(count): @@ -155,8 +162,13 @@ class TDTestCase(TBase): if node == "TAG": break # check - tdSql.checkData(i, 4, defEncodes[i]) - tdSql.checkData(i, 5, compressDefaultDict[tdSql.getData(i, 2)]) + tdLog.info(f"check default encode {tdSql.getData(i, 1)}") + #tdLog.info(f"check default encode compressDefaultDict[tdSql.getData(i, 2)]") + defaultValue = compressDefaultDict[tdSql.getData(i, 1)] + if defaultValue == None: + defaultValue = self.defCompress + tdLog.info(f"check default compress {tdSql.getData(i, 1)} {defaultValue}") + tdSql.checkData(i, 5, defaultValue) tdSql.checkData(i, 6, self.defLevel) # geometry encode is disabled @@ -198,8 +210,8 @@ class TDTestCase(TBase): for i in range(self.colCnt - 1): col = f"c{i}" sql = f"alter table {tbname} modify column {col} COMPRESS '{comp}';" - tdSql.execute(sql, show=True) - self.checkDataDesc(tbname, i + 1, 5, comp) + #tdSql.execute(sql, show=False) + #self.checkDataDesc(tbname, i + 1, 5, comp) self.writeData(1000) # alter float(c9) double(c10) to tsz @@ -337,6 +349,7 @@ class TDTestCase(TBase): while offset < count: sql = f"select * from {tbname} limit {step} offset {offset}" + tdLog.info(sql) tdSql.query(sql) self.autoGen.dataCorrect(tdSql.res, tdSql.getRows(), step) offset += step From ded0e954ba317285d23b38da070c760287f6bd3c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 14 Nov 2024 21:12:19 +0800 Subject: [PATCH 04/19] update case --- utils/test/c/tmq_taosx_ci.c | 56 ++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index cd70dd88f5..370b18ae68 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -65,12 +65,12 @@ static void msg_process(TAOS_RES* msg) { if (g_fp && strcmp(result, "") != 0) { // RES_TYPE__TMQ_BATCH_META if ((*(int8_t*)msg) == 5) { - cJSON* pJson = cJSON_Parse(result); - cJSON* pJsonArray = cJSON_GetObjectItem(pJson, "metas"); + cJSON* pJson = cJSON_Parse(result); + cJSON* pJsonArray = cJSON_GetObjectItem(pJson, "metas"); int32_t num = cJSON_GetArraySize(pJsonArray); for (int32_t i = 0; i < num; i++) { cJSON* pJsonItem = cJSON_GetArrayItem(pJsonArray, i); - char* itemStr = cJSON_PrintUnformatted(pJsonItem); + char* itemStr = cJSON_PrintUnformatted(pJsonItem); taosFprintfFile(g_fp, itemStr); tmq_free_json_meta(itemStr); taosFprintfFile(g_fp, "\n"); @@ -489,10 +489,11 @@ int buildStable(TAOS* pConn, TAOS_RES* pRes) { } taos_free_result(pRes); #else - pRes = taos_query(pConn, - "create stream meters_summary_s trigger at_once IGNORE EXPIRED 0 fill_history 1 into meters_summary as select " - "_wstart, max(current) as current, " - "groupid, location from meters partition by groupid, location interval(10m)"); + pRes = taos_query( + pConn, + "create stream meters_summary_s trigger at_once IGNORE EXPIRED 0 fill_history 1 into meters_summary as select " + "_wstart, max(current) as current, " + "groupid, location from meters partition by groupid, location interval(10m)"); if (taos_errno(pRes) != 0) { printf("failed to create super table meters_summary, reason:%s\n", taos_errstr(pRes)); return -1; @@ -632,8 +633,8 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "auto.offset.reset", "earliest"); tmq_conf_set(conf, "msg.consume.excluded", "1"); -// tmq_conf_set(conf, "session.timeout.ms", "1000000"); -// tmq_conf_set(conf, "max.poll.interval.ms", "20000"); + // tmq_conf_set(conf, "session.timeout.ms", "1000000"); + // tmq_conf_set(conf, "max.poll.interval.ms", "20000"); if (g_conf.snapShot) { tmq_conf_set(conf, "experimental.snapshot.enable", "true"); @@ -722,7 +723,7 @@ void initLogFile() { "\"level\":\"medium\"},{" "\"name\":\"groupid\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\"," "\"level\":\"medium\"},{\"name\":" - "\"location\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\"," + "\"location\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\"," "\"level\":\"medium\"}],\"tags\":[{\"name\":\"group_id\"," "\"type\":14}" "]}", @@ -750,7 +751,7 @@ void initLogFile() { "\"level\":\"medium\"}" ",{" "\"name\":\"c3\",\"type\":8,\"length\":64,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":" - "\"lz4\",\"level\":\"medium\"},{" + "\"zstd\",\"level\":\"medium\"},{" "\"name\":\"c4\",\"type\":5,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":" "\"medium\"}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":" "\"t3\"," @@ -772,7 +773,7 @@ void initLogFile() { "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\"," "\"type\":9," "\"isPrimarykey\":false,\"encode\":\"delta-i\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c2\"," - "\"type\":10,\"length\":8,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\",\"level\":" + "\"type\":10,\"length\":8,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":" "\"medium\"},{\"name\":\"cc3\",\"type\":5," "\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":\"medium\"}],\"tags\":[]}", "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\"," @@ -794,7 +795,7 @@ void initLogFile() { "\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":" "false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{" "\"name\":\"c3\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":" - "\"lz4\",\"level\":\"medium\"}]," + "\"zstd\",\"level\":\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":" "1}]}", @@ -806,7 +807,7 @@ void initLogFile() { "\"name\":\"c2\",\"type\":6," "\"isPrimarykey\":false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":" "\"c3\"," - "\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\",\"level\":" + "\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":" "\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":" @@ -863,7 +864,7 @@ void initLogFile() { "\"level\":\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":false,\"encode\":\"delta-d\"," "\"compress\":\"lz4\",\"level\":\"medium\"},{" "\"name\":\"c3\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":" - "\"lz4\",\"level\":\"medium\"}]," + "\"zstd\",\"level\":\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":" "1}]}", @@ -883,7 +884,8 @@ void initLogFile() { "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\"," "\"colType\":5}", "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":6,\"colName\":\"c4\"}", - "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\"," + "\"colType\":5}", "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\"," "\"colType\":8,\"colLength\":64}", "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\"," @@ -896,7 +898,7 @@ void initLogFile() { "9,\"isPrimarykey\":false,\"encode\":\"delta-i\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":" "\"c1\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":" "\"medium\"},{\"name\":\"c2\",\"type\":10,\"length\":4," - "\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\",\"level\":\"medium\"}],\"tags\":[]}", + "\"isPrimarykey\":false,\"encode\":\"disabled\",\"zstd\":\"lz4\",\"level\":\"medium\"}],\"tags\":[]}", "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\"," "\"colType\":5}", "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\"," @@ -921,7 +923,7 @@ void initLogFile() { "{\"name\":\"c1\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":" "\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":" "false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c3\",\"type\":8," - "\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\",\"level\":\"medium\"}]," + "\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":1}]}", "{\"type\":\"drop\",\"tableType\":\"super\",\"tableName\":\"st1\"}", @@ -931,7 +933,7 @@ void initLogFile() { "\"level\":\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":" "false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{" "\"name\":\"c3\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":" - "\"lz4\",\"level\":\"medium\"}]," + "\"zstd\",\"level\":\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":" "1}]}", @@ -941,7 +943,7 @@ void initLogFile() { "\"c1\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":" "\"medium\"},{\"name\":\"c2\",\"type\":6," "\"isPrimarykey\":false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c3\"," - "\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\",\"level\":" + "\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":" "\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":" @@ -985,7 +987,8 @@ void initLogFile() { "\"lz4\",\"level\":\"medium\"},{" "\"name\":\"groupid\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\"," "\"level\":\"medium\"},{\"name\":" - "\"location\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\"," + "\"location\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":" + "\"zstd\"," "\"level\":\"medium\"}],\"tags\":[{\"name\":\"group_id\"," "\"type\":" "14}]}", @@ -1012,7 +1015,7 @@ void initLogFile() { "\"level\":\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":" "false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{" "\"name\":\"c3\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":" - "\"lz4\",\"level\":\"medium\"}]," + "\"zstd\",\"level\":\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":" "1}]}", @@ -1032,7 +1035,8 @@ void initLogFile() { "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\"," "\"colType\":5}", "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":6,\"colName\":\"c4\"}", - "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\"," + "\"colType\":5}", "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\"," "\"colType\":8,\"colLength\":64}", "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\"," @@ -1044,7 +1048,7 @@ void initLogFile() { "9,\"isPrimarykey\":false,\"encode\":\"delta-i\",\"compress\":\"lz4\",\"level\":\"medium\"}" ",{\"name\":\"c1\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\"," "\"level\":\"medium\"},{\"name\":\"c2\",\"type\":10,\"length\":4," - "\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\",\"level\":\"medium\"}],\"tags\":[]}", + "\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":\"medium\"}],\"tags\":[]}", "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\"," "\"colType\":5}", "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\"," @@ -1069,7 +1073,7 @@ void initLogFile() { "\"level\":\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":" "false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{" "\"name\":\"c3\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":" - "\"lz4\",\"level\":\"medium\"}]," + "\"zstd\",\"level\":\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":" "1}]}", @@ -1079,7 +1083,7 @@ void initLogFile() { "\"c1\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":" "\"medium\"},{\"name\":\"c2\",\"type\":6," "\"isPrimarykey\":false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c3\"," - "\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"lz4\",\"level\":" + "\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":" "\"medium\"}]," "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\"," "\"type\":" From f9d67724d92027082afc7d1306cdbce7783cd7c3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 14 Nov 2024 22:20:25 +0800 Subject: [PATCH 05/19] fix test case --- utils/test/c/tmq_taosx_ci.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 370b18ae68..117f9fa2e1 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -898,7 +898,7 @@ void initLogFile() { "9,\"isPrimarykey\":false,\"encode\":\"delta-i\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":" "\"c1\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":" "\"medium\"},{\"name\":\"c2\",\"type\":10,\"length\":4," - "\"isPrimarykey\":false,\"encode\":\"disabled\",\"zstd\":\"lz4\",\"level\":\"medium\"}],\"tags\":[]}", + "\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":\"medium\"}],\"tags\":[]}", "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\"," "\"colType\":5}", "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\"," From 1a4e1c9d9e972805317d9099ec72d246348d9e86 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 15 Nov 2024 08:44:59 +0800 Subject: [PATCH 06/19] update tese case --- tests/army/storage/compressBasic.py | 51 ++++++++++++++++------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/tests/army/storage/compressBasic.py b/tests/army/storage/compressBasic.py index 5882c9ffbd..fc9c9d11fb 100644 --- a/tests/army/storage/compressBasic.py +++ b/tests/army/storage/compressBasic.py @@ -35,6 +35,25 @@ class TDTestCase(TBase): # compress compresses = ["lz4","zlib","zstd","disabled","xz"] + compressDefaultDict = {}; + compressDefaultDict["BOOL"] = "zstd" + compressDefaultDict["TINYINT"] = "zlib" + compressDefaultDict["SMALLINT"] = "zlib" + compressDefaultDict["INT"] = "lz4" + compressDefaultDict["BIGINT"] = "lz4" + compressDefaultDict["FLOAT"] = "lz4" + compressDefaultDict["DOUBLE"] = "lz4" + compressDefaultDict["VARCHAR"] = "zstd" + compressDefaultDict["TIMESTAMP"] = "lz4" + compressDefaultDict["NCHAR"] = "zstd" + compressDefaultDict["TINYINT UNSIGNED"] = "zlib" + compressDefaultDict["SMALLINT UNSIGNED"] = "zlib" + compressDefaultDict["INT UNSIGNED"] = "lz4" + compressDefaultDict["BIGINT UNSIGNED"] = "lz4" + compressDefaultDict["NCHAR"] = "zstd" + compressDefaultDict["BLOB"] = "lz4" + compressDefaultDict["VARBINARY"] = "zstd" + # level levels = ["high","medium","low"] @@ -137,24 +156,6 @@ class TDTestCase(TBase): defEncodes = [ "delta-i","delta-i","simple8b","simple8b","simple8b","simple8b","simple8b","simple8b", "simple8b","simple8b","delta-d","delta-d","bit-packing", "disabled","disabled","disabled","disabled"] - compressDefaultDict = {}; - compressDefaultDict["BOOL"] = "zstd" - compressDefaultDict["TINYINT"] = "zlib" - compressDefaultDict["SMALLINT"] = "zlib" - compressDefaultDict["INT"] = "lz4" - compressDefaultDict["BIGINT"] = "lz4" - compressDefaultDict["FLOAT"] = "lz4" - compressDefaultDict["DOUBLE"] = "lz4" - compressDefaultDict["VARCHAR"] = "zstd" - compressDefaultDict["TIMESTAMP"] = "lz4" - compressDefaultDict["NCHAR"] = "zstd" - compressDefaultDict["TINYINT UNSIGNED"] = "zlib" - compressDefaultDict["SMALLINT UNSIGNED"] = "zlib" - compressDefaultDict["INT UNSIGNED"] = "lz4" - compressDefaultDict["BIGINT UNSIGNED"] = "lz4" - compressDefaultDict["NCHAR"] = "zstd" - compressDefaultDict["BLOB"] = "lz4" - compressDefaultDict["VARBINARY"] = "zstd" count = tdSql.getRows() for i in range(count): @@ -164,7 +165,7 @@ class TDTestCase(TBase): # check tdLog.info(f"check default encode {tdSql.getData(i, 1)}") #tdLog.info(f"check default encode compressDefaultDict[tdSql.getData(i, 2)]") - defaultValue = compressDefaultDict[tdSql.getData(i, 1)] + defaultValue = self.compressDefaultDict[tdSql.getData(i, 1)] if defaultValue == None: defaultValue = self.defCompress tdLog.info(f"check default compress {tdSql.getData(i, 1)} {defaultValue}") @@ -209,9 +210,15 @@ class TDTestCase(TBase): for comp in comps: for i in range(self.colCnt - 1): col = f"c{i}" - sql = f"alter table {tbname} modify column {col} COMPRESS '{comp}';" - #tdSql.execute(sql, show=False) - #self.checkDataDesc(tbname, i + 1, 5, comp) + sql2= f"desc {tbname}" + tdSql.execute(sql2, show=True) + + defaultValue = self.compressDefaultDict[tdSql.getData(i, 1)] + if defaultValue == None: + defaultValue = self.defCompress + if defaultValue != comp: + tdSql.execute(sql, show=False) + self.checkDataDesc(tbname, i + 1, 5, comp) self.writeData(1000) # alter float(c9) double(c10) to tsz From 2a83a7d263cced3be43d2376ea31a363d30ea5c6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 15 Nov 2024 08:49:36 +0800 Subject: [PATCH 07/19] update tese case --- tests/army/storage/compressBasic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/army/storage/compressBasic.py b/tests/army/storage/compressBasic.py index fc9c9d11fb..565bd312a6 100644 --- a/tests/army/storage/compressBasic.py +++ b/tests/army/storage/compressBasic.py @@ -217,6 +217,7 @@ class TDTestCase(TBase): if defaultValue == None: defaultValue = self.defCompress if defaultValue != comp: + sql = f"alter table {tbname} modify column {col} COMPRESS '{comp}';" tdSql.execute(sql, show=False) self.checkDataDesc(tbname, i + 1, 5, comp) self.writeData(1000) From c1a610fd1969dfbbc79a899b84f318e5d5bf30d0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 15 Nov 2024 09:17:13 +0800 Subject: [PATCH 08/19] update test case --- tests/army/storage/compressBasic.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/army/storage/compressBasic.py b/tests/army/storage/compressBasic.py index 565bd312a6..446cb920fb 100644 --- a/tests/army/storage/compressBasic.py +++ b/tests/army/storage/compressBasic.py @@ -209,17 +209,6 @@ class TDTestCase(TBase): comps.append(self.compresses[0]) # add lz4 for comp in comps: for i in range(self.colCnt - 1): - col = f"c{i}" - sql2= f"desc {tbname}" - tdSql.execute(sql2, show=True) - - defaultValue = self.compressDefaultDict[tdSql.getData(i, 1)] - if defaultValue == None: - defaultValue = self.defCompress - if defaultValue != comp: - sql = f"alter table {tbname} modify column {col} COMPRESS '{comp}';" - tdSql.execute(sql, show=False) - self.checkDataDesc(tbname, i + 1, 5, comp) self.writeData(1000) # alter float(c9) double(c10) to tsz From d2ec0c37ac7f5d7b2ba156d13063b1a129673f0e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 15 Nov 2024 14:01:16 +0800 Subject: [PATCH 09/19] doc: update compress readme --- docs/en/14-reference/03-taos-sql/31-compress.md | 7 ++++--- docs/zh/14-reference/03-taos-sql/32-compress.md | 9 +++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/en/14-reference/03-taos-sql/31-compress.md b/docs/en/14-reference/03-taos-sql/31-compress.md index 39abfe69bd..f726c8bbe6 100644 --- a/docs/en/14-reference/03-taos-sql/31-compress.md +++ b/docs/en/14-reference/03-taos-sql/31-compress.md @@ -30,11 +30,12 @@ In this article, it specifically refers to the level within the secondary compre | Data Type | Optional Encoding Algorithm | Default Encoding Algorithm | Optional Compression Algorithm|Default Compression Algorithm| Default Compression Level| | :-----------:|:----------:|:-------:|:-------:|:----------:|:----:| -| tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| +| int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| +| tinyint/untinyint/smallint/usmallint | simple8b| simple8b | lz4/zlib/zstd/xz| zlib| medium| | bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium| |float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium| -|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium| -|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium| +|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| zstd| +|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| zstd| ## SQL diff --git a/docs/zh/14-reference/03-taos-sql/32-compress.md b/docs/zh/14-reference/03-taos-sql/32-compress.md index 0f2b260832..51650c9123 100644 --- a/docs/zh/14-reference/03-taos-sql/32-compress.md +++ b/docs/zh/14-reference/03-taos-sql/32-compress.md @@ -31,11 +31,12 @@ description: 可配置压缩算法 | 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法|压缩算法默认值| 压缩等级默认值| | :-----------:|:----------:|:-------:|:-------:|:----------:|:----:| -| tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| -| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium| +| int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| +| tinyint/untinyint/smallint/usmallint | simple8b| simple8b | lz4/zlib/zstd/xz| zlib | medium| +| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium| |float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium| -|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium| -|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium| +|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| zstd| medium| +|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| zstd| medium| ## SQL 语法 From 9fafdd8c47b4d62e4c72840b472cade118d2f956 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 16 Nov 2024 14:34:00 +0800 Subject: [PATCH 10/19] Update index.md minor change --- docs/zh/06-advanced/06-TDgpt/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/zh/06-advanced/06-TDgpt/index.md b/docs/zh/06-advanced/06-TDgpt/index.md index 6c7a2534c3..dde7014a0d 100644 --- a/docs/zh/06-advanced/06-TDgpt/index.md +++ b/docs/zh/06-advanced/06-TDgpt/index.md @@ -15,6 +15,7 @@ TDgpt 运行在集群中的 AI Node (Anode)中，集群中可以部署若干个 在查询处理过程中，Vnode中运行的查询引擎会根据查询处理物理执行计划，按需向 Anode 请求高级时序数据分析服务。因此用户可通过 SQL 语句与 Anode 节点交互并使用其提供的全部分析服务。需要注意的是 Anode 不直接接受用户的数据分析请求。同时 Anode 具备分析算法动态注册机制，其算法扩展过程完全不影响 TDengine 集群的服务，仅在非常小的（秒级）时间窗口内影响涉及高级分析的查询服务。 +目前 TDgpt 提供如下的高级分析服务： - 时序数据异常检测。TDengine 中定义了新的时间窗口——异常（状态）窗口——来提供异常检测服务。异常窗口可以视为一种特殊的**事件窗口（Event Window）**，即异常检测算法确定的连续异常时间序列数据所在的时间窗口。与普通事件窗口区别在于——时间窗口的起始时间和结束时间均是分析算法确定，不是用户指定的表达式判定。异常窗口使用方式与其他类型的时间窗口（例如状态窗口、会话窗口等）类似。因此时间窗口内可使用的查询操作均可应用在异常窗口上。 - 时序数据预测。定义了一个新函数`FORECAST`，基于输入的（历史）时间序列数据调用指定（或默认）预测算法给出输入时序数据后续时间序列的**预测**数据。 From 8c401e3f84ea789aa5335e0d6e8a5d8b71e8fa3a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 16 Nov 2024 14:45:58 +0800 Subject: [PATCH 11/19] Update 02-management.md --- docs/zh/06-advanced/06-TDgpt/02-management.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/zh/06-advanced/06-TDgpt/02-management.md b/docs/zh/06-advanced/06-TDgpt/02-management.md index 07f1136549..51d0aafa0a 100644 --- a/docs/zh/06-advanced/06-TDgpt/02-management.md +++ b/docs/zh/06-advanced/06-TDgpt/02-management.md @@ -4,7 +4,7 @@ sidebar_label: "安装部署" --- ### 环境准备 -使用 TDgpt 的高级时序数据分析功能需要在 TDengine 集群中安装部署 AI node（Anode）。Anode 可以运行在 Linux/Windows/MacOS 等平台上，同时需要 3.10 或以上版本的 Python 环境支持。 +使用 TDgpt 的高级时序数据分析功能需要在 TDengine 集群中安装部署 AI node（Anode）。Anode 可以运行在 Linux/Windows/MacOS 等平台上，同时需要 Python 3.10 或以上版本的环境支持。 ### 安装及卸载 不同操作系统上安装及部署 Anode 有一些差异，主要是卸载操作、安装路径、服务启停等方面。本文以 Linux 系统为例，说明安装部署的流程。 @@ -48,7 +48,7 @@ Anode 的服务需要使用 uWSGI 驱动驱动运行，因此 Anode 和 uWSGI ```ini [uwsgi] -# Anode HTTP service ip:port +# Anode RESTful service ip:port http = 127.0.0.1:6050 # base directory for Anode python files， do NOT modified this @@ -95,6 +95,7 @@ Anode 运行配置主要是以下： ### Anode 基本操作 +对于 Anode 的管理，用户需要通过 TDengine 的命令行接口 taos 进行。因此下述介绍的管理命令都需要先打开 taos, 链接到 TDengine 运行实例。 #### 创建 Anode ```sql CREATE ANODE {node_url} From 354f0b75b76e1befdf13321815bbdfcb9024730c Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 16 Nov 2024 14:52:46 +0800 Subject: [PATCH 12/19] Update 03-preprocess.md minor change --- docs/zh/06-advanced/06-TDgpt/03-preprocess.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/06-advanced/06-TDgpt/03-preprocess.md b/docs/zh/06-advanced/06-TDgpt/03-preprocess.md index 20940dc17f..3f791fb2e9 100644 --- a/docs/zh/06-advanced/06-TDgpt/03-preprocess.md +++ b/docs/zh/06-advanced/06-TDgpt/03-preprocess.md @@ -7,11 +7,11 @@ import activity from './pic/activity.png'; import wndata from './pic/white-noise-data.png' ### 分析流程 -时序数据分析之前需要有预处理的过程，整体的流程如下图所示。 +时序数据分析之前需要有预处理的过程，为减轻分析算法的负担，TDgpt 在将时序数据发给具体分析算法进行分析时，已经对数据做了预处理，整体的流程如下图所示。 预处理流程 -首先对输入数据进行白噪声检查（White Noise Data check）, 检查通过以后针对预测分析，还需要进行输入（历史）数据的重采样和时间戳对齐处理操作，如果是异常检测则跳过数据重采样和时间戳对齐的预处理。 +TDgpt 首先对输入数据进行白噪声检查（White Noise Data check）, 检查通过以后针对预测分析，还需要进行输入（历史）数据的重采样和时间戳对齐处理操作，如果是异常检测则跳过数据重采样和时间戳对齐的预处理。 预处理完成以后，在进行后续的预测或异常检测操作。 ### 白噪声检查 From b99daf7d586038e15c2f691f5d26fc5e0d54e961 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 16 Nov 2024 14:56:03 +0800 Subject: [PATCH 13/19] Update index.md --- docs/zh/06-advanced/06-TDgpt/04-forecast/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md index 41461a26fd..a2b5d0662e 100644 --- a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md +++ b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md @@ -3,7 +3,7 @@ title: 预测算法 description: 预测算法 --- -时序数据预测处理以持续一个时间段的时序数据作为输入，预测接下来一个连续时间区间内时间序列数据趋势。用户可以指定输出的（预测）时间序列数据点的数量，因此其输出的结果行数不确定。为此，TDengine 使用新函数 `FORECAST` 提供时序数据预测服务。基础数据（用于预测的历史时间序列数据）是该函数的输入，预测结果是该函数的输出。用户可以通过 `FORECAST` 函数调用 Anode 提供的预测算法提供的服务。 +时序数据预测处理以持续一个时间段的时序数据作为输入，预测接下来一个连续时间区间内时间序列数据趋势。用户可以指定输出的（预测）时间序列数据点的数量，因此其输出的结果行数不确定。为此，TDengine 使用新 SQL 函数 `FORECAST` 提供时序数据预测服务。基础数据（用于预测的历史时间序列数据）是该函数的输入，预测结果是该函数的输出。用户可以通过 `FORECAST` 函数调用 Anode 提供的预测算法提供的服务。 ### 语法 ```SQL From 7f6ff450ffde4ba1edb192f007fa12da7e8b086f Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 16 Nov 2024 15:07:09 +0800 Subject: [PATCH 14/19] Update index.md --- docs/zh/06-advanced/06-TDgpt/06-dev/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/index.md b/docs/zh/06-advanced/06-TDgpt/06-dev/index.md index c6e6e9097a..ead1c3bb6e 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/index.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/index.md @@ -2,13 +2,13 @@ title: "算法开发者指南" sidebar_label: "算法开发者指南" --- -TDgpt 是一个可扩展的时序数据高级分析平台，用户遵循简易的步骤就能将自己开发的分析算法添加到分析平台并通过SQL语句直接调用。目前 TDpgt 平台只支持使用 Python 语言开发的分析算法。 +TDgpt 是一个可扩展的时序数据高级分析平台，用户遵循简易的步骤就能将自己开发的分析算法添加到分析平台, 各种应用就可以通过SQL语句直接调用, 让高级分析算法的使用门槛降到几乎为零。目前 TDpgt 平台只支持使用 Python 语言开发的分析算法。 Anode 采用类动态加载模式，在启动的时候扫描特定目录内满足约定条件的所有代码文件，并将其加载到系统中。因此，开发者只需要遵循以下几步就能完成新算法的添加工作： 1. 开发完成符合要求的分析算法类 2. 将代码文件放入对应目录，然后重启 Anode -3. 使用SQL命令，向 Mnode 同步新的算法列表 +3. 使用SQL命令"CREATE ANODE"，将 Anode 添加到 TDengine -此时就完成了新算法的添加工作，之后就可以直接使用SQL语句调用新算法。得益于 TDgpt 与 TDengine主进程 `taosd` 的松散耦合，Anode算法升级对 `taosd` 完全没有影响。应用系统只需要调整对应的SQL语句调用新（升级的）算法，就能够快速完成分析功能和分析算法的升级。 +此时就完成了新算法的添加工作，之后应用就可以直接使用SQL语句调用新算法。得益于 TDgpt 与 TDengine主进程 `taosd` 的松散耦合，Anode算法升级对 `taosd` 完全没有影响。应用系统只需要调整对应的SQL语句调用新（升级的）算法，就能够快速完成分析功能和分析算法的升级。 这种方式能够按需扩展分析算法，极大地拓展 TDgpt 的适应范围，用户可以按需将更契合业务场景的、更准确的（预测、异常检测）分析算法动态嵌入到 TDgpt，并通过 SQL 语句进行调用。在基本不用更改应用系统代码的前提下，就能够快速完成分析功能的平滑升级。 From 38c78f6d32e42d76f9a0058c3bffeddf65475325 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 16 Nov 2024 15:18:36 +0800 Subject: [PATCH 15/19] Update 02-forecast.md --- docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md index e66eb309dd..dc58598f50 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md @@ -77,14 +77,8 @@ class _MyForecastService(AbstractForecastService): """该算法无需任何输入参数，直接重载父类该函数，不处理算法参数设置逻辑""" pass ``` -将该文件保存在 `./taosanalytics/algo/ad/` 目录下，然后重启 taosanode 服务。然后就可以通过 SQL 语句调用该检测算法。 -```SQL ---- 对 col 列进行异常检测，通过指定 algo 参数为 myad 来调用新添加的异常检测类 -SELECT COUNT(*) FROM foo ANOMALY_DETECTION(col, 'algo=myad') -``` - -将该文件保存在 `./taosanalytics/algo/fc/` 目录下，然后重启 taosanode 服务。通过执行 `SHOW ANODES FULL` 能够看到新加入的算法，通过 SQL 语句调用该预测算法。 +将该文件保存在 `./taosanalytics/algo/fc/` 目录下，然后重启 taosanode 服务。在 TDengine 命令行接口中执行 `SHOW ANODES FULL` 能够看到新加入的算法，通过 SQL 语句调用该预测算法。 ```SQL --- 对 col 列进行异常检测，通过指定 algo 参数为 myfc 来调用新添加的预测类 @@ -92,6 +86,7 @@ SELECT _flow, _fhigh, _frowts, FORECAST(col_name, "algo=myfc") FROM foo; ``` +如果是第一次启动该 Anode, 请按照 [TDgpt 安装部署](../management/) 里的步骤先将该 Anode 添加到 TDengine 系统中。 ### 单元测试 From 376654eed740f70af1d1094729a32ba43de5cb57 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 16 Nov 2024 15:27:51 +0800 Subject: [PATCH 16/19] Update 02-forecast.md --- docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md index dc58598f50..e8f6a8470a 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md @@ -78,7 +78,7 @@ class _MyForecastService(AbstractForecastService): pass ``` -将该文件保存在 `./taosanalytics/algo/fc/` 目录下，然后重启 taosanode 服务。在 TDengine 命令行接口中执行 `SHOW ANODES FULL` 能够看到新加入的算法，通过 SQL 语句调用该预测算法。 +将该文件保存在 `./taosanalytics/algo/fc/` 目录下，然后重启 taosanode 服务。在 TDengine 命令行接口中执行 `SHOW ANODES FULL` 能够看到新加入的算法。应用就可以通过 SQL 语句调用该预测算法。 ```SQL --- 对 col 列进行异常检测，通过指定 algo 参数为 myfc 来调用新添加的预测类 From db6620621fd778558605fea0b26dff36b971efb0 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 16 Nov 2024 15:30:47 +0800 Subject: [PATCH 17/19] Update 03-ad.md --- docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md index 8068931653..5ed48c9e43 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md @@ -48,13 +48,13 @@ class _MyAnomalyDetectionService(AbstractAnomalyDetectionService): pass ``` -将该文件保存在 `./taosanalytics/algo/ad/` 目录下，然后重启 taosanode 服务。然后就可以通过 SQL 语句调用该检测算法。 +将该文件保存在 `./taosanalytics/algo/ad/` 目录下，然后重启 taosanode 服务。在 TDengine 命令行接口 taos 中执行 `SHOW ANODES FULL` 就能够看到新加入的算法，然后应用就可以通过 SQL 语句调用该检测算法。 ```SQL --- 对 col 列进行异常检测，通过指定 algo 参数为 myad 来调用新添加的异常检测类 SELECT COUNT(*) FROM foo ANOMALY_DETECTION(col, 'algo=myad') ``` - +如果是第一次启动该 Anode, 请按照 [TDgpt 安装部署](../management/) 里的步骤先将该 Anode 添加到 TDengine 系统中。 ### 单元测试 From d274dbfd8c16d077e84608afe9b8941ef109d709 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 16 Nov 2024 15:44:12 +0800 Subject: [PATCH 18/19] Update 02-forecast.md --- docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md index e8f6a8470a..954076c8fd 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md @@ -86,7 +86,7 @@ SELECT _flow, _fhigh, _frowts, FORECAST(col_name, "algo=myfc") FROM foo; ``` -如果是第一次启动该 Anode, 请按照 [TDgpt 安装部署](../management/) 里的步骤先将该 Anode 添加到 TDengine 系统中。 +如果是第一次启动该 Anode, 请按照 [TDgpt 安装部署](../../management/) 里的步骤先将该 Anode 添加到 TDengine 系统中。 ### 单元测试 From 1f3aa4428bffe2044b1a9d01636a1f4581d7e207 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 16 Nov 2024 15:44:48 +0800 Subject: [PATCH 19/19] Update 03-ad.md --- docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md index 5ed48c9e43..dc0a534706 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md @@ -54,7 +54,7 @@ class _MyAnomalyDetectionService(AbstractAnomalyDetectionService): --- 对 col 列进行异常检测，通过指定 algo 参数为 myad 来调用新添加的异常检测类 SELECT COUNT(*) FROM foo ANOMALY_DETECTION(col, 'algo=myad') ``` -如果是第一次启动该 Anode, 请按照 [TDgpt 安装部署](../management/) 里的步骤先将该 Anode 添加到 TDengine 系统中。 +如果是第一次启动该 Anode, 请按照 [TDgpt 安装部署](../../management/) 里的步骤先将该 Anode 添加到 TDengine 系统中。 ### 单元测试