From 16a79dd86d1de701a779e551274787d0998d4dd4 Mon Sep 17 00:00:00 2001 From: "pengrongkun94@qq.com" Date: Wed, 25 Dec 2024 10:59:37 +0800 Subject: [PATCH 01/35] add stmt test case --- source/client/test/CMakeLists.txt | 16 ++ source/client/test/stmt2Test.cpp | 417 ++++++++++++++++++++++++++++++ 2 files changed, 433 insertions(+) create mode 100644 source/client/test/stmt2Test.cpp diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index fecddbbff4..62b7bb3e91 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -41,6 +41,12 @@ TARGET_LINK_LIBRARIES( PUBLIC ${TAOS_LIB} ) +ADD_EXECUTABLE(stmt2Test stmt2Test.cpp) +TARGET_LINK_LIBRARIES( + stmt2Test + os util common transport parser catalog scheduler gtest ${TAOS_LIB_STATIC} qcom executor function +) + TARGET_INCLUDE_DIRECTORIES( clientTest PUBLIC "${TD_SOURCE_DIR}/include/client/" @@ -82,6 +88,12 @@ TARGET_INCLUDE_DIRECTORIES( # PRIVATE "${TD_SOURCE_DIR}/source/client/inc" #) +TARGET_INCLUDE_DIRECTORIES( + stmt2Test + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" +) + add_test( NAME smlTest COMMAND smlTest @@ -97,3 +109,7 @@ add_test( COMMAND userOperTest ) +add_test( + NAME stmt2Test + COMMAND stmt2Test +) diff --git a/source/client/test/stmt2Test.cpp b/source/client/test/stmt2Test.cpp new file mode 100644 index 0000000000..c325ed3a38 --- /dev/null +++ b/source/client/test/stmt2Test.cpp @@ -0,0 +1,417 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "clientInt.h" +#include "osSemaphore.h" +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "executor.h" +#include "taos.h" + +namespace { +void getFieldsSuccess(TAOS* taos, const char* sql, TAOS_FIELD_ALL* expectedFields, int expectedFieldNum) { + TAOS_STMT2_OPTION option = {0}; + TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); + int code = taos_stmt2_prepare(stmt, sql, 0); + ASSERT_EQ(code, 0); + + int fieldNum = 0; + TAOS_FIELD_ALL* pFields = NULL; + code = taos_stmt2_get_fields(stmt, &fieldNum, &pFields); + ASSERT_EQ(code, 0); + ASSERT_EQ(fieldNum, expectedFieldNum); + + for (int i = 0; i < fieldNum; i++) { + ASSERT_STREQ(pFields[i].name, expectedFields[i].name); + ASSERT_EQ(pFields[i].type, expectedFields[i].type); + ASSERT_EQ(pFields[i].field_type, expectedFields[i].field_type); + ASSERT_EQ(pFields[i].precision, expectedFields[i].precision); + ASSERT_EQ(pFields[i].bytes, expectedFields[i].bytes); + ASSERT_EQ(pFields[i].scale, expectedFields[i].scale); + } + taos_stmt2_free_fields(stmt, pFields); + taos_stmt2_close(stmt); +} + +void getQueryFields(TAOS* taos, const char* sql) { + TAOS_STMT2_OPTION option = {0}; + TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); + int code = taos_stmt2_prepare(stmt, sql, 0); + if (code != 0) { + printf("failed to execute taos_stmt2_prepare. error:%s\n", taos_stmt2_error(stmt)); + taos_stmt2_close(stmt); + return; + } + int fieldNum = 0; + TAOS_FIELD_ALL* pFields = NULL; + code = taos_stmt2_get_fields(stmt, &fieldNum, &pFields); + if (code != 0) { + printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt)); + } else { + printf("bind nums:%d\n", fieldNum); + } + printf("====================================\n"); + taos_stmt2_free_fields(stmt, pFields); + taos_stmt2_close(stmt); +} + +void do_query(TAOS* taos, const char* sql) { + TAOS_RES* result = taos_query(taos, sql); + int code = taos_errno(result); + ASSERT_EQ(code, 0); + + taos_free_result(result); +} + +// void do_stmt(TAOS* taos) { + +// printf("=================error test===================\n"); + +// // case 14 : INSERT INTO db.d0 using db.stb values(?,?) +// // none para for ctbname +// sql = "INSERT INTO db.d0 using db.stb values(?,?)"; +// printf("case 14 (no tags error): %s\n", sql); +// getFields(taos, sql); + +// // case 15 : insert into db.stb(t1,t2,tbname) values(?,?,?) +// // no value +// sql = "insert into db.stb(t1,t2,tbname) values(?,?,?)"; +// printf("case 15 (no PK error): %s\n", sql); +// getFields(taos, sql); + +// // case 16 : insert into db.stb(ts,b,tbname) values(?,?,?,?,?) +// // wrong para nums +// sql = "insert into db.stb(ts,b,tbname) values(?,?,?,?,?)"; +// printf("case 16 (wrong para nums): %s\n", sql); +// getFields(taos, sql); + +// // case 17 : insert into db.? values(?,?) +// // normal table must have tbnam +// sql = "insert into db.? values(?,?)"; +// printf("case 17 (normal table must have tbname): %s\n", sql); +// getFields(taos, sql); + +// // case 18 : INSERT INTO db.stb(t1,t2,ts,b) values(?,?,?,?) +// // no tbname error +// sql = "INSERT INTO db.stb(t1,t2,ts,b) values(?,?,?,?)"; +// printf("case 18 (no tbname error): %s\n", sql); +// getFields(taos, sql); + +// // case 19 : insert into db.ntb(nts,ni) values(?,?,?,?,?) +// // wrong para nums +// sql = "insert into ntb(nts,ni) values(?,?,?,?,?)"; +// printf("case 19 : %s\n", sql); +// getFields(taos, sql); + +// // case 20 : insert into db.stb(t1,t2,ts,b,tbname) values(*,*,*,*,*) +// // wrong simbol +// sql = "insert into db.stb(t1,t2,ts,b,tbname) values(*,*,*,*,*)"; +// printf("=================normal test===================\n"); +// printf("case 20 : %s\n", sql); +// getFields(taos, sql); + +// // case 21 : INSERT INTO ! using db.stb TAGS(?,?) VALUES(?,?) +// // wrong simbol +// sql = "insert into ! using db.stb tags(?, ?) values(?,?)"; +// printf("case 21 : %s\n", sql); +// getFields(taos, sql); + +// // case 22 : INSERT INTO ! using db.stb TAGS(?,?) VALUES(?,?) +// // wrong tbname +// sql = "insert into db.stb values(?,?)"; +// printf("case 22 : %s\n", sql); +// getFields(taos, sql); + +// // case 23 : select * from ? where ts = ? +// // wrong query type +// sql = "select * from ? where ts = ?"; +// printf("case 23 : %s\n", sql); +// getQueryFields(taos, sql); +// } + +} // namespace + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(clientCase, driverInit_Test) { + // taosInitGlobalCfg(); + // taos_init(); +} + +TEST(stmt2Case, insert_stb_get_fields_Test) { + TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + do_query(taos, "drop database if exists db"); + do_query(taos, "create database db PRECISION 'ns'"); + do_query(taos, "use db"); + do_query(taos, + "create table db.stb (ts timestamp, b binary(10)) tags(t1 " + "int, t2 binary(10))"); + do_query( + taos, + "create table if not exists all_stb(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 " + "tinyint unsigned, v7 smallint unsigned, v8 int unsigned, v9 bigint unsigned, v10 float, v11 double, v12 " + "binary(20), v13 varbinary(20), v14 geometry(100), v15 nchar(20))tags(tts timestamp, tv1 bool, tv2 tinyint, tv3 " + "smallint, tv4 int, tv5 bigint, tv6 tinyint unsigned, tv7 smallint unsigned, tv8 int unsigned, tv9 bigint " + "unsigned, tv10 float, tv11 double, tv12 binary(20), tv13 varbinary(20), tv14 geometry(100), tv15 nchar(20));"); + + // // case 1 : test super table + // { + // const char* sql = "insert into db.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)"; + // TAOS_FIELD_ALL expectedFields[5] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + // {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + // {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, + // {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}}; + // printf("case 1 : %s\n", sql); + // getFieldsSuccess(taos, sql, expectedFields, 5); + // } + + // case 2 : no tbname + { + const char* sql = "insert into db.stb(t1,t2,ts,b,tbname) values(1,?,?,'abc',?)"; + TAOS_FIELD_ALL expectedFields[3] = {{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}}; + printf("case 2 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 3); + } + + { + // case 3 : insert into db.stb(ts,b,tbname) values(?,?,?) + // no tag + const char* sql = "insert into db.stb(ts,b,tbname) values(?,?,?)"; + TAOS_FIELD_ALL expectedFields[3] = { + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, + {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + }; + printf("case 5 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 3); + } + + // case 12 : insert into ? using all_stb tags(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + // values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + // test all types + { + const char* sql = + "insert into ? using all_stb tags(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + TAOS_FIELD_ALL expectedFields[33] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + {"tts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_TAG}, + {"tv1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_TAG}, + {"tv2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_TAG}, + {"tv3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_TAG}, + {"tv4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + {"tv5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_TAG}, + {"tv6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_TAG}, + {"tv7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_TAG}, + {"tv8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_TAG}, + {"tv9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_TAG}, + {"tv10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_TAG}, + {"tv11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_TAG}, + {"tv12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_TAG}, + {"tv13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_TAG}, + {"tv14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_TAG}, + {"tv15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_TAG}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"v1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_COL}, + {"v2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_COL}, + {"v3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_COL}, + {"v4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL}, + {"v5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_COL}, + {"v6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_COL}, + {"v7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_COL}, + {"v8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_COL}, + {"v9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_COL}, + {"v10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_COL}, + {"v11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_COL}, + {"v12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_COL}, + {"v13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_COL}, + {"v14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_COL}, + {"v15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_COL}}; + printf("case 12 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 33); + } + + taos_close(taos); + taos_cleanup(); +} + +TEST(stmt2Case, insert_ctb_using_get_fields_Test) { + // do_query(taos, "CREATE TABLE db.d0 USING db.stb (t1,t2) TAGS (7,'Cali');"); + + // // case 2 : INSERT INTO db.d0 VALUES (?,?) + // // test child table + // { + // const char* sql = "INSERT INTO db.d0(ts,b) VALUES (?,?)"; + // TAOS_FIELD_ALL expectedFields[2] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + // {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; + // printf("case 2 : %s\n", sql); + // getFieldsSuccess(taos, sql, expectedFields, 2); + // } + + // // case 6 : INSERT INTO db.? using db.stb (t1,t2)TAGS(?,?) (ts,b)VALUES(?,?) + // // normal insert clause + // { + // const char* sql = "INSERT INTO db.? using db.stb (t1,t2)TAGS(?,?) (ts,b)VALUES(?,?)"; + // TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + // {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + // {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + // {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; + // printf("case 6 : %s\n", sql); + // getFieldsSuccess(taos, sql, expectedFields, 5); + // } + + // // case 7 : insert into db.? using db.stb(t2,t1) tags(?, ?) (b,ts)values(?,?) + // // disordered + // { + // const char* sql = "insert into db.? using db.stb(t2,t1) tags(?, ?) (b,ts)values(?,?)"; + // TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + // {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + // {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + // {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, + // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}}; + // printf("case 7 : %s\n", sql); + // getFieldsSuccess(taos, sql, expectedFields, 5); + // } + + // // case 8 : insert into db.? using db.stb tags(?, ?) values(?,?) + // // no field name + // { + // const char* sql = "insert into db.? using db.stb tags(?, ?) values(?,?)"; + // TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + // {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + // {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + // {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; + // printf("case 8 : %s\n", sql); + // getFieldsSuccess(taos, sql, expectedFields, 5); + // } + + // // case 9 : insert into db.? using db.stb (t2)tags(?) (ts)values(?) + // // less para + // { + // const char* sql = "insert into db.? using db.stb (t2)tags(?) (ts)values(?)"; + // TAOS_FIELD_ALL expectedFields[3] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + // {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}}; + // printf("case 9 : %s\n", sql); + // getFieldsSuccess(taos, sql, expectedFields, 3); + // } + + // // case 10 : insert into db.d0 (ts)values(?) + // // less para + // { + // const char* sql = "insert into db.d0 (ts)values(?)"; + // TAOS_FIELD_ALL expectedFields[1] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}}; + // printf("case 10 : %s\n", sql); + // getFieldsSuccess(taos, sql, expectedFields, 1); + // } + // // case 11 : insert into abc using stb tags(?, ?) values(?,?) + // // insert create table + // { + // const char* sql = "insert into abc using stb tags(?, ?) values(?,?)"; + // TAOS_FIELD_ALL expectedFields[4] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + // {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + // {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; + // printf("case 11 : %s\n", sql); + // getFieldsSuccess(taos, sql, expectedFields, 4); + // } + + // // // case 13 : insert into ? using all_stb tags(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + // // values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + // // // test all types + // // sql = + // // "insert into all_stb " + // // + // "(tbname,tts,tv1,tv2,tv3,tv4,tv5,tv6,tv7,tv8,tv9,tv10,tv11,tv12,tv13,tv14,tv15,ts,v1,v2,v3,v4,v5,v6,v7,v8,v9,v10," + // // "v11,v12,v13,v14,v15) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + // // printf("case 13 : %s\n", sql); + // // getFields(taos, sql); +} + +TEST(stmt2Case, insert_ntb_get_fields_Test) { + // do_query(taos, "CREATE TABLE db.ntb(nts timestamp, nb binary(10),nvc varchar(16),ni int);"); + + // // // case 3 : INSERT INTO db.ntb VALUES(?,?,?,?) + // // // test normal table + // { + // const char* sql = "INSERT INTO db.ntb VALUES(?,?,?,?)"; + // TAOS_FIELD_ALL expectedFields[4] = {{"nts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + // {"nb", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, + // {"nvc", TSDB_DATA_TYPE_BINARY, 0, 0, 18, TAOS_FIELD_COL}, + // {"ni", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL}}; + // printf("case 3 : %s\n", sql); + // getFieldsSuccess(taos, sql, expectedFields, 4); + // } + + // { + // // case 4 : insert into db.stb(t1,tbname,ts,t2,b) values(?,?,?,?,?) + // // test random order + // const char* sql = "insert into db.stb(t1,tbname,ts,t2,b) values(?,?,?,?,?)"; + // TAOS_FIELD_ALL expectedFields[5] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + // {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + // {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + // {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; + // printf("case 4 : %s\n", sql); + // getFieldsSuccess(taos, sql, expectedFields, 5); + // } +} + +TEST(stmt2Case, select_get_fields_Test) { + // // case 14 : select * from ntb where ts = ? + // // query type + // sql = "select * from ntb where ts = ?"; + // printf("case 14 : %s\n", sql); + // getQueryFields(taos, sql); + + // // case 15 : select * from ntb where ts = ? and b = ? + // // query type + // sql = "select * from ntb where ts = ? and b = ?"; + // printf("case 15 : %s\n", sql); + // getQueryFields(taos, sql); +} + +TEST(stmt2Case, db_get_fields_Test) {} + +TEST(stmt2Case, get_fields_error_Test) { + // { + // const char* sql = "insert into db.stb(t1,t2,ts,b,tbname) values(1,?,?,'abc',?)"; + // TAOS_FIELD_ALL expectedFields[3] = {{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + // {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}}; + // printf("case 2 : %s\n", sql); + // getFieldsSuccess(taos, sql, expectedFields, 3); + // } +} + +#pragma GCC diagnostic pop From 0d1a99d7d07bdf4c5e8a5c5f149e83326dd4e2e4 Mon Sep 17 00:00:00 2001 From: "pengrongkun94@qq.com" Date: Thu, 26 Dec 2024 16:30:50 +0800 Subject: [PATCH 02/35] remove some unused code --- source/client/inc/clientStmt2.h | 3 - source/client/src/clientMain.c | 82 +++---- source/client/src/clientStmt2.c | 161 -------------- source/client/test/stmt2Test.cpp | 352 ++++++++++++++++++++++--------- 4 files changed, 281 insertions(+), 317 deletions(-) diff --git a/source/client/inc/clientStmt2.h b/source/client/inc/clientStmt2.h index 0fe813473d..05a4c849f8 100644 --- a/source/client/inc/clientStmt2.h +++ b/source/client/inc/clientStmt2.h @@ -221,11 +221,8 @@ int stmtPrepare2(TAOS_STMT2 *stmt, const char *sql, unsigned long length int stmtSetTbName2(TAOS_STMT2 *stmt, const char *tbName); int stmtSetTbTags2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *tags); int stmtBindBatch2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *bind, int32_t colIdx); -int stmtGetTagFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields); -int stmtGetColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields); int stmtGetStbColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_ALL **fields); int stmtGetParamNum2(TAOS_STMT2 *stmt, int *nums); -int stmtGetParamTbName(TAOS_STMT2 *stmt, int *nums); int stmtIsInsert2(TAOS_STMT2 *stmt, int *insert); TAOS_RES *stmtUseResult2(TAOS_STMT2 *stmt); const char *stmtErrstr2(TAOS_STMT2 *stmt); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index f041e4b030..83aff351dd 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -24,13 +24,13 @@ #include "query.h" #include "scheduler.h" #include "tcompare.h" +#include "tconv.h" #include "tdatablock.h" #include "tglobal.h" #include "tmsg.h" #include "tref.h" #include "trpc.h" #include "version.h" -#include "tconv.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -56,12 +56,12 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) { } #ifndef WINDOWS -static void freeTz(void *p){ +static void freeTz(void *p) { timezone_t tz = *(timezone_t *)p; tzfree(tz); } -int32_t tzInit(){ +int32_t tzInit() { pTimezoneMap = taosHashInit(0, MurmurHash3_32, false, HASH_ENTRY_LOCK); if (pTimezoneMap == NULL) { return terrno; @@ -75,15 +75,15 @@ int32_t tzInit(){ return 0; } -void tzCleanup(){ +void tzCleanup() { taosHashCleanup(pTimezoneMap); taosHashCleanup(pTimezoneNameMap); } -static timezone_t setConnnectionTz(const char* val){ - timezone_t tz = NULL; +static timezone_t setConnnectionTz(const char *val) { + timezone_t tz = NULL; timezone_t *tmp = taosHashGet(pTimezoneMap, val, strlen(val)); - if (tmp != NULL && *tmp != NULL){ + if (tmp != NULL && *tmp != NULL) { tz = *tmp; goto END; } @@ -100,20 +100,20 @@ static timezone_t setConnnectionTz(const char* val){ } } int32_t code = taosHashPut(pTimezoneMap, val, strlen(val), &tz, sizeof(timezone_t)); - if (code != 0){ + if (code != 0) { tscError("%s put timezone to tz map error:%d", __func__, code); tzfree(tz); tz = NULL; goto END; } - time_t tx1 = taosGetTimestampSec(); - char output[TD_TIMEZONE_LEN] = {0}; + time_t tx1 = taosGetTimestampSec(); + char output[TD_TIMEZONE_LEN] = {0}; code = taosFormatTimezoneStr(tx1, val, tz, output); - if (code == 0){ + if (code == 0) { code = taosHashPut(pTimezoneNameMap, &tz, sizeof(timezone_t), output, strlen(output) + 1); } - if (code != 0){ + if (code != 0) { tscError("failed to put timezone %s to map", val); } @@ -122,18 +122,18 @@ END: } #endif -static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, const char* val){ +static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, const char *val) { if (taos == NULL) { return terrno = TSDB_CODE_INVALID_PARA; } #ifdef WINDOWS - if (option == TSDB_OPTION_CONNECTION_TIMEZONE){ + if (option == TSDB_OPTION_CONNECTION_TIMEZONE) { return terrno = TSDB_CODE_NOT_SUPPORTTED_IN_WINDOWS; } #endif - if (option < TSDB_OPTION_CONNECTION_CLEAR || option >= TSDB_MAX_OPTIONS_CONNECTION){ + if (option < TSDB_OPTION_CONNECTION_CLEAR || option >= TSDB_MAX_OPTIONS_CONNECTION) { return terrno = TSDB_CODE_INVALID_PARA; } @@ -149,7 +149,7 @@ static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, co return terrno; } - if (option == TSDB_OPTION_CONNECTION_CLEAR){ + if (option == TSDB_OPTION_CONNECTION_CLEAR) { val = NULL; } @@ -165,19 +165,19 @@ static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, co goto END; } pObj->optionInfo.charsetCxt = tmp; - }else{ + } else { pObj->optionInfo.charsetCxt = NULL; } } if (option == TSDB_OPTION_CONNECTION_TIMEZONE || option == TSDB_OPTION_CONNECTION_CLEAR) { #ifndef WINDOWS - if (val != NULL){ - if (val[0] == 0){ + if (val != NULL) { + if (val[0] == 0) { val = "UTC"; } timezone_t tz = setConnnectionTz(val); - if (tz == NULL){ + if (tz == NULL) { code = terrno; goto END; } @@ -199,7 +199,7 @@ static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, co if (option == TSDB_OPTION_CONNECTION_USER_IP || option == TSDB_OPTION_CONNECTION_CLEAR) { if (val != NULL) { pObj->optionInfo.userIp = taosInetAddr(val); - if (pObj->optionInfo.userIp == INADDR_NONE){ + if (pObj->optionInfo.userIp == INADDR_NONE) { code = TSDB_CODE_INVALID_PARA; goto END; } @@ -213,7 +213,7 @@ END: return terrno = code; } -int taos_options_connection(TAOS *taos, TSDB_OPTION_CONNECTION option, const void *arg, ...){ +int taos_options_connection(TAOS *taos, TSDB_OPTION_CONNECTION option, const void *arg, ...) { return setConnectionOption(taos, option, (const char *)arg); } @@ -2129,6 +2129,11 @@ int taos_stmt_close(TAOS_STMT *stmt) { } TAOS_STMT2 *taos_stmt2_init(TAOS *taos, TAOS_STMT2_OPTION *option) { + if (NULL == taos) { + tscError("NULL parameter for %s", __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } STscObj *pObj = acquireTscObj(*(int64_t *)taos); if (NULL == pObj) { tscError("invalid parameter for %s", __FUNCTION__); @@ -2257,16 +2262,7 @@ int taos_stmt2_close(TAOS_STMT2 *stmt) { return stmtClose2(stmt); } -/* -int taos_stmt2_param_count(TAOS_STMT2 *stmt, int *nums) { - if (stmt == NULL || nums == NULL) { - tscError("NULL parameter for %s", __FUNCTION__); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return stmtGetParamNum2(stmt, nums); -} -*/ + int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert) { if (stmt == NULL || insert == NULL) { tscError("NULL parameter for %s", __FUNCTION__); @@ -2277,28 +2273,6 @@ int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert) { return stmtIsInsert2(stmt, insert); } -// int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields) { -// if (stmt == NULL || count == NULL) { -// tscError("NULL parameter for %s", __FUNCTION__); -// terrno = TSDB_CODE_INVALID_PARA; -// return terrno; -// } - -// if (field_type == TAOS_FIELD_COL) { -// return stmtGetColFields2(stmt, count, fields); -// } else if (field_type == TAOS_FIELD_TAG) { -// return stmtGetTagFields2(stmt, count, fields); -// } else if (field_type == TAOS_FIELD_QUERY) { -// return stmtGetParamNum2(stmt, count); -// } else if (field_type == TAOS_FIELD_TBNAME) { -// return stmtGetParamTbName(stmt, count); -// } else { -// tscError("invalid parameter for %s", __FUNCTION__); -// terrno = TSDB_CODE_INVALID_PARA; -// return terrno; -// } -// } - int taos_stmt2_get_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields) { if (stmt == NULL || count == NULL) { tscError("NULL parameter for %s", __FUNCTION__); diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index acd118acc9..f7f4cbb087 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1036,28 +1036,6 @@ int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) { return TSDB_CODE_SUCCESS; } -static int stmtFetchTagFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields) { - if (pStmt->errCode != TSDB_CODE_SUCCESS) { - return pStmt->errCode; - } - - if (STMT_TYPE_QUERY == pStmt->sql.type) { - tscError("invalid operation to get query tag fileds"); - STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); - } - - STableDataCxt** pDataBlock = - (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); - if (NULL == pDataBlock) { - tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_APP_ERROR); - } - - STMT_ERR_RET(qBuildStmtTagFields(*pDataBlock, pStmt->bInfo.boundTags, fieldNum, fields)); - - return TSDB_CODE_SUCCESS; -} - static int stmtFetchColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields) { if (pStmt->errCode != TSDB_CODE_SUCCESS) { return pStmt->errCode; @@ -1819,47 +1797,6 @@ int stmtAffectedRows(TAOS_STMT* stmt) { return ((STscStmt2*)stmt)->affectedRows; int stmtAffectedRowsOnce(TAOS_STMT* stmt) { return ((STscStmt2*)stmt)->exec.affectedRows; } */ -int stmtGetTagFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_E** fields) { - int32_t code = 0; - STscStmt2* pStmt = (STscStmt2*)stmt; - int32_t preCode = pStmt->errCode; - - STMT_DLOG_E("start to get tag fields"); - - if (pStmt->errCode != TSDB_CODE_SUCCESS) { - return pStmt->errCode; - } - - if (STMT_TYPE_QUERY == pStmt->sql.type) { - STMT_ERRI_JRET(TSDB_CODE_TSC_STMT_API_ERROR); - } - - STMT_ERRI_JRET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); - - if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && - STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { - pStmt->bInfo.needParse = false; - } - - if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) { - taos_free_result(pStmt->exec.pRequest); - pStmt->exec.pRequest = NULL; - } - - STMT_ERRI_JRET(stmtCreateRequest(pStmt)); - - if (pStmt->bInfo.needParse) { - STMT_ERRI_JRET(stmtParseSql(pStmt)); - } - - STMT_ERRI_JRET(stmtFetchTagFields2(stmt, nums, fields)); - -_return: - - pStmt->errCode = preCode; - - return code; -} int stmtParseColFields2(TAOS_STMT2* stmt) { int32_t code = 0; @@ -1902,15 +1839,6 @@ _return: return code; } -int stmtGetColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_E** fields) { - int32_t code = stmtParseColFields2(stmt); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - return stmtFetchColFields2(stmt, nums, fields); -} - int stmtGetStbColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_ALL** fields) { int32_t code = stmtParseColFields2(stmt); if (code != TSDB_CODE_SUCCESS) { @@ -1956,95 +1884,6 @@ int stmtGetParamNum2(TAOS_STMT2* stmt, int* nums) { return TSDB_CODE_SUCCESS; } -int stmtGetParamTbName(TAOS_STMT2* stmt, int* nums) { - STscStmt2* pStmt = (STscStmt2*)stmt; - int32_t code = 0; - int32_t preCode = pStmt->errCode; - - STMT_DLOG_E("start to get param num"); - - if (pStmt->errCode != TSDB_CODE_SUCCESS) { - return pStmt->errCode; - } - - STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); - - if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && - STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { - pStmt->bInfo.needParse = false; - } - - if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) { - taos_free_result(pStmt->exec.pRequest); - pStmt->exec.pRequest = NULL; - } - - STMT_ERR_RET(stmtCreateRequest(pStmt)); - - if (pStmt->bInfo.needParse) { - STMT_ERRI_JRET(stmtParseSql(pStmt)); - } - - *nums = STMT_TYPE_MULTI_INSERT == pStmt->sql.type ? 1 : 0; - -_return: - if (TSDB_CODE_TSC_STMT_TBNAME_ERROR == code) { - *nums = 1; - code = TSDB_CODE_SUCCESS; - } - - pStmt->errCode = preCode; - return code; -} -/* -int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) { - STscStmt2* pStmt = (STscStmt2*)stmt; - - STMT_DLOG_E("start to get param"); - - if (pStmt->errCode != TSDB_CODE_SUCCESS) { - return pStmt->errCode; - } - - if (STMT_TYPE_QUERY == pStmt->sql.type) { - STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR); - } - - STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); - - if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && - STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { - pStmt->bInfo.needParse = false; - } - - if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) { - taos_free_result(pStmt->exec.pRequest); - pStmt->exec.pRequest = NULL; - } - - STMT_ERR_RET(stmtCreateRequest(pStmt)); - - if (pStmt->bInfo.needParse) { - STMT_ERR_RET(stmtParseSql(pStmt)); - } - - int32_t nums = 0; - TAOS_FIELD_E* pField = NULL; - STMT_ERR_RET(stmtFetchColFields(stmt, &nums, &pField)); - if (idx >= nums) { - tscError("idx %d is too big", idx); - taosMemoryFree(pField); - STMT_ERR_RET(TSDB_CODE_INVALID_PARA); - } - - *type = pField[idx].type; - *bytes = pField[idx].bytes; - - taosMemoryFree(pField); - - return TSDB_CODE_SUCCESS; -} -*/ TAOS_RES* stmtUseResult2(TAOS_STMT2* stmt) { STscStmt2* pStmt = (STscStmt2*)stmt; diff --git a/source/client/test/stmt2Test.cpp b/source/client/test/stmt2Test.cpp index c325ed3a38..267b777ff3 100644 --- a/source/client/test/stmt2Test.cpp +++ b/source/client/test/stmt2Test.cpp @@ -27,10 +27,17 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" +#include "../inc/clientStmt.h" +#include "../inc/clientStmt2.h" #include "executor.h" #include "taos.h" namespace { +void stmtAsyncQueryCb(void* param, TAOS_RES* pRes, int code) { + int affected_rows = taos_affected_rows(pRes); + return; +} + void getFieldsSuccess(TAOS* taos, const char* sql, TAOS_FIELD_ALL* expectedFields, int expectedFieldNum) { TAOS_STMT2_OPTION option = {0}; TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); @@ -55,28 +62,35 @@ void getFieldsSuccess(TAOS* taos, const char* sql, TAOS_FIELD_ALL* expectedField taos_stmt2_close(stmt); } -void getQueryFields(TAOS* taos, const char* sql) { +void getFieldsError(TAOS* taos, const char* sql, int errorCode) { TAOS_STMT2_OPTION option = {0}; TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); int code = taos_stmt2_prepare(stmt, sql, 0); - if (code != 0) { - printf("failed to execute taos_stmt2_prepare. error:%s\n", taos_stmt2_error(stmt)); - taos_stmt2_close(stmt); - return; - } + ASSERT_EQ(code, 0); + int fieldNum = 0; TAOS_FIELD_ALL* pFields = NULL; code = taos_stmt2_get_fields(stmt, &fieldNum, &pFields); - if (code != 0) { - printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt)); - } else { - printf("bind nums:%d\n", fieldNum); - } - printf("====================================\n"); + ASSERT_EQ(code, errorCode); taos_stmt2_free_fields(stmt, pFields); taos_stmt2_close(stmt); } +void getQueryFields(TAOS* taos, const char* sql, int expectedFieldNum) { + TAOS_STMT2_OPTION option = {0}; + TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); + int code = taos_stmt2_prepare(stmt, sql, 0); + ASSERT_EQ(code, 0); + + int fieldNum = 0; + TAOS_FIELD_ALL* pFields = NULL; + code = taos_stmt2_get_fields(stmt, &fieldNum, NULL); + ASSERT_EQ(code, 0); + ASSERT_EQ(fieldNum, expectedFieldNum); + taos_stmt2_free_fields(stmt, NULL); + taos_stmt2_close(stmt); +} + void do_query(TAOS* taos, const char* sql) { TAOS_RES* result = taos_query(taos, sql); int code = taos_errno(result); @@ -169,94 +183,134 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { do_query(taos, "drop database if exists db"); do_query(taos, "create database db PRECISION 'ns'"); - do_query(taos, "use db"); do_query(taos, "create table db.stb (ts timestamp, b binary(10)) tags(t1 " "int, t2 binary(10))"); do_query( taos, - "create table if not exists all_stb(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 " + "create table if not exists db.all_stb(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 " "tinyint unsigned, v7 smallint unsigned, v8 int unsigned, v9 bigint unsigned, v10 float, v11 double, v12 " "binary(20), v13 varbinary(20), v14 geometry(100), v15 nchar(20))tags(tts timestamp, tv1 bool, tv2 tinyint, tv3 " "smallint, tv4 int, tv5 bigint, tv6 tinyint unsigned, tv7 smallint unsigned, tv8 int unsigned, tv9 bigint " "unsigned, tv10 float, tv11 double, tv12 binary(20), tv13 varbinary(20), tv14 geometry(100), tv15 nchar(20));"); + printf("support case \n"); - // // case 1 : test super table - // { - // const char* sql = "insert into db.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)"; - // TAOS_FIELD_ALL expectedFields[5] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, - // {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, - // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, - // {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, - // {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}}; - // printf("case 1 : %s\n", sql); - // getFieldsSuccess(taos, sql, expectedFields, 5); - // } - - // case 2 : no tbname + // case 1 : test super table { - const char* sql = "insert into db.stb(t1,t2,ts,b,tbname) values(1,?,?,'abc',?)"; - TAOS_FIELD_ALL expectedFields[3] = {{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + const char* sql = "insert into db.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)"; + TAOS_FIELD_ALL expectedFields[5] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, + {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}}; + printf("case 1 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 5); + } + + { + // case 2 : no tag + const char* sql = "insert into db.stb(ts,b,tbname) values(?,?,?)"; + TAOS_FIELD_ALL expectedFields[3] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}}; printf("case 2 : %s\n", sql); getFieldsSuccess(taos, sql, expectedFields, 3); } + // case 3 : random order { - // case 3 : insert into db.stb(ts,b,tbname) values(?,?,?) - // no tag - const char* sql = "insert into db.stb(ts,b,tbname) values(?,?,?)"; - TAOS_FIELD_ALL expectedFields[3] = { - {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, - {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, - {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, - }; - printf("case 5 : %s\n", sql); - getFieldsSuccess(taos, sql, expectedFields, 3); + const char* sql = "insert into db.stb(tbname,ts,t2,b,t1) values(?,?,?,?,?)"; + TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, + {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}}; + printf("case 3 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 5); } - // case 12 : insert into ? using all_stb tags(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) - // values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) - // test all types + // case 4 : random order { - const char* sql = - "insert into ? using all_stb tags(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; - TAOS_FIELD_ALL expectedFields[33] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, - {"tts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_TAG}, - {"tv1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_TAG}, - {"tv2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_TAG}, - {"tv3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_TAG}, - {"tv4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, - {"tv5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_TAG}, - {"tv6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_TAG}, - {"tv7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_TAG}, - {"tv8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_TAG}, - {"tv9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_TAG}, - {"tv10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_TAG}, - {"tv11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_TAG}, - {"tv12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_TAG}, - {"tv13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_TAG}, - {"tv14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_TAG}, - {"tv15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_TAG}, - {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, - {"v1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_COL}, - {"v2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_COL}, - {"v3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_COL}, - {"v4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL}, - {"v5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_COL}, - {"v6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_COL}, - {"v7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_COL}, - {"v8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_COL}, - {"v9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_COL}, - {"v10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_COL}, - {"v11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_COL}, - {"v12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_COL}, - {"v13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_COL}, - {"v14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_COL}, - {"v15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_COL}}; - printf("case 12 : %s\n", sql); - getFieldsSuccess(taos, sql, expectedFields, 33); + const char* sql = "insert into db.stb(tbname,ts,t2,b,t1) values(?,?,?,?,?)"; + TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, + {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}}; + printf("case 4 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 5); + } + + // case 5 : 'db'.'stb' + { + const char* sql = "insert into 'db'.'stb'(t1,t2,ts,b,tbname) values(?,?,?,?,?)"; + TAOS_FIELD_ALL expectedFields[5] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, + {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}}; + printf("case 5 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 5); + } + + // case 6 : use db + { + do_query(taos, "use db"); + const char* sql = "insert into stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)"; + TAOS_FIELD_ALL expectedFields[5] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, + {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}}; + printf("case 6 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 5); + } + + // case 7 : less param + { + const char* sql = "insert into db.stb(ts,tbname) values(?,?)"; + TAOS_FIELD_ALL expectedFields[2] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}}; + printf("case 7 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 2); + } + + // not support case + printf("not support case \n"); + + // case 5 : add in main TD-33353 + // { + // const char* sql = "insert into db.stb(t1,t2,ts,b,tbname) values(1,?,?,'abc',?)"; + // printf("case 2 : %s\n", sql); + // getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); + // } + + // case 7 : no pk + { + const char* sql = "insert into db.stb(b,tbname) values(?,?)"; + printf("case 7 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); + } + + // case 5 : no tbname and tag(not support bind) + { + const char* sql = "insert into db.stb(ts,b) values(?,?)"; + printf("case 5 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); + } + + // case 5 : no tbname and tag(not support bind) + { + const char* sql = "insert into db.stb(tbname) values(?)"; + printf("case 5 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); + } + + // case 8 : no param + { + const char* sql = "insert into db.stb(?,?,?,?,?)"; + printf("case 8 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR); } taos_close(taos); @@ -356,6 +410,49 @@ TEST(stmt2Case, insert_ctb_using_get_fields_Test) { // // "v11,v12,v13,v14,v15) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; // // printf("case 13 : %s\n", sql); // // getFields(taos, sql); + + // case 12 : insert into ? using all_stb tags(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + // values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + // test all types + // { + // const char* sql = + // "insert into ? using all_stb tags(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + // TAOS_FIELD_ALL expectedFields[33] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + // {"tts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_TAG}, + // {"tv1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_TAG}, + // {"tv2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_TAG}, + // {"tv3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_TAG}, + // {"tv4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + // {"tv5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_TAG}, + // {"tv6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_TAG}, + // {"tv7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_TAG}, + // {"tv8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_TAG}, + // {"tv9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_TAG}, + // {"tv10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_TAG}, + // {"tv11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_TAG}, + // {"tv12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_TAG}, + // {"tv13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_TAG}, + // {"tv14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_TAG}, + // {"tv15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_TAG}, + // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + // {"v1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_COL}, + // {"v2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_COL}, + // {"v3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_COL}, + // {"v4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL}, + // {"v5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_COL}, + // {"v6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_COL}, + // {"v7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_COL}, + // {"v8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_COL}, + // {"v9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_COL}, + // {"v10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_COL}, + // {"v11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_COL}, + // {"v12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_COL}, + // {"v13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_COL}, + // {"v14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_COL}, + // {"v15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_COL}}; + // printf("case 12 : %s\n", sql); + // getFieldsSuccess(taos, sql, expectedFields, 33); + // } } TEST(stmt2Case, insert_ntb_get_fields_Test) { @@ -388,30 +485,87 @@ TEST(stmt2Case, insert_ntb_get_fields_Test) { } TEST(stmt2Case, select_get_fields_Test) { - // // case 14 : select * from ntb where ts = ? - // // query type - // sql = "select * from ntb where ts = ?"; - // printf("case 14 : %s\n", sql); - // getQueryFields(taos, sql); + TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + do_query(taos, "drop database if exists db"); + do_query(taos, "create database db PRECISION 'ns'"); + do_query(taos, "use db"); + do_query(taos, "CREATE TABLE db.ntb(nts timestamp, nb binary(10),nvc varchar(16),ni int);"); + { + // case 1 : select * from ntb where ts = ? + // query type + const char* sql = "select * from ntb where ts = ?"; + printf("case 1 : %s\n", sql); + getQueryFields(taos, sql, 1); + } - // // case 15 : select * from ntb where ts = ? and b = ? - // // query type - // sql = "select * from ntb where ts = ? and b = ?"; - // printf("case 15 : %s\n", sql); - // getQueryFields(taos, sql); + { + // case 2 : select * from ntb where ts = ? and b = ? + // query type + const char* sql = "select * from ntb where ts = ? and b = ?"; + printf("case 2 : %s\n", sql); + getQueryFields(taos, sql, 2); + } } -TEST(stmt2Case, db_get_fields_Test) {} - TEST(stmt2Case, get_fields_error_Test) { - // { - // const char* sql = "insert into db.stb(t1,t2,ts,b,tbname) values(1,?,?,'abc',?)"; - // TAOS_FIELD_ALL expectedFields[3] = {{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, - // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, - // {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}}; - // printf("case 2 : %s\n", sql); - // getFieldsSuccess(taos, sql, expectedFields, 3); - // } + // case 1 : + { + printf("case 1 : NULL param \n"); + int code = taos_stmt2_get_fields(NULL, NULL, NULL); + ASSERT_EQ(code, TSDB_CODE_INVALID_PARA); + } } +TEST(stmt2Case, stmt2_init_prepare_Test) { + TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + { + (void)taos_stmt2_init(NULL, NULL); + ASSERT_EQ(terrno, TSDB_CODE_INVALID_PARA); + terrno = 0; + } + + { + (void)taos_stmt2_prepare(NULL, NULL, 0); + ASSERT_EQ(terrno, TSDB_CODE_INVALID_PARA); + terrno = 0; + } + + { + TAOS_STMT2_OPTION option = {0, true, true, NULL, NULL}; + TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); + ASSERT_EQ(terrno, 0); + ASSERT_NE(stmt, nullptr); + int code = taos_stmt2_prepare(stmt, "wrong sql", 0); + ASSERT_EQ(terrno, 0); + ASSERT_NE(stmt, nullptr); + ASSERT_EQ(((STscStmt2*)stmt)->db, nullptr); + + code = taos_stmt2_prepare(stmt, "insert into 'db'.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)", 0); + ASSERT_EQ(terrno, 0); + ASSERT_NE(stmt, nullptr); + // ASSERT_STREQ(((STscStmt2*)stmt)->db, "db"); //add in main TD-33332 + taos_stmt2_close(stmt); + } + + { + TAOS_STMT2_OPTION option = {0, true, false, NULL, NULL}; + TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); + ASSERT_EQ(terrno, 0); + ASSERT_NE(stmt, nullptr); + taos_stmt2_close(stmt); + } + + { + TAOS_STMT2_OPTION option = {0, true, true, stmtAsyncQueryCb, NULL}; + TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); + ASSERT_EQ(terrno, 0); + ASSERT_NE(stmt, nullptr); + taos_stmt2_close(stmt); + } +} + +TEST(stmt2Case, stmt2_status_Test) {} + #pragma GCC diagnostic pop From 05712e2ddeb10847900f687a186bb5c7f9046f8c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 26 Dec 2024 17:24:17 +0800 Subject: [PATCH 03/35] test(analytics): add tests for tdanalytics. --- source/util/src/tanalytics.c | 6 +-- tests/script/sh/stop_dnodes.sh | 8 +-- tests/script/tsim/analytics/basic0.sim | 67 +++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 8 deletions(-) diff --git a/source/util/src/tanalytics.c b/source/util/src/tanalytics.c index e68edd4b76..bf2cb4fd07 100644 --- a/source/util/src/tanalytics.c +++ b/source/util/src/tanalytics.c @@ -20,7 +20,7 @@ #ifdef USE_ANALYTICS #include -#define ANAL_ALGO_SPLIT "," +#define ANALYTICS_ALOG_SPLIT_CHAR "," typedef struct { int64_t ver; @@ -136,7 +136,7 @@ bool taosAnalGetOptStr(const char *option, const char *optName, char *optValue, return false; } - pEnd = strstr(pStart, ANAL_ALGO_SPLIT); + pEnd = strstr(pStart, ANALYTICS_ALOG_SPLIT_CHAR); if (optMaxLen > 0) { if (pEnd > pStart) { int32_t len = (int32_t)(pEnd - pStart); @@ -168,7 +168,7 @@ bool taosAnalGetOptInt(const char *option, const char *optName, int64_t *optValu int32_t bufLen = tsnprintf(buf, sizeof(buf), "%s=", optName); char *pos1 = strstr(option, buf); - char *pos2 = strstr(option, ANAL_ALGO_SPLIT); + char *pos2 = strstr(option, ANALYTICS_ALOG_SPLIT_CHAR); if (pos1 != NULL) { *optValue = taosStr2Int64(pos1 + bufLen, NULL, 10); return true; diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index 8923804547..da2083b013 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -13,7 +13,7 @@ if [ -n "$PID" ]; then systemctl stop taosd fi -PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` +PID=`ps -ef|grep -w taosd | grep -v grep | grep -v taosanode | awk '{print $2}'` while [ -n "$PID" ]; do echo kill -9 $PID #pkill -9 taosd @@ -38,10 +38,10 @@ while [ -n "$PID" ]; do else lsof -nti:6030 | xargs kill -9 fi - PID=`ps -ef|grep -w taos | grep -v grep | awk '{print $2}'` + PID=`ps -ef|grep -w taos | grep -v grep |grep -v taosanode| awk '{print $2}'` done -PID=`ps -ef|grep -w tmq_sim | grep -v grep | awk '{print $2}'` +PID=`ps -ef|grep -w tmq_sim | grep -v grep | grep -v taosanode|awk '{print $2}'` while [ -n "$PID" ]; do echo kill -9 $PID #pkill -9 tmq_sim @@ -52,5 +52,5 @@ while [ -n "$PID" ]; do else lsof -nti:6030 | xargs kill -9 fi - PID=`ps -ef|grep -w tmq_sim | grep -v grep | awk '{print $2}'` + PID=`ps -ef|grep -w tmq_sim | grep -v grep | grep -v taosanode| awk '{print $2}'` done \ No newline at end of file diff --git a/tests/script/tsim/analytics/basic0.sim b/tests/script/tsim/analytics/basic0.sim index 77c9184e8f..35774e7682 100644 --- a/tests/script/tsim/analytics/basic0.sim +++ b/tests/script/tsim/analytics/basic0.sim @@ -3,7 +3,17 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -print =============== create anode +print =============== failed to create anode on '127.0.0.1:1101' +sql_error create anode '127.0.0.1:1101' + +sql show anodes +if $rows != 0 then + return -1 +endi + +sql_error drop anode 1 + +print ================ create anode sql create anode '192.168.1.116:6050' sql show anodes @@ -58,6 +68,61 @@ if $data00 != 1 then return -1 endi +sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 +sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=119,wncheck=1,period=0') from ct1 +sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters1,conf=0.5,wncheck=1,period=0') from ct1 +sql_error select forecast(c1, 'conf=50 ,algo = arima, rows=0') from ct1 +sql_error select forecast(c1, 'conf=50 ,algo = arima, rows=-10') from ct1 +sql_error select forecast(c1, 'conf=50 ,algo = arima, every=0') from ct1 + +sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters, conf=50 ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, ' algo=holtwinters , conf=50 ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, ' algo = holtwinters , conf = 50 ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, ,') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, , ,') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, a =') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, = a ,') from ct1 +sql select forecast(c1, 'conf=50 ,algo = arima') from ct1 +sql select forecast(c1, 'conf=50 ,algo = arima, rows=1') from ct1 + +sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=50,wncheck=1,period=0,start=1700000000000,every=2') from ct1 +if $rows != 10 then + return -1 +endi + +if $data03 != 28 then + return -1 +endi + +if $data00 != @23-11-15 06:13:20.000@ then + print expect 23-11-15 06:13:20.000 , actual $data00 + return -1 +endi + +if $data10 != @23-11-15 06:13:20.002@ then + print expect 23-11-15 06:13:20.002 , actual $data10 + return -1 +endi + +if $data20 != @23-11-15 06:13:20.004@ then + return -1 +endi + +print test the every option and rows option + +sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=50,wncheck=1,period=0,start=1700000000000,every=100,rows=5') from ct1 +if $rows != 5 then + return -1 +endi + +if $data00 != @23-11-15 06:13:20.000@ then + return -1 +endi + +if $data10 != @23-11-15 06:13:20.100@ then + return -1 +endi sql drop anode 1 sql show anodes From 8b0a371aa699d264a1e3cb014de4701c64a58c1e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 26 Dec 2024 18:17:02 +0800 Subject: [PATCH 04/35] test: add test case for anamaly detection in tdanalytics. --- .../libs/executor/src/anomalywindowoperator.c | 49 +++++++++---------- tests/script/tsim/analytics/basic0.sim | 44 +++++++++++++++-- 2 files changed, 62 insertions(+), 31 deletions(-) diff --git a/source/libs/executor/src/anomalywindowoperator.c b/source/libs/executor/src/anomalywindowoperator.c index dd1a52022e..eb72edb964 100644 --- a/source/libs/executor/src/anomalywindowoperator.c +++ b/source/libs/executor/src/anomalywindowoperator.c @@ -61,9 +61,13 @@ static int32_t anomalyCacheBlock(SAnomalyWindowOperatorInfo* pInfo, SSDataBlock* int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) { QRY_PARAM_CHECK(pOptrInfo); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + size_t keyBufSize = 0; + int32_t num = 0; + SExprInfo* pExprInfo = NULL; + const char* id = GET_TASKID(pTaskInfo); - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; SAnomalyWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAnomalyWindowOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SAnomalyWindowPhysiNode* pAnomalyNode = (SAnomalyWindowPhysiNode*)physiNode; @@ -74,13 +78,13 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p } if (!taosAnalGetOptStr(pAnomalyNode->anomalyOpt, "algo", pInfo->algoName, sizeof(pInfo->algoName))) { - qError("failed to get anomaly_window algorithm name from %s", pAnomalyNode->anomalyOpt); + qError("%s failed to get anomaly_window algorithm name from %s", id, pAnomalyNode->anomalyOpt); code = TSDB_CODE_ANA_ALGO_NOT_FOUND; goto _error; } if (taosAnalGetAlgoUrl(pInfo->algoName, ANAL_ALGO_TYPE_ANOMALY_DETECT, pInfo->algoUrl, sizeof(pInfo->algoUrl)) != 0) { - qError("failed to get anomaly_window algorithm url from %s", pInfo->algoName); + qError("%s failed to get anomaly_window algorithm url from %s", id, pInfo->algoName); code = TSDB_CODE_ANA_ALGO_NOT_LOAD; goto _error; } @@ -94,20 +98,18 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p SExprInfo* pScalarExprInfo = NULL; code = createExprInfo(pAnomalyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalarExpr); QUERY_CHECK_CODE(code, lino, _error); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); QUERY_CHECK_CODE(code, lino, _error); } - size_t keyBufSize = 0; - int32_t num = 0; - SExprInfo* pExprInfo = NULL; code = createExprInfo(pAnomalyNode->window.pFuncs, NULL, &pExprInfo, &num); QUERY_CHECK_CODE(code, lino, _error); initResultSizeInfo(&pOperator->resultInfo, 4096); - code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); + code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, id, pTaskInfo->streamInfo.pState, + &pTaskInfo->storageAPI.functionStore); QUERY_CHECK_CODE(code, lino, _error); SSDataBlock* pResBlock = createDataBlockFromDescNode(pAnomalyNode->window.node.pOutputDataBlockDesc); @@ -124,27 +126,19 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p pInfo->anomalyCol = extractColumnFromColumnNode(pColNode); pInfo->anomalyKey.type = pInfo->anomalyCol.type; pInfo->anomalyKey.bytes = pInfo->anomalyCol.bytes; + pInfo->anomalyKey.pData = taosMemoryCalloc(1, pInfo->anomalyCol.bytes); - if (pInfo->anomalyKey.pData == NULL) { - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalyKey.pData, code, lino, _error, terrno) int32_t itemSize = sizeof(int32_t) + pInfo->aggSup.resultRowSize + pInfo->anomalyKey.bytes; pInfo->anomalySup.pResultRow = taosMemoryCalloc(1, itemSize); - if (pInfo->anomalySup.pResultRow == NULL) { - code = terrno; - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalySup.pResultRow, code, lino, _error, terrno) + pInfo->anomalySup.blocks = taosArrayInit(16, sizeof(SSDataBlock*)); - if (pInfo->anomalySup.blocks == NULL) { - code = terrno; - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalySup.blocks, code, lino, _error, terrno) + pInfo->anomalySup.windows = taosArrayInit(16, sizeof(STimeWindow)); - if (pInfo->anomalySup.windows == NULL) { - code = terrno; - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalySup.windows, code, lino, _error, terrno) code = filterInitFromNode((SNode*)pAnomalyNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); QUERY_CHECK_CODE(code, lino, _error); @@ -162,18 +156,21 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p *pOptrInfo = pOperator; - qDebug("anomaly_window operator is created, algo:%s url:%s opt:%s", pInfo->algoName, pInfo->algoUrl, + qDebug("%s anomaly_window operator is created, algo:%s url:%s opt:%s", id, pInfo->algoName, pInfo->algoUrl, pInfo->anomalyOpt); return TSDB_CODE_SUCCESS; _error: + qError("%s failed to create anomaly_window operator, line:%d algo:%s code:%s", id, lino, pAnomalyNode->anomalyOpt, + tstrerror(code)); + if (pInfo != NULL) { anomalyDestroyOperatorInfo(pInfo); } destroyOperatorAndDownstreams(pOperator, &downstream, 1); pTaskInfo->code = code; - qError("failed to create anomaly_window operator, algo:%s code:0x%x", pInfo->algoName, code); + return code; } diff --git a/tests/script/tsim/analytics/basic0.sim b/tests/script/tsim/analytics/basic0.sim index 35774e7682..3ac49b1fc3 100644 --- a/tests/script/tsim/analytics/basic0.sim +++ b/tests/script/tsim/analytics/basic0.sim @@ -40,7 +40,7 @@ print $data00 $data01 $data02 sql use d0 print =============== create super table, include column type for count/sum/min/max/first -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double, c4 tinyint, c5 bigint, c6 varchar(12)) tags (t1 int unsigned) sql show stables if $rows != 1 then @@ -52,10 +52,11 @@ sql create table ct1 using stb tags(1000) print ==================== insert data # input_list = [5, 14, 15, 15, 14, 19, 17, 16, 20, 22, 8, 21, 28, 11, 9, 29, 40] -sql insert into ct1(ts, c1) values(now-1a, 5)(now+1a, 14)(now+2a, 15)(now+3a, 15)(now+4a, 14) -sql insert into ct1(ts, c1) values(now+5a, 19)(now+6a, 17)(now+7a, 16)(now+8a, 20)(now+9a, 22) -sql insert into ct1(ts, c1) values(now+10a, 8)(now+11a, 21)(now+12a, 28)(now+13a, 11)(now+14a, 9) -sql insert into ct1(ts, c1) values(now+15a, 29)(now+16a, 40) +sql insert into ct1(ts, c1, c2, c3, c4, c5, c6) values(now-1a, 5, 5, 5, 5, 5, 'a')(now+1a, 14, 14, 14, 14, 14, 'a')(now+2a, 15, 15, 15, 15, 15, 'a') +sql insert into ct1 values(now+3a, 15, 15, 15, 15, 15, 'a')(now+4a, 14, 14, 14, 14, 14, 'a')(now+5a, 19, 19, 19, 19, 19, 'a')(now+6a, 17, 17, 17, 17, 17, 'a') +sql insert into ct1 values(now+7a, 16, 16, 16, 16, 16, 'a')(now+8a, 20, 20, 20, 20, 20, 'a')(now+9a, 22, 22, 22, 22, 22, 'a') +sql insert into ct1 values(now+10a, 8, 8, 8, 8, 8, 'a')(now+11a, 21, 21, 21, 21, 21, 'a')(now+12a, 28, 28, 28, 28, 28, 'a')(now+13a, 11, 11, 11, 11, 11, 'a')(now+14a, 9, 9, 9, 9, 9, 'a') +sql insert into ct1 values(now+15a, 29, 29, 29, 29, 29, 'a')(now+16a, 40, 40, 40, 40, 40, 'a') sql select count(*) from ct1 if $data00 != 17 then @@ -68,6 +69,25 @@ if $data00 != 1 then return -1 endi +print ================= try every loaded anomaly detection algorithm +sql select count(*) from ct1 anomaly_window(c1, 'algo=iqr'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=ksigma'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=lof'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=shesd'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=grubbs'); + +print ================= try every column type of column +sql select count(*) from ct1 anomaly_window(c1, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c2, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c3, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c4, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c5, 'algo=ksigma,k=2'); + +print =================== invalid column type +sql_error select count(*) from ct1 anomaly_window(c6, 'algo=ksigma,k=2'); +sql_error select forecast(c6, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 + + sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=119,wncheck=1,period=0') from ct1 sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters1,conf=0.5,wncheck=1,period=0') from ct1 @@ -83,8 +103,15 @@ sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, ,' sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, , ,') from ct1 sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, a =') from ct1 sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, = a ,') from ct1 + +print =================== valid column type sql select forecast(c1, 'conf=50 ,algo = arima') from ct1 sql select forecast(c1, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c2, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c3, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c4, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c5, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c5, 'conf=50 ,algo = arima, rows=1') from ct1 sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=50,wncheck=1,period=0,start=1700000000000,every=2') from ct1 if $rows != 10 then @@ -131,6 +158,13 @@ if $rows != 0 then return -1 endi +sleep 1000 + +print ===================== query without anodes +sql_error select forecast(c5, 'conf=50 ,algo = arima, rows=1') from ct1 +sql_error select count(*) from ct1 anomaly_window(c1, 'algo=iqr'); + + _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check From 69aac6d778c23a4f93ff517133b6d880e2ecaf5a Mon Sep 17 00:00:00 2001 From: "pengrongkun94@qq.com" Date: Thu, 26 Dec 2024 18:40:07 +0800 Subject: [PATCH 05/35] finish stmt2 get fiels test --- source/client/test/stmt2Test.cpp | 615 +++++++++++++++++++------------ 1 file changed, 380 insertions(+), 235 deletions(-) diff --git a/source/client/test/stmt2Test.cpp b/source/client/test/stmt2Test.cpp index 267b777ff3..bf05f7788b 100644 --- a/source/client/test/stmt2Test.cpp +++ b/source/client/test/stmt2Test.cpp @@ -103,68 +103,12 @@ void do_query(TAOS* taos, const char* sql) { // printf("=================error test===================\n"); -// // case 14 : INSERT INTO db.d0 using db.stb values(?,?) -// // none para for ctbname -// sql = "INSERT INTO db.d0 using db.stb values(?,?)"; -// printf("case 14 (no tags error): %s\n", sql); -// getFields(taos, sql); - -// // case 15 : insert into db.stb(t1,t2,tbname) values(?,?,?) -// // no value -// sql = "insert into db.stb(t1,t2,tbname) values(?,?,?)"; -// printf("case 15 (no PK error): %s\n", sql); -// getFields(taos, sql); - -// // case 16 : insert into db.stb(ts,b,tbname) values(?,?,?,?,?) -// // wrong para nums -// sql = "insert into db.stb(ts,b,tbname) values(?,?,?,?,?)"; -// printf("case 16 (wrong para nums): %s\n", sql); -// getFields(taos, sql); - -// // case 17 : insert into db.? values(?,?) -// // normal table must have tbnam -// sql = "insert into db.? values(?,?)"; -// printf("case 17 (normal table must have tbname): %s\n", sql); -// getFields(taos, sql); - -// // case 18 : INSERT INTO db.stb(t1,t2,ts,b) values(?,?,?,?) -// // no tbname error -// sql = "INSERT INTO db.stb(t1,t2,ts,b) values(?,?,?,?)"; -// printf("case 18 (no tbname error): %s\n", sql); -// getFields(taos, sql); - -// // case 19 : insert into db.ntb(nts,ni) values(?,?,?,?,?) -// // wrong para nums -// sql = "insert into ntb(nts,ni) values(?,?,?,?,?)"; -// printf("case 19 : %s\n", sql); -// getFields(taos, sql); - -// // case 20 : insert into db.stb(t1,t2,ts,b,tbname) values(*,*,*,*,*) -// // wrong simbol -// sql = "insert into db.stb(t1,t2,ts,b,tbname) values(*,*,*,*,*)"; -// printf("=================normal test===================\n"); -// printf("case 20 : %s\n", sql); -// getFields(taos, sql); - -// // case 21 : INSERT INTO ! using db.stb TAGS(?,?) VALUES(?,?) -// // wrong simbol -// sql = "insert into ! using db.stb tags(?, ?) values(?,?)"; -// printf("case 21 : %s\n", sql); -// getFields(taos, sql); - // // case 22 : INSERT INTO ! using db.stb TAGS(?,?) VALUES(?,?) // // wrong tbname // sql = "insert into db.stb values(?,?)"; // printf("case 22 : %s\n", sql); // getFields(taos, sql); -// // case 23 : select * from ? where ts = ? -// // wrong query type -// sql = "select * from ? where ts = ?"; -// printf("case 23 : %s\n", sql); -// getQueryFields(taos, sql); -// } - } // namespace int main(int argc, char** argv) { @@ -229,13 +173,13 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { getFieldsSuccess(taos, sql, expectedFields, 5); } - // case 4 : random order + // case 4 : random order 2 { - const char* sql = "insert into db.stb(tbname,ts,t2,b,t1) values(?,?,?,?,?)"; - TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, - {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, - {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + const char* sql = "insert into db.stb(ts,tbname,b,t2,t1) values(?,?,?,?,?)"; + TAOS_FIELD_ALL expectedFields[5] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, + {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}}; printf("case 4 : %s\n", sql); getFieldsSuccess(taos, sql, expectedFields, 5); @@ -275,6 +219,50 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { getFieldsSuccess(taos, sql, expectedFields, 2); } + // case 8 : test all types + { + const char* sql = + "insert into " + "all_stb(tbname,tts,tv1,tv2,tv3,tv4,tv5,tv6,tv7,tv8,tv9,tv10,tv11,tv12,tv13,tv14,tv15,ts,v1,v2,v3,v4,v5,v6,v7," + "v8,v9,v10," + "v11,v12,v13,v14,v15) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + TAOS_FIELD_ALL expectedFields[33] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + {"tts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_TAG}, + {"tv1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_TAG}, + {"tv2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_TAG}, + {"tv3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_TAG}, + {"tv4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + {"tv5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_TAG}, + {"tv6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_TAG}, + {"tv7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_TAG}, + {"tv8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_TAG}, + {"tv9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_TAG}, + {"tv10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_TAG}, + {"tv11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_TAG}, + {"tv12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_TAG}, + {"tv13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_TAG}, + {"tv14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_TAG}, + {"tv15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_TAG}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"v1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_COL}, + {"v2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_COL}, + {"v3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_COL}, + {"v4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL}, + {"v5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_COL}, + {"v6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_COL}, + {"v7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_COL}, + {"v8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_COL}, + {"v9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_COL}, + {"v10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_COL}, + {"v11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_COL}, + {"v12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_COL}, + {"v13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_COL}, + {"v14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_COL}, + {"v15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_COL}}; + printf("case 8 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 33); + } + // not support case printf("not support case \n"); @@ -285,32 +273,60 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { // getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); // } - // case 7 : no pk + // case 2 : no pk { const char* sql = "insert into db.stb(b,tbname) values(?,?)"; + printf("case 2 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); + } + + // case 3 : no tbname and tag(not support bind) + { + const char* sql = "insert into db.stb(ts,b) values(?,?)"; + printf("case 3 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); + } + + // case 4 : no col and tag(not support bind) + { + const char* sql = "insert into db.stb(tbname) values(?)"; + printf("case 4 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); + } + + // case 5 : no field name + { + const char* sql = "insert into db.stb(?,?,?,?,?)"; + printf("case 5 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR); + } + + // case 6 : test super table not exist + { + const char* sql = "insert into db.nstb(?,?,?,?,?)"; + printf("case 6 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR); + } + + // case 7 : no col + { + const char* sql = "insert into db.stb(t1,t2,tbname) values(?,?,?)"; printf("case 7 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); } - // case 5 : no tbname and tag(not support bind) + // case 8 : wrong para nums { - const char* sql = "insert into db.stb(ts,b) values(?,?)"; - printf("case 5 : %s\n", sql); - getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); - } - - // case 5 : no tbname and tag(not support bind) - { - const char* sql = "insert into db.stb(tbname) values(?)"; - printf("case 5 : %s\n", sql); - getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); - } - - // case 8 : no param - { - const char* sql = "insert into db.stb(?,?,?,?,?)"; + const char* sql = "insert into db.stb(ts,b,tbname) values(?,?,?,?,?)"; printf("case 8 : %s\n", sql); - getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR); + getFieldsError(taos, sql, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); + } + + // case 9 : wrong simbol + { + const char* sql = "insert into db.stb(t1,t2,ts,b,tbname) values(*,*,*,*,*)"; + printf("case 9 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_TSC_SQL_SYNTAX_ERROR); } taos_close(taos); @@ -318,170 +334,295 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { } TEST(stmt2Case, insert_ctb_using_get_fields_Test) { - // do_query(taos, "CREATE TABLE db.d0 USING db.stb (t1,t2) TAGS (7,'Cali');"); + TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); - // // case 2 : INSERT INTO db.d0 VALUES (?,?) - // // test child table - // { - // const char* sql = "INSERT INTO db.d0(ts,b) VALUES (?,?)"; - // TAOS_FIELD_ALL expectedFields[2] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, - // {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; - // printf("case 2 : %s\n", sql); - // getFieldsSuccess(taos, sql, expectedFields, 2); - // } + do_query(taos, "drop database if exists db"); + do_query(taos, "create database db PRECISION 'ns'"); + do_query(taos, + "create table db.stb (ts timestamp, b binary(10)) tags(t1 " + "int, t2 binary(10))"); + do_query( + taos, + "create table if not exists db.all_stb(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 " + "tinyint unsigned, v7 smallint unsigned, v8 int unsigned, v9 bigint unsigned, v10 float, v11 double, v12 " + "binary(20), v13 varbinary(20), v14 geometry(100), v15 nchar(20))tags(tts timestamp, tv1 bool, tv2 tinyint, tv3 " + "smallint, tv4 int, tv5 bigint, tv6 tinyint unsigned, tv7 smallint unsigned, tv8 int unsigned, tv9 bigint " + "unsigned, tv10 float, tv11 double, tv12 binary(20), tv13 varbinary(20), tv14 geometry(100), tv15 nchar(20));"); + do_query(taos, "CREATE TABLE db.t0 USING db.stb (t1,t2) TAGS (7,'Cali');"); - // // case 6 : INSERT INTO db.? using db.stb (t1,t2)TAGS(?,?) (ts,b)VALUES(?,?) - // // normal insert clause - // { - // const char* sql = "INSERT INTO db.? using db.stb (t1,t2)TAGS(?,?) (ts,b)VALUES(?,?)"; - // TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, - // {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, - // {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, - // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, - // {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; - // printf("case 6 : %s\n", sql); - // getFieldsSuccess(taos, sql, expectedFields, 5); - // } + printf("support case \n"); + // case 1 : test child table already exist + { + const char* sql = "INSERT INTO db.t0(ts,b)using db.stb (t1,t2) TAGS(?,?) VALUES (?,?)"; + TAOS_FIELD_ALL expectedFields[4] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; + printf("case 1 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 4); + } - // // case 7 : insert into db.? using db.stb(t2,t1) tags(?, ?) (b,ts)values(?,?) - // // disordered - // { - // const char* sql = "insert into db.? using db.stb(t2,t1) tags(?, ?) (b,ts)values(?,?)"; - // TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, - // {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, - // {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, - // {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, - // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}}; - // printf("case 7 : %s\n", sql); - // getFieldsSuccess(taos, sql, expectedFields, 5); - // } + // case 2 : insert clause + { + const char* sql = "INSERT INTO db.? using db.stb (t1,t2) TAGS(?,?) (ts,b)VALUES(?,?)"; + TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; + printf("case 2 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 5); + } - // // case 8 : insert into db.? using db.stb tags(?, ?) values(?,?) - // // no field name - // { - // const char* sql = "insert into db.? using db.stb tags(?, ?) values(?,?)"; - // TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, - // {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, - // {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, - // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, - // {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; - // printf("case 8 : %s\n", sql); - // getFieldsSuccess(taos, sql, expectedFields, 5); - // } + // case 3 : insert child table not exist + { + const char* sql = "INSERT INTO db.d1 using db.stb (t1,t2)TAGS(?,?) (ts,b)VALUES(?,?)"; + TAOS_FIELD_ALL expectedFields[4] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; + printf("case 3 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 4); + } - // // case 9 : insert into db.? using db.stb (t2)tags(?) (ts)values(?) - // // less para - // { - // const char* sql = "insert into db.? using db.stb (t2)tags(?) (ts)values(?)"; - // TAOS_FIELD_ALL expectedFields[3] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, - // {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, - // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}}; - // printf("case 9 : %s\n", sql); - // getFieldsSuccess(taos, sql, expectedFields, 3); - // } + // case 4 : random order + { + const char* sql = "INSERT INTO db.? using db.stb (t2,t1)TAGS(?,?) (b,ts)VALUES(?,?)"; + TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}}; + printf("case 4 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 5); + } - // // case 10 : insert into db.d0 (ts)values(?) - // // less para - // { - // const char* sql = "insert into db.d0 (ts)values(?)"; - // TAOS_FIELD_ALL expectedFields[1] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}}; - // printf("case 10 : %s\n", sql); - // getFieldsSuccess(taos, sql, expectedFields, 1); - // } - // // case 11 : insert into abc using stb tags(?, ?) values(?,?) - // // insert create table - // { - // const char* sql = "insert into abc using stb tags(?, ?) values(?,?)"; - // TAOS_FIELD_ALL expectedFields[4] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, - // {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, - // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, - // {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; - // printf("case 11 : %s\n", sql); - // getFieldsSuccess(taos, sql, expectedFields, 4); - // } + // case 5 : less para + { + const char* sql = "insert into db.? using db.stb (t2)tags(?) (ts)values(?)"; + TAOS_FIELD_ALL expectedFields[3] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}}; + printf("case 5 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 3); + } - // // // case 13 : insert into ? using all_stb tags(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) - // // values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) - // // // test all types - // // sql = - // // "insert into all_stb " - // // - // "(tbname,tts,tv1,tv2,tv3,tv4,tv5,tv6,tv7,tv8,tv9,tv10,tv11,tv12,tv13,tv14,tv15,ts,v1,v2,v3,v4,v5,v6,v7,v8,v9,v10," - // // "v11,v12,v13,v14,v15) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; - // // printf("case 13 : %s\n", sql); - // // getFields(taos, sql); + // case 6 : insert into db.? using db.stb tags(?, ?) values(?,?) + // no field name + { + const char* sql = "insert into db.? using db.stb tags(?, ?) values(?,?)"; + TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; + printf("case 6 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 5); + } - // case 12 : insert into ? using all_stb tags(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) - // values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) - // test all types - // { - // const char* sql = - // "insert into ? using all_stb tags(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; - // TAOS_FIELD_ALL expectedFields[33] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, - // {"tts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_TAG}, - // {"tv1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_TAG}, - // {"tv2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_TAG}, - // {"tv3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_TAG}, - // {"tv4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, - // {"tv5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_TAG}, - // {"tv6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_TAG}, - // {"tv7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_TAG}, - // {"tv8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_TAG}, - // {"tv9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_TAG}, - // {"tv10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_TAG}, - // {"tv11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_TAG}, - // {"tv12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_TAG}, - // {"tv13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_TAG}, - // {"tv14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_TAG}, - // {"tv15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_TAG}, - // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, - // {"v1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_COL}, - // {"v2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_COL}, - // {"v3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_COL}, - // {"v4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL}, - // {"v5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_COL}, - // {"v6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_COL}, - // {"v7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_COL}, - // {"v8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_COL}, - // {"v9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_COL}, - // {"v10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_COL}, - // {"v11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_COL}, - // {"v12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_COL}, - // {"v13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_COL}, - // {"v14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_COL}, - // {"v15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_COL}}; - // printf("case 12 : %s\n", sql); - // getFieldsSuccess(taos, sql, expectedFields, 33); - // } + // case 7 : insert into db.d0 (ts)values(?) + // less para + { + const char* sql = "insert into db.t0 (ts)values(?)"; + TAOS_FIELD_ALL expectedFields[1] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}}; + printf("case 7 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 1); + } + + // case 8 : 'db' 'stb' + { + const char* sql = "INSERT INTO 'db'.? using 'db'.'stb' (t1,t2) TAGS(?,?) (ts,b)VALUES(?,?)"; + TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; + printf("case 8 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 5); + } + + // case 9 : use db + { + do_query(taos, "use db"); + const char* sql = "INSERT INTO ? using stb (t1,t2) TAGS(?,?) (ts,b)VALUES(?,?)"; + TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; + printf("case 9 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 5); + } + + // case 10 : test all types + { + const char* sql = + "insert into ? using all_stb tags(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + TAOS_FIELD_ALL expectedFields[33] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, + {"tts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_TAG}, + {"tv1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_TAG}, + {"tv2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_TAG}, + {"tv3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_TAG}, + {"tv4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, + {"tv5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_TAG}, + {"tv6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_TAG}, + {"tv7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_TAG}, + {"tv8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_TAG}, + {"tv9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_TAG}, + {"tv10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_TAG}, + {"tv11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_TAG}, + {"tv12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_TAG}, + {"tv13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_TAG}, + {"tv14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_TAG}, + {"tv15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_TAG}, + {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, + {"v1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_COL}, + {"v2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_COL}, + {"v3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_COL}, + {"v4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL}, + {"v5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_COL}, + {"v6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_COL}, + {"v7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_COL}, + {"v8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_COL}, + {"v9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_COL}, + {"v10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_COL}, + {"v11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_COL}, + {"v12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_COL}, + {"v13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_COL}, + {"v14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_COL}, + {"v15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_COL}}; + printf("case 10 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 33); + } + printf("not support case \n"); + + // case 1 : test super table not exist + { + const char* sql = "INSERT INTO db.?(ts,b)using db.nstb (t1,t2) TAGS(?,?) VALUES (?,?)"; + printf("case 1 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR); + } + + // case 2 : no pk + { + const char* sql = "INSERT INTO db.?(ts,b)using db.nstb (t1,t2) TAGS(?,?) (n)VALUES (?)"; + printf("case 2 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR); + } + + // case 3 : less param and no filed name + { + const char* sql = "INSERT INTO db.?(ts,b)using db.stb TAGS(?)VALUES (?,?)"; + printf("case 3 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR); + } + + // case 4 : none para for ctbname + { + const char* sql = "INSERT INTO db.d0 using db.stb values(?,?)"; + printf("case 4 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_TSC_SQL_SYNTAX_ERROR); + } + + // case 5 : none para for ctbname + { + const char* sql = "insert into ! using db.stb tags(?, ?) values(?,?)"; + printf("case 5 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_TSC_SQL_SYNTAX_ERROR); + } } TEST(stmt2Case, insert_ntb_get_fields_Test) { - // do_query(taos, "CREATE TABLE db.ntb(nts timestamp, nb binary(10),nvc varchar(16),ni int);"); + TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); - // // // case 3 : INSERT INTO db.ntb VALUES(?,?,?,?) - // // // test normal table - // { - // const char* sql = "INSERT INTO db.ntb VALUES(?,?,?,?)"; - // TAOS_FIELD_ALL expectedFields[4] = {{"nts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, - // {"nb", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, - // {"nvc", TSDB_DATA_TYPE_BINARY, 0, 0, 18, TAOS_FIELD_COL}, - // {"ni", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL}}; - // printf("case 3 : %s\n", sql); - // getFieldsSuccess(taos, sql, expectedFields, 4); - // } + do_query(taos, "drop database if exists db"); + do_query(taos, "create database db PRECISION 'ms'"); + do_query(taos, "CREATE TABLE db.ntb(nts timestamp, nb binary(10),nvc varchar(16),ni int);"); + do_query( + taos, + "create table if not exists db.all_ntb(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 " + "tinyint unsigned, v7 smallint unsigned, v8 int unsigned, v9 bigint unsigned, v10 float, v11 double, v12 " + "binary(20), v13 varbinary(20), v14 geometry(100), v15 nchar(20));"); - // { - // // case 4 : insert into db.stb(t1,tbname,ts,t2,b) values(?,?,?,?,?) - // // test random order - // const char* sql = "insert into db.stb(t1,tbname,ts,t2,b) values(?,?,?,?,?)"; - // TAOS_FIELD_ALL expectedFields[5] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, - // {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, - // {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, - // {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, - // {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}}; - // printf("case 4 : %s\n", sql); - // getFieldsSuccess(taos, sql, expectedFields, 5); - // } + printf("support case \n"); + + // case 1 : test normal table no field name + { + const char* sql = "INSERT INTO db.ntb VALUES(?,?,?,?)"; + TAOS_FIELD_ALL expectedFields[4] = {{"nts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0, 8, TAOS_FIELD_COL}, + {"nb", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, + {"nvc", TSDB_DATA_TYPE_BINARY, 0, 0, 18, TAOS_FIELD_COL}, + {"ni", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL}}; + printf("case 1 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 4); + } + + // case 2 : test random order + { + const char* sql = "INSERT INTO db.ntb (ni,nb,nvc,nts)VALUES(?,?,?,?)"; + TAOS_FIELD_ALL expectedFields[4] = {{"ni", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL}, + {"nb", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, + {"nvc", TSDB_DATA_TYPE_BINARY, 0, 0, 18, TAOS_FIELD_COL}, + {"nts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0, 8, TAOS_FIELD_COL}}; + printf("case 2 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 4); + } + + // case 3 : less param + { + const char* sql = "INSERT INTO db.ntb (nts)VALUES(?)"; + TAOS_FIELD_ALL expectedFields[1] = {{"nts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0, 8, TAOS_FIELD_COL}}; + printf("case 3 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 1); + } + + // case 4 : test all types + { + const char* sql = "insert into db.all_ntb values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + TAOS_FIELD_ALL expectedFields[16] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0, 8, TAOS_FIELD_COL}, + {"v1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_COL}, + {"v2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_COL}, + {"v3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_COL}, + {"v4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL}, + {"v5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_COL}, + {"v6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_COL}, + {"v7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_COL}, + {"v8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_COL}, + {"v9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_COL}, + {"v10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_COL}, + {"v11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_COL}, + {"v12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_COL}, + {"v13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_COL}, + {"v14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_COL}, + {"v15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_COL}}; + printf("case 4 : %s\n", sql); + getFieldsSuccess(taos, sql, expectedFields, 16); + } + + printf("not support case \n"); + + // case 1 : wrong db + { + const char* sql = "insert into ntb values(?,?,?,?)"; + printf("case 1 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); + } + + // case 2 : normal table must have tbnam + { + const char* sql = "insert into db.? values(?,?)"; + printf("case 2 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_PAR_TABLE_NOT_EXIST); + } + + // case 3 : wrong para nums + { + const char* sql = "insert into db.ntb(nts,ni) values(?,?,?,?,?)"; + printf("case 3 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); + } } TEST(stmt2Case, select_get_fields_Test) { @@ -492,20 +633,25 @@ TEST(stmt2Case, select_get_fields_Test) { do_query(taos, "use db"); do_query(taos, "CREATE TABLE db.ntb(nts timestamp, nb binary(10),nvc varchar(16),ni int);"); { - // case 1 : select * from ntb where ts = ? - // query type + // case 1 : const char* sql = "select * from ntb where ts = ?"; printf("case 1 : %s\n", sql); getQueryFields(taos, sql, 1); } { - // case 2 : select * from ntb where ts = ? and b = ? - // query type + // case 2 : const char* sql = "select * from ntb where ts = ? and b = ?"; printf("case 2 : %s\n", sql); getQueryFields(taos, sql, 2); } + + { + // case 3 : + const char* sql = "select * from ? where ts = ?"; + printf("case 3 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR); + } } TEST(stmt2Case, get_fields_error_Test) { @@ -519,7 +665,6 @@ TEST(stmt2Case, get_fields_error_Test) { TEST(stmt2Case, stmt2_init_prepare_Test) { TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - { (void)taos_stmt2_init(NULL, NULL); ASSERT_EQ(terrno, TSDB_CODE_INVALID_PARA); From 34a4608c7de04d01d8decadf2b9cf10c2602d2a6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 26 Dec 2024 20:10:13 +0800 Subject: [PATCH 06/35] enh: add test case --- tests/army/cluster/tsdbSnapshot.py | 264 +++++++++++++++++++++++++++++ tests/army/frame/common.py | 10 ++ tests/parallel_test/cases.task | 1 + 3 files changed, 275 insertions(+) create mode 100644 tests/army/cluster/tsdbSnapshot.py diff --git a/tests/army/cluster/tsdbSnapshot.py b/tests/army/cluster/tsdbSnapshot.py new file mode 100644 index 0000000000..5c4a6d2c80 --- /dev/null +++ b/tests/army/cluster/tsdbSnapshot.py @@ -0,0 +1,264 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from datetime import timedelta +import sys +import time +import random + +import taos +import frame +import frame.etool + + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.srvCtl import * +from frame.clusterCommonCheck import clusterComCheck + + +class TDTestCase(TBase): + def init(self, conn, logSql, replicaVar=3): + # super(TDTestCase, self).init(conn, logSql, replicaVar=3, db="db") + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), logSql) + self.vgroupNum = 3 + self.dbName = 'test' + self.dnode1Path = tdCom.getTaosdPath() + self.dnode1Cfg = f'{self.dnode1Path}/cfg' + self.dnode1Log = f'{self.dnode1Path}/log' + + def _write_bulk_data(self): + tdLog.info("============== write bulk data ===============") + json_content = f""" +{{ + "filetype": "insert", + "cfgdir": "{self.dnode1Cfg}", + "host": "localhost", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "thread_count": 16, + "create_table_thread_count": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 5, + "num_of_records_per_req": 1540, + "prepared_rand": 10000, + "chinese": "no", + "databases": [ + {{ + "dbinfo": {{ + "name": "{self.dbName}", + "drop": "yes", + "vgroups": 1, + "duration": "10d", + "wal_retention_period": 0, + "replica": 3, + "stt_trigger": 2 + }}, + "super_tables": [ + {{ + "name": "stb", + "child_table_exists": "no", + "childtable_count": 100, + "childtable_prefix": "ctb", + "escape_character": "yes", + "auto_create_table": "no", + "batch_create_tbl_num": 500, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 10000, + "interlace_rows": 1, + "insert_interval": 0, + "partial_col_num": 0, + "disorder_ratio": 0, + "disorder_range": 0, + "timestamp_step": 1000, + "start_timestamp": "{(datetime.now() - timedelta(days=1)).replace(hour=10, minute=0, second=0, microsecond=0).strftime('%Y-%m-%d %H:%M:%S')}", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + {{ + "type": "bigint", + "count": 10 + }} + ], + "tags": [ + {{ + "type": "TINYINT", + "name": "groupid", + "max": 10, + "min": 1 + }}, + {{ + "name": "location", + "type": "BINARY", + "len": 16, + "values": [ + "beijing", + "shanghai" + ] + }} + ] + }} + ] + }} + ] +}} +""" + json_file = '/tmp/test.json' + with open(json_file, 'w') as f: + f.write(json_content) + # Use subprocess.run() to wait for the command to finish + subprocess.run(f'taosBenchmark -f {json_file}', shell=True, check=True) + + def _write_bulk_data2(self): + tdLog.info("============== write bulk data ===============") + json_content = f""" +{{ + "filetype": "insert", + "cfgdir": "{self.dnode1Cfg}", + "host": "localhost", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "thread_count": 16, + "create_table_thread_count": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 5, + "num_of_records_per_req": 1540, + "prepared_rand": 10000, + "chinese": "no", + "databases": [ + {{ + "dbinfo": {{ + "name": "{self.dbName}", + "drop": "no", + "vgroups": 1, + "duration": "10d", + "wal_retention_period": 0, + "replica": 3, + "stt_trigger": 2 + }}, + "super_tables": [ + {{ + "name": "stb", + "child_table_exists": "yes", + "childtable_count": 100, + "childtable_prefix": "ctb", + "escape_character": "yes", + "auto_create_table": "no", + "batch_create_tbl_num": 500, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 10000, + "interlace_rows": 1, + "insert_interval": 0, + "partial_col_num": 0, + "disorder_ratio": 0, + "disorder_range": 0, + "timestamp_step": 1000, + "start_timestamp": "{(datetime.now() - timedelta(days=1)).replace(hour=14, minute=0, second=0, microsecond=0).strftime('%Y-%m-%d %H:%M:%S')}", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + {{ + "type": "bigint", + "count": 10 + }} + ], + "tags": [ + {{ + "type": "TINYINT", + "name": "groupid", + "max": 10, + "min": 1 + }}, + {{ + "name": "location", + "type": "BINARY", + "len": 16, + "values": [ + "beijing", + "shanghai" + ] + }} + ] + }} + ] + }} + ] +}} +""" + json_file = '/tmp/test.json' + with open(json_file, 'w') as f: + f.write(json_content) + # Use subprocess.run() to wait for the command to finish + subprocess.run(f'taosBenchmark -f {json_file}', shell=True, check=True) + + def run(self): + tdLog.info("============== write bulk data ===============") + self._write_bulk_data() + + tdSql.execute(f'flush database {self.dbName}') + tdLog.sleep(10) + + tdLog.info("============== stop dnode 3 ===============") + cluster.dnodes[2].stoptaosd() + tdLog.sleep(10) + + # tdLog.info("============== write more data ===============") + self._write_bulk_data2() + + tdSql.execute(f'flush database {self.dbName}') + tdLog.sleep(10) + + cluster.dnodes[0].stoptaosd() + cluster.dnodes[1].stoptaosd() + + dnode1_wal = f'{self.dnode1Path}/data/vnode/vnode2/wal' + dnode2_wal = f'{self.dnode1Path}/../dnode2/data/vnode/vnode2/wal' + + tdLog.info("============== remove wal files ===============") + tdLog.info(f"{dnode1_wal}") + tdLog.info(f"{dnode2_wal}") + os.system(f'rm -rf {dnode1_wal}/*') + os.system(f'rm -rf {dnode2_wal}/*') + + tdLog.info("============== restart cluster ===============") + cluster.dnodes[0].starttaosd() + cluster.dnodes[1].starttaosd() + cluster.dnodes[2].starttaosd() + + tdLog.sleep(10) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/common.py b/tests/army/frame/common.py index 21dc568713..2a7fcfdb9c 100644 --- a/tests/army/frame/common.py +++ b/tests/army/frame/common.py @@ -214,6 +214,16 @@ class TDCom: telnet_url = "http://127.0.0.1:6041/opentsdb/v1/put/telnet" return header, sql_url, sqlt_url, sqlutc_url, influx_url, telnet_url + def getTaosdPath(self, dnodeID="dnode1"): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + taosdPath = buildPath + "/../sim/" + dnodeID + tdLog.info("taosdPath: %s" % taosdPath) + return taosdPath + def genTcpParam(self): MaxBytes = 1024*1024 host ='127.0.0.1' diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index d58d7afe12..2035d80705 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -34,6 +34,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f cluster/test_drop_table_by_uid.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f cluster/incSnapshot.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f cluster/clusterBasic.py -N 5 +,,y,army,./pytest.sh python3 ./test.py -f cluster/tsdbSnapshot.py -N 3 -M 3 ,,y,army,./pytest.sh python3 ./test.py -f query/query_basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/accuracy/test_query_accuracy.py ,,y,army,./pytest.sh python3 ./test.py -f query/accuracy/test_ts5400.py From 7283d54770d52c8bc5a8a09349446f989d30779e Mon Sep 17 00:00:00 2001 From: "pengrongkun94@qq.com" Date: Thu, 26 Dec 2024 22:22:06 +0800 Subject: [PATCH 07/35] add bind exec test --- source/client/test/stmt2Test.cpp | 111 ++++++++++++++++++++++++++++--- 1 file changed, 103 insertions(+), 8 deletions(-) diff --git a/source/client/test/stmt2Test.cpp b/source/client/test/stmt2Test.cpp index bf05f7788b..4542f953ab 100644 --- a/source/client/test/stmt2Test.cpp +++ b/source/client/test/stmt2Test.cpp @@ -14,7 +14,7 @@ */ #include -#include +#include #include "clientInt.h" #include "osSemaphore.h" #include "taoserror.h" @@ -99,15 +99,100 @@ void do_query(TAOS* taos, const char* sql) { taos_free_result(result); } -// void do_stmt(TAOS* taos) { +void do_stmt(TAOS* taos, const char* sql, int CTB_NUMS, int ROW_NUMS, int CYC_NUMS, bool createTable) { + do_query(taos, "drop database if exists db"); + do_query(taos, "create database db"); + do_query(taos, "create table db.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))"); + do_query(taos, "use db"); -// printf("=================error test===================\n"); + TAOS_STMT2_OPTION option = {0, true, true, NULL, NULL}; -// // case 22 : INSERT INTO ! using db.stb TAGS(?,?) VALUES(?,?) -// // wrong tbname -// sql = "insert into db.stb values(?,?)"; -// printf("case 22 : %s\n", sql); -// getFields(taos, sql); + TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); + ASSERT_NE(stmt, nullptr); + int code = taos_stmt2_prepare(stmt, sql, 0); + ASSERT_EQ(code, 0); + ASSERT_EQ(terrno, 0); + + // tbname + char** tbs = (char**)taosMemoryMalloc(CTB_NUMS * sizeof(char*)); + for (int i = 0; i < CTB_NUMS; i++) { + tbs[i] = (char*)taosMemoryMalloc(sizeof(char) * 20); + sprintf(tbs[i], "ctb_%d", i); + if (createTable) { + char* tmp = (char*)taosMemoryMalloc(sizeof(char) * 100); + sprintf(tmp, "create table db.%s using db.stb tags(0, 'after')", tbs[i]); + do_query(taos, tmp); + } + } + for (int r = 0; r < CYC_NUMS; r++) { + // col params + int64_t** ts = (int64_t**)taosMemoryMalloc(CTB_NUMS * sizeof(int64_t*)); + char** b = (char**)taosMemoryMalloc(CTB_NUMS * sizeof(char*)); + int* ts_len = (int*)taosMemoryMalloc(ROW_NUMS * sizeof(int)); + int* b_len = (int*)taosMemoryMalloc(ROW_NUMS * sizeof(int)); + for (int i = 0; i < ROW_NUMS; i++) { + ts_len[i] = sizeof(int64_t); + b_len[i] = 1; + } + for (int i = 0; i < CTB_NUMS; i++) { + ts[i] = (int64_t*)taosMemoryMalloc(ROW_NUMS * sizeof(int64_t)); + b[i] = (char*)taosMemoryMalloc(ROW_NUMS * sizeof(char)); + for (int j = 0; j < ROW_NUMS; j++) { + ts[i][j] = 1591060628000 + r * 100000 + j; + b[i][j] = 'a' + j; + } + } + // tag params + int t1 = 0; + int t1len = sizeof(int); + int t2len = 3; + // TAOS_STMT2_BIND* tagv[2] = {&tags[0][0], &tags[1][0]}; + + // bind params + TAOS_STMT2_BIND** paramv = (TAOS_STMT2_BIND**)taosMemoryMalloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*)); + TAOS_STMT2_BIND** tags = (TAOS_STMT2_BIND**)taosMemoryMalloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*)); + for (int i = 0; i < CTB_NUMS; i++) { + // create tags + tags[i] = (TAOS_STMT2_BIND*)taosMemoryMalloc(2 * sizeof(TAOS_STMT2_BIND)); + tags[i][0] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_INT, &t1, &t1len, NULL, 0}; + tags[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, (void*)"after", &t2len, NULL, 0}; + + // create col params + paramv[i] = (TAOS_STMT2_BIND*)taosMemoryMalloc(2 * sizeof(TAOS_STMT2_BIND)); + paramv[i][0] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_TIMESTAMP, &ts[i][0], &ts_len[0], NULL, ROW_NUMS}; + paramv[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, &b[i][0], &b_len[0], NULL, ROW_NUMS}; + } + // bind + TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, tags, paramv}; + code = taos_stmt2_bind_param(stmt, &bindv, -1); + ASSERT_EQ(code, 0); + ASSERT_EQ(errno, 0); + + // exec + code = taos_stmt2_exec(stmt, NULL); + ASSERT_EQ(code, 0); + ASSERT_EQ(errno, 0); + + for (int i = 0; i < CTB_NUMS; i++) { + taosMemoryFree(tags[i]); + taosMemoryFree(paramv[i]); + taosMemoryFree(ts[i]); + taosMemoryFree(b[i]); + } + taosMemoryFree(ts); + taosMemoryFree(b); + taosMemoryFree(ts_len); + taosMemoryFree(b_len); + taosMemoryFree(paramv); + taosMemoryFree(tags); + } + for (int i = 0; i < CTB_NUMS; i++) { + taosMemoryFree(tbs[i]); + } + taosMemoryFree(tbs); + + taos_stmt2_close(stmt); +} } // namespace @@ -665,6 +750,7 @@ TEST(stmt2Case, get_fields_error_Test) { TEST(stmt2Case, stmt2_init_prepare_Test) { TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); { (void)taos_stmt2_init(NULL, NULL); ASSERT_EQ(terrno, TSDB_CODE_INVALID_PARA); @@ -711,6 +797,15 @@ TEST(stmt2Case, stmt2_init_prepare_Test) { } } +TEST(stmt2Case, stmt2_all) { + TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 0); + ASSERT_NE(taos, nullptr); + + { do_stmt(taos, "insert into db.stb (tbname,ts,b,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true); } + taos_close(taos); + taos_cleanup(); +} + TEST(stmt2Case, stmt2_status_Test) {} #pragma GCC diagnostic pop From 68ece75ad050a0972934a94a4c6e63bf031ac66d Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 27 Dec 2024 10:27:05 +0800 Subject: [PATCH 08/35] fix stream state operator issue --- .../executor/src/streamtimewindowoperator.c | 17 ++++++- tests/script/tsim/stream/basic5.sim | 49 ++++++++++++++----- 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 91af218a7e..031d2e8bdc 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -4466,6 +4466,19 @@ _end: return code; } +static bool isWinResult(SSessionKey* pKey, SSHashObj* pSeUpdate, SSHashObj* pResults) { + SSessionKey checkKey = {0}; + getSessionHashKey(pKey, &checkKey); + if (tSimpleHashGet(pSeUpdate, &checkKey, sizeof(SSessionKey)) != NULL) { + return true; + } + + if (tSimpleHashGet(pResults, &checkKey, sizeof(SSessionKey)) != NULL) { + return true; + } + return false; +} + static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated, SSHashObj* pStDeleted) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -4518,7 +4531,9 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin); QUERY_CHECK_CODE(code, lino, _end); - releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore); + if (isWinResult(&nextWin.winInfo.sessionWin, pSeUpdated, pAggSup->pResultRows) == false) { + releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore); + } setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); code = updateStateWindowInfo(pAggSup, &curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual, diff --git a/tests/script/tsim/stream/basic5.sim b/tests/script/tsim/stream/basic5.sim index d7dd603d3c..5d90740b13 100644 --- a/tests/script/tsim/stream/basic5.sim +++ b/tests/script/tsim/stream/basic5.sim @@ -2,13 +2,28 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c debugflag -v 135 system sh/cfg.sh -n dnode1 -c streamBufferSize -v 10 +system sh/cfg.sh -n dnode1 -c checkpointinterval -v 60 +system sh/cfg.sh -n dnode1 -c snodeAddress -v 127.0.0.1:873 system sh/exec.sh -n dnode1 -s start sleep 500 sql connect +print step1 ============= -print step1============= +print ================ create snode +sql show snodes +if $rows != 0 then + return -1 +endi + +sql create snode on dnode 1; +sql show snodes; +if $rows != 1 then + return -1 +endi + +print ============== snode created , create db sql create database test3 vgroups 1; sql use test3; @@ -57,7 +72,7 @@ loop8: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -87,7 +102,7 @@ loop9: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -127,7 +142,7 @@ loop10: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -162,7 +177,7 @@ loop11: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -194,7 +209,7 @@ loop11: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -239,7 +254,7 @@ loop12: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -315,7 +330,7 @@ loop13: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -369,7 +384,6 @@ if $data24 != 1 then endi print step4============= - sql create database test6 vgroups 4; sql use test6; sql create stable st(ts timestamp,a int,b int,c int,d int) tags(ta int,tb int,tc int); @@ -396,7 +410,7 @@ loop14: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -449,4 +463,17 @@ if $data25 != 2 then goto loop14 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +print sleep for 1min for checkpoint generate +sleep 60000 + +print ================== restart to load checkpoint from snode + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +sleep 500 +sql connect + +sleep 30000 + +sql select start_ver, checkpoint_ver from information_schema.ins_stream_tasks; From de62d9bb836745035e7d477ddcae0283888fef24 Mon Sep 17 00:00:00 2001 From: "pengrongkun94@qq.com" Date: Fri, 27 Dec 2024 10:44:02 +0800 Subject: [PATCH 09/35] fix some case --- source/client/test/stmt2Test.cpp | 34 +++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/source/client/test/stmt2Test.cpp b/source/client/test/stmt2Test.cpp index 4542f953ab..ada3b66dc9 100644 --- a/source/client/test/stmt2Test.cpp +++ b/source/client/test/stmt2Test.cpp @@ -33,6 +33,19 @@ #include "taos.h" namespace { + +void checkRows(TAOS* pConn, const char* sql, int32_t expectedRows) { + TAOS_RES* pRes = taos_query(pConn, sql); + ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); + TAOS_ROW pRow = NULL; + int rows = 0; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + rows++; + } + ASSERT_EQ(rows, expectedRows); + taos_free_result(pRes); +} + void stmtAsyncQueryCb(void* param, TAOS_RES* pRes, int code) { int affected_rows = taos_affected_rows(pRes); return; @@ -186,6 +199,8 @@ void do_stmt(TAOS* taos, const char* sql, int CTB_NUMS, int ROW_NUMS, int CYC_NU taosMemoryFree(paramv); taosMemoryFree(tags); } + + checkRows(taos, "select * from db.stb", CYC_NUMS * ROW_NUMS * CTB_NUMS); for (int i = 0; i < CTB_NUMS; i++) { taosMemoryFree(tbs[i]); } @@ -352,11 +367,11 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { printf("not support case \n"); // case 5 : add in main TD-33353 - // { - // const char* sql = "insert into db.stb(t1,t2,ts,b,tbname) values(1,?,?,'abc',?)"; - // printf("case 2 : %s\n", sql); - // getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); - // } + { + const char* sql = "insert into db.stb(t1,t2,ts,b,tbname) values(1,?,?,'abc',?)"; + printf("case 2 : %s\n", sql); + getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); + } // case 2 : no pk { @@ -411,11 +426,10 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { { const char* sql = "insert into db.stb(t1,t2,ts,b,tbname) values(*,*,*,*,*)"; printf("case 9 : %s\n", sql); - getFieldsError(taos, sql, TSDB_CODE_TSC_SQL_SYNTAX_ERROR); + getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); } taos_close(taos); - taos_cleanup(); } TEST(stmt2Case, insert_ctb_using_get_fields_Test) { @@ -616,6 +630,7 @@ TEST(stmt2Case, insert_ctb_using_get_fields_Test) { printf("case 5 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_TSC_SQL_SYNTAX_ERROR); } + taos_close(taos); } TEST(stmt2Case, insert_ntb_get_fields_Test) { @@ -737,6 +752,7 @@ TEST(stmt2Case, select_get_fields_Test) { printf("case 3 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR); } + taos_close(taos); } TEST(stmt2Case, get_fields_error_Test) { @@ -776,7 +792,7 @@ TEST(stmt2Case, stmt2_init_prepare_Test) { code = taos_stmt2_prepare(stmt, "insert into 'db'.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)", 0); ASSERT_EQ(terrno, 0); ASSERT_NE(stmt, nullptr); - // ASSERT_STREQ(((STscStmt2*)stmt)->db, "db"); //add in main TD-33332 + ASSERT_STREQ(((STscStmt2*)stmt)->db, "db"); // add in main TD-33332 taos_stmt2_close(stmt); } @@ -795,6 +811,7 @@ TEST(stmt2Case, stmt2_init_prepare_Test) { ASSERT_NE(stmt, nullptr); taos_stmt2_close(stmt); } + taos_close(taos); } TEST(stmt2Case, stmt2_all) { @@ -803,7 +820,6 @@ TEST(stmt2Case, stmt2_all) { { do_stmt(taos, "insert into db.stb (tbname,ts,b,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true); } taos_close(taos); - taos_cleanup(); } TEST(stmt2Case, stmt2_status_Test) {} From 7a6eb1d9697975b520a31ee55f7da395f59ca33f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 27 Dec 2024 11:19:24 +0800 Subject: [PATCH 10/35] test: update the sim, set the output as uint, instead of int to avoid overflow. --- source/libs/function/src/builtinsimpl.c | 2 +- tests/script/tsim/stream/basic5.sim | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1c77d12fb2..707018ac65 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -6429,7 +6429,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return code; } - len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Inmem_Rows=[%d] Stt_Rows=[%d] ", + len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Inmem_Rows=[%u] Stt_Rows=[%u] ", pData->numOfInmemRows, pData->numOfSttRows); varDataSetLen(st, len); code = colDataSetVal(pColInfo, row++, st, false); diff --git a/tests/script/tsim/stream/basic5.sim b/tests/script/tsim/stream/basic5.sim index 5d90740b13..866fbd3ebe 100644 --- a/tests/script/tsim/stream/basic5.sim +++ b/tests/script/tsim/stream/basic5.sim @@ -476,4 +476,8 @@ sql connect sleep 30000 -sql select start_ver, checkpoint_ver from information_schema.ins_stream_tasks; +sql select start_ver, checkpoint_ver from information_schema.ins_stream_tasks where level='source'; +sleep 500 + +system sh/exec.sh -n dnode1 -s stop -x SIGINT + From 5ab873832d0e240d8cec06bca02298321264066f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 27 Dec 2024 11:35:49 +0800 Subject: [PATCH 11/35] change case --- tests/army/cluster/tsdbSnapshot.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/army/cluster/tsdbSnapshot.py b/tests/army/cluster/tsdbSnapshot.py index 5c4a6d2c80..80d8d9555d 100644 --- a/tests/army/cluster/tsdbSnapshot.py +++ b/tests/army/cluster/tsdbSnapshot.py @@ -66,7 +66,7 @@ class TDTestCase(TBase): "dbinfo": {{ "name": "{self.dbName}", "drop": "yes", - "vgroups": 1, + "vgroups": 5, "duration": "10d", "wal_retention_period": 0, "replica": 3, @@ -86,7 +86,7 @@ class TDTestCase(TBase): "non_stop_mode": "no", "line_protocol": "line", "insert_rows": 10000, - "interlace_rows": 1, + "interlace_rows": 0, "insert_interval": 0, "partial_col_num": 0, "disorder_ratio": 0, @@ -155,7 +155,7 @@ class TDTestCase(TBase): "dbinfo": {{ "name": "{self.dbName}", "drop": "no", - "vgroups": 1, + "vgroups": 5, "duration": "10d", "wal_retention_period": 0, "replica": 3, @@ -175,7 +175,7 @@ class TDTestCase(TBase): "non_stop_mode": "no", "line_protocol": "line", "insert_rows": 10000, - "interlace_rows": 1, + "interlace_rows": 0, "insert_interval": 0, "partial_col_num": 0, "disorder_ratio": 0, @@ -257,7 +257,7 @@ class TDTestCase(TBase): def stop(self): tdSql.close() - tdLog.success("%s successfully executed" % __file__) + tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) From 0bf9fb1e312a206d21f70993aef3c7c3400397ef Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 27 Dec 2024 15:11:19 +0800 Subject: [PATCH 12/35] add reset stream --- include/common/tmsg.h | 10 ++++++ include/common/tmsgdef.h | 1 + include/libs/nodes/cmdnodes.h | 6 ++++ source/common/src/msg/tmsg.c | 37 +++++++++++++++++++++ source/common/test/msgTypeTable.ini | 1 + source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 1 + source/dnode/mnode/impl/src/mndStream.c | 31 +++++++++++++++++ source/libs/nodes/src/nodesCodeFuncs.c | 28 ++++++++++++++++ source/libs/nodes/src/nodesUtilFuncs.c | 4 +++ source/libs/parser/inc/parAst.h | 1 + source/libs/parser/inc/sql.y | 1 + source/libs/parser/src/parAstCreater.c | 14 ++++++++ source/libs/parser/src/parTranslater.c | 13 ++++++++ 13 files changed, 148 insertions(+) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0b6a8b3f1b..6f9ff1ec97 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -480,6 +480,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_STREAM_ANOMALY, QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERP_FUNC, + QUERY_NODE_RESET_STREAM_STMT, } ENodeType; typedef struct { @@ -3913,6 +3914,15 @@ typedef struct { int32_t tSerializeSMResumeStreamReq(void* buf, int32_t bufLen, const SMResumeStreamReq* pReq); int32_t tDeserializeSMResumeStreamReq(void* buf, int32_t bufLen, SMResumeStreamReq* pReq); +typedef struct { + char name[TSDB_STREAM_FNAME_LEN]; + int8_t igNotExists; + int8_t igUntreated; +} SMResetStreamReq; + +int32_t tSerializeSMResetStreamReq(void* buf, int32_t bufLen, const SMResetStreamReq* pReq); +int32_t tDeserializeSMResetStreamReq(void* buf, int32_t bufLen, SMResetStreamReq* pReq); + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char stb[TSDB_TABLE_FNAME_LEN]; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 93bfe306b6..e325d42ecf 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -261,6 +261,7 @@ TD_DEF_MSG_TYPE(TDMT_MND_UPDATE_DNODE_INFO, "update-dnode-info", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_AUDIT, "audit", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CONFIG, "init-config", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_RESET_STREAM, "reset-stream", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8 diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 8eb30b8184..d8584711a5 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -595,6 +595,12 @@ typedef struct SResumeStreamStmt { bool ignoreUntreated; } SResumeStreamStmt; +typedef struct SResetStreamStmt { + ENodeType type; + char streamName[TSDB_TABLE_NAME_LEN]; + bool ignoreNotExists; +} SResetStreamStmt; + typedef struct SCreateFunctionStmt { ENodeType type; bool orReplace; diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index 2193c7983f..aae0c013f7 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -12018,6 +12018,43 @@ _exit: return code; } +int32_t tSerializeSMResetStreamReq(void *buf, int32_t bufLen, const SMResetStreamReq *pReq) { + SEncoder encoder = {0}; + int32_t code = 0; + int32_t lino; + int32_t tlen; + tEncoderInit(&encoder, buf, bufLen); + TAOS_CHECK_EXIT(tStartEncode(&encoder)); + TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->name)); + TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->igNotExists)); + tEndEncode(&encoder); + +_exit: + if (code) { + tlen = code; + } else { + tlen = encoder.pos; + } + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMResetStreamReq(void *buf, int32_t bufLen, SMResetStreamReq *pReq) { + SDecoder decoder = {0}; + int32_t code = 0; + int32_t lino; + + tDecoderInit(&decoder, buf, bufLen); + TAOS_CHECK_EXIT(tStartDecode(&decoder)); + TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->name)); + TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->igNotExists)); + tEndDecode(&decoder); + +_exit: + tDecoderClear(&decoder); + return code; +} + int32_t tEncodeMqSubTopicEp(void **buf, const SMqSubTopicEp *pTopicEp) { int32_t tlen = 0; tlen += taosEncodeString(buf, pTopicEp->topic); diff --git a/source/common/test/msgTypeTable.ini b/source/common/test/msgTypeTable.ini index 0325183d8b..e31f21fd27 100644 --- a/source/common/test/msgTypeTable.ini +++ b/source/common/test/msgTypeTable.ini @@ -438,6 +438,7 @@ TDMT_STREAM_DROP = 1053 TDMT_STREAM_DROP_RSP = 1054 TDMT_STREAM_RETRIEVE_TRIGGER = 1055 TDMT_STREAM_RETRIEVE_TRIGGER_RSP = 1056 +TDMT_MND_RESET_STREAM = 1057 TDMT_SYNC_TIMEOUT = 1537 TDMT_SYNC_TIMEOUT_RSP = 1538 TDMT_SYNC_TIMEOUT_ELECTION = 1539 diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 8e595f76c9..869d68ed94 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -182,6 +182,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TB_WITH_TSMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_RESET_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_ANAL_ALGO, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 3bee82e3e7..963f69de64 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -52,6 +52,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock static void mndCancelGetNextStreamTask(SMnode *pMnode, void *pIter); static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq); static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq); +static int32_t mndProcessResetStreamReq(SRpcMsg *pReq); static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int32_t nodeId, int64_t checkpointId, int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger); static int32_t mndProcessNodeCheck(SRpcMsg *pReq); @@ -130,6 +131,7 @@ int32_t mndInitStream(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_PAUSE_STREAM, mndProcessPauseStreamReq); mndSetMsgHandle(pMnode, TDMT_MND_RESUME_STREAM, mndProcessResumeStreamReq); + mndSetMsgHandle(pMnode, TDMT_MND_RESET_STREAM, mndProcessResetStreamReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStream); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream); @@ -1898,6 +1900,35 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } +static int32_t mndProcessResetStreamReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SStreamObj *pStream = NULL; + int32_t code = 0; + + if ((code = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { + return code; + } + + SMResetStreamReq resetReq = {0}; + if (tDeserializeSMResetStreamReq(pReq->pCont, pReq->contLen, &resetReq) < 0) { + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + code = mndAcquireStream(pMnode, resetReq.name, &pStream); + if (pStream == NULL || code != 0) { + if (resetReq.igNotExists) { + mInfo("stream:%s, not exist, not pause stream", resetReq.name); + return 0; + } else { + mError("stream:%s not exist, failed to pause stream", resetReq.name); + TAOS_RETURN(TSDB_CODE_MND_STREAM_NOT_EXIST); + } + } + + //todo(liao hao jun) + return TSDB_CODE_ACTION_IN_PROGRESS; +} + static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo, bool includeAllNodes) { SSdb *pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 6d4d89607f..369e6de85b 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -193,6 +193,8 @@ const char* nodesNodeName(ENodeType type) { return "PauseStreamStmt"; case QUERY_NODE_RESUME_STREAM_STMT: return "ResumeStreamStmt"; + case QUERY_NODE_RESET_STREAM_STMT: + return "ResetStreamStmt"; case QUERY_NODE_BALANCE_VGROUP_STMT: return "BalanceVgroupStmt"; case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: @@ -7287,6 +7289,32 @@ static int32_t jsonToDropStreamStmt(const SJson* pJson, void* pObj) { return code; } +static const char* jkResetStreamStmtStreamName = "StreamName"; +static const char* jkResetStreamStmtIgnoreNotExists = "IgnoreNotExists"; + +static int32_t resetStreamStmtToJson(const void* pObj, SJson* pJson) { + const SResetStreamStmt* pNode = (const SResetStreamStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkResetStreamStmtStreamName, pNode->streamName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkResetStreamStmtIgnoreNotExists, pNode->ignoreNotExists); + } + + return code; +} + +static int32_t jsonToResetStreamStmt(const SJson* pJson, void* pObj) { + SResetStreamStmt* pNode = (SResetStreamStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkResetStreamStmtStreamName, pNode->streamName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkResetStreamStmtIgnoreNotExists, &pNode->ignoreNotExists); + } + + return code; +} + + static const char* jkMergeVgroupStmtVgroupId1 = "VgroupId1"; static const char* jkMergeVgroupStmtVgroupId2 = "VgroupId2"; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 7beaeaa46c..5c22ff00a3 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -615,6 +615,9 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) { case QUERY_NODE_RESUME_STREAM_STMT: code = makeNode(type, sizeof(SResumeStreamStmt), &pNode); break; + case QUERY_NODE_RESET_STREAM_STMT: + code = makeNode(type, sizeof(SResetStreamStmt), &pNode); + break; case QUERY_NODE_BALANCE_VGROUP_STMT: code = makeNode(type, sizeof(SBalanceVgroupStmt), &pNode); break; @@ -1480,6 +1483,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_DROP_STREAM_STMT: // no pointer field case QUERY_NODE_PAUSE_STREAM_STMT: // no pointer field case QUERY_NODE_RESUME_STREAM_STMT: // no pointer field + case QUERY_NODE_RESET_STREAM_STMT: // no pointer field case QUERY_NODE_BALANCE_VGROUP_STMT: // no pointer field case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: // no pointer field case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT: // no pointer field diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index e69a3da4a9..c6d617fc3d 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -301,6 +301,7 @@ SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName); SNode* createPauseStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName); SNode* createResumeStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, bool ignoreUntreated, SToken* pStreamName); +SNode* createResetStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName); SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId); SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId); SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 63eb09d509..0354d3c590 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -783,6 +783,7 @@ cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A) stream_options(B) INTO cmd ::= DROP STREAM exists_opt(A) stream_name(B). { pCxt->pRootNode = createDropStreamStmt(pCxt, A, &B); } cmd ::= PAUSE STREAM exists_opt(A) stream_name(B). { pCxt->pRootNode = createPauseStreamStmt(pCxt, A, &B); } cmd ::= RESUME STREAM exists_opt(A) ignore_opt(C) stream_name(B). { pCxt->pRootNode = createResumeStreamStmt(pCxt, A, C, &B); } +cmd ::= RESET STREAM exists_opt(A) stream_name(B). { pCxt->pRootNode = createResetStreamStmt(pCxt, A, &B); } %type col_list_opt { SNodeList* } %destructor col_list_opt { nodesDestroyList($$); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a13472620b..b78d6baede 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -3714,6 +3714,20 @@ _err: return NULL; } +SNode* createResetStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName) { + CHECK_PARSER_STATUS(pCxt); + CHECK_NAME(checkStreamName(pCxt, pStreamName)); + SPauseStreamStmt* pStmt = NULL; + pCxt->errCode = nodesMakeNode(QUERY_NODE_RESET_STREAM_STMT, (SNode**)&pStmt); + CHECK_MAKE_NODE(pStmt); + COPY_STRING_FORM_ID_TOKEN(pStmt->streamName, pStreamName); + pStmt->ignoreNotExists = ignoreNotExists; + return (SNode*)pStmt; +_err: + return NULL; +} + + SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId) { CHECK_PARSER_STATUS(pCxt); SKillStmt* pStmt = NULL; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f191080512..951fabb0f6 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -12346,6 +12346,16 @@ static int32_t translateResumeStream(STranslateContext* pCxt, SResumeStreamStmt* return buildCmdMsg(pCxt, TDMT_MND_RESUME_STREAM, (FSerializeFunc)tSerializeSMResumeStreamReq, &req); } +static int32_t translateResetStream(STranslateContext* pCxt, SResetStreamStmt* pStmt) { + SMResetStreamReq req = {0}; + SName name; + int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->streamName, strlen(pStmt->streamName)); + if (TSDB_CODE_SUCCESS != code) return code; + (void)tNameGetFullDbName(&name, req.name); + req.igNotExists = pStmt->ignoreNotExists; + return buildCmdMsg(pCxt, TDMT_MND_RESET_STREAM, (FSerializeFunc)tSerializeSMResetStreamReq, &req); +} + static int32_t validateCreateView(STranslateContext* pCxt, SCreateViewStmt* pStmt) { if (QUERY_NODE_SELECT_STMT != nodeType(pStmt->pQuery) && QUERY_NODE_SET_OPERATOR != nodeType(pStmt->pQuery)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VIEW_QUERY, "Invalid view query type"); @@ -13380,6 +13390,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { case QUERY_NODE_RESUME_STREAM_STMT: code = translateResumeStream(pCxt, (SResumeStreamStmt*)pNode); break; + case QUERY_NODE_RESET_STREAM_STMT: + code = translateResetStream(pCxt, (SResetStreamStmt*)pNode); + break; case QUERY_NODE_CREATE_FUNCTION_STMT: code = translateCreateFunction(pCxt, (SCreateFunctionStmt*)pNode); break; From fbcfde3989a846cf59098f818877985a0c1c6665 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 27 Dec 2024 15:54:12 +0800 Subject: [PATCH 13/35] enh/TD-29974-improve-trans-add-case --- tests/parallel_test/cases.task | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 38dd080ef5..eb99fb9266 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -409,8 +409,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/subscribe_stream_privilege.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/empty_identifier.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_transaction_detail.py -N 3 -# ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/kill_balance_leader.py -N 3 -# ,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/kill_restore_dnode.py -N 5 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/kill_balance_leader.py -N 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/kill_restore_dnode.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/persisit_config.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/qmemCtrl.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact_vgroups.py From 41548ceaf59b832eea355182030207df569e1acb Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 27 Dec 2024 17:30:09 +0800 Subject: [PATCH 14/35] add lock --- source/libs/sync/src/syncSnapshot.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 1720935e9e..33cc543b09 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -601,8 +601,10 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap // write data sRInfo(pReceiver, "snapshot receiver write about to finish, blockLen:%d seq:%d", pMsg->dataLen, pMsg->seq); if (pMsg->dataLen > 0) { + (void)taosThreadMutexLock(&pReceiver->writerMutex); code = pReceiver->pSyncNode->pFsm->FpSnapshotDoWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); + (void)taosThreadMutexUnlock(&pReceiver->writerMutex); if (code != 0) { sRError(pReceiver, "failed to finish snapshot receiver write since %s", tstrerror(code)); TAOS_RETURN(code); From 42410d6c85034aa18a61ac309ee4afa30cf553e7 Mon Sep 17 00:00:00 2001 From: "pengrongkun94@qq.com" Date: Fri, 27 Dec 2024 15:25:42 +0800 Subject: [PATCH 15/35] only linux run stmt2Test --- source/client/test/CMakeLists.txt | 11 +++++------ source/client/test/stmt2Test.cpp | 8 ++++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 62b7bb3e91..ce1cc064db 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -68,6 +68,10 @@ IF(${TD_LINUX}) NAME connectOptionsTest COMMAND connectOptionsTest ) + add_test( + NAME stmt2Test + COMMAND stmt2Test + ) ENDIF () TARGET_INCLUDE_DIRECTORIES( @@ -107,9 +111,4 @@ add_test( add_test( NAME userOperTest COMMAND userOperTest -) - -add_test( - NAME stmt2Test - COMMAND stmt2Test -) +) \ No newline at end of file diff --git a/source/client/test/stmt2Test.cpp b/source/client/test/stmt2Test.cpp index ada3b66dc9..4fea2452d0 100644 --- a/source/client/test/stmt2Test.cpp +++ b/source/client/test/stmt2Test.cpp @@ -167,13 +167,13 @@ void do_stmt(TAOS* taos, const char* sql, int CTB_NUMS, int ROW_NUMS, int CYC_NU for (int i = 0; i < CTB_NUMS; i++) { // create tags tags[i] = (TAOS_STMT2_BIND*)taosMemoryMalloc(2 * sizeof(TAOS_STMT2_BIND)); - tags[i][0] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_INT, &t1, &t1len, NULL, 0}; - tags[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, (void*)"after", &t2len, NULL, 0}; + tags[i][0] = {TSDB_DATA_TYPE_INT, &t1, &t1len, NULL, 0}; + tags[i][1] = {TSDB_DATA_TYPE_BINARY, (void*)"after", &t2len, NULL, 0}; // create col params paramv[i] = (TAOS_STMT2_BIND*)taosMemoryMalloc(2 * sizeof(TAOS_STMT2_BIND)); - paramv[i][0] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_TIMESTAMP, &ts[i][0], &ts_len[0], NULL, ROW_NUMS}; - paramv[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, &b[i][0], &b_len[0], NULL, ROW_NUMS}; + paramv[i][0] = {TSDB_DATA_TYPE_TIMESTAMP, &ts[i][0], &ts_len[0], NULL, ROW_NUMS}; + paramv[i][1] = {TSDB_DATA_TYPE_BINARY, &b[i][0], &b_len[0], NULL, ROW_NUMS}; } // bind TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, tags, paramv}; From 847e873620c27882594f018336a1c568ffdea172 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 30 Dec 2024 08:47:52 +0800 Subject: [PATCH 16/35] test case --- tests/pytest/util/tserror.py | 2 + tests/system-test/2-query/join.py | 64 ++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/tests/pytest/util/tserror.py b/tests/pytest/util/tserror.py index 35d74153c3..d930c82b1d 100644 --- a/tests/pytest/util/tserror.py +++ b/tests/pytest/util/tserror.py @@ -11,3 +11,5 @@ TSDB_CODE_UDF_FUNC_EXEC_FAILURE = (TAOS_DEF_ERROR_CODE | 0x290A) TSDB_CODE_TSC_INTERNAL_ERROR = (TAOS_DEF_ERROR_CODE | 0x02FF) + +TSDB_CODE_TSC_INVALID_OPERATION = (TAOS_DEF_ERROR_CODE | 0x0200) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 1c303b6d96..be741ab959 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -6,6 +6,7 @@ from util.log import * from util.sql import * from util.cases import * from util.dnodes import * +from util.tserror import * PRIMARY_COL = "ts" @@ -352,7 +353,66 @@ class TDTestCase: tdSql.execute( f"insert into {dbname}.nt1 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )" ) tdSql.execute( f"insert into {dbname}.nt1 values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )" ) - + def join_semantic_test(self, dbname=DBNAME): + tdSql.query("select ct1.c_int from db.ct1 as ct1 join db1.ct1 as cy1 on ct1.ts=cy1.ts") + tdSql.checkRows(self.rows) + tdSql.error("select ct1.c_int from db.ct1 as ct1 semi join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION) + tdSql.error("select ct1.c_int from db.ct1 as ct1 anti join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION) + tdSql.error("select ct1.c_int from db.ct1 as ct1 outer join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION) + tdSql.error("select ct1.c_int from db.ct1 as ct1 asof join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION) + tdSql.error("select ct1.c_int from db.ct1 as ct1 window join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION) + + tdSql.query("select ct1.c_int from db.ct1 as ct1 join db1.ct1 as cy1 on ct1.ts=cy1.ts") + tdSql.checkRows(self.rows) + tdSql.query("select ct1.c_int from db.ct1 as ct1 left join db1.ct1 as cy1 on ct1.ts=cy1.ts") + tdSql.checkRows(self.rows) + tdSql.query("select ct1.c_int from db.ct1 as ct1 left semi join db1.ct1 as cy1 on ct1.ts=cy1.ts") + tdSql.checkRows(self.rows) + tdSql.query("select ct1.c_int from db.ct1 as ct1 left anti join db1.ct1 as cy1 on ct1.ts=cy1.ts") + tdSql.checkRows(0) + tdSql.query("select ct1.c_int from db.ct1 as ct1 left outer join db1.ct1 as cy1 on ct1.ts=cy1.ts") + tdSql.checkRows(self.rows) + tdSql.query("select ct1.c_int from db.ct1 as ct1 left asof join db1.ct1 as cy1 on ct1.ts=cy1.ts") + tdSql.checkRows(self.rows) + tdSql.error("select ct1.c_int from db.ct1 as ct1 left window join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION) + + tdSql.query("select ct1.c_int from db.ct1 as ct1 right join db1.ct1 as cy1 on ct1.ts=cy1.ts") + tdSql.checkRows(self.rows) + tdSql.query("select ct1.c_int from db.ct1 as ct1 right semi join db1.ct1 as cy1 on ct1.ts=cy1.ts") + tdSql.checkRows(self.rows) + tdSql.query("select ct1.c_int from db.ct1 as ct1 right anti join db1.ct1 as cy1 on ct1.ts=cy1.ts") + tdSql.checkRows(0) + tdSql.query("select ct1.c_int from db.ct1 as ct1 right outer join db1.ct1 as cy1 on ct1.ts=cy1.ts") + tdSql.checkRows(self.rows) + tdSql.query("select ct1.c_int from db.ct1 as ct1 right asof join db1.ct1 as cy1 on ct1.ts=cy1.ts") + tdSql.checkRows(self.rows) + tdSql.error("select ct1.c_int from db.ct1 as ct1 right window join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION) + + tdSql.query("select ct1.c_int from db.ct1 as ct1 full join db1.ct1 as cy1 on ct1.ts=cy1.ts") + tdSql.checkRows(self.rows) + tdSql.error("select ct1.c_int from db.ct1 as ct1 full semi join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION) + tdSql.error("select ct1.c_int from db.ct1 as ct1 full anti join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION) + tdSql.query("select ct1.c_int from db.ct1 as ct1 full outer join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION) + tdSql.checkRows(self.rows) + tdSql.error("select ct1.c_int from db.ct1 as ct1 full asof join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION) + tdSql.error("select ct1.c_int from db.ct1 as ct1 full window join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION) + + + tdSql.query("select ct1.c_int from db.ct1 as ct1 full join db1.ct1 as cy1 on ct1.ts=cy1.ts join db1.ct1 as cy2 on ct1.ts=cy2.ts") + tdSql.checkRows(self.rows) + tdSql.query("select * from db.ct1 join db.ct2 join db.ct3 on ct2.ts=ct3.ts on ct1.ts=ct2.ts") + tdSql.checkRows(0) + tdSql.execute(f'create table db.ct1_2 using db.stb1 tags ( 102 )') + tdSql.execute(f'create table db.ct1_3 using db.stb1 tags ( 103 )') + tdSql.execute(f'insert into db.ct1_2 (select * from db.ct1)') + tdSql.execute(f'insert into db.ct1_3 (select * from db.ct1)') + tdSql.query("select * from db.ct1 join db.ct1_2 join db.ct1_3 on ct1_2.ts=ct1_3.ts on ct1.ts=ct1_2.ts") + tdSql.checkRows(self.rows) + + tdSql.execute("drop table db.ct1_2") + tdSql.execute("drop table db.ct1_3") + + def run(self): tdSql.prepare() @@ -374,6 +434,8 @@ class TDTestCase: tdSql.execute(f"use {dbname1}") self.__create_tb(dbname=dbname1) self.__insert_data(dbname=dbname1) + + self.join_semantic_test({dbname1}) tdSql.query("select ct1.c_int from db.ct1 as ct1 join db1.ct1 as cy1 on ct1.ts=cy1.ts") tdSql.checkRows(self.rows) From f03e2305e03cced5ad5526bae328fe362caddcde Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 30 Dec 2024 14:26:20 +0800 Subject: [PATCH 17/35] fix(stream): add logs and set the reset-stream rsp. --- source/common/test/msgTypeTable.ini | 1 + source/dnode/mnode/impl/src/mndStream.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/source/common/test/msgTypeTable.ini b/source/common/test/msgTypeTable.ini index e31f21fd27..add1128850 100644 --- a/source/common/test/msgTypeTable.ini +++ b/source/common/test/msgTypeTable.ini @@ -439,6 +439,7 @@ TDMT_STREAM_DROP_RSP = 1054 TDMT_STREAM_RETRIEVE_TRIGGER = 1055 TDMT_STREAM_RETRIEVE_TRIGGER_RSP = 1056 TDMT_MND_RESET_STREAM = 1057 +TDMT_MND_RESET_STREAM_RSP = 1058 TDMT_SYNC_TIMEOUT = 1537 TDMT_SYNC_TIMEOUT_RSP = 1538 TDMT_SYNC_TIMEOUT_ELECTION = 1539 diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 963f69de64..577efd155a 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1914,6 +1914,8 @@ static int32_t mndProcessResetStreamReq(SRpcMsg *pReq) { TAOS_RETURN(TSDB_CODE_INVALID_MSG); } + mDebug("recv reset stream req, stream:%s", resetReq.name); + code = mndAcquireStream(pMnode, resetReq.name, &pStream); if (pStream == NULL || code != 0) { if (resetReq.igNotExists) { From d8963b467a3cf4aa5e39de668ab8a8c2c96ef90a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 30 Dec 2024 14:39:52 +0800 Subject: [PATCH 18/35] fix(stream): remove msg info. --- source/common/test/msgTypeTable.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/common/test/msgTypeTable.ini b/source/common/test/msgTypeTable.ini index add1128850..0325183d8b 100644 --- a/source/common/test/msgTypeTable.ini +++ b/source/common/test/msgTypeTable.ini @@ -438,8 +438,6 @@ TDMT_STREAM_DROP = 1053 TDMT_STREAM_DROP_RSP = 1054 TDMT_STREAM_RETRIEVE_TRIGGER = 1055 TDMT_STREAM_RETRIEVE_TRIGGER_RSP = 1056 -TDMT_MND_RESET_STREAM = 1057 -TDMT_MND_RESET_STREAM_RSP = 1058 TDMT_SYNC_TIMEOUT = 1537 TDMT_SYNC_TIMEOUT_RSP = 1538 TDMT_SYNC_TIMEOUT_ELECTION = 1539 From 625b1f5f2f0f9d18e1f433e33326b34b92c11e5d Mon Sep 17 00:00:00 2001 From: qevolg <2227465945@qq.com> Date: Mon, 30 Dec 2024 16:31:11 +0800 Subject: [PATCH 19/35] fix(keeper): add gitinfo --- tools/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index d058d7a52f..1ee2bc4ce6 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -251,7 +251,7 @@ IF(TD_BUILD_KEEPER) PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND go build -a -ldflags "-X 'github.com/taosdata/taoskeeper/version.Version=${taos_version}' -X 'github.com/taosdata/taoskeeper/version.CommitID=${taoskeeper_commit_sha1}' -X 'github.com/taosdata/taoskeeper/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" + COMMAND go build -a -ldflags "-X 'github.com/taosdata/taoskeeper/version.Version=${taos_version}' -X 'github.com/taosdata/taoskeeper/version.Gitinfo=${taoskeeper_commit_sha1}' -X 'github.com/taosdata/taoskeeper/version.CommitID=${taoskeeper_commit_sha1}' -X 'github.com/taosdata/taoskeeper/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" INSTALL_COMMAND COMMAND cmake -E echo "Comparessing taoskeeper.exe" COMMAND cmake -E time upx taoskeeper.exe @@ -278,7 +278,7 @@ IF(TD_BUILD_KEEPER) PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND go build -a -ldflags "-X 'github.com/taosdata/taoskeeper/version.Version=${taos_version}' -X 'github.com/taosdata/taoskeeper/version.CommitID=${taoskeeper_commit_sha1}' -X 'github.com/taosdata/taoskeeper/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" + COMMAND go build -a -ldflags "-X 'github.com/taosdata/taoskeeper/version.Version=${taos_version}' -X 'github.com/taosdata/taoskeeper/version.Gitinfo=${taoskeeper_commit_sha1}' -X 'github.com/taosdata/taoskeeper/version.CommitID=${taoskeeper_commit_sha1}' -X 'github.com/taosdata/taoskeeper/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" INSTALL_COMMAND COMMAND cmake -E echo "Copy taoskeeper" COMMAND cmake -E copy taoskeeper ${CMAKE_BINARY_DIR}/build/bin From 5c192cbcb52518035517012a389d7346775c397f Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 27 Dec 2024 17:43:42 +0800 Subject: [PATCH 20/35] Refator walMeta.c. --- include/util/tutil.h | 19 +++ source/libs/wal/inc/walInt.h | 1 + source/libs/wal/src/walMeta.c | 208 ++++++++------------------- source/libs/wal/src/walWrite.c | 2 +- source/libs/wal/test/walMetaTest.cpp | 45 ++++++ 5 files changed, 127 insertions(+), 148 deletions(-) diff --git a/include/util/tutil.h b/include/util/tutil.h index 32fc9f215a..b17bdab1ac 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -187,6 +187,15 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, } \ } while (0) +#define TAOS_CHECK_RETURN_SET_CODE(CMD, CODE, ERRNO) \ + do { \ + int32_t __c = (CMD); \ + if (__c != TSDB_CODE_SUCCESS) { \ + (CODE) = (ERRNO); \ + TAOS_RETURN(__c); \ + } \ + } while (0) + #define TAOS_CHECK_RETURN_WITH_RELEASE(CMD, PTR1, PTR2) \ do { \ int32_t __c = (CMD); \ @@ -225,6 +234,16 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, } \ } while (0) +#define TAOS_CHECK_EXIT_SET_CODE(CMD, CODE, ERRNO) \ + do { \ + code = (CMD); \ + if (code < TSDB_CODE_SUCCESS) { \ + (CODE) = (ERRNO); \ + lino = __LINE__; \ + goto _exit; \ + } \ + } while (0) + #define TAOS_UNUSED(expr) (void)(expr) bool taosIsBigChar(char c); diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 14f6503941..8042713b3b 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -156,6 +156,7 @@ static inline void walResetVer(SWalVer* pVer) { int32_t walLoadMeta(SWal* pWal); int32_t walSaveMeta(SWal* pWal); int32_t walRemoveMeta(SWal* pWal); +int32_t walRollImpl(SWal* pWal); int32_t walRollFileInfo(SWal* pWal); int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* lastVer); int32_t walCheckAndRepairMeta(SWal* pWal); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 470a6b3f40..23bf92f422 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -57,18 +57,10 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); int64_t fileSize = 0; - if (taosStatFile(fnameStr, &fileSize, NULL, NULL) != 0) { - wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); - code = terrno; - goto _err; - } + TAOS_CHECK_GOTO(taosStatFile(fnameStr, &fileSize, NULL, NULL), &lino, _err); TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); - if (pFile == NULL) { - wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); - *lastVer = retVer; - TAOS_RETURN(terrno); - } + TSDB_CHECK_NULL(pFile, code, lino, _err, terrno); // ensure size as non-negative pFileInfo->fileSize = TMAX(0, pFileInfo->fileSize); @@ -102,16 +94,10 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* capacity = readSize + sizeof(magic); ptr = taosMemoryRealloc(buf, capacity); - if (ptr == NULL) { - TAOS_CHECK_GOTO(terrno, &lino, _err); - } + TSDB_CHECK_NULL(ptr, code, lino, _err, terrno); buf = ptr; - int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET); - if (ret < 0) { - wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), offset); - TAOS_CHECK_GOTO(terrno, &lino, _err); - } + TAOS_CHECK_GOTO(taosLSeekFile(pFile, offset, SEEK_SET), &lino, _err); if (readSize != taosReadFile(pFile, buf, readSize)) { wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), @@ -166,18 +152,10 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* if (capacity < readSize + extraSize + sizeof(magic)) { capacity += extraSize; void* ptr = taosMemoryRealloc(buf, capacity); - if (ptr == NULL) { - TAOS_CHECK_GOTO(terrno, &lino, _err); - } + TSDB_CHECK_NULL(ptr, code, lino, _err, terrno); buf = ptr; } - int64_t ret = taosLSeekFile(pFile, offset + readSize, SEEK_SET); - if (ret < 0) { - wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(terrno), - offset); - code = terrno; - break; - } + TAOS_CHECK_GOTO(taosLSeekFile(pFile, offset + readSize, SEEK_SET), &lino, _err); if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) { wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", extraSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), offset + readSize, extraSize, fnameStr); @@ -187,10 +165,7 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* } logContent = (SWalCkHead*)(buf + pos); - code = decryptBody(&pWal->cfg, logContent, logContent->head.bodyLen, __FUNCTION__); - if (code) { - break; - } + TAOS_CHECK_GOTO(decryptBody(&pWal->cfg, logContent, logContent->head.bodyLen, __FUNCTION__), &lino, _err); if (walValidBodyCksum(logContent) != 0) { code = TSDB_CODE_WAL_CHKSUM_MISMATCH; @@ -217,10 +192,6 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* if (forwardStage && (terrno != TSDB_CODE_SUCCESS || end == fileSize)) break; } - if (retVer < 0) { - code = TSDB_CODE_WAL_LOG_NOT_EXIST; - } - // truncate file if (lastEntryEndOffset != fileSize) { wWarn("vgId:%d, repair meta truncate file %s to %" PRId64 ", orig size %" PRId64, pWal->cfg.vgId, fnameStr, @@ -240,6 +211,9 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* pFileInfo->fileSize = lastEntryEndOffset; _err: + if (retVer < 0) { + code = TSDB_CODE_WAL_LOG_NOT_EXIST; + } taosCloseFile(&pFile); taosMemoryFree(buf); *lastVer = retVer; @@ -371,45 +345,37 @@ static int32_t walLogEntriesComplete(const SWal* pWal) { } static int32_t walTrimIdxFile(SWal* pWal, int32_t fileIdx) { - int32_t code = TSDB_CODE_SUCCESS; - TdFilePtr pFile = NULL; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + TdFilePtr pFile = NULL; + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); - if (!pFileInfo) { - TAOS_RETURN(TSDB_CODE_FAILED); - } + TSDB_CHECK_NULL(pFileInfo, code, lino, _exit, terrno); char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); int64_t fileSize = 0; - if (taosStatFile(fnameStr, &fileSize, NULL, NULL) != 0) { - wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); - code = terrno; - goto _exit; - } + TAOS_CHECK_EXIT(taosStatFile(fnameStr, &fileSize, NULL, NULL) != 0); + int64_t records = TMAX(0, pFileInfo->lastVer - pFileInfo->firstVer + 1); int64_t lastEndOffset = records * sizeof(SWalIdxEntry); - if (fileSize <= lastEndOffset) { - TAOS_RETURN(TSDB_CODE_SUCCESS); - } + if (fileSize <= lastEndOffset) TAOS_RETURN(TSDB_CODE_SUCCESS); pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); - if (pFile == NULL) { - code = terrno; - goto _exit; - } + TSDB_CHECK_NULL(pFile, code, lino, _exit, terrno); wInfo("vgId:%d, trim idx file. file: %s, size: %" PRId64 ", offset: %" PRId64, pWal->cfg.vgId, fnameStr, fileSize, lastEndOffset); - code = taosFtruncateFile(pFile, lastEndOffset); - if (code < 0) { - wError("vgId:%d, failed to truncate file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); - goto _exit; - } + TAOS_CHECK_EXIT(taosFtruncateFile(pFile, lastEndOffset)); _exit: + if (code != TSDB_CODE_SUCCESS) { + wError("vgId:%d, failed to trim idx file %s due to %s", pWal->cfg.vgId, fnameStr, tstrerror(code)); + } + (void)taosCloseFile(&pFile); TAOS_RETURN(code); } @@ -428,32 +394,25 @@ static void printFileSet(int32_t vgId, SArray* fileSet, const char* str) { int32_t walCheckAndRepairMeta(SWal* pWal) { // load log files, get first/snapshot/last version info int32_t code = 0; + int32_t lino = 0; const char* logPattern = "^[0-9]+.log$"; const char* idxPattern = "^[0-9]+.idx$"; regex_t logRegPattern; regex_t idxRegPattern; + TdDirPtr pDir = NULL; + SArray* actualLog = NULL; wInfo("vgId:%d, begin to repair meta, wal path:%s, firstVer:%" PRId64 ", lastVer:%" PRId64 ", snapshotVer:%" PRId64, pWal->cfg.vgId, pWal->path, pWal->vers.firstVer, pWal->vers.lastVer, pWal->vers.snapshotVer); - if (regcomp(&logRegPattern, logPattern, REG_EXTENDED) != 0) { - wError("failed to compile log pattern, error:%s", tstrerror(terrno)); - return terrno; - } - if (regcomp(&idxRegPattern, idxPattern, REG_EXTENDED) != 0) { - wError("failed to compile idx pattern"); - return terrno; - } + TAOS_CHECK_EXIT_SET_CODE(regcomp(&logRegPattern, logPattern, REG_EXTENDED), code, terrno); - TdDirPtr pDir = taosOpenDir(pWal->path); - if (pDir == NULL) { - regfree(&logRegPattern); - regfree(&idxRegPattern); - wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - return terrno; - } + TAOS_CHECK_EXIT_SET_CODE(regcomp(&idxRegPattern, idxPattern, REG_EXTENDED), code, terrno); - SArray* actualLog = taosArrayInit(8, sizeof(SWalFileInfo)); + pDir = taosOpenDir(pWal->path); + TSDB_CHECK_NULL(pDir, code, lino, _exit, terrno); + + actualLog = taosArrayInit(8, sizeof(SWalFileInfo)); // scan log files and build new meta TdDirEntryPtr pDirEntry; @@ -464,28 +423,10 @@ int32_t walCheckAndRepairMeta(SWal* pWal) { SWalFileInfo fileInfo; (void)memset(&fileInfo, -1, sizeof(SWalFileInfo)); (void)sscanf(name, "%" PRId64 ".log", &fileInfo.firstVer); - if (!taosArrayPush(actualLog, &fileInfo)) { - regfree(&logRegPattern); - regfree(&idxRegPattern); - int32_t ret = taosCloseDir(&pDir); - if (ret != 0) { - wError("failed to close dir, ret:%s", tstrerror(ret)); - return terrno; - } - - return terrno; - } + TSDB_CHECK_NULL(taosArrayPush(actualLog, &fileInfo), code, lino, _exit, terrno); } } - int32_t ret = taosCloseDir(&pDir); - if (ret != 0) { - wError("failed to close dir, ret:%s", tstrerror(ret)); - return terrno; - } - regfree(&logRegPattern); - regfree(&idxRegPattern); - taosArraySort(actualLog, compareWalFileInfo); wInfo("vgId:%d, actual log file, wal path:%s, num:%d", pWal->cfg.vgId, pWal->path, @@ -500,11 +441,7 @@ int32_t walCheckAndRepairMeta(SWal* pWal) { bool updateMeta = (metaFileNum != actualFileNum); // rebuild meta of file info - code = walRebuildFileInfoSet(pWal->fileInfoSet, actualLog); - taosArrayDestroy(actualLog); - if (code) { - TAOS_RETURN(code); - } + TAOS_CHECK_EXIT(walRebuildFileInfoSet(pWal->fileInfoSet, actualLog)); wInfo("vgId:%d, log file in meta, wal path:%s, num:%d", pWal->cfg.vgId, pWal->path, (int32_t)taosArrayGetSize(pWal->fileInfoSet)); @@ -521,12 +458,7 @@ int32_t walCheckAndRepairMeta(SWal* pWal) { SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); - int32_t code = taosStatFile(fnameStr, &fileSize, NULL, NULL); - if (code < 0) { - wError("failed to stat file since %s. file:%s", terrstr(), fnameStr); - - TAOS_RETURN(terrno); - } + TAOS_CHECK_EXIT(taosStatFile(fnameStr, &fileSize, NULL, NULL)); if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) { totSize += pFileInfo->fileSize; @@ -581,22 +513,24 @@ int32_t walCheckAndRepairMeta(SWal* pWal) { wInfo("vgId:%d, success to repair meta, wal path:%s, firstVer:%" PRId64 ", lastVer:%" PRId64 ", snapshotVer:%" PRId64, pWal->cfg.vgId, pWal->path, pWal->vers.firstVer, pWal->vers.lastVer, pWal->vers.snapshotVer); +_exit: + if (code != TSDB_CODE_SUCCESS) { + wError("vgId:%d, failed to repair meta due to %s, at line:%d", pWal->cfg.vgId, tstrerror(code), lino); + } + taosArrayDestroy(actualLog); + TAOS_UNUSED(taosCloseDir(&pDir)); + regfree(&logRegPattern); + regfree(&idxRegPattern); return code; } static int32_t walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) { - if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) { - TAOS_RETURN(terrno); - } + if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) return terrno; - if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { - TAOS_RETURN(terrno); - } + if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) return terrno; - if (walValidHeadCksum(pCkHead) != 0) { - TAOS_RETURN(TSDB_CODE_WAL_CHKSUM_MISMATCH); - } + if (walValidHeadCksum(pCkHead) != 0) return TSDB_CODE_WAL_CHKSUM_MISMATCH; return TSDB_CODE_SUCCESS; } @@ -838,29 +772,17 @@ int32_t walMetaSerialize(SWal* pWal, char** serialized) { TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } - if (!cJSON_AddItemToObject(pRoot, "meta", pMeta)) { - wInfo("vgId:%d, failed to add meta to root", pWal->cfg.vgId); - } + if (!cJSON_AddItemToObject(pRoot, "meta", pMeta)) goto _err; snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pWal->vers.firstVer); - if (cJSON_AddStringToObject(pMeta, "firstVer", buf) == NULL) { - wInfo("vgId:%d, failed to add firstVer to meta", pWal->cfg.vgId); - } + if (cJSON_AddStringToObject(pMeta, "firstVer", buf) == NULL) goto _err; (void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pWal->vers.snapshotVer); - if (cJSON_AddStringToObject(pMeta, "snapshotVer", buf) == NULL) { - wInfo("vgId:%d, failed to add snapshotVer to meta", pWal->cfg.vgId); - } + if (cJSON_AddStringToObject(pMeta, "snapshotVer", buf) == NULL) goto _err; (void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pWal->vers.commitVer); - if (cJSON_AddStringToObject(pMeta, "commitVer", buf) == NULL) { - wInfo("vgId:%d, failed to add commitVer to meta", pWal->cfg.vgId); - } + if (cJSON_AddStringToObject(pMeta, "commitVer", buf) == NULL) goto _err; (void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pWal->vers.lastVer); - if (cJSON_AddStringToObject(pMeta, "lastVer", buf) == NULL) { - wInfo("vgId:%d, failed to add lastVer to meta", pWal->cfg.vgId); - } + if (cJSON_AddStringToObject(pMeta, "lastVer", buf) == NULL) goto _err; - if (!cJSON_AddItemToObject(pRoot, "files", pFiles)) { - wInfo("vgId:%d, failed to add files to root", pWal->cfg.vgId); - } + if (!cJSON_AddItemToObject(pRoot, "files", pFiles)) goto _err; SWalFileInfo* pData = pWal->fileInfoSet->pData; for (int i = 0; i < sz; i++) { SWalFileInfo* pInfo = &pData[i]; @@ -869,31 +791,20 @@ int32_t walMetaSerialize(SWal* pWal, char** serialized) { } if (pField == NULL) { cJSON_Delete(pRoot); - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } // cjson only support int32_t or double // string are used to prohibit the loss of precision (void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pInfo->firstVer); - if (cJSON_AddStringToObject(pField, "firstVer", buf) == NULL) { - wInfo("vgId:%d, failed to add firstVer to field", pWal->cfg.vgId); - } + if (cJSON_AddStringToObject(pField, "firstVer", buf) == NULL) goto _err; (void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pInfo->lastVer); - if (cJSON_AddStringToObject(pField, "lastVer", buf) == NULL) { - wInfo("vgId:%d, failed to add lastVer to field", pWal->cfg.vgId); - } + if (cJSON_AddStringToObject(pField, "lastVer", buf) == NULL) goto _err; (void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pInfo->createTs); - if (cJSON_AddStringToObject(pField, "createTs", buf) == NULL) { - wInfo("vgId:%d, failed to add createTs to field", pWal->cfg.vgId); - } + if (cJSON_AddStringToObject(pField, "createTs", buf) == NULL) goto _err; (void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pInfo->closeTs); - if (cJSON_AddStringToObject(pField, "closeTs", buf) == NULL) { - wInfo("vgId:%d, failed to add closeTs to field", pWal->cfg.vgId); - } + if (cJSON_AddStringToObject(pField, "closeTs", buf) == NULL) goto _err; (void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pInfo->fileSize); - if (cJSON_AddStringToObject(pField, "fileSize", buf) == NULL) { - wInfo("vgId:%d, failed to add fileSize to field", pWal->cfg.vgId); - } + if (cJSON_AddStringToObject(pField, "fileSize", buf) == NULL) goto _err; } char* pSerialized = cJSON_Print(pRoot); cJSON_Delete(pRoot); @@ -901,6 +812,9 @@ int32_t walMetaSerialize(SWal* pWal, char** serialized) { *serialized = pSerialized; TAOS_RETURN(TSDB_CODE_SUCCESS); +_err: + cJSON_Delete(pRoot); + return TSDB_CODE_FAILED; } int32_t walMetaDeserialize(SWal* pWal, const char* bytes) { diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 998668beac..b8f0de16b4 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -274,7 +274,7 @@ _exit: TAOS_RETURN(code); } -static int32_t walRollImpl(SWal *pWal) { +int32_t walRollImpl(SWal *pWal) { int32_t code = 0, lino = 0; if (pWal->cfg.level == TAOS_WAL_SKIP && pWal->pIdxFile != NULL && pWal->pLogFile != NULL) { diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index b2875bdca1..9a8f5d4ec3 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -627,6 +627,51 @@ TEST_F(WalKeepEnv, walRollback) { ASSERT_EQ(code, 0); } +TEST_F(WalCleanEnv, walRepairLogFileTs2) { + int code; + + int i; + for (i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + + code = walRollImpl(pWal); + ASSERT_EQ(code, 0); + + for (i = 100; i < 200; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + + code = walRollImpl(pWal); + ASSERT_EQ(code, 0); + + for (i = 200; i < 300; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + + code = walRollImpl(pWal); + ASSERT_EQ(code, 0); + + // Try to step in ts repair logic. + SWalFileInfo* pFileInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 2); + pFileInfo->closeTs = -1; + + code = walCheckAndRepairMeta(pWal); + ASSERT_EQ(code, 0); +} + TEST_F(WalRetentionEnv, repairMeta1) { walResetEnv(); int code; From fd7204b04b772579f377b9137fac93a0b11b74a8 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 27 Dec 2024 17:50:52 +0800 Subject: [PATCH 21/35] Fix logic error. --- source/libs/wal/src/walMeta.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 23bf92f422..bb9a3336c8 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -192,6 +192,10 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* if (forwardStage && (terrno != TSDB_CODE_SUCCESS || end == fileSize)) break; } + if (retVer < 0) { + code = TSDB_CODE_WAL_LOG_NOT_EXIST; + } + // truncate file if (lastEntryEndOffset != fileSize) { wWarn("vgId:%d, repair meta truncate file %s to %" PRId64 ", orig size %" PRId64, pWal->cfg.vgId, fnameStr, @@ -211,8 +215,8 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* pFileInfo->fileSize = lastEntryEndOffset; _err: - if (retVer < 0) { - code = TSDB_CODE_WAL_LOG_NOT_EXIST; + if (code != 0) { + wError("vgId:%d, failed to scan log file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); } taosCloseFile(&pFile); taosMemoryFree(buf); From eb710ebc5f9597781c7eb96e7dca303fb282bada Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 30 Dec 2024 11:20:16 +0800 Subject: [PATCH 22/35] Fix meta logic error. --- source/libs/wal/src/walMeta.c | 76 +++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index bb9a3336c8..4ddec1ceb2 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -97,7 +97,11 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* TSDB_CHECK_NULL(ptr, code, lino, _err, terrno); buf = ptr; - TAOS_CHECK_GOTO(taosLSeekFile(pFile, offset, SEEK_SET), &lino, _err); + int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET); + if (ret < 0) { + wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), offset); + TAOS_CHECK_GOTO(terrno, &lino, _err); + } if (readSize != taosReadFile(pFile, buf, readSize)) { wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), @@ -155,7 +159,13 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* TSDB_CHECK_NULL(ptr, code, lino, _err, terrno); buf = ptr; } - TAOS_CHECK_GOTO(taosLSeekFile(pFile, offset + readSize, SEEK_SET), &lino, _err); + int64_t ret = taosLSeekFile(pFile, offset + readSize, SEEK_SET); + if (ret < 0) { + wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(terrno), + offset); + code = terrno; + break; + } if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) { wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", extraSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), offset + readSize, extraSize, fnameStr); @@ -208,7 +218,7 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pFile) < 0) { wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr); - TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err); + TAOS_CHECK_GOTO(terrno, &lino, _err); } } @@ -395,14 +405,20 @@ static void printFileSet(int32_t vgId, SArray* fileSet, const char* str) { } } +void walRegfree(regex_t* ptr) { + if (ptr == NULL) { + return; + } + regfree(ptr); +} + int32_t walCheckAndRepairMeta(SWal* pWal) { // load log files, get first/snapshot/last version info int32_t code = 0; int32_t lino = 0; const char* logPattern = "^[0-9]+.log$"; const char* idxPattern = "^[0-9]+.idx$"; - regex_t logRegPattern; - regex_t idxRegPattern; + regex_t logRegPattern, idxRegPattern; TdDirPtr pDir = NULL; SArray* actualLog = NULL; @@ -523,8 +539,8 @@ _exit: } taosArrayDestroy(actualLog); TAOS_UNUSED(taosCloseDir(&pDir)); - regfree(&logRegPattern); - regfree(&idxRegPattern); + walRegfree(&logRegPattern); + walRegfree(&idxRegPattern); return code; } @@ -1027,8 +1043,12 @@ _err: } int32_t walLoadMeta(SWal* pWal) { - int32_t code = 0; - int n = 0; + int32_t code = 0; + int n = 0; + int32_t lino = 0; + char* buf = NULL; + TdFilePtr pFile = NULL; + // find existing meta file int metaVer = walFindCurMetaVer(pWal); if (metaVer == -1) { @@ -1043,11 +1063,7 @@ int32_t walLoadMeta(SWal* pWal) { } // read metafile int64_t fileSize = 0; - if (taosStatFile(fnameStr, &fileSize, NULL, NULL) != 0) { - wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); - code = terrno; - TAOS_RETURN(code); - } + TAOS_CHECK_EXIT(taosStatFile(fnameStr, &fileSize, NULL, NULL) != 0); if (fileSize == 0) { code = taosRemoveFile(fnameStr); if (code) { @@ -1059,37 +1075,37 @@ int32_t walLoadMeta(SWal* pWal) { TAOS_RETURN(TSDB_CODE_FAILED); } - int size = (int)fileSize; - char* buf = taosMemoryMalloc(size + 5); - if (buf == NULL) { - TAOS_RETURN(terrno); - } + int size = (int)fileSize; + buf = taosMemoryMalloc(size + 5); + TSDB_CHECK_NULL(buf, code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); + (void)memset(buf, 0, size + 5); - TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); - if (pFile == NULL) { - taosMemoryFree(buf); + pFile = taosOpenFile(fnameStr, TD_FILE_READ); + TSDB_CHECK_NULL(pFile, code, lino, _exit, terrno); - TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED); - } if (taosReadFile(pFile, buf, size) != size) { - (void)taosCloseFile(&pFile); - taosMemoryFree(buf); - - TAOS_RETURN(terrno); + code = terrno; + goto _exit; } + // load into fileInfoSet code = walMetaDeserialize(pWal, buf); if (code < 0) { wError("failed to deserialize wal meta. file:%s", fnameStr); code = TSDB_CODE_WAL_FILE_CORRUPTED; } - (void)taosCloseFile(&pFile); - taosMemoryFree(buf); wInfo("vgId:%d, meta file loaded: %s, firstVer:%" PRId64 ", lastVer:%" PRId64 ", fileInfoSet size:%d", pWal->cfg.vgId, fnameStr, pWal->vers.firstVer, pWal->vers.lastVer, (int32_t)taosArrayGetSize(pWal->fileInfoSet)); printFileSet(pWal->cfg.vgId, pWal->fileInfoSet, "file in meta"); +_exit: + if (code != TSDB_CODE_SUCCESS) { + wError("vgId:%d, failed to load meta file due to %s, at line:%d", pWal->cfg.vgId, tstrerror(code), lino); + } + + taosMemoryFree(buf); + (void)taosCloseFile(&pFile); TAOS_RETURN(code); } From bc95f823730a1de93da2dea1fab45af4f135d92f Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 30 Dec 2024 15:48:11 +0800 Subject: [PATCH 23/35] add ut . --- include/util/tconfig.h | 2 - source/libs/wal/inc/walInt.h | 1 + source/libs/wal/src/walWrite.c | 116 ++++++--------- source/libs/wal/test/walMetaTest.cpp | 17 +++ source/util/src/tconfig.c | 210 +++++++++++---------------- source/util/test/cfgTest.cpp | 12 ++ source/util/test/memPoolTest.cpp | 19 --- 7 files changed, 159 insertions(+), 218 deletions(-) diff --git a/include/util/tconfig.h b/include/util/tconfig.h index b1bd144d77..7d00fe97ba 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -160,8 +160,6 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl SArray *taosGetLocalCfg(SConfig *pCfg); SArray *taosGetGlobalCfg(SConfig *pCfg); -void taosSetLocalCfg(SConfig *pCfg, SArray *pArray); -void taosSetGlobalCfg(SConfig *pCfg, SArray *pArray); #ifdef __cplusplus } #endif diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 8042713b3b..3bc69a6393 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -160,6 +160,7 @@ int32_t walRollImpl(SWal* pWal); int32_t walRollFileInfo(SWal* pWal); int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* lastVer); int32_t walCheckAndRepairMeta(SWal* pWal); +int64_t walChangeWrite(SWal* pWal, int64_t ver); int32_t walCheckAndRepairIdx(SWal* pWal); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index b8f0de16b4..4eaf0a9399 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -92,18 +92,15 @@ void walApplyVer(SWal *pWal, int64_t ver) { } int32_t walCommit(SWal *pWal, int64_t ver) { - if (ver < pWal->vers.commitVer) { - TAOS_RETURN(TSDB_CODE_SUCCESS); - } - if (ver > pWal->vers.lastVer || pWal->vers.commitVer < pWal->vers.snapshotVer) { - TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER); - } + if (ver < pWal->vers.commitVer) TAOS_RETURN(TSDB_CODE_SUCCESS); + if (ver > pWal->vers.lastVer || pWal->vers.commitVer < pWal->vers.snapshotVer) TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER); + pWal->vers.commitVer = ver; TAOS_RETURN(TSDB_CODE_SUCCESS); } -static int64_t walChangeWrite(SWal *pWal, int64_t ver) { +int64_t walChangeWrite(SWal *pWal, int64_t ver) { int code; TdFilePtr pIdxTFile, pLogTFile; char fnameStr[WAL_FILE_LEN]; @@ -161,6 +158,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); wInfo("vgId:%d, wal rollback for version %" PRId64, pWal->cfg.vgId, ver); int32_t code = 0; + int32_t lino = 0; int64_t ret; char fnameStr[WAL_FILE_LEN]; TdFilePtr pIdxFile = NULL, pLogFile = NULL; @@ -172,11 +170,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // find correct file if (ver < walGetLastFileFirstVer(pWal)) { // change current files - ret = walChangeWrite(pWal, ver); - if (ret < 0) { - code = terrno; - goto _exit; - } + TAOS_CHECK_EXIT_SET_CODE(walChangeWrite(pWal, ver), code, terrno); // delete files in descending order int fileSetSize = taosArrayGetSize(pWal->fileInfoSet); @@ -198,10 +192,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); pIdxFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND); - if (pIdxFile == NULL) { - code = terrno; - goto _exit; - } + TSDB_CHECK_NULL(pIdxFile, code, lino, _exit, terrno); int64_t idxOff = walGetVerIdxOffset(pWal, ver); ret = taosLSeekFile(pIdxFile, idxOff, SEEK_SET); if (ret < 0) { @@ -218,11 +209,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) { walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); pLogFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND); wDebug("vgId:%d, wal truncate file %s", pWal->cfg.vgId, fnameStr); - if (pLogFile == NULL) { - // TODO - code = terrno; - goto _exit; - } + TSDB_CHECK_NULL(pLogFile, code, lino, _exit, terrno); + ret = taosLSeekFile(pLogFile, entry.offset, SEEK_SET); if (ret < 0) { // TODO @@ -238,35 +226,26 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } code = walValidHeadCksum(&head); - if (code != 0) { - code = TSDB_CODE_WAL_FILE_CORRUPTED; - goto _exit; - } - if (head.head.version != ver) { + if (code != 0 || head.head.version != ver) { code = TSDB_CODE_WAL_FILE_CORRUPTED; goto _exit; } // truncate old files - code = taosFtruncateFile(pLogFile, entry.offset); - if (code < 0) { - goto _exit; - } - code = taosFtruncateFile(pIdxFile, idxOff); - if (code < 0) { - goto _exit; - } + if ((code = taosFtruncateFile(pLogFile, entry.offset)) < 0) goto _exit; + + if ((code = taosFtruncateFile(pIdxFile, idxOff)) < 0) goto _exit; + pWal->vers.lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; - code = walSaveMeta(pWal); - if (code < 0) { - wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr()); - goto _exit; - } + TAOS_CHECK_EXIT(walSaveMeta(pWal)); _exit: + if (code != 0) { + wError("vgId:%d, %s failed at line %d since %s", pWal->cfg.vgId, __func__, lino, tstrerror(code)); + } TAOS_UNUSED(taosCloseFile(&pIdxFile)); TAOS_UNUSED(taosCloseFile(&pLogFile)); TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); @@ -306,15 +285,12 @@ int32_t walRollImpl(SWal *pWal) { char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, newFileFirstVer, fnameStr); pIdxFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); - if (pIdxFile == NULL) { - TAOS_CHECK_GOTO(terrno, &lino, _exit); - } + TSDB_CHECK_NULL(pIdxFile, code, lino, _exit, terrno); + walBuildLogName(pWal, newFileFirstVer, fnameStr); pLogFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); wDebug("vgId:%d, wal create new file for write:%s", pWal->cfg.vgId, fnameStr); - if (pLogFile == NULL) { - TAOS_CHECK_GOTO(terrno, &lino, _exit); - } + TSDB_CHECK_NULL(pLogFile, code, lino, _exit, terrno); TAOS_CHECK_GOTO(walRollFileInfo(pWal), &lino, _exit); @@ -358,6 +334,7 @@ static FORCE_INLINE int32_t walCheckAndRoll(SWal *pWal) { int32_t walBeginSnapshot(SWal *pWal, int64_t ver, int64_t logRetention) { int32_t code = 0; + int32_t lino = 0; if (pWal->cfg.level == TAOS_WAL_SKIP) { TAOS_RETURN(TSDB_CODE_SUCCESS); @@ -375,16 +352,13 @@ int32_t walBeginSnapshot(SWal *pWal, int64_t ver, int64_t logRetention) { ", last ver %" PRId64, pWal->cfg.vgId, ver, pWal->vers.logRetention, pWal->vers.firstVer, pWal->vers.lastVer); // check file rolling - if (walGetLastFileSize(pWal) != 0) { - if ((code = walRollImpl(pWal)) < 0) { - wError("vgId:%d, failed to roll wal files since %s", pWal->cfg.vgId, terrstr()); - goto _exit; - } - } + if (walGetLastFileSize(pWal) != 0 && (code = walRollImpl(pWal)) < 0) goto _exit; _exit: + if (code) { + wError("vgId:%d, %s failed since %s at line %d", pWal->cfg.vgId, __func__, tstrerror(code), lino); + } TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); - TAOS_RETURN(code); } @@ -515,6 +489,13 @@ _exit: return code; } +static void walStopDnode(SWal *pWal) { + if (pWal->stopDnode != NULL) { + wWarn("vgId:%d, set stop dnode flag", pWal->cfg.vgId); + pWal->stopDnode(); + } +} + static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { int32_t code = 0; @@ -528,12 +509,7 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { int64_t size = taosWriteFile(pWal->pIdxFile, &entry, sizeof(SWalIdxEntry)); if (size != sizeof(SWalIdxEntry)) { wError("vgId:%d, failed to write idx entry due to %s. ver:%" PRId64, pWal->cfg.vgId, strerror(errno), ver); - - if (pWal->stopDnode != NULL) { - wWarn("vgId:%d, set stop dnode flag", pWal->cfg.vgId); - pWal->stopDnode(); - } - + walStopDnode(pWal); TAOS_RETURN(terrno); } @@ -579,12 +555,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy code = terrno; wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); - - if (pWal->stopDnode != NULL) { - wWarn("vgId:%d, set stop dnode flag", pWal->cfg.vgId); - pWal->stopDnode(); - } - + walStopDnode(pWal); TAOS_CHECK_GOTO(code, &lino, _exit); } @@ -597,12 +568,8 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy cyptedBodyLen = ENCRYPTED_LEN(cyptedBodyLen); newBody = taosMemoryMalloc(cyptedBodyLen); - if (newBody == NULL) { - wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), - strerror(errno)); + TSDB_CHECK_NULL(newBody, code, lino, _exit, terrno); - TAOS_CHECK_GOTO(terrno, &lino, _exit); - } (void)memset(newBody, 0, cyptedBodyLen); (void)memcpy(newBody, body, plainBodyLen); @@ -641,10 +608,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy taosMemoryFreeClear(newBodyEncrypted); } - if (pWal->stopDnode != NULL) { - wWarn("vgId:%d, set stop dnode flag", pWal->cfg.vgId); - pWal->stopDnode(); - } + walStopDnode(pWal); TAOS_CHECK_GOTO(code, &lino, _exit); } @@ -652,8 +616,6 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy if (pWal->cfg.encryptAlgorithm == DND_CA_SM4) { taosMemoryFreeClear(newBody); taosMemoryFreeClear(newBodyEncrypted); - // wInfo("vgId:%d, free newBody newBodyEncrypted %s", - // pWal->cfg.vgId, __FUNCTION__); } // set status @@ -668,6 +630,10 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy return 0; _exit: + if (code) { + wError("vgId:%d, %s failed at line %d since %s", pWal->cfg.vgId, __func__, lino, tstrerror(code)); + } + // recover in a reverse order if (taosFtruncateFile(pWal->pLogFile, offset) < 0) { wFatal("vgId:%d, failed to recover WAL logfile from write error since %s, offset:%" PRId64, pWal->cfg.vgId, diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 9a8f5d4ec3..9e364717f4 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -627,6 +627,23 @@ TEST_F(WalKeepEnv, walRollback) { ASSERT_EQ(code, 0); } +TEST_F(WalKeepEnv, walChangeWrite) { + walResetEnv(); + int code; + + int i; + for (i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + + code = walChangeWrite(pWal, 50); + ASSERT_EQ(code, 0); +} + TEST_F(WalCleanEnv, walRepairLogFileTs2) { int code; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 9ca53c8202..d683e8e056 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -46,24 +46,29 @@ int32_t cfgSetItemVal(SConfigItem *pItem, const char *name, const char *value, E extern char **environ; int32_t cfgInit(SConfig **ppCfg) { - SConfig *pCfg = taosMemoryCalloc(1, sizeof(SConfig)); - if (pCfg == NULL) { - TAOS_RETURN(terrno); - } + int32_t code = 0; + int32_t lino = 0; + SConfig *pCfg = NULL; + pCfg = taosMemoryCalloc(1, sizeof(SConfig)); + if (pCfg == NULL) return terrno; + + pCfg->localArray = NULL, pCfg->globalArray = NULL; pCfg->localArray = taosArrayInit(64, sizeof(SConfigItem)); - if (pCfg->localArray == NULL) { - taosMemoryFree(pCfg); - TAOS_RETURN(terrno); - } + TSDB_CHECK_NULL(pCfg->localArray, code, lino, _exit, terrno); + pCfg->globalArray = taosArrayInit(64, sizeof(SConfigItem)); - if (pCfg->globalArray == NULL) { - taosMemoryFree(pCfg); - TAOS_RETURN(terrno); - } + TSDB_CHECK_NULL(pCfg->globalArray, code, lino, _exit, terrno); TAOS_CHECK_RETURN(taosThreadMutexInit(&pCfg->lock, NULL)); *ppCfg = pCfg; + +_exit: + if (code != 0) { + uError("failed to init config, since %s ,at line %d", tstrerror(code), lino); + cfgCleanup(pCfg); + } + TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -187,14 +192,11 @@ int32_t cfgGetGlobalSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->globalAr static int32_t cfgCheckAndSetConf(SConfigItem *pItem, const char *conf) { cfgItemFreeVal(pItem); - if (!(pItem->str == NULL)) { - return TSDB_CODE_INVALID_PARA; - } + if (!(pItem->str == NULL)) return TSDB_CODE_INVALID_PARA; pItem->str = taosStrdup(conf); - if (pItem->str == NULL) { - TAOS_RETURN(terrno); - } + + if (pItem->str == NULL) return terrno; TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -209,9 +211,8 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { taosMemoryFreeClear(pItem->str); pItem->str = taosStrdup(fullDir); - if (pItem->str == NULL) { - TAOS_RETURN(terrno); - } + + if (pItem->str == NULL) return terrno; TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -219,9 +220,8 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) { int32_t code = 0; bool tmp = false; - if (strcasecmp(value, "true") == 0) { - tmp = true; - } + if (strcasecmp(value, "true") == 0) tmp = true; + int32_t val = 0; if ((code = taosStr2int32(value, &val)) == 0 && val > 0) { tmp = true; @@ -441,9 +441,7 @@ static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool rese // logflag names that should 'not' be set by 'debugFlag' if (pDebugFlagItem->array == NULL) { pDebugFlagItem->array = taosArrayInit(16, sizeof(SLogVar)); - if (pDebugFlagItem->array == NULL) { - TAOS_RETURN(terrno); - } + if (pDebugFlagItem->array == NULL) return terrno; } taosArrayClear(pDebugFlagItem->array); TAOS_RETURN(TSDB_CODE_SUCCESS); @@ -454,9 +452,7 @@ static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool rese if (pDebugFlagItem->array != NULL) { SLogVar logVar = {0}; tstrncpy(logVar.name, name, TSDB_LOG_VAR_LEN); - if (NULL == taosArrayPush(pDebugFlagItem->array, &logVar)) { - TAOS_RETURN(terrno); - } + if (NULL == taosArrayPush(pDebugFlagItem->array, &logVar)) return terrno; } TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -518,9 +514,8 @@ _exit: int32_t cfgSetItemVal(SConfigItem *pItem, const char *name, const char *value, ECfgSrcType stype) { int32_t code = TSDB_CODE_SUCCESS; - if (pItem == NULL) { - TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); - } + if (pItem == NULL) return TSDB_CODE_CFG_NOT_FOUND; + switch (pItem->dtype) { case CFG_DTYPE_BOOL: { code = cfgSetBool(pItem, value, stype); @@ -589,10 +584,7 @@ SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName) { } void cfgLock(SConfig *pCfg) { - if (pCfg == NULL) { - return; - } - + if (pCfg == NULL) return; (void)taosThreadMutexLock(&pCfg->lock); } @@ -600,7 +592,7 @@ void cfgUnLock(SConfig *pCfg) { (void)taosThreadMutexUnlock(&pCfg->lock); } int32_t checkItemDyn(SConfigItem *pItem, bool isServer) { if (pItem->dynScope == CFG_DYN_NONE) { - return TSDB_CODE_SUCCESS; + return TSDB_CODE_INVALID_CFG; } if (isServer) { if (pItem->dynScope == CFG_DYN_CLIENT || pItem->dynScope == CFG_DYN_CLIENT_LAZY) { @@ -617,39 +609,33 @@ int32_t checkItemDyn(SConfigItem *pItem, bool isServer) { int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer, CfgAlterType alterType) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; cfgLock(pCfg); SConfigItem *pItem = cfgGetItem(pCfg, name); - if (pItem == NULL) { - cfgUnLock(pCfg); - TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); - } - int32_t code = checkItemDyn(pItem, isServer); - if (code != TSDB_CODE_SUCCESS) { - cfgUnLock(pCfg); - TAOS_RETURN(code); - } + TSDB_CHECK_NULL(pItem, code, lino, _exit, TSDB_CODE_CFG_NOT_FOUND); + + TAOS_CHECK_EXIT(checkItemDyn(pItem, isServer)); if ((pItem->category == CFG_CATEGORY_GLOBAL) && alterType == CFG_ALTER_DNODE) { uError("failed to config:%s, not support update global config on only one dnode", name); - cfgUnLock(pCfg); - TAOS_RETURN(TSDB_CODE_INVALID_CFG); + code = TSDB_CODE_INVALID_CFG; + goto _exit; } switch (pItem->dtype) { case CFG_DTYPE_STRING: { if (strcasecmp(name, "slowLogScope") == 0) { char *tmp = taosStrdup(pVal); if (!tmp) { - cfgUnLock(pCfg); - uError("failed to config:%s since %s", name, terrstr()); - TAOS_RETURN(terrno); + code = terrno; + goto _exit; } int32_t scope = 0; - int32_t code = taosSetSlowLogScope(tmp, &scope); + code = taosSetSlowLogScope(tmp, &scope); if (TSDB_CODE_SUCCESS != code) { - cfgUnLock(pCfg); taosMemoryFree(tmp); - TAOS_RETURN(code); + goto _exit; } taosMemoryFree(tmp); } @@ -659,13 +645,13 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p code = taosStr2int32(pVal, &ival); if (code != 0 || (ival != 0 && ival != 1)) { uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name, cfgDtypeStr(pItem->dtype), ival); - cfgUnLock(pCfg); - TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + code = TSDB_CODE_OUT_OF_RANGE; + goto _exit; } } break; case CFG_DTYPE_INT32: { int32_t ival; - int32_t code = (int32_t)taosStrHumanToInt32(pVal, &ival); + code = (int32_t)taosStrHumanToInt32(pVal, &ival); if (code != TSDB_CODE_SUCCESS) { cfgUnLock(pCfg); return code; @@ -673,13 +659,13 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p if (ival < pItem->imin || ival > pItem->imax) { uError("cfg:%s, type:%s value:%d out of range[%" PRId64 ", %" PRId64 "]", pItem->name, cfgDtypeStr(pItem->dtype), ival, pItem->imin, pItem->imax); - cfgUnLock(pCfg); - TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + code = TSDB_CODE_OUT_OF_RANGE; + goto _exit; } } break; case CFG_DTYPE_INT64: { int64_t ival; - int32_t code = taosStrHumanToInt64(pVal, &ival); + code = taosStrHumanToInt64(pVal, &ival); if (code != TSDB_CODE_SUCCESS) { cfgUnLock(pCfg); TAOS_RETURN(code); @@ -687,31 +673,32 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p if (ival < pItem->imin || ival > pItem->imax) { uError("cfg:%s, type:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 "]", pItem->name, cfgDtypeStr(pItem->dtype), ival, pItem->imin, pItem->imax); - cfgUnLock(pCfg); - TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + code = TSDB_CODE_OUT_OF_RANGE; + goto _exit; } } break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: { - float dval = 0; - int32_t code = parseCfgReal(pVal, &dval); - if (code != TSDB_CODE_SUCCESS) { - cfgUnLock(pCfg); - TAOS_RETURN(code); - } + float dval = 0; + TAOS_CHECK_EXIT(parseCfgReal(pVal, &dval)); + if (dval < pItem->fmin || dval > pItem->fmax) { uError("cfg:%s, type:%s value:%g out of range[%g, %g]", pItem->name, cfgDtypeStr(pItem->dtype), dval, pItem->fmin, pItem->fmax); - cfgUnLock(pCfg); - TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); + code = TSDB_CODE_OUT_OF_RANGE; + goto _exit; } } break; default: break; } +_exit: + if (code != TSDB_CODE_SUCCESS) { + uError("failed to check range for cfg:%s, value:%s, since %s at line:%d", name, pVal, tstrerror(code), __LINE__); + } cfgUnLock(pCfg); - TAOS_RETURN(TSDB_CODE_SUCCESS); + TAOS_RETURN(code); } static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { @@ -720,9 +707,7 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { pItem->stype = CFG_STYPE_DEFAULT; pItem->name = taosStrdup(name); - if (pItem->name == NULL) { - TAOS_RETURN(terrno); - } + if (pItem->name == NULL) return terrno; int32_t size = taosArrayGetSize(array); for (int32_t i = 0; i < size; ++i) { @@ -819,9 +804,8 @@ int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, in int8_t category) { SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope, .dynScope = dynScope, .category = category}; item.str = taosStrdup(defaultVal); - if (item.str == NULL) { - TAOS_RETURN(terrno); - } + if (item.str == NULL) return terrno; + return cfgAddItem(pCfg, &item, name); } @@ -943,13 +927,9 @@ int32_t cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t break; } - if (len < 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); - } + if (len < 0) return terrno; - if (len > bufSize) { - len = bufSize; - } + if (len > bufSize) len = bufSize; *pLen = len; TAOS_RETURN(TSDB_CODE_SUCCESS); @@ -1310,9 +1290,7 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { } TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) { - TAOS_RETURN(terrno); - } + if (pFile == NULL) return terrno; while (!taosEOFFile(pFile)) { name = value = value2 = value3 = value4 = NULL; @@ -1466,8 +1444,10 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { } int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { - char *cfgLineBuf = NULL, *name, *value, *value2, *value3, *value4; - SJson *pJson = NULL; + char *cfgLineBuf = NULL, *buf = NULL, *name, *value, *value2, *value3, *value4; + SJson *pJson = NULL; + TdFilePtr pFile = NULL; + int32_t olen, vlen, vlen2, vlen3, vlen4; int32_t code = 0, lino = 0; if (url == NULL || strlen(url) == 0) { @@ -1490,36 +1470,28 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { } TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); - if (pFile == NULL) { - TAOS_CHECK_EXIT(terrno); - } + TSDB_CHECK_NULL(pFile, code, lino, _exit, terrno); + size_t fileSize = taosLSeekFile(pFile, 0, SEEK_END); if (fileSize <= 0) { - (void)taosCloseFile(&pFile); - (void)printf("load json file error: %s\n", filepath); - TAOS_CHECK_EXIT(terrno); - } - char *buf = taosMemoryMalloc(fileSize + 1); - if (!buf) { - (void)taosCloseFile(&pFile); - (void)printf("load json file error: %s, failed to alloc memory\n", filepath); - TAOS_RETURN(terrno); + code = terrno; + goto _exit; } + buf = taosMemoryMalloc(fileSize + 1); + TSDB_CHECK_NULL(buf, code, lino, _exit, terrno); + buf[fileSize] = 0; if (taosLSeekFile(pFile, 0, SEEK_SET) < 0) { - (void)taosCloseFile(&pFile); - (void)printf("load json file error: %s\n", filepath); - taosMemoryFreeClear(buf); - TAOS_RETURN(terrno); + code = terrno; + goto _exit; } + if (taosReadFile(pFile, buf, fileSize) <= 0) { - (void)taosCloseFile(&pFile); - (void)printf("load json file error: %s\n", filepath); - taosMemoryFreeClear(buf); - TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT); + code = TSDB_CODE_INVALID_DATA_FMT; + goto _exit; } - (void)taosCloseFile(&pFile); + pJson = tjsonParse(buf); if (NULL == pJson) { const char *jsonParseError = tjsonGetError(); @@ -1529,7 +1501,6 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { taosMemoryFreeClear(buf); TAOS_CHECK_EXIT(TSDB_CODE_INVALID_DATA_FMT); } - taosMemoryFreeClear(buf); int32_t jsonArraySize = tjsonGetArraySize(pJson); for (int32_t i = 0; i < jsonArraySize; i++) { @@ -1596,12 +1567,13 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { TAOS_RETURN(TSDB_CODE_INVALID_PARA); } - taosMemoryFree(cfgLineBuf); uInfo("load from apoll url not implemented yet"); - TAOS_RETURN(TSDB_CODE_SUCCESS); _exit: taosMemoryFree(cfgLineBuf); + taosMemoryFree(buf); + (void)taosCloseFile(&pFile); + tjsonDelete(pJson); if (code != 0) { (void)printf("failed to load from apollo url:%s at line %d since %s\n", url, lino, tstrerror(code)); @@ -1701,9 +1673,7 @@ struct SConfigIter { int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter) { SConfigIter *pIter = taosMemoryCalloc(1, sizeof(SConfigIter)); - if (pIter == NULL) { - TAOS_RETURN(terrno); - } + if (pIter == NULL) return terrno; pIter->pConf = pConf; @@ -1721,14 +1691,10 @@ SConfigItem *cfgNextIter(SConfigIter *pIter) { } void cfgDestroyIter(SConfigIter *pIter) { - if (pIter == NULL) { - return; - } + if (pIter == NULL) return; taosMemoryFree(pIter); } SArray *taosGetLocalCfg(SConfig *pCfg) { return pCfg->localArray; } -SArray *taosGetGlobalCfg(SConfig *pCfg) { return pCfg->globalArray; } -void taosSetLocalCfg(SConfig *pCfg, SArray *pArray) { pCfg->localArray = pArray; }; -void taosSetGlobalCfg(SConfig *pCfg, SArray *pArray) { pCfg->globalArray = pArray; }; \ No newline at end of file +SArray *taosGetGlobalCfg(SConfig *pCfg) { return pCfg->globalArray; } \ No newline at end of file diff --git a/source/util/test/cfgTest.cpp b/source/util/test/cfgTest.cpp index 74c34f5c91..27343c9531 100644 --- a/source/util/test/cfgTest.cpp +++ b/source/util/test/cfgTest.cpp @@ -337,6 +337,12 @@ TEST_F(CfgTest, cfgLoadFromEnvFile) { ASSERT_EQ(code, TSDB_CODE_SUCCESS); ASSERT_NE(pConfig, nullptr); + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 0, 0), 0); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0, 0, 0), 0); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0, 0, 0), 0); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0, 0, 0), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0, 0, 0), 0); + TdFilePtr envFile = NULL; const char *envFilePath = TD_TMP_DIR_PATH "envFile"; envFile = taosOpenFile(envFilePath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); @@ -355,6 +361,12 @@ TEST_F(CfgTest, cfgLoadFromApollUrl) { ASSERT_EQ(code, TSDB_CODE_SUCCESS); ASSERT_NE(pConfig, nullptr); + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 0, 0), 0); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0, 0, 0), 0); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0, 0, 0), 0); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0, 0, 0), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0, 0, 0), 0); + TdFilePtr jsonFile = NULL; const char *jsonFilePath = TD_TMP_DIR_PATH "envJson.json"; jsonFile = taosOpenFile(jsonFilePath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); diff --git a/source/util/test/memPoolTest.cpp b/source/util/test/memPoolTest.cpp index a8da96711d..6302bc4a2b 100644 --- a/source/util/test/memPoolTest.cpp +++ b/source/util/test/memPoolTest.cpp @@ -2298,23 +2298,4 @@ TEST(functionsTest, internalFunc) { #endif - - - - - - - - - - -int main(int argc, char** argv) { - taosSeedRand(taosGetTimestampSec()); - mptInit(); - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - - - #pragma GCC diagnosti From 1934376b84f118336249d7fc2035beb070fa58ce Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 30 Dec 2024 17:49:25 +0800 Subject: [PATCH 24/35] fix error code. --- source/util/test/memPoolTest.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/source/util/test/memPoolTest.cpp b/source/util/test/memPoolTest.cpp index 6302bc4a2b..a8da96711d 100644 --- a/source/util/test/memPoolTest.cpp +++ b/source/util/test/memPoolTest.cpp @@ -2298,4 +2298,23 @@ TEST(functionsTest, internalFunc) { #endif + + + + + + + + + + +int main(int argc, char** argv) { + taosSeedRand(taosGetTimestampSec()); + mptInit(); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + + #pragma GCC diagnosti From 85b41440fddf4d1833f8850a1f8e5efcbb3d5f82 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 30 Dec 2024 17:24:13 +0800 Subject: [PATCH 25/35] Fix some logic errors. --- source/common/src/tglobal.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d6d3e3a443..c5bbc330c2 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -14,12 +14,12 @@ */ #define _DEFAULT_SOURCE -#include "tglobal.h" #include "cJSON.h" #include "defines.h" #include "os.h" #include "osString.h" #include "tconfig.h" +#include "tglobal.h" #include "tgrant.h" #include "tjson.h" #include "tlog.h" @@ -858,9 +858,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "syncLogBufferMemoryAllowed", tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); @@ -911,7 +911,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlBatchDropNum", tsTtlBatchDropNum, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); @@ -919,11 +919,11 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "s3MigrateEnabled", tsS3MigrateEnabled, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); @@ -939,11 +939,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - SConfigItem *pItem = NULL; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "fPrecision"); - tsFPrecision = pItem->fval; - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); From 836687e1a37cb8f4595a32b8d1e3ba547db87626 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 30 Dec 2024 17:39:25 +0800 Subject: [PATCH 26/35] Fix some logic errors. --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c5bbc330c2..ae8b14d84e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1929,7 +1929,7 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char * SConfig *pCfg = NULL; TAOS_CHECK_RETURN(cfgInit(&pCfg)); - TAOS_CHECK_GOTO(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL), &lino, + TAOS_CHECK_GOTO(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL), &lino, _exit); TAOS_CHECK_GOTO( cfgAddInt32(pCfg, "debugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL), &lino, From 6d09d4864dbda643822f14b4044a953dbf4166ca Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 30 Dec 2024 21:15:35 +0800 Subject: [PATCH 27/35] Fix ut errors. --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index ae8b14d84e..24fc3b2a96 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -825,7 +825,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsLogBufferMemoryAllowed = TRANGE(tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); // clang-format off - TAOS_CHECK_RETURN(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER, CFG_CATEGORY_LOCAL)); From 304511aaa07941fe66fe6c3dc4f75e17ff66a305 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 31 Dec 2024 09:53:16 +0800 Subject: [PATCH 28/35] Fix ci errors. --- source/util/src/tconfig.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index d683e8e056..54d9be4df6 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -1573,12 +1573,14 @@ _exit: taosMemoryFree(cfgLineBuf); taosMemoryFree(buf); (void)taosCloseFile(&pFile); - tjsonDelete(pJson); - if (code != 0) { - (void)printf("failed to load from apollo url:%s at line %d since %s\n", url, lino, tstrerror(code)); + if (code == TSDB_CODE_CFG_NOT_FOUND) { + uTrace("load from apoll url success"); + TAOS_RETURN(TSDB_CODE_SUCCESS); + } else { + (void)printf("failed to load from apoll url:%s at line %d since %s\n", url, lino, tstrerror(code)); + TAOS_RETURN(code); } - TAOS_RETURN(code); } int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl) { From 1688eae59aa1b736594cee0f2ba2e6321be5b6fc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 31 Dec 2024 11:21:00 +0800 Subject: [PATCH 29/35] fix(docs/s3): make OBS explicitly --- docs/zh/08-operation/12-multi.md | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/docs/zh/08-operation/12-multi.md b/docs/zh/08-operation/12-multi.md index 122fb9c2f3..994192e8fc 100644 --- a/docs/zh/08-operation/12-multi.md +++ b/docs/zh/08-operation/12-multi.md @@ -60,11 +60,15 @@ dataDir /mnt/data6 2 0 ## 对象存储 -本节介绍在 TDengine Enterprise 如何使用 S3 对象存储，本功能基于通用 S3 SDK 实现，对各个 S3 平台的访问参数进行了兼容适配，可以访问如 Amazon S3、Azure Blob、华为 OBS、腾讯云 COS、阿里云 OSS、minio等对象存储服务。通过适当的参数配置，可以把大部分较冷的时序数据存储到 S3 服务中。 +本节介绍在 TDengine Enterprise 版本中如何使用对象存储功能，如 Amazon S3、Azure Blob Storage、华为 OBS、腾讯云 COS、阿里云 OSS、MinIO 等对象存储服务。 **注意** 在配合多级存储使用时，每一级存储介质上保存的数据都有可能被按规则备份到远程对象存储中并删除本地数据文件。 -### 配置方式 +### S3 对象存储 + +本功能基于通用 S3 SDK 实现，对各个 S3 平台的访问参数进行了兼容适配，通过适当的参数配置，可以把大部分较冷的时序数据存储到 S3 服务中。 + +#### 配置方式 在配置文件 /etc/taos/taos.cfg 中，添加用于 S3 访问的参数： @@ -78,7 +82,7 @@ dataDir /mnt/data6 2 0 | s3MigrateIntervalSec | 本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600 | | s3MigrateEnabled | 是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1。 | -### 检查配置参数可用性 +#### 检查配置参数可用性 在 taos.cfg 中完成对 S3 的配置后，通过 taosd 命令的 checks3 参数可以检查所配置的 S3 服务是否可用： @@ -88,7 +92,7 @@ taosd --checks3 如果配置的 S3 服务无法访问，此命令会在运行过程中输出相应的错误信息。 -### 创建使用 S3 的 DB +#### 创建使用 S3 的 DB 完成配置后，即可启动 TDengine 集群，创建使用 S3 的数据库，比如： @@ -112,11 +116,11 @@ s3migrate database ; | 2 | s3_chunkpages | 131072 | 131072 | 1048576 | 上传对象的大小阈值，与 tsdb_pagesize 参数一样，不可修改，单位为 TSDB 页 | | 3 | s3_compact | 1 | 0 | 1 | TSDB 文件组首次上传 S3 时，是否自动进行 compact 操作 | -### 对象存储读写次数估算 +#### 对象存储读写次数估算 对象存储服务的使用成本与存储的数据量及请求次数相关，下面分别介绍数据的上传及下载过程。 -#### 数据上传 +##### 数据上传 当 TSDB 时序数据超过 `s3_keeplocal` 参数指定的时间，相关的数据文件会被切分成多个文件块，每个文件块的默认大小是 512M 字节 (`s3_chunkpages * tsdb_pagesize`)。除了最后一个文件块保留在本地文件系统外，其余的文件块会被上传到对象存储服务。 @@ -128,7 +132,7 @@ s3migrate database ; 其它类型的文件如 head, stt, sma 等，保留在本地文件系统，以加速预计算相关查询。 -#### 数据下载 +##### 数据下载 在查询操作中，如果需要访问对象存储中的数据，TSDB 不会下载整个数据文件，而是计算所需数据在文件中的位置，只下载相应的数据到 TSDB 页缓存中，然后将数据返回给查询执行引擎。后续查询首先检查页缓存，查看数据是否已被缓存。如果数据已缓存，则直接使用缓存中的数据，而无需重复从对象存储下载，从而有效降低从对象存储下载数据的次数。 @@ -140,15 +144,15 @@ s3migrate database ; 页缓存是内存缓存，节点重启后，再次查询需要重新下载数据。缓存采用 LRU (Least Recently Used) 策略，当缓存空间不足时，最近最少使用的数据将被淘汰。缓存的大小可以通过 `s3PageCacheSize` 参数进行调整，通常来说，缓存越大，下载次数越少。 -## Azure Blob 存储 -本节介绍在 TDengine Enterprise 如何使用微软 Azure Blob 对象存储。本功能是上一小节‘对象存储’功能的扩展，需额外依赖 Flexify 服务提供的 S3 网关。通过适当的参数配置，可以把大部分较冷的时序数据存储到 Azure Blob 服务中。 +### Azure Blob 存储 +本节介绍在 TDengine Enterprise 版本中如何使用微软 Azure Blob 存储。本功能可以通过两个方式使用：利用 Flexify 服务提供的 S3 网关功能和不依赖 Flexify 服务。通过配置参数，可以把大部分较冷的时序数据存储到 Azure Blob 服务中。 -### Flexify 服务 +#### Flexify 服务 Flexify 是 Azure Marketplace 中的一款应用程序，允许兼容 S3 的应用程序通过标准 S3 API 在 Azure Blob Storage 中存储数据。可使用多个 Flexify 服务对同一个 Blob 存储建立多个 S3 网关。 部署方式请参考 [Flexify](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/flexify.azure-s3-api?tab=Overview) 应用页面说明。 -### 配置方式 +##### 配置方式 在配置文件 /etc/taos/taos.cfg 中，添加用于 S3 访问的参数： @@ -164,7 +168,7 @@ s3BucketName td-test - 在某一 S3 服务上操作失败后会切换至其他服务，全部服务都失败后将返回最后产生的错误码 - 最大支持的 S3 服务配置数为 10 -### 不依赖 Flexify 服务 +#### 不依赖 Flexify 服务 用户界面同 S3，不同的地方在于下面三个参数的配置： From b6a018485c31c37a2efd291c8cc259da39bc1b17 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 Date: Tue, 31 Dec 2024 13:46:39 +0800 Subject: [PATCH 30/35] add drop view test --- .../0-others/view/non_marterial_view/test_view.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/system-test/0-others/view/non_marterial_view/test_view.py b/tests/system-test/0-others/view/non_marterial_view/test_view.py index 7e062143d2..059b016c3d 100644 --- a/tests/system-test/0-others/view/non_marterial_view/test_view.py +++ b/tests/system-test/0-others/view/non_marterial_view/test_view.py @@ -566,8 +566,18 @@ class TDTestCase: tdSql.execute(f"drop database {paraDict['dbName']}") tdSql.execute("drop database cdb;") tdLog.debug("Finish test case 'test_tmq_from_view'") + def test_TD_33390(self): + tdSql.execute('create database test') + tdSql.execute('create table test.nt(ts timestamp, c1 int)') + for i in range(0, 200): + tdSql.execute(f'create view test.view{i} as select * from test.nt') + tdSql.query("show test.views") + + for i in range(0, 200): + tdSql.execute(f'drop view test.view{i}') def run(self): + self.test_TD_33390() self.test_create_view_from_one_database() self.test_create_view_from_multi_database() self.test_create_view_name_params() From 0ce3d91abd18af182c34fe9fc749edd7a6256816 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 Date: Tue, 31 Dec 2024 13:48:13 +0800 Subject: [PATCH 31/35] modify Jenkinsfile2 --- Jenkinsfile2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 1b2f28908c..5c620658a5 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -165,7 +165,7 @@ def pre_test(){ script { sh ''' cd ${WK} - git checkout ''' + env.CHANGE_TARGET + ''' + git checkout fix/3.0/TD-33390 cd ${WKC} git checkout ''' + env.CHANGE_TARGET + ''' ''' @@ -274,7 +274,7 @@ def pre_test_win(){ script { bat ''' cd %WIN_INTERNAL_ROOT% - git checkout ''' + env.CHANGE_TARGET + ''' + git checkout fix/3.0/TD-33390 ''' bat ''' cd %WIN_COMMUNITY_ROOT% From 56417b3a92e18421e656e98c132140ddbbc5e16b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 31 Dec 2024 14:30:54 +0800 Subject: [PATCH 32/35] more --- tests/army/cluster/tsdbSnapshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/cluster/tsdbSnapshot.py b/tests/army/cluster/tsdbSnapshot.py index 80d8d9555d..c008261b00 100644 --- a/tests/army/cluster/tsdbSnapshot.py +++ b/tests/army/cluster/tsdbSnapshot.py @@ -253,7 +253,7 @@ class TDTestCase(TBase): cluster.dnodes[1].starttaosd() cluster.dnodes[2].starttaosd() - tdLog.sleep(10) + tdLog.sleep(60) def stop(self): tdSql.close() From b378d83194f018daef21ea7771dbd3f1970cd90c Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 31 Dec 2024 15:12:53 +0800 Subject: [PATCH 33/35] Fix review errors for error return. --- source/libs/wal/src/walMeta.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 4ddec1ceb2..b4eaa467af 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -226,7 +226,7 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* _err: if (code != 0) { - wError("vgId:%d, failed to scan log file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + wError("vgId:%d, failed to scan log file due to %s, file:%s", pWal->cfg.vgId, tstrerror(terrno), fnameStr); } taosCloseFile(&pFile); taosMemoryFree(buf); @@ -1063,7 +1063,7 @@ int32_t walLoadMeta(SWal* pWal) { } // read metafile int64_t fileSize = 0; - TAOS_CHECK_EXIT(taosStatFile(fnameStr, &fileSize, NULL, NULL) != 0); + TAOS_CHECK_EXIT(taosStatFile(fnameStr, &fileSize, NULL, NULL)); if (fileSize == 0) { code = taosRemoveFile(fnameStr); if (code) { From bc6502d47b2fe345c9e46531d6eae03bff995aa2 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 2 Jan 2025 02:13:24 +0000 Subject: [PATCH 34/35] fix/TS-5651-skip-sync-heartbeat --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index acd95d4b43..6fefd47a6f 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -254,7 +254,15 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { pRpc->info.wrapper = pWrapper; - EQItype itype = IsReq(pRpc) ? RPC_QITEM : DEF_QITEM; // rsp msg is not restricted by tsQueueMemoryUsed + EQItype itype = RPC_QITEM; // rsp msg is not restricted by tsQueueMemoryUsed + if (IsReq(pRpc)) { + if (pRpc->msgType == TDMT_SYNC_HEARTBEAT || pRpc->msgType == TDMT_SYNC_HEARTBEAT_REPLY) + itype = DEF_QITEM; + else + itype = RPC_QITEM; + } else { + itype = DEF_QITEM; + } code = taosAllocateQitem(sizeof(SRpcMsg), itype, pRpc->contLen, (void **)&pMsg); if (code) goto _OVER; From 6e3adaecee097ec35b533fa097a7d872b811c42e Mon Sep 17 00:00:00 2001 From: wangjiaming0909 Date: Thu, 2 Jan 2025 13:45:46 +0800 Subject: [PATCH 35/35] Revert "modify Jenkinsfile2" This reverts commit 0ce3d91abd18af182c34fe9fc749edd7a6256816. --- Jenkinsfile2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 5c620658a5..1b2f28908c 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -165,7 +165,7 @@ def pre_test(){ script { sh ''' cd ${WK} - git checkout fix/3.0/TD-33390 + git checkout ''' + env.CHANGE_TARGET + ''' cd ${WKC} git checkout ''' + env.CHANGE_TARGET + ''' ''' @@ -274,7 +274,7 @@ def pre_test_win(){ script { bat ''' cd %WIN_INTERNAL_ROOT% - git checkout fix/3.0/TD-33390 + git checkout ''' + env.CHANGE_TARGET + ''' ''' bat ''' cd %WIN_COMMUNITY_ROOT%