diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 5321dd8f10..a9c829660a 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -426,6 +426,10 @@ pipeline { cd ${WKC}/tests/parallel_test ./run_check_assert_container.sh -d ${WKDIR} ''' + sh ''' + cd ${WKC}/tests/parallel_test + ./run_check_void_container.sh -d ${WKDIR} + ''' sh ''' date rm -rf ${WKC}/debug diff --git a/cmake/cmake.install b/cmake/cmake.install index 67634625ce..119bda1c38 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -7,7 +7,17 @@ ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ - COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo firstEp localhost:6030 > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo fqdn localhost >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo serverPort 6030 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo debugFlag 135 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo asyncLog 0 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo supportVnodes 1024 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo numOfLogLines 300000000 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo logKeepDays -1 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo checkpointInterval 60 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo snodeAddress 127.0.0.1:873 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg COMMAND ${CMAKE_COMMAND} -E echo monitor 0 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg diff --git a/docs/examples/c-ws/.gitignore b/docs/examples/c-ws/.gitignore new file mode 100644 index 0000000000..afe9743149 --- /dev/null +++ b/docs/examples/c-ws/.gitignore @@ -0,0 +1,3 @@ +* +!*.c +!.gitignore diff --git a/docs/examples/c-ws/connect_example.c b/docs/examples/c-ws/connect_example.c new file mode 100644 index 0000000000..5a9cd64292 --- /dev/null +++ b/docs/examples/c-ws/connect_example.c @@ -0,0 +1,21 @@ +// compile with +// gcc connect_example.c -o connect_example -ltaos +#include +#include +#include "taosws.h" + +int main() { + ws_enable_log("debug"); + char *dsn = "ws://localhost:6041"; + WS_TAOS *taos = ws_connect(dsn); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return -1; + } + fprintf(stdout, "Connected to %s successfully.\n", dsn); + + /* put your code here for read and write */ + + // close & clean + ws_close(taos); +} diff --git a/docs/examples/c-ws/create_db_demo.c b/docs/examples/c-ws/create_db_demo.c new file mode 100755 index 0000000000..13f038f1df --- /dev/null +++ b/docs/examples/c-ws/create_db_demo.c @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o create_db_demo create_db_demo.c -ltaos + +#include +#include +#include +#include +#include "taosws.h" + +static int DemoCreateDB() { + ws_enable_log("debug"); + // ANCHOR: create_db_and_table + int code = 0; + char *dsn = "ws://localhost:6041"; + // connect + WS_TAOS *taos = ws_connect(dsn); + + if (taos == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return -1; + } + + // create database + WS_RES *result = ws_query(taos, "CREATE DATABASE IF NOT EXISTS power"); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to create database power, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(result)); + ws_close(taos); + return -1; + } + ws_free_result(result); + fprintf(stdout, "Create database power successfully.\n"); + + // create table + const char *sql = + "CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId " + "INT, location BINARY(24))"; + result = ws_query(taos, sql); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to create stable power.meters, ErrCode: 0x%x, ErrMessage: %s\n.", code, ws_errstr(result)); + ws_close(taos); + return -1; + } + ws_free_result(result); + fprintf(stdout, "Create stable power.meters successfully.\n"); + + // close & clean + ws_close(taos); + return 0; + // ANCHOR_END: create_db_and_table +} + +int main(int argc, char *argv[]) { return DemoCreateDB(); } diff --git a/docs/examples/c-ws/insert_data_demo.c b/docs/examples/c-ws/insert_data_demo.c new file mode 100644 index 0000000000..4abaea4c5e --- /dev/null +++ b/docs/examples/c-ws/insert_data_demo.c @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o insert_data_demo insert_data_demo.c -ltaos + +#include +#include +#include +#include +#include "taosws.h" + +static int DemoInsertData() { + // ANCHOR: insert_data + int code = 0; + char *dsn = "ws://localhost:6041"; + // connect + WS_TAOS *taos = ws_connect(dsn); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return -1; + } + + // insert data, please make sure the database and table are already created + const char *sql = + "INSERT INTO " + "power.d1001 USING power.meters TAGS(2,'California.SanFrancisco') " + "VALUES " + "(NOW + 1a, 10.30000, 219, 0.31000) " + "(NOW + 2a, 12.60000, 218, 0.33000) " + "(NOW + 3a, 12.30000, 221, 0.31000) " + "power.d1002 USING power.meters TAGS(3, 'California.SanFrancisco') " + "VALUES " + "(NOW + 1a, 10.30000, 218, 0.25000) "; + WS_RES *result = ws_query(taos, sql); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to insert data to power.meters, sql: %s, ErrCode: 0x%x, ErrMessage: %s\n.", sql, code, + ws_errstr(result)); + ws_close(taos); + return -1; + } + ws_free_result(result); + + // you can check affectedRows here + int rows = ws_affected_rows(result); + fprintf(stdout, "Successfully inserted %d rows into power.meters.\n", rows); + + // close & clean + ws_close(taos); + return 0; + // ANCHOR_END: insert_data +} + +int main(int argc, char *argv[]) { return DemoInsertData(); } diff --git a/docs/examples/c-ws/query_data_demo.c b/docs/examples/c-ws/query_data_demo.c new file mode 100644 index 0000000000..e70e15458d --- /dev/null +++ b/docs/examples/c-ws/query_data_demo.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o query_data_demo query_data_demo.c -ltaos + +#include +#include +#include +#include +#include "taosws.h" + +static int DemoQueryData() { + // ANCHOR: query_data + int code = 0; + char *dsn = "ws://localhost:6041"; + + // connect + WS_TAOS *taos = ws_connect(dsn); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return -1; + } + + // query data, please make sure the database and table are already created + const char *sql = "SELECT ts, current, location FROM power.meters limit 100"; + WS_RES *result = ws_query(taos, sql); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to query data from power.meters, sql: %s, ErrCode: 0x%x, ErrMessage: %s\n.", sql, code, + ws_errstr(result)); + ws_close(taos); + return -1; + } + + WS_ROW row = NULL; + int rows = 0; + int num_fields = ws_field_count(result); + const WS_FIELD *fields = ws_fetch_fields(result); + + fprintf(stdout, "query successfully, got %d fields, the sql is: %s.\n", num_fields, sql); + + // fetch the records row by row + while ((row = ws_fetch_row(result))) { + // Add your data processing logic here + + rows++; + } + fprintf(stdout, "total rows: %d\n", rows); + ws_free_result(result); + + // close & clean + ws_close(taos); + return 0; + // ANCHOR_END: query_data +} + +int main(int argc, char *argv[]) { return DemoQueryData(); } diff --git a/docs/examples/c-ws/sml_insert_demo.c b/docs/examples/c-ws/sml_insert_demo.c new file mode 100644 index 0000000000..1149aa2543 --- /dev/null +++ b/docs/examples/c-ws/sml_insert_demo.c @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o sml_insert_demo sml_insert_demo.c -ltaos + +#include +#include +#include +#include "taosws.h" + +static int DemoSmlInsert() { + // ANCHOR: schemaless + int code = 0; + char *dsn = "ws://localhost:6041"; + + // connect + WS_TAOS *taos = ws_connect(dsn); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return -1; + } + + // create database + WS_RES *result = ws_query(taos, "CREATE DATABASE IF NOT EXISTS power"); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to create database power, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(result)); + ws_close(taos); + return -1; + } + ws_free_result(result); + + // use database + result = ws_query(taos, "USE power"); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to execute use power, ErrCode: 0x%x, ErrMessage: %s\n.", code, ws_errstr(result)); + ws_close(taos); + return -1; + } + ws_free_result(result); + + // schemaless demo data + char *line_demo = + "meters,groupid=2,location=California.SanFrancisco current=10.3000002f64,voltage=219i32,phase=0.31f64 " + "1626006833639"; + char *telnet_demo = "metric_telnet 1707095283260 4 host=host0 interface=eth0"; + char *json_demo = + "{\"metric\": \"metric_json\",\"timestamp\": 1626846400,\"value\": 10.3, \"tags\": {\"groupid\": 2, " + "\"location\": \"California.SanFrancisco\", \"id\": \"d1001\"}}"; + + // influxdb line protocol + char *lines[] = {line_demo}; + int totalLines = 0; + result = ws_schemaless_insert_raw(taos, line_demo, strlen(line_demo), &totalLines, WS_TSDB_SML_LINE_PROTOCOL, + WS_TSDB_SML_TIMESTAMP_MILLI_SECONDS); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to insert schemaless line data, data: %s, ErrCode: 0x%x, ErrMessage: %s\n.", line_demo, + code, ws_errstr(result)); + ws_close(taos); + return -1; + } + + fprintf(stdout, "Insert %d rows of schemaless line data successfully.\n", totalLines); + ws_free_result(result); + + // opentsdb telnet protocol + totalLines = 0; + result = ws_schemaless_insert_raw(taos, telnet_demo, strlen(telnet_demo), &totalLines, WS_TSDB_SML_TELNET_PROTOCOL, + WS_TSDB_SML_TIMESTAMP_MILLI_SECONDS); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to insert schemaless telnet data, data: %s, ErrCode: 0x%x, ErrMessage: %s\n.", telnet_demo, + code, ws_errstr(result)); + ws_close(taos); + return -1; + } + + fprintf(stdout, "Insert %d rows of schemaless telnet data successfully.\n", totalLines); + ws_free_result(result); + + // opentsdb json protocol + char *jsons[1] = {0}; + // allocate memory for json data. can not use static memory. + totalLines = 0; + result = ws_schemaless_insert_raw(taos, json_demo, strlen(json_demo), &totalLines, WS_TSDB_SML_JSON_PROTOCOL, + WS_TSDB_SML_TIMESTAMP_MILLI_SECONDS); + code = ws_errno(result); + if (code != 0) { + free(jsons[0]); + fprintf(stderr, "Failed to insert schemaless json data, Server: %s, ErrCode: 0x%x, ErrMessage: %s\n.", json_demo, + code, ws_errstr(result)); + ws_close(taos); + return -1; + } + free(jsons[0]); + + fprintf(stdout, "Insert %d rows of schemaless json data successfully.\n", totalLines); + ws_free_result(result); + + // close & clean + ws_close(taos); + return 0; + // ANCHOR_END: schemaless +} + +int main(int argc, char *argv[]) { return DemoSmlInsert(); } diff --git a/docs/examples/c-ws/stmt_insert_demo.c b/docs/examples/c-ws/stmt_insert_demo.c new file mode 100644 index 0000000000..0ddfb53303 --- /dev/null +++ b/docs/examples/c-ws/stmt_insert_demo.c @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o stmt_insert_demo stmt_insert_demo.c -ltaos + +#include +#include +#include +#include +#include "taosws.h" + +/** + * @brief execute sql only. + * + * @param taos + * @param sql + */ +void executeSQL(WS_TAOS *taos, const char *sql) { + WS_RES *res = ws_query(taos, sql); + int code = ws_errno(res); + if (code != 0) { + fprintf(stderr, "%s\n", ws_errstr(res)); + ws_free_result(res); + ws_close(taos); + exit(EXIT_FAILURE); + } + ws_free_result(res); +} + +/** + * @brief check return status and exit program when error occur. + * + * @param stmt + * @param code + * @param msg + */ +void checkErrorCode(WS_STMT *stmt, int code, const char *msg) { + if (code != 0) { + fprintf(stderr, "%s. code: %d, error: %s\n", msg, code, ws_stmt_errstr(stmt)); + ws_stmt_close(stmt); + exit(EXIT_FAILURE); + } +} + +typedef struct { + int64_t ts; + float current; + int voltage; + float phase; +} Row; + +int num_of_sub_table = 10; +int num_of_row = 10; +int total_affected = 0; +/** + * @brief insert data using stmt API + * + * @param taos + */ +void insertData(WS_TAOS *taos) { + // init + WS_STMT *stmt = ws_stmt_init(taos); + if (stmt == NULL) { + fprintf(stderr, "Failed to init ws_stmt, error: %s\n", ws_stmt_errstr(NULL)); + exit(EXIT_FAILURE); + } + // prepare + const char *sql = "INSERT INTO ? USING meters TAGS(?,?) VALUES (?,?,?,?)"; + int code = ws_stmt_prepare(stmt, sql, 0); + checkErrorCode(stmt, code, "Failed to execute ws_stmt_prepare"); + for (int i = 1; i <= num_of_sub_table; i++) { + char table_name[20]; + sprintf(table_name, "d_bind_%d", i); + char location[20]; + sprintf(location, "location_%d", i); + + // set table name and tags + WS_MULTI_BIND tags[2]; + // groupId + tags[0].buffer_type = TSDB_DATA_TYPE_INT; + tags[0].buffer_length = sizeof(int); + tags[0].length = (int32_t *)&tags[0].buffer_length; + tags[0].buffer = &i; + tags[0].is_null = NULL; + tags[0].num = 1; + // location + tags[1].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[1].buffer_length = strlen(location); + tags[1].length = (int32_t *)&tags[1].buffer_length; + tags[1].buffer = location; + tags[1].is_null = NULL; + tags[1].num = 1; + code = ws_stmt_set_tbname_tags(stmt, table_name, tags, 2); + checkErrorCode(stmt, code, "Failed to set table name and tags\n"); + + // insert rows + WS_MULTI_BIND params[4]; + // ts + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(int64_t); + params[0].length = (int32_t *)¶ms[0].buffer_length; + params[0].is_null = NULL; + params[0].num = 1; + // current + params[1].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[1].buffer_length = sizeof(float); + params[1].length = (int32_t *)¶ms[1].buffer_length; + params[1].is_null = NULL; + params[1].num = 1; + // voltage + params[2].buffer_type = TSDB_DATA_TYPE_INT; + params[2].buffer_length = sizeof(int); + params[2].length = (int32_t *)¶ms[2].buffer_length; + params[2].is_null = NULL; + params[2].num = 1; + // phase + params[3].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[3].buffer_length = sizeof(float); + params[3].length = (int32_t *)¶ms[3].buffer_length; + params[3].is_null = NULL; + params[3].num = 1; + + for (int j = 0; j < num_of_row; j++) { + struct timeval tv; + gettimeofday(&tv, NULL); + long long milliseconds = tv.tv_sec * 1000LL + tv.tv_usec / 1000; // current timestamp in milliseconds + int64_t ts = milliseconds + j; + float current = (float)rand() / RAND_MAX * 30; + int voltage = rand() % 300; + float phase = (float)rand() / RAND_MAX; + params[0].buffer = &ts; + params[1].buffer = ¤t; + params[2].buffer = &voltage; + params[3].buffer = &phase; + // bind param + code = ws_stmt_bind_param_batch(stmt, params, 4); + checkErrorCode(stmt, code, "Failed to bind param"); + } + // add batch + code = ws_stmt_add_batch(stmt); + checkErrorCode(stmt, code, "Failed to add batch"); + // execute batch + int affected_rows = 0; + code = ws_stmt_execute(stmt, &affected_rows); + checkErrorCode(stmt, code, "Failed to exec stmt"); + // get affected rows + int affected = ws_stmt_affected_rows_once(stmt); + total_affected += affected; + } + fprintf(stdout, "Successfully inserted %d rows to power.meters.\n", total_affected); + ws_stmt_close(stmt); +} + +int main() { + int code = 0; + char *dsn = "ws://localhost:6041"; + WS_TAOS *taos = ws_connect(dsn); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + exit(EXIT_FAILURE); + } + // create database and table + executeSQL(taos, "CREATE DATABASE IF NOT EXISTS power"); + executeSQL(taos, "USE power"); + executeSQL(taos, + "CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS " + "(groupId INT, location BINARY(24))"); + insertData(taos); + ws_close(taos); +} diff --git a/docs/examples/c-ws/tmq_demo.c b/docs/examples/c-ws/tmq_demo.c new file mode 100644 index 0000000000..e84c75fa3a --- /dev/null +++ b/docs/examples/c-ws/tmq_demo.c @@ -0,0 +1,488 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// to compile: gcc -o tmq_demo tmq_demo.c -ltaos -lpthread + +#include +#include +#include +#include +#include +#include +#include +#include "taosws.h" + +volatile int thread_stop = 0; +static int running = 1; +static int count = 0; +const char* topic_name = "topic_meters"; + +typedef struct { + const char* enable_auto_commit; + const char* auto_commit_interval_ms; + const char* group_id; + const char* client_id; + const char* td_connect_host; + const char* td_connect_port; + const char* td_connect_user; + const char* td_connect_pass; + const char* auto_offset_reset; +} ConsumerConfig; + +ConsumerConfig config = {.enable_auto_commit = "true", + .auto_commit_interval_ms = "1000", + .group_id = "group1", + .client_id = "client1", + .td_connect_host = "localhost", + .td_connect_port = "6030", + .td_connect_user = "root", + .td_connect_pass = "taosdata", + .auto_offset_reset = "latest"}; + +void* prepare_data(void* arg) { + int code = 0; + char* dsn = "ws://localhost:6041"; + WS_TAOS* pConn = ws_connect(dsn); + if (pConn == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return NULL; + } + + WS_RES* pRes; + int i = 1; + + while (!thread_stop) { + char buf[200] = {0}; + i++; + snprintf( + buf, sizeof(buf), + "INSERT INTO power.d1001 USING power.meters TAGS(2,'California.SanFrancisco') VALUES (NOW + %da, 10.30000, " + "219, 0.31000)", + i); + + pRes = ws_query(pConn, buf); + code = ws_errno(pRes); + if (code != 0) { + fprintf(stderr, "Failed to insert data to power.meters, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + } + ws_free_result(pRes); + sleep(1); + } + fprintf(stdout, "Prepare data thread exit\n"); + return NULL; +} + +// ANCHOR: msg_process +int32_t msg_process(WS_RES* msg) { + int32_t rows = 0; + const char* topicName = ws_tmq_get_topic_name(msg); + const char* dbName = ws_tmq_get_db_name(msg); + int32_t vgroupId = ws_tmq_get_vgroup_id(msg); + + while (true) { + // get one row data from message + WS_ROW row = ws_fetch_row(msg); + if (row == NULL) break; + + // Add your data processing logic here + + rows++; + } + + return rows; +} +// ANCHOR_END: msg_process + +WS_TAOS* init_env() { + int code = 0; + char* dsn = "ws://localhost:6041"; + WS_TAOS* pConn = ws_connect(dsn); + if (pConn == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return NULL; + } + + WS_RES* pRes; + // drop database if exists + pRes = ws_query(pConn, "DROP TOPIC IF EXISTS topic_meters"); + code = ws_errno(pRes); + if (code != 0) { + fprintf(stderr, "Failed to drop topic_meters, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + goto END; + } + ws_free_result(pRes); + + pRes = ws_query(pConn, "DROP DATABASE IF EXISTS power"); + code = ws_errno(pRes); + if (code != 0) { + fprintf(stderr, "Failed to drop database power, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + goto END; + } + ws_free_result(pRes); + + // create database + pRes = ws_query(pConn, "CREATE DATABASE power PRECISION 'ms' WAL_RETENTION_PERIOD 3600"); + code = ws_errno(pRes); + if (code != 0) { + fprintf(stderr, "Failed to create power, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + goto END; + } + ws_free_result(pRes); + + // create super table + pRes = + ws_query(pConn, + "CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS " + "(groupId INT, location BINARY(24))"); + code = ws_errno(pRes); + if (code != 0) { + fprintf(stderr, "Failed to create super table meters, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + goto END; + } + ws_free_result(pRes); + + return pConn; + +END: + ws_free_result(pRes); + ws_close(pConn); + return NULL; +} + +void deinit_env(WS_TAOS* pConn) { + if (pConn) ws_close(pConn); +} + +int32_t create_topic(WS_TAOS* pConn) { + WS_RES* pRes; + int code = 0; + + if (!pConn) { + fprintf(stderr, "Invalid input parameter.\n"); + return -1; + } + + pRes = ws_query(pConn, "USE power"); + code = ws_errno(pRes); + if (ws_errno(pRes) != 0) { + fprintf(stderr, "Failed to use power, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + return -1; + } + ws_free_result(pRes); + + pRes = ws_query( + pConn, + "CREATE TOPIC IF NOT EXISTS topic_meters AS SELECT ts, current, voltage, phase, groupid, location FROM meters"); + code = ws_errno(pRes); + if (code != 0) { + fprintf(stderr, "Failed to create topic topic_meters, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + return -1; + } + ws_free_result(pRes); + return 0; +} + +int32_t drop_topic(WS_TAOS* pConn) { + WS_RES* pRes; + int code = 0; + + if (!pConn) { + fprintf(stderr, "Invalid input parameter.\n"); + return -1; + } + + pRes = ws_query(pConn, "USE power"); + code = ws_errno(pRes); + if (ws_errno(pRes) != 0) { + fprintf(stderr, "Failed to use power, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + return -1; + } + ws_free_result(pRes); + + pRes = ws_query(pConn, "DROP TOPIC IF EXISTS topic_meters"); + code = ws_errno(pRes); + if (code != 0) { + fprintf(stderr, "Failed to drop topic topic_meters, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + return -1; + } + ws_free_result(pRes); + return 0; +} + +void tmq_commit_cb_print(ws_tmq_t* tmq, int32_t code, void* param) { + count += 1; + fprintf(stdout, "tmq_commit_cb_print() code: %d, tmq: %p, param: %p, count: %d.\n", code, tmq, param, count); +} + +// ANCHOR: create_consumer_1 +ws_tmq_t* build_consumer(const ConsumerConfig* config) { + ws_tmq_conf_res_t code; + ws_tmq_t* tmq = NULL; + + // create a configuration object + ws_tmq_conf_t* conf = ws_tmq_conf_new(); + + // set the configuration parameters + code = ws_tmq_conf_set(conf, "enable.auto.commit", config->enable_auto_commit); + if (WS_TMQ_CONF_OK != code) { + ws_tmq_conf_destroy(conf); + return NULL; + } + code = ws_tmq_conf_set(conf, "auto.commit.interval.ms", config->auto_commit_interval_ms); + if (WS_TMQ_CONF_OK != code) { + ws_tmq_conf_destroy(conf); + return NULL; + } + code = ws_tmq_conf_set(conf, "group.id", config->group_id); + if (WS_TMQ_CONF_OK != code) { + ws_tmq_conf_destroy(conf); + return NULL; + } + code = ws_tmq_conf_set(conf, "client.id", config->client_id); + if (WS_TMQ_CONF_OK != code) { + ws_tmq_conf_destroy(conf); + return NULL; + } + + code = ws_tmq_conf_set(conf, "auto.offset.reset", config->auto_offset_reset); + if (WS_TMQ_CONF_OK != code) { + ws_tmq_conf_destroy(conf); + return NULL; + } + + // create a consumer object + tmq = ws_tmq_consumer_new(conf, "taos://localhost:6041", NULL, 0); + +_end: + // destroy the configuration object + ws_tmq_conf_destroy(conf); + return tmq; +} +// ANCHOR_END: create_consumer_1 + +// ANCHOR: build_topic_list +// build a topic list used to subscribe +ws_tmq_list_t* build_topic_list() { + // create a empty topic list + ws_tmq_list_t* topicList = ws_tmq_list_new(); + + // append topic name to the list + int32_t code = ws_tmq_list_append(topicList, topic_name); + if (code) { + // if failed, destroy the list and return NULL + ws_tmq_list_destroy(topicList); + fprintf(stderr, + "Failed to create topic_list, topic: %s, groupId: %s, clientId: %s, ErrCode: 0x%x, ErrMessage: %s.\n", + topic_name, config.group_id, config.client_id, code, ws_tmq_errstr(NULL)); + return NULL; + } + // if success, return the list + return topicList; +} +// ANCHOR_END: build_topic_list + +// ANCHOR: basic_consume_loop +void basic_consume_loop(ws_tmq_t* tmq) { + int32_t totalRows = 0; // total rows consumed + int32_t msgCnt = 0; // total messages consumed + int32_t timeout = 5000; // poll timeout + + while (running) { + // poll message from TDengine + WS_RES* tmqmsg = ws_tmq_consumer_poll(tmq, timeout); + if (tmqmsg) { + msgCnt++; + + // Add your data processing logic here + totalRows += msg_process(tmqmsg); + + // free the message + ws_free_result(tmqmsg); + } + if (msgCnt > 50) { + // consume 50 messages and break + break; + } + } + + // print the result: total messages and total rows consumed + fprintf(stdout, "%d msg consumed, include %d rows\n", msgCnt, totalRows); +} +// ANCHOR_END: basic_consume_loop + +// ANCHOR: consume_repeatly +void consume_repeatly(ws_tmq_t* tmq) { + int32_t numOfAssignment = 0; + ws_tmq_topic_assignment* pAssign = NULL; + + // get the topic assignment + int32_t code = ws_tmq_get_topic_assignment(tmq, topic_name, &pAssign, &numOfAssignment); + if (code != 0 || pAssign == NULL || numOfAssignment == 0) { + fprintf(stderr, "Failed to get assignment, topic: %s, groupId: %s, clientId: %s, ErrCode: 0x%x, ErrMessage: %s.\n", + topic_name, config.group_id, config.client_id, code, ws_tmq_errstr(tmq)); + return; + } + + // seek to the earliest offset + for (int32_t i = 0; i < numOfAssignment; ++i) { + ws_tmq_topic_assignment* p = &pAssign[i]; + + code = ws_tmq_offset_seek(tmq, topic_name, p->vgId, p->begin); + if (code != 0) { + fprintf(stderr, + "Failed to seek offset, topic: %s, groupId: %s, clientId: %s, vgId: %d, ErrCode: 0x%x, ErrMessage: %s.\n", + topic_name, config.group_id, config.client_id, p->vgId, code, ws_tmq_errstr(tmq)); + break; + } + } + if (code == 0) fprintf(stdout, "Assignment seek to beginning successfully.\n"); + + // free the assignment array + ws_tmq_free_assignment(pAssign, numOfAssignment); + + // let's consume the messages again + basic_consume_loop(tmq); +} +// ANCHOR_END: consume_repeatly + +// ANCHOR: manual_commit +void manual_commit(ws_tmq_t* tmq) { + int32_t totalRows = 0; // total rows consumed + int32_t msgCnt = 0; // total messages consumed + int32_t timeout = 5000; // poll timeout + + while (running) { + // poll message from TDengine + WS_RES* tmqmsg = ws_tmq_consumer_poll(tmq, timeout); + if (tmqmsg) { + msgCnt++; + // process the message + totalRows += msg_process(tmqmsg); + // commit the message + int32_t code = ws_tmq_commit_sync(tmq, tmqmsg); + if (code) { + fprintf(stderr, + "Failed to commit offset, topic: %s, groupId: %s, clientId: %s, ErrCode: 0x%x, ErrMessage: %s.\n", + topic_name, config.group_id, config.client_id, code, ws_tmq_errstr(tmq)); + // free the message + ws_free_result(tmqmsg); + break; + } else { + fprintf(stdout, "Commit offset manually successfully.\n"); + } + // free the message + ws_free_result(tmqmsg); + } + if (msgCnt > 50) { + // consume 50 messages and break + break; + } + } + + // print the result: total messages and total rows consumed + fprintf(stdout, "%d msg consumed, include %d rows.\n", msgCnt, totalRows); +} +// ANCHOR_END: manual_commit + +int main(int argc, char* argv[]) { + int32_t code; + pthread_t thread_id; + + WS_TAOS* pConn = init_env(); + if (pConn == NULL) { + fprintf(stderr, "Failed to init env.\n"); + return -1; + } + + if (create_topic(pConn) < 0) { + fprintf(stderr, "Failed to create topic.\n"); + return -1; + } + + if (pthread_create(&thread_id, NULL, &prepare_data, NULL)) { + fprintf(stderr, "Failed to create thread.\n"); + return -1; + } + + // ANCHOR: create_consumer_2 + ws_tmq_t* tmq = build_consumer(&config); + if (NULL == tmq) { + fprintf(stderr, "Failed to create native consumer, host: %s, groupId: %s, , clientId: %s.\n", + config.td_connect_host, config.group_id, config.client_id); + return -1; + } else { + fprintf(stdout, "Create consumer successfully, host: %s, groupId: %s, clientId: %s.\n", config.td_connect_host, + config.group_id, config.client_id); + } + + // ANCHOR_END: create_consumer_2 + + // ANCHOR: subscribe_3 + ws_tmq_list_t* topic_list = build_topic_list(); + if (NULL == topic_list) { + fprintf(stderr, "Failed to create topic_list, topic: %s, groupId: %s, clientId: %s.\n", topic_name, config.group_id, + config.client_id); + return -1; + } + + if ((code = ws_tmq_subscribe(tmq, topic_list))) { + fprintf(stderr, + "Failed to subscribe topic_list, topic: %s, groupId: %s, clientId: %s, ErrCode: 0x%x, ErrMessage: %s.\n", + topic_name, config.group_id, config.client_id, code, ws_tmq_errstr(tmq)); + } else { + fprintf(stdout, "Subscribe topics successfully.\n"); + } + + ws_tmq_list_destroy(topic_list); + + basic_consume_loop(tmq); + // ANCHOR_END: subscribe_3 + + consume_repeatly(tmq); + + manual_commit(tmq); + + // ANCHOR: unsubscribe_and_close + // unsubscribe the topic + code = ws_tmq_unsubscribe(tmq); + if (code) { + fprintf(stderr, + "Failed to unsubscribe consumer, topic: %s, groupId: %s, clientId: %s, ErrCode: 0x%x, ErrMessage: %s.\n", + topic_name, config.group_id, config.client_id, code, ws_tmq_errstr(tmq)); + } else { + fprintf(stdout, "Consumer unsubscribed successfully.\n"); + } + + // close the consumer + code = ws_tmq_consumer_close(tmq); + if (code) { + fprintf(stderr, "Failed to close consumer, topic: %s, groupId: %s, clientId: %s, ErrCode: 0x%x, ErrMessage: %s.\n", + topic_name, config.group_id, config.client_id, code, ws_tmq_errstr(tmq)); + } else { + fprintf(stdout, "Consumer closed successfully.\n"); + } + // ANCHOR_END: unsubscribe_and_close + + thread_stop = 1; + pthread_join(thread_id, NULL); + + if (drop_topic(pConn) < 0) { + fprintf(stderr, "Failed to drop topic.\n"); + return -1; + } + + deinit_env(pConn); + return 0; +} diff --git a/docs/examples/c-ws/with_reqid_demo.c b/docs/examples/c-ws/with_reqid_demo.c new file mode 100644 index 0000000000..7dae526a9e --- /dev/null +++ b/docs/examples/c-ws/with_reqid_demo.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o with_reqid_demo with_reqid_demo.c -ltaos + +#include +#include +#include +#include +#include "taosws.h" + +static int DemoWithReqId() { + // ANCHOR: with_reqid + int code = 0; + char *dsn = "ws://localhost:6041"; + + // connect + WS_TAOS *taos = ws_connect(dsn); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return -1; + } + + const char *sql = "SELECT ts, current, location FROM power.meters limit 1"; + // query data with reqid + long reqid = 3L; + WS_RES *result = ws_query_with_reqid(taos, sql, reqid); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to execute sql withQID: %ld, ErrCode: 0x%x, ErrMessage: %s\n.", reqid, code, + ws_errstr(result)); + ws_close(taos); + return -1; + } + + WS_ROW row = NULL; + int rows = 0; + int num_fields = ws_field_count(result); + const WS_FIELD *fields = ws_fetch_fields(result); + + fprintf(stdout, "query successfully, got %d fields, the sql is: %s.\n", num_fields, sql); + + // fetch the records row by row + while ((row = ws_fetch_row(result))) { + // Add your data processing logic here + + rows++; + } + fprintf(stdout, "total rows: %d\n", rows); + ws_free_result(result); + + // close & clean + ws_close(taos); + return 0; + // ANCHOR_END: with_reqid +} + +int main(int argc, char *argv[]) { return DemoWithReqId(); } diff --git a/docs/examples/c/CMakeLists.txt b/docs/examples/c/CMakeLists.txt new file mode 100644 index 0000000000..f636084bab --- /dev/null +++ b/docs/examples/c/CMakeLists.txt @@ -0,0 +1,101 @@ +PROJECT(TDengine) + +IF (TD_LINUX) + INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc) + AUX_SOURCE_DIRECTORY(. SRC) + + add_executable(docs_connect_example "") + add_executable(docs_create_db_demo "") + add_executable(docs_insert_data_demo "") + add_executable(docs_query_data_demo "") + add_executable(docs_with_reqid_demo "") + add_executable(docs_sml_insert_demo "") + add_executable(docs_stmt_insert_demo "") + add_executable(docs_tmq_demo "") + + target_sources(docs_connect_example + PRIVATE + "connect_example.c" + ) + + target_sources(docs_create_db_demo + PRIVATE + "create_db_demo.c" + ) + + target_sources(docs_insert_data_demo + PRIVATE + "insert_data_demo.c" + ) + + target_sources(docs_query_data_demo + PRIVATE + "query_data_demo.c" + ) + + target_sources(docs_with_reqid_demo + PRIVATE + "with_reqid_demo.c" + ) + + target_sources(docs_sml_insert_demo + PRIVATE + "sml_insert_demo.c" + ) + + target_sources(docs_stmt_insert_demo + PRIVATE + "stmt_insert_demo.c" + ) + + target_sources(docs_tmq_demo + PRIVATE + "tmq_demo.c" + ) + + target_link_libraries(docs_connect_example + taos + ) + + target_link_libraries(docs_create_db_demo + taos + ) + + target_link_libraries(docs_insert_data_demo + taos + ) + + target_link_libraries(docs_query_data_demo + taos + ) + + target_link_libraries(docs_with_reqid_demo + taos + ) + + target_link_libraries(docs_sml_insert_demo + taos + ) + + target_link_libraries(docs_stmt_insert_demo + taos + ) + + target_link_libraries(docs_tmq_demo + taos + pthread + ) + + SET_TARGET_PROPERTIES(docs_connect_example PROPERTIES OUTPUT_NAME docs_connect_example) + SET_TARGET_PROPERTIES(docs_create_db_demo PROPERTIES OUTPUT_NAME docs_create_db_demo) + SET_TARGET_PROPERTIES(docs_insert_data_demo PROPERTIES OUTPUT_NAME docs_insert_data_demo) + SET_TARGET_PROPERTIES(docs_query_data_demo PROPERTIES OUTPUT_NAME docs_query_data_demo) + SET_TARGET_PROPERTIES(docs_with_reqid_demo PROPERTIES OUTPUT_NAME docs_with_reqid_demo) + SET_TARGET_PROPERTIES(docs_sml_insert_demo PROPERTIES OUTPUT_NAME docs_sml_insert_demo) + SET_TARGET_PROPERTIES(docs_stmt_insert_demo PROPERTIES OUTPUT_NAME docs_stmt_insert_demo) + SET_TARGET_PROPERTIES(docs_tmq_demo PROPERTIES OUTPUT_NAME docs_tmq_demo) +ENDIF () +IF (TD_DARWIN) + INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc) + AUX_SOURCE_DIRECTORY(. SRC) +ENDIF () diff --git a/docs/examples/c/Makefile b/docs/examples/c/Makefile new file mode 100644 index 0000000000..9fda575ec6 --- /dev/null +++ b/docs/examples/c/Makefile @@ -0,0 +1,34 @@ +# Makefile for building TDengine examples on TD Linux platform + +INCLUDE_DIRS = + +TARGETS = connect_example \ + create_db_demo \ + insert_data_demo \ + query_data_demo \ + with_reqid_demo \ + sml_insert_demo \ + stmt_insert_demo \ + tmq_demo + +SOURCES = connect_example.c \ + create_db_demo.c \ + insert_data_demo.c \ + query_data_demo.c \ + with_reqid_demo.c \ + sml_insert_demo.c \ + stmt_insert_demo.c \ + tmq_demo.c + +LIBS = -ltaos -lpthread + + +CFLAGS = -g + +all: $(TARGETS) + +$(TARGETS): + $(CC) $(CFLAGS) -o $@ $(wildcard $(@F).c) $(LIBS) + +clean: + rm -f $(TARGETS) \ No newline at end of file diff --git a/docs/examples/node/package.json b/docs/examples/node/package.json index d00d71d99f..3f5f54e9d7 100644 --- a/docs/examples/node/package.json +++ b/docs/examples/node/package.json @@ -4,7 +4,6 @@ "main": "index.js", "license": "MIT", "dependencies": { - "@tdengine/client": "^3.0.1", - "@tdengine/rest": "^3.0.0" + "@tdengine/websocket": "^3.1.0" } } diff --git a/docs/examples/node/websocketexample/all_type_query.js b/docs/examples/node/websocketexample/all_type_query.js index 266d110d24..6524d35718 100644 --- a/docs/examples/node/websocketexample/all_type_query.js +++ b/docs/examples/node/websocketexample/all_type_query.js @@ -34,6 +34,7 @@ async function json_tag_example() { } catch (err) { console.error(`Failed to create database example_json_tag or stable stb, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (wsSql) { await wsSql.close(); @@ -78,9 +79,10 @@ async function all_type_example() { let row = wsRows.getData(); console.log(row); } - + } catch (err) { console.error(`Failed to create database all_type_example or stable stb, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (wsSql) { await wsSql.close(); @@ -91,7 +93,7 @@ async function all_type_example() { async function test() { await json_tag_example() - await all_type_example() + await all_type_example() taos.destroy(); } diff --git a/docs/examples/node/websocketexample/all_type_stmt.js b/docs/examples/node/websocketexample/all_type_stmt.js index 8a0dcf21e1..f095bee090 100644 --- a/docs/examples/node/websocketexample/all_type_stmt.js +++ b/docs/examples/node/websocketexample/all_type_stmt.js @@ -46,6 +46,7 @@ async function json_tag_example() { } catch (err) { console.error(`Failed to create database example_json_tag or stable stb, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err } finally { if (wsSql) { await wsSql.close(); @@ -125,6 +126,7 @@ async function all_type_example() { } catch (err) { console.error(`Failed to create database all_type_example or stable stb, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (stmt) { await stmt.close(); @@ -136,10 +138,7 @@ async function all_type_example() { } - - -async function test() { - taos.setLevel("debug") +async function test() { await json_tag_example() await all_type_example() taos.destroy(); diff --git a/docs/examples/node/websocketexample/json_line_example.js b/docs/examples/node/websocketexample/json_line_example.js deleted file mode 100644 index e6587eaa45..0000000000 --- a/docs/examples/node/websocketexample/json_line_example.js +++ /dev/null @@ -1,53 +0,0 @@ -const taos = require("@tdengine/websocket"); - -var host = null; -for(var i = 2; i < global.process.argv.length; i++){ - var key = global.process.argv[i].split("=")[0]; - var value = global.process.argv[i].split("=")[1]; - if("host" == key){ - host = value; - } -} - -if(host == null){ - console.log("Usage: node nodejsChecker.js host= port="); - process.exit(0); - } - -let dbData = ["{\"metric\": \"meter_current\",\"timestamp\": 1626846402,\"value\": 10.3, \"tags\": {\"groupid\": 2, \"location\": \"California.SanFrancisco\", \"id\": \"d1001\"}}", - "{\"metric\": \"meter_current\",\"timestamp\": 1626846403,\"value\": 10.3, \"tags\": {\"groupid\": 2, \"location\": \"California.SanFrancisco\", \"id\": \"d1002\"}}", - "{\"metric\": \"meter_current\",\"timestamp\": 1626846404,\"value\": 10.3, \"tags\": {\"groupid\": 2, \"location\": \"California.SanFrancisco\", \"id\": \"d1003\"}}"] - -async function createConnect() { - let dsn = 'ws://' + host + ':6041' - let conf = new taos.WSConfig(dsn); - conf.setUser('root'); - conf.setPwd('taosdata'); - conf.setDb('power'); - return await taos.sqlConnect(conf); -} - -async function test() { - let wsSql = null; - let wsRows = null; - let reqId = 0; - try { - wsSql = await createConnect() - await wsSql.exec('CREATE DATABASE IF NOT EXISTS power KEEP 3650 DURATION 10 BUFFER 16 WAL_LEVEL 1;', reqId++); - await wsSql.schemalessInsert([dbData], taos.SchemalessProto.OpenTSDBJsonFormatProtocol, taos.Precision.SECONDS, 0); - } - catch (err) { - console.error(err.code, err.message); - } - finally { - if (wsRows) { - await wsRows.close(); - } - if (wsSql) { - await wsSql.close(); - } - taos.destroy(); - } -} - -test() \ No newline at end of file diff --git a/docs/examples/node/websocketexample/line_example.js b/docs/examples/node/websocketexample/line_example.js index ac3083d358..97570475b0 100644 --- a/docs/examples/node/websocketexample/line_example.js +++ b/docs/examples/node/websocketexample/line_example.js @@ -15,8 +15,8 @@ async function createConnect() { return wsSql; } + async function test() { - let dsn = 'ws://localhost:6041' let wsSql = null; let wsRows = null; let ttl = 0; @@ -29,6 +29,7 @@ async function test() { } catch (err) { console.error(`Failed to insert data with schemaless, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (wsRows) { @@ -40,4 +41,5 @@ async function test() { taos.destroy(); } } + test() diff --git a/docs/examples/node/websocketexample/nodejsChecker.js b/docs/examples/node/websocketexample/nodejsChecker.js index d81aeb585f..9f1d40f0cc 100644 --- a/docs/examples/node/websocketexample/nodejsChecker.js +++ b/docs/examples/node/websocketexample/nodejsChecker.js @@ -10,11 +10,9 @@ for(var i = 2; i < global.process.argv.length; i++){ } if(host == null){ - console.log("Usage: node nodejsChecker.js host= port="); - process.exit(0); + host = 'localhost'; } - async function createConnect() { let dsn = 'ws://' + host + ':6041' console.log(dsn) @@ -41,7 +39,7 @@ async function test() { taosResult = await wsSql.exec('USE power', reqId++); console.log(taosResult); - taosResult = await wsSql.exec('CREATE STABLE IF NOT EXISTS meters (_ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);', reqId++); + taosResult = await wsSql.exec('CREATE STABLE IF NOT EXISTS meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);', reqId++); console.log(taosResult); taosResult = await wsSql.exec('DESCRIBE meters', reqId++); @@ -62,6 +60,7 @@ async function test() { } catch (err) { console.error(err.code, err.message); + throw err; } finally { if (wsRows) { diff --git a/docs/examples/node/websocketexample/sql_example.js b/docs/examples/node/websocketexample/sql_example.js index 8eb8af989d..0a09228d97 100644 --- a/docs/examples/node/websocketexample/sql_example.js +++ b/docs/examples/node/websocketexample/sql_example.js @@ -41,6 +41,7 @@ async function createDbAndTable() { console.log("Create stable power.meters successfully"); } catch (err) { console.error(`Failed to create database power or stable meters, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (wsSql) { await wsSql.close(); @@ -53,21 +54,23 @@ async function createDbAndTable() { // ANCHOR: insertData async function insertData() { let wsSql = null + let insertQuery = "INSERT INTO " + + "power.d1001 USING power.meters (location, groupId) TAGS('California.SanFrancisco', 2) " + + "VALUES " + + "(NOW + 1a, 10.30000, 219, 0.31000) " + + "(NOW + 2a, 12.60000, 218, 0.33000) " + + "(NOW + 3a, 12.30000, 221, 0.31000) " + + "power.d1002 USING power.meters (location, groupId) TAGS('California.SanFrancisco', 3) " + + "VALUES " + + "(NOW + 1a, 10.30000, 218, 0.25000) "; + try { wsSql = await createConnect(); - let insertQuery = "INSERT INTO " + - "power.d1001 USING power.meters (location, groupId) TAGS('California.SanFrancisco', 2) " + - "VALUES " + - "(NOW + 1a, 10.30000, 219, 0.31000) " + - "(NOW + 2a, 12.60000, 218, 0.33000) " + - "(NOW + 3a, 12.30000, 221, 0.31000) " + - "power.d1002 USING power.meters TAGS('California.SanFrancisco', 3) " + - "VALUES " + - "(NOW + 1a, 10.30000, 218, 0.25000) "; taosResult = await wsSql.exec(insertQuery); console.log("Successfully inserted " + taosResult.getAffectRows() + " rows to power.meters."); } catch (err) { console.error(`Failed to insert data to power.meters, sql: ${insertQuery}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (wsSql) { await wsSql.close(); @@ -91,6 +94,7 @@ async function queryData() { } catch (err) { console.error(`Failed to query data from power.meters, sql: ${sql}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (wsRows) { @@ -118,6 +122,7 @@ async function sqlWithReqid() { } catch (err) { console.error(`Failed to query data from power.meters, reqId: ${reqId}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (wsRows) { @@ -135,7 +140,7 @@ async function test() { await insertData(); await queryData(); await sqlWithReqid(); - taos.destroy(); + taos.destroy(); } test() diff --git a/docs/examples/node/websocketexample/stmt_example.js b/docs/examples/node/websocketexample/stmt_example.js index e3bb3c4dda..6f98bb7974 100644 --- a/docs/examples/node/websocketexample/stmt_example.js +++ b/docs/examples/node/websocketexample/stmt_example.js @@ -23,7 +23,7 @@ async function prepare() { return wsSql } -(async () => { +async function test() { let stmt = null; let connector = null; try { @@ -60,6 +60,7 @@ async function prepare() { } catch (err) { console.error(`Failed to insert to table meters using stmt, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (stmt) { @@ -70,4 +71,6 @@ async function prepare() { } taos.destroy(); } -})(); +} + +test() \ No newline at end of file diff --git a/docs/examples/node/websocketexample/telnet_line_example.js b/docs/examples/node/websocketexample/telnet_line_example.js deleted file mode 100644 index 924137e162..0000000000 --- a/docs/examples/node/websocketexample/telnet_line_example.js +++ /dev/null @@ -1,58 +0,0 @@ -const taos = require("@tdengine/websocket"); - -var host = null; -for(var i = 2; i < global.process.argv.length; i++){ - var key = global.process.argv[i].split("=")[0]; - var value = global.process.argv[i].split("=")[1]; - if("host" == key){ - host = value; - } -} - -if(host == null){ - console.log("Usage: node nodejsChecker.js host= port="); - process.exit(0); - } - - let dbData = ["meters.current 1648432611249 10.3 location=California.SanFrancisco groupid=2", - "meters.current 1648432611250 12.6 location=California.SanFrancisco groupid=2", - "meters.current 1648432611249 10.8 location=California.LosAngeles groupid=3", - "meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3", - "meters.voltage 1648432611249 219 location=California.SanFrancisco groupid=2", - "meters.voltage 1648432611250 218 location=California.SanFrancisco groupid=2", - "meters.voltage 1648432611249 221 location=California.LosAngeles groupid=3", - "meters.voltage 1648432611250 217 location=California.LosAngeles groupid=3",]; - -async function createConnect() { - let dsn = 'ws://' + host + ':6041' - let conf = new taos.WSConfig(dsn); - conf.setUser('root'); - conf.setPwd('taosdata'); - - return await taos.sqlConnect(conf); -} - -async function test() { - let wsSql = null; - let wsRows = null; - let reqId = 0; - try { - wsSql = await createConnect() - await wsSql.exec('create database if not exists power KEEP 3650 DURATION 10 BUFFER 16 WAL_LEVEL 1;', reqId++); - await wsSql.exec('use power', reqId++); - await wsSql.schemalessInsert(dbData, taos.SchemalessProto.OpenTSDBTelnetLineProtocol, taos.Precision.MILLI_SECONDS, 0); - } - catch (err) { - console.error(err.code, err.message); - } - finally { - if (wsRows) { - await wsRows.close(); - } - if (wsSql) { - await wsSql.close(); - } - taos.destroy(); - } -} -test() \ No newline at end of file diff --git a/docs/examples/node/websocketexample/tmq_example.js b/docs/examples/node/websocketexample/tmq_example.js index 5097402e6a..4f7f099c31 100644 --- a/docs/examples/node/websocketexample/tmq_example.js +++ b/docs/examples/node/websocketexample/tmq_example.js @@ -1,3 +1,4 @@ +const { sleep } = require("@tdengine/websocket"); const taos = require("@tdengine/websocket"); // ANCHOR: create_consumer @@ -49,12 +50,20 @@ async function prepare() { let createTopic = `CREATE TOPIC IF NOT EXISTS ${topics[0]} AS SELECT * FROM ${db}.${stable}`; await wsSql.exec(createTopic); + await wsSql.close(); +} - - for (let i = 0; i < 10; i++) { +async function insert() { + let conf = new taos.WSConfig('ws://localhost:6041'); + conf.setUser('root'); + conf.setPwd('taosdata'); + conf.setDb('power'); + let wsSql = await taos.sqlConnect(conf); + for (let i = 0; i < 50; i++) { await wsSql.exec(`INSERT INTO d1001 USING ${stable} (location, groupId) TAGS ("California.SanFrancisco", 3) VALUES (NOW, ${10 + i}, ${200 + i}, ${0.32 + i})`); + await sleep(100); } - wsSql.close(); + await wsSql.close(); } async function subscribe(consumer) { @@ -82,13 +91,17 @@ async function test() { let consumer = null; try { await prepare(); - consumer = await createConsumer() - await subscribe(consumer) + consumer = await createConsumer(); + const allPromises = []; + allPromises.push(subscribe(consumer)); + allPromises.push(insert()); + await Promise.all(allPromises); await consumer.unsubscribe(); console.log("Consumer unsubscribed successfully."); } catch (err) { console.error(`Failed to unsubscribe consumer, topic: ${topic}, groupId: ${groupId}, clientId: ${clientId}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (consumer) { diff --git a/docs/examples/node/websocketexample/tmq_seek_example.js b/docs/examples/node/websocketexample/tmq_seek_example.js index b2bd569d92..f676efe36f 100644 --- a/docs/examples/node/websocketexample/tmq_seek_example.js +++ b/docs/examples/node/websocketexample/tmq_seek_example.js @@ -1,41 +1,45 @@ +const { sleep } = require("@tdengine/websocket"); const taos = require("@tdengine/websocket"); const db = 'power'; const stable = 'meters'; +const url = 'ws://localhost:6041'; const topic = 'topic_meters' const topics = [topic]; const groupId = "group1"; const clientId = "client1"; - -// ANCHOR: create_consumer async function createConsumer() { + + let groupId = "group1"; + let clientId = "client1"; let configMap = new Map([ - [taos.TMQConstants.GROUP_ID, "group1"], - [taos.TMQConstants.CLIENT_ID, 'client1'], + [taos.TMQConstants.GROUP_ID, groupId], + [taos.TMQConstants.CLIENT_ID, clientId], [taos.TMQConstants.CONNECT_USER, "root"], [taos.TMQConstants.CONNECT_PASS, "taosdata"], [taos.TMQConstants.AUTO_OFFSET_RESET, "latest"], - [taos.TMQConstants.WS_URL, 'ws://localhost:6041'], + [taos.TMQConstants.WS_URL, url], [taos.TMQConstants.ENABLE_AUTO_COMMIT, 'true'], [taos.TMQConstants.AUTO_COMMIT_INTERVAL_MS, '1000'] ]); try { - return await taos.tmqConnect(configMap); + conn = await taos.tmqConnect(configMap); + console.log(`Create consumer successfully, host: ${url}, groupId: ${groupId}, clientId: ${clientId}`) + return conn; } catch (err) { - console.error(err); + console.error(`Failed to create websocket consumer, topic: ${topic}, groupId: ${groupId}, clientId: ${clientId}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); throw err; } } -// ANCHOR_END: create_consumer async function prepare() { - let conf = new taos.WSConfig('ws://localhost:6041'); + let conf = new taos.WSConfig('ws://192.168.1.98:6041'); conf.setUser('root'); conf.setPwd('taosdata'); conf.setDb('power'); - const createDB = `CREATE DATABASE IF NOT EXISTS ${db} KEEP 3650 DURATION 10 BUFFER 16 WAL_LEVEL 1;`; + const createDB = `CREATE DATABASE IF NOT EXISTS ${db}`; const createStable = `CREATE STABLE IF NOT EXISTS ${db}.${stable} (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);`; let wsSql = await taos.sqlConnect(conf); @@ -44,58 +48,63 @@ async function prepare() { let createTopic = `CREATE TOPIC IF NOT EXISTS ${topics[0]} AS SELECT * FROM ${db}.${stable}`; await wsSql.exec(createTopic); + await wsSql.close(); +} - - for (let i = 0; i < 10; i++) { +async function insert() { + let conf = new taos.WSConfig('ws://localhost:6041'); + conf.setUser('root'); + conf.setPwd('taosdata'); + conf.setDb('power'); + let wsSql = await taos.sqlConnect(conf); + for (let i = 0; i < 1; i++) { await wsSql.exec(`INSERT INTO d1001 USING ${stable} (location, groupId) TAGS ("California.SanFrancisco", 3) VALUES (NOW, ${10 + i}, ${200 + i}, ${0.32 + i})`); } await wsSql.close(); } -// ANCHOR: subscribe +// ANCHOR: offset async function subscribe(consumer) { try { - await consumer.subscribe(['topic_meters']); - for (let i = 0; i < 50; i++) { - let res = await consumer.poll(100); - for (let [key, value] of res) { - // Add your data processing logic here - console.log(`data: ${key} ${value}`); - } - } - } catch (err) { - console.error(`Failed to poll data, topic: ${topic}, groupId: ${groupId}, clientId: ${clientId}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); - throw err; - } - -} -// ANCHOR_END: subscribe - -// ANCHOR: offset -async function test() { - let consumer = null; - try { - await prepare(); - let consumer = await createConsumer() await consumer.subscribe(['topic_meters']); let res = new Map(); while (res.size == 0) { res = await consumer.poll(100); + await consumer.commit(); } - let assignment = await consumer.assignment(); await consumer.seekToBeginning(assignment); console.log("Assignment seek to beginning successfully"); + } catch (err) { + console.error(`Failed to seek offset, topic: ${topic}, groupId: ${groupId}, clientId: ${clientId}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; + } +} +// ANCHOR_END: offset + +async function test() { + let consumer = null; + try { + await prepare(); + consumer = await createConsumer(); + const allPromises = []; + allPromises.push(subscribe(consumer)); + allPromises.push(insert()); + await Promise.all(allPromises); + await consumer.unsubscribe(); + console.log("Consumer unsubscribed successfully."); } catch (err) { - console.error(`Failed to seek offset, topic: ${topic}, groupId: ${groupId}, clientId: ${clientId}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + console.error(`Failed to consumer, topic: ${topic}, groupId: ${groupId}, clientId: ${clientId}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (consumer) { await consumer.close(); + console.log("Consumer closed successfully."); } taos.destroy(); } } -// ANCHOR_END: offset + test() diff --git a/docs/examples/python/connect_example.py b/docs/examples/python/connect_example.py index ce8b306024..e1f1a63d57 100644 --- a/docs/examples/python/connect_example.py +++ b/docs/examples/python/connect_example.py @@ -15,6 +15,7 @@ def create_connection(): print(f"Connected to {host}:{port} successfully."); except Exception as err: print(f"Failed to connect to {host}:{port} , ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/connect_websocket_examples.py b/docs/examples/python/connect_websocket_examples.py index 56d208f5db..fe8f407865 100644 --- a/docs/examples/python/connect_websocket_examples.py +++ b/docs/examples/python/connect_websocket_examples.py @@ -15,7 +15,7 @@ def create_connection(): print(f"Connected to {host}:{port} successfully."); except Exception as err: print(f"Failed to connect to {host}:{port} , ErrMessage:{err}") - + raise err return conn # ANCHOR_END: connect @@ -28,6 +28,7 @@ def create_db_table(conn): conn.execute("CREATE TABLE IF NOT EXISTS `d0` USING `meters` (groupId, location) TAGS(0, 'Los Angles')") except Exception as err: print(f'Exception {err}') + raise err # ANCHOR_END: create_db def insert(conn): @@ -42,9 +43,10 @@ def insert(conn): """ try: inserted = conn.execute(sql) - assert inserted == 8 + assert inserted == 4 except Exception as err: print(f'Exception111 {err}') + raise err # ANCHOR_END: insert def query(conn): @@ -58,6 +60,7 @@ def query(conn): print(row) except Exception as err: print(f'Exception {err}') + raise err # ANCHOR_END: query if __name__ == "__main__": diff --git a/docs/examples/python/create_db_native.py b/docs/examples/python/create_db_native.py index 34dabfabe2..cf263b7b27 100644 --- a/docs/examples/python/create_db_native.py +++ b/docs/examples/python/create_db_native.py @@ -21,6 +21,7 @@ try: except Exception as err: print(f"Failed to create database power or stable meters, ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/create_db_rest.py b/docs/examples/python/create_db_rest.py index 4b98c991a3..4a247dfaec 100644 --- a/docs/examples/python/create_db_rest.py +++ b/docs/examples/python/create_db_rest.py @@ -20,6 +20,7 @@ try: except Exception as err: print(f"Failed to create database power or stable meters, ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/create_db_ws.py b/docs/examples/python/create_db_ws.py index ddbacb4b1f..29ee95dc65 100644 --- a/docs/examples/python/create_db_ws.py +++ b/docs/examples/python/create_db_ws.py @@ -21,6 +21,7 @@ try: except Exception as err: print(f"Failed to create database power or stable meters, ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/insert_native.py b/docs/examples/python/insert_native.py index 19dafa3f23..b49860dfb1 100644 --- a/docs/examples/python/insert_native.py +++ b/docs/examples/python/insert_native.py @@ -22,6 +22,7 @@ try: except Exception as err: print(f"Failed to insert data to power.meters, sql: {sql}, ErrMessage: {err}.") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/insert_rest.py b/docs/examples/python/insert_rest.py index 526c3a6a69..115ec1a702 100644 --- a/docs/examples/python/insert_rest.py +++ b/docs/examples/python/insert_rest.py @@ -21,6 +21,7 @@ try: except Exception as err: print(f"Failed to insert data to power.meters, sql:{sql}, ErrMessage:{err}.") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/insert_ws.py b/docs/examples/python/insert_ws.py index 886dda1c10..9fec00e02b 100644 --- a/docs/examples/python/insert_ws.py +++ b/docs/examples/python/insert_ws.py @@ -22,6 +22,7 @@ try: except Exception as err: print(f"Failed to insert data to power.meters, sql: {sql}, ErrMessage: {err}.") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/query_native.py b/docs/examples/python/query_native.py index 072807986e..7c2015fe04 100644 --- a/docs/examples/python/query_native.py +++ b/docs/examples/python/query_native.py @@ -16,6 +16,7 @@ try: except Exception as err: print(f"Failed to query data from power.meters, sql: {sql}, ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/query_rest.py b/docs/examples/python/query_rest.py index 85a70fd382..b864a4ccd6 100644 --- a/docs/examples/python/query_rest.py +++ b/docs/examples/python/query_rest.py @@ -15,3 +15,4 @@ try: except Exception as err: print(f"Failed to query data from power.meters, sql: {sql}, ErrMessage:{err}") + raise err diff --git a/docs/examples/python/query_ws.py b/docs/examples/python/query_ws.py index afab438ad9..52484b5308 100644 --- a/docs/examples/python/query_ws.py +++ b/docs/examples/python/query_ws.py @@ -15,6 +15,7 @@ try: except Exception as err: print(f"Failed to query data from power.meters, sql: {sql}, ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/reqid_native.py b/docs/examples/python/reqid_native.py index 7f16093835..57f438d3f2 100644 --- a/docs/examples/python/reqid_native.py +++ b/docs/examples/python/reqid_native.py @@ -18,7 +18,7 @@ try: except Exception as err: print(f"Failed to execute sql with reqId:{reqId}, ErrMessage:{err}") - + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/reqid_rest.py b/docs/examples/python/reqid_rest.py index 570e671092..a5f752113b 100644 --- a/docs/examples/python/reqid_rest.py +++ b/docs/examples/python/reqid_rest.py @@ -16,3 +16,4 @@ try: except Exception as err: print(f"Failed to execute sql with reqId:{reqId}, ErrMessage:{err}") + raise err diff --git a/docs/examples/python/reqid_ws.py b/docs/examples/python/reqid_ws.py index 7c74104169..217ff0b54b 100644 --- a/docs/examples/python/reqid_ws.py +++ b/docs/examples/python/reqid_ws.py @@ -19,6 +19,7 @@ try: except Exception as err: print(f"Failed to execute sql with reqId:{reqId}, ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/schemaless_native.py b/docs/examples/python/schemaless_native.py index 96d8f3177f..9654ee02dd 100644 --- a/docs/examples/python/schemaless_native.py +++ b/docs/examples/python/schemaless_native.py @@ -35,6 +35,7 @@ try: print("Inserted data with schemaless successfully."); except Exception as err: print(f"Failed to insert data with schemaless, ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/schemaless_ws.py b/docs/examples/python/schemaless_ws.py index 39de55393d..3033e9b670 100644 --- a/docs/examples/python/schemaless_ws.py +++ b/docs/examples/python/schemaless_ws.py @@ -75,8 +75,6 @@ def schemaless_insert(): conn.close() if __name__ == "__main__": - try: - prepare() - schemaless_insert() - except Exception as err: - print(f"Failed to insert data with schemaless, err:{err}") + prepare() + schemaless_insert() + diff --git a/docs/examples/python/stmt_native.py b/docs/examples/python/stmt_native.py index a1af7d1dd7..3e4475f1f7 100644 --- a/docs/examples/python/stmt_native.py +++ b/docs/examples/python/stmt_native.py @@ -57,6 +57,7 @@ try: except Exception as err: print(f"Failed to insert to table meters using stmt, ErrMessage:{err}") + raise err finally: if stmt: stmt.close() diff --git a/docs/examples/python/stmt_ws.py b/docs/examples/python/stmt_ws.py index 45d9222315..82686abda4 100644 --- a/docs/examples/python/stmt_ws.py +++ b/docs/examples/python/stmt_ws.py @@ -62,6 +62,7 @@ try: except Exception as err: print(f"Failed to insert to table meters using stmt, ErrMessage:{err}") + raise err finally: if stmt: stmt.close() diff --git a/docs/examples/python/tmq_native.py b/docs/examples/python/tmq_native.py index d4ccfda138..34bac26d95 100644 --- a/docs/examples/python/tmq_native.py +++ b/docs/examples/python/tmq_native.py @@ -152,6 +152,7 @@ def unsubscribe(consumer): print("Consumer unsubscribed successfully."); except Exception as err: print(f"Failed to unsubscribe consumer. topic: {topic}, groupId: {groupId}, clientId: {clientId}, ErrMessage:{err}.") + raise err finally: if consumer: consumer.close() @@ -166,7 +167,6 @@ if __name__ == "__main__": subscribe(consumer) seek_offset(consumer) commit_offset(consumer) - except Exception as err: - print(f"Failed to execute consumer example, topic: {topic}, groupId: {groupId}, clientId: {clientId}, ErrMessage:{err}.") finally: - unsubscribe(consumer); + if consumer: + unsubscribe(consumer); diff --git a/docs/examples/python/tmq_websocket_example.py b/docs/examples/python/tmq_websocket_example.py index c9c7924661..7bd3f57da6 100644 --- a/docs/examples/python/tmq_websocket_example.py +++ b/docs/examples/python/tmq_websocket_example.py @@ -31,7 +31,7 @@ def prepareMeta(): # create super table rowsAffected = conn.execute( - "CREATE TABLE IF NOT EXISTS `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` BINARY(16))" + "CREATE TABLE IF NOT EXISTS `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` BINARY(64))" ) assert rowsAffected == 0 @@ -155,6 +155,7 @@ def unsubscribe(consumer): print("Consumer unsubscribed successfully."); except Exception as err: print(f"Failed to unsubscribe consumer. topic: {topic}, groupId: {groupId}, clientId: {clientId}, ErrMessage:{err}.") + raise err finally: if consumer: consumer.close() @@ -170,7 +171,6 @@ if __name__ == "__main__": subscribe(consumer) seek_offset(consumer) commit_offset(consumer) - except Exception as err: - print(f"Failed to execute consumer example, topic: {topic}, groupId: {groupId}, clientId: {clientId}, ErrMessage:{err}.") finally: - unsubscribe(consumer) + if consumer: + unsubscribe(consumer) diff --git a/docs/examples/rust/restexample/examples/tmq.rs b/docs/examples/rust/restexample/examples/tmq.rs index 61416133e3..8586bbc5c7 100644 --- a/docs/examples/rust/restexample/examples/tmq.rs +++ b/docs/examples/rust/restexample/examples/tmq.rs @@ -83,7 +83,6 @@ async fn main() -> anyhow::Result<()> { eprintln!("Failed to execute insert: {:?}", e); } tokio::time::sleep(Duration::from_millis(10)).await; - println!("Succed to execute insert 1 row"); } }); }); diff --git a/docs/zh/05-basic/03-query.md b/docs/zh/05-basic/03-query.md index 1b4c3731e6..6afdba0997 100644 --- a/docs/zh/05-basic/03-query.md +++ b/docs/zh/05-basic/03-query.md @@ -181,7 +181,7 @@ INTERVAL(interval_val [, interval_offset]) - FILL：用于指定窗口区间数据缺失的情况下，数据的填充模式。 对于时间窗口，interval_val 和 sliding_val 都表示时间段， 语法上支持三种方式。例如： -1. INTERVAL(1s, 500a) SLIDING(1s)，带时间单位的形式，其中的时间单位是单字符表示， 分别为： a （毫秒）， b （纳秒）， d （天）， h （小时）， m （分钟）， n （月）， s （秒）， u （微妙）， w （周）， y （年）； +1. INTERVAL(1s, 500a) SLIDING(1s)，带时间单位的形式，其中的时间单位是单字符表示， 分别为： a （毫秒）， b （纳秒）， d （天）， h （小时）， m （分钟）， n （月）， s （秒）， u （微秒）， w （周）， y （年）； 2. INTERVAL(1000, 500) SLIDING(1000)，不带时间单位的形式，将使用查询库的时间精度作为默认时间单位，当存在多个库时默认采用精度更高的库； 3. INTERVAL('1s', '500a') SLIDING('1s')，带时间单位的字符串形式，字符串内部不能有任何空格等其它字符。 diff --git a/docs/zh/06-advanced/05-data-in/03-pi.md b/docs/zh/06-advanced/05-data-in/03-pi.md index 6b65b1337f..8a2c5ada35 100644 --- a/docs/zh/06-advanced/05-data-in/03-pi.md +++ b/docs/zh/06-advanced/05-data-in/03-pi.md @@ -27,7 +27,7 @@ PI 系统是一套用于数据收集、查找、分析、传递和可视化的 在数据写入页面中，点击 **+新增数据源** 按钮，进入新增数据源页面。 -![kafka-01.png](./kafka-01.png) +![new.png](./pic/pi-01-new.png) ### 基本配置 diff --git a/docs/zh/06-advanced/05-data-in/05-opcua.md b/docs/zh/06-advanced/05-data-in/05-opcua.md index 7ae7f153e9..5795528d01 100644 --- a/docs/zh/06-advanced/05-data-in/05-opcua.md +++ b/docs/zh/06-advanced/05-data-in/05-opcua.md @@ -208,3 +208,15 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下 ### 8. 创建完成 点击 **提交** 按钮，完成创建 OPC UA 到 TDengine 的数据同步任务，回到**数据源列表**页面可查看任务执行情况。 + +## 增加数据点位 + +在任务运行中，点击 **编辑**，点击 **增加数据点位** 按钮，追加数据点位到 CSV 文件中。 + +![增加数据点位](./pic/opc-08-add-point.png) + +在弹出的表单中，填写数据点位的信息。 + +![数据点位表单](./pic/opc-09-add-point.png) + +点击 **确定** 按钮，完成数据点位的追加。 \ No newline at end of file diff --git a/docs/zh/06-advanced/05-data-in/06-opcda.md b/docs/zh/06-advanced/05-data-in/06-opcda.md index e84876d7ed..7da5b89fe6 100644 --- a/docs/zh/06-advanced/05-data-in/06-opcda.md +++ b/docs/zh/06-advanced/05-data-in/06-opcda.md @@ -182,3 +182,15 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下 ### 7. 创建完成 点击 **提交** 按钮，完成创建 OPC DA 到 TDengine 的数据同步任务，回到**数据源列表**页面可查看任务执行情况。 + +## 增加数据点位 + +在任务运行中，点击 **编辑**，点击 **增加数据点位** 按钮，追加数据点位到 CSV 文件中。 + +![增加数据点位](./pic/opc-08-add-point.png) + +在弹出的表单中，填写数据点位的信息。 + +![数据点位表单](./pic/opc-09-add-point.png) + +点击 **确定** 按钮，完成数据点位的追加。 \ No newline at end of file diff --git a/docs/zh/06-advanced/05-data-in/07-mqtt.md b/docs/zh/06-advanced/05-data-in/07-mqtt.md index af99cd3621..f54086b61b 100644 --- a/docs/zh/06-advanced/05-data-in/07-mqtt.md +++ b/docs/zh/06-advanced/05-data-in/07-mqtt.md @@ -33,13 +33,14 @@ TDengine 可以通过 MQTT 连接器从 MQTT 代理订阅数据并将其写入 T ### 3. 配置连接和认证信息 -在 **MQTT地址** 中填写 MQTT 代理的地址，例如：`192.168.1.42:1883` +在 **MQTT 地址** 中填写 MQTT 代理的地址，例如：`192.168.1.42` + +在 **MQTT 端口** 中填写 MQTT 代理的端口，例如：`1883` 在 **用户** 中填写 MQTT 代理的用户名。 在 **密码** 中填写 MQTT 代理的密码。 -点击 **连通性检查** 按钮，检查数据源是否可用。 ![mqtt-03.png](./mqtt-03.png) @@ -64,6 +65,8 @@ TDengine 可以通过 MQTT 连接器从 MQTT 代理订阅数据并将其写入 T 在 **订阅主题及 QoS 配置** 中填写要消费的 Topic 名称。使用如下格式设置： `topic1::0,topic2::1`。 +点击 **检查连通性** 按钮，检查数据源是否可用。 + ![mqtt-05.png](./mqtt-05.png) ### 6. 配置 MQTT Payload 解析 diff --git a/docs/zh/06-advanced/05-data-in/08-kafka.md b/docs/zh/06-advanced/05-data-in/08-kafka.md index e05c205f6e..8cca24930e 100644 --- a/docs/zh/06-advanced/05-data-in/08-kafka.md +++ b/docs/zh/06-advanced/05-data-in/08-kafka.md @@ -44,8 +44,50 @@ TDengine 可以高效地从 Kafka 读取数据并将其写入 TDengine，以实 如果服务端开启了 SASL 认证机制，此处需要启用 SASL 并配置相关内容，目前支持 PLAIN/SCRAM-SHA-256/GSSAPI 三种认证机制，请按实际情况进行选择。 +#### 4.1. PLAIN 认证 + +选择 `PLAIN` 认证机制，输入用户名和密码： + +![kafka-04-sasl-plain.png](./kafka-04-sasl-plain.png) + +#### 4.1. SCRAM(SCRAM-SHA-256) 认证 + +选择 `SCRAM-SHA-256` 认证机制，输入用户名和密码： + ![kafka-04.png](./kafka-04.png) +#### 4.3. GSSAPI 认证 + +选择 `GSSAPI` ，将通过 [RDkafka 客户端](https://github.com/confluentinc/librdkafka) 调用 GSSAPI 应用 Kerberos 认证机制： + +![kafka-04-sasl-gssapi.png](./kafka-04-sasl-gssapi.png) + +需要输入的信息有： + +- Kerberos 服务名，一般是 `kafka`； +- Kerberos 认证主体，即认证用户名，例如 `kafkaclient`； +- Kerberos 初始化命令（可选，一般不用填写）； +- Kerberos 密钥表，需提供文件并上传； + +以上信息均需由 Kafka 服务管理者提供。 + +除此之外，在服务器上需要配置 [Kerberos](https://web.mit.edu/kerberos/) 认证服务。在 Ubuntu 下使用 `apt install krb5-user` ；在 CentOS 下，使用 `yum install krb5-workstation`；即可。 + +配置完成后，可以使用 [kcat](https://github.com/edenhill/kcat) 工具进行 Kafka 主题消费验证： + +```bash +kcat \ + -b \ + -G kcat \ + -X security.protocol=SASL_PLAINTEXT \ + -X sasl.mechanism=GSSAPI \ + -X sasl.kerberos.keytab= \ + -X sasl.kerberos.principal= \ + -X sasl.kerberos.service.name=kafka +``` + +如果出现错误：“Server xxxx not found in kerberos database”，则需要配置 Kafka 节点对应的域名并在 Kerberos 客户端配置文件 `/etc/krb5.conf` 中配置反向域名解析 `rdns = true`。 + ### 5. 配置 SSL 证书 如果服务端开启了 SSL 加密认证,此处需要启用 SSL 并配置相关内容。 @@ -60,7 +102,7 @@ TDengine 可以高效地从 Kafka 读取数据并将其写入 TDengine，以实 在 **主题** 中填写要消费的 Topic 名称。可以配置多个 Topic ， Topic 之间用逗号分隔。例如：`tp1,tp2`。 -在 **Client ID** 中填写客户端标识，填写后会生成带有 `taosx` 前缀的客户端 ID （例如，如果填写的标识为 `foo`，则生成的客户端 ID 为 `taosxfoo`）。如果打开末尾处的开关，则会把当前任务的任务 ID 拼接到 `taosx` 之后，输入的标识之前（生成的客户端 ID 形如 `taosx100foo`）。连接到同一个 Kafka 集群的所有客户端 ID 必须保证唯一。 +在 **Client ID** 中填写客户端标识，填写后会生成带有 `taosx` 前缀的客户端 ID （例如，如果填写的标识为 `foo`，则生成的客户端 ID 为 `taosxfoo`）。如果打开末尾处的开关，则会把当前任务的任务 ID 拼接到 `taosx` 之后，输入的标识之前（生成的客户端 ID 形如 `taosx100foo`）。需要注意的是，当使用多个 taosX 订阅同一 Topic 需要进行负载均衡时，必须填写一致的客户端 ID 才能达到均衡效果。 在 **消费者组 ID** 中填写消费者组标识，填写后会生成带有 `taosx` 前缀的消费者组 ID （例如，如果填写的标识为 `foo`，则生成的消费者组 ID 为 `taosxfoo`）。如果打开末尾处的开关，则会把当前任务的任务 ID 拼接到 `taosx` 之后，输入的标识之前（生成的消费者组 ID 形如 `taosx100foo`）。 @@ -160,4 +202,4 @@ json 数据支持 JSONObject 或者 JSONArray，使用 json 解析器可以解 ### 9. 创建完成 -点击 **提交** 按钮，完成创建 Kafka 到 TDengine 的数据同步任务，回到**数据源列表**页面可查看任务执行情况。 \ No newline at end of file +点击 **提交** 按钮，完成创建 Kafka 到 TDengine 的数据同步任务，回到**数据源列表**页面可查看任务执行情况。 diff --git a/docs/zh/06-advanced/05-data-in/csv-03.png b/docs/zh/06-advanced/05-data-in/csv-03.png index 4165469db5..1e0bd97a51 100644 Binary files a/docs/zh/06-advanced/05-data-in/csv-03.png and b/docs/zh/06-advanced/05-data-in/csv-03.png differ diff --git a/docs/zh/06-advanced/05-data-in/kafka-04-sasl-gssapi.png b/docs/zh/06-advanced/05-data-in/kafka-04-sasl-gssapi.png new file mode 100644 index 0000000000..051efda8ab Binary files /dev/null and b/docs/zh/06-advanced/05-data-in/kafka-04-sasl-gssapi.png differ diff --git a/docs/zh/06-advanced/05-data-in/kafka-04-sasl-plain.png b/docs/zh/06-advanced/05-data-in/kafka-04-sasl-plain.png new file mode 100644 index 0000000000..3c11dcf7e1 Binary files /dev/null and b/docs/zh/06-advanced/05-data-in/kafka-04-sasl-plain.png differ diff --git a/docs/zh/06-advanced/05-data-in/migrate-step2.png b/docs/zh/06-advanced/05-data-in/migrate-step2.png index 54412d0536..03cfa205a2 100644 Binary files a/docs/zh/06-advanced/05-data-in/migrate-step2.png and b/docs/zh/06-advanced/05-data-in/migrate-step2.png differ diff --git a/docs/zh/06-advanced/05-data-in/migrate-step3.png b/docs/zh/06-advanced/05-data-in/migrate-step3.png index 17b3024ca7..8ebfae8d1e 100644 Binary files a/docs/zh/06-advanced/05-data-in/migrate-step3.png and b/docs/zh/06-advanced/05-data-in/migrate-step3.png differ diff --git a/docs/zh/06-advanced/05-data-in/migrate-step4.png b/docs/zh/06-advanced/05-data-in/migrate-step4.png index e6b88ed080..273e588def 100644 Binary files a/docs/zh/06-advanced/05-data-in/migrate-step4.png and b/docs/zh/06-advanced/05-data-in/migrate-step4.png differ diff --git a/docs/zh/06-advanced/05-data-in/mqtt-02.png b/docs/zh/06-advanced/05-data-in/mqtt-02.png index 5c25b27c67..d2d813144d 100644 Binary files a/docs/zh/06-advanced/05-data-in/mqtt-02.png and b/docs/zh/06-advanced/05-data-in/mqtt-02.png differ diff --git a/docs/zh/06-advanced/05-data-in/mqtt-03.png b/docs/zh/06-advanced/05-data-in/mqtt-03.png index c53da8f14d..292fed0d1a 100644 Binary files a/docs/zh/06-advanced/05-data-in/mqtt-03.png and b/docs/zh/06-advanced/05-data-in/mqtt-03.png differ diff --git a/docs/zh/06-advanced/05-data-in/mqtt-04.png b/docs/zh/06-advanced/05-data-in/mqtt-04.png index 430c5b8fef..e099875679 100644 Binary files a/docs/zh/06-advanced/05-data-in/mqtt-04.png and b/docs/zh/06-advanced/05-data-in/mqtt-04.png differ diff --git a/docs/zh/06-advanced/05-data-in/mqtt-05.png b/docs/zh/06-advanced/05-data-in/mqtt-05.png index d362e8be86..2b4cfcabb3 100644 Binary files a/docs/zh/06-advanced/05-data-in/mqtt-05.png and b/docs/zh/06-advanced/05-data-in/mqtt-05.png differ diff --git a/docs/zh/06-advanced/05-data-in/mqtt-14.png b/docs/zh/06-advanced/05-data-in/mqtt-14.png index b1fe456354..0388d8a705 100644 Binary files a/docs/zh/06-advanced/05-data-in/mqtt-14.png and b/docs/zh/06-advanced/05-data-in/mqtt-14.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/InfluxDB-02zh-SelectTheTypeAsInfluxDB.png b/docs/zh/06-advanced/05-data-in/pic/InfluxDB-02zh-SelectTheTypeAsInfluxDB.png index 71393ec24e..d9b806926e 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/InfluxDB-02zh-SelectTheTypeAsInfluxDB.png and b/docs/zh/06-advanced/05-data-in/pic/InfluxDB-02zh-SelectTheTypeAsInfluxDB.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/InfluxDB-03zh-FillInTheConnectionInformation.png b/docs/zh/06-advanced/05-data-in/pic/InfluxDB-03zh-FillInTheConnectionInformation.png index 70c1ffe89e..9fbc6dccaa 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/InfluxDB-03zh-FillInTheConnectionInformation.png and b/docs/zh/06-advanced/05-data-in/pic/InfluxDB-03zh-FillInTheConnectionInformation.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/InfluxDB-04zh-SelectVersion1.x.png b/docs/zh/06-advanced/05-data-in/pic/InfluxDB-04zh-SelectVersion1.x.png index d3542a38a1..836b566ec4 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/InfluxDB-04zh-SelectVersion1.x.png and b/docs/zh/06-advanced/05-data-in/pic/InfluxDB-04zh-SelectVersion1.x.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/InfluxDB-05zh-SelectVersion2.x.png b/docs/zh/06-advanced/05-data-in/pic/InfluxDB-05zh-SelectVersion2.x.png index 161705aa44..1ebf13601c 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/InfluxDB-05zh-SelectVersion2.x.png and b/docs/zh/06-advanced/05-data-in/pic/InfluxDB-05zh-SelectVersion2.x.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/InfluxDB-08zh-GetSchemaAndSelectOneBucket.png b/docs/zh/06-advanced/05-data-in/pic/InfluxDB-08zh-GetSchemaAndSelectOneBucket.png index 21714b6b51..55bee503a0 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/InfluxDB-08zh-GetSchemaAndSelectOneBucket.png and b/docs/zh/06-advanced/05-data-in/pic/InfluxDB-08zh-GetSchemaAndSelectOneBucket.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/InfluxDB-10zh-AdvancedOptionsExpand.png b/docs/zh/06-advanced/05-data-in/pic/InfluxDB-10zh-AdvancedOptionsExpand.png index 02c3f73629..dbb188852c 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/InfluxDB-10zh-AdvancedOptionsExpand.png and b/docs/zh/06-advanced/05-data-in/pic/InfluxDB-10zh-AdvancedOptionsExpand.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-02zh-SelectTheTypeAsOpenTSDB.png b/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-02zh-SelectTheTypeAsOpenTSDB.png index 7b039fbe77..9d2569880b 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-02zh-SelectTheTypeAsOpenTSDB.png and b/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-02zh-SelectTheTypeAsOpenTSDB.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-03zh-FillInTheConnectionInformation.png b/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-03zh-FillInTheConnectionInformation.png index 25f5aaca83..8b3bbd1ffa 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-03zh-FillInTheConnectionInformation.png and b/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-03zh-FillInTheConnectionInformation.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-06zh-GetAndSelectMetrics.png b/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-06zh-GetAndSelectMetrics.png index 447ba589cc..18101c256b 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-06zh-GetAndSelectMetrics.png and b/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-06zh-GetAndSelectMetrics.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-08zh-AdvancedOptionsExpand.png b/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-08zh-AdvancedOptionsExpand.png index 1c57bc19af..ea5dc538e5 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-08zh-AdvancedOptionsExpand.png and b/docs/zh/06-advanced/05-data-in/pic/OpenTSDB-08zh-AdvancedOptionsExpand.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-02.png b/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-02.png index edbaff6595..8fa2419c56 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-02.png and b/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-02.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-03.png b/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-03.png index 3183a02c26..3645668e3b 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-03.png and b/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-03.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-04.png b/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-04.png index 15a126fe46..776e37b153 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-04.png and b/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-04.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-05.png b/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-05.png index 12380aa4b6..1abe21436e 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-05.png and b/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-05.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-06.png b/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-06.png index 9a1533fe4d..74ffb467d2 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-06.png and b/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-06.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-08.png b/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-08.png index d941b9b700..234f2805bc 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-08.png and b/docs/zh/06-advanced/05-data-in/pic/avevaHistorian-08.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/opc-08-add-point.png b/docs/zh/06-advanced/05-data-in/pic/opc-08-add-point.png new file mode 100644 index 0000000000..832b815d53 Binary files /dev/null and b/docs/zh/06-advanced/05-data-in/pic/opc-08-add-point.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/opc-09-add-point.png b/docs/zh/06-advanced/05-data-in/pic/opc-09-add-point.png new file mode 100644 index 0000000000..dd901e4352 Binary files /dev/null and b/docs/zh/06-advanced/05-data-in/pic/opc-09-add-point.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/opcda-01-basic.png b/docs/zh/06-advanced/05-data-in/pic/opcda-01-basic.png index 4c8808fd10..bcb7566cee 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/opcda-01-basic.png and b/docs/zh/06-advanced/05-data-in/pic/opcda-01-basic.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/opcda-02-endpoint.png b/docs/zh/06-advanced/05-data-in/pic/opcda-02-endpoint.png index d58e59f38a..f732fe633f 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/opcda-02-endpoint.png and b/docs/zh/06-advanced/05-data-in/pic/opcda-02-endpoint.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/opcda-03-connect.png b/docs/zh/06-advanced/05-data-in/pic/opcda-03-connect.png deleted file mode 100644 index 323e3adc6e..0000000000 Binary files a/docs/zh/06-advanced/05-data-in/pic/opcda-03-connect.png and /dev/null differ diff --git a/docs/zh/06-advanced/05-data-in/pic/opcda-05-csv.png b/docs/zh/06-advanced/05-data-in/pic/opcda-05-csv.png deleted file mode 100644 index 8286355c0d..0000000000 Binary files a/docs/zh/06-advanced/05-data-in/pic/opcda-05-csv.png and /dev/null differ diff --git a/docs/zh/06-advanced/05-data-in/pic/opcua-01-basic.png b/docs/zh/06-advanced/05-data-in/pic/opcua-01-basic.png index dc2c994b2e..38bde91ce4 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/opcua-01-basic.png and b/docs/zh/06-advanced/05-data-in/pic/opcua-01-basic.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/opcua-02-endpoint.png b/docs/zh/06-advanced/05-data-in/pic/opcua-02-endpoint.png index 0901431eea..53c67c47d2 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/opcua-02-endpoint.png and b/docs/zh/06-advanced/05-data-in/pic/opcua-02-endpoint.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/opcua-03-connect.png b/docs/zh/06-advanced/05-data-in/pic/opcua-03-connect.png deleted file mode 100644 index 5654c99f9a..0000000000 Binary files a/docs/zh/06-advanced/05-data-in/pic/opcua-03-connect.png and /dev/null differ diff --git a/docs/zh/06-advanced/05-data-in/pic/opcua-04-auth.png b/docs/zh/06-advanced/05-data-in/pic/opcua-04-auth.png index 51911fe88a..0fda46face 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/opcua-04-auth.png and b/docs/zh/06-advanced/05-data-in/pic/opcua-04-auth.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/opcua-05-csv.png b/docs/zh/06-advanced/05-data-in/pic/opcua-05-csv.png deleted file mode 100644 index 40c1149c68..0000000000 Binary files a/docs/zh/06-advanced/05-data-in/pic/opcua-05-csv.png and /dev/null differ diff --git a/docs/zh/06-advanced/05-data-in/pic/opcua-07-advance.png b/docs/zh/06-advanced/05-data-in/pic/opcua-07-advance.png index 6a96573fae..22cb89a6da 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/opcua-07-advance.png and b/docs/zh/06-advanced/05-data-in/pic/opcua-07-advance.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/pi-01-agent.png b/docs/zh/06-advanced/05-data-in/pic/pi-01-agent.png index 4ea3274ea5..e5a3b19880 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/pi-01-agent.png and b/docs/zh/06-advanced/05-data-in/pic/pi-01-agent.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/pi-01-new.png b/docs/zh/06-advanced/05-data-in/pic/pi-01-new.png new file mode 100644 index 0000000000..386b2462ee Binary files /dev/null and b/docs/zh/06-advanced/05-data-in/pic/pi-01-new.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/pi-02-connect-archive-only.png b/docs/zh/06-advanced/05-data-in/pic/pi-02-connect-archive-only.png index dc530cb9ba..2f210b0c9d 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/pi-02-connect-archive-only.png and b/docs/zh/06-advanced/05-data-in/pic/pi-02-connect-archive-only.png differ diff --git a/docs/zh/06-advanced/05-data-in/pic/pi-03-connect-af.png b/docs/zh/06-advanced/05-data-in/pic/pi-03-connect-af.png index e38fffc4b6..8b62c36574 100644 Binary files a/docs/zh/06-advanced/05-data-in/pic/pi-03-connect-af.png and b/docs/zh/06-advanced/05-data-in/pic/pi-03-connect-af.png differ diff --git a/docs/zh/06-advanced/05-data-in/tmq-step2.png b/docs/zh/06-advanced/05-data-in/tmq-step2.png index 01dded1372..d0e158f883 100644 Binary files a/docs/zh/06-advanced/05-data-in/tmq-step2.png and b/docs/zh/06-advanced/05-data-in/tmq-step2.png differ diff --git a/docs/zh/06-advanced/05-data-in/tmq-step3.png b/docs/zh/06-advanced/05-data-in/tmq-step3.png index 67461efe17..6d7abeaa95 100644 Binary files a/docs/zh/06-advanced/05-data-in/tmq-step3.png and b/docs/zh/06-advanced/05-data-in/tmq-step3.png differ diff --git a/docs/zh/06-advanced/05-data-in/tmq-step4.png b/docs/zh/06-advanced/05-data-in/tmq-step4.png index 0ec896de3e..f55856f28f 100644 Binary files a/docs/zh/06-advanced/05-data-in/tmq-step4.png and b/docs/zh/06-advanced/05-data-in/tmq-step4.png differ diff --git a/docs/zh/06-advanced/05-data-in/tmq-step5.png b/docs/zh/06-advanced/05-data-in/tmq-step5.png index 568658bdaf..336f888d02 100644 Binary files a/docs/zh/06-advanced/05-data-in/tmq-step5.png and b/docs/zh/06-advanced/05-data-in/tmq-step5.png differ diff --git a/docs/zh/07-develop/01-connect/index.md b/docs/zh/07-develop/01-connect/index.md index d15f481b05..1dfb95d169 100644 --- a/docs/zh/07-develop/01-connect/index.md +++ b/docs/zh/07-develop/01-connect/index.md @@ -387,7 +387,19 @@ DSN 的详细说明和如何使用详见 [连接功能](../../reference/connecto - `reconnectIntervalMs`：重连间隔毫秒时间，默认为 2000。 -C/C++ 语言连接器使用 `taos_connect()` 函数用于建立与 TDengine 数据库的连接。其参数详细说明如下： +**Websocket 连接** +C/C++ 语言连接器 Websocket 连接方式使用 `ws_connect()` 函数用于建立与 TDengine 数据库的连接。其参数为 DSN 描述字符串，其基本结构如下： + +```text +[+]://[[:@]:][/][?=[&=]] +|------|------------|---|-----------|-----------|------|------|------------|-----------------------| +|driver| protocol | | username | password | host | port | database | params | +``` + +DSN 的详细说明和如何使用详见 [连接功能](../../reference/connector/cpp/#dsn) + +**原生连接** +C/C++ 语言连接器原生连接方式使用 `taos_connect()` 函数用于建立与 TDengine 数据库的连接。其参数详细说明如下： - `host`：要连接的数据库服务器的主机名或IP地址。如果是本地数据库，可以使用 `"localhost"`。 - `user`：用于登录数据库的用户名。 @@ -440,7 +452,10 @@ C/C++ 语言连接器使用 `taos_connect()` 函数用于建立与 TDengine 数 ``` -不支持 +```c +{{#include docs/examples/c-ws/connect_example.c}} +``` + 不支持 diff --git a/docs/zh/07-develop/02-sql.md b/docs/zh/07-develop/02-sql.md index be44458c5b..5461c975dd 100644 --- a/docs/zh/07-develop/02-sql.md +++ b/docs/zh/07-develop/02-sql.md @@ -68,9 +68,15 @@ REST API：直接调用 `taosadapter` 提供的 REST API 接口，进行数据 ``` -```c + +```c title="Websocket 连接" +{{#include docs/examples/c-ws/create_db_demo.c:create_db_and_table}} +``` + +```c title="原生连接" {{#include docs/examples/c/create_db_demo.c:create_db_and_table}} ``` + @@ -144,7 +150,12 @@ NOW 为系统内部函数，默认为客户端所在计算机当前时间。 NOW ``` -```c + +```c title="Websocket 连接" +{{#include docs/examples/c-ws/insert_data_demo.c:insert_data}} +``` + +```c title="原生连接" {{#include docs/examples/c/insert_data_demo.c:insert_data}} ``` @@ -218,7 +229,12 @@ rust 连接器还支持使用 **serde** 进行反序列化行为结构体的结 ``` -```c + +```c title="Websocket 连接" +{{#include docs/examples/c-ws/query_data_demo.c:query_data}} +``` + +```c title="原生连接" {{#include docs/examples/c/query_data_demo.c:query_data}} ``` @@ -293,9 +309,15 @@ reqId 可用于请求链路追踪，reqId 就像分布式系统中的 traceId ``` -```c + +```c "Websocket 连接" +{{#include docs/examples/c-ws/with_reqid_demo.c:with_reqid}} +``` + +```c "原生连接" {{#include docs/examples/c/with_reqid_demo.c:with_reqid}} ``` + diff --git a/docs/zh/07-develop/04-schemaless.md b/docs/zh/07-develop/04-schemaless.md index 17a377950c..a865b58b28 100644 --- a/docs/zh/07-develop/04-schemaless.md +++ b/docs/zh/07-develop/04-schemaless.md @@ -237,7 +237,10 @@ writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO ``` -不支持 + +```c +{{#include docs/examples/c-ws/sml_insert_demo.c:schemaless}} +``` 不支持 diff --git a/docs/zh/07-develop/05-stmt.md b/docs/zh/07-develop/05-stmt.md index e659177c94..624600ba4d 100644 --- a/docs/zh/07-develop/05-stmt.md +++ b/docs/zh/07-develop/05-stmt.md @@ -64,7 +64,9 @@ import TabItem from "@theme/TabItem"; ``` -不支持 +```c +{{#include docs/examples/c-ws/stmt_insert_demo.c}} +``` 不支持 diff --git a/docs/zh/07-develop/07-tmq.md b/docs/zh/07-develop/07-tmq.md index 96f34c6d5d..c668203259 100644 --- a/docs/zh/07-develop/07-tmq.md +++ b/docs/zh/07-develop/07-tmq.md @@ -28,21 +28,21 @@ TDengine 消费者的概念跟 Kafka 类似，消费者通过订阅主题来接 ### 创建参数 创建消费者的参数较多，非常灵活的支持了各种连接类型、 Offset 提交方式、压缩、重连、反序列化等特性。各语言连接器都适用的通用基础配置项如下表所示： -| 参数名称 | 类型 | 参数说明 | 备注 | -| :-----------------------: | :-----: | ----------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `td.connect.ip` | string | 服务端的 IP 地址 | | -| `td.connect.user` | string | 用户名 | | -| `td.connect.pass` | string | 密码 | | -| `td.connect.port` | integer | 服务端的端口号 | | -| `group.id` | string | 消费组 ID，同一消费组共享消费进度 |
**必填项**。最大长度：192。
每个topic最多可建立 100 个 consumer group | -| `client.id` | string | 客户端 ID | 最大长度：192 | -| `auto.offset.reset` | enum | 消费组订阅的初始位置 |
`earliest`: default(version < 3.2.0.0);从头开始订阅;
`latest`: default(version >= 3.2.0.0);仅从最新数据开始订阅;
`none`: 没有提交的 offset 无法订阅 | -| `enable.auto.commit` | boolean | 是否启用消费位点自动提交，true: 自动提交，客户端应用无需commit；false：客户端应用需要自行commit | 默认值为 true | -| `auto.commit.interval.ms` | integer | 消费记录自动提交消费位点时间间隔，单位为毫秒 | 默认值为 5000 | -| `msg.with.table.name` | boolean | 是否允许从消息中解析表名, 不适用于列订阅（列订阅时可将 tbname 作为列写入 subquery 语句）（从3.2.0.0版本该参数废弃，恒为true） | 默认关闭 | -| `enable.replay` | boolean | 是否开启数据回放功能 | 默认关闭 | -| `session.timeout.ms` | integer | consumer 心跳丢失后超时时间，超时后会触发 rebalance 逻辑，成功后该 consumer 会被删除（从3.3.3.0版本开始支持） | 默认值为 12000，取值范围 [6000， 1800000] | -| `max.poll.interval.ms` | integer | consumer poll 拉取数据间隔的最长时间，超过该时间，会认为该 consumer 离线，触发rebalance 逻辑，成功后该 consumer 会被删除（从3.3.3.0版本开始支持） | 默认值为 300000，[1000，INT32_MAX] | +| 参数名称 | 类型 | 参数说明 | 备注 | +| :-----------------------: | :-----: | ------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `td.connect.ip` | string | 服务端的 IP 地址 | | +| `td.connect.user` | string | 用户名 | | +| `td.connect.pass` | string | 密码 | | +| `td.connect.port` | integer | 服务端的端口号 | | +| `group.id` | string | 消费组 ID，同一消费组共享消费进度 |
**必填项**。最大长度：192。
每个topic最多可建立 100 个 consumer group | +| `client.id` | string | 客户端 ID | 最大长度：192 | +| `auto.offset.reset` | enum | 消费组订阅的初始位置 |
`earliest`: default(version < 3.2.0.0);从头开始订阅;
`latest`: default(version >= 3.2.0.0);仅从最新数据开始订阅;
`none`: 没有提交的 offset 无法订阅 | +| `enable.auto.commit` | boolean | 是否启用消费位点自动提交，true: 自动提交，客户端应用无需commit；false：客户端应用需要自行commit | 默认值为 true | +| `auto.commit.interval.ms` | integer | 消费记录自动提交消费位点时间间隔，单位为毫秒 | 默认值为 5000 | +| `msg.with.table.name` | boolean | 是否允许从消息中解析表名, 不适用于列订阅（列订阅时可将 tbname 作为列写入 subquery 语句）（从3.2.0.0版本该参数废弃，恒为true） | 默认关闭 | +| `enable.replay` | boolean | 是否开启数据回放功能 | 默认关闭 | +| `session.timeout.ms` | integer | consumer 心跳丢失后超时时间，超时后会触发 rebalance 逻辑，成功后该 consumer 会被删除（从3.3.3.0版本开始支持） | 默认值为 12000，取值范围 [6000， 1800000] | +| `max.poll.interval.ms` | integer | consumer poll 拉取数据间隔的最长时间，超过该时间，会认为该 consumer 离线，触发rebalance 逻辑，成功后该 consumer 会被删除（从3.3.3.0版本开始支持） | 默认值为 300000，[1000，INT32_MAX] | 下面是各语言连接器创建参数： @@ -94,8 +94,8 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 - -同通用基础配置项。 +- Websocket 连接: 因为使用 dsn，不需要 `td.connect.ip`，`td.connect.port`，`td.connect.user` 和 `td.connect.pass` 四个配置项，其余同通用配置项。 +- 原生连接: 同通用基础配置项。 @@ -154,7 +154,15 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 ``` -不支持 +```c +{{#include docs/examples/c-ws/tmq_demo.c:create_consumer_1}} +``` + +```c +{{#include docs/examples/c-ws/tmq_demo.c:create_consumer_2}} +``` + +调用 `build_consumer` 函数尝试获取消费者实例 `tmq`。成功则打印成功日志，失败则打印失败日志。 不支持 @@ -283,7 +291,29 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 ``` -不支持 +```c +{{#include docs/examples/c-ws/tmq_demo.c:build_topic_list}} +``` + +```c +{{#include docs/examples/c-ws/tmq_demo.c:basic_consume_loop}} +``` + +```c +{{#include docs/examples/c-ws/tmq_demo.c:msg_process}} +``` + +```c +{{#include docs/examples/c-ws/tmq_demo.c:subscribe_3}} +``` + +订阅消费数据步骤： + 1. 调用 `ws_build_topic_list` 函数创建一个主题列表 `topic_list`。 + 2. 如果 `topic_list` 为 `NULL`，表示创建失败，函数返回 `-1`。 + 3. 使用 `ws_tmq_subscribe` 函数订阅 `tmq` 指定的主题列表。如果订阅失败，打印错误信息。 + 4. 销毁主题列表 `topic_list` 以释放资源。 + 5. 调用 `basic_consume_loop` 函数开始基本的消费循环，处理订阅的消息。 + 不支持 @@ -427,7 +457,17 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 ``` -不支持 +```c +{{#include docs/examples/c-ws/tmq_demo.c:consume_repeatly}} +``` + +1. 通过 `ws_tmq_get_topic_assignment` 函数获取特定主题的分配信息，包括分配的数量和具体分配详情。 +2. 如果获取分配信息失败，则打印错误信息并返回。 +3. 对于每个分配，使用 `ws_tmq_offset_seek` 函数将消费者的偏移量设置到最早的偏移量。 +4. 如果设置偏移量失败，则打印错误信息。 +5. 释放分配信息数组以释放资源。 +6. 调用 `basic_consume_loop` 函数开始新的消费循环，处理消息。 + 不支持 @@ -554,7 +594,12 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 ``` -不支持 +```c +{{#include docs/examples/c-ws/tmq_demo.c:manual_commit}} +``` + +可以通过 `ws_tmq_commit_sync` 函数来手工提交消费进度。 + 不支持 @@ -662,7 +707,9 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 ``` -不支持 +```c +{{#include docs/examples/c-ws/tmq_demo.c:unsubscribe_and_close}} +``` 不支持 @@ -777,7 +824,13 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 -不支持 +
+完整代码示例 +```c +{{#include docs/examples/c-ws/tmq_demo.c}} +``` +
+ 不支持 diff --git a/docs/zh/07-develop/index.md b/docs/zh/07-develop/index.md index 9d4adce01c..4759027344 100644 --- a/docs/zh/07-develop/index.md +++ b/docs/zh/07-develop/index.md @@ -14,7 +14,8 @@ description: 让开发者能够快速上手的指南 7. 在很多场景下（如车辆管理），应用需要获取每个数据采集点的最新状态，那么建议你采用 TDengine 的 Cache 功能，而不用单独部署 Redis 等缓存软件。 8. 如果你发现 TDengine 的函数无法满足你的要求，那么你可以使用用户自定义函数（UDF）来解决问题。 -本部分内容就是按照上述顺序组织的。为便于理解，TDengine 为每个功能和每个支持的编程语言都提供了示例代码。如果你希望深入了解 SQL 的使用，需要查看[SQL 手册](../reference/taos-sql/)。如果想更深入地了解各连接器的使用，请阅读[连接器参考指南](../reference/connector/)。如果还希望想将 TDengine 与第三方系统集成起来，比如 Grafana, 请参考[第三方工具](../third-party/)。 +本部分内容就是按照上述顺序组织的。为便于理解，TDengine 为每个功能和每个支持的编程语言都提供了示例代码，位于 [示例代码](https://github.com/taosdata/TDengine/tree/main/docs/examples)。所有示例代码都会有 CI 保证正确性，脚本位于 [示例代码 CI](https://github.com/taosdata/TDengine/tree/main/tests/docs-examples-test)。 +如果你希望深入了解 SQL 的使用，需要查看[SQL 手册](../reference/taos-sql/)。如果想更深入地了解各连接器的使用，请阅读[连接器参考指南](../reference/connector/)。如果还希望想将 TDengine 与第三方系统集成起来，比如 Grafana, 请参考[第三方工具](../third-party/)。 如果在开发过程中遇到任何问题，请点击每个页面下方的["反馈问题"](https://github.com/taosdata/TDengine/issues/new/choose), 在 GitHub 上直接递交 Issue。 diff --git a/docs/zh/14-reference/01-components/04-taosx.md b/docs/zh/14-reference/01-components/04-taosx.md index e378c18800..114a6b1ce5 100644 --- a/docs/zh/14-reference/01-components/04-taosx.md +++ b/docs/zh/14-reference/01-components/04-taosx.md @@ -239,40 +239,45 @@ d4,2017-07-14T10:40:00.006+08:00,-2.740636,10,-0.893545,7,California.LosAngles - `plugins_home`：外部数据源连接器所在目录。 - `data_dir`：数据文件存放目录。 -- `logs_home`：日志文件存放目录，`taosX` 日志文件的前缀为 `taosx.log`，外部数据源有自己的日志文件名前缀。 -- `log_level`：日志等级，可选级别包括 `error`、`warn`、`info`、`debug`、`trace`，默认值为 `info`。 -- `log_keep_days`：日志的最大存储天数，`taosX` 日志将按天划分为不同的文件。 +- `instanceId`：当前 taosX 服务的实例 ID，如果同一台机器上启动了多个 taosX 实例，必须保证各个实例的实例 ID 互不相同。 +- `logs_home`：日志文件存放目录，`taosX` 日志文件的前缀为 `taosx.log`，外部数据源有自己的日志文件名前缀。已弃用，请使用 `log.path` 代替。 +- `log_level`：日志等级，可选级别包括 `error`、`warn`、`info`、`debug`、`trace`，默认值为 `info`。已弃用，请使用 `log.level` 代替。 +- `log_keep_days`：日志的最大存储天数，`taosX` 日志将按天划分为不同的文件。已弃用，请使用 `log.keepDays` 代替。 - `jobs`：每个运行时的最大线程数。在服务模式下，线程总数为 `jobs*2`，默认线程数为`当前服务器内核*2`。 - `serve.listen`：是 `taosX` REST API 监听地址，默认值为 `0.0.0.0:6050`。 - `serve.database_url`：`taosX` 数据库的地址，格式为 `sqlite:`。 +- `serve.request_timeout`：全局接口 API 超时时间。 - `monitor.fqdn`：`taosKeeper` 服务的 FQDN，没有默认值，置空则关闭监控功能。 - `monitor.port`：`taosKeeper` 服务的端口，默认`6043`。 - `monitor.interval`：向 `taosKeeper` 发送指标的频率，默认为每 10 秒一次，只有 1 到 10 之间的值才有效。 +- `log.path`：日志文件存放的目录。 +- `log.level`：日志级别，可选值为 "error", "warn", "info", "debug", "trace"。 +- `log.compress`：日志文件滚动后的文件是否进行压缩。 +- `log.rotationCount`：日志文件目录下最多保留的文件数，超出数量的旧文件被删除。 +- `log.rotationSize`：触发日志文件滚动的文件大小（单位为字节），当日志文件超出此大小后会生成一个新文件，新的日志会写入新文件。 +- `log.reservedDiskSize`：日志所在磁盘停止写入日志的阈值（单位为字节），当磁盘剩余空间达到此大小后停止写入日志。 +- `log.keepDays`：日志文件保存的天数，超过此天数的旧日志文件会被删除。 +- `log.watching`：是否对日志文件中 `log.loggers` 配置内容的变更进行监听并尝试重载。 +- `log.loggers`：指定模块的日志输出级别，格式为 `"modname" = "level"`，同时适配 tracing 库语法，可以根据 `modname[span{field=value}]=level`，其中 `level` 为日志级别。 如下所示： ```toml -# plugins home -#plugins_home = "/usr/local/taos/plugins" # on linux/macOS -#plugins_home = "C:\\TDengine\\plugins" # on windows - # data dir #data_dir = "/var/lib/taos/taosx" # on linux/macOS #data_dir = "C:\\TDengine\\data\\taosx" # on windows -# logs home -#logs_home = "/var/log/taos" # on linux/macOS -#logs_home = "C:\\TDengine\\log" # on windows - -# log level: off/error/warn/info/debug/trace -#log_level = "info" - -# log keep days -#log_keep_days = 30 - -# number of jobs, default to 0, will use `jobs` number of works for TMQ +# number of threads used for tokio workers, default to 0 (means cores * 2) #jobs = 0 +# enable OpenTelemetry tracing and metrics exporter +#otel = false + +# server instance id +# +# The instanceId of each instance is unique on the host +# instanceId = 16 + [serve] # listen to ip:port address #listen = "0.0.0.0:6050" @@ -280,13 +285,66 @@ d4,2017-07-14T10:40:00.006+08:00,-2.740636,10,-0.893545,7,California.LosAngles # database url #database_url = "sqlite:taosx.db" +# default global request timeout which unit is second. This parameter takes effect for certain interfaces that require a timeout setting +#request_timeout = 30 + [monitor] # FQDN of taosKeeper service, no default value #fqdn = "localhost" -# port of taosKeeper service, default 6043 + +# Port of taosKeeper service, default 6043 #port = 6043 -# how often to send metrics to taosKeeper, default every 10 seconds. Only value from 1 to 10 is valid. + +# How often to send metrics to taosKeeper, default every 10 seconds. Only value from 1 to 10 is valid. #interval = 10 + + +# log configuration +[log] +# All log files are stored in this directory +# +#path = "/var/log/taos" # on linux/macOS +#path = "C:\\TDengine\\log" # on windows + +# log filter level +# +#level = "info" + +# Compress archived log files or not +# +#compress = false + +# The number of log files retained by the current explorer server instance in the `path` directory +# +#rotationCount = 30 + +# Rotate when the log file reaches this size +# +#rotationSize = "1GB" + +# Log downgrade when the remaining disk space reaches this size, only logging `ERROR` level logs +# +#reservedDiskSize = "1GB" + +# The number of days log files are retained +# +#keepDays = 30 + +# Watching the configuration file for log.loggers changes, default to true. +# +#watching = true + +# Customize the log output level of modules, and changes will be applied after modifying the file when log.watching is enabled +# +# ## Examples: +# +# crate = "error" +# crate::mod1::mod2 = "info" +# crate::span[field=value] = "warn" +# +[log.loggers] +#"actix_server::accept" = "warn" +#"taos::query" = "warn" ``` ### 启动 @@ -421,7 +479,6 @@ taosX 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeep | write_raw_fails | 本次运行写入 raw meta 失败的次数 | | success_blocks | 本次写入成功的数据块数 | - ### taosX 其他数据源 任务 这些数据源包括： InfluxDB，OpenTSDB，OPC UA，OPC DA，PI，CSV，MQTT，AVEVA Historian 和 Kafka。 @@ -452,3 +509,103 @@ taosX 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeep | written_blocks | 本次运行此任务写人成功的 raw block 数 | | failed_blocks | 本次运行此任务写入失败的 raw block 数 | +### Kafka 数据源相关指标 + +| 字段 | 描述 | +| ----------------------------- | ---------------------------- | +| kafka_consumers | 本次运行任务 Kafka 消费者数 | +| kafka_total_partitions | Kafka 主题总分区数 | +| kafka_consuming_partitions | 本次运行任务正在消费的分区数 | +| kafka_consumed_messages | 本次运行任务已经消费的消息数 | +| total_kafka_consumed_messages | 累计消费的消息总数 | + +## taosX 数据解析插件 + +接入 kafka / mqtt 消息中间件时，需要对原始数据进行解析，如果使用 json/regex 等模式解析器无法满足解析需求，同时 UDT(自定义解析脚本) 也无法满足性能要求时，可以自定义数据解析插件。 + +### 插件概述 + +taosX Parser 插件是一个要求用 C/Rust 语言开发的 C ABI 兼容动态库，该动态库要实现约定的 API 并编译为在 taosX 所在运行环境中能够正确运行的动态库，然后复制到约定位置由 taosX 在运行时加载，并在处理数据的 Parsing 阶段调用。 + +### 插件部署 + +完成插件开发后，编译环境需要和目标运行环境兼容，将编译好的插件动态库复制到插件目录下，taosX 启动后，系统首次使用插件时初始化加载插件。可以在 explorer 的 kafka 或者 mqtt 数据接入配置页面中，检查是否加载成功。如下图，如果加载成功，则在解析器选择列表中展示出来。 + +![插件示例](./plugin-01.png) + +插件目录在 `taosx.toml` 配置文件中复用 plugins 配置，追加`/parsers`作为插件安装路径，默认值在 UNIX 环境下为 `/usr/local/taos/plugins/parsers`，在 Windows 下为 `C:\TDengine\plugins\parsers`。 + +### 插件 api 说明 + +#### 1. 获取插件名称 + +获取插件名，用于前端显示。 + +**函数签名**：const char* parser_name() + +**返回值**：字符串。 + +#### 2. 获取插件版本 + +插件版本，方便问题定位。 + +**函数签名**：const char* parser_version() + +**返回值**：字符串。 + +#### 3. 配置解析器 + +将一个字符串参数解析为一个配置对象，仅插件内部使用。 + +**函数签名**：parser_resp_t parser_new(char* ctx, uint32_t len); + +char* ctx: 用户自定义配置字符串。 + +uint32_t len: 该字符串的二进制长度（不含 `\0`）。 + +**返回值**： + +``` c +struct parser_resp_t { + int e; // 0 if success. + void* p; // Success if contains. +} +``` + +当创建对象失败时，e 不为 0。 + +当创建成功时，e = 0，p 为解析器对象。 + +#### 4. 解析数据 + +**函数签名**： + +对输入 payload 进行解析，返回结果为 JSON 格式 [u8] 。返回的 JSON 将使用默认的 JSON 解析器进行完全解码（展开根数组和所有的对象）。 + +``` c +const char* parser_mutate( + void* parser, + const uint8_t* in_ptr, uint32_t in_len, + const void* uint8_t* out_ptr, uint32_t* out_len +); +``` + +`void* parser`: parser_new 生成的对象指针; + +`const uint8_t* in_ptr`：输入 Payload 的指针; + +`uint32_t in_len`: 输入 Payload 的 bytes 长度（不含 `\0`）; + +`const void* uint8_t* out_ptr`:输出 JSON 字符串的指针（不含 \0）。当 out_ptr 指向为空时，表示输出为空。 + +`uint32_t * out_len`：输出 JSON 字符串长度。 + +**返回值**： 当调用成功时，返回值为 NULL。 + +#### 5. 释放解析器 + +释放解析器对象内存。 + +**函数签名**： void parser_free(void* parser); + +void* parser: parser_new 生成的对象指针。 diff --git a/docs/zh/14-reference/01-components/05-taosx-agent.md b/docs/zh/14-reference/01-components/05-taosx-agent.md index da1c395b3d..bf2e6f7e78 100644 --- a/docs/zh/14-reference/01-components/05-taosx-agent.md +++ b/docs/zh/14-reference/01-components/05-taosx-agent.md @@ -11,18 +11,69 @@ sidebar_label: taosX-Agent - `endpoint`: 必填，`taosX` 的 GRPC 服务地址。 - `token`: 必填，在 `Explorer` 上创建 `Agent` 时，产生的 Token。 +- `instanceId`：当前 taosx-agent 服务的实例 ID，如果同一台机器上启动了多个 taosx-agent 实例，必须保证各个实例的实例 ID 互不相同。 - `compression`: 非必填，可配置为 `ture` 或 `false`, 默认为 `false`。配置为`true`, 则开启 `Agent` 和 `taosX` 通信数据压缩。 -- `log_level`: 非必填，日志级别，默认为 `info`, 同 `taosX` 一样，支持 `error`，`warn`，`info`，`debug`，`trace` 五级。 -- `log_keep_days`：非必填，日志保存天数，默认为 `30` 天。 +- `log_level`: 非必填，日志级别，默认为 `info`, 同 `taosX` 一样，支持 `error`，`warn`，`info`，`debug`，`trace` 五级。已弃用，请使用 `log.level` 代替。 +- `log_keep_days`：非必填，日志保存天数，默认为 `30` 天。已弃用，请使用 `log.keepDays` 代替。 +- `log.path`：日志文件存放的目录。 +- `log.level`：日志级别，可选值为 "error", "warn", "info", "debug", "trace"。 +- `log.compress`：日志文件滚动后的文件是否进行压缩。 +- `log.rotationCount`：日志文件目录下最多保留的文件数，超出数量的旧文件被删除。 +- `log.rotationSize`：触发日志文件滚动的文件大小（单位为字节），当日志文件超出此大小后会生成一个新文件，新的日志会写入新文件。 +- `log.reservedDiskSize`：日志所在磁盘停止写入日志的阈值（单位为字节），当磁盘剩余空间达到此大小后停止写入日志。 +- `log.keepDays`：日志文件保存的天数，超过此天数的旧日志文件会被删除。 如下所示： ```TOML -endpoint = "grpc://:6055" -token = "" -compression = true -log_level = "info" -log_keep_days = 30 +# taosX service endpoint +# +#endpoint = "http://localhost:6055" + +# !important! +# Uncomment it and copy-paste the token generated in Explorer. +# +#token = "" + +# server instance id +# +# The instanceId of each instance is unique on the host +# instanceId = 48 + +# enable communication data compression between Agent and taosX +# +#compression = true + +# log configuration +[log] +# All log files are stored in this directory +# +#path = "/var/log/taos" # on linux/macOS +#path = "C:\\TDengine\\log" # on windows + +# log filter level +# +#level = "info" + +# Compress archived log files or not +# +#compress = false + +# The number of log files retained by the current explorer server instance in the `path` directory +# +#rotationCount = 30 + +# Rotate when the log file reaches this size +# +#rotationSize = "1GB" + +# Log downgrade when the remaining disk space reaches this size, only logging `ERROR` level logs +# +#reservedDiskSize = "1GB" + +# The number of days log files are retained +# +#keepDays = 30 ``` 您不必对配置文件如何设置感到疑惑，阅读并跟随 `Explorer` 中创建 `Agent` 的提示进行操作，您可以对配置文件进行查看、修改和检查。 diff --git a/docs/zh/14-reference/01-components/07-explorer.md b/docs/zh/14-reference/01-components/07-explorer.md index 6a8972deea..c63bc703e2 100644 --- a/docs/zh/14-reference/01-components/07-explorer.md +++ b/docs/zh/14-reference/01-components/07-explorer.md @@ -15,36 +15,111 @@ taosEexplorer 无需单独安装，从 TDengine 3.3.0.0 版本开始，它随着 在启动 taosExplorer 之前，请确保配置文件中的内容正确。 ```TOML -# listen port +# This is a automacically generated configuration file for Explorer in [TOML](https://toml.io/) format. +# +# Here is a full list of available options. + +# Explorer server port to listen on. +# Default is 6060. +# port = 6060 -# listen address for IPv4 +# IPv4 listen address. +# Default is 0.0.0.0 addr = "0.0.0.0" -# listen address for IPv4 -#ipv6 = "::1" +# IPv6 listen address. -# log level. Possible: error,warn,info,debug,trace +# ipv6 = "::1" + +# explorer server instance id +# +# The instanceId of each instance is unique on the host +# instanceId = 1 + +# Explorer server log level. +# Default is "info" +# +# Deprecated: use log.level instead log_level = "info" -# taosAdapter address. +# All data files are stored in this directory +# data_dir = "/var/lib/taos/explorer" # Default for Linux +# data_dir = "C:\\TDengine\\data\\explorer" # Default for Windows + +# REST API endpoint to connect to the cluster. +# This configuration is also the target for data migration tasks. +# +# Default is "http://localhost:6041" - the default endpoint for REST API. +# cluster = "http://localhost:6041" -# taosX gRPC address +# native endpoint to connect to the cluster. +# Default is disabled. To enable it, set it to the native API URL like "taos://localhost:6030" and uncomment it. +# If you enable it, you will get more performance for data migration tasks. +# +# cluster_native = "taos://localhost:6030" + +# API endpoint for data replication/backup/data sources. No default option. +# Set it to API URL like "http://localhost:6050". +# x_api = "http://localhost:6050" # GRPC endpoint for "Agent"s. +# Default is "http://localhost:6055" - the default endpoint for taosX grpc API. +# You should set it to public IP or FQDN name like: +# "http://192.168.111.111:6055" or "http://node1.company.domain:6055" and +# ensure to add the port to the exception list of the firewall if it enabled. grpc = "http://localhost:6055" # CORS configuration switch, it allows cross-origin access -cors = false +cors = true -# Enable ssl: if the following two files exist, enable ssl protocol +# Enable ssl +# If the following two files exist, enable ssl protocol +# [ssl] + # SSL certificate -#certificate = "/path/to/ca.file" +# +# certificate = "/path/to/ca.file" # on linux/macOS +# certificate = "C:\\path\\to\\ca.file" # on windows + # SSL certificate key -#certificate_key = "/path/to/key.file" +# +# certificate_key = "/path/to/key.file" # on linux/macOS +# certificate_key = "C:\\path\\to\\key.file" # on windows + +# log configuration +[log] +# All log files are stored in this directory +# +# path = "/var/log/taos" # on linux/macOS +# path = "C:\\TDengine\\log" # on windows + +# log filter level +# +# level = "info" + +# Compress archived log files or not +# +# compress = false + +# The number of log files retained by the current explorer server instance in the `path` directory +# +# rotationCount = 30 + +# Rotate when the log file reaches this size +# +# rotationSize = "1GB" + +# Log downgrade when the remaining disk space reaches this size, only logging `ERROR` level logs +# +# reservedDiskSize = "1GB" + +# The number of days log files are retained +# +# keepDays = 30 ``` 说明： @@ -52,13 +127,23 @@ cors = false - `port`：taosExplorer 服务绑定的端口。 - `addr`：taosExplorer 服务绑定的 IPv4 地址，默认为 `0.0.0.0`。如需修改，请配置为 `localhost` 之外的地址以对外提供服务。 - `ipv6`：taosExplorer 服务绑定的 IPv6 地址，默认不绑定 IPv6 地址。 -- `log_level`：日志级别，可选值为 "error", "warn", "info", "debug", "trace"。 +- `instanceId`：当前 explorer 服务的实例 ID，如果同一台机器上启动了多个 explorer 实例，必须保证各个实例的实例 ID 互不相同。 +- `log_level`：日志级别，可选值为 "error", "warn", "info", "debug", "trace"。此参数已弃用，请使用 `log.level` 代替。 - `cluster`：TDengine 集群的 taosAdapter 地址。 +- `cluster_native`：TDengine 集群的原生连接地址，默认关闭。 - `x_api`：taosX 的 gRPC 地址。 -- `grpc`: taosX 代理向 taosX 建立连接的 gRPC 地址. +- `grpc`：taosX 代理向 taosX 建立连接的 gRPC 地址。 - `cors`：CORS 配置开关，默认为 `false`。当为 `true` 时，允许跨域访问。 -- `ssl.certificate`: SSL 证书（如果同时设置了 certificate 与 certificate_key 两个参数，则启用 HTTPS 服务，否则不启用）。 -- `ssl.certificate_key`: SSL 证书密钥。 +- `ssl.certificate`：SSL 证书（如果同时设置了 certificate 与 certificate_key 两个参数，则启用 HTTPS 服务，否则不启用）。 +- `ssl.certificate_key`：SSL 证书密钥。 +- `log.path`：日志文件存放的目录。 +- `log.level`：日志级别，可选值为 "error", "warn", "info", "debug", "trace"。 +- `log.compress`：日志文件滚动后的文件是否进行压缩。 +- `log.rotationCount`：日志文件目录下最多保留的文件数，超出数量的旧文件被删除。 +- `log.rotationSize`：触发日志文件滚动的文件大小（单位为字节），当日志文件超出此大小后会生成一个新文件，新的日志会写入新文件。 +- `log.reservedDiskSize`：日志所在磁盘停止写入日志的阈值（单位为字节），当磁盘剩余空间达到此大小后停止写入日志。 +- `log.keepDays`：日志文件保存的天数，超过此天数的旧日志文件会被删除。 + ## 启动停止 diff --git a/docs/zh/14-reference/01-components/plugin-01.png b/docs/zh/14-reference/01-components/plugin-01.png new file mode 100644 index 0000000000..18bc2ca242 Binary files /dev/null and b/docs/zh/14-reference/01-components/plugin-01.png differ diff --git a/docs/zh/14-reference/03-taos-sql/02-database.md b/docs/zh/14-reference/03-taos-sql/02-database.md index d2e9ba0646..7d040a2c44 100644 --- a/docs/zh/14-reference/03-taos-sql/02-database.md +++ b/docs/zh/14-reference/03-taos-sql/02-database.md @@ -80,7 +80,6 @@ database_option: { ```sql create database if not exists db vgroups 10 buffer 10 - ``` 以上示例创建了一个有 10 个 vgroup 名为 db 的数据库， 其中每个 vnode 分配 10MB 的写入缓存 diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index 007e1dd64a..ac69f64f08 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -1214,7 +1214,7 @@ TO_TIMESTAMP(ts_str_literal, format_str_literal) - 如果没有指定完整的时间，那么默认时间值为指定或默认时区的 `1970-01-01 00:00:00`, 未指定部分使用该默认值中的对应部分. 暂不支持只指定年日而不指定月日的格式, 如'yyyy-mm-DDD', 支持'yyyy-mm-DD'. - 如果格式串中有`AM`, `PM`等, 那么小时必须是12小时制, 范围必须是01-12. - `to_timestamp`转换具有一定的容错机制, 在格式串和时间戳串不完全对应时, 有时也可转换, 如: `to_timestamp('200101/2', 'yyyyMM1/dd')`, 格式串中多出来的1会被丢弃. 格式串与时间戳串中多余的空格字符(空格, tab等)也会被 自动忽略. 如`to_timestamp(' 23 年 - 1 月 - 01 日 ', 'yy 年-MM月-dd日')` 可以被成功转换. 虽然`MM`等字段需要两个数字对应(只有一位时前面补0), 在`to_timestamp`时, 一个数字也可以成功转换. -- 输出时间戳的精度与查询表的精度相同, 若查询未指定表, 则输出精度为毫秒. 如`select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns')`的输出将会把微妙和纳秒进行截断. 如果指定一张纳秒表, 那么就不会发生截断, 如`select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns') from db_ns.table_ns limit 1`. +- 输出时间戳的精度与查询表的精度相同, 若查询未指定表, 则输出精度为毫秒. 如`select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns')`的输出将会把微秒和纳秒进行截断. 如果指定一张纳秒表, 那么就不会发生截断, 如`select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns') from db_ns.table_ns limit 1`. ### 时间和日期函数 diff --git a/docs/zh/14-reference/03-taos-sql/16-operators.md b/docs/zh/14-reference/03-taos-sql/16-operators.md index 76af4037c8..96a35e9ebf 100644 --- a/docs/zh/14-reference/03-taos-sql/16-operators.md +++ b/docs/zh/14-reference/03-taos-sql/16-operators.md @@ -41,9 +41,11 @@ TDengine 支持 `UNION ALL` 和 `UNION` 操作符。UNION ALL 将查询返回的 | 5 | IS [NOT] NULL | 所有类型 | 是否为空值 | | 6 | [NOT] BETWEEN AND | 除 BOOL、BLOB、MEDIUMBLOB、JSON 和 GEOMETRY 外的所有类型 | 闭区间比较 | | 7 | IN | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型，且不可以为表的时间戳主键列 | 与列表内的任意值相等 | -| 8 | LIKE | BINARY、NCHAR 和 VARCHAR | 通配符匹配 | -| 9 | MATCH, NMATCH | BINARY、NCHAR 和 VARCHAR | 正则表达式匹配 | -| 10 | CONTAINS | JSON | JSON 中是否存在某键 | +| 8 | NOT IN | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型，且不可以为表的时间戳主键列 | 与列表内的任意值都不相等 | +| 9 | LIKE | BINARY、NCHAR 和 VARCHAR | 通配符匹配所指定的模式串 | +| 10 | NOT LIKE | BINARY、NCHAR 和 VARCHAR | 通配符不匹配所指定的模式串 | +| 11 | MATCH, NMATCH | BINARY、NCHAR 和 VARCHAR | 正则表达式匹配 | +| 12 | CONTAINS | JSON | JSON 中是否存在某键 | LIKE 条件使用通配符字符串进行匹配检查，规则如下： diff --git a/docs/zh/14-reference/03-taos-sql/28-index.md b/docs/zh/14-reference/03-taos-sql/28-index.md index 14c45b6585..ef625de1e7 100644 --- a/docs/zh/14-reference/03-taos-sql/28-index.md +++ b/docs/zh/14-reference/03-taos-sql/28-index.md @@ -28,7 +28,7 @@ TSMA只能基于超级表和普通表创建, 不能基于子表创建. 由于TSMA输出为一张超级表, 因此输出表的行长度受最大行长度限制, 不同函数的`中间结果`大小各异, 一般都大于原始数据大小, 若输出表的行长度大于最大行长度限制, 将会报`Row length exceeds max length`错误. 此时需要减少函数个数或者将常用的函数进行分组拆分到多个TSMA中. -窗口大小的限制为[1m ~ 1y/12n]. INTERVAL 的单位与查询中INTERVAL子句相同, 如 a (毫秒), b (纳秒), h (小时), m (分钟), s (秒), u (微妙), d (天), w(周), n(月), y(年). +窗口大小的限制为[1m ~ 1y/12n]. INTERVAL 的单位与查询中INTERVAL子句相同, 如 a (毫秒), b (纳秒), h (小时), m (分钟), s (秒), u (微秒), d (天), w(周), n(月), y(年). TSMA为库内对象, 但名字全局唯一. 集群内一共可创建TSMA个数受参数`maxTsmaNum`限制, 参数默认值为3, 范围: [0-3]. 注意, 由于TSMA后台计算使用流计算, 因此每创建一条TSMA, 将会创建一条流, 因此能够创建的TSMA条数也受当前已经存在的流条数和最大可创建流条数限制. diff --git a/docs/zh/14-reference/05-connector/10-cpp.mdx b/docs/zh/14-reference/05-connector/10-cpp.mdx index 7c0da088a6..0df6ed924c 100644 --- a/docs/zh/14-reference/05-connector/10-cpp.mdx +++ b/docs/zh/14-reference/05-connector/10-cpp.mdx @@ -4,8 +4,615 @@ title: C/C++ Connector toc_max_heading_level: 4 --- -C/C++ 开发人员可以使用 TDengine 的客户端驱动，即 C/C++连接器 （以下都用 TDengine 客户端驱动表示），开发自己的应用来连接 TDengine 集群完成数据存储、查询以及其他功能。TDengine 客户端驱动的 API 类似于 MySQL 的 C API。应用程序使用时，需要包含 TDengine 头文件 _taos.h_，里面列出了提供的 API 的函数原型；应用程序还要链接到所在平台上对应的动态库。 +C/C++ 开发人员可以使用 TDengine 的客户端驱动，即 C/C++连接器 （以下都用 TDengine 客户端驱动表示），开发自己的应用来连接 TDengine 集群完成数据存储、查询以及其他功能。TDengine 客户端驱动的 API 类似于 MySQL 的 C API。应用程序使用时，需要包含 TDengine 头文件，里面列出了提供的 API 的函数原型；应用程序还要链接到所在平台上对应的动态库。 +TDengine 的客户端驱动提供了 taosws 和 taos 两个动态库，分别支持 Websocket 连接和原生连接。 Websocket 连接和原生连接的区别是 Websocket 连接方式不要求客户端和服务端版本完全匹配，而原生连接要求，在性能上 Websocket 连接方式也接近于原生连接，一般我们推荐使用 Websocket 连接方式。 +下面我们分开介绍两种连接方式的使用方法。 + + +## Websocket 连接方式 + +Websocket 连接方式需要使用 taosws.h 头文件和 taosws 动态库。 + +```c +#include +``` + +TDengine 服务端或客户端安装后，`taosws.h` 位于： + +- Linux：`/usr/local/taos/include` +- Windows：`C:\TDengine\include` +- macOS：`/usr/local/include` + +TDengine 客户端驱动的动态库位于： + +- Linux: `/usr/local/taos/driver/libtaosws.so` +- Windows: `C:\TDengine\taosws.dll` +- macOS: `/usr/local/lib/libtaosws.dylib` + +### 支持的平台 + +请参考[支持的平台列表](../#支持的平台) + +### 版本历史 + +| TDengine 客户端版本 | 主要变化 | TDengine 版本 | +| :------------------: | :---------------------------: | :----------------: | +| 3.3.3.0 | 首次发布，提供了 SQL执行，参数绑定，无模式写入和数据订阅等全面功能支持。 | 3.3.2.0及更高版本 | + + +### 错误码 + +在 C 接口的设计中，错误码采用整数类型表示，每个错误码都对应一个特定的错误状态。如未特别说明，当 API 的返回值是整数时，_0_ 代表成功，其它是代表失败原因的错误码，当返回值是指针时， _NULL_ 表示失败。 +Websocket 连接方式单独的错误码在 `taosws.h` 中， + + +| 错误码 | 错误描述 | 可能的出错场景或者可能的原因 | 建议用户采取的措施 | +| ------- | -------- | ---------------------------- | ------------------ | +| 0xE000 | DSN 错误 | DSN 不符合规范 | 检查 dsn 字符串是否符合规范 | +| 0xE001 | 内部错误 | 不确定 | 保留现场和日志，github上报issue | +| 0xE002 | 连接关闭 | 网络断开 | 请检查网络状况，查看 `taosadapter` 日志。 | +| 0xE003 | 发送超时 | 网络断开 | 请检查网络状况 | +| 0xE004 | 接收超时 | 慢查询，或者网络断开 | 排查 `taosadapter` 日志 | + +其余错误码请参考同目录下 `taoserror.h` 文件，详细的原生连接错误码说明参考：[错误码](../../../reference/error-code)。 +:::info +WebSocket 连接方式错误码只保留了原生连接错误码的后两个字节。 +::: + +### 示例程序 + +本节展示了使用客户端驱动访问 TDengine 集群的常见访问方式的示例代码。 + +- 同步查询示例：[同步查询](https://github.com/taosdata/TDengine/tree/main/docs/examples/c-ws/query_data_demo.c) + +- 参数绑定示例：[参数绑定](https://github.com/taosdata/TDengine/tree/main/docs/examples/c-ws/stmt_insert_demo.c) + +- 无模式写入示例：[无模式写入](https://github.com/taosdata/TDengine/tree/main/docs/examples/c-ws/sml_insert_demo.c) + +- 订阅和消费示例：[订阅和消费](https://github.com/taosdata/TDengine/tree/main/docs/examples/c-ws/tmq_demo.c) + +:::info +更多示例代码及下载请见 [GitHub](https://github.com/taosdata/TDengine/tree/main/docs/examples/c-ws)。 +::: + +### API 参考 + +以下分别介绍 TDengine 客户端驱动的 DSN、基础 API、同步查询 API、参数绑定 API、无模式写入 API 和 数据订阅订阅 API。 + +#### DSN + +C/C++ Websocket 连接器通过 DSN 连接描述字符串来表示连接信息。 +DSN 描述字符串基本结构如下： + +```text +[+]://[[:@]:][/][?=[&=]] +|------|------------|---|-----------|-----------|------|------|------------|-----------------------| +|driver| protocol | | username | password | host | port | database | params | +``` + +各部分意义见下表： + +- **driver**: 必须指定驱动名以便连接器选择何种方式创建连接，支持如下驱动名： + - **taos**: 默认驱动，支持 SQL 执行，参数绑定，无模式写入。 + - **tmq**: 使用 TMQ 订阅数据。 +- **protocol**: 显示指定以何种方式建立连接，例如：`taos+ws://localhost:6041` 指定以 Websocket 方式建立连接。 + - **http/ws**: 使用 Websocket 协议。 + - **https/wss**: 在 Websocket 连接方式下显示启用 SSL/TLS 协议。 + +- **username/password**: 用于创建连接的用户名及密码。 +- **host/port**: 指定创建连接的服务器及端口，当不指定服务器地址及端口时 Websocket 连接默认为 `localhost:6041` 。 +- **database**: 指定默认连接的数据库名，可选参数。 +- **params**：其他可选参数。 + +一个完整的 DSN 描述字符串示例如下：`taos+ws://localhost:6041/test`， 表示使用 Websocket（`ws`）方式通过 `6041` 端口连接服务器 `localhost`，并指定默认数据库为 `test`。 + +#### 基础 API + +基础 API 用于完成创建数据库连接等工作，为其它 API 的执行提供运行时环境。 + +- `char *ws_get_client_info()` + - **接口说明**：获取客户端版本信息。 + - **返回值**：返回客户端版本信息。 + +- `WS_TAOS *ws_connect(const char *dsn)` + - **接口说明**：创建数据库连接，初始化连接上下文。 + - **参数说明**： + - dsn：[入参] 连接信息，见上文 DSN 章节。 + - **返回值**：返回数据库连接，返回值为空表示失败。应用程序需要保存返回的参数，以便后续使用。 + :::info + 同一进程可以根据不同的 dsn 连接多个 TDengine 集群 + ::: + +- `const char *ws_get_server_info(WS_TAOS *taos)` + - **接口说明**：获取服务端版本信息。 + - **参数说明**： + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - **返回值**：返回获取服务端版本信息。 + +- `int32_t ws_select_db(WS_TAOS *taos, const char *db)` + - **接口说明**：将当前的缺省数据库设置为 `db`。 + - **参数说明**： + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - db：[入参] 数据库名称。 + - **返回值**：`0`：成功，非 `0`：失败，详情请参考错误码页面。 + +- `int32_t ws_get_current_db(WS_TAOS *taos, char *database, int len, int *required)` + - **接口说明**：获取当前数据库名称。 + - **参数说明**： + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - database：[出参] 存储当前数据库名称。 + - len：[入参] database 的空间大小。 + - required：[出参] 存储当前数据库名称所需的空间（包含最后的'\0'）。 + - **返回值**：`0`：成功，`-1`：失败，可调用函数 ws_errstr(NULL) 获取更详细的错误信息。 + - 如果，database == NULL 或者 len\<=0 返回失败。 + - 如果，len 小于 存储数据库名称所需的空间（包含最后的'\0'），返回失败，database 里赋值截断的数据，以'\0'结尾。 + - 如果，len 大于等于 存储数据库名称所需的空间（包含最后的'\0'），返回成功，database 里赋值以'\0‘结尾数据库名称。 + +- `int32_t ws_close(WS_TAOS *taos);` + - **接口说明**：关闭连接。 + - **参数说明**： + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - **返回值**：`0`：成功，非 `0`：失败，详情请参考错误码页面。 + +#### 同步查询 + +本小节介绍 API 均属于同步接口。应用调用后，会阻塞等待响应，直到获得返回结果或错误信息。 + +- `WS_RES *ws_query(WS_TAOS *taos, const char *sql)` + - **接口说明**：执行 SQL 语句，可以是 DQL、DML 或 DDL 语句。 + - **参数说明**： + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - sql：[入参] 需要执行 SQL 语句。 + - **返回值**：不能通过返回值是否是 `NULL` 来判断执行结果是否失败，而是需要调用 `ws_errno()` 函数解析结果集中的错误代码来进行判断。 + - ws_errno 返回值：`0`：成功，`-1`：失败，详情请调用 ws_errstr 函数来获取错误提示。 + +- `int32_t ws_result_precision(const WS_RES *rs)` + - **接口说明**：返回结果集时间戳字段的精度类别。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：`0`：毫秒，`1`：微秒，`2`：纳秒。 + +- `WS_ROW ws_fetch_row(WS_RES *rs)` + - **接口说明**：按行获取查询结果集中的数据。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：非 `NULL`：成功，`NULL`：失败，可调用函数 ws_errstr(NULL) 获取更详细的错误信息。 + +- `int32_t ws_fetch_raw_block(WS_RES *rs, const void **pData, int32_t *numOfRows)` + - **接口说明**：批量获取查询结果集中的数据。 + - **参数说明**： + - res：[入参] 结果集。 + - pData：[出参] 用于存储从结果集中获取一个数据块。 + - numOfRows：[出参] 用于存储从结果集中获取数据块包含的行数。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int32_t ws_num_fields(const WS_RES *rs)` 和 `int32_t ws_field_count(const WS_RES *rs)` + - **接口说明**：这两个 API 等价，用于获取查询结果集中的列数。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：返回值为结果集中列的数量。 + +- `int32_t ws_affected_rows(const WS_RES *rs)` + - **接口说明**：获取被所执行的 SQL 语句影响的行数。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：返回值表示受影响的行数。 + +- `int64_t ws_affected_rows64(const WS_RES *rs)` + - **接口说明**：获取被所执行的 SQL 语句影响的行数。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：返回值表示受影响的行数。 + +- `const struct WS_FIELD *ws_fetch_fields(WS_RES *rs)` + - **接口说明**：获取查询结果集每列数据的属性（列的名称、列的数据类型、列的长度），与 `ws_num_fields()` 配合使用，可用来解析 `ws_fetch_row()` 返回的一个元组(一行)的数据。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：非 `NULL`：成功，返回一个指向 WS_FIELD 结构体的指针，每个元素代表一列的元数据。`NULL`：失败。 + +- `int32_t ws_stop_query(WS_RES *rs)` + - **接口说明**：停止当前查询的执行。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int32_t ws_free_result(WS_RES *rs)` + - **接口说明**：释放查询结果集以及相关的资源。查询完成后，务必调用该 API 释放资源，否则可能导致应用内存泄露。但也需注意，释放资源后，如果再调用 `ws_fetch_fields()` 等获取查询结果的函数，将导致应用崩溃。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `const char *ws_errstr(WS_RES *rs)` + - **接口说明**：获取最近一次 API 调用失败的原因，返回值为字符串标识的错误提示信息。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：字符串标识的错误提示信息。 + +- `int32_t ws_errno(WS_RES *rs)` + - **接口说明**：获取最近一次 API 调用失败的原因，返回值为错误代码。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：字符串标识的错误提示信息。 + +:::note +TDengine 推荐数据库应用的每个线程都建立一个独立的连接，或基于线程建立连接池。不要在应用中将该连接 (WS_TAOS\*) 结构体传递到不同的线程共享使用。 +另一个需要注意的是，在上述同步 API 执行过程中，不能调用类似 pthread_cancel 之类的 API 来强制结束线程，因为涉及一些模块的同步操作，如果强制结束线程有可能造成包括但不限于死锁等异常状况。 + +::: + +#### 参数绑定 + +除了直接调用 `ws_query()` 通过执行 SQL 进行数据写入，TDengine 也提供了支持参数绑定的 Prepare API，风格与 MySQL 类似，目前也仅支持用问号 `?` 来代表待绑定的参数。 + +通过参数绑定接口写入数据时，可以避免 SQL 语法解析的资源消耗，从而在绝大多数情况下显著提升写入性能。此时的典型操作步骤如下： + +1. 调用 `ws_stmt_init()` 创建参数绑定对象； +2. 调用 `ws_stmt_prepare()` 解析 INSERT 语句； +3. 如果 INSERT 语句中预留了表名但没有预留 TAGS，那么调用 `ws_stmt_set_tbname()` 来设置表名； +4. 如果 INSERT 语句中既预留了表名又预留了 TAGS（例如 INSERT 语句采取的是自动建表的方式），那么调用 `ws_stmt_set_tbname_tags()` 来设置表名和 TAGS 的值； +5. 调用 `ws_stmt_bind_param_batch()` 以多行的方式设置 VALUES 的值； +6. 调用 `ws_stmt_add_batch()` 把当前绑定的参数加入批处理； +7. 可以重复第 3 ～ 6 步，为批处理加入更多的数据行； +8. 调用 `ws_stmt_execute()` 执行已经准备好的批处理指令； +9. 执行完毕，调用 `ws_stmt_close()` 释放所有资源。 + +说明：如果 `ws_stmt_execute()` 执行成功，假如不需要改变 SQL 语句的话，那么是可以复用 `ws_stmt_prepare()` 的解析结果，直接进行第 3 ～ 6 步绑定新数据的。但如果执行出错，那么并不建议继续在当前的环境上下文下继续工作，而是建议释放资源，然后从 `ws_stmt_init()` 步骤重新开始。 + +接口相关的具体函数如下（也可以参考 [stmt_insert_demo.c](https://github.com/taosdata/TDengine/blob/develop/docs/examples/c-ws/stmt_insert_demo.c) 文件中使用对应函数的方式）： + +- `WS_STMT *ws_stmt_init(const WS_TAOS *taos)` + - **接口说明**：初始化一个预编译的 SQL 语句对象。 + - **参数说明**： + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - **返回值**：非 `NULL`：成功，返回一个指向 WS_STMT 结构体的指针，该结构体表示预编译的 SQL 语句对象。`NULL`：失败，详情请调用 ws_stmt_errstr() 函数来获取错误提示。 + +- `int ws_stmt_prepare(WS_STMT *stmt, const char *sql, unsigned long len)` + - **接口说明**：解析一条预编译的 SQL 语句，将解析结果和参数信息绑定到 stmt 上。 + - **参数说明**： + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - sql：[入参] 需要解析的 SQL 语句。 + - len：[入参] 参数 sql 的长度。如果参数 len 大于 0，将使用此参数作为 SQL 语句的长度，如等于 0，将自动判断 SQL 语句的长度。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int ws_stmt_bind_param_batch(WS_STMT *stmt, const WS_MULTI_BIND *bind, uint32_t len)` + - **接口说明**：以多列的方式传递待绑定的数据，需要保证这里传递的数据列的顺序、列的数量与 SQL 语句中的 VALUES 参数完全一致。 + - **参数说明**： + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - bind：[入参] 指向一个有效的 WS_MULTI_BIND 结构体指针，该结构体包含了要批量绑定到 SQL 语句中的参数列表。 + - len: [入参] bind 数组的元素个数。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int ws_stmt_set_tbname(WS_STMT *stmt, const char *name)` + - **接口说明**：（仅支持用于替换 INSERT 语句中的参数值）当 SQL 语句中的表名使用了 `?` 占位时，可以使用此函数绑定一个具体的表名。 + - **参数说明**： + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - name：[入参] 指向一个包含子表名称的字符串常量。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int ws_stmt_set_tbname_tags(WS_STMT *stmt, + const char *name, + const WS_MULTI_BIND *bind, + uint32_t len);` + - **接口说明**：（仅支持用于替换 INSERT 语句中的参数值）当 SQL 语句中的表名和 TAGS 都使用了 `?` 占位时，可以使用此函数绑定具体的表名和具体的 TAGS 取值。最典型的使用场景是使用了自动建表功能的 INSERT 语句（目前版本不支持指定具体的 TAGS 列）。TAGS 参数中的列数量需要与 SQL 语句中要求的 TAGS 数量完全一致。 + - **参数说明**： + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - name：[入参] 指向一个包含子表名称的字符串常量。 + - tags：[入参] 指向一个有效的 WS_MULTI_BIND 结构体指针，该结构体包含了子表标签的值。 + - len：[入参] bind 数组的元素个数。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int ws_stmt_add_batch(WS_STMT *stmt)` + - **接口说明**：将当前绑定的参数加入批处理中，调用此函数后，可以再次调用 `ws_stmt_bind_param_batch()` 绑定新的参数。需要注意，此函数仅支持 INSERT/IMPORT 语句，如果是 SELECT 等其他 SQL 语句，将返回错误。 + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int ws_stmt_execute(WS_STMT *stmt, int32_t *affected_rows)` + - **接口说明**：执行准备好的语句。目前，一条语句只能执行一次。 + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - affected_rows：[出参] 成功写入的行数。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int ws_stmt_affected_rows(WS_STMT *stmt)` + - **接口说明**：获取执行预编译 SQL 语句后受影响的行数。 + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - **返回值**：返回受影响的行数。 + +- `int ws_stmt_affected_rows_once(WS_STMT *stmt)` + - **接口说明**：获取执行一次绑定语句影响的行数。 + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - **返回值**：返回受影响的行数。 + +- `int32_t ws_stmt_close(WS_STMT *stmt)` + - **接口说明**：执行完毕，释放所有资源。 + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `const char *ws_stmt_errstr(WS_STMT *stmt)` + - **接口说明**：用于在其他 STMT API 返回错误（返回错误码或空指针）时获取错误信息。 + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - **返回值**：返回一个指向包含错误信息的字符串的指针。 + +#### 无模式写入 + +除了使用 SQL 方式或者使用参数绑定 API 写入数据外，还可以使用 Schemaless 的方式完成写入。Schemaless 可以免于预先创建超级表/数据子表的数据结构，而是可以直接写入数据，TDengine 系统会根据写入的数据内容自动创建和维护所需要的表结构。Schemaless 的使用方式详见 [Schemaless 写入](../../../develop/schemaless/) 章节，这里介绍与之配套使用的 C/C++ API。 +- `WS_RES *ws_schemaless_insert_raw(WS_TAOS *taos, + const char *lines, + int len, + int32_t *totalRows, + int protocal, + int precision)` + - **接口说明**：执行无模式的批量插入操作，将行协议的文本数据写入到 TDengine 中。通过传递的参数lines指针和长度len来表示数据，为了解决原始接口数据包含'\0'而被截断的问题。 + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - lines：[入参] 文本数据。满足解析格式要求的无模式文本字符串。 + - len：[入参] 数据缓冲区 lines 的总长度（字节数）。 + - totalRows：[出参] 指向一个整数指针，用于返回成功插入的记录总数。 + - protocol：[入参] 行协议类型，用于标识文本数据格式。 + - precision：[入参] 文本数据中的时间戳精度字符串。 + - **返回值**：返回一个指向 WS_RES 结构体的指针，该结构体包含了插入操作的结果。应用可以通过使用 `ws_errstr()` 获得错误信息，也可以使用 `ws_errno()` 获得错误码。在某些情况下，返回的 WS_RES 为 `NULL`，此时仍然可以调用 `ws_errno()` 来安全地获得错误码信息。 + 返回的 WS_RES 需要调用方来负责释放，否则会出现内存泄漏。 + + **说明** + 协议类型是枚举类型，包含以下三种格式： + + - WS_TSDB_SML_LINE_PROTOCOL：InfluxDB 行协议（Line Protocol) + - WS_TSDB_SML_TELNET_PROTOCOL: OpenTSDB Telnet 文本行协议 + - WS_TSDB_SML_JSON_PROTOCOL: OpenTSDB Json 协议格式 + + 时间戳分辨率的定义，定义在 `taosws.h` 文件中，具体内容如下： + + - WS_TSDB_SML_TIMESTAMP_NOT_CONFIGURED = 0, + - WS_TSDB_SML_TIMESTAMP_HOURS, + - WS_TSDB_SML_TIMESTAMP_MINUTES, + - WS_TSDB_SML_TIMESTAMP_SECONDS, + - WS_TSDB_SML_TIMESTAMP_MILLI_SECONDS, + - WS_TSDB_SML_TIMESTAMP_MICRO_SECONDS, + - WS_TSDB_SML_TIMESTAMP_NANO_SECONDS + + 需要注意的是，时间戳分辨率参数只在协议类型为 `WS_SML_LINE_PROTOCOL` 的时候生效。 + 对于 OpenTSDB 的文本协议，时间戳的解析遵循其官方解析规则 — 按照时间戳包含的字符的数量来确认时间精度。 + + **schemaless 其他相关的接口** + +- `WS_RES *ws_schemaless_insert_raw_with_reqid(WS_TAOS *taos, + const char *lines, + int len, + int32_t *totalRows, + int protocal, + int precision, + uint64_t reqid)` + - **接口说明**：执行无模式的批量插入操作，将行协议的文本数据写入到 TDengine 中。通过传递的参数lines指针和长度len来表示数据，为了解决原始接口数据包含'\0'而被截断的问题。通过传递参数reqid来跟踪整个的函数调用链情况。 + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - lines：[入参] 文本数据。满足解析格式要求的无模式文本字符串。 + - len：[入参] 数据缓冲区 lines 的总长度（字节数）。 + - totalRows：[出参] 指向一个整数指针，用于返回成功插入的记录总数。 + - protocol：[入参] 行协议类型，用于标识文本数据格式。 + - precision：[入参] 文本数据中的时间戳精度字符串。 + - reqid：[入参] 指定的请求 ID，用于跟踪调用请求。请求 ID (reqid) 可以用于在客户端和服务器端之间建立请求和响应之间的关联，对于分布式系统中的跟踪和调试非常有用。 + - **返回值**：返回一个指向 WS_RES 结构体的指针，该结构体包含了插入操作的结果。应用可以通过使用 `ws_errstr()` 获得错误信息，也可以使用 `ws_errno()` 获得错误码。在某些情况下，返回的 WS_RES 为 `NULL`，此时仍然可以调用 `ws_errno()` 来安全地获得错误码信息。 + 返回的 WS_RES 需要调用方来负责释放，否则会出现内存泄漏。 + +- `WS_RES *ws_schemaless_insert_raw_ttl(WS_TAOS *taos, + const char *lines, + int len, + int32_t *totalRows, + int protocal, + int precision, + int ttl)` + - **接口说明**：执行无模式的批量插入操作，将行协议的文本数据写入到 TDengine 中。通过传递的参数lines指针和长度len来表示数据，为了解决原始接口数据包含'\0'而被截断的问题。通过传递ttl参数来控制建表的ttl到期时间。 + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - lines：[入参] 文本数据。满足解析格式要求的无模式文本字符串。 + - len：[入参] 数据缓冲区 lines 的总长度（字节数）。 + - totalRows：[出参] 指向一个整数指针，用于返回成功插入的记录总数。 + - protocol：[入参] 行协议类型，用于标识文本数据格式。 + - precision：[入参] 文本数据中的时间戳精度字符串。 + - ttl：[入参] 指定的生存时间（TTL），单位为天。记录在超过这个生存时间后会被自动删除。 + - **返回值**：返回一个指向 WS_RES 结构体的指针，该结构体包含了插入操作的结果。应用可以通过使用 `ws_errstr()` 获得错误信息，也可以使用 `ws_errno()` 获得错误码。在某些情况下，返回的 WS_RES 为 `NULL`，此时仍然可以调用 `ws_errno()` 来安全地获得错误码信息。 + 返回的 WS_RES 需要调用方来负责释放，否则会出现内存泄漏。 + +- `WS_RES *ws_schemaless_insert_raw_ttl_with_reqid(WS_TAOS *taos, + const char *lines, + int len, + int32_t *totalRows, + int protocal, + int precision, + int ttl, + uint64_t reqid)` + - **接口说明**：执行无模式的批量插入操作，将行协议的文本数据写入到 TDengine 中。通过传递的参数lines指针和长度len来表示数据，为了解决原始接口数据包含'\0'而被截断的问题。通过传递ttl参数来控制建表的ttl到期时间。通过传递参数reqid来跟踪整个的函数调用链情况。 + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - lines：[入参] 文本数据。满足解析格式要求的无模式文本字符串。 + - len：[入参] 数据缓冲区 lines 的总长度（字节数）。 + - totalRows：[出参] 指向一个整数指针，用于返回成功插入的记录总数。 + - protocol：[入参] 行协议类型，用于标识文本数据格式。 + - precision：[入参] 文本数据中的时间戳精度字符串。 + - ttl：[入参] 指定的生存时间（TTL），单位为天。记录在超过这个生存时间后会被自动删除。 + - reqid：[入参] 指定的请求 ID，用于跟踪调用请求。请求 ID (reqid) 可以用于在客户端和服务器端之间建立请求和响应之间的关联，对于分布式系统中的跟踪和调试非常有用。 + - **返回值**：返回一个指向 WS_RES 结构体的指针，该结构体包含了插入操作的结果。应用可以通过使用 `ws_errstr()` 获得错误信息，也可以使用 `ws_errno()` 获得错误码。在某些情况下，返回的 WS_RES 为 `NULL`，此时仍然可以调用 `ws_errno()` 来安全地获得错误码信息。 + 返回的 WS_RES 需要调用方来负责释放，否则会出现内存泄漏。 + + **说明** + - 上面这3个接口是扩展接口，主要用于在 schemaless 写入时传递 ttl、reqid 参数，可以根据需要使用。 + - 带 ttl 的接口可以传递 ttl 参数来控制建表的ttl到期时间。 + - 带 reqid 的接口可以通过传递 reqid 参数来追踪整个的调用链。 + +#### 数据订阅 +- `const char *ws_tmq_errstr(ws_tmq_t *tmq)` + - **接口说明**：用于获取数据订阅的错误信息。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - **返回值**：返回一个指向包含错误信息字符串的指针，返回值非NULL，但是错误信息可能为空字符串。 + +- `ws_tmq_conf_t *ws_tmq_conf_new(void);` + - **接口说明**：创建一个新的 TMQ 配置对象。 + - **返回值**：非 `NULL`：成功，返回一个指向 ws_tmq_conf_t 结构体的指针，该结构体用于配置 TMQ 的行为和特性。`NULL`：失败，可调用函数 ws_errstr(NULL) 获取更详细的错误信息。 + +- `enum ws_tmq_conf_res_t ws_tmq_conf_set(ws_tmq_conf_t *conf, const char *key, const char *value)` + - **接口说明**：设置 TMQ 配置对象中的配置项，用于配置消费参数。 + - conf：[入参] 指向一个有效的 ws_tmq_conf_t 结构体指针，该结构体代表一个 TMQ 配置对象。 + - key：[入参] 数配置项的键名。 + - value：[入参] 配置项的值。 + - **返回值**：返回一个 ws_tmq_conf_res_t 枚举值，表示配置设置的结果。 + - WS_TMQ_CONF_OK：成功设置配置项。 + - WS_TMQ_CONF_INVALID_KEY：键值无效。 + - WS_TMQ_CONF_UNKNOWN：键名无效。 + +- `int32_t ws_tmq_conf_destroy(ws_tmq_conf_t *conf)` + - **接口说明**：销毁一个 TMQ 配置对象并释放相关资源。 + - conf：[入参] 指向一个有效的 ws_tmq_conf_t 结构体指针，该结构体代表一个 TMQ 配置对象。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(NULL)` 获取更详细的错误信息。 + +- `ws_tmq_list_t *ws_tmq_list_new(void)` + - **接口说明**：用于创建一个 ws_tmq_list_t 结构体，用于存储订阅的 topic。 + - **返回值**：非 `NULL`：成功，返回一个指向 ws_tmq_list_t 结构体的指针。`NULL`：失败，可调用函数 `ws_tmq_errstr(NULL)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_list_append(ws_tmq_list_t *list, const char *topic)` + - **接口说明**：用于向 ws_tmq_list_t 结构体中添加一个 topic。 + - list：[入参] 指向一个有效的 ws_tmq_list_t 结构体指针，该结构体代表一个 TMQ 列表对象。 + - topic：[入参] topic 名称。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(NULL)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_list_destroy(ws_tmq_list_t *list);` + - **接口说明**：用于销毁 ws_tmq_list_t 结构体，ws_tmq_list_new 的结果需要通过该接口销毁。 + - list：[入参] 指向一个有效的 ws_tmq_list_t 结构体指针，该结构体代表一个 TMQ 列表对象。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(NULL)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_list_get_size(ws_tmq_list_t *list);` + - **接口说明**：用于获取 ws_tmq_list_t 结构体中 topic 的个数。 + - list：[入参] 指向一个有效的 ws_tmq_list_t 结构体指针，该结构体代表一个 TMQ 列表对象。 + - **返回值**：`>=0`：成功，返回 ws_tmq_list_t 结构体中 topic 的个数。`-1`：失败，表示输入参数 list 为 NULL 。 + +- `char **ws_tmq_list_to_c_array(const ws_tmq_list_t *list, uint32_t *topic_num);` + - **接口说明**：用于将 ws_tmq_list_t 结构体转换为 C 数组，数组每个元素为字符串指针。 + - list：[入参] 指向一个有效的 ws_tmq_list_t 结构体指针，该结构体代表一个 TMQ 列表对象。 + - topic_num：[入参] list 的元素个数。 + - **返回值**：非 `NULL`：成功，返回 c 数组, 每个元素是字符串指针，代表一个 topic 名称。`NULL`：失败，表示输入参数 list 为 NULL 。 + +- `ws_tmq_t *ws_tmq_consumer_new(ws_tmq_conf_t *conf, const char *dsn, char *errstr, int errstr_len)` + - **接口说明**：用于创建一个 ws_tmq_t 结构体，用于消费数据，消费完数据后需调用 tmq_consumer_close 关闭消费者。 + - conf：[入参] 指向一个有效的 ws_tmq_conf_t 结构体指针，该结构体代表一个 TMQ 配置对象。 + - dsn：[入参] dsn 信息字符串，具体可参考上面 DSN 章节。一个常见的合法 dsn 为 "tmq+ws://root:taosdata@localhost:6041"。 + - errstr：[出参] 指向一个有效的字符缓冲区指针，用于接收创建过程中可能产生的错误信息。内存的申请/释放由调用者负责。 + - errstrLen：[入参] 指定 errstr 缓冲区的大小（以字节为单位）。 + - **返回值**：非 `NULL`：成功，返回一个指向 ws_tmq_t 结构体的指针，该结构体代表一个 TMQ 消费者对象。。`NULL`：失败，错误信息存储在参数 errstr 中 。 + +- `int32_t ws_tmq_subscribe(ws_tmq_t *tmq, const ws_tmq_list_t *topic_list)` + - **接口说明**：用于订阅 topic 列表，消费完数据后，需调用 ws_tmq_subscribe 取消订阅。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - topic_list：[入参] 指向一个有效的 ws_tmq_list_t 结构体指针，该结构体包含一个或多个主题名称，目前仅支持一个主题名称。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_unsubscribe(ws_tmq_t *tmq)` + - **接口说明**：用于取消订阅的 topic 列表。需与 ws_tmq_subscribe 配合使用。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `WS_RES *ws_tmq_consumer_poll(ws_tmq_t *tmq, int64_t timeout)` + - **接口说明**：用于轮询消费数据，每一个消费者，只能单线程调用该接口。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - timeout：[入参] 轮询的超时时间，单位为毫秒，负数表示默认超时1秒。 + - **返回值**：非 `NULL`：成功，返回一个指向 WS_RES 结构体的指针，该结构体包含了接收到的消息。`NULL`：失败，表示没有数据。WS_RES 结果和 taos_query 返回结果一致，可通过查询的各种接口获取 WS_RES 里的信息，比如 schema 等。 + +- `int32_t ws_tmq_consumer_close(ws_tmq_t *tmq)` + - **接口说明**：用于关闭 ws_tmq_t 结构体。需与 ws_tmq_consumer_new 配合使用。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_get_topic_assignment(ws_tmq_t *tmq, + const char *pTopicName, + struct ws_tmq_topic_assignment **assignment, + int32_t *numOfAssignment)` + - **接口说明**：返回当前 consumer 分配的 vgroup 的信息，每个 vgroup 的信息包括 vgId，wal 的最大最小 offset，以及当前消费到的 offset。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - pTopicName：[入参] 要查询分配信息的主题名称。 + - assignment：[出参] 指向一个 tmq_topic_assignment 结构体指针的指针，用于接收分配信息。数据大小为 numOfAssignment，需要通过 tmq_free_assignment 接口释放。 + - numOfAssignment：[出参] 指向一个整数指针，用于接收分配给该 consumer 有效的 vgroup 个数。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_free_assignment(struct ws_tmq_topic_assignment *pAssignment, int32_t numOfAssignment)` + - **接口说明**：返回当前consumer分配的vgroup的信息，每个vgroup的信息包括vgId，wal的最大最小offset，以及当前消费到的offset。 + - pAssignment：[入参] 指向一个有效的 ws_tmq_topic_assignment 结构体数组的指针，该数组包含了 vgroup 分配信息。 + - numOfAssignment：[入参] pAssignment 指向的数组元素个数。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int64_t ws_tmq_committed(ws_tmq_t *tmq, const char *pTopicName, int32_t vgId)` + - **接口说明**：获取 TMQ 消费者对象对特定 topic 和 vgroup 的已提交偏移量。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - pTopicName：[入参] 要查询已提交偏移量的主题名称。 + - vgId：[入参] vgroup 的 ID。 + - **返回值**：`>=0`：成功，返回一个 int64_t 类型的值，表示已提交的偏移量。`<0`：失败，返回值就是错误码，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_commit_sync(ws_tmq_t *tmq, const WS_RES *rs)` + - **接口说明**：同步提交 TMQ 消费者对象处理的消息偏移量。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - rs：[入参] 指向一个有效的 WS_RES 结构体指针，该结构体包含了已处理的消息。如果为 NULL，提交当前 consumer 所有消费的 vgroup 的当前进度。 + - **返回值**：`0`：成功，已经成功提交偏移量。非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_commit_offset_sync(ws_tmq_t *tmq, + const char *pTopicName, + int32_t vgId, + int64_t offset)` + - **接口说明**：同步提交 TMQ 消费者对象的特定主题和 vgroup 的偏移量。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - pTopicName：[入参] 要提交偏移量的主题名称。 + - vgId：[入参] 虚拟组 vgroup 的 ID。 + - offset：[入参] 要提交的偏移量。 + - **返回值**：`0`：成功，已经成功提交偏移量。非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int64_t ws_tmq_position(ws_tmq_t *tmq, const char *pTopicName, int32_t vgId)` + - **接口说明**：获取当前消费位置，即已消费到的数据位置的下一个位置. + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - pTopicName：[入参] 要查询当前位置的主题名称。 + - vgId：[入参] 虚拟组 vgroup 的 ID。 + - **返回值**：`>=0`：成功，返回一个 int64_t 类型的值，表示当前位置的偏移量。`<0`：失败，返回值就是错误码，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + + - `int32_t ws_tmq_offset_seek(ws_tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset)` + - **接口说明**：将 TMQ 消费者对象在某个特定 topic 和 vgroup 的偏移量设置到指定的位置。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - pTopicName：[入参] 要查询当前位置的主题名称。 + - vgId：[入参] 虚拟组 vgroup 的 ID。 + - offset：[入参] 虚拟组 vgroup 的 ID。 + - **返回值**：`0`：成功，非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int64_t ws_tmq_get_vgroup_offset(const WS_RES *rs)` + - **接口说明**：从 TMQ 消费者获取的消息结果中提取虚拟组（vgroup）的当前消费数据位置的偏移量。 + - res：[入参] 指向一个有效的 WS_RES 结构体指针，该结构体包含了从 TMQ 消费者轮询得到的消息。 + - **返回值**：`>=0`：成功，返回一个 int64_t 类型的值，表示当前消费位置的偏移量。`<0`：失败，返回值就是错误码，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_get_vgroup_id(const WS_RES *rs)` + - **接口说明**：从 TMQ 消费者获取的消息结果中提取所属虚拟组（vgroup）的 ID。 + - res：[入参] 指向一个有效的 WS_RES 结构体指针，该结构体包含了从 TMQ 消费者轮询得到的消息。 + - **返回值**：`>=0`：成功，返回一个 int32_t 类型的值，表示虚拟组（vgroup）的 ID。`<0`：失败，返回值就是错误码，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `const char *ws_tmq_get_table_name(const WS_RES *rs)` + - **接口说明**：从 TMQ 消费者获取的消息结果中获取所属的表名。 + - res：[入参] 指向一个有效的 WS_RES 结构体指针，该结构体包含了从 TMQ 消费者轮询得到的消息。 + - **返回值**：非 `NULL`：成功，返回一个 const char * 类型的指针，指向表名字符串。`NULL`：失败，非法的输入参数。 + +- `enum ws_tmq_res_t ws_tmq_get_res_type(const WS_RES *rs)` + - **接口说明**：从 TMQ 消费者获取的消息结果中获取消息类型。 + - res：[入参] 指向一个有效的 WS_RES 结构体指针，该结构体包含了从 TMQ 消费者轮询得到的消息。 + - **返回值**：返回一个 ws_tmq_res_t 类型的枚举值，表示消息类型。 + - ws_tmq_res_t 表示消费到的数据类型，定义如下： + ``` + typedef enum ws_tmq_res_t { + WS_TMQ_RES_INVALID = -1, // 无效 + WS_TMQ_RES_DATA = 1, // 数据类型 + WS_TMQ_RES_TABLE_META = 2, // 元数据类型 + WS_TMQ_RES_METADATA = 3 // 既有元数据类型又有数据类型，即自动建表 + } tmq_res_t; + ``` + +- `const char *ws_tmq_get_topic_name(const WS_RES *rs)` + - **接口说明**：从 TMQ 消费者获取的消息结果中获取所属的 topic 名称。 + - res：[入参] 指向一个有效的 WS_RES 结构体指针，该结构体包含了从 TMQ 消费者轮询得到的消息。 + - **返回值**：非 `NULL`：成功，返回一个 const char * 类型的指针，指向 topic 名称字符串。`NULL`：失败，非法的输入参数。 + +- `const char *ws_tmq_get_db_name(const WS_RES *rs)` + - **接口说明**：从 TMQ 消费者获取的消息结果中获取所属的数据库名称。 + - res：[入参] 指向一个有效的 WS_RES 结构体指针，该结构体包含了从 TMQ 消费者轮询得到的消息。 + - **返回值**：非 `NULL`：成功，返回一个 const char * 类型的指针，指向数据库名称字符串。`NULL`：失败，非法的输入参数。 + +## 原生连接方式 +原生连接方式需要使用 taos.h 头文件和 taos 动态库。 ```c #include ``` @@ -22,21 +629,21 @@ TDengine 客户端驱动的动态库位于： - Windows: `C:\TDengine\taos.dll` - macOS: `/usr/local/lib/libtaos.dylib` -## 支持的平台 +### 支持的平台 请参考[支持的平台列表](../#支持的平台) -## 支持的版本 +### 支持的版本 TDengine 客户端驱动的版本号与 TDengine 服务端的版本号是一一对应的强对应关系，建议使用与 TDengine 服务端完全相同的客户端驱动。虽然低版本的客户端驱动在前三段版本号一致（即仅第四段版本号不同）的情况下也能够与高版本的服务端相兼容，但这并非推荐用法。强烈不建议使用高版本的客户端驱动访问低版本的服务端。 -## 错误码 +### 错误码 在 C 接口的设计中，错误码采用整数类型表示，每个错误码都对应一个特定的错误状态。如未特别说明，当 API 的返回值是整数时，_0_ 代表成功，其它是代表失败原因的错误码，当返回值是指针时， _NULL_ 表示失败。 所有的错误码以及对应的原因描述在 `taoserror.h` 文件中。 详细的错误码说明参考：[错误码](../../../reference/error-code) -## 示例程序 +### 示例程序 本节展示了使用客户端驱动访问 TDengine 集群的常见访问方式的示例代码。 @@ -57,11 +664,11 @@ TDengine 客户端驱动的版本号与 TDengine 服务端的版本号是一一 ::: -## API 参考 +### API 参考 -以下分别介绍 TDengine 客户端驱动的基础 API、同步 API、异步 API、订阅 API 和无模式写入 API。 +以下分别介绍 TDengine 客户端驱动的基础 API、同步 API、异步 API、参数绑定 API，无模式写入 API 和数据订阅 API。 -### 基础 API +#### 基础 API 基础 API 用于完成创建数据库连接等工作，为其它 API 的执行提供运行时环境。 @@ -145,7 +752,7 @@ TDengine 客户端驱动的版本号与 TDengine 服务端的版本号是一一 - **参数说明**： - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `taos_connect()` 函数建立。 -### 同步查询 +#### 同步查询 本小节介绍 API 均属于同步接口。应用调用后，会阻塞等待响应，直到获得返回结果或错误信息。 @@ -228,7 +835,7 @@ TDengine 客户端驱动的版本号与 TDengine 服务端的版本号是一一 ::: -### 异步查询 +#### 异步查询 TDengine 还提供性能更高的异步 API 处理数据插入、查询操作。在软硬件环境相同的情况下，异步 API 处理数据插入的速度比同步 API 快 2 ～ 4 倍。异步 API 采用非阻塞式的调用方式，在系统真正完成某个具体数据库操作前，立即返回。调用的线程可以去处理其他工作，从而可以提升整个应用的性能。异步 API 在网络延迟严重的情况下，优势尤为突出。 @@ -252,7 +859,7 @@ TDengine 还提供性能更高的异步 API 处理数据插入、查询操作。 TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多线程同时打开多张表，并可以同时对每张打开的表进行查询或者插入操作。需要指出的是，**客户端应用必须确保对同一张表的操作完全串行化**，即对同一个表的插入或查询操作未完成时（未返回时），不能够执行第二个插入或查询操作。 -### 参数绑定 +#### 参数绑定 除了直接调用 `taos_query()` 进行查询，TDengine 也提供了支持参数绑定的 Prepare API，风格与 MySQL 类似，目前也仅支持用问号 `?` 来代表待绑定的参数。 @@ -350,7 +957,7 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多 - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 - **返回值**：返回一个指向包含错误信息的字符串的指针。 -### 无模式写入 +#### 无模式写入 除了使用 SQL 方式或者使用参数绑定 API 写入数据外，还可以使用 Schemaless 的方式完成写入。Schemaless 可以免于预先创建超级表/数据子表的数据结构，而是可以直接写入数据，TDengine 系统会根据写入的数据内容自动创建和维护所需要的表结构。Schemaless 的使用方式详见 [Schemaless 写入](../../../develop/schemaless/) 章节，这里介绍与之配套使用的 C/C++ API。 @@ -474,7 +1081,7 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多 - 带_ttl的接口可以传递ttl参数来控制建表的ttl到期时间。 - 带_reqid的接口可以通过传递reqid参数来追踪整个的调用链。 -### 数据订阅 +#### 数据订阅 - `const char *tmq_err2str(int32_t code)` - **接口说明**：用于将数据订阅的错误码转换为错误信息。 - code：[入参] 数据订阅的错误码。 @@ -584,7 +1191,7 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多 - **接口说明**：获取 TMQ 消费者对象对特定 topic 和 vgroup 的已提交偏移量。 - tmq：[入参] 指向一个有效的 tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 - pTopicName：[入参] 要查询已提交偏移量的主题名称。 - - assignment：[入参] vgroup 的 ID。 + - vgId：[入参] vgroup 的 ID。 - **返回值**：`>=0`：成功，返回一个 int64_t 类型的值，表示已提交的偏移量。`<0`：失败，返回值就是错误码，可调用函数 `char *tmq_err2str(int32_t code)` 获取更详细的错误信息。 - `int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg)` @@ -604,6 +1211,8 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多 - **接口说明**：同步提交 TMQ 消费者对象的特定主题和 vgroup 的偏移量。 - tmq：[入参] 指向一个有效的 tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 - pTopicName：[入参] 要提交偏移量的主题名称。 + - vgId：[入参] 虚拟组 vgroup 的 ID。 + - offset：[入参] 要提交的偏移量。 - **返回值**：`0`：成功，已经成功提交偏移量。非 `0`：失败，可调用函数 `char *tmq_err2str(int32_t code)` 获取更详细的错误信息。 - `void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param)` diff --git a/docs/zh/14-reference/05-connector/26-rust.mdx b/docs/zh/14-reference/05-connector/26-rust.mdx index c5d2a165d4..1fcc1e3dcd 100644 --- a/docs/zh/14-reference/05-connector/26-rust.mdx +++ b/docs/zh/14-reference/05-connector/26-rust.mdx @@ -113,11 +113,11 @@ DSN 描述字符串基本结构如下： 各部分意义见下表： - **driver**: 必须指定驱动名以便连接器选择何种方式创建连接，支持如下驱动名： - - **taos**: 表名使用 TDengine 连接器驱动。 + - **taos**: 使用 TDengine 连接器驱动，默认是使用 taos 驱动。 - **tmq**: 使用 TMQ 订阅数据。 +- **protocol**: 显示指定以何种方式建立连接，例如：`taos+ws://localhost:6041` 指定以 Websocket 方式建立连接。 - **http/ws**: 使用 Websocket 创建连接。 - **https/wss**: 在 Websocket 连接方式下显示启用 SSL/TLS 连接。 -- **protocol**: 显示指定以何种方式建立连接，例如：`taos+ws://localhost:6041` 指定以 Websocket 方式建立连接。 - **username/password**: 用于创建连接的用户名及密码。 - **host/port**: 指定创建连接的服务器及端口，当不指定服务器地址及端口时（`taos://`），原生连接默认为 `localhost:6030`，Websocket 连接默认为 `localhost:6041` 。 - **database**: 指定默认连接的数据库名，可选参数。 @@ -522,6 +522,6 @@ Offset 结构体提供了获取当前消息所属的数据库，主题和分区 ## 附录 - Rust 连接器文档：https://docs.rs/taos -- Rust 连接器项目地址: https://github.com/taosdata/rust-connector-taos +- Rust 连接器项目地址: https://github.com/taosdata/taos-connector-rust - deadpool 连接池: https://crates.io/crates/deadpool - r2d2 连接池: https://crates.io/crates/r2d2 diff --git a/docs/zh/14-reference/05-connector/35-node.mdx b/docs/zh/14-reference/05-connector/35-node.mdx index 3d8e82086e..bd2ca537e3 100644 --- a/docs/zh/14-reference/05-connector/35-node.mdx +++ b/docs/zh/14-reference/05-connector/35-node.mdx @@ -85,8 +85,6 @@ Node.js 连接器目前仅支持 Websocket 连接器， 其通过 taosAdapter | [sql_example](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/sql_example.js) | 基本的使用如如建立连接，执行 SQL 等操作。 | | [stmt_example](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/stmt_example.js) | 绑定参数插入的示例。 | | | [line_example](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/line_example.js) | 行协议写入示例。 | -| [telnet_line_example](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/telnet_line_example.js) | OpenTSDB Telnet 行协议写入示例。 | -| [json_line_example](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/json_line_example.js) | OpenTSDB JSON 行协议写入示例。 | | [tmq_example](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/tmq_example.js) | 订阅的使用示例。 | | [all_type_query](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/all_type_query.js) | 支持全部类型示例。 | | [all_type_stmt](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/all_type_stmt.js) | 参数绑定支持全部类型示例。 | diff --git a/docs/zh/14-reference/05-connector/50-odbc.mdx b/docs/zh/14-reference/05-connector/50-odbc.mdx index ee69cf9364..38dd88b86d 100644 --- a/docs/zh/14-reference/05-connector/50-odbc.mdx +++ b/docs/zh/14-reference/05-connector/50-odbc.mdx @@ -60,6 +60,8 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式：Websocket 连接与 4.6【密码】仅供第5步测试连接使用，选填，数据库用户密码，如果不填，TDengine 默认 taosdata + 4.7【兼容软件】支持对工业软件 KingSCADA、Kepware 等的兼容性适配，通常情况下，选择默认值 General 即可 + 5. 点【测试连接】测试连接情况，如果成功，提示"成功连接到URL" 6. 点【确定】，即可保存配置并退出 @@ -90,12 +92,449 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式：Websocket 连接与 4.6 【密码】仅供第5步测试连接使用，选填，数据库用户密码，如果不填，TDengine 默认 taosdata + 4.7【兼容软件】支持对工业软件 KingSCADA、Kepware 等的兼容性适配，通常情况下，选择默认值 General 即可 + 5. 点【测试连接】测试连接情况，如果成功，提示"连接成功" 6. 点【确定】，即可保存配置并退出 7. 也可以在第2步选择已经配置好的数据源名通过【配置】按钮进入配置页面，修改已有配置 + +## 支持的平台 + +原生连接方式支持的平台和 TDengine Windows X64版 客户端驱动支持的平台一致。 +WebSocket 连接方式除此之外还支持 Windows X64系统上运行的 32 位应用程序上使用。 + + +## 版本历史 + +| taos_odbc版本 | 主要变化 | TDengine 版本 | +| :----------- | :-------------------------------------------------------------------------------------------------- | :---------------- | +| v1.1.0 | 1. 支持视图功能；
2. 支持 VARBINARY/GEOMETRY 数据类型； | 3.3.3.0及更高版本 | +| v1.0.2 | 支持 CP1252 字符编码； | 3.2.3.0及更高版本 | +| v1.0.1 | 1. 支持 DSN 设置 BI 模式，在 BI 模式下 TDengine 数据库不返回系统数据库和超级表子表信息；
2. 重构字符集转换模块，提升读写性能；
3. ODBC 数据源配置对话框中默认修改默认连接方式为“WebSocket”；
4. ODBC 数据源配置对话框增加“测试连接”控件；
5. ODBC 数据源配置支持中文/英文界面； | - | +| v1.0.0.0 | 发布初始版本，支持与Tdengine数据库交互以读写数据，具体请参考“API 参考”一节 | 3.2.2.0及更高版本 | + + +## 数据类型映射 + +下表说明了 ODBC 连接器如何将服务器数据类型映射到默认的 SQL 和 C 数据类型。 + +| TDengine Type | SQL Type | C Type | +|--------------------|-------------------|-------------------| +| TIMESTAMP | SQL_TYPE_TIMESTAMP| SQL_C_TIMESTAMP | +| INT | SQL_INTEGER | SQL_C_SLONG | +| INT UNSIGNED | SQL_INTEGER | SQL_C_ULONG | +| BIGINT | SQL_BIGINT | SQL_C_SBIGINT | +| BIGINT UNSIGNED | SQL_BIGINT | SQL_C_UBIGINT | +| FLOAT | SQL_REAL | SQL_C_FLOAT | +| DOUBLE | SQL_REAL | SQL_C_DOUBLE | +| BINARY | SQL_BINARY | SQL_C_BINARY | +| SMALLINT | SQL_SMALLINT | SQL_C_SSHORT | +| SMALLINT UNSIGNED | SQL_SMALLINT | SQL_C_USHORT | +| TINYINT | SQL_TINYINT | SQL_C_STINYINT | +| TINYINT UNSIGNED | SQL_TINYINT | SQL_C_UTINYINT | +| BOOL | SQL_BIT | SQL_C_BIT | +| NCHAR | SQL_VARCHAR | SQL_C_CHAR | +| JSON | SQL_VARCHAR | SQL_C_CHAR | +| VARCHAR | SQL_VARCHAR | SQL_C_CHAR | +| GEOMETRY | SQL_VARBINARY | SQL_C_BINARY | +| VARBINARY | SQL_VARBINARY | SQL_C_BINARY | + + +## API 参考 + +本节按功能分类汇总了 ODBC API，关于完整的 ODBC API 参考，请访问 http://msdn.microsoft.com/en-us/library/ms714177.aspx 的ODBC程序员参考页面。 + +### 数据源和驱动程序管理 + +- API: ConfigDSN + - **是否支持**: 支持 + - **标准**: ODBC + - **作用**: 配置数据源 + +- API: ConfigDriver + - **是否支持**: 支持 + - **标准**: ODBC + - **作用**: 用于执行与特定驱动程序相关的安装和配置任务 + +- API: ConfigTranslator + - **是否支持**: 支持 + - **标准**: ODBC + - **作用**: 用于解析DSN的配置，在DSN配置和实际数据库驱动程序配置之间进行翻译或转换 + + +### 连接到数据源 + +- API: SQLAllocHandle + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 分配环境、连接、语句或描述符句柄 + +- API: SQLConnect + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 通过数据源名称、用户 ID 和密码连接到特定驱动程序 + +- API: SQLDriverConnect + - **是否支持**: 支持 + - **标准**: ODBC + - **作用**: 通过连接字符串连接到特定驱动程序，支持更多连接信息 + +- API: SQLBrowseConnect + - **是否支持**: 不支持 + - **标准**: ODBC + - **作用**: 用于发现和枚举连接到数据源所需的特性和属性值。每次调用 SQLBrowseConnect 都会返回属性和属性值的连续级别 + +- API: SQLAllocEnv + - **是否支持**: 不支持 + - **标准**: 弃用 + - **作用**: 在 ODBC 3.x 中，ODBC 2.x 函数 SQLAllocEnv 已替换为 SQLAllocHandle + +- API: SQLAllocConnect + - **是否支持**: 不支持 + - **标准**: 弃用 + - **作用**: 在 ODBC 3.x 中，ODBC 2.x 函数 SQLAllocConnect 已替换为 SQLAllocHandle + + +### 获取有关驱动程序和数据源的信息 + +- API: SQLDataSources + - **是否支持**: 不支持 + - **标准**: ISO 92 + - **作用**: 返回可用数据源的列表，由驱动程序管理器处理 + +- API: SQLDrivers + - **是否支持**: 不支持 + - **标准**: ISO 92 + - **作用**: 返回由驱动程序管理器处理的已安装驱动程序及其属性的列表 + +- API: SQLGetInfo + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 返回有关数据库环境的详细信息，如数据库产品名称、驱动程序名、数据库的SQL语法特性、连接能力等等 + +- API: SQLGetFunctions + - **是否支持**: 不支持 + - **标准**: ISO 92 + - **作用**: 用于查询驱动程序支持的函数 + +- API: SQLGetTypeInfo + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 返回有关支持的数据类型的信息 + + +### 设置和检索驱动程序属性 + +- API: SQLSetConnectAttr + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 设置连接属性，当设置SQL_ATTR_AUTOCOMMIT属性时，用于控制自动提交模式 + +- API: SQLGetConnectAttr + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 返回连接属性的值 + +- API: SQLSetConnectOption + - **是否支持**: 不支持 + - **标准**: 弃用 + - **作用**: 在 ODBC 3.x 中，ODBC 2.0 函数 SQLSetConnectOption 已替换为 SQLSetConnectAttr + +- API: SQLGetConnectOption + - **是否支持**: 不支持 + - **标准**: 弃用 + - **作用**: 在 ODBC 3.x 中，ODBC 2.0 函数 SQLSetConnectOption 已替换为 SQLGetConnectAttr + +- API: SQLSetEnvAttr + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 设置控制环境的属性 + +- API: SQLGetEnvAttr + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 返回环境属性的当前设置 + +- API: SQLSetStmtAttr + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 设置与语句相关的属性 + +- API: SQLGetStmtAttr + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 返回语句属性的当前设置 + +- API: SQLSetStmtOption + - **是否支持**: 不支持 + - **标准**: 弃用 + - **作用**: 在 ODBC 3.x 中，ODBC 2.0 函数 SQLSetStmtOption 已替换为 SQLSetStmtAttr + +- API: SQLGetStmtOption + - **是否支持**: 不支持 + - **标准**: 弃用 + - **作用**: 在 ODBC 3.x 中，ODBC 2.0 函数 SQLSetStmtOption 已替换为 SQLGetStmtAttr + + +### 准备SQL请求 + +- API: SQLAllocStmt + - **是否支持**: 不支持 + - **标准**: 弃用 + - **作用**: 在 ODBC 3.x 中，ODBC 2.x 函数 SQLAllocStmt 已替换为 SQLAllocHandle + +- API: SQLPrepare + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 用于预处理SQL语句，这通常是SQLExecute之前的一个步骤 + +- API: SQLBindCol + - **是否支持**: 支持 + - **标准**: ODBC + - **作用**: 用于将结果集中的列绑定到应用程序缓冲区 + +- API: SQLBindParameter + - **是否支持**: 支持 + - **标准**: ODBC + - **作用**: 用于将SQL语句的参数绑定到应用程序缓冲区 + +- API: SQLGetCursorName + - **是否支持**: 不支持 + - **标准**: ISO 92 + - **作用**: 返回与指定语句关联的游标名称 + +- API: SQLSetCursorName + - **是否支持**: 不支持 + - **标准**: ISO 92 + - **作用**: 设置游标名称，允许在查询中使用命名游标 + +- API: SQLSetScrollOptions + - **是否支持**: 不支持 + - **标准**: ODBC + - **作用**: 设置控制光标行为的选项 + + +### 提交请求 + +- API: SQLExecute + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 用于执行之前通过 SQLPrepare 准备好的SQL语句 + +- API: SQLExecDirect + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 用于执行包含SQL语句的字符串 + +- API: SQLNativeSql + - **是否支持**: 不支持 + - **标准**: ODBC + - **作用**: 用于将应用程序提供的SQL语句转换为数据库驱动程序的本机SQL语法 + +- API: SQLDescribeParam + - **是否支持**: 支持 + - **标准**: ODBC + - **作用**: 返回语句中特定参数的描述 + +- API: SQLNumParams + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 用于查询预编译SQL语句中的参数数量 + +- API: SQLParamData + - **是否支持**: 不支持 + - **标准**: ISO 92 + - **作用**: 用于从参数数据流中获取下一个参数值 + +- API: SQLPutData + - **是否支持**: 不支持 + - **标准**: ISO 92 + - **作用**: 当使用流输入方式时，可以用于向输出参数发送数据块 + + +### 检索结果和关于结果的信息 + +- API: SQLRowCount + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 返回受插入或删除请求影响的行数 + +- API: SQLNumResultCols + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 返回结果集中的列数 + +- API: SQLDescribeCol + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 用于描述结果集中列的属性。它提供了关于列的数据类型、列名、列的最大宽度、小数位数和是否可为空等信息 + +- API: SQLColAttribute + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 返回结果集中列的描述符信息，如标题、排序规则等 + +- API: SQLColAttributes + - **是否支持**: 不支持 + - **标准**: 弃用 + - **作用**: 在 ODBC 3.x 中，ODBC 2.0 函数 SQLColAttributes 已替换为 SQLColAttribute + +- API: SQLGetData + - **是否支持**: 支持 + - **标准**: ODBC + - **作用**: 用于从结果集中的当前行获取特定列的数据 + +- API: SQLMoreResults + - **是否支持**: 支持 + - **标准**: ODBC + - **作用**: 多个结果集的 SQL 语句执行后（例如：一个批处理或存储过程），移动到下一个结果集 + +- API: SQLFetch + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 用于从结果集中提取下一行数据，并返回所有绑定列的数据 + +- API: SQLFetchScroll + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 用于从结果集中提取指定的数据行集，并返回所有绑定列的数据 + +- API: SQLExtendedFetch + - **是否支持**: 不支持 + - **标准**: 弃用 + - **作用**: 在 ODBC 3.x 中，SQLExtendedFetch 已替换为 SQLFetchScroll + +- API: SQLSetPos + - **是否支持**: 支持 + - **标准**: ODBC + - **作用**: 设置行集中的游标位置，并允许应用程序更新数据集中的行 + +- API: SQLBulkOperations + - **是否支持**: 不支持 + - **标准**: ODBC + - **作用**: 执行批量插入和批量书签操作，包括更新、删除和按书签提取 + + +### 检索错误或诊断信息 + +- API: SQLError + - **是否支持**: 不支持 + - **标准**: 弃用 + - **作用**: 在 ODBC 3.x 中，ODBC 2.x 函数 SQLError 已替换为 SQLGetDiagRec + +- API: SQLGetDiagField + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 返回附加诊断信息（单条诊断结果） + +- API: SQLGetDiagRec + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 返回附加诊断信息（多条诊断结果） + + +### 获取有关数据源的系统表项的信息 + +- API: SQLColumnPrivileges + - **是否支持**: 不支持 + - **标准**: ODBC + - **作用**: 用于检索指定表中列的权限信息，如哪些用户或角色拥有对特定列的读取、插入、更新或删除权限 + +- API: SQLColumns + - **是否支持**: 支持 + - **标准**: X/Open + - **作用**: 返回指定表中的列名列表 + +- API: SQLForeignKeys + - **是否支持**: 不支持 + - **标准**: ODBC + - **作用**: 检索外键关系的详细信息 + +- API: SQLPrimaryKeys + - **是否支持**: 支持 + - **标准**: ODBC + - **作用**: 返回构成表主键的列名列表 + +- API: SQLSpecialColumns + - **是否支持**: 不支持 + - **标准**: X/Open + - **作用**: 返回数据库中特殊列的信息，如唯一键或索引列 + +- API: SQLStatistics + - **是否支持**: 不支持 + - **标准**: ISO 92 + - **作用**: 返回关于表的统计信息，如行数、列数、平均行宽等 + +- API: SQLTablePrivileges + - **是否支持**: 不支持 + - **标准**: ODBC + - **作用**: 返回用户在特定表上的权限，如SELECT、INSERT、UPDATE等 + +- API: SQLTables + - **是否支持**: 支持 + - **标准**: X/Open + - **作用**: 返回存储在数据源的当前数据库中的表信息 + +- API: SQLProcedures + - **是否支持**: 不支持 + - **标准**: ODBC + - **作用**: 返回数据库中可用的存储过程信息，包括名称和类型 + +- API: SQLProcedureColumns + - **是否支持**: 不支持 + - **标准**: ODBC + - **作用**: 返回存储过程的列信息，包括输入输出参数的详细信息 + + +### 执行事务 + +- API: SQLTransact + - **是否支持**: 不支持 + - **标准**: 弃用 + - **作用**: 在 ODBC 3.x 中，ODBC 2.x 函数 SQLTransact 已替换为 SQLEndTran + +- API: SQLEndTran + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 用于提交或回滚事务，TDengine 不支持事务，因此不支持回滚操作 + + +### 终止连接 + +- API: SQLDisconnect + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 断开数据库连接 + +- API: SQLFreeHandle + - **是否支持**: 支持 + - **标准**: ISO 92 + - **作用**: 释放与特定环境、连接、语句或描述符句柄关联的资源 + +- API: SQLFreeConnect + - **是否支持**: 不支持 + - **标准**: 弃用 + - **作用**: 在 ODBC 3.x 中，ODBC 2.0 函数 SQLFreeConnect 已替换为 SQLFreeHandle + +- API: SQLFreeEnv + - **是否支持**: 不支持 + - **标准**: 弃用 + - **作用**: 在 ODBC 3.x 中，ODBC 2.0 函数 SQLFreeEnv 已替换为 SQLFreeHandle + +- API: SQLFreeStmt + - **是否支持**: 支持 + - **标准**: ODBC + - **作用**: 结束语句处理，丢弃挂起的结果，并且可以选择释放与语句句柄关联的所有资源 + +- API: SQLCloseCursor + - **是否支持**: 支持 + - **标准**: ODBC + - **作用**: 关闭与当前语句句柄关联的游标，并释放游标所使用的所有资源 + + ## 与第三方集成 作为使用 TDengine ODBC driver 的一个示例，你可以使用 Power BI 与 TDengine 分析时序数据。更多细节请参考 [Power BI](../../../third-party/bi/powerbi) diff --git a/docs/zh/14-reference/05-connector/assets/odbc-native-config-zh.webp b/docs/zh/14-reference/05-connector/assets/odbc-native-config-zh.webp index 5589bc6cf7..ee8c48b4ef 100644 Binary files a/docs/zh/14-reference/05-connector/assets/odbc-native-config-zh.webp and b/docs/zh/14-reference/05-connector/assets/odbc-native-config-zh.webp differ diff --git a/docs/zh/14-reference/05-connector/assets/odbc-ws-config-zh.webp b/docs/zh/14-reference/05-connector/assets/odbc-ws-config-zh.webp index 6a9cece9d9..70c0fc8b8d 100644 Binary files a/docs/zh/14-reference/05-connector/assets/odbc-ws-config-zh.webp and b/docs/zh/14-reference/05-connector/assets/odbc-ws-config-zh.webp differ diff --git a/docs/zh/14-reference/05-connector/index.md b/docs/zh/14-reference/05-connector/index.md index c4e5628686..04a2ef6c1f 100644 --- a/docs/zh/14-reference/05-connector/index.md +++ b/docs/zh/14-reference/05-connector/index.md @@ -28,14 +28,14 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 TDengine 版本更新往往会增加新的功能特性，列表中的连接器版本为连接器最佳适配版本。 -| **TDengine 版本** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | -| ---------------------- | ----------- | ------------------------------------------- | ------------ | ------------- | --------------- | -------- | -| **3.3.0.0 及以上** | 3.3.0及以上 | taospy 2.7.15及以上，taos-ws-py 0.3.2及以上 | 3.5.5及以上 | 3.1.3及以上 | 3.1.0及以上 | 当前版本 | -| **3.0.0.0 及以上** | 3.0.2以上 | 当前版本 | 3.0 分支 | 3.0.0 | 3.1.0 | 当前版本 | -| **2.4.0.14 及以上** | 2.0.38 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | -| **2.4.0.4 - 2.4.0.13** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | -| **2.2.x.x ** | 2.0.36 | 当前版本 | master 分支 | n/a | 2.0.7 - 2.0.9 | 当前版本 | -| **2.0.x.x ** | 2.0.34 | 当前版本 | master 分支 | n/a | 2.0.1 - 2.0.6 | 当前版本 | +| **TDengine 版本** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | **C/C++** | +| ---------------------- | ----------- | ------------------------------------------- | ------------ | ------------- | --------------- | -------- | -------------------- | +| **3.3.0.0 及以上** | 3.3.0及以上 | taospy 2.7.15及以上，taos-ws-py 0.3.2及以上 | 3.5.5及以上 | 3.1.3及以上 | 3.1.0及以上 | 当前版本 | 与 TDengine 相同版本 | +| **3.0.0.0 及以上** | 3.0.2以上 | 当前版本 | 3.0 分支 | 3.0.0 | 3.1.0 | 当前版本 | 与 TDengine 相同版本 | +| **2.4.0.14 及以上** | 2.0.38 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | 与 TDengine 相同版本 | +| **2.4.0.4 - 2.4.0.13** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | 与 TDengine 相同版本 | +| **2.2.x.x ** | 2.0.36 | 当前版本 | master 分支 | n/a | 2.0.7 - 2.0.9 | 当前版本 | 与 TDengine 相同版本 | +| **2.0.x.x ** | 2.0.34 | 当前版本 | master 分支 | n/a | 2.0.1 - 2.0.6 | 当前版本 | 与 TDengine 相同版本 | ## 功能特性 @@ -43,13 +43,13 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 ### 使用原生接口（taosc） -| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Rust** | -| ------------------- | -------- | ---------- | ------ | ------ | -------- | -| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | -| **执行 SQL** | 支持 | 支持 | 支持 | 支持 | 支持 | -| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | -| **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 支持 | 支持 | -| **无模式写入** | 支持 | 支持 | 支持 | 支持 | 支持 | +| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Rust** | **C/C++** | +| ------------------- | -------- | ---------- | ------ | ------ | -------- | --------- | +| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **执行 SQL** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **无模式写入** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | :::info 由于不同编程语言数据库框架规范不同，并不意味着所有 C/C++ 接口都需要对应封装支持。 @@ -64,13 +64,13 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 ### 使用 Websocket 接口 -| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | -| ------------------- | -------- | ---------- | ------ | ------ | ----------- | -------- | -| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **执行 SQL** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **无模式写入** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | **C/C++** | +| ------------------- | -------- | ---------- | ------ | ------ | ----------- | -------- | --------- | +| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **执行 SQL** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **无模式写入** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | :::warning - 无论选用何种编程语言的连接器，2.0 及以上版本的 TDengine 推荐数据库应用的每个线程都建立一个独立的连接，或基于线程建立连接池，以避免连接内的“USE statement”状态量在线程之间相互干扰（但连接的查询和写入操作都是线程安全的）。 diff --git a/docs/zh/14-reference/07-supported.md b/docs/zh/14-reference/07-supported.md index 47c874c907..10ca237653 100644 --- a/docs/zh/14-reference/07-supported.md +++ b/docs/zh/14-reference/07-supported.md @@ -7,11 +7,11 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" ## TDengine 服务端支持的平台列表 | | **Windows server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18 以上** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **macOS** | -| ------------ | ---------------------------- | ----------------- | ---------------- | ---------------- | ------------ | ----------------- | ---------------- | --------- | -| X64 | ●/E | ●/E | ● | ● | ●/E | ●/E | ●/E | ● | -| 树莓派 ARM64 | | | ● | | | | | | -| 华为云 ARM64 | | | | ● | | | | | -| M1 | | | | | | | | ● | +| ------------ | ---------------------------- | ----------------- | ---------------- | ------------------ | ------------ | ----------------- | ---------------- | --------- | +| X64 | ●/E | ●/E | ● | ● | ●/E | ●/E | ●/E | ● | +| 树莓派 ARM64 | | | ● | | | | | | +| 华为云 ARM64 | | | | ● | | | | | +| M1 | | | | | | | | ● | 注：1) ● 表示经过官方测试验证， ○ 表示非官方测试验证，E 表示仅企业版支持。 2) 社区版仅支持主流操作系统的较新版本，包括 Ubuntu 18+/CentOS 7+/RedHat/Debian/CoreOS/FreeBSD/OpenSUSE/SUSE Linux/Fedora/macOS 等。如果有其他操作系统及版本的需求，请联系企业版支持。 @@ -31,6 +31,7 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" | **Go** | ● | ● | ● | ● | ● | | **NodeJs** | ● | ● | ● | ○ | ○ | | **C#** | ● | ● | ○ | ○ | ○ | +| **Rust** | ● | ● | ○ | ● | ● | | **RESTful** | ● | ● | ● | ● | ● | 注：● 表示官方测试验证通过，○ 表示非官方测试验证通过，-- 表示未经验证。 diff --git a/include/client/taos.h b/include/client/taos.h index 8b0111ae97..4bd0744c9e 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -389,7 +389,7 @@ DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES *res); DLL_EXPORT const char *tmq_err2str(int32_t code); -/* ------------------------------ TAOSX -----------------------------------*/ +/* ------------------------------ TAOSX INTERFACE -----------------------------------*/ typedef struct tmq_raw_data { void *raw; uint32_t raw_len; diff --git a/include/common/cos.h b/include/common/cos.h index 17c48d594b..b336a1e5ee 100644 --- a/include/common/cos.h +++ b/include/common/cos.h @@ -32,7 +32,6 @@ extern int32_t tsS3PageCacheSize; extern int32_t tsS3UploadDelaySec; int32_t s3Init(); -void s3CleanUp(); int32_t s3CheckCfg(); int32_t s3PutObjectFromFile(const char *file, const char *object); int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp); diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index d8a88d038d..19f3e222d1 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -115,7 +115,7 @@ typedef struct { } SValueColumnCompressInfo; int32_t tValueColumnInit(SValueColumn *valCol); -int32_t tValueColumnDestroy(SValueColumn *valCol); +void tValueColumnDestroy(SValueColumn *valCol); void tValueColumnClear(SValueColumn *valCol); int32_t tValueColumnAppend(SValueColumn *valCol, const SValue *value); int32_t tValueColumnUpdate(SValueColumn *valCol, int32_t idx, const SValue *value); @@ -153,7 +153,6 @@ char *tTagValToData(const STagVal *pTagVal, bool isJson); int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag); int32_t tTagToValArray(const STag *pTag, SArray **ppArray); -void tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid); void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bc61e21d16..1a10f02c96 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1684,6 +1684,7 @@ typedef struct { typedef struct { int32_t openVnodes; + int32_t dropVnodes; int32_t totalVnodes; int32_t masterNum; int64_t numOfSelectReqs; @@ -2822,8 +2823,8 @@ enum { TOPIC_SUB_TYPE__COLUMN, }; -#define DEFAULT_MAX_POLL_INTERVAL 300000 -#define DEFAULT_SESSION_TIMEOUT 12000 +#define DEFAULT_MAX_POLL_INTERVAL 300000 +#define DEFAULT_SESSION_TIMEOUT 12000 typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; // accout.topic @@ -4048,37 +4049,39 @@ void tDeleteMqMetaRsp(SMqMetaRsp* pRsp); #define MQ_DATA_RSP_VERSION 100 typedef struct { - SMqRspHead head; - STqOffsetVal reqOffset; - STqOffsetVal rspOffset; - int32_t blockNum; - int8_t withTbName; - int8_t withSchema; - SArray* blockDataLen; - SArray* blockData; - SArray* blockTbName; - SArray* blockSchema; -} SMqDataRspCommon; + struct { + SMqRspHead head; + STqOffsetVal rspOffset; + STqOffsetVal reqOffset; + int32_t blockNum; + int8_t withTbName; + int8_t withSchema; + SArray* blockDataLen; + SArray* blockData; + SArray* blockTbName; + SArray* blockSchema; + }; + + union{ + struct{ + int64_t sleepTime; + }; + struct{ + int32_t createTableNum; + SArray* createTableLen; + SArray* createTableReq; + }; + }; -typedef struct { - SMqDataRspCommon common; - int64_t sleepTime; } SMqDataRsp; -int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const void* pRsp); -int32_t tDecodeMqDataRsp(SDecoder* pDecoder, void* pRsp); -void tDeleteMqDataRsp(void* pRsp); +int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pObj); +int32_t tDecodeMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); +void tDeleteMqDataRsp(SMqDataRsp* pRsp); -typedef struct { - SMqDataRspCommon common; - int32_t createTableNum; - SArray* createTableLen; - SArray* createTableReq; -} STaosxRsp; - -int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const void* pRsp); -int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, void* pRsp); -void tDeleteSTaosxRsp(void* pRsp); +int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); +int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); +void tDeleteSTaosxRsp(SMqDataRsp* pRsp); typedef struct SMqBatchMetaRsp { SMqRspHead head; // not serialize @@ -4163,6 +4166,7 @@ typedef struct { typedef struct { SArray* topicPrivileges; // SArray + int32_t debugFlag; } SMqHbRsp; typedef struct { diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 7a845e43c3..8e88a1a278 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -382,7 +382,7 @@ typedef struct SStateStore { int32_t (*streamStateCountWinAddIfNotExist)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal, int32_t* pVLen, int32_t* pWinCode); - int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); + int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen); int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); diff --git a/include/libs/function/function.h b/include/libs/function/function.h index a71a2a6b7f..ec01cf1f6f 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -258,6 +258,7 @@ typedef struct SqlFunctionCtx { SFuncInputRowIter rowIter; bool bInputFinished; bool hasWindowOrGroup; // denote that the function is used with time window or group + bool needCleanup; // denote that the function need to be cleaned up } SqlFunctionCtx; typedef struct tExprNode { diff --git a/include/libs/monitorfw/taos_collector_registry.h b/include/libs/monitorfw/taos_collector_registry.h index 81d1f8050c..5ac849530f 100644 --- a/include/libs/monitorfw/taos_collector_registry.h +++ b/include/libs/monitorfw/taos_collector_registry.h @@ -37,9 +37,9 @@ extern taos_collector_registry_t *TAOS_COLLECTOR_REGISTRY_DEFAULT; /** * @brief Initializes the default collector registry and enables metric collection on the executing process - * @return A non-zero integer value upon failure + * @return */ -int taos_collector_registry_default_init(void); +void taos_collector_registry_default_init(void); /** * @brief Constructs a taos_collector_registry_t* diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 7271da8ff6..c98b188d49 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -176,7 +176,7 @@ int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsS STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int32_t msgBufLen); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); -int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD* fields, +int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD* fields, int numFields, bool needChangeLength, char* errstr, int32_t errstrLen); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 46874b7c65..f9469a449d 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -87,7 +87,7 @@ void streamStateFreeVal(void* val); // count window int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal, int32_t* pVLen, int32_t* pWinCode); -int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); +int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen); SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key); SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index cb10aeb6a0..e6d750468e 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -70,6 +70,8 @@ typedef struct SActiveCheckpointInfo SActiveCheckpointInfo; #define SSTREAM_TASK_NEED_CONVERT_VER 2 #define SSTREAM_TASK_SUBTABLE_CHANGED_VER 3 +extern int32_t streamMetaId; + enum { STREAM_STATUS__NORMAL = 0, STREAM_STATUS__STOP, @@ -135,11 +137,6 @@ enum { STREAM_QUEUE__PROCESSING, }; -enum { - STREAM_META_WILL_STOP = 1, - STREAM_META_OK_TO_STOP = 2, -}; - typedef enum EStreamTaskEvent { TASK_EVENT_INIT = 0x1, TASK_EVENT_INIT_SCANHIST = 0x2, @@ -282,7 +279,6 @@ typedef enum { } EConsenChkptStatus; typedef struct SConsenChkptInfo { -// bool alreadySendChkptId; EConsenChkptStatus status; int64_t statusTs; int32_t consenChkptTransId; diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index 6f1a1b3b98..a265ae7e60 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -110,7 +110,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch // count window int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen, int32_t* pWinCode); -int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen); +int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen); // function int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen, diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 07d56f9b07..50c096258e 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -47,7 +47,7 @@ extern "C" { #define SYNC_HEARTBEAT_SLOW_MS 1500 #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 #define SYNC_SNAP_RESEND_MS 1000 * 60 -#define SYNC_SNAP_TIMEOUT_MS 1000 * 300 +#define SYNC_SNAP_TIMEOUT_MS 1000 * 180 #define SYNC_VND_COMMIT_MIN_MS 3000 diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 5a2cf3a3a0..74ab0bf484 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -177,7 +177,7 @@ int32_t walRollback(SWal *, int64_t ver); int32_t walBeginSnapshot(SWal *, int64_t ver, int64_t logRetention); int32_t walEndSnapshot(SWal *); int32_t walRestoreFromSnapshot(SWal *, int64_t ver); -int32_t walApplyVer(SWal *, int64_t ver); +void walApplyVer(SWal *, int64_t ver); // wal reader SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond, int64_t id); diff --git a/include/os/osDef.h b/include/os/osDef.h index 439f4b5c6a..ff30265afa 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -76,7 +76,6 @@ typedef int (*__compar_fn_t)(const void *, const void *); char *strsep(char **stringp, const char *delim); char *getpass(const char *prefix); -char *strndup(const char *s, int n); // for send function in tsocket.c #define MSG_NOSIGNAL 0 diff --git a/include/os/osString.h b/include/os/osString.h index a64fb34f1e..30bfd61b62 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -51,6 +51,12 @@ typedef enum { M2C = 0, C2M } ConvType; #define strtod STR_TO_LD_FUNC_TAOS_FORBID #define strtold STR_TO_D_FUNC_TAOS_FORBID #define strtof STR_TO_F_FUNC_TAOS_FORBID + +#ifdef strndup +#undef strndup +#endif +#define strndup STR_TO_F_FUNC_TAOS_FORBID + #endif #define tstrncpy(dst, src, size) \ @@ -101,8 +107,9 @@ int8_t taosStr2Int8(const char *str, char **pEnd, int32_t radix); uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix); double taosStr2Double(const char *str, char **pEnd); float taosStr2Float(const char *str, char **pEnd); -int32_t taosHex2Ascii(const char *z, uint32_t n, void** data, uint32_t* size); -int32_t taosAscii2Hex(const char *z, uint32_t n, void** data, uint32_t* size); +int32_t taosHex2Ascii(const char *z, uint32_t n, void **data, uint32_t *size); +int32_t taosAscii2Hex(const char *z, uint32_t n, void **data, uint32_t *size); +char *taosStrndup(const char *s, int n); //int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size); bool isHex(const char* z, uint32_t n); bool isValidateHex(const char* z, uint32_t n); diff --git a/include/util/tcompression.h b/include/util/tcompression.h index cea648a3a6..fef6c0713c 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -72,8 +72,8 @@ extern "C" { #ifdef TD_TSZ extern bool lossyFloat; extern bool lossyDouble; -int32_t tsCompressInit(char *lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals, - uint32_t intervals, int32_t ifAdtFse, const char *compressor); +void tsCompressInit(char *lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals, uint32_t intervals, + int32_t ifAdtFse, const char *compressor); void tsCompressExit(); @@ -153,11 +153,10 @@ int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int int32_t getWordLength(char type); int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type); -int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output); -int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output); -int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, - bool bigEndian); -int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, bool bigEndian); +void tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output); +void tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output); +void tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, bool bigEndian); +void tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, bool bigEndian); /************************************************************************* * REGULAR COMPRESSION 2 @@ -268,7 +267,7 @@ typedef struct { uint8_t lvl[3]; // l[0] = 'low', l[1] = 'mid', l[2] = 'high' } TCmprLvlSet; -int32_t tcompressDebug(uint32_t cmprAlg, uint8_t *l1Alg, uint8_t *l2Alg, uint8_t *level); +void tcompressDebug(uint32_t cmprAlg, uint8_t *l1Alg, uint8_t *l2Alg, uint8_t *level); #define DEFINE_VAR(cmprAlg) \ uint8_t l1 = COMPRESS_L1_TYPE_U32(cmprAlg); \ diff --git a/include/util/tlog.h b/include/util/tlog.h index 832dc7dbc1..e80e94de32 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -69,6 +69,8 @@ extern int32_t tdbDebugFlag; extern int32_t sndDebugFlag; extern int32_t simDebugFlag; +extern int32_t tqClientDebug; + int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc); void taosCloseLog(); void taosResetLog(); diff --git a/include/util/tlrucache.h b/include/util/tlrucache.h index e5e59d0876..97c51b5b88 100644 --- a/include/util/tlrucache.h +++ b/include/util/tlrucache.h @@ -25,6 +25,7 @@ extern "C" { typedef struct SLRUCache SLRUCache; typedef void (*_taos_lru_deleter_t)(const void *key, size_t keyLen, void *value, void *ud); +typedef void (*_taos_lru_overwriter_t)(const void *key, size_t keyLen, void *value, void *ud); typedef int (*_taos_lru_functor_t)(const void *key, size_t keyLen, void *value, void *ud); typedef struct LRUHandle LRUHandle; @@ -42,7 +43,8 @@ SLRUCache *taosLRUCacheInit(size_t capacity, int numShardBits, double highPriPoo void taosLRUCacheCleanup(SLRUCache *cache); LRUStatus taosLRUCacheInsert(SLRUCache *cache, const void *key, size_t keyLen, void *value, size_t charge, - _taos_lru_deleter_t deleter, LRUHandle **handle, LRUPriority priority, void *ud); + _taos_lru_deleter_t deleter, _taos_lru_overwriter_t overwriter, LRUHandle **handle, + LRUPriority priority, void *ud); LRUHandle *taosLRUCacheLookup(SLRUCache *cache, const void *key, size_t keyLen); void taosLRUCacheErase(SLRUCache *cache, const void *key, size_t keyLen); diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 6643363339..9ba9529146 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -116,23 +116,14 @@ function install_bin() { ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/* #Make link - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : + [ -x ${install_main_dir}/bin/${clientName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName2} ${bin_link_dir}/${clientName2} || : if [ "$osType" != "Darwin" ]; then [ -x ${install_main_dir}/bin/${demoName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${demoName2} ${bin_link_dir}/${demoName2} || : fi [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript} || : - [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : - - if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then - #Make link - [ -x ${install_main_dir}/bin/${clientName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName2} ${bin_link_dir}/${clientName2} || : - if [ "$osType" != "Darwin" ]; then - [ -x ${install_main_dir}/bin/${demoName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${demoName2} ${bin_link_dir}/${demoName2} || : - [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2} || : - [ -x ${install_main_dir}/bin/${dumpName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName2} ${bin_link_dir}/${dumpName2} || : - fi - [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript2} || : - fi + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : + [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2} || : + [ -x ${install_main_dir}/bin/${dumpName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName2} ${bin_link_dir}/${dumpName2} || : } function clean_lib() { diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index ea19125bf5..13447bd5e4 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -616,8 +616,8 @@ function update_TDengine() { [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}" else - echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" - echo -e "${GREEN_DARK}To start Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" + echo -e "${GREEN_DARK}To start service ${NC}: sudo launchctl start com.tdengine.taosd${NC}" + echo -e "${GREEN_DARK}To start Adapter ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}" fi fi @@ -668,8 +668,8 @@ function install_TDengine() { [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}" else - echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" - echo -e "${GREEN_DARK}To start Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" + echo -e "${GREEN_DARK}To start service ${NC}: sudo launchctl start com.tdengine.taosd${NC}" + echo -e "${GREEN_DARK}To start Adapter ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}" fi fi diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 9811003254..8d45e8b4a8 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -26,10 +26,10 @@ extern "C" { #include "query.h" #include "taos.h" #include "tcommon.h" -#include "tmisce.h" #include "tdef.h" #include "thash.h" #include "tlist.h" +#include "tmisce.h" #include "tmsg.h" #include "tmsgtype.h" #include "trpc.h" @@ -86,7 +86,7 @@ typedef struct { int8_t threadStop; int8_t quitByKill; TdThread thread; - TdThreadMutex lock; // used when app init and cleanup + TdThreadMutex lock; // used when app init and cleanup SHashObj* appSummary; SHashObj* appHbHash; // key: clusterId SArray* appHbMgrs; // SArray one for each cluster @@ -95,11 +95,11 @@ typedef struct { } SClientHbMgr; typedef struct SQueryExecMetric { - int64_t start; // start timestamp, us - int64_t ctgStart; // start to parse, us - int64_t execStart; // start to parse, us + int64_t start; // start timestamp, us + int64_t ctgStart; // start to parse, us + int64_t execStart; // start to parse, us - int64_t parseCostUs; + int64_t parseCostUs; int64_t ctgCostUs; int64_t analyseCostUs; int64_t planCostUs; @@ -193,7 +193,7 @@ typedef struct SReqResultInfo { char** convertBuf; TAOS_ROW row; SResultColumn* pCol; - uint64_t numOfRows; // from int32_t change to int64_t + uint64_t numOfRows; // from int32_t change to int64_t uint64_t totalRows; uint64_t current; bool localResultFetched; @@ -226,31 +226,17 @@ typedef struct { SSchemaWrapper schema; int32_t resIter; SReqResultInfo resInfo; -} SMqRspObjCommon; - -typedef struct { - SMqRspObjCommon common; - SMqDataRsp rsp; + union{ + struct{ + SMqRspHead head; + STqOffsetVal rspOffset; + }; + SMqDataRsp dataRsp; + SMqMetaRsp metaRsp; + SMqBatchMetaRsp batchMetaRsp; + }; } SMqRspObj; -typedef struct { - int8_t resType; - char topic[TSDB_TOPIC_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int32_t vgId; - SMqMetaRsp metaRsp; -} SMqMetaRspObj; - -typedef struct { - SMqRspObjCommon common; - STaosxRsp rsp; -} SMqTaosxRspObj; - -typedef struct { - SMqRspObjCommon common; - SMqBatchMetaRsp rsp; -} SMqBatchMetaRspObj; - typedef struct SReqRelInfo { uint64_t userRefId; uint64_t prevRefId; @@ -319,21 +305,23 @@ void syncCatalogFn(SMetaData* pResult, void* param, int32_t code); TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t source); TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly, int64_t reqid); -void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source); +void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, + int8_t source); void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int64_t reqid); -void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param); -int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effectiveUser, SParseSqlRes* pRes); -void syncQueryFn(void* param, void* res, int32_t code); +void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param); +int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effectiveUser, + SParseSqlRes* pRes); +void syncQueryFn(void* param, void* res, int32_t code); int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols); static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; - return (SReqResultInfo*)&msg->common.resInfo; + return (SReqResultInfo*)&msg->resInfo; } -int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pResInfo); +int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pResInfo); static FORCE_INLINE SReqResultInfo* tscGetCurResInfo(TAOS_RES* res) { if (TD_RES_QUERY(res)) return &(((SRequestObj*)res)->body.resInfo); return tmqGetCurResInfo(res); @@ -349,8 +337,8 @@ __async_send_cb_fn_t getMsgRspHandle(int32_t msgType); SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); -int32_t createTscObj(const char *user, const char *auth, const char *db, int32_t connType, SAppInstInfo *pAppInfo, - STscObj **p); +int32_t createTscObj(const char* user, const char* auth, const char* db, int32_t connType, SAppInstInfo* pAppInfo, + STscObj** p); void destroyTscObj(void* pObj); STscObj* acquireTscObj(int64_t rid); void releaseTscObj(int64_t rid); @@ -358,7 +346,7 @@ void destroyAppInst(void* pAppInfo); uint64_t generateRequestId(); -int32_t createRequest(uint64_t connId, int32_t type, int64_t reqid, SRequestObj **pRequest); +int32_t createRequest(uint64_t connId, int32_t type, int64_t reqid, SRequestObj** pRequest); void destroyRequest(SRequestObj* pRequest); SRequestObj* acquireRequest(int64_t rid); int32_t releaseRequest(int64_t rid); @@ -372,9 +360,9 @@ void resetConnectDB(STscObj* pTscObj); int taos_options_imp(TSDB_OPTION option, const char* str); -int32_t openTransporter(const char* user, const char* auth, int32_t numOfThreads, void **pDnodeConn); -void tscStopCrashReport(); -void cleanupAppInfo(); +int32_t openTransporter(const char* user, const char* auth, int32_t numOfThreads, void** pDnodeConn); +void tscStopCrashReport(); +void cleanupAppInfo(); typedef struct AsyncArg { SRpcMsg msg; @@ -402,17 +390,17 @@ int32_t hbMgrInit(); void hbMgrCleanUp(); // cluster level -int32_t appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key, SAppHbMgr **pAppHbMgr); -void appHbMgrCleanup(void); -void hbRemoveAppHbMrg(SAppHbMgr** pAppHbMgr); -void destroyAllRequests(SHashObj* pRequests); -void stopAllRequests(SHashObj* pRequests); +int32_t appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key, SAppHbMgr** pAppHbMgr); +void appHbMgrCleanup(void); +void hbRemoveAppHbMrg(SAppHbMgr** pAppHbMgr); +void destroyAllRequests(SHashObj* pRequests); +void stopAllRequests(SHashObj* pRequests); -//SAppInstInfo* getAppInstInfo(const char* clusterKey); +// SAppInstInfo* getAppInstInfo(const char* clusterKey); // conn level int32_t hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); -void hbDeregisterConn(STscObj* pTscObj, SClientHbKey connKey); +void hbDeregisterConn(STscObj* pTscObj, SClientHbKey connKey); typedef struct SSqlCallbackWrapper { SParseContext* pParseCtx; @@ -421,9 +409,9 @@ typedef struct SSqlCallbackWrapper { void* pPlanInfo; } SSqlCallbackWrapper; -void setQueryRequest(int64_t rId); -SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res); -int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList); +void setQueryRequest(int64_t rId); +void launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res); +int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList); void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta, SSqlCallbackWrapper* pWrapper); int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest); int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList); @@ -431,20 +419,21 @@ void doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta); int32_t removeMeta(STscObj* pTscObj, SArray* tbList, bool isView); int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog); int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog); -int32_t qnodeRequired(SRequestObj* pRequest, bool *required); +int32_t qnodeRequired(SRequestObj* pRequest, bool* required); void continueInsertFromCsv(SSqlCallbackWrapper* pWrapper, SRequestObj* pRequest); void destorySqlCallbackWrapper(SSqlCallbackWrapper* pWrapper); -void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, int32_t code); -void restartAsyncQuery(SRequestObj *pRequest, int32_t code); -int32_t buildPreviousRequest(SRequestObj *pRequest, const char* sql, SRequestObj** pNewRequest); -int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce); +void handleQueryAnslyseRes(SSqlCallbackWrapper* pWrapper, SMetaData* pResultMeta, int32_t code); +void restartAsyncQuery(SRequestObj* pRequest, int32_t code); +int32_t buildPreviousRequest(SRequestObj* pRequest, const char* sql, SRequestObj** pNewRequest); +int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper** ppWrapper, SRequestObj* pRequest, bool updateMetaForce); void returnToUser(SRequestObj* pRequest); -void stopAllQueries(SRequestObj *pRequest); +void stopAllQueries(SRequestObj* pRequest); void doRequestCallback(SRequestObj* pRequest, int32_t code); void freeQueryParam(SSyncQueryParam* param); #ifdef TD_ENTERPRISE -int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effeciveUser, SParseSqlRes* pRes); +int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effeciveUser, + SParseSqlRes* pRes); #endif #define TSC_ERR_RET(c) \ @@ -474,13 +463,9 @@ int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, boo void slowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost); -enum { - MONITORSQLTYPESELECT = 0, - MONITORSQLTYPEINSERT = 1, - MONITORSQLTYPEDELETE = 2 -}; +enum { MONITORSQLTYPESELECT = 0, MONITORSQLTYPEINSERT = 1, MONITORSQLTYPEDELETE = 2 }; -void sqlReqLog(int64_t rid, bool killed, int32_t code, int8_t type); +void sqlReqLog(int64_t rid, bool killed, int32_t code, int8_t type); void tmqMgmtClose(void); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 53be1fccf4..3b755c2921 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -197,7 +197,12 @@ static int32_t generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString(""))); } - char *value = cJSON_PrintUnformatted(json); + char *value = cJSON_PrintUnformatted(json); + if (value == NULL) { + tscError("failed to print json"); + code = TSDB_CODE_FAILED; + goto _end; + } MonitorSlowLogData data = {0}; data.clusterId = pTscObj->pAppInfo->clusterId; data.type = SLOW_LOG_WRITE; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 18cdd58639..15bd5795e2 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -679,9 +679,15 @@ _return: return TSDB_CODE_SUCCESS; } +void freeVgList(void* list) { + SArray* pList = *(SArray**)list; + taosArrayDestroy(pList); +} + int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SMetaData* pResultMeta) { SArray* pDbVgList = NULL; SArray* pQnodeList = NULL; + FDelete fp = NULL; int32_t code = 0; switch (tsQueryPolicy) { @@ -705,6 +711,43 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray goto _return; } } + } else { + fp = freeVgList; + + int32_t dbNum = taosArrayGetSize(pRequest->dbList); + if (dbNum > 0) { + SCatalog* pCtg = NULL; + SAppInstInfo* pInst = pRequest->pTscObj->pAppInfo; + code = catalogGetHandle(pInst->clusterId, &pCtg); + if (code != TSDB_CODE_SUCCESS) { + goto _return; + } + + pDbVgList = taosArrayInit(dbNum, POINTER_BYTES); + if (NULL == pDbVgList) { + code = terrno; + goto _return; + } + SArray* pVgList = NULL; + for (int32_t i = 0; i < dbNum; ++i) { + char* dbFName = taosArrayGet(pRequest->dbList, i); + SRequestConnInfo conn = {.pTrans = pInst->pTransporter, + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pInst->mgmtEp)}; + + // catalogGetDBVgList will handle dbFName == null. + code = catalogGetDBVgList(pCtg, &conn, dbFName, &pVgList); + if (code) { + goto _return; + } + + if (NULL == taosArrayPush(pDbVgList, &pVgList)) { + code = terrno; + goto _return; + } + } + } } code = buildVnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pDbVgList); @@ -745,17 +788,12 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray } _return: - taosArrayDestroy(pDbVgList); + taosArrayDestroyEx(pDbVgList, fp); taosArrayDestroy(pQnodeList); return code; } -void freeVgList(void* list) { - SArray* pList = *(SArray**)list; - taosArrayDestroy(pList); -} - int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList) { SArray* pDbVgList = NULL; SArray* pQnodeList = NULL; @@ -1211,7 +1249,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { } } -SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res) { +void launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res) { int32_t code = 0; if (pQuery->pRoot) { @@ -1297,8 +1335,6 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue *res = pRequest->body.resInfo.execRes.res; pRequest->body.resInfo.execRes.res = NULL; } - - return pRequest; } static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta, @@ -2405,10 +2441,9 @@ char* getDbOfConnection(STscObj* pObj) { (void)taosThreadMutexLock(&pObj->mutex); size_t len = strlen(pObj->db); if (len > 0) { - p = strndup(pObj->db, tListLen(pObj->db)); + p = taosStrndup(pObj->db, tListLen(pObj->db)); if (p == NULL) { - tscError("failed to strndup db name"); - terrno = TSDB_CODE_OUT_OF_MEMORY; + tscError("failed to taosStrndup db name"); } } @@ -2895,6 +2930,10 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t s taosMemoryFree(param); return NULL; } + code = tsem_destroy(¶m->sem); + if (TSDB_CODE_SUCCESS != code) { + tscError("failed to destroy semaphore since %s", tstrerror(code)); + } SRequestObj* pRequest = NULL; if (param->pRequest != NULL) { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 6957874642..48bf8d129c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -373,25 +373,22 @@ void taos_free_result(TAOS_RES *res) { SRequestObj *pRequest = (SRequestObj *)res; tscDebug("0x%" PRIx64 " taos_free_result start to free query", pRequest->requestId); destroyRequest(pRequest); - } else if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj *pRsp = (SMqTaosxRspObj *)res; - tDeleteSTaosxRsp(&pRsp->rsp); - doFreeReqResultInfo(&pRsp->common.resInfo); - taosMemoryFree(pRsp); - } else if (TD_RES_TMQ(res)) { - SMqRspObj *pRsp = (SMqRspObj *)res; - tDeleteMqDataRsp(&pRsp->rsp); - doFreeReqResultInfo(&pRsp->common.resInfo); - taosMemoryFree(pRsp); - } else if (TD_RES_TMQ_META(res)) { - SMqMetaRspObj *pRspObj = (SMqMetaRspObj *)res; - tDeleteMqMetaRsp(&pRspObj->metaRsp); - taosMemoryFree(pRspObj); - } else if (TD_RES_TMQ_BATCH_META(res)) { - SMqBatchMetaRspObj *pBtRspObj = (SMqBatchMetaRspObj *)res; - tDeleteMqBatchMetaRsp(&pBtRspObj->rsp); - taosMemoryFree(pBtRspObj); + return; } + SMqRspObj *pRsp = (SMqRspObj *)res; + if (TD_RES_TMQ(res)) { + tDeleteMqDataRsp(&pRsp->dataRsp); + doFreeReqResultInfo(&pRsp->resInfo); + } else if (TD_RES_TMQ_METADATA(res)) { + tDeleteSTaosxRsp(&pRsp->dataRsp); + doFreeReqResultInfo(&pRsp->resInfo); + } else if (TD_RES_TMQ_META(res)) { + tDeleteMqMetaRsp(&pRsp->metaRsp); + } else if (TD_RES_TMQ_BATCH_META(res)) { + tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp); + } + taosMemoryFree(pRsp); + } void taos_kill_query(TAOS *taos) { @@ -454,7 +451,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { SMqRspObj *msg = ((SMqRspObj *)res); SReqResultInfo *pResultInfo = NULL; - if (msg->common.resIter == -1) { + if (msg->resIter == -1) { if (tmqGetNextResInfo(res, true, &pResultInfo) != 0) { return NULL; } @@ -1316,9 +1313,9 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d,QID:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); code = refreshMeta(pRequest->pTscObj, pRequest); - if (code != 0){ - tscWarn("0x%" PRIx64 " refresh meta failed, code:%d - %s,QID:0x%" PRIx64, pRequest->self, code, - tstrerror(code), pRequest->requestId); + if (code != 0) { + tscWarn("0x%" PRIx64 " refresh meta failed, code:%d - %s,QID:0x%" PRIx64, pRequest->self, code, tstrerror(code), + pRequest->requestId); } pRequest->prevCode = code; doAsyncQuery(pRequest, true); @@ -1985,7 +1982,9 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col STscStmt2 *pStmt = (STscStmt2 *)stmt; if (pStmt->options.asyncExecFn && !pStmt->semWaited) { - (void)tsem_wait(&pStmt->asyncQuerySem); + if (tsem_wait(&pStmt->asyncQuerySem) != 0) { + tscError("wait async query sem failed"); + } pStmt->semWaited = true; } diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index f424e4d1a2..6667c4c741 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -447,20 +447,19 @@ static char* readFile(TdFilePtr pFile, int64_t* offset, int64_t size) { char* pCont = NULL; int64_t totalSize = 0; if (size - *offset >= SLOW_LOG_SEND_SIZE_MAX) { - pCont = taosMemoryCalloc(1, 4 + SLOW_LOG_SEND_SIZE_MAX); // 4 reserved for [] totalSize = 4 + SLOW_LOG_SEND_SIZE_MAX; } else { - pCont = taosMemoryCalloc(1, 4 + (size - *offset)); totalSize = 4 + (size - *offset); } + pCont = taosMemoryCalloc(1, totalSize); // 4 reserved for [] if (pCont == NULL) { tscError("failed to allocate memory for slow log, size:%" PRId64, totalSize); return NULL; } char* buf = pCont; (void)strcat(buf++, "["); - int64_t readSize = taosReadFile(pFile, buf, SLOW_LOG_SEND_SIZE_MAX); + int64_t readSize = taosReadFile(pFile, buf, totalSize - 4); // 4 reserved for [] if (readSize <= 0) { if (readSize < 0) { tscError("failed to read len from file:%p since %s", pFile, terrstr()); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 856f4f59c1..17ebd9b558 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -47,7 +47,7 @@ } \ } while (0) -#define LOG_ID_TAG "connId:0x%" PRIx64 ",qid:0x%" PRIx64 +#define LOG_ID_TAG "connId:0x%" PRIx64 ",QID:0x%" PRIx64 #define LOG_ID_VALUE *(int64_t*)taos, pRequest->requestId #define TMQ_META_VERSION "1.0" @@ -360,8 +360,8 @@ static void processCreateStb(SMqMetaRsp* metaRsp, cJSON** pJson) { } buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE, &req.colCmpr, pJson); - end: - uDebug("create stable return, sql json:%s", cJSON_PrintUnformatted(*pJson)); +end: + uDebug("create stable return"); tDecoderClear(&coder); } @@ -380,8 +380,8 @@ static void processAlterStb(SMqMetaRsp* metaRsp, cJSON** pJson) { } buildAlterSTableJson(req.alterOriData, req.alterOriDataLen, pJson); - end: - uDebug("alter stable return, sql json:%s", cJSON_PrintUnformatted(*pJson)); +end: + uDebug("alter stable return"); tDecoderClear(&coder); } @@ -393,8 +393,8 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { int64_t id = pCreateReq->uid; uint8_t tagNum = pCreateReq->ctb.tagNum; int32_t code = 0; - - cJSON* tableName = cJSON_CreateString(name); + cJSON* tags = NULL; + cJSON* tableName = cJSON_CreateString(name); RAW_NULL_CHECK(tableName); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableName", tableName)); cJSON* using = cJSON_CreateString(sname); @@ -404,7 +404,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { RAW_NULL_CHECK(tagNumJson); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tagNum", tagNumJson)); - cJSON* tags = cJSON_CreateArray(); + tags = cJSON_CreateArray(); RAW_NULL_CHECK(tags); SArray* pTagVals = NULL; RAW_RETURN_CHECK(tTagToValArray(pTag, &pTagVals)); @@ -417,7 +417,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { } char* pJson = NULL; parseTagDatatoJson(pTag, &pJson); - if(pJson == NULL) { + if (pJson == NULL) { uError("parseTagDatatoJson failed, pJson == NULL"); goto end; } @@ -542,13 +542,13 @@ static void processCreateTable(SMqMetaRsp* metaRsp, cJSON** pJson) { } } - end: - uDebug("create table return, sql json:%s", cJSON_PrintUnformatted(*pJson)); +end: + uDebug("create table return"); tDeleteSVCreateTbBatchReq(&req); tDecoderClear(&decoder); } -static void processAutoCreateTable(STaosxRsp* rsp, char** string) { +static void processAutoCreateTable(SMqDataRsp* rsp, char** string) { SDecoder* decoder = NULL; SVCreateTbReq* pCreateReq = NULL; int32_t code = 0; @@ -731,7 +731,7 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { goto end; } parseTagDatatoJson(vAlterTbReq.pTagVal, &buf); - if(buf == NULL) { + if (buf == NULL) { uError("parseTagDatatoJson failed, buf == NULL"); goto end; } @@ -771,8 +771,8 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { break; } - end: - uDebug("alter table return, sql json:%s", cJSON_PrintUnformatted(json)); +end: + uDebug("alter table return"); tDecoderClear(&decoder); *pJson = json; } @@ -806,8 +806,8 @@ static void processDropSTable(SMqMetaRsp* metaRsp, cJSON** pJson) { RAW_NULL_CHECK(tableName); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableName", tableName)); - end: - uDebug("processDropSTable return, sql json:%s", cJSON_PrintUnformatted(json)); +end: + uDebug("processDropSTable return"); tDecoderClear(&decoder); *pJson = json; } @@ -842,8 +842,8 @@ static void processDeleteTable(SMqMetaRsp* metaRsp, cJSON** pJson) { RAW_NULL_CHECK(sqlJson); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", sqlJson)); - end: - uDebug("processDeleteTable return, sql json:%s", cJSON_PrintUnformatted(json)); +end: + uDebug("processDeleteTable return"); tDecoderClear(&coder); *pJson = json; } @@ -879,8 +879,8 @@ static void processDropTable(SMqMetaRsp* metaRsp, cJSON** pJson) { } RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableNameList", tableNameList)); - end: - uDebug("processDropTable return, json sql:%s", cJSON_PrintUnformatted(json)); +end: + uDebug("processDropTable return"); tDecoderClear(&decoder); *pJson = json; } @@ -978,7 +978,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { pQuery.msgType = pQuery.pCmdMsg->msgType; pQuery.stableQuery = true; - (void)launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest + launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest taosMemoryFree(pCmdMsg.pMsg); @@ -1082,7 +1082,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { pQuery.msgType = pQuery.pCmdMsg->msgType; pQuery.stableQuery = true; - (void)launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest + launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest taosMemoryFree(pCmdMsg.pMsg); if (pRequest->code == TSDB_CODE_SUCCESS) { // ignore the error code @@ -1120,7 +1120,8 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { SRequestObj* pRequest = NULL; SQuery* pQuery = NULL; SHashObj* pVgroupHashmap = NULL; - + SArray* pTagList = taosArrayInit(0, POINTER_BYTES); + RAW_NULL_CHECK(pTagList); RAW_RETURN_CHECK(buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0)); uDebug(LOG_ID_TAG " create table, meta:%p, metaLen:%d", LOG_ID_VALUE, meta, metaLen); @@ -1186,6 +1187,14 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { } pCreateReq->ctb.suid = pTableMeta->uid; + SArray* pTagVals = NULL; + code = tTagToValArray((STag*)pCreateReq->ctb.pTag, &pTagVals); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pTableMeta); + goto end; + } + + bool rebuildTag = false; for (int32_t i = 0; i < taosArrayGetSize(pCreateReq->ctb.tagName); i++) { char* tName = taosArrayGet(pCreateReq->ctb.tagName, i); if (tName == NULL) { @@ -1195,11 +1204,35 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { j < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; j++) { SSchema* tag = &pTableMeta->schema[j]; if (strcmp(tag->name, tName) == 0 && tag->type != TSDB_DATA_TYPE_JSON) { - tTagSetCid((STag*)pCreateReq->ctb.pTag, i, tag->colId); + STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, i); + if (pTagVal) { + if (pTagVal->cid != tag->colId) { + pTagVal->cid = tag->colId; + rebuildTag = true; + } + } else { + uError("create tb invalid data %s, size:%d index:%d cid:%d", pCreateReq->name, + (int)taosArrayGetSize(pTagVals), i, tag->colId); + } } } } taosMemoryFreeClear(pTableMeta); + if (rebuildTag) { + STag* ppTag = NULL; + code = tTagNew(pTagVals, 1, false, &ppTag); + taosArrayDestroy(pTagVals); + pTagVals = NULL; + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + if (NULL == taosArrayPush(pTagList, &ppTag)) { + tTagFree(ppTag); + goto end; + } + pCreateReq->ctb.pTag = (uint8_t*)ppTag; + } + taosArrayDestroy(pTagVals); } RAW_NULL_CHECK(taosArrayPush(pRequest->tableList, &pName)); @@ -1236,7 +1269,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { RAW_RETURN_CHECK(rewriteToVnodeModifyOpStmt(pQuery, pBufArray)); - (void)launchQueryImpl(pRequest, pQuery, true, NULL); + launchQueryImpl(pRequest, pQuery, true, NULL); if (pRequest->code == TSDB_CODE_SUCCESS) { RAW_RETURN_CHECK(removeMeta(pTscObj, pRequest->tableList, false)); } @@ -1251,6 +1284,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { destroyRequest(pRequest); tDecoderClear(&coder); qDestroyQuery(pQuery); + taosArrayDestroyP(pTagList, taosMemoryFree); return code; } @@ -1365,7 +1399,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { if (TSDB_CODE_SUCCESS != code) goto end; RAW_RETURN_CHECK(rewriteToVnodeModifyOpStmt(pQuery, pBufArray)); - (void)launchQueryImpl(pRequest, pQuery, true, NULL); + launchQueryImpl(pRequest, pQuery, true, NULL); if (pRequest->code == TSDB_CODE_SUCCESS) { RAW_RETURN_CHECK(removeMeta(pTscObj, pRequest->tableList, false)); } @@ -1510,7 +1544,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { if (TSDB_CODE_SUCCESS != code) goto end; RAW_RETURN_CHECK(rewriteToVnodeModifyOpStmt(pQuery, pArray)); - (void)launchQueryImpl(pRequest, pQuery, true, NULL); + launchQueryImpl(pRequest, pQuery, true, NULL); pVgData = NULL; pArray = NULL; @@ -1587,7 +1621,7 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0)); RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash)); - (void)launchQueryImpl(pRequest, pQuery, true, NULL); + launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; end: @@ -1647,7 +1681,7 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0)); RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash)); - (void)launchQueryImpl(pRequest, pQuery, true, NULL); + launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; end: @@ -1670,119 +1704,119 @@ static void* getRawDataFromRes(void* pRetrieve) { return rawData; } -static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { - if (taos == NULL || data == NULL) { - SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data); - return TSDB_CODE_INVALID_PARA; - } - int32_t code = TSDB_CODE_SUCCESS; - SHashObj* pVgHash = NULL; - SQuery* pQuery = NULL; - SMqRspObj rspObj = {0}; - SDecoder decoder = {0}; - STableMeta* pTableMeta = NULL; +//static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { +// if (taos == NULL || data == NULL) { +// SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data); +// return TSDB_CODE_INVALID_PARA; +// } +// int32_t code = TSDB_CODE_SUCCESS; +// SHashObj* pVgHash = NULL; +// SQuery* pQuery = NULL; +// SMqRspObj rspObj = {0}; +// SDecoder decoder = {0}; +// STableMeta* pTableMeta = NULL; +// +// SRequestObj* pRequest = NULL; +// RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest)); +// +// uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen); +// pRequest->syncQuery = true; +// rspObj.resIter = -1; +// rspObj.resType = RES_TYPE__TMQ; +// +// int8_t dataVersion = *(int8_t*)data; +// if (dataVersion >= MQ_DATA_RSP_VERSION) { +// data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t)); +// dataLen -= sizeof(int8_t) + sizeof(int32_t); +// } +// tDecoderInit(&decoder, data, dataLen); +// code = tDecodeMqDataRsp(&decoder, &rspObj.dataRsp); +// if (code != 0) { +// SET_ERROR_MSG("decode mq data rsp failed"); +// code = TSDB_CODE_INVALID_MSG; +// goto end; +// } +// +// if (!pRequest->pDb) { +// code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; +// goto end; +// } +// +// struct SCatalog* pCatalog = NULL; +// RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog)); +// +// SRequestConnInfo conn = {0}; +// conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter; +// conn.requestId = pRequest->requestId; +// conn.requestObjRefId = pRequest->self; +// conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); +// +// RAW_RETURN_CHECK(smlInitHandle(&pQuery)); +// pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); +// RAW_NULL_CHECK(pVgHash); +// while (++rspObj.resIter < rspObj.dataRsp.blockNum) { +// void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter); +// RAW_NULL_CHECK(pRetrieve); +// if (!rspObj.dataRsp.withSchema) { +// goto end; +// } +// +// const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter); +// RAW_NULL_CHECK(tbName); +// +// SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; +// (void)strcpy(pName.dbname, pRequest->pDb); +// (void)strcpy(pName.tname, tbName); +// +// RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta)); +// +// SVgroupInfo vg = {0}; +// RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg)); +// +// void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); +// if (hData == NULL) { +// RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); +// } +// +// SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter); +// RAW_NULL_CHECK(pSW); +// TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); +// RAW_NULL_CHECK(fields); +// for (int i = 0; i < pSW->nCols; i++) { +// fields[i].type = pSW->pSchema[i].type; +// fields[i].bytes = pSW->pSchema[i].bytes; +// tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name)); +// } +// void* rawData = getRawDataFromRes(pRetrieve); +// char err[ERR_MSG_LEN] = {0}; +// code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true, err, ERR_MSG_LEN); +// taosMemoryFree(fields); +// taosMemoryFreeClear(pTableMeta); +// if (code != TSDB_CODE_SUCCESS) { +// SET_ERROR_MSG("table:%s, err:%s", tbName, err); +// goto end; +// } +// } +// +// RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash)); +// +// launchQueryImpl(pRequest, pQuery, true, NULL); +// code = pRequest->code; +// +// end: +// uDebug(LOG_ID_TAG " write raw data return, msg:%s", LOG_ID_VALUE, tstrerror(code)); +// tDeleteMqDataRsp(&rspObj.dataRsp); +// tDecoderClear(&decoder); +// qDestroyQuery(pQuery); +// destroyRequest(pRequest); +// taosHashCleanup(pVgHash); +// taosMemoryFreeClear(pTableMeta); +// return code; +//} - SRequestObj* pRequest = NULL; - RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest)); - - uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen); - pRequest->syncQuery = true; - rspObj.common.resIter = -1; - rspObj.common.resType = RES_TYPE__TMQ; - - int8_t dataVersion = *(int8_t*)data; - if (dataVersion >= MQ_DATA_RSP_VERSION) { - data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t)); - dataLen -= sizeof(int8_t) + sizeof(int32_t); - } - tDecoderInit(&decoder, data, dataLen); - code = tDecodeMqDataRsp(&decoder, &rspObj.rsp); - if (code != 0) { - SET_ERROR_MSG("decode mq data rsp failed"); - code = TSDB_CODE_INVALID_MSG; - goto end; - } - - if (!pRequest->pDb) { - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; - } - - struct SCatalog* pCatalog = NULL; - RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog)); - - SRequestConnInfo conn = {0}; - conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter; - conn.requestId = pRequest->requestId; - conn.requestObjRefId = pRequest->self; - conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); - - RAW_RETURN_CHECK(smlInitHandle(&pQuery)); - pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - RAW_NULL_CHECK(pVgHash); - while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) { - void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter); - RAW_NULL_CHECK(pRetrieve); - if (!rspObj.rsp.common.withSchema) { - goto end; - } - - const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.common.blockTbName, rspObj.common.resIter); - RAW_NULL_CHECK(tbName); - - SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; - (void)strcpy(pName.dbname, pRequest->pDb); - (void)strcpy(pName.tname, tbName); - - RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta)); - - SVgroupInfo vg = {0}; - RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg)); - - void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); - if (hData == NULL) { - RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); - } - - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter); - RAW_NULL_CHECK(pSW); - TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); - RAW_NULL_CHECK(fields); - for (int i = 0; i < pSW->nCols; i++) { - fields[i].type = pSW->pSchema[i].type; - fields[i].bytes = pSW->pSchema[i].bytes; - tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name)); - } - void* rawData = getRawDataFromRes(pRetrieve); - char err[ERR_MSG_LEN] = {0}; - code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true, err, ERR_MSG_LEN); - taosMemoryFree(fields); - taosMemoryFreeClear(pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - SET_ERROR_MSG("table:%s, err:%s", tbName, err); - goto end; - } - } - - RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash)); - - (void)launchQueryImpl(pRequest, pQuery, true, NULL); - code = pRequest->code; - - end: - uDebug(LOG_ID_TAG " write raw data return, msg:%s", LOG_ID_VALUE, tstrerror(code)); - tDeleteMqDataRsp(&rspObj.rsp); - tDecoderClear(&decoder); - qDestroyQuery(pQuery); - destroyRequest(pRequest); - taosHashCleanup(pVgHash); - taosMemoryFreeClear(pTableMeta); - return code; -} - -static int32_t buildCreateTbMap(STaosxRsp* rsp, SHashObj* pHashObj) { +static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) { // find schema data info - int32_t code = 0; + int32_t code = 0; SVCreateTbReq pCreateReq = {0}; SDecoder decoderTmp = {0}; @@ -1799,19 +1833,15 @@ static int32_t buildCreateTbMap(STaosxRsp* rsp, SHashObj* pHashObj) { code = TSDB_CODE_INVALID_MSG; goto end; } - SVCreateTbReq** pCreateReqDst = taosHashGet(pHashObj, pCreateReq.name, strlen(pCreateReq.name)); - if (pCreateReqDst == NULL){ - RAW_RETURN_CHECK(cloneSVreateTbReq(&pCreateReq, pCreateReqDst)); - code = taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), &pCreateReqDst, POINTER_BYTES); - if (code != 0){ - tdDestroySVCreateTbReq(*pCreateReqDst); - goto end; - } + if (taosHashGet(pHashObj, pCreateReq.name, strlen(pCreateReq.name)) == NULL) { + RAW_RETURN_CHECK( + taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), &pCreateReq, sizeof(SVCreateTbReq))); + } else { + tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); + pCreateReq = (SVCreateTbReq){0}; } tDecoderClear(&decoderTmp); - tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); - pCreateReq = (SVCreateTbReq){0}; } return 0; @@ -1826,22 +1856,22 @@ static threadlocal SHashObj* pCreateTbHash = NULL; static threadlocal SHashObj* pNameHash = NULL; static threadlocal SHashObj* pMetaHash = NULL; -static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) { +static int32_t tmqWriteRawImpl(TAOS* taos, uint16_t type, void* data, int32_t dataLen) { if (taos == NULL || data == NULL) { SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data); return TSDB_CODE_INVALID_PARA; } int32_t code = TSDB_CODE_SUCCESS; SQuery* pQuery = NULL; - SMqTaosxRspObj rspObj = {0}; + SMqRspObj rspObj = {0}; SDecoder decoder = {0}; SRequestObj* pRequest = NULL; RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest)); uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen); pRequest->syncQuery = true; - rspObj.common.resIter = -1; - rspObj.common.resType = RES_TYPE__TMQ_METADATA; + rspObj.resIter = -1; +// rspObj.resType = RES_TYPE__TMQ_METADATA; int8_t dataVersion = *(int8_t*)data; if (dataVersion >= MQ_DATA_RSP_VERSION) { @@ -1850,7 +1880,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) } tDecoderInit(&decoder, data, dataLen); - code = tDecodeSTaosxRsp(&decoder, &rspObj.rsp); + code = (type == RES_TYPE__TMQ_METADATA) ? tDecodeSTaosxRsp(&decoder, &rspObj.dataRsp) : tDecodeMqDataRsp(&decoder, &rspObj.dataRsp); if (code != 0) { SET_ERROR_MSG("decode mq taosx data rsp failed"); code = TSDB_CODE_INVALID_MSG; @@ -1890,31 +1920,30 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) RAW_RETURN_CHECK(smlInitHandle(&pQuery)); - uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum); - while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) { - void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter); + uDebug(LOG_ID_TAG " write raw data type:%d block num:%d", LOG_ID_VALUE, type, rspObj.dataRsp.blockNum); + while (++rspObj.resIter < rspObj.dataRsp.blockNum) { + void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter); RAW_NULL_CHECK(pRetrieve); - if (!rspObj.rsp.common.withSchema) { + if (!rspObj.dataRsp.withSchema) { goto end; } - const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.common.blockTbName, rspObj.common.resIter); - if (!tbName) { - SET_ERROR_MSG("block tbname is null"); - code = TSDB_CODE_TMQ_INVALID_MSG; - goto end; - } + const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter); + RAW_NULL_CHECK(tbName); - uDebug(LOG_ID_TAG " write raw metadata block tbname:%s", LOG_ID_VALUE, tbName); + uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName); SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; (void)strcpy(pName.dbname, pRequest->pDb); (void)strcpy(pName.tname, tbName); // find schema data info - SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName)); - if (pCreateReqDst == NULL) { - RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.rsp, pCreateTbHash)); + SVCreateTbReq* pCreateReqDst = NULL; + if (type == RES_TYPE__TMQ_METADATA){ pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName)); + if (pCreateReqDst == NULL) { + RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash)); + pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName)); + } } int32_t vgId = 0; @@ -1945,31 +1974,21 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) } } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter); - RAW_NULL_CHECK(pSW); - TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); - if (fields == NULL) { - SET_ERROR_MSG("calloc fields failed"); - code = terrno; - goto end; - } - for (int i = 0; i < pSW->nCols; i++) { - fields[i].type = pSW->pSchema[i].type; - fields[i].bytes = pSW->pSchema[i].bytes; - tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name)); - } +// SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter); +// RAW_NULL_CHECK(pSW); +// TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); +// RAW_NULL_CHECK(fields); +// +// for (int i = 0; i < pSW->nCols; i++) { +// fields[i].type = pSW->pSchema[i].type; +// fields[i].bytes = pSW->pSchema[i].bytes; +// tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name)); +// } void* rawData = getRawDataFromRes(pRetrieve); char err[ERR_MSG_LEN] = {0}; - SVCreateTbReq* pCreateReqTmp = NULL; - if (pCreateReqDst){ - RAW_RETURN_CHECK(cloneSVreateTbReq(pCreateReqDst, &pCreateReqTmp)); - } - code = rawBlockBindData(pQuery, *pTableMeta, rawData, &pCreateReqTmp, fields, pSW->nCols, true, err, ERR_MSG_LEN); - if (pCreateReqTmp != NULL) { - tdDestroySVCreateTbReq(pCreateReqTmp); - taosMemoryFree(pCreateReqTmp); - } - taosMemoryFree(fields); + code = rawBlockBindData(pQuery, *pTableMeta, rawData, pCreateReqDst, NULL, 0, true, err, ERR_MSG_LEN); +// code = rawBlockBindData(pQuery, *pTableMeta, rawData, pCreateReqDst, fields, pSW->nCols, true, err, ERR_MSG_LEN); +// taosMemoryFree(fields); if (code != TSDB_CODE_SUCCESS) { SET_ERROR_MSG("table:%s, err:%s", tbName, err); goto end; @@ -1978,16 +1997,19 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash)); - (void)launchQueryImpl(pRequest, pQuery, true, NULL); + launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; end: uDebug(LOG_ID_TAG " write raw metadata return, msg:%s", LOG_ID_VALUE, tstrerror(code)); - tDeleteSTaosxRsp(&rspObj.rsp); + if (type == RES_TYPE__TMQ_METADATA){ + tDeleteSTaosxRsp(&rspObj.dataRsp); + }else { + tDeleteMqDataRsp(&rspObj.dataRsp); + } tDecoderClear(&decoder); qDestroyQuery(pQuery); destroyRequest(pRequest); - return code; } @@ -2075,30 +2097,28 @@ char* tmq_get_json_meta(TAOS_RES* res) { return NULL; } + char* string = NULL; + SMqRspObj* rspObj = (SMqRspObj*)res; if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj* pMetaDataRspObj = (SMqTaosxRspObj*)res; - char* string = NULL; - processAutoCreateTable(&pMetaDataRspObj->rsp, &string); - return string; + processAutoCreateTable(&rspObj->dataRsp, &string); } else if (TD_RES_TMQ_BATCH_META(res)) { - SMqBatchMetaRspObj* pBatchMetaRspObj = (SMqBatchMetaRspObj*)res; - char* string = NULL; - processBatchMetaToJson(&pBatchMetaRspObj->rsp, &string); - return string; + processBatchMetaToJson(&rspObj->batchMetaRsp, &string); + } else if (TD_RES_TMQ_META(res)) { + cJSON* pJson = NULL; + processSimpleMeta(&rspObj->metaRsp, &pJson); + string = cJSON_PrintUnformatted(pJson); + cJSON_Delete(pJson); + } else{ + uError("tmq_get_json_meta res:%d, invalid type", *(int8_t*)res); } - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - cJSON* pJson = NULL; - processSimpleMeta(&pMetaRspObj->metaRsp, &pJson); - char* string = cJSON_PrintUnformatted(pJson); - cJSON_Delete(pJson); + uDebug("tmq_get_json_meta string:%s", string); return string; } void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); } -static int32_t getOffSetLen(const void* rsp) { - const SMqDataRspCommon* pRsp = rsp; +static int32_t getOffSetLen(const SMqDataRsp* pRsp) { SEncoder coder = {0}; tEncoderInit(&coder, NULL, 0); if (tEncodeSTqOffsetVal(&coder, &pRsp->reqOffset) < 0) return -1; @@ -2108,9 +2128,8 @@ static int32_t getOffSetLen(const void* rsp) { return pos; } -typedef int32_t __encode_func__(SEncoder* pEncoder, const void* pRsp); - -static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, void* rspObj, tmq_raw_data* raw) { +typedef int32_t __encode_func__(SEncoder* pEncoder, const SMqDataRsp* pRsp); +static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, SMqDataRsp* rspObj, tmq_raw_data* raw) { int32_t len = 0; int32_t code = 0; SEncoder encoder = {0}; @@ -2159,15 +2178,14 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { if (!raw || !res) { return TSDB_CODE_INVALID_PARA; } + SMqRspObj* rspObj = ((SMqRspObj*)res); if (TD_RES_TMQ_META(res)) { - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - raw->raw = pMetaRspObj->metaRsp.metaRsp; - raw->raw_len = pMetaRspObj->metaRsp.metaRspLen; - raw->raw_type = pMetaRspObj->metaRsp.resMsgType; + raw->raw = rspObj->metaRsp.metaRsp; + raw->raw_len = rspObj->metaRsp.metaRspLen; + raw->raw_type = rspObj->metaRsp.resMsgType; uDebug("tmq get raw type meta:%p", raw); } else if (TD_RES_TMQ(res)) { - SMqRspObj* rspObj = ((SMqRspObj*)res); - int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->rsp, raw); + int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->dataRsp, raw); if (code != 0) { uError("tmq get raw type error:%d", terrno); return code; @@ -2175,9 +2193,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { raw->raw_type = RES_TYPE__TMQ; uDebug("tmq get raw type data:%p", raw); } else if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj* rspObj = ((SMqTaosxRspObj*)res); - - int32_t code = encodeMqDataRsp(tEncodeSTaosxRsp, &rspObj->rsp, raw); + int32_t code = encodeMqDataRsp(tEncodeSTaosxRsp, &rspObj->dataRsp, raw); if (code != 0) { uError("tmq get raw type error:%d", terrno); return code; @@ -2185,10 +2201,9 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { raw->raw_type = RES_TYPE__TMQ_METADATA; uDebug("tmq get raw type metadata:%p", raw); } else if (TD_RES_TMQ_BATCH_META(res)) { - SMqBatchMetaRspObj* pBtMetaRspObj = (SMqBatchMetaRspObj*)res; - raw->raw = pBtMetaRspObj->rsp.pMetaBuff; - raw->raw_len = pBtMetaRspObj->rsp.metaBuffLen; - raw->raw_type = RES_TYPE__TMQ_BATCH_META; + raw->raw = rspObj->batchMetaRsp.pMetaBuff; + raw->raw_len = rspObj->batchMetaRsp.metaBuffLen; + raw->raw_type = rspObj->resType; uDebug("tmq get raw batch meta:%p", raw); } else { uError("tmq get raw error type:%d", *(int8_t*)res); @@ -2220,10 +2235,8 @@ static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type) return taosDropTable(taos, buf, len); } else if (type == TDMT_VND_DELETE) { return taosDeleteData(taos, buf, len); - } else if (type == RES_TYPE__TMQ) { - return tmqWriteRawDataImpl(taos, buf, len); - } else if (type == RES_TYPE__TMQ_METADATA) { - return tmqWriteRawMetaDataImpl(taos, buf, len); + } else if (type == RES_TYPE__TMQ_METADATA || type == RES_TYPE__TMQ) { + return tmqWriteRawImpl(taos, type, buf, len); } else if (type == RES_TYPE__TMQ_BATCH_META) { return tmqWriteBatchMetaDataImpl(taos, buf, len); } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 6a19f61383..f3a22bff75 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -112,7 +112,7 @@ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo *conn, const char SUserAuthInfo pAuth = {0}; (void)snprintf(pAuth.user, sizeof(pAuth.user), "%s", info->taos->user); if (NULL == pTabName) { - if (tNameSetDbName(&pAuth.tbName, info->taos->acctId, info->pRequest->pDb, strlen(info->pRequest->pDb)) != 0){ + if (tNameSetDbName(&pAuth.tbName, info->taos->acctId, info->pRequest->pDb, strlen(info->pRequest->pDb)) != 0) { return TSDB_CODE_SML_INVALID_DATA; } } else { @@ -165,7 +165,7 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u return convertTimePrecision(tsInt64, fromPrecision, toPrecision); } -int32_t smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen, SSmlTableInfo** tInfo) { +int32_t smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen, SSmlTableInfo **tInfo) { SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); if (!tag) { return terrno; @@ -203,13 +203,13 @@ static void smlDestroySTableMeta(void *para) { taosMemoryFree(meta); } -int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta) { - int32_t code = TSDB_CODE_SUCCESS; - char *measure = currElement->measure; - int measureLen = currElement->measureLen; +int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta **sMeta) { + int32_t code = TSDB_CODE_SUCCESS; + char *measure = currElement->measure; + int measureLen = currElement->measureLen; if (currElement->measureEscaped) { measure = (char *)taosMemoryMalloc(measureLen); - if (measure == NULL){ + if (measure == NULL) { return terrno; } (void)memcpy(measure, currElement->measure, measureLen); @@ -233,7 +233,7 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml } (*sMeta)->tableMeta = pTableMeta; code = taosHashPut(info->superTables, currElement->measure, currElement->measureLen, sMeta, POINTER_BYTES); - if (code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { smlDestroySTableMeta(*sMeta); return code; } @@ -250,11 +250,11 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml } if (i < pTableMeta->tableInfo.numOfColumns) { - if(taosArrayPush((*sMeta)->cols, &kv) == NULL){ + if (taosArrayPush((*sMeta)->cols, &kv) == NULL) { return terrno; } } else { - if(taosArrayPush((*sMeta)->tags, &kv) == NULL){ + if (taosArrayPush((*sMeta)->tags, &kv) == NULL) { return terrno; } } @@ -277,7 +277,7 @@ bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { goto END; } SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt); - if (maxKV == NULL){ + if (maxKV == NULL) { goto END; } if (unlikely(!IS_SAME_KEY)) { @@ -336,9 +336,9 @@ int32_t smlJoinMeasureTag(SSmlLineInfo *elements) { return TSDB_CODE_SUCCESS; } -static bool smlIsPKTable(STableMeta *pTableMeta){ - for(int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++){ - if(pTableMeta->schema[i].flags & COL_IS_KEY){ +static bool smlIsPKTable(STableMeta *pTableMeta) { + for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { + if (pTableMeta->schema[i].flags & COL_IS_KEY) { return true; } } @@ -368,14 +368,14 @@ int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { info->maxTagKVs = sMeta->tags; info->maxColKVs = sMeta->cols; - if(smlIsPKTable(sMeta->tableMeta)){ + if (smlIsPKTable(sMeta->tableMeta)) { return TSDB_CODE_SML_NOT_SUPPORT_PK; } return 0; } int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); SSmlTableInfo *tinfo = NULL; @@ -385,19 +385,19 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { return code; } code = taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES); - if(code != 0){ + if (code != 0) { smlDestroyTableInfo(&tinfo); return code; } tinfo->tags = taosArrayDup(info->preLineTagKV, NULL); - if(tinfo->tags == NULL){ + if (tinfo->tags == NULL) { smlDestroyTableInfo(&tinfo); return TSDB_CODE_OUT_OF_MEMORY; } for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i); - if(kv == NULL){ + if (kv == NULL) { smlDestroyTableInfo(&tinfo); return TSDB_CODE_SML_INVALID_DATA; } @@ -406,12 +406,12 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { } code = smlSetCTableName(tinfo, info->tbnameKey); - if (code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { smlDestroyTableInfo(&tinfo); return code; } code = getTableUid(info, elements, tinfo); - if (code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { smlDestroyTableInfo(&tinfo); return code; } @@ -458,10 +458,10 @@ int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv * return terrno; } } - if (taosArrayPush(elements->colArray, kvTs) == NULL){ + if (taosArrayPush(elements->colArray, kvTs) == NULL) { return terrno; } - if (taosArrayPush(elements->colArray, kv) == NULL){ + if (taosArrayPush(elements->colArray, kv) == NULL) { return terrno; } } @@ -495,11 +495,11 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnameKey) { bool autoChildName = false; size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); - if(delimiter > 0 && tbnameKey == NULL){ + if (delimiter > 0 && tbnameKey == NULL) { size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - if(tag == NULL){ + if (tag == NULL) { return TSDB_CODE_SML_INVALID_DATA; } totalNameLen += tag->length; @@ -512,7 +512,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - if(tag == NULL){ + if (tag == NULL) { return TSDB_CODE_SML_INVALID_DATA; } (void)strncat(childTableName, tag->value, tag->length); @@ -523,8 +523,8 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam if (tsSmlDot2Underline) { smlStrReplace(childTableName, strlen(childTableName)); } - }else{ - if (tbnameKey == NULL){ + } else { + if (tbnameKey == NULL) { tbnameKey = tsSmlChildTableName; } size_t childTableNameLen = strlen(tbnameKey); @@ -532,13 +532,14 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - if(tag == NULL){ + if (tag == NULL) { return TSDB_CODE_SML_INVALID_DATA; } // handle child table name if (childTableNameLen == tag->keyLen && strncmp(tag->key, tbnameKey, tag->keyLen) == 0) { (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - (void)strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + (void)strncpy(childTableName, tag->value, + (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); if (tsSmlDot2Underline) { smlStrReplace(childTableName, strlen(childTableName)); } @@ -553,7 +554,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { int32_t code = smlParseTableName(oneTable->tags, oneTable->childTableName, tbnameKey); - if(code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { return code; } @@ -562,7 +563,7 @@ int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { if (dst == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - if(oneTable->sTableNameLen >= TSDB_TABLE_NAME_LEN){ + if (oneTable->sTableNameLen >= TSDB_TABLE_NAME_LEN) { uError("SML:smlSetCTableName super table name is too long"); taosArrayDestroy(dst); return TSDB_CODE_SML_INTERNAL_ERROR; @@ -578,7 +579,7 @@ int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { } code = buildChildTableName(&rName); - if (code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { return code; } taosArrayDestroy(dst); @@ -906,13 +907,13 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { return result; } -static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *checkDumplicateCols, - ESchemaAction *action, bool isTag) { +static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, + SArray *checkDumplicateCols, ESchemaAction *action, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; for (int j = 0; j < taosArrayGetSize(cols); ++j) { if (j == 0 && !isTag) continue; SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); - if (kv == NULL){ + if (kv == NULL) { return TSDB_CODE_SML_INVALID_DATA; } code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); @@ -923,10 +924,10 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH for (int j = 0; j < taosArrayGetSize(checkDumplicateCols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(checkDumplicateCols, j); - if (kv == NULL){ + if (kv == NULL) { return TSDB_CODE_SML_INVALID_DATA; } - if(taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL){ + if (taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; } } @@ -934,16 +935,16 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH } static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (hashTmp == NULL) { code = terrno; goto END; } - int32_t i = 0; + int32_t i = 0; for (; i < length; i++) { code = taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); - if (code != 0){ + if (code != 0) { goto END; } } @@ -955,7 +956,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool } for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL){ + if (kv == NULL) { code = TSDB_CODE_SML_INVALID_DATA; goto END; } @@ -982,8 +983,8 @@ static int32_t getBytes(uint8_t type, int32_t length) { static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *results, int32_t numOfCols, bool isTag) { for (int j = 0; j < taosArrayGetSize(cols); ++j) { - SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); - if (kv == NULL){ + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + if (kv == NULL) { return TSDB_CODE_SML_INVALID_DATA; } ESchemaAction action = SCHEMA_ACTION_NULL; @@ -996,7 +997,7 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO field.type = kv->type; field.bytes = getBytes(kv->type, kv->length); (void)memcpy(field.name, kv->key, kv->keyLen); - if (taosArrayPush(results, &field) == NULL){ + if (taosArrayPush(results, &field) == NULL) { return terrno; } } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { @@ -1008,7 +1009,7 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO uint16_t newIndex = *index; if (isTag) newIndex -= numOfCols; SField *field = (SField *)taosArrayGet(results, newIndex); - if (field == NULL){ + if (field == NULL) { return TSDB_CODE_SML_INVALID_DATA; } field->bytes = getBytes(kv->type, kv->length); @@ -1019,7 +1020,7 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO int32_t len = 0; for (int j = 0; j < taosArrayGetSize(results); ++j) { SField *field = taosArrayGet(results, j); - if (field == NULL){ + if (field == NULL) { return TSDB_CODE_SML_INVALID_DATA; } len += field->bytes; @@ -1051,14 +1052,14 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, } for (int32_t i = 0; i < pReq.numOfColumns; ++i) { SField *pField = taosArrayGet(pColumns, i); - if (pField == NULL){ + if (pField == NULL) { code = TSDB_CODE_SML_INVALID_DATA; goto end; } SFieldWithOptions fieldWithOption = {0}; setFieldWithOptions(&fieldWithOption, pField); setDefaultOptionsForField(&fieldWithOption); - if (taosArrayPush(pReq.pColumns, &fieldWithOption) == NULL){ + if (taosArrayPush(pReq.pColumns, &fieldWithOption) == NULL) { code = terrno; goto end; } @@ -1105,7 +1106,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, field.type = TSDB_DATA_TYPE_NCHAR; field.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; tstrncpy(field.name, tsSmlTagName, sizeof(field.name)); - if (taosArrayPush(pReq.pTags, &field) == NULL){ + if (taosArrayPush(pReq.pTags, &field) == NULL) { code = terrno; goto end; } @@ -1121,7 +1122,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); pCmdMsg.msgType = TDMT_MND_CREATE_STB; pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); - if (pCmdMsg.msgLen < 0){ + if (pCmdMsg.msgLen < 0) { code = TSDB_CODE_OUT_OF_MEMORY; goto end; } @@ -1131,7 +1132,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, goto end; } - if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) < 0){ + if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) < 0) { code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pCmdMsg.pMsg); goto end; @@ -1144,11 +1145,11 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pQuery.msgType = pQuery.pCmdMsg->msgType; pQuery.stableQuery = true; - (void)launchQueryImpl(pRequest, &pQuery, true, NULL); // no need to check return value + launchQueryImpl(pRequest, &pQuery, true, NULL); // no need to check return value if (pRequest->code == TSDB_CODE_SUCCESS) { code = catalogRemoveTableMeta(info->pCatalog, pName); - if (code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { goto end; } } @@ -1187,7 +1188,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { size_t superTableLen = 0; void *superTable = taosHashGetKey(tmp, &superTableLen); char *measure = taosMemoryMalloc(superTableLen); - if (measure == NULL){ + if (measure == NULL) { code = terrno; goto end; } @@ -1246,28 +1247,28 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { goto end; } } else if (code == TSDB_CODE_SUCCESS) { - - if(smlIsPKTable(pTableMeta)){ + if (smlIsPKTable(pTableMeta)) { code = TSDB_CODE_SML_NOT_SUPPORT_PK; goto end; } hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (hashTmp == NULL){ + if (hashTmp == NULL) { code = terrno; goto end; } for (uint16_t i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { code = taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); - if (code != 0){ + if (code != 0) { goto end; } } ESchemaAction action = SCHEMA_ACTION_NULL; - code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, sTableData->cols, &action, true); + code = + smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, sTableData->cols, &action, true); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -1280,13 +1281,13 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { action); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - if (pColumns == NULL){ + if (pColumns == NULL) { code = terrno; goto end; } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - if (pTags == NULL){ + if (pTags == NULL) { taosArrayDestroy(pColumns); code = terrno; goto end; @@ -1297,14 +1298,14 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { field.bytes = pTableMeta->schema[i].bytes; tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); if (i < pTableMeta->tableInfo.numOfColumns) { - if (taosArrayPush(pColumns, &field) == NULL){ + if (taosArrayPush(pColumns, &field) == NULL) { taosArrayDestroy(pColumns); taosArrayDestroy(pTags); code = terrno; goto end; } } else { - if (taosArrayPush(pTags, &field) == NULL){ + if (taosArrayPush(pTags, &field) == NULL) { taosArrayDestroy(pColumns); taosArrayDestroy(pTags); code = terrno; @@ -1363,7 +1364,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } } action = SCHEMA_ACTION_NULL; - code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false); + code = + smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -1376,13 +1378,13 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { action); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - if (pColumns == NULL){ + if (pColumns == NULL) { code = terrno; goto end; } SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - if (pTags == NULL){ + if (pTags == NULL) { taosArrayDestroy(pColumns); code = terrno; goto end; @@ -1393,14 +1395,14 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { field.bytes = pTableMeta->schema[i].bytes; tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); if (i < pTableMeta->tableInfo.numOfColumns) { - if (taosArrayPush(pColumns, &field) == NULL){ + if (taosArrayPush(pColumns, &field) == NULL) { taosArrayDestroy(pColumns); taosArrayDestroy(pTags); code = terrno; goto end; } } else { - if (taosArrayPush(pTags, &field) == NULL){ + if (taosArrayPush(pTags, &field) == NULL) { taosArrayDestroy(pColumns); taosArrayDestroy(pTags); code = terrno; @@ -1483,7 +1485,7 @@ end: taosHashCancelIterate(info->superTables, tmp); taosHashCleanup(hashTmp); taosMemoryFreeClear(pTableMeta); - (void)catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); // ignore refresh meta code if there is an error + (void)catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); // ignore refresh meta code if there is an error uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code, tstrerror(code), info->dataFormat, info->needModifySchema); @@ -1494,34 +1496,35 @@ static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols terrno = 0; for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL){ + if (kv == NULL) { return TSDB_CODE_SML_INVALID_DATA; } - int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); + int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); if (ret == 0) { - if (taosArrayPush(metaArray, kv) == NULL){ + if (taosArrayPush(metaArray, kv) == NULL) { return terrno; } - if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { + if (taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; } - }else if(terrno == TSDB_CODE_DUP_KEY){ + } else if (terrno == TSDB_CODE_DUP_KEY) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; } } return TSDB_CODE_SUCCESS; } -static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg, SHashObj* checkDuplicate) { +static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg, + SHashObj *checkDuplicate) { for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL){ + if (kv == NULL) { return TSDB_CODE_SML_INVALID_DATA; } int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); if (index) { SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index); - if (value == NULL){ + if (value == NULL) { return TSDB_CODE_SML_INVALID_DATA; } @@ -1549,13 +1552,13 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols int16_t size = tmp; int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); if (ret == 0) { - if(taosArrayPush(metaArray, kv) == NULL){ + if (taosArrayPush(metaArray, kv) == NULL) { return terrno; } - if(taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { + if (taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; } - }else{ + } else { return ret; } } @@ -1586,7 +1589,7 @@ void freeSSmlKv(void *data) { void smlDestroyInfo(SSmlHandle *info) { if (!info) return; -// qDestroyQuery(info->pQuery); + // qDestroyQuery(info->pQuery); taosHashCleanup(info->pVgHash); taosHashCleanup(info->childTables); @@ -1657,7 +1660,7 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { info->id = smlGenId(); code = smlInitHandle(&info->pQuery); - if (code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { goto FAILED; } info->dataFormat = true; @@ -1688,7 +1691,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { } for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL){ + if (kv == NULL) { taosHashCleanup(kvHash); return TSDB_CODE_SML_INVALID_DATA; } @@ -1698,7 +1701,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { taosHashCleanup(kvHash); return TSDB_CODE_PAR_DUPLICATED_COLUMN; } - if (code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { taosHashCleanup(kvHash); return code; } @@ -1759,9 +1762,11 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { if (tableMeta) { // update meta uDebug("SML:0x%" PRIx64 " smlParseLineBottom update meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf, (*tableMeta)->tagHash); + ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf, + (*tableMeta)->tagHash); if (ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf, (*tableMeta)->colHash); + ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf, + (*tableMeta)->colHash); } if (ret != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlUpdateMeta failed, ret:%d", info->id, ret); @@ -1801,17 +1806,17 @@ static int32_t smlInsertData(SSmlHandle *info) { if (info->pRequest->dbList == NULL) { info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN); - if (info->pRequest->dbList == NULL){ + if (info->pRequest->dbList == NULL) { return terrno; } } char *data = (char *)taosArrayReserve(info->pRequest->dbList, 1); - if (data == NULL){ + if (data == NULL) { return terrno; } SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); - (void)tNameGetFullDbName(&pName, data); //ignore + (void)tNameGetFullDbName(&pName, data); // ignore SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); while (oneTable) { @@ -1819,7 +1824,7 @@ static int32_t smlInsertData(SSmlHandle *info) { int measureLen = tableData->sTableNameLen; char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); - if (measure == NULL){ + if (measure == NULL) { return terrno; } (void)memcpy(measure, tableData->sTableName, tableData->sTableNameLen); @@ -1830,11 +1835,11 @@ static int32_t smlInsertData(SSmlHandle *info) { if (info->pRequest->tableList == NULL) { info->pRequest->tableList = taosArrayInit(1, sizeof(SName)); - if (info->pRequest->tableList == NULL){ + if (info->pRequest->tableList == NULL) { return terrno; } } - if (taosArrayPush(info->pRequest->tableList, &pName) == NULL){ + if (taosArrayPush(info->pRequest->tableList, &pName) == NULL) { return terrno; } @@ -1862,7 +1867,7 @@ static int32_t smlInsertData(SSmlHandle *info) { return code; } code = taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); - if (code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " taosHashPut failed. table name: %s", info->id, tableData->childTableName); taosMemoryFree(measure); taosHashCancelIterate(info->childTables, oneTable); @@ -1904,9 +1909,9 @@ static int32_t smlInsertData(SSmlHandle *info) { info->cost.insertRpcTime = taosGetTimestampUs(); SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; - (void)atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); // no need to check return code + (void)atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); // no need to check return code - (void)launchQueryImpl(info->pRequest, info->pQuery, true, NULL); // no need to check return code + launchQueryImpl(info->pRequest, info->pQuery, true, NULL); // no need to check return code uDebug("SML:0x%" PRIx64 " smlInsertData end, format:%d, code:%d,%s", info->id, info->dataFormat, info->pRequest->code, tstrerror(info->pRequest->code)); @@ -1975,12 +1980,12 @@ static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLi if (*rawLine != NULL && (uDebugFlag & DEBUG_DEBUG)) { char *print = taosMemoryCalloc(*len + 1, 1); - if (print != NULL){ + if (print != NULL) { (void)memcpy(print, *tmp, *len); uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, numLines, info->protocol, *len, print); taosMemoryFree(print); - } else{ + } else { uError("SML:0x%" PRIx64 " smlParseLine taosMemoryCalloc failed", info->id); } } else { @@ -2228,7 +2233,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, uInfo("SML:%" PRIx64 " retry:%d/10,ver is old retry or object is creating code:%d, msg:%s", info->id, cnt, code, tstrerror(code)); code = refreshMeta(request->pTscObj, request); - if (code != 0){ + if (code != 0) { uInfo("SML:%" PRIx64 " refresh meta error code:%d, msg:%s", info->id, code, tstrerror(code)); } smlDestroyInfo(info); @@ -2266,7 +2271,7 @@ end: */ TAOS_RES *taos_schemaless_insert_ttl_with_reqid_tbname_key(TAOS *taos, char *lines[], int numLines, int protocol, - int precision, int32_t ttl, int64_t reqid, char *tbnameKey){ + int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { return taos_schemaless_insert_inner(taos, lines, NULL, NULL, numLines, protocol, precision, ttl, reqid, tbnameKey); } @@ -2306,14 +2311,17 @@ static void getRawLineLen(char *lines, int len, int32_t *totalRows, int protocol } TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid_tbname_key(TAOS *taos, char *lines, int len, int32_t *totalRows, - int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey){ + int protocol, int precision, int32_t ttl, int64_t reqid, + char *tbnameKey) { getRawLineLen(lines, len, totalRows, protocol); - return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid, tbnameKey); + return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid, + tbnameKey); } TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int32_t ttl, int64_t reqid) { - return taos_schemaless_insert_raw_ttl_with_reqid_tbname_key(taos, lines, len, totalRows, protocol, precision, ttl, reqid, NULL); + return taos_schemaless_insert_raw_ttl_with_reqid_tbname_key(taos, lines, len, totalRows, protocol, precision, ttl, + reqid, NULL); } TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index ee6e2d71a0..866d0cc272 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -25,7 +25,7 @@ static FORCE_INLINE int32_t stmtAllocQNodeFromBuf(STableBufInfo* pTblBuf, void** return terrno; } - if(taosArrayPush(pTblBuf->pBufList, &buff) == NULL){ + if (taosArrayPush(pTblBuf->pBufList, &buff) == NULL) { return terrno; } @@ -184,10 +184,16 @@ int32_t stmtBackupQueryFields(STscStmt* pStmt) { int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD); pRes->fields = taosMemoryMalloc(size); - pRes->userFields = taosMemoryMalloc(size); - if (NULL == pRes->fields || NULL == pRes->userFields) { + if (pRes->fields == NULL) { STMT_ERR_RET(terrno); } + + pRes->userFields = taosMemoryMalloc(size); + if (pRes->userFields == NULL) { + taosMemoryFreeClear(pRes->fields); + STMT_ERR_RET(terrno); + } + (void)memcpy(pRes->fields, pStmt->exec.pRequest->body.resInfo.fields, size); (void)memcpy(pRes->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size); @@ -224,8 +230,8 @@ int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, bool autoCreateTbl) { STscStmt* pStmt = (STscStmt*)stmt; char tbFName[TSDB_TABLE_FNAME_LEN]; - int32_t code = tNameExtractFullName(tbName, tbFName); - if (code != 0){ + int32_t code = tNameExtractFullName(tbName, tbFName); + if (code != 0) { return code; } @@ -772,7 +778,7 @@ void* stmtBindThreadFunc(void* param) { } int ret = stmtAsyncOutput(pStmt, asyncParam); - if (ret != 0){ + if (ret != 0) { qError("stmtAsyncOutput failed, reason:%s", tstrerror(ret)); } } @@ -821,7 +827,7 @@ int32_t stmtInitTableBuf(STableBufInfo* pTblBuf) { return terrno; } - if (taosArrayPush(pTblBuf->pBufList, &buff) == NULL){ + if (taosArrayPush(pTblBuf->pBufList, &buff) == NULL) { return terrno; } @@ -923,9 +929,9 @@ int stmtPrepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { length = strlen(sql); } - pStmt->sql.sqlStr = strndup(sql, length); + pStmt->sql.sqlStr = taosStrndup(sql, length); if (!pStmt->sql.sqlStr) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pStmt->sql.sqlLen = length; pStmt->sql.stbInterlaceMode = pStmt->stbInterlaceMode; @@ -967,7 +973,7 @@ int32_t stmtInitStbInterlaceTableInfo(STscStmt* pStmt) { } int stmtSetDbName(TAOS_STMT* stmt, const char* dbName) { - STscStmt *pStmt = (STscStmt *) stmt; + STscStmt* pStmt = (STscStmt*)stmt; STMT_DLOG("start to set dbName: %s", dbName); @@ -1045,7 +1051,7 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); - SBoundColInfo *tags_info = (SBoundColInfo*)pStmt->bInfo.boundTags; + SBoundColInfo* tags_info = (SBoundColInfo*)pStmt->bInfo.boundTags; if (tags_info->numOfBound <= 0 || tags_info->numOfCols <= 0) { tscWarn("no tags bound in sql, will not bound tags"); return TSDB_CODE_SUCCESS; @@ -1192,7 +1198,7 @@ static FORCE_INLINE int32_t stmtGetTableColsFromCache(STscStmt* pStmt, SArray** return terrno; } - if (taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols) == NULL){ + if (taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols) == NULL) { return terrno; } } @@ -1216,7 +1222,6 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); - if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { pStmt->bInfo.needParse = false; @@ -1256,7 +1261,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { if (pStmt->sql.pQuery->haveResultSet) { STMT_ERR_RET(setResSchemaInfo(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->pResSchema, - pStmt->sql.pQuery->numOfResCols)); + pStmt->sql.pQuery->numOfResCols)); taosMemoryFreeClear(pStmt->sql.pQuery->pResSchema); setResPrecision(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->precision); } @@ -1549,7 +1554,7 @@ int stmtExec(TAOS_STMT* stmt) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); if (STMT_TYPE_QUERY == pStmt->sql.type) { - (void)launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } else { if (pStmt->sql.stbInterlaceMode) { int64_t startTs = taosGetTimestampUs(); @@ -1571,7 +1576,7 @@ int stmtExec(TAOS_STMT* stmt) { STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pBlockHash)); } - (void)launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) { diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index a9738e4e9c..a0fd49ac86 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -698,7 +698,9 @@ static void* stmtBindThreadFunc(void* param) { continue; } - (void)stmtAsyncOutput(pStmt, asyncParam); + if (stmtAsyncOutput(pStmt, asyncParam) != 0) { + qError("stmt async output failed"); + } } qInfo("stmt bind thread stopped"); @@ -822,7 +824,11 @@ TAOS_STMT2* stmtInit2(STscObj* taos, TAOS_STMT2_OPTION* pOptions) { pStmt->sql.siInfo.tableColsReady = true; if (pStmt->options.asyncExecFn) { - (void)tsem_init(&pStmt->asyncQuerySem, 0, 1); + if (tsem_init(&pStmt->asyncQuerySem, 0, 1) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + (void)stmtClose(pStmt); + return NULL; + } } pStmt->semWaited = false; @@ -868,9 +874,9 @@ int stmtPrepare2(TAOS_STMT2* stmt, const char* sql, unsigned long length) { length = strlen(sql); } - pStmt->sql.sqlStr = strndup(sql, length); + pStmt->sql.sqlStr = taosStrndup(sql, length); if (!pStmt->sql.sqlStr) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pStmt->sql.sqlLen = length; pStmt->sql.stbInterlaceMode = pStmt->stbInterlaceMode; @@ -1603,7 +1609,9 @@ static void asyncQueryCb(void* userdata, TAOS_RES* res, int code) { (void)stmtCleanExecInfo(pStmt, (code ? false : true), false); ++pStmt->sql.runTimes; - (void)tsem_post(&pStmt->asyncQuerySem); + if (tsem_post(&pStmt->asyncQuerySem) != 0) { + tscError("failed to post asyncQuerySem"); + } } int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) { @@ -1645,7 +1653,7 @@ int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) { __taos_async_fn_t fp = pStmt->options.asyncExecFn; if (!fp) { - (void)launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) { code = refreshMeta(pStmt->exec.pRequest->pTscObj, pStmt->exec.pRequest); @@ -1710,7 +1718,9 @@ int stmtClose2(TAOS_STMT2* stmt) { } if (pStmt->options.asyncExecFn && !pStmt->semWaited) { - (void)tsem_wait(&pStmt->asyncQuerySem); + if (tsem_wait(&pStmt->asyncQuerySem) != 0) { + tscError("failed to wait asyncQuerySem"); + } } STMT_DLOG("stmt %p closed, stbInterlaceMode: %d, statInfo: ctgGetTbMetaNum=>%" PRId64 ", getCacheTbInfo=>%" PRId64 @@ -1727,7 +1737,9 @@ int stmtClose2(TAOS_STMT2* stmt) { STMT_ERR_RET(stmtCleanSQLInfo(pStmt)); if (pStmt->options.asyncExecFn) { - (void)tsem_destroy(&pStmt->asyncQuerySem); + if (tsem_destroy(&pStmt->asyncQuerySem) != 0) { + tscError("failed to destroy asyncQuerySem"); + } } taosMemoryFree(stmt); @@ -1873,6 +1885,8 @@ int stmtGetParamNum2(TAOS_STMT2* stmt, int* nums) { int stmtGetParamTbName(TAOS_STMT2* stmt, int* nums) { STscStmt2* pStmt = (STscStmt2*)stmt; + int32_t code = 0; + int32_t preCode = pStmt->errCode; STMT_DLOG_E("start to get param num"); @@ -1895,17 +1909,19 @@ int stmtGetParamTbName(TAOS_STMT2* stmt, int* nums) { STMT_ERR_RET(stmtCreateRequest(pStmt)); if (pStmt->bInfo.needParse) { - STMT_ERR_RET(stmtParseSql(pStmt)); + STMT_ERRI_JRET(stmtParseSql(pStmt)); } - if (TSDB_CODE_TSC_STMT_TBNAME_ERROR == pStmt->errCode) { + *nums = STMT_TYPE_MULTI_INSERT == pStmt->sql.type ? 1 : 0; + +_return: + if (TSDB_CODE_TSC_STMT_TBNAME_ERROR == code) { *nums = 1; - pStmt->errCode = TSDB_CODE_SUCCESS; - } else { - *nums = STMT_TYPE_MULTI_INSERT == pStmt->sql.type ? 1 : 0; + code = TSDB_CODE_SUCCESS; } - return TSDB_CODE_SUCCESS; + pStmt->errCode = preCode; + return code; } /* int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) { diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 2c92824855..975d14f3ee 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -24,33 +24,61 @@ #include "tref.h" #include "ttimer.h" +#define tqFatalC(...) do { if (cDebugFlag & DEBUG_FATAL || tqClientDebug) { taosPrintLog("TQ FATAL ", DEBUG_FATAL, tqDebugFlag, __VA_ARGS__); }} while(0) +#define tqErrorC(...) do { if (cDebugFlag & DEBUG_ERROR || tqClientDebug) { taosPrintLog("TQ ERROR ", DEBUG_ERROR, tqDebugFlag, __VA_ARGS__); }} while(0) +#define tqWarnC(...) do { if (cDebugFlag & DEBUG_WARN || tqClientDebug) { taosPrintLog("TQ WARN ", DEBUG_WARN, tqDebugFlag, __VA_ARGS__); }} while(0) +#define tqInfoC(...) do { if (cDebugFlag & DEBUG_INFO || tqClientDebug) { taosPrintLog("TQ ", DEBUG_INFO, tqDebugFlag, __VA_ARGS__); }} while(0) +#define tqDebugC(...) do { if (cDebugFlag & DEBUG_DEBUG || tqClientDebug) { taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0) +#define tqTraceC(...) do { if (cDebugFlag & DEBUG_TRACE || tqClientDebug) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) + #define EMPTY_BLOCK_POLL_IDLE_DURATION 10 #define DEFAULT_AUTO_COMMIT_INTERVAL 5000 #define DEFAULT_HEARTBEAT_INTERVAL 3000 #define DEFAULT_ASKEP_INTERVAL 1000 +#define DEFAULT_COMMIT_CNT 1 +#define SUBSCRIBE_RETRY_MAX_COUNT 240 +#define SUBSCRIBE_RETRY_INTERVAL 500 -struct SMqMgmt { - tmr_h timer; - int32_t rsetId; + +#define SET_ERROR_MSG_TMQ(MSG) \ + if (errstr != NULL) (void)snprintf(errstr, errstrLen, MSG); + +#define PROCESS_POLL_RSP(FUNC,DATA) \ + SDecoder decoder = {0}; \ + tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); \ + if (FUNC(&decoder, DATA) < 0) { \ + tDecoderClear(&decoder); \ + code = TSDB_CODE_OUT_OF_MEMORY; \ + goto END;\ + }\ + tDecoderClear(&decoder);\ + (void)memcpy(DATA, pMsg->pData, sizeof(SMqRspHead)); + +#define DELETE_POLL_RSP(FUNC,DATA) \ + SMqPollRspWrapper* pRsp = &rspWrapper->pollRsp;\ + taosMemoryFreeClear(pRsp->pEpset);\ + FUNC(DATA); + +enum { + TMQ_VG_STATUS__IDLE = 0, + TMQ_VG_STATUS__WAIT, }; -static TdThreadOnce tmqInit = PTHREAD_ONCE_INIT; // initialize only once -volatile int32_t tmqInitRes = 0; // initialize rsp code -static struct SMqMgmt tmqMgmt = {0}; -static int8_t pollFlag = 0; +enum { + TMQ_CONSUMER_STATUS__INIT = 0, + TMQ_CONSUMER_STATUS__READY, + TMQ_CONSUMER_STATUS__CLOSED, +}; + +enum { + TMQ_DELAYED_TASK__ASK_EP = 1, + TMQ_DELAYED_TASK__COMMIT, +}; typedef struct { - int32_t code; - int8_t tmqRspType; - int32_t epoch; -} SMqRspWrapper; - -typedef struct { - int32_t code; - int8_t tmqRspType; - int32_t epoch; - SMqAskEpRsp msg; -} SMqAskEpRspWrapper; + tmr_h timer; + int32_t rsetId; +} SMqMgmt; struct tmq_list_t { SArray container; @@ -119,30 +147,12 @@ struct tmq_t { tsem2_t rspSem; }; -typedef struct SAskEpInfo { +typedef struct { int32_t code; tsem2_t sem; } SAskEpInfo; -enum { - TMQ_VG_STATUS__IDLE = 0, - TMQ_VG_STATUS__WAIT, -}; - -enum { - TMQ_CONSUMER_STATUS__INIT = 0, - TMQ_CONSUMER_STATUS__READY, - TMQ_CONSUMER_STATUS__NO_TOPIC, - TMQ_CONSUMER_STATUS__RECOVER, - TMQ_CONSUMER_STATUS__CLOSED, -}; - -enum { - TMQ_DELAYED_TASK__ASK_EP = 1, - TMQ_DELAYED_TASK__COMMIT, -}; - -typedef struct SVgOffsetInfo { +typedef struct { STqOffsetVal committedOffset; STqOffsetVal endOffset; // the last version in TAOS_RES + 1 STqOffsetVal beginOffset; // the first version in TAOS_RES @@ -173,23 +183,32 @@ typedef struct { } SMqClientTopic; typedef struct { - int32_t code; - int8_t tmqRspType; - int32_t epoch; // epoch can be used to guard the vgHandle int32_t vgId; char topicName[TSDB_TOPIC_FNAME_LEN]; - SMqClientVg* vgHandle; SMqClientTopic* topicHandle; uint64_t reqId; SEpSet* pEpset; union { + struct{ + SMqRspHead head; + STqOffsetVal rspOffset; + }; SMqDataRsp dataRsp; SMqMetaRsp metaRsp; - STaosxRsp taosxRsp; SMqBatchMetaRsp batchMetaRsp; }; } SMqPollRspWrapper; +typedef struct { + int32_t code; + int8_t tmqRspType; + int32_t epoch; + union{ + SMqPollRspWrapper pollRsp; + SMqAskEpRsp epRsp; + }; +} SMqRspWrapper; + typedef struct { tsem2_t rspSem; int32_t rspErr; @@ -208,7 +227,7 @@ typedef struct { uint64_t requestId; // request id for debug purpose } SMqPollCbParam; -typedef struct SMqVgCommon { +typedef struct { tsem2_t rsp; int32_t numOfRsp; SArray* pList; @@ -218,18 +237,18 @@ typedef struct SMqVgCommon { int32_t code; } SMqVgCommon; -typedef struct SMqSeekParam { +typedef struct { tsem2_t sem; int32_t code; } SMqSeekParam; -typedef struct SMqCommittedParam { +typedef struct { tsem2_t sem; int32_t code; SMqVgOffset vgOffset; } SMqCommittedParam; -typedef struct SMqVgWalInfoParam { +typedef struct { int32_t vgId; int32_t epoch; int32_t totalReq; @@ -252,17 +271,23 @@ typedef struct { int64_t consumerId; } SMqCommitCbParam; -typedef struct SSyncCommitInfo { +typedef struct { tsem2_t sem; int32_t code; } SSyncCommitInfo; -static int32_t syncAskEp(tmq_t* tmq); -static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet); -static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffsetVal* offset, const char* pTopicName, - SMqCommitCbParamSet* pParamSet); -static int32_t commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId); -static int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpset); +typedef struct { + STqOffsetVal currentOffset; + STqOffsetVal commitOffset; + STqOffsetVal seekOffset; + int64_t numOfRows; + int32_t vgStatus; +} SVgroupSaveInfo; + +static TdThreadOnce tmqInit = PTHREAD_ONCE_INIT; // initialize only once +volatile int32_t tmqInitRes = 0; // initialize rsp code +static SMqMgmt tmqMgmt = {0}; +static int8_t pollFlag = 0; tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); @@ -300,7 +325,7 @@ void tmq_conf_destroy(tmq_conf_t* conf) { tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { int32_t code = 0; if (conf == NULL || key == NULL || value == NULL) { - tscError("tmq_conf_set null, conf:%p key:%p value:%p", conf, key, value); + tqErrorC("tmq_conf_set null, conf:%p key:%p value:%p", conf, key, value); return TMQ_CONF_INVALID; } if (strcasecmp(key, "group.id") == 0) { @@ -321,7 +346,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value conf->autoCommit = false; return TMQ_CONF_OK; } else { - tscError("invalid value for enable.auto.commit: %s", value); + tqErrorC("invalid value for enable.auto.commit: %s", value); return TMQ_CONF_INVALID; } } @@ -330,7 +355,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value int64_t tmp; code = taosStr2int64(value, &tmp); if (tmp < 0 || code != 0) { - tscError("invalid value for auto.commit.interval.ms: %s", value); + tqErrorC("invalid value for auto.commit.interval.ms: %s", value); return TMQ_CONF_INVALID; } conf->autoCommitInterval = (tmp > INT32_MAX ? INT32_MAX : tmp); @@ -341,7 +366,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value int64_t tmp; code = taosStr2int64(value, &tmp); if (tmp < 6000 || tmp > 1800000 || code != 0) { - tscError("invalid value for session.timeout.ms: %s", value); + tqErrorC("invalid value for session.timeout.ms: %s", value); return TMQ_CONF_INVALID; } conf->sessionTimeoutMs = tmp; @@ -352,7 +377,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value int64_t tmp; code = taosStr2int64(value, &tmp); if (tmp < 1000 || tmp >= conf->sessionTimeoutMs || code != 0) { - tscError("invalid value for heartbeat.interval.ms: %s", value); + tqErrorC("invalid value for heartbeat.interval.ms: %s", value); return TMQ_CONF_INVALID; } conf->heartBeatIntervalMs = tmp; @@ -363,7 +388,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value int32_t tmp; code = taosStr2int32(value, &tmp); if (tmp < 1000 || code != 0) { - tscError("invalid value for max.poll.interval.ms: %s", value); + tqErrorC("invalid value for max.poll.interval.ms: %s", value); return TMQ_CONF_INVALID; } conf->maxPollIntervalMs = tmp; @@ -381,7 +406,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value conf->resetOffset = TMQ_OFFSET__RESET_LATEST; return TMQ_CONF_OK; } else { - tscError("invalid value for auto.offset.reset: %s", value); + tqErrorC("invalid value for auto.offset.reset: %s", value); return TMQ_CONF_INVALID; } } @@ -394,7 +419,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value conf->withTbName = false; return TMQ_CONF_OK; } else { - tscError("invalid value for msg.with.table.name: %s", value); + tqErrorC("invalid value for msg.with.table.name: %s", value); return TMQ_CONF_INVALID; } } @@ -407,7 +432,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value conf->snapEnable = false; return TMQ_CONF_OK; } else { - tscError("invalid value for experimental.snapshot.enable: %s", value); + tqErrorC("invalid value for experimental.snapshot.enable: %s", value); return TMQ_CONF_INVALID; } } @@ -415,7 +440,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value if (strcasecmp(key, "td.connect.ip") == 0) { void *tmp = taosStrdup(value); if (tmp == NULL) { - tscError("tmq_conf_set out of memory:%d", terrno); + tqErrorC("tmq_conf_set out of memory:%d", terrno); return TMQ_CONF_INVALID; } conf->ip = tmp; @@ -425,7 +450,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value if (strcasecmp(key, "td.connect.user") == 0) { void *tmp = taosStrdup(value); if (tmp == NULL) { - tscError("tmq_conf_set out of memory:%d", terrno); + tqErrorC("tmq_conf_set out of memory:%d", terrno); return TMQ_CONF_INVALID; } conf->user = tmp; @@ -435,7 +460,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value if (strcasecmp(key, "td.connect.pass") == 0) { void *tmp = taosStrdup(value); if (tmp == NULL) { - tscError("tmq_conf_set out of memory:%d", terrno); + tqErrorC("tmq_conf_set out of memory:%d", terrno); return TMQ_CONF_INVALID; } conf->pass = tmp; @@ -446,7 +471,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value int64_t tmp; code = taosStr2int64(value, &tmp); if (tmp <= 0 || tmp > 65535 || code != 0) { - tscError("invalid value for td.connect.port: %s", value); + tqErrorC("invalid value for td.connect.port: %s", value); return TMQ_CONF_INVALID; } @@ -462,7 +487,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value conf->replayEnable = false; return TMQ_CONF_OK; } else { - tscError("invalid value for enable.replay: %s", value); + tqErrorC("invalid value for enable.replay: %s", value); return TMQ_CONF_INVALID; } } @@ -484,16 +509,22 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_OK; } - tscError("unknown key: %s", key); + tqErrorC("unknown key: %s", key); return TMQ_CONF_UNKNOWN; } -tmq_list_t* tmq_list_new() { return (tmq_list_t*)taosArrayInit(0, sizeof(void*)); } +tmq_list_t* tmq_list_new() { + return (tmq_list_t*)taosArrayInit(0, sizeof(void*)); +} int32_t tmq_list_append(tmq_list_t* list, const char* src) { - if (list == NULL) return TSDB_CODE_INVALID_PARA; + if (list == NULL) { + return TSDB_CODE_INVALID_PARA; + } SArray* container = &list->container; - if (src == NULL || src[0] == 0) return TSDB_CODE_INVALID_PARA; + if (src == NULL || src[0] == 0) { + return TSDB_CODE_INVALID_PARA; + } char* topic = taosStrdup(src); if (topic == NULL) return terrno; if (taosArrayPush(container, &topic) == NULL) { @@ -510,24 +541,66 @@ void tmq_list_destroy(tmq_list_t* list) { } int32_t tmq_list_get_size(const tmq_list_t* list) { - if (list == NULL) return -1; + if (list == NULL) { + return TSDB_CODE_INVALID_PARA; + } const SArray* container = &list->container; return taosArrayGetSize(container); } char** tmq_list_to_c_array(const tmq_list_t* list) { - if (list == NULL) return NULL; + if (list == NULL) { + return NULL; + } const SArray* container = &list->container; return container->pData; } +static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { + int64_t refId = pParamSet->refId; + int32_t code = 0; + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); + if (tmq == NULL) { + code = TSDB_CODE_TMQ_CONSUMER_CLOSED; + } + + // if no more waiting rsp + if (pParamSet->callbackFn != NULL) { + pParamSet->callbackFn(tmq, pParamSet->code, pParamSet->userParam); + } + + taosMemoryFree(pParamSet); + if (tmq != NULL) { + code = taosReleaseRef(tmqMgmt.rsetId, refId); + } + + return code; +} + +static int32_t commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId) { + int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); + if (waitingRspNum == 0) { + tqDebugC("consumer:0x%" PRIx64 " topic:%s vgId:%d all commit-rsp received, commit completed", consumerId, pTopic, + vgId); + return tmqCommitDone(pParamSet); + } else { + tqDebugC("consumer:0x%" PRIx64 " topic:%s vgId:%d commit-rsp received, remain:%d", consumerId, pTopic, vgId, + waitingRspNum); + } + return 0; +} + static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { + if (pBuf){ + taosMemoryFreeClear(pBuf->pData); + taosMemoryFreeClear(pBuf->pEpSet); + } + if(param == NULL){ + return TSDB_CODE_INVALID_PARA; + } SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; - taosMemoryFree(pBuf->pData); - taosMemoryFree(pBuf->pEpSet); - return commitRspCountDown(pParamSet, pParam->consumerId, pParam->topicName, pParam->vgId); } @@ -598,7 +671,6 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, NULL, pMsgSendInfo); if (code != 0) { (void)atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); - return code; } return code; } @@ -614,7 +686,7 @@ static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic return 0; } - tscError("consumer:0x%" PRIx64 ", total:%d, failed to find topic:%s", tmq->consumerId, numOfTopics, pTopicName); + tqErrorC("consumer:0x%" PRIx64 ", total:%d, failed to find topic:%s", tmq->consumerId, numOfTopics, pTopicName); return TSDB_CODE_TMQ_INVALID_TOPIC; } @@ -638,7 +710,7 @@ static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClient SMqClientTopic* pTopic = NULL; int32_t code = getTopicByName(tmq, pTopicName, &pTopic); if (code != 0) { - tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); + tqErrorC("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); return code; } @@ -654,22 +726,15 @@ static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClient return *pVg == NULL ? TSDB_CODE_TMQ_INVALID_VGID : TSDB_CODE_SUCCESS; } -static int32_t asyncCommitOffset(tmq_t* tmq, char* pTopicName, int32_t vgId, STqOffsetVal* offsetVal, - tmq_commit_cb* pCommitFp, void* userParam) { - tscInfo("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); - taosRLockLatch(&tmq->lock); - SMqClientVg* pVg = NULL; - int32_t code = getClientVg(tmq, pTopicName, vgId, &pVg); - if (code != 0) { - goto end; - } +static int32_t innerCommit(tmq_t* tmq, char* pTopicName, STqOffsetVal* offsetVal, SMqClientVg* pVg, SMqCommitCbParamSet* pParamSet){ + int32_t code = 0; if (offsetVal->type <= 0) { code = TSDB_CODE_TMQ_INVALID_MSG; - goto end; + return code; } if (tOffsetEqual(offsetVal, &pVg->offsetInfo.committedOffset)) { code = TSDB_CODE_TMQ_SAME_COMMITTED_VALUE; - goto end; + return code; } char offsetBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(offsetBuf, tListLen(offsetBuf), offsetVal); @@ -677,26 +742,39 @@ static int32_t asyncCommitOffset(tmq_t* tmq, char* pTopicName, int32_t vgId, STq char commitBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); - SMqCommitCbParamSet* pParamSet = NULL; - code = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 0, &pParamSet); - if (code != 0) { - goto end; - } - code = doSendCommitMsg(tmq, pVg->vgId, &pVg->epSet, offsetVal, pTopicName, pParamSet); if (code != TSDB_CODE_SUCCESS) { - tscError("consumer:0x%" PRIx64 " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s", + tqErrorC("consumer:0x%" PRIx64 " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s", tmq->consumerId, pTopicName, pVg->vgId, offsetBuf, commitBuf, tstrerror(terrno)); - taosMemoryFree(pParamSet); - goto end; + return code; } - tscInfo("consumer:0x%" PRIx64 " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s", + tqDebugC("consumer:0x%" PRIx64 " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s", tmq->consumerId, pTopicName, pVg->vgId, offsetBuf, commitBuf); tOffsetCopy(&pVg->offsetInfo.committedOffset, offsetVal); + return code; +} -end: +static int32_t asyncCommitOffset(tmq_t* tmq, char* pTopicName, int32_t vgId, STqOffsetVal* offsetVal, + tmq_commit_cb* pCommitFp, void* userParam) { + tqInfoC("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); + SMqCommitCbParamSet* pParamSet = NULL; + int32_t code = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 0, &pParamSet); + if (code != 0){ + return code; + } + + taosRLockLatch(&tmq->lock); + SMqClientVg* pVg = NULL; + code = getClientVg(tmq, pTopicName, vgId, &pVg); + if (code == 0) { + code = innerCommit(tmq, pTopicName, offsetVal, pVg, pParamSet); + } taosRUnLockLatch(&tmq->lock); + + if (code != 0){ + taosMemoryFree(pParamSet); + } return code; } @@ -711,26 +789,12 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c goto end; } - if (TD_RES_TMQ(pRes)) { + if (TD_RES_TMQ(pRes) || TD_RES_TMQ_META(pRes) || + TD_RES_TMQ_METADATA(pRes) || TD_RES_TMQ_BATCH_META(pRes)) { SMqRspObj* pRspObj = (SMqRspObj*)pRes; - pTopicName = pRspObj->common.topic; - vgId = pRspObj->common.vgId; - offsetVal = pRspObj->rsp.common.rspOffset; - } else if (TD_RES_TMQ_META(pRes)) { - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)pRes; - pTopicName = pMetaRspObj->topic; - vgId = pMetaRspObj->vgId; - offsetVal = pMetaRspObj->metaRsp.rspOffset; - } else if (TD_RES_TMQ_METADATA(pRes)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)pRes; - pTopicName = pRspObj->common.topic; - vgId = pRspObj->common.vgId; - offsetVal = pRspObj->rsp.common.rspOffset; - } else if (TD_RES_TMQ_BATCH_META(pRes)) { - SMqBatchMetaRspObj* pBtRspObj = (SMqBatchMetaRspObj*)pRes; - pTopicName = pBtRspObj->common.topic; - vgId = pBtRspObj->common.vgId; - offsetVal = pBtRspObj->rsp.rspOffset; + pTopicName = pRspObj->topic; + vgId = pRspObj->vgId; + offsetVal = pRspObj->rspOffset; } else { code = TSDB_CODE_TMQ_INVALID_MSG; goto end; @@ -745,83 +809,61 @@ end: } } -static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam) { +static int32_t innerCommitAll(tmq_t* tmq, SMqCommitCbParamSet* pParamSet){ int32_t code = 0; - // init as 1 to prevent concurrency issue - SMqCommitCbParamSet* pParamSet = NULL; - code = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 1, &pParamSet); - if (code != 0) { - goto end; - } taosRLockLatch(&tmq->lock); int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); - tscDebug("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics); + tqDebugC("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics); for (int32_t i = 0; i < numOfTopics; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); if (pTopic == NULL) { code = TSDB_CODE_TMQ_INVALID_TOPIC; - taosRUnLockLatch(&tmq->lock); - goto end; + goto END; } int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); - tscDebug("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, - numOfVgroups); + tqDebugC("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, numOfVgroups); for (int32_t j = 0; j < numOfVgroups; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); if (pVg == NULL) { code = TSDB_CODE_INVALID_PARA; - taosRUnLockLatch(&tmq->lock); - goto end; + goto END; } - if (pVg->offsetInfo.endOffset.type > 0 && - !tOffsetEqual(&pVg->offsetInfo.endOffset, &pVg->offsetInfo.committedOffset)) { - char offsetBuf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(offsetBuf, tListLen(offsetBuf), &pVg->offsetInfo.endOffset); - char commitBuf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); - - code = doSendCommitMsg(tmq, pVg->vgId, &pVg->epSet, &pVg->offsetInfo.endOffset, pTopic->topicName, pParamSet); - if (code != TSDB_CODE_SUCCESS) { - tscError("consumer:0x%" PRIx64 - " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, tstrerror(terrno), j + 1, - numOfVgroups); - continue; - } - - tscDebug("consumer:0x%" PRIx64 - " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s, ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, j + 1, numOfVgroups); - tOffsetCopy(&pVg->offsetInfo.committedOffset, &pVg->offsetInfo.endOffset); - } else { - tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, current:%" PRId64 ", ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups); + code = innerCommit(tmq, pTopic->topicName, &pVg->offsetInfo.endOffset, pVg, pParamSet); + if (code != 0){ + tqDebugC("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, code:%s, current offset version:%" PRId64 ", ordinal:%d/%d", + tmq->consumerId, pTopic->topicName, pVg->vgId, tstrerror(code), pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups); } } } - taosRUnLockLatch(&tmq->lock); - - tscDebug("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1, + tqDebugC("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - DEFAULT_COMMIT_CNT, numOfTopics); +END: + taosRUnLockLatch(&tmq->lock); + return code; +} - // request is sent - if (pParamSet->waitingRspNum != 1) { - // count down since waiting rsp num init as 1 - code = commitRspCountDown(pParamSet, tmq->consumerId, "", 0); - if (code != 0) { - tscError("consumer:0x%" PRIx64 " commit rsp count down failed, code:%s", tmq->consumerId, tstrerror(code)); - pParamSet = NULL; - goto end; +static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam) { + int32_t code = 0; + SMqCommitCbParamSet* pParamSet = NULL; + // init waitingRspNum as DEFAULT_COMMIT_CNT to prevent concurrency issue + code = prepareCommitCbParamSet(tmq, pCommitFp, userParam, DEFAULT_COMMIT_CNT, &pParamSet); + if (code != 0) { + tqErrorC("consumer:0x%" PRIx64 " prepareCommitCbParamSet failed, code:%s", tmq->consumerId, tstrerror(code)); + if (pCommitFp != NULL) { + pCommitFp(tmq, code, userParam); } return; } + code = innerCommitAll(tmq, pParamSet); + if (code != 0){ + tqErrorC("consumer:0x%" PRIx64 " innerCommitAll failed, code:%s", tmq->consumerId, tstrerror(code)); + } -end: - taosMemoryFree(pParamSet); - if (pCommitFp != NULL) { - pCommitFp(tmq, code, userParam); + code = commitRspCountDown(pParamSet, tmq->consumerId, "init", -1); + if (code != 0) { + tqErrorC("consumer:0x%" PRIx64 " commit rsp count down failed, code:%s", tmq->consumerId, tstrerror(code)); } return; } @@ -839,7 +881,7 @@ static void generateTimedTask(int64_t refId, int32_t type) { *pTaskType = type; if (taosWriteQitem(tmq->delayedTask, pTaskType) == 0) { if (tsem2_post(&tmq->rspSem) != 0){ - tscError("consumer:0x%" PRIx64 " failed to post sem, type:%d", tmq->consumerId, type); + tqErrorC("consumer:0x%" PRIx64 " failed to post sem, type:%d", tmq->consumerId, type); } }else{ taosFreeQitem(pTaskType); @@ -848,7 +890,7 @@ static void generateTimedTask(int64_t refId, int32_t type) { code = taosReleaseRef(tmqMgmt.rsetId, refId); if (code != 0){ - tscError("failed to release ref:%"PRId64 ", type:%d, code:%d", refId, type, code); + tqErrorC("failed to release ref:%"PRId64 ", type:%d, code:%d", refId, type, code); } } @@ -863,11 +905,11 @@ void tmqReplayTask(void* param, void* tmrId) { if (tmq == NULL) return; if (tsem2_post(&tmq->rspSem) != 0){ - tscError("consumer:0x%" PRIx64 " failed to post sem, replay", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 " failed to post sem, replay", tmq->consumerId); } int32_t code = taosReleaseRef(tmqMgmt.rsetId, refId); if (code != 0){ - tscError("failed to release ref:%"PRId64 ", code:%d", refId, code); + tqErrorC("failed to release ref:%"PRId64 ", code:%d", refId, code); } } @@ -877,11 +919,11 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) { } int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { - if (code != 0) { - goto END; + if (pMsg == NULL) { + return TSDB_CODE_INVALID_PARA; } - if (pMsg == NULL || param == NULL) { - code = TSDB_CODE_INVALID_PARA; + + if (param == NULL || code != 0){ goto END; } @@ -902,7 +944,7 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { for (int32_t j = 0; j < topicNumCur; j++) { SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j); if (pTopicCur && strcmp(pTopicCur->topicName, privilege->topic) == 0) { - tscInfo("consumer:0x%" PRIx64 ", has no privilege, topic:%s", tmq->consumerId, privilege->topic); + tqInfoC("consumer:0x%" PRIx64 ", has no privilege, topic:%s", tmq->consumerId, privilege->topic); pTopicCur->noPrivilege = 1; } } @@ -911,10 +953,11 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { taosWUnLockLatch(&tmq->lock); code = taosReleaseRef(tmqMgmt.rsetId, refId); if (code != 0){ - tscError("failed to release ref:%"PRId64 ", code:%d", refId, code); + tqErrorC("failed to release ref:%"PRId64 ", code:%d", refId, code); } } + tqClientDebug = rsp.debugFlag; tDestroySMqHbRsp(&rsp); END: @@ -937,7 +980,7 @@ void tmqSendHbReq(void* param, void* tmrId) { req.pollFlag = atomic_load_8(&pollFlag); req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows)); if (req.topics == NULL) { - return; + goto END; } taosRLockLatch(&tmq->lock); for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { @@ -970,7 +1013,7 @@ void tmqSendHbReq(void* param, void* tmrId) { offRows->ever = pVg->offsetInfo.walVerEnd == -1 ? 0 : pVg->offsetInfo.walVerEnd; char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset); - tscDebug("consumer:0x%" PRIx64 ",report offset, group:%s vgId:%d, offset:%s/%" PRId64 ", rows:%" PRId64, + tqDebugC("consumer:0x%" PRIx64 ",report offset, group:%s vgId:%d, offset:%s/%" PRId64 ", rows:%" PRId64, tmq->consumerId, tmq->groupId, offRows->vgId, buf, offRows->ever, offRows->rows); } } @@ -978,26 +1021,26 @@ void tmqSendHbReq(void* param, void* tmrId) { int32_t tlen = tSerializeSMqHbReq(NULL, 0, &req); if (tlen < 0) { - tscError("tSerializeSMqHbReq failed, size:%d", tlen); - goto OVER; + tqErrorC("tSerializeSMqHbReq failed, size:%d", tlen); + goto END; } void* pReq = taosMemoryCalloc(1, tlen); if (pReq == NULL) { - tscError("failed to malloc MqHbReq msg, code:%d", terrno); - goto OVER; + tqErrorC("failed to malloc MqHbReq msg, code:%d", terrno); + goto END; } if (tSerializeSMqHbReq(pReq, tlen, &req) < 0) { - tscError("tSerializeSMqHbReq %d failed", tlen); + tqErrorC("tSerializeSMqHbReq %d failed", tlen); taosMemoryFree(pReq); - goto OVER; + goto END; } SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { taosMemoryFree(pReq); - goto OVER; + goto END; } sendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = tlen, .handle = NULL}; @@ -1012,35 +1055,351 @@ void tmqSendHbReq(void* param, void* tmrId) { int32_t code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); if (code != 0) { - tscError("tmqSendHbReq asyncSendMsgToServer failed"); + tqErrorC("tmqSendHbReq asyncSendMsgToServer failed"); } - (void)atomic_val_compare_exchange_8(&pollFlag, 1, 0); -OVER: + +END: tDestroySMqHbReq(&req); if (tmrId != NULL) { bool ret = taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer); - tscDebug("reset timer fo tmq hb:%d", ret); + tqDebugC("reset timer fo tmq hb:%d", ret); } int32_t ret = taosReleaseRef(tmqMgmt.rsetId, refId); if (ret != 0){ - tscError("failed to release ref:%"PRId64 ", code:%d", refId, ret); + tqErrorC("failed to release ref:%"PRId64 ", code:%d", refId, ret); } } static void defaultCommitCbFn(tmq_t* pTmq, int32_t code, void* param) { if (code != 0) { - tscError("consumer:0x%" PRIx64 ", failed to commit offset, code:%s", pTmq->consumerId, tstrerror(code)); + tqErrorC("consumer:0x%" PRIx64 ", failed to commit offset, code:%s", pTmq->consumerId, tstrerror(code)); } } +static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { + if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { + tDeleteSMqAskEpRsp(&rspWrapper->epRsp); + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { + DELETE_POLL_RSP(tDeleteMqDataRsp, &pRsp->dataRsp) + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP){ + DELETE_POLL_RSP(tDeleteSTaosxRsp, &pRsp->dataRsp) + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { + DELETE_POLL_RSP(tDeleteMqMetaRsp,&pRsp->metaRsp) + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { + DELETE_POLL_RSP(tDeleteMqBatchMetaRsp,&pRsp->batchMetaRsp) + } +} + +static void freeClientVg(void* param) { + SMqClientVg* pVg = param; + tOffsetDestroy(&pVg->offsetInfo.endOffset); + tOffsetDestroy(&pVg->offsetInfo.beginOffset); + tOffsetDestroy(&pVg->offsetInfo.committedOffset); +} +static void freeClientTopic(void* param) { + SMqClientTopic* pTopic = param; + taosMemoryFreeClear(pTopic->schema.pSchema); + taosArrayDestroyEx(pTopic->vgs, freeClientVg); +} + +static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopicEp, SHashObj* pVgOffsetHashMap, + tmq_t* tmq) { + pTopic->schema = pTopicEp->schema; + pTopicEp->schema.nCols = 0; + pTopicEp->schema.pSchema = NULL; + + char vgKey[TSDB_TOPIC_FNAME_LEN + 22] = {0}; + int32_t vgNumGet = taosArrayGetSize(pTopicEp->vgs); + + tstrncpy(pTopic->topicName, pTopicEp->topic, TSDB_TOPIC_FNAME_LEN); + tstrncpy(pTopic->db, pTopicEp->db, TSDB_DB_FNAME_LEN); + + tqInfoC("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d", tmq->consumerId, pTopic->topicName, vgNumGet); + pTopic->vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); + if (pTopic->vgs == NULL) { + tqErrorC("consumer:0x%" PRIx64 ", failed to init vgs for topic:%s", tmq->consumerId, pTopic->topicName); + return; + } + for (int32_t j = 0; j < vgNumGet; j++) { + SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); + if (pVgEp == NULL) { + continue; + } + (void)sprintf(vgKey, "%s:%d", pTopic->topicName, pVgEp->vgId); + SVgroupSaveInfo* pInfo = taosHashGet(pVgOffsetHashMap, vgKey, strlen(vgKey)); + + STqOffsetVal offsetNew = {0}; + offsetNew.type = tmq->resetOffsetCfg; + + tqInfoC("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d, num:%d, port:%d", tmq->consumerId, + pTopic->topicName, vgNumGet, pVgEp->epSet.numOfEps, pVgEp->epSet.eps[pVgEp->epSet.inUse].port); + + SMqClientVg clientVg = { + .pollCnt = 0, + .vgId = pVgEp->vgId, + .epSet = pVgEp->epSet, + .vgStatus = pInfo ? pInfo->vgStatus : TMQ_VG_STATUS__IDLE, + .vgSkipCnt = 0, + .emptyBlockReceiveTs = 0, + .blockReceiveTs = 0, + .blockSleepForReplay = 0, + .numOfRows = pInfo ? pInfo->numOfRows : 0, + }; + + clientVg.offsetInfo.walVerBegin = -1; + clientVg.offsetInfo.walVerEnd = -1; + clientVg.seekUpdated = false; + if (pInfo) { + tOffsetCopy(&clientVg.offsetInfo.endOffset, &pInfo->currentOffset); + tOffsetCopy(&clientVg.offsetInfo.committedOffset, &pInfo->commitOffset); + tOffsetCopy(&clientVg.offsetInfo.beginOffset, &pInfo->seekOffset); + } else { + clientVg.offsetInfo.endOffset = offsetNew; + clientVg.offsetInfo.committedOffset = offsetNew; + clientVg.offsetInfo.beginOffset = offsetNew; + } + if (taosArrayPush(pTopic->vgs, &clientVg) == NULL) { + tqErrorC("consumer:0x%" PRIx64 ", failed to push vg:%d into topic:%s", tmq->consumerId, pVgEp->vgId, + pTopic->topicName); + freeClientVg(&clientVg); + } + } +} + +static void buildNewTopicList(tmq_t* tmq, SArray* newTopics, const SMqAskEpRsp* pRsp){ + SHashObj* pVgOffsetHashMap = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); + if (pVgOffsetHashMap == NULL) { + tqErrorC("consumer:0x%" PRIx64 " taos hash init null, code:%d", tmq->consumerId, terrno); + return; + } + + int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); + for (int32_t i = 0; i < topicNumCur; i++) { + // find old topic + SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); + if (pTopicCur && pTopicCur->vgs) { + int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); + tqInfoC("consumer:0x%" PRIx64 ", current vg num: %d", tmq->consumerId, vgNumCur); + for (int32_t j = 0; j < vgNumCur; j++) { + SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); + if (pVgCur == NULL) { + continue; + } + char vgKey[TSDB_TOPIC_FNAME_LEN + 22] = {0}; + (void)sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); + + char buf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &pVgCur->offsetInfo.endOffset); + tqInfoC("consumer:0x%" PRIx64 ", vgId:%d vgKey:%s, offset:%s", tmq->consumerId, pVgCur->vgId, vgKey, buf); + + SVgroupSaveInfo info = {.currentOffset = pVgCur->offsetInfo.endOffset, + .seekOffset = pVgCur->offsetInfo.beginOffset, + .commitOffset = pVgCur->offsetInfo.committedOffset, + .numOfRows = pVgCur->numOfRows, + .vgStatus = pVgCur->vgStatus}; + if (taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)) != 0) { + tqErrorC("consumer:0x%" PRIx64 ", failed to put vg:%d into hashmap", tmq->consumerId, pVgCur->vgId); + } + } + } + } + + for (int32_t i = 0; i < taosArrayGetSize(pRsp->topics); i++) { + SMqClientTopic topic = {0}; + SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i); + if (pTopicEp == NULL) { + continue; + } + initClientTopicFromRsp(&topic, pTopicEp, pVgOffsetHashMap, tmq); + if (taosArrayPush(newTopics, &topic) == NULL) { + tqErrorC("consumer:0x%" PRIx64 ", failed to push topic:%s into new topics", tmq->consumerId, topic.topicName); + freeClientTopic(&topic); + } + } + + taosHashCleanup(pVgOffsetHashMap); +} + +static void doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { + int32_t topicNumGet = taosArrayGetSize(pRsp->topics); + // vnode transform (epoch == tmq->epoch && topicNumGet != 0) + // ask ep rsp (epoch == tmq->epoch && topicNumGet == 0) + if (epoch < tmq->epoch || (epoch == tmq->epoch && topicNumGet == 0)) { + tqDebugC("consumer:0x%" PRIx64 " no update ep epoch from %d to epoch %d, incoming topics:%d", tmq->consumerId, + tmq->epoch, epoch, topicNumGet); + return; + } + + SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); + if (newTopics == NULL) { + tqErrorC("consumer:0x%" PRIx64 " taos array init null, code:%d", tmq->consumerId, terrno); + return; + } + tqInfoC("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d", + tmq->consumerId, tmq->epoch, epoch, topicNumGet, (int)taosArrayGetSize(tmq->clientTopics)); + + taosWLockLatch(&tmq->lock); + if (topicNumGet > 0){ + buildNewTopicList(tmq, newTopics, pRsp); + } + // destroy current buffered existed topics info + if (tmq->clientTopics) { + taosArrayDestroyEx(tmq->clientTopics, freeClientTopic); + } + tmq->clientTopics = newTopics; + taosWUnLockLatch(&tmq->lock); + + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); + atomic_store_32(&tmq->epoch, epoch); + + tqInfoC("consumer:0x%" PRIx64 " update topic info completed", tmq->consumerId); +} + +static int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { + SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; + if (pParam == NULL) { + goto FAIL; + } + + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); + if (tmq == NULL) { + code = TSDB_CODE_TMQ_CONSUMER_CLOSED; + goto FAIL; + } + + if (code != TSDB_CODE_SUCCESS) { + tqErrorC("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code)); + goto END; + } + + if (pMsg == NULL) { + goto END; + } + SMqRspHead* head = pMsg->pData; + int32_t epoch = atomic_load_32(&tmq->epoch); + tqDebugC("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch); + if (pParam->sync) { + SMqAskEpRsp rsp = {0}; + if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp) != NULL) { + doUpdateLocalEp(tmq, head->epoch, &rsp); + } + tDeleteSMqAskEpRsp(&rsp); + } else { + SMqRspWrapper* pWrapper = NULL; + code = taosAllocateQitem(sizeof(SMqRspWrapper), DEF_QITEM, 0, (void**)&pWrapper); + if (code) { + goto END; + } + + pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP; + pWrapper->epoch = head->epoch; + (void)memcpy(&pWrapper->epRsp, pMsg->pData, sizeof(SMqRspHead)); + if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->epRsp) == NULL) { + tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); + taosFreeQitem(pWrapper); + } else { + code = taosWriteQitem(tmq->mqueue, pWrapper); + if (code != 0) { + tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); + taosFreeQitem(pWrapper); + tqErrorC("consumer:0x%" PRIx64 " put ep res into mqueue failed, code:%d", tmq->consumerId, code); + } + } + } + + END: + { + int32_t ret = taosReleaseRef(tmqMgmt.rsetId, pParam->refId); + if (ret != 0){ + tqErrorC("failed to release ref:%"PRId64 ", code:%d", pParam->refId, ret); + } + } + + FAIL: + if (pParam && pParam->sync) { + SAskEpInfo* pInfo = pParam->pParam; + if (pInfo) { + pInfo->code = code; + if (tsem2_post(&pInfo->sem) != 0){ + tqErrorC("failed to post rsp sem askep cb"); + } + } + } + + if (pMsg) { + taosMemoryFree(pMsg->pEpSet); + taosMemoryFree(pMsg->pData); + } + + return code; +} + +static int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { + SMqAskEpReq req = {0}; + req.consumerId = pTmq->consumerId; + req.epoch = updateEpSet ? -1 : pTmq->epoch; + tstrncpy(req.cgroup, pTmq->groupId, TSDB_CGROUP_LEN); + int code = 0; + SMqAskEpCbParam* pParam = NULL; + void* pReq = NULL; + + int32_t tlen = tSerializeSMqAskEpReq(NULL, 0, &req); + if (tlen < 0) { + tqErrorC("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq failed", pTmq->consumerId); + return TSDB_CODE_INVALID_PARA; + } + + pReq = taosMemoryCalloc(1, tlen); + if (pReq == NULL) { + tqErrorC("consumer:0x%" PRIx64 ", failed to malloc askEpReq msg, size:%d", pTmq->consumerId, tlen); + return terrno; + } + + if (tSerializeSMqAskEpReq(pReq, tlen, &req) < 0) { + tqErrorC("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq %d failed", pTmq->consumerId, tlen); + taosMemoryFree(pReq); + return TSDB_CODE_INVALID_PARA; + } + + pParam = taosMemoryCalloc(1, sizeof(SMqAskEpCbParam)); + if (pParam == NULL) { + tqErrorC("consumer:0x%" PRIx64 ", failed to malloc subscribe param", pTmq->consumerId); + taosMemoryFree(pReq); + return terrno; + } + + pParam->refId = pTmq->refId; + pParam->sync = sync; + pParam->pParam = param; + + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + taosMemoryFree(pReq); + taosMemoryFree(pParam); + return terrno; + } + + sendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = tlen, .handle = NULL}; + sendInfo->requestId = generateRequestId(); + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->paramFreeFp = taosMemoryFree; + sendInfo->fp = askEpCb; + sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; + + SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp); + tqDebugC("consumer:0x%" PRIx64 " ask ep from mnode,QID:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId); + return asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); +} + void tmqHandleAllDelayedTask(tmq_t* pTmq) { STaosQall* qall = NULL; int32_t code = 0; code = taosAllocateQall(&qall); if (code) { - tscError("consumer:0x%" PRIx64 ", failed to allocate qall, code:%s", pTmq->consumerId, tstrerror(code)); + tqErrorC("consumer:0x%" PRIx64 ", failed to allocate qall, code:%s", pTmq->consumerId, tstrerror(code)); return; } @@ -1050,29 +1409,30 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) { return; } - tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, numOfItems); + tqDebugC("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, numOfItems); int8_t* pTaskType = NULL; while (taosGetQitem(qall, (void**)&pTaskType) != 0) { if (*pTaskType == TMQ_DELAYED_TASK__ASK_EP) { + tqDebugC("consumer:0x%" PRIx64 " retrieve ask ep timer", pTmq->consumerId); code = askEp(pTmq, NULL, false, false); if (code != 0) { - tscError("consumer:0x%" PRIx64 " failed to ask ep, code:%s", pTmq->consumerId, tstrerror(code)); + tqErrorC("consumer:0x%" PRIx64 " failed to ask ep, code:%s", pTmq->consumerId, tstrerror(code)); continue; } - tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId); + tqDebugC("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId); bool ret = taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, - &pTmq->epTimer); - tscDebug("reset timer fo tmq ask ep:%d", ret); + &pTmq->epTimer); + tqDebugC("reset timer fo tmq ask ep:%d", ret); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { - tmq_commit_cb* pCallbackFn = pTmq->commitCb ? pTmq->commitCb : defaultCommitCbFn; + tmq_commit_cb* pCallbackFn = (pTmq->commitCb != NULL) ? pTmq->commitCb : defaultCommitCbFn; asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam); - tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId, + tqDebugC("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId, pTmq->autoCommitInterval / 1000.0); bool ret = taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, - &pTmq->commitTimer); - tscDebug("reset timer fo commit:%d", ret); + &pTmq->commitTimer); + tqDebugC("reset timer fo commit:%d", ret); } else { - tscError("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType); + tqErrorC("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType); } taosFreeQitem(pTaskType); @@ -1081,35 +1441,11 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) { taosFreeQall(qall); } -static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { - if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { - SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; - tDeleteSMqAskEpRsp(&pEpRspWrapper->msg); - } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { - SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; - taosMemoryFreeClear(pRsp->pEpset); - - tDeleteMqDataRsp(&pRsp->dataRsp); - } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { - SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; - taosMemoryFreeClear(pRsp->pEpset); - tDeleteMqMetaRsp(&pRsp->metaRsp); - } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { - SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; - taosMemoryFreeClear(pRsp->pEpset); - tDeleteSTaosxRsp(&pRsp->taosxRsp); - } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { - SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; - taosMemoryFreeClear(pRsp->pEpset); - tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp); - } -} - void tmqClearUnhandleMsg(tmq_t* tmq) { SMqRspWrapper* rspWrapper = NULL; while (taosGetQitem(tmq->qall, (void**)&rspWrapper) != 0) { - tmqFreeRspWrapper(rspWrapper); - taosFreeQitem(rspWrapper); + tmqFreeRspWrapper(rspWrapper); + taosFreeQitem(rspWrapper); } rspWrapper = NULL; @@ -1123,6 +1459,10 @@ void tmqClearUnhandleMsg(tmq_t* tmq) { } int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) { + if (pMsg) { + taosMemoryFreeClear(pMsg->pEpSet); + } + if (param == NULL) { return code; } @@ -1130,11 +1470,8 @@ int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) { SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; pParam->rspErr = code; - if (pMsg) { - taosMemoryFree(pMsg->pEpSet); - } if (tsem2_post(&pParam->rspSem) != 0){ - tscError("failed to post sem, subscribe cb"); + tqErrorC("failed to post sem, subscribe cb"); } return 0; } @@ -1151,16 +1488,16 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i); if (topic == NULL) { - tscError("topic is null"); + tqErrorC("topic is null"); continue; } char* tmp = strchr(topic->topicName, '.'); if (tmp == NULL) { - tscError("topic name is invalid:%s", topic->topicName); + tqErrorC("topic name is invalid:%s", topic->topicName); continue; } if (tmq_list_append(*topics, tmp + 1) != 0) { - tscError("failed to append topic:%s", tmp + 1); + tqErrorC("failed to append topic:%s", tmp + 1); continue; } } @@ -1168,18 +1505,6 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { return 0; } -static void freeClientVg(void* param) { - SMqClientVg* pVg = param; - tOffsetDestroy(&pVg->offsetInfo.endOffset); - tOffsetDestroy(&pVg->offsetInfo.beginOffset); - tOffsetDestroy(&pVg->offsetInfo.committedOffset); -} -static void freeClientTopic(void* param) { - SMqClientTopic* pTopic = param; - taosMemoryFreeClear(pTopic->schema.pSchema); - taosArrayDestroyEx(pTopic->vgs, freeClientVg); -} - void tmqFreeImpl(void* handle) { tmq_t* tmq = (tmq_t*)handle; int64_t id = tmq->consumerId; @@ -1195,7 +1520,7 @@ void tmqFreeImpl(void* handle) { taosFreeQall(tmq->qall); if(tsem2_destroy(&tmq->rspSem) != 0) { - tscError("failed to destroy sem in free tmq"); + tqErrorC("failed to destroy sem in free tmq"); } taosArrayDestroyEx(tmq->clientTopics, freeClientTopic); @@ -1203,22 +1528,22 @@ void tmqFreeImpl(void* handle) { if (tmq->commitTimer) { if (!taosTmrStopA(&tmq->commitTimer)) { - tscError("failed to stop commit timer"); + tqErrorC("failed to stop commit timer"); } } if (tmq->epTimer) { if (!taosTmrStopA(&tmq->epTimer)) { - tscError("failed to stop ep timer"); + tqErrorC("failed to stop ep timer"); } } if (tmq->hbLiveTimer) { if (!taosTmrStopA(&tmq->hbLiveTimer)) { - tscError("failed to stop hb timer"); + tqErrorC("failed to stop hb timer"); } } taosMemoryFree(tmq); - tscDebug("consumer:0x%" PRIx64 " closed", id); + tqInfoC("consumer:0x%" PRIx64 " closed", id); } static void tmqMgmtInit(void) { @@ -1247,9 +1572,6 @@ void tmqMgmtClose(void) { } } -#define SET_ERROR_MSG_TMQ(MSG) \ - if (errstr != NULL) (void)snprintf(errstr, errstrLen, MSG); - tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { int32_t code = 0; @@ -1269,7 +1591,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); if (pTmq == NULL) { - tscError("failed to create consumer, groupId:%s", conf->groupId); + tqErrorC("failed to create consumer, groupId:%s", conf->groupId); SET_ERROR_MSG_TMQ("malloc tmq failed") return NULL; } @@ -1279,13 +1601,13 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); if (pTmq->clientTopics == NULL) { - tscError("failed to create consumer, groupId:%s", conf->groupId); + tqErrorC("failed to create consumer, groupId:%s", conf->groupId); SET_ERROR_MSG_TMQ("malloc client topics failed") goto _failed; } code = taosOpenQueue(&pTmq->mqueue); if (code) { - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), + tqErrorC("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); SET_ERROR_MSG_TMQ("open queue failed") goto _failed; @@ -1293,7 +1615,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { code = taosOpenQueue(&pTmq->delayedTask); if (code) { - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), + tqErrorC("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); SET_ERROR_MSG_TMQ("open delayed task queue failed") goto _failed; @@ -1301,14 +1623,14 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { code = taosAllocateQall(&pTmq->qall); if (code) { - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), + tqErrorC("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); SET_ERROR_MSG_TMQ("allocate qall failed") goto _failed; } if (conf->groupId[0] == 0) { - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), + tqErrorC("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); SET_ERROR_MSG_TMQ("malloc tmq element failed or group is empty") goto _failed; @@ -1320,8 +1642,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->epoch = 0; // set conf - (void)strcpy(pTmq->clientId, conf->clientId); - (void)strcpy(pTmq->groupId, conf->groupId); + tstrncpy(pTmq->clientId, conf->clientId, TSDB_CLIENT_ID_LEN); + tstrncpy(pTmq->groupId, conf->groupId, TSDB_CGROUP_LEN); pTmq->withTbName = conf->withTbName; pTmq->useSnapshot = conf->snapEnable; pTmq->autoCommit = conf->autoCommit; @@ -1337,7 +1659,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->enableBatchMeta = conf->enableBatchMeta; tstrncpy(pTmq->user, user, TSDB_USER_LEN); if (taosGetFqdn(pTmq->fqdn) != 0) { - (void)strcpy(pTmq->fqdn, "localhost"); + tstrncpy(pTmq->fqdn, "localhost", TSDB_FQDN_LEN); } if (conf->replayEnable) { pTmq->autoCommit = false; @@ -1349,7 +1671,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { // init semaphore if (tsem2_init(&pTmq->rspSem, 0, 0) != 0) { - tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, + tqErrorC("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, tstrerror(TAOS_SYSTEM_ERROR(errno)), pTmq->groupId); SET_ERROR_MSG_TMQ("init t_sem failed") goto _failed; @@ -1359,7 +1681,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { code = taos_connect_internal(conf->ip, user, pass, NULL, NULL, conf->port, CONN_TYPE__TMQ, &pTmq->pTscObj); if (code) { terrno = code; - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); + tqErrorC("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); SET_ERROR_MSG_TMQ("init tscObj failed") goto _failed; } @@ -1378,7 +1700,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { char buf[TSDB_OFFSET_LEN] = {0}; STqOffsetVal offset = {.type = pTmq->resetOffsetCfg}; tFormatOffset(buf, tListLen(buf), &offset); - tscInfo("consumer:0x%" PRIx64 " is setup, refId:%" PRId64 + tqInfoC("consumer:0x%" PRIx64 " is setup, refId:%" PRId64 ", groupId:%s, snapshot:%d, autoCommit:%d, commitInterval:%dms, offset:%s", pTmq->consumerId, pTmq->refId, pTmq->groupId, pTmq->useSnapshot, pTmq->autoCommit, pTmq->autoCommitInterval, buf); @@ -1390,9 +1712,31 @@ _failed: return NULL; } +static int32_t syncAskEp(tmq_t* pTmq) { + SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo)); + if (pInfo == NULL) return terrno; + if (tsem2_init(&pInfo->sem, 0, 0) != 0) { + taosMemoryFree(pInfo); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } + + int32_t code = askEp(pTmq, pInfo, true, false); + if (code == 0) { + if (tsem2_wait(&pInfo->sem) != 0){ + tqErrorC("consumer:0x%" PRIx64 ", failed to wait for sem", pTmq->consumerId); + } + code = pInfo->code; + } + + if(tsem2_destroy(&pInfo->sem) != 0) { + tqErrorC("failed to destroy sem sync ask ep"); + } + taosMemoryFree(pInfo); + return code; +} + int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { if (tmq == NULL || topic_list == NULL) return TSDB_CODE_INVALID_PARA; - const int32_t MAX_RETRY_COUNT = 120 * 2; // let's wait for 2 mins at most const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); void* buf = NULL; @@ -1400,7 +1744,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SCMSubscribeReq req = {0}; int32_t code = 0; - tscInfo("consumer:0x%" PRIx64 " cgroup:%s, subscribe %d topics", tmq->consumerId, tmq->groupId, sz); + tqInfoC("consumer:0x%" PRIx64 " cgroup:%s, subscribe %d topics", tmq->consumerId, tmq->groupId, sz); req.consumerId = tmq->consumerId; tstrncpy(req.clientId, tmq->clientId, TSDB_CLIENT_ID_LEN); @@ -1411,7 +1755,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { req.topicNames = taosArrayInit(sz, sizeof(void*)); if (req.topicNames == NULL) { code = terrno; - goto FAIL; + goto END; } req.withTbName = tmq->withTbName; @@ -1426,43 +1770,43 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { for (int32_t i = 0; i < sz; i++) { char* topic = taosArrayGetP(container, i); if (topic == NULL) { - code = TSDB_CODE_INVALID_PARA; - goto FAIL; + code = terrno; + goto END; } SName name = {0}; code = tNameSetDbName(&name, tmq->pTscObj->acctId, topic, strlen(topic)); if (code) { - tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to set topic name, code:%d", tmq->consumerId, tmq->groupId, + tqErrorC("consumer:0x%" PRIx64 " cgroup:%s, failed to set topic name, code:%d", tmq->consumerId, tmq->groupId, code); - goto FAIL; + goto END; } char* topicFName = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); if (topicFName == NULL) { code = terrno; - goto FAIL; + goto END; } code = tNameExtractFullName(&name, topicFName); if (code) { - tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to extract topic name, code:%d", tmq->consumerId, tmq->groupId, + tqErrorC("consumer:0x%" PRIx64 " cgroup:%s, failed to extract topic name, code:%d", tmq->consumerId, tmq->groupId, code); taosMemoryFree(topicFName); - goto FAIL; + goto END; } if (taosArrayPush(req.topicNames, &topicFName) == NULL) { code = terrno; taosMemoryFree(topicFName); - goto FAIL; + goto END; } - tscInfo("consumer:0x%" PRIx64 " subscribe topic:%s", tmq->consumerId, topicFName); + tqInfoC("consumer:0x%" PRIx64 " subscribe topic:%s", tmq->consumerId, topicFName); } int32_t tlen = tSerializeSCMSubscribeReq(NULL, &req); buf = taosMemoryMalloc(tlen); if (buf == NULL) { code = terrno; - goto FAIL; + goto END; } void* abuf = buf; @@ -1472,7 +1816,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { if (sendInfo == NULL) { code = terrno; taosMemoryFree(buf); - goto FAIL; + goto END; } SMqSubscribeCbParam param = {.rspErr = 0}; @@ -1480,7 +1824,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { code = TSDB_CODE_TSC_INTERNAL_ERROR; taosMemoryFree(buf); taosMemoryFree(sendInfo); - goto FAIL; + goto END; } sendInfo->msgInfo = (SDataBuf){.pData = buf, .len = tlen, .handle = NULL}; @@ -1494,47 +1838,49 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); if (code != 0) { - goto FAIL; + goto END; } if (tsem2_wait(¶m.rspSem) != 0){ - tscError("consumer:0x%" PRIx64 ", failed to wait semaphore in subscribe", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 ", failed to wait semaphore in subscribe", tmq->consumerId); } if(tsem2_destroy(¶m.rspSem) != 0) { - tscError("consumer:0x%" PRIx64 ", failed to destroy semaphore in subscribe", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 ", failed to destroy semaphore in subscribe", tmq->consumerId); } if (param.rspErr != 0) { code = param.rspErr; - goto FAIL; + goto END; } int32_t retryCnt = 0; while ((code = syncAskEp(tmq)) != 0) { - if (retryCnt++ > MAX_RETRY_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { - tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes, code:%s", + if (retryCnt++ > SUBSCRIBE_RETRY_MAX_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { + tqErrorC("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes, code:%s", tmq->consumerId, tstrerror(code)); if (code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { code = 0; } - goto FAIL; + goto END; } - tscInfo("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt); - taosMsleep(500); + tqInfoC("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt); + taosMsleep(SUBSCRIBE_RETRY_INTERVAL); } - tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(tmq->refId), tmqMgmt.timer); - tmq->commitTimer = - taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer); + if (tmq->epTimer == NULL){ + tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(tmq->refId), tmqMgmt.timer); + } + if (tmq->commitTimer == NULL){ + tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer); + } if (tmq->epTimer == NULL || tmq->commitTimer == NULL) { code = TSDB_CODE_TSC_INTERNAL_ERROR; - goto FAIL; + goto END; } -FAIL: +END: taosArrayDestroyP(req.topicNames, taosMemoryFree); - return code; } @@ -1572,19 +1918,9 @@ static SMqClientTopic* getTopicInfo(tmq_t* tmq, char* topicName) { return NULL; } -static void setVgIdle(tmq_t* tmq, char* topicName, int32_t vgId) { - taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = NULL; - getVgInfo(tmq, topicName, vgId, &pVg); - if (pVg) { - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - } - taosWUnLockLatch(&tmq->lock); -} - int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tmq_t* tmq = NULL; - SMqPollRspWrapper* pRspWrapper = NULL; + SMqRspWrapper* pRspWrapper = NULL; int8_t rspType = 0; int32_t vgId = 0; uint64_t requestId = 0; @@ -1593,21 +1929,19 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { return TSDB_CODE_TSC_INTERNAL_ERROR; } if (pParam == NULL) { - taosMemoryFreeClear(pMsg->pData); - taosMemoryFreeClear(pMsg->pEpSet); - return TSDB_CODE_TSC_INTERNAL_ERROR; + code = TSDB_CODE_TSC_INTERNAL_ERROR; + goto EXIT; } int64_t refId = pParam->refId; vgId = pParam->vgId; requestId = pParam->requestId; tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { - taosMemoryFreeClear(pMsg->pData); - taosMemoryFreeClear(pMsg->pEpSet); - return TSDB_CODE_TMQ_CONSUMER_CLOSED; + code = TSDB_CODE_TMQ_CONSUMER_CLOSED; + goto EXIT; } - int32_t ret = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0, (void**)&pRspWrapper); + int32_t ret = taosAllocateQitem(sizeof(SMqRspWrapper), DEF_QITEM, 0, (void**)&pRspWrapper); if (ret) { code = ret; tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d, since out of memory", tmq->consumerId, vgId); @@ -1619,280 +1953,72 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } if (pMsg->pData == NULL) { - tscError("consumer:0x%" PRIx64 " msg discard from vgId:%d, since msg is NULL", tmq->consumerId, vgId); + tqErrorC("consumer:0x%" PRIx64 " msg discard from vgId:%d, since msg is NULL", tmq->consumerId, vgId); code = TSDB_CODE_TSC_INTERNAL_ERROR; goto END; } int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch; int32_t clientEpoch = atomic_load_32(&tmq->epoch); - if (msgEpoch < clientEpoch) { - // do not write into queue since updating epoch reset - tscWarn("consumer:0x%" PRIx64 - " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d,QID:0x%" PRIx64, - tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId); - code = TSDB_CODE_TMQ_CONSUMER_MISMATCH; - goto END; - } if (msgEpoch != clientEpoch) { - tscError("consumer:0x%" PRIx64 - " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64, + tqErrorC("consumer:0x%" PRIx64 + " msg discard from vgId:%d since epoch not equal, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64, tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId); code = TSDB_CODE_TMQ_CONSUMER_MISMATCH; goto END; } - // handle meta rsp rspType = ((SMqRspHead*)pMsg->pData)->mqMsgType; - pRspWrapper->tmqRspType = rspType; - pRspWrapper->reqId = requestId; - pRspWrapper->pEpset = pMsg->pEpSet; - pMsg->pEpSet = NULL; - + tqDebugC("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, type %d,QID:0x%" PRIx64, tmq->consumerId, vgId, rspType, requestId); if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if (tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp) < 0) { - tDecoderClear(&decoder); - code = TSDB_CODE_OUT_OF_MEMORY; - goto END; - } - tDecoderClear(&decoder); - (void)memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); - - char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.common.rspOffset); - tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d,QID:0x%" PRIx64, - tmq->consumerId, vgId, pRspWrapper->dataRsp.common.reqOffset.version, buf, rspType, requestId); + PROCESS_POLL_RSP(tDecodeMqDataRsp, &pRspWrapper->pollRsp.dataRsp) } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if (tDecodeMqMetaRsp(&decoder, &pRspWrapper->metaRsp) < 0) { - tDecoderClear(&decoder); - code = TSDB_CODE_OUT_OF_MEMORY; - goto END; - } - tDecoderClear(&decoder); - (void)memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead)); + PROCESS_POLL_RSP(tDecodeMqMetaRsp, &pRspWrapper->pollRsp.metaRsp) } else if (rspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if (tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp) < 0) { - tDecoderClear(&decoder); - code = TSDB_CODE_OUT_OF_MEMORY; - goto END; - } - tDecoderClear(&decoder); - (void)memcpy(&pRspWrapper->taosxRsp, pMsg->pData, sizeof(SMqRspHead)); + PROCESS_POLL_RSP(tDecodeSTaosxRsp, &pRspWrapper->pollRsp.dataRsp) } else if (rspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if (tSemiDecodeMqBatchMetaRsp(&decoder, &pRspWrapper->batchMetaRsp) < 0) { - tDecoderClear(&decoder); - code = TSDB_CODE_OUT_OF_MEMORY; - goto END; - } - tDecoderClear(&decoder); - (void)memcpy(&pRspWrapper->batchMetaRsp, pMsg->pData, sizeof(SMqRspHead)); - tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d,QID:0x%" PRIx64, tmq->consumerId, vgId, - requestId); + PROCESS_POLL_RSP(tSemiDecodeMqBatchMetaRsp, &pRspWrapper->pollRsp.batchMetaRsp) } else { // invalid rspType - tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType); + tqErrorC("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType); + code = TSDB_CODE_TSC_INTERNAL_ERROR; + goto END; } + pRspWrapper->tmqRspType = rspType; + pRspWrapper->pollRsp.reqId = requestId; + pRspWrapper->pollRsp.pEpset = pMsg->pEpSet; + pMsg->pEpSet = NULL; END: if (pRspWrapper) { pRspWrapper->code = code; - pRspWrapper->vgId = vgId; - (void)strcpy(pRspWrapper->topicName, pParam->topicName); + pRspWrapper->pollRsp.vgId = vgId; + (void)strcpy(pRspWrapper->pollRsp.topicName, pParam->topicName); code = taosWriteQitem(tmq->mqueue, pRspWrapper); if (code != 0) { - tmqFreeRspWrapper((SMqRspWrapper*)pRspWrapper); + tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pRspWrapper); - tscError("consumer:0x%" PRIx64 " put poll res into mqueue failed, code:%d", tmq->consumerId, code); + tqErrorC("consumer:0x%" PRIx64 " put poll res into mqueue failed, code:%d", tmq->consumerId, code); + } else { + tqDebugC("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d,QID:0x%" PRIx64, + tmq ? tmq->consumerId : 0, rspType, vgId, taosQueueItemSize(tmq->mqueue), requestId); } } - int32_t total = taosQueueItemSize(tmq->mqueue); - tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d,QID:0x%" PRIx64, - tmq ? tmq->consumerId : 0, rspType, vgId, total, requestId); - if (tmq) { - if (tsem2_post(&tmq->rspSem) != 0){ - tscError("failed to post rsp sem, consumer:0x%" PRIx64, tmq->consumerId); - } + + if (tsem2_post(&tmq->rspSem) != 0){ + tqErrorC("failed to post rsp sem, consumer:0x%" PRIx64, tmq->consumerId); } - if (pMsg) taosMemoryFreeClear(pMsg->pData); - if (pMsg) taosMemoryFreeClear(pMsg->pEpSet); ret = taosReleaseRef(tmqMgmt.rsetId, refId); if (ret != 0){ - tscError("failed to release ref:%"PRId64 ", code:%d", refId, ret); + tqErrorC("failed to release ref:%"PRId64 ", code:%d", refId, ret); } +EXIT: + taosMemoryFreeClear(pMsg->pData); + taosMemoryFreeClear(pMsg->pEpSet); return code; } -typedef struct SVgroupSaveInfo { - STqOffsetVal currentOffset; - STqOffsetVal commitOffset; - STqOffsetVal seekOffset; - int64_t numOfRows; - int32_t vgStatus; -} SVgroupSaveInfo; - -static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopicEp, SHashObj* pVgOffsetHashMap, - tmq_t* tmq) { - pTopic->schema = pTopicEp->schema; - pTopicEp->schema.nCols = 0; - pTopicEp->schema.pSchema = NULL; - - char vgKey[TSDB_TOPIC_FNAME_LEN + 22] = {0}; - int32_t vgNumGet = taosArrayGetSize(pTopicEp->vgs); - - tstrncpy(pTopic->topicName, pTopicEp->topic, TSDB_TOPIC_FNAME_LEN); - tstrncpy(pTopic->db, pTopicEp->db, TSDB_DB_FNAME_LEN); - - tscInfo("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d", tmq->consumerId, pTopic->topicName, vgNumGet); - pTopic->vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); - if (pTopic->vgs == NULL) { - tscError("consumer:0x%" PRIx64 ", failed to init vgs for topic:%s", tmq->consumerId, pTopic->topicName); - return; - } - for (int32_t j = 0; j < vgNumGet; j++) { - SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); - if (pVgEp == NULL) { - continue; - } - (void)sprintf(vgKey, "%s:%d", pTopic->topicName, pVgEp->vgId); - SVgroupSaveInfo* pInfo = taosHashGet(pVgOffsetHashMap, vgKey, strlen(vgKey)); - - STqOffsetVal offsetNew = {0}; - offsetNew.type = tmq->resetOffsetCfg; - - tscInfo("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d, num:%d, port:%d", tmq->consumerId, - pTopic->topicName, vgNumGet, pVgEp->epSet.numOfEps, pVgEp->epSet.eps[pVgEp->epSet.inUse].port); - - SMqClientVg clientVg = { - .pollCnt = 0, - .vgId = pVgEp->vgId, - .epSet = pVgEp->epSet, - .vgStatus = pInfo ? pInfo->vgStatus : TMQ_VG_STATUS__IDLE, - .vgSkipCnt = 0, - .emptyBlockReceiveTs = 0, - .blockReceiveTs = 0, - .blockSleepForReplay = 0, - .numOfRows = pInfo ? pInfo->numOfRows : 0, - }; - - clientVg.offsetInfo.walVerBegin = -1; - clientVg.offsetInfo.walVerEnd = -1; - clientVg.seekUpdated = false; - if (pInfo) { - tOffsetCopy(&clientVg.offsetInfo.endOffset, &pInfo->currentOffset); - tOffsetCopy(&clientVg.offsetInfo.committedOffset, &pInfo->commitOffset); - tOffsetCopy(&clientVg.offsetInfo.beginOffset, &pInfo->seekOffset); - } else { - clientVg.offsetInfo.endOffset = offsetNew; - clientVg.offsetInfo.committedOffset = offsetNew; - clientVg.offsetInfo.beginOffset = offsetNew; - } - if (taosArrayPush(pTopic->vgs, &clientVg) == NULL) { - tscError("consumer:0x%" PRIx64 ", failed to push vg:%d into topic:%s", tmq->consumerId, pVgEp->vgId, - pTopic->topicName); - freeClientVg(&clientVg); - } - } -} - -static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { - bool set = false; - - int32_t topicNumGet = taosArrayGetSize(pRsp->topics); - if (epoch < tmq->epoch || (epoch == tmq->epoch && topicNumGet == 0)) { - tscDebug("consumer:0x%" PRIx64 " no update ep epoch from %d to epoch %d, incoming topics:%d", tmq->consumerId, - tmq->epoch, epoch, topicNumGet); - if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) { - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); - } - return false; - } - - SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); - if (newTopics == NULL) { - return false; - } - - SHashObj* pVgOffsetHashMap = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); - if (pVgOffsetHashMap == NULL) { - taosArrayDestroy(newTopics); - return false; - } - - taosWLockLatch(&tmq->lock); - int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); - - char vgKey[TSDB_TOPIC_FNAME_LEN + 22] = {0}; - tscInfo("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d", - tmq->consumerId, tmq->epoch, epoch, topicNumGet, topicNumCur); - for (int32_t i = 0; i < topicNumCur; i++) { - // find old topic - SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); - if (pTopicCur && pTopicCur->vgs) { - int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); - tscInfo("consumer:0x%" PRIx64 ", current vg num: %d", tmq->consumerId, vgNumCur); - for (int32_t j = 0; j < vgNumCur; j++) { - SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); - if (pVgCur == NULL) { - continue; - } - (void)sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); - - char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pVgCur->offsetInfo.endOffset); - tscInfo("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, - vgKey, buf); - - SVgroupSaveInfo info = {.currentOffset = pVgCur->offsetInfo.endOffset, - .seekOffset = pVgCur->offsetInfo.beginOffset, - .commitOffset = pVgCur->offsetInfo.committedOffset, - .numOfRows = pVgCur->numOfRows, - .vgStatus = pVgCur->vgStatus}; - if (taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)) != 0) { - tscError("consumer:0x%" PRIx64 ", failed to put vg:%d into hashmap", tmq->consumerId, pVgCur->vgId); - } - } - } - } - - for (int32_t i = 0; i < topicNumGet; i++) { - SMqClientTopic topic = {0}; - SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i); - if (pTopicEp == NULL) { - continue; - } - initClientTopicFromRsp(&topic, pTopicEp, pVgOffsetHashMap, tmq); - if (taosArrayPush(newTopics, &topic) == NULL) { - tscError("consumer:0x%" PRIx64 ", failed to push topic:%s into new topics", tmq->consumerId, topic.topicName); - freeClientTopic(&topic); - } - } - - taosHashCleanup(pVgOffsetHashMap); - - // destroy current buffered existed topics info - if (tmq->clientTopics) { - taosArrayDestroyEx(tmq->clientTopics, freeClientTopic); - } - tmq->clientTopics = newTopics; - taosWUnLockLatch(&tmq->lock); - - int8_t flag = (topicNumGet == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY; - atomic_store_8(&tmq->status, flag); - atomic_store_32(&tmq->epoch, epoch); - - tscInfo("consumer:0x%" PRIx64 " update topic info completed", tmq->consumerId); - return set; -} - void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { (void)snprintf(pReq->subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", tmq->groupId, TMQ_SEPARATOR, pTopic->topicName); pReq->withTbName = tmq->withTbName; @@ -1908,35 +2034,6 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl pReq->enableBatchMeta = tmq->enableBatchMeta; } -void tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqMetaRspObj** ppRspObj) { - SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj)); - if (pRspObj == NULL) { - return; - } - pRspObj->resType = RES_TYPE__TMQ_META; - tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); - tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); - pRspObj->vgId = pWrapper->vgHandle->vgId; - - (void)memcpy(&pRspObj->metaRsp, &pWrapper->metaRsp, sizeof(SMqMetaRsp)); - *ppRspObj = pRspObj; -} - -void tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqBatchMetaRspObj** ppRspObj) { - SMqBatchMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqBatchMetaRspObj)); - if (pRspObj == NULL) { - return; - } - pRspObj->common.resType = RES_TYPE__TMQ_BATCH_META; - tstrncpy(pRspObj->common.topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); - tstrncpy(pRspObj->common.db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); - pRspObj->common.vgId = pWrapper->vgHandle->vgId; - - (void)memcpy(&pRspObj->rsp, &pWrapper->batchMetaRsp, sizeof(SMqBatchMetaRsp)); - tscDebug("build batchmeta Rsp from wrapper"); - *ppRspObj = pRspObj; -} - void changeByteEndian(char* pData) { if (pData == NULL) { return; @@ -1947,7 +2044,7 @@ void changeByteEndian(char* pData) { // length | version: int32_t blockVersion = *(int32_t*)p; if (blockVersion != BLOCK_VERSION_1) { - tscError("invalid block version:%d", blockVersion); + tqErrorC("invalid block version:%d", blockVersion); return; } *(int32_t*)p = BLOCK_VERSION_2; @@ -1988,23 +2085,18 @@ static void tmqGetRawDataRowsPrecisionFromRes(void* pRetrieve, void** rawData, i } static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, - SMqRspObjCommon* pRspObj, SMqDataRspCommon* pDataRsp) { - (*numOfRows) = 0; - tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); - tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); - - pRspObj->vgId = pWrapper->vgHandle->vgId; + SMqRspObj* pRspObj) { pRspObj->resIter = -1; - pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; + SMqDataRsp* pDataRsp = &pRspObj->dataRsp; bool needTransformSchema = !pDataRsp->withSchema; if (!pDataRsp->withSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable pDataRsp->withSchema = true; pDataRsp->blockSchema = taosArrayInit(pDataRsp->blockNum, sizeof(void*)); if (pDataRsp->blockSchema == NULL) { - tscError("failed to allocate memory for blockSchema"); + tqErrorC("failed to allocate memory for blockSchema"); return; } } @@ -2023,7 +2115,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); if (schema) { if (taosArrayPush(pDataRsp->blockSchema, &schema) == NULL) { - tscError("failed to push schema into blockSchema"); + tqErrorC("failed to push schema into blockSchema"); continue; } } @@ -2031,31 +2123,6 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg } } -void tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, - SMqRspObj** ppRspObj) { - SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); - if (pRspObj == NULL) { - return; - } - pRspObj->common.resType = RES_TYPE__TMQ; - (void)memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); - tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common); - *ppRspObj = pRspObj; -} - -void tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, - SMqTaosxRspObj** ppRspObj) { - SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); - if (pRspObj == NULL) { - return; - } - pRspObj->common.resType = RES_TYPE__TMQ_METADATA; - (void)memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp)); - - tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common); - *ppRspObj = pRspObj; -} - static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* pVg, int64_t timeout) { SMqPollReq req = {0}; char* msg = NULL; @@ -2108,11 +2175,10 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p sendInfo->fp = tmqPollCb; sendInfo->msgType = TDMT_VND_TMQ_CONSUME; - // int64_t transporterId = 0; char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.endOffset); code = asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, NULL, sendInfo); - tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s,QID:0x%" PRIx64, pTmq->consumerId, + tqDebugC("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s,QID:0x%" PRIx64, pTmq->consumerId, pTopic->topicName, pVg->vgId, code, pTmq->epoch, offsetFormatBuf, req.reqId); if (code != 0) { return code; @@ -2127,14 +2193,11 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p // broadcast the poll request to all related vnodes static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { - if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) { - return 0; - } int32_t code = 0; taosWLockLatch(&tmq->lock); int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); - tscDebug("consumer:0x%" PRIx64 " start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics); + tqDebugC("consumer:0x%" PRIx64 " start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics); for (int i = 0; i < numOfTopics; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); @@ -2143,7 +2206,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { } int32_t numOfVg = taosArrayGetSize(pTopic->vgs); if (pTopic->noPrivilege) { - tscDebug("consumer:0x%" PRIx64 " has no privilegr for topic:%s", tmq->consumerId, pTopic->topicName); + tqDebugC("consumer:0x%" PRIx64 " has no privilegr for topic:%s", tmq->consumerId, pTopic->topicName); continue; } for (int j = 0; j < numOfVg; j++) { @@ -2153,14 +2216,14 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { } int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs; if (elapsed < EMPTY_BLOCK_POLL_IDLE_DURATION && elapsed >= 0) { // less than 10ms - tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, + tqDebugC("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, tmq->epoch, pVg->vgId); continue; } elapsed = taosGetTimestampMs() - pVg->blockReceiveTs; if (tmq->replayEnable && elapsed < pVg->blockSleepForReplay && elapsed >= 0) { - tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for %" PRId64 "ms before start next poll when replay", + tqDebugC("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for %" PRId64 "ms before start next poll when replay", tmq->consumerId, tmq->epoch, pVg->vgId, pVg->blockSleepForReplay); continue; } @@ -2168,7 +2231,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus == TMQ_VG_STATUS__WAIT) { int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1); - tscDebug("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, + tqDebugC("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, vgSkipCnt); continue; } @@ -2183,20 +2246,20 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { end: taosWUnLockLatch(&tmq->lock); - tscDebug("consumer:0x%" PRIx64 " end to poll data, code:%d", tmq->consumerId, code); + tqDebugC("consumer:0x%" PRIx64 " end to poll data, code:%d", tmq->consumerId, code); return code; } static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal* rspOffset, int64_t sver, int64_t ever, int64_t consumerId, bool hasData) { if (!pVg->seekUpdated) { - tscDebug("consumer:0x%" PRIx64 " local offset is update, since seekupdate not set", consumerId); + tqDebugC("consumer:0x%" PRIx64 " local offset is update, since seekupdate not set", consumerId); if (hasData) { tOffsetCopy(&pVg->offsetInfo.beginOffset, reqOffset); } tOffsetCopy(&pVg->offsetInfo.endOffset, rspOffset); } else { - tscDebug("consumer:0x%" PRIx64 " local offset is NOT update, since seekupdate is set", consumerId); + tqDebugC("consumer:0x%" PRIx64 " local offset is NOT update, since seekupdate is set", consumerId); } // update the status @@ -2207,9 +2270,134 @@ static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal pVg->offsetInfo.walVerEnd = ever + 1; } -static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { - tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, taosQallItemSize(tmq->qall)); +static SMqRspObj* buildRsp(SMqPollRspWrapper* pollRspWrapper){ + typedef union { + SMqDataRsp dataRsp; + SMqMetaRsp metaRsp; + SMqBatchMetaRsp batchMetaRsp; + } MEMSIZE; + SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); + if (pRspObj == NULL) { + tqErrorC("buildRsp:failed to allocate memory"); + return NULL; + } + (void)memcpy(&pRspObj->dataRsp, &pollRspWrapper->dataRsp, sizeof(MEMSIZE)); + tstrncpy(pRspObj->topic, pollRspWrapper->topicName, TSDB_TOPIC_FNAME_LEN); + tstrncpy(pRspObj->db, pollRspWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); + pRspObj->vgId = pollRspWrapper->vgId; + (void)memset(&pollRspWrapper->dataRsp, 0, sizeof(MEMSIZE)); + return pRspObj; +} + +static void processMqRspError(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ + SMqPollRspWrapper* pollRspWrapper = &pRspWrapper->pollRsp; + + if (pRspWrapper->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { // for vnode transform + int32_t code = askEp(tmq, NULL, false, true); + if (code != 0) { + tqErrorC("consumer:0x%" PRIx64 " failed to ask ep, code:%s", tmq->consumerId, tstrerror(code)); + } + } else if (pRspWrapper->code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) { + int32_t code = askEp(tmq, NULL, false, false); + if (code != 0) { + tqErrorC("consumer:0x%" PRIx64 " failed to ask ep, code:%s", tmq->consumerId, tstrerror(code)); + } + } + tqInfoC("consumer:0x%" PRIx64 " msg from vgId:%d discarded, since %s", tmq->consumerId, pollRspWrapper->vgId, + tstrerror(pRspWrapper->code)); + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = NULL; + getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); + if (pVg) { + pVg->emptyBlockReceiveTs = taosGetTimestampMs(); + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + } + taosWUnLockLatch(&tmq->lock); +} +static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ + SMqRspObj* pRspObj = NULL; + + if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { + tqDebugC("consumer:0x%" PRIx64 " ep msg received", tmq->consumerId); + SMqAskEpRsp* rspMsg = &pRspWrapper->epRsp; + doUpdateLocalEp(tmq, pRspWrapper->epoch, rspMsg); + return pRspObj; + } + + SMqPollRspWrapper* pollRspWrapper = &pRspWrapper->pollRsp; + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = NULL; + getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); + if(pVg == NULL) { + tqErrorC("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + goto END; + } + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if (pollRspWrapper->pEpset != NULL) { + pVg->epSet = *pollRspWrapper->pEpset; + } + + if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP || pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { + updateVgInfo(pVg, &pollRspWrapper->dataRsp.reqOffset, &pollRspWrapper->dataRsp.rspOffset, pollRspWrapper->head.walsver, pollRspWrapper->head.walever, + tmq->consumerId, pollRspWrapper->dataRsp.blockNum != 0); + + char buf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &pollRspWrapper->rspOffset); + if (pollRspWrapper->dataRsp.blockNum == 0) { + tqDebugC("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64 + ", total:%" PRId64 ",QID:0x%" PRIx64, + tmq->consumerId, pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); + pVg->emptyBlockReceiveTs = taosGetTimestampMs(); + } else { + pRspObj = buildRsp(pollRspWrapper); + if (pRspObj == NULL) { + tqErrorC("consumer:0x%" PRIx64 " failed to allocate memory for meta rsp", tmq->consumerId); + goto END; + } + pRspObj->resType = pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP ? RES_TYPE__TMQ : RES_TYPE__TMQ_METADATA; + int64_t numOfRows = 0; + tmqBuildRspFromWrapperInner(pollRspWrapper, pVg, &numOfRows, pRspObj); + tmq->totalRows += numOfRows; + pVg->emptyBlockReceiveTs = 0; + if (tmq->replayEnable) { + pVg->blockReceiveTs = taosGetTimestampMs(); + pVg->blockSleepForReplay = pRspObj->dataRsp.sleepTime; + if (pVg->blockSleepForReplay > 0) { + if (taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, (void*)(tmq->refId), tmqMgmt.timer) == NULL) { + tqErrorC("consumer:0x%" PRIx64 " failed to start replay timer, vgId:%d, sleep:%" PRId64, + tmq->consumerId, pVg->vgId, pVg->blockSleepForReplay); + } + } + } + tqDebugC("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 + ", vg total:%" PRId64 ", total:%" PRId64 ",QID:0x%" PRIx64, + tmq->consumerId, pVg->vgId, buf, pRspObj->dataRsp.blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, + pollRspWrapper->reqId); + } + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP || pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { + updateVgInfo(pVg, &pollRspWrapper->rspOffset, &pollRspWrapper->rspOffset, + pollRspWrapper->head.walsver, pollRspWrapper->head.walever, tmq->consumerId, true); + + + pRspObj = buildRsp(pollRspWrapper); + if (pRspObj == NULL) { + tqErrorC("consumer:0x%" PRIx64 " failed to allocate memory for meta rsp", tmq->consumerId); + goto END; + } + pRspObj->resType = pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP ? RES_TYPE__TMQ_META : RES_TYPE__TMQ_BATCH_META; + } + + END: + taosWUnLockLatch(&tmq->lock); + return pRspObj; +} + +static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { + tqDebugC("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, taosQallItemSize(tmq->qall)); + + void* returnVal = NULL; while (1) { SMqRspWrapper* pRspWrapper = NULL; if (taosGetQitem(tmq->qall, (void**)&pRspWrapper) == 0) { @@ -2221,253 +2409,20 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { } } - tscDebug("consumer:0x%" PRIx64 " handle rsp, type:%d", tmq->consumerId, pRspWrapper->tmqRspType); - + tqDebugC("consumer:0x%" PRIx64 " handle rsp, type:%d", tmq->consumerId, pRspWrapper->tmqRspType); if (pRspWrapper->code != 0) { - SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; - if (pRspWrapper->code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) { - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER); - tscDebug("consumer:0x%" PRIx64 " wait for the rebalance, set status to be RECOVER", tmq->consumerId); - } else if (pRspWrapper->code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { - tscInfo("consumer:0x%" PRIx64 " return null since no committed offset", tmq->consumerId); - } else { - if (pRspWrapper->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { // for vnode transform - int32_t code = askEp(tmq, NULL, false, true); - if (code != 0) { - tscError("consumer:0x%" PRIx64 " failed to ask ep, code:%s", tmq->consumerId, tstrerror(code)); - } - } - tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, since %s", tmq->consumerId, pollRspWrapper->vgId, - tstrerror(pRspWrapper->code)); - taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = NULL; - getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); - if (pVg) pVg->emptyBlockReceiveTs = taosGetTimestampMs(); - taosWUnLockLatch(&tmq->lock); - } - setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); - taosMemoryFreeClear(pollRspWrapper->pEpset); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { - SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; - - int32_t consumerEpoch = atomic_load_32(&tmq->epoch); - SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->dataRsp; - - if (pDataRsp->head.epoch == consumerEpoch) { - taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = NULL; - getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); - pollRspWrapper->vgHandle = pVg; - pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); - if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { - tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, - pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return NULL; - } - // update the epset - if (pollRspWrapper->pEpset != NULL) { - SEp* pEp = GET_ACTIVE_EP(pollRspWrapper->pEpset); - SEp* pOld = GET_ACTIVE_EP(&(pVg->epSet)); - tscDebug("consumer:0x%" PRIx64 " update epset vgId:%d, ep:%s:%d, old ep:%s:%d", tmq->consumerId, pVg->vgId, - pEp->fqdn, pEp->port, pOld->fqdn, pOld->port); - pVg->epSet = *pollRspWrapper->pEpset; - } - - updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset, pDataRsp->head.walsver, pDataRsp->head.walever, - tmq->consumerId, pDataRsp->blockNum != 0); - - char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset); - if (pDataRsp->blockNum == 0) { - tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64 - ", total:%" PRId64 ",QID:0x%" PRIx64, - tmq->consumerId, pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); - pVg->emptyBlockReceiveTs = taosGetTimestampMs(); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - } else { // build rsp - int64_t numOfRows = 0; - SMqRspObj* pRsp = NULL; - tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp); - tmq->totalRows += numOfRows; - pVg->emptyBlockReceiveTs = 0; - if (pRsp && tmq->replayEnable) { - pVg->blockReceiveTs = taosGetTimestampMs(); - pVg->blockSleepForReplay = pRsp->rsp.sleepTime; - if (pVg->blockSleepForReplay > 0) { - if (taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, (void*)(tmq->refId), tmqMgmt.timer) == NULL) { - tscError("consumer:0x%" PRIx64 " failed to start replay timer, vgId:%d, sleep:%" PRId64, - tmq->consumerId, pVg->vgId, pVg->blockSleepForReplay); - } - } - } - tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 - ", vg total:%" PRId64 ", total:%" PRId64 ",QID:0x%" PRIx64, - tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, - pollRspWrapper->reqId); - taosMemoryFreeClear(pollRspWrapper->pEpset); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return pRsp; - } - } else { - tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch); - setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - } - } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { - SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; - int32_t consumerEpoch = atomic_load_32(&tmq->epoch); - - tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId); - - if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { - taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = NULL; - getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); - pollRspWrapper->vgHandle = pVg; - pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); - if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { - tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, - pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return NULL; - } - - updateVgInfo(pVg, &pollRspWrapper->metaRsp.rspOffset, &pollRspWrapper->metaRsp.rspOffset, - pollRspWrapper->metaRsp.head.walsver, pollRspWrapper->metaRsp.head.walever, tmq->consumerId, true); - // build rsp - SMqMetaRspObj* pRsp = NULL; - tmqBuildMetaRspFromWrapper(pollRspWrapper, &pRsp); - taosMemoryFreeClear(pollRspWrapper->pEpset); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return pRsp; - } else { - tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->metaRsp.head.epoch, consumerEpoch); - setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - } - } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { - SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; - int32_t consumerEpoch = atomic_load_32(&tmq->epoch); - - tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId); - - if (pollRspWrapper->batchMetaRsp.head.epoch == consumerEpoch) { - taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = NULL; - getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); - pollRspWrapper->vgHandle = pVg; - pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); - if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { - tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, - pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return NULL; - } - - // build rsp - updateVgInfo(pVg, &pollRspWrapper->batchMetaRsp.rspOffset, &pollRspWrapper->batchMetaRsp.rspOffset, - pollRspWrapper->batchMetaRsp.head.walsver, pollRspWrapper->batchMetaRsp.head.walever, - tmq->consumerId, true); - SMqBatchMetaRspObj* pRsp = NULL; - tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp); - taosMemoryFreeClear(pollRspWrapper->pEpset); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return pRsp; - } else { - tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->batchMetaRsp.head.epoch, consumerEpoch); - setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - } - } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { - SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; - int32_t consumerEpoch = atomic_load_32(&tmq->epoch); - SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->taosxRsp; - - if (pDataRsp->head.epoch == consumerEpoch) { - taosWLockLatch(&tmq->lock); - SMqClientVg* pVg = NULL; - getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId, &pVg); - pollRspWrapper->vgHandle = pVg; - pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); - if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { - tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, - pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return NULL; - } - - updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset, pDataRsp->head.walsver, pDataRsp->head.walever, - tmq->consumerId, pDataRsp->blockNum != 0); - - if (pDataRsp->blockNum == 0) { - tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ",QID:0x%" PRIx64, - tmq->consumerId, pVg->vgId, pVg->numOfRows, pollRspWrapper->reqId); - pVg->emptyBlockReceiveTs = taosGetTimestampMs(); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - continue; - } else { - pVg->emptyBlockReceiveTs = 0; // reset the ts - } - - // build rsp - int64_t numOfRows = 0; - SMqTaosxRspObj* pRsp = NULL; - tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp); - tmq->totalRows += numOfRows; - - char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset); - tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 - ", vg total:%" PRId64 ", total:%" PRId64 ",QID:0x%" PRIx64, - tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, - pollRspWrapper->reqId); - - taosMemoryFreeClear(pollRspWrapper->pEpset); - taosFreeQitem(pRspWrapper); - taosWUnLockLatch(&tmq->lock); - return pRsp; - } else { - tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch); - setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - } - } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { - tscDebug("consumer:0x%" PRIx64 " ep msg received", tmq->consumerId); - SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)pRspWrapper; - SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg; - (void)doUpdateLocalEp(tmq, pEpRspWrapper->epoch, rspMsg); - tmqFreeRspWrapper(pRspWrapper); - taosFreeQitem(pRspWrapper); - } else { - tscError("consumer:0x%" PRIx64 " invalid msg received:%d", tmq->consumerId, pRspWrapper->tmqRspType); + processMqRspError(tmq, pRspWrapper); + }else{ + returnVal = processMqRsp(tmq, pRspWrapper); + } + tmqFreeRspWrapper(pRspWrapper); + taosFreeQitem(pRspWrapper); + if(returnVal != NULL){ + break; } } + + return returnVal; } TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { @@ -2476,45 +2431,28 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { void* rspObj = NULL; int64_t startTime = taosGetTimestampMs(); - tscDebug("consumer:0x%" PRIx64 " start to poll at %" PRId64 ", timeout:%" PRId64, tmq->consumerId, startTime, + tqDebugC("consumer:0x%" PRIx64 " start to poll at %" PRId64 ", timeout:%" PRId64, tmq->consumerId, startTime, timeout); // in no topic status, delayed task also need to be processed if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) { - tscInfo("consumer:0x%" PRIx64 " poll return since consumer is init", tmq->consumerId); + tqInfoC("consumer:0x%" PRIx64 " poll return since consumer is init", tmq->consumerId); taosMsleep(500); // sleep for a while return NULL; } - while (1) { - if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__RECOVER) { - break; - } - tscInfo("consumer:0x%" PRIx64 " tmq status is recover", tmq->consumerId); - - int32_t retryCnt = 0; - while (syncAskEp(tmq) != 0) { - if (retryCnt++ > 40) { - return NULL; - } - - tscInfo("consumer:0x%" PRIx64 " not ready, retry:%d/40 in 500ms", tmq->consumerId, retryCnt); - taosMsleep(500); - } - } - (void)atomic_val_compare_exchange_8(&pollFlag, 0, 1); while (1) { tmqHandleAllDelayedTask(tmq); if (tmqPollImpl(tmq, timeout) < 0) { - tscError("consumer:0x%" PRIx64 " return due to poll error", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 " return due to poll error", tmq->consumerId); } rspObj = tmqHandleAllRsp(tmq, timeout); if (rspObj) { - tscDebug("consumer:0x%" PRIx64 " return rsp %p", tmq->consumerId, rspObj); + tqDebugC("consumer:0x%" PRIx64 " return rsp %p", tmq->consumerId, rspObj); return (TAOS_RES*)rspObj; } @@ -2522,7 +2460,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { int64_t currentTime = taosGetTimestampMs(); int64_t elapsedTime = currentTime - startTime; if (elapsedTime > timeout || elapsedTime < 0) { - tscDebug("consumer:0x%" PRIx64 " (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64, + tqDebugC("consumer:0x%" PRIx64 " (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64, tmq->consumerId, tmq->epoch, startTime, currentTime); return NULL; } @@ -2536,77 +2474,68 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { static void displayConsumeStatistics(tmq_t* pTmq) { taosRLockLatch(&pTmq->lock); int32_t numOfTopics = taosArrayGetSize(pTmq->clientTopics); - tscDebug("consumer:0x%" PRIx64 " closing poll:%" PRId64 " rows:%" PRId64 " topics:%d, final epoch:%d", + tqInfoC("consumer:0x%" PRIx64 " closing poll:%" PRId64 " rows:%" PRId64 " topics:%d, final epoch:%d", pTmq->consumerId, pTmq->pollCnt, pTmq->totalRows, numOfTopics, pTmq->epoch); - tscDebug("consumer:0x%" PRIx64 " rows dist begin: ", pTmq->consumerId); + tqInfoC("consumer:0x%" PRIx64 " rows dist begin: ", pTmq->consumerId); for (int32_t i = 0; i < numOfTopics; ++i) { SMqClientTopic* pTopics = taosArrayGet(pTmq->clientTopics, i); if (pTopics == NULL) continue; - tscDebug("consumer:0x%" PRIx64 " topic:%d", pTmq->consumerId, i); + tqInfoC("consumer:0x%" PRIx64 " topic:%d", pTmq->consumerId, i); int32_t numOfVgs = taosArrayGetSize(pTopics->vgs); for (int32_t j = 0; j < numOfVgs; ++j) { SMqClientVg* pVg = taosArrayGet(pTopics->vgs, j); if (pVg == NULL) continue; - tscDebug("topic:%s, %d. vgId:%d rows:%" PRId64, pTopics->topicName, j, pVg->vgId, pVg->numOfRows); + tqInfoC("topic:%s, %d. vgId:%d rows:%" PRId64, pTopics->topicName, j, pVg->vgId, pVg->numOfRows); } } taosRUnLockLatch(&pTmq->lock); - tscDebug("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId); + tqInfoC("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId); } -static int32_t innerClose(tmq_t* tmq) { - if (tmq->status != TMQ_CONSUMER_STATUS__READY) { - tscInfo("consumer:0x%" PRIx64 " not in ready state, unsubscribe it directly", tmq->consumerId); - return 0; +int32_t tmq_unsubscribe(tmq_t* tmq) { + if (tmq == NULL) return TSDB_CODE_INVALID_PARA; + int32_t code = 0; + int8_t status = atomic_load_8(&tmq->status); + tqInfoC("consumer:0x%" PRIx64 " start to unsubscribe consumer, status:%d", tmq->consumerId, status); + + displayConsumeStatistics(tmq); + if (status != TMQ_CONSUMER_STATUS__READY) { + tqInfoC("consumer:0x%" PRIx64 " status:%d, already closed or not in ready state, no need unsubscribe", tmq->consumerId, status); + goto END; } if (tmq->autoCommit) { - int32_t code = tmq_commit_sync(tmq, NULL); + code = tmq_commit_sync(tmq, NULL); if (code != 0) { - return code; + goto END; } } tmqSendHbReq((void*)(tmq->refId), NULL); tmq_list_t* lst = tmq_list_new(); if (lst == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; } - int32_t code = tmq_subscribe(tmq, lst); + code = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); - return code; -} -int32_t tmq_unsubscribe(tmq_t* tmq) { - if (tmq == NULL) return TSDB_CODE_INVALID_PARA; - - tscInfo("consumer:0x%" PRIx64 " start to unsubscribe consumer, status:%d", tmq->consumerId, tmq->status); - int32_t code = 0; - if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) { - code = innerClose(tmq); - if (code == 0) { - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED); - } + if(code != 0){ + goto END; } + +END: return code; } int32_t tmq_consumer_close(tmq_t* tmq) { if (tmq == NULL) return TSDB_CODE_INVALID_PARA; - - tscInfo("consumer:0x%" PRIx64 " start to close consumer, status:%d", tmq->consumerId, tmq->status); - displayConsumeStatistics(tmq); - int32_t code = 0; - if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) { - code = innerClose(tmq); - if (code == 0) { - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED); - } - } - + tqInfoC("consumer:0x%" PRIx64 " start to close consumer, status:%d", tmq->consumerId, tmq->status); + int32_t code = tmq_unsubscribe(tmq); if (code == 0) { + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED); code = taosRemoveRef(tmqMgmt.rsetId, tmq->refId); if (code != 0){ - tscError("tmq close failed to remove ref:%" PRId64 ", code:%d", tmq->refId, code); + tqErrorC("tmq close failed to remove ref:%" PRId64 ", code:%d", tmq->refId, code); } } return code; @@ -2648,14 +2577,9 @@ const char* tmq_get_topic_name(TAOS_RES* res) { if (res == NULL) { return NULL; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { - char* tmp = strchr(((SMqRspObjCommon*)res)->topic, '.'); - if (tmp == NULL) { - return NULL; - } - return tmp + 1; - } else if (TD_RES_TMQ_META(res)) { - char* tmp = strchr(((SMqMetaRspObj*)res)->topic, '.'); + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || + TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + char* tmp = strchr(((SMqRspObj*)res)->topic, '.'); if (tmp == NULL) { return NULL; } @@ -2670,14 +2594,9 @@ const char* tmq_get_db_name(TAOS_RES* res) { return NULL; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { - char* tmp = strchr(((SMqRspObjCommon*)res)->db, '.'); - if (tmp == NULL) { - return NULL; - } - return tmp + 1; - } else if (TD_RES_TMQ_META(res)) { - char* tmp = strchr(((SMqMetaRspObj*)res)->db, '.'); + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || + TD_RES_TMQ_BATCH_META(res) || TD_RES_TMQ_META(res)) { + char* tmp = strchr(((SMqRspObj*)res)->db, '.'); if (tmp == NULL) { return NULL; } @@ -2691,10 +2610,9 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { if (res == NULL) { return TSDB_CODE_INVALID_PARA; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { - return ((SMqRspObjCommon*)res)->vgId; - } else if (TD_RES_TMQ_META(res)) { - return ((SMqMetaRspObj*)res)->vgId; + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || + TD_RES_TMQ_BATCH_META(res) || TD_RES_TMQ_META(res)) { + return ((SMqRspObj*)res)->vgId; } else { return TSDB_CODE_INVALID_PARA; } @@ -2705,25 +2623,20 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { return TSDB_CODE_INVALID_PARA; } if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { - SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); - STqOffsetVal* pOffset = &common->reqOffset; - if (common->reqOffset.type == TMQ_OFFSET__LOG) { - return common->reqOffset.version; + SMqRspObj* pRspObj = (SMqRspObj*)res; + STqOffsetVal* pOffset = &pRspObj->dataRsp.reqOffset; + if (pOffset->type == TMQ_OFFSET__LOG) { + return pOffset->version; } else { - tscError("invalid offset type:%d", common->reqOffset.type); + tqErrorC("invalid offset type:%d", pOffset->type); } - } else if (TD_RES_TMQ_META(res)) { - SMqMetaRspObj* pRspObj = (SMqMetaRspObj*)res; - if (pRspObj->metaRsp.rspOffset.type == TMQ_OFFSET__LOG) { - return pRspObj->metaRsp.rspOffset.version; - } - } else if (TD_RES_TMQ_BATCH_META(res)) { - SMqBatchMetaRspObj* pBtRspObj = (SMqBatchMetaRspObj*)res; - if (pBtRspObj->rsp.rspOffset.type == TMQ_OFFSET__LOG) { - return pBtRspObj->rsp.rspOffset.version; + } else if (TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + SMqRspObj* pRspObj = (SMqRspObj*)res; + if (pRspObj->rspOffset.type == TMQ_OFFSET__LOG) { + return pRspObj->rspOffset.version; } } else { - tscError("invalid tmq type:%d", *(int8_t*)res); + tqErrorC("invalid tmq type:%d", *(int8_t*)res); } // data from tsdb, no valid offset info @@ -2735,21 +2648,20 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { - SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); - - SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res; - if (!common->withTbName || common->blockTbName == NULL || pRspObj->resIter < 0 || - pRspObj->resIter >= common->blockNum) { + SMqRspObj* pRspObj = (SMqRspObj*)res; + SMqDataRsp* data = &pRspObj->dataRsp; + if (!data->withTbName || data->blockTbName == NULL || pRspObj->resIter < 0 || + pRspObj->resIter >= data->blockNum) { return NULL; } - return (const char*)taosArrayGetP(common->blockTbName, pRspObj->resIter); + return (const char*)taosArrayGetP(data->blockTbName, pRspObj->resIter); } return NULL; } void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* param) { if (tmq == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); if (cb != NULL) { cb(tmq, TSDB_CODE_INVALID_PARA, param); } @@ -2766,13 +2678,13 @@ static void commitCallBackFn(tmq_t* UNUSED_PARAM(tmq), int32_t code, void* param SSyncCommitInfo* pInfo = (SSyncCommitInfo*)param; pInfo->code = code; if (tsem2_post(&pInfo->sem) != 0){ - tscError("failed to post rsp sem in commit cb"); + tqErrorC("failed to post rsp sem in commit cb"); } } int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { if (tmq == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } @@ -2780,11 +2692,11 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); if (pInfo == NULL) { - tscError("failed to allocate memory for sync commit"); + tqErrorC("failed to allocate memory for sync commit"); return terrno; } if (tsem2_init(&pInfo->sem, 0, 0) != 0) { - tscError("failed to init sem for sync commit"); + tqErrorC("failed to init sem for sync commit"); taosMemoryFree(pInfo); return TSDB_CODE_OUT_OF_MEMORY; } @@ -2797,28 +2709,28 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { } if (tsem2_wait(&pInfo->sem) != 0){ - tscError("failed to wait sem for sync commit"); + tqErrorC("failed to wait sem for sync commit"); } code = pInfo->code; if(tsem2_destroy(&pInfo->sem) != 0) { - tscError("failed to destroy sem for sync commit"); + tqErrorC("failed to destroy sem for sync commit"); } taosMemoryFree(pInfo); - tscInfo("consumer:0x%" PRIx64 " sync res commit done, code:%s", tmq->consumerId, tstrerror(code)); + tqInfoC("consumer:0x%" PRIx64 " sync res commit done, code:%s", tmq->consumerId, tstrerror(code)); return code; } // wal range will be ok after calling tmq_get_topic_assignment or poll interface static int32_t checkWalRange(SVgOffsetInfo* offset, int64_t value) { if (offset->walVerBegin == -1 || offset->walVerEnd == -1) { - tscError("Assignment or poll interface need to be called first"); + tqErrorC("Assignment or poll interface need to be called first"); return TSDB_CODE_TMQ_NEED_INITIALIZED; } if (value != -1 && (value < offset->walVerBegin || value > offset->walVerEnd)) { - tscError("invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", value, + tqErrorC("invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", value, offset->walVerBegin, offset->walVerEnd); return TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE; } @@ -2828,7 +2740,7 @@ static int32_t checkWalRange(SVgOffsetInfo* offset, int64_t value) { int32_t tmq_commit_offset_sync(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_t offset) { if (tmq == NULL || pTopicName == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } @@ -2856,7 +2768,7 @@ int32_t tmq_commit_offset_sync(tmq_t* tmq, const char* pTopicName, int32_t vgId, SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); if (pInfo == NULL) { - tscError("consumer:0x%" PRIx64 " failed to prepare seek operation", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 " failed to prepare seek operation", tmq->consumerId); return terrno; } @@ -2869,18 +2781,18 @@ int32_t tmq_commit_offset_sync(tmq_t* tmq, const char* pTopicName, int32_t vgId, code = asyncCommitOffset(tmq, tname, vgId, &offsetVal, commitCallBackFn, pInfo); if (code == 0) { if (tsem2_wait(&pInfo->sem) != 0){ - tscError("failed to wait sem for sync commit offset"); + tqErrorC("failed to wait sem for sync commit offset"); } code = pInfo->code; } if (code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS; if(tsem2_destroy(&pInfo->sem) != 0) { - tscError("failed to destroy sem for sync commit offset"); + tqErrorC("failed to destroy sem for sync commit offset"); } taosMemoryFree(pInfo); - tscInfo("consumer:0x%" PRIx64 " sync send commit to vgId:%d, offset:%" PRId64 " code:%s", tmq->consumerId, vgId, + tqInfoC("consumer:0x%" PRIx64 " sync send commit to vgId:%d, offset:%" PRId64 " code:%s", tmq->consumerId, vgId, offset, tstrerror(code)); return code; @@ -2890,7 +2802,7 @@ void tmq_commit_offset_async(tmq_t* tmq, const char* pTopicName, int32_t vgId, i void* param) { int32_t code = 0; if (tmq == NULL || pTopicName == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); code = TSDB_CODE_INVALID_PARA; goto end; } @@ -2919,7 +2831,7 @@ void tmq_commit_offset_async(tmq_t* tmq, const char* pTopicName, int32_t vgId, i code = asyncCommitOffset(tmq, tname, vgId, &offsetVal, cb, param); - tscInfo("consumer:0x%" PRIx64 " async send commit to vgId:%d, offset:%" PRId64 " code:%s", tmq->consumerId, vgId, + tqInfoC("consumer:0x%" PRIx64 " async send commit to vgId:%d, offset:%" PRId64 " code:%s", tmq->consumerId, vgId, offset, tstrerror(code)); end: @@ -2929,210 +2841,21 @@ end: } } -int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { - SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; - if (pParam == NULL) { - goto FAIL; - } - - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); - if (tmq == NULL) { - code = TSDB_CODE_TMQ_CONSUMER_CLOSED; - goto FAIL; - } - - if (code != TSDB_CODE_SUCCESS) { - tscError("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code)); - goto END; - } - - if (pMsg == NULL) { - goto END; - } - SMqRspHead* head = pMsg->pData; - int32_t epoch = atomic_load_32(&tmq->epoch); - tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch); - if (pParam->sync) { - SMqAskEpRsp rsp = {0}; - if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp) != NULL) { - (void)doUpdateLocalEp(tmq, head->epoch, &rsp); - } - tDeleteSMqAskEpRsp(&rsp); - } else { - SMqAskEpRspWrapper* pWrapper = NULL; - code = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0, (void**)&pWrapper); - if (code) { - goto END; - } - - pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP; - pWrapper->epoch = head->epoch; - (void)memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); - if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg) == NULL) { - tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); - taosFreeQitem(pWrapper); - } else { - code = taosWriteQitem(tmq->mqueue, pWrapper); - if (code != 0) { - tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); - taosFreeQitem(pWrapper); - tscError("consumer:0x%" PRIx64 " put ep res into mqueue failed, code:%d", tmq->consumerId, code); - } - } - } - -END: - { - int32_t ret = taosReleaseRef(tmqMgmt.rsetId, pParam->refId); - if (ret != 0){ - tscError("failed to release ref:%"PRId64 ", code:%d", pParam->refId, ret); - } - } -FAIL: - if (pParam && pParam->sync) { - SAskEpInfo* pInfo = pParam->pParam; - if (pInfo) { - pInfo->code = code; - if (tsem2_post(&pInfo->sem) != 0){ - tscError("failed to post rsp sem askep cb"); - } - } - } - - if (pMsg) { - taosMemoryFree(pMsg->pEpSet); - taosMemoryFree(pMsg->pData); - } - - return code; -} - -int32_t syncAskEp(tmq_t* pTmq) { - SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo)); - if (pInfo == NULL) return terrno; - if (tsem2_init(&pInfo->sem, 0, 0) != 0) { - taosMemoryFree(pInfo); - return TSDB_CODE_TSC_INTERNAL_ERROR; - } - - int32_t code = askEp(pTmq, pInfo, true, false); - if (code == 0) { - if (tsem2_wait(&pInfo->sem) != 0){ - tscError("consumer:0x%" PRIx64 ", failed to wait for sem", pTmq->consumerId); - } - code = pInfo->code; - } - - if(tsem2_destroy(&pInfo->sem) != 0) { - tscError("failed to destroy sem sync ask ep"); - } - taosMemoryFree(pInfo); - return code; -} - -int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { - SMqAskEpReq req = {0}; - req.consumerId = pTmq->consumerId; - req.epoch = updateEpSet ? -1 : pTmq->epoch; - (void)strcpy(req.cgroup, pTmq->groupId); - int code = 0; - SMqAskEpCbParam* pParam = NULL; - void* pReq = NULL; - - int32_t tlen = tSerializeSMqAskEpReq(NULL, 0, &req); - if (tlen < 0) { - tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq failed", pTmq->consumerId); - return TSDB_CODE_INVALID_PARA; - } - - pReq = taosMemoryCalloc(1, tlen); - if (pReq == NULL) { - tscError("consumer:0x%" PRIx64 ", failed to malloc askEpReq msg, size:%d", pTmq->consumerId, tlen); - return terrno; - } - - if (tSerializeSMqAskEpReq(pReq, tlen, &req) < 0) { - tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq %d failed", pTmq->consumerId, tlen); - taosMemoryFree(pReq); - return TSDB_CODE_INVALID_PARA; - } - - pParam = taosMemoryCalloc(1, sizeof(SMqAskEpCbParam)); - if (pParam == NULL) { - tscError("consumer:0x%" PRIx64 ", failed to malloc subscribe param", pTmq->consumerId); - taosMemoryFree(pReq); - return terrno; - } - - pParam->refId = pTmq->refId; - pParam->sync = sync; - pParam->pParam = param; - - SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (sendInfo == NULL) { - taosMemoryFree(pReq); - taosMemoryFree(pParam); - return terrno; - } - - sendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = tlen, .handle = NULL}; - sendInfo->requestId = generateRequestId(); - sendInfo->requestObjRefId = 0; - sendInfo->param = pParam; - sendInfo->paramFreeFp = taosMemoryFree; - sendInfo->fp = askEpCb; - sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; - - SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp); - tscDebug("consumer:0x%" PRIx64 " ask ep from mnode,QID:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId); - return asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); -} - -int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { - int64_t refId = pParamSet->refId; - - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); - if (tmq == NULL) { - taosMemoryFree(pParamSet); - return TSDB_CODE_TMQ_CONSUMER_CLOSED; - } - - // if no more waiting rsp - if (pParamSet->callbackFn != NULL) { - pParamSet->callbackFn(tmq, pParamSet->code, pParamSet->userParam); - } - - taosMemoryFree(pParamSet); - return taosReleaseRef(tmqMgmt.rsetId, refId); -} - -int32_t commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId) { - int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); - if (waitingRspNum == 0) { - tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d all commit-rsp received, commit completed", consumerId, pTopic, - vgId); - return tmqCommitDone(pParamSet); - } else { - tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d commit-rsp received, remain:%d", consumerId, pTopic, vgId, - waitingRspNum); - } - return 0; -} - int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pResInfo) { - SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); - SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res; + SMqRspObj* pRspObj = (SMqRspObj*)res; + SMqDataRsp* data = &pRspObj->dataRsp; + pRspObj->resIter++; - if (pRspObj->resIter < common->blockNum) { - if (common->withSchema) { + if (pRspObj->resIter < data->blockNum) { + if (data->withSchema) { doFreeReqResultInfo(&pRspObj->resInfo); - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(common->blockSchema, pRspObj->resIter); + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(data->blockSchema, pRspObj->resIter); if (pSW) { TAOS_CHECK_RETURN(setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols)); } } - void* pRetrieve = taosArrayGetP(common->blockData, pRspObj->resIter); + void* pRetrieve = taosArrayGetP(data->blockData, pRspObj->resIter); void* rawData = NULL; int64_t rows = 0; int32_t precision = 0; @@ -3165,7 +2888,7 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t total = atomic_add_fetch_32(&pCommon->numOfRsp, 1); if (code != TSDB_CODE_SUCCESS) { - tscError("consumer:0x%" PRIx64 " failed to get the wal info from vgId:%d for topic:%s", pCommon->consumerId, + tqErrorC("consumer:0x%" PRIx64 " failed to get the wal info from vgId:%d for topic:%s", pCommon->consumerId, pParam->vgId, pCommon->pTopicName); } else { @@ -3181,12 +2904,12 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { SMqRspHead* pHead = pMsg->pData; tmq_topic_assignment assignment = {.begin = pHead->walsver, .end = pHead->walever + 1, - .currentOffset = rsp.common.rspOffset.version, + .currentOffset = rsp.rspOffset.version, .vgId = pParam->vgId}; (void)taosThreadMutexLock(&pCommon->mutex); if (taosArrayPush(pCommon->pList, &assignment) == NULL) { - tscError("consumer:0x%" PRIx64 " failed to push the wal info from vgId:%d for topic:%s", pCommon->consumerId, + tqErrorC("consumer:0x%" PRIx64 " failed to push the wal info from vgId:%d for topic:%s", pCommon->consumerId, pParam->vgId, pCommon->pTopicName); code = TSDB_CODE_TSC_INTERNAL_ERROR; } @@ -3197,7 +2920,7 @@ END: pCommon->code = code; if (total == pParam->totalReq) { if (tsem2_post(&pCommon->rsp) != 0) { - tscError("failed to post semaphore in get wal cb"); + tqErrorC("failed to post semaphore in get wal cb"); } } @@ -3215,7 +2938,7 @@ static void destroyCommonInfo(SMqVgCommon* pCommon) { } taosArrayDestroy(pCommon->pList); if(tsem2_destroy(&pCommon->rsp) != 0) { - tscError("failed to destroy semaphore for topic:%s", pCommon->pTopicName); + tqErrorC("failed to destroy semaphore for topic:%s", pCommon->pTopicName); } (void)taosThreadMutexDestroy(&pCommon->mutex); taosMemoryFree(pCommon->pTopicName); @@ -3253,7 +2976,7 @@ end: } pParam->code = code; if (tsem2_post(&pParam->sem) != 0){ - tscError("failed to post semaphore in tmCommittedCb"); + tqErrorC("failed to post semaphore in tmCommittedCb"); } return code; } @@ -3319,14 +3042,14 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, NULL, sendInfo); if (code != 0) { if(tsem2_destroy(&pParam->sem) != 0) { - tscError("failed to destroy semaphore in get committed from server1"); + tqErrorC("failed to destroy semaphore in get committed from server1"); } taosMemoryFree(pParam); return code; } if (tsem2_wait(&pParam->sem) != 0){ - tscError("failed to wait semaphore in get committed from server"); + tqErrorC("failed to wait semaphore in get committed from server"); } code = pParam->code; if (code == TSDB_CODE_SUCCESS) { @@ -3338,7 +3061,7 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep } } if(tsem2_destroy(&pParam->sem) != 0) { - tscError("failed to destroy semaphore in get committed from server2"); + tqErrorC("failed to destroy semaphore in get committed from server2"); } taosMemoryFree(pParam); @@ -3347,7 +3070,7 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep int64_t tmq_position(tmq_t* tmq, const char* pTopicName, int32_t vgId) { if (tmq == NULL || pTopicName == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } @@ -3367,7 +3090,7 @@ int64_t tmq_position(tmq_t* tmq, const char* pTopicName, int32_t vgId) { SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; int32_t type = pOffsetInfo->endOffset.type; if (isInSnapshotMode(type, tmq->useSnapshot)) { - tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, position error", tmq->consumerId, type); + tqErrorC("consumer:0x%" PRIx64 " offset type:%d not wal version, position error", tmq->consumerId, type); taosWUnLockLatch(&tmq->lock); return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } @@ -3397,16 +3120,16 @@ int64_t tmq_position(tmq_t* tmq, const char* pTopicName, int32_t vgId) { position = code; } } else { - tscError("consumer:0x%" PRIx64 " offset type:%d can not be reach here", tmq->consumerId, type); + tqErrorC("consumer:0x%" PRIx64 " offset type:%d can not be reach here", tmq->consumerId, type); } - tscInfo("consumer:0x%" PRIx64 " tmq_position vgId:%d position:%" PRId64, tmq->consumerId, vgId, position); + tqInfoC("consumer:0x%" PRIx64 " tmq_position vgId:%d position:%" PRId64, tmq->consumerId, vgId, position); return position; } int64_t tmq_committed(tmq_t* tmq, const char* pTopicName, int32_t vgId) { if (tmq == NULL || pTopicName == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } @@ -3425,14 +3148,14 @@ int64_t tmq_committed(tmq_t* tmq, const char* pTopicName, int32_t vgId) { SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; if (isInSnapshotMode(pOffsetInfo->endOffset.type, tmq->useSnapshot)) { - tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, + tqErrorC("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, pOffsetInfo->endOffset.type); taosWUnLockLatch(&tmq->lock); return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } if (isInSnapshotMode(pOffsetInfo->committedOffset.type, tmq->useSnapshot)) { - tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, + tqErrorC("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, pOffsetInfo->committedOffset.type); taosWUnLockLatch(&tmq->lock); return TSDB_CODE_TMQ_SNAPSHOT_ERROR; @@ -3450,14 +3173,14 @@ int64_t tmq_committed(tmq_t* tmq, const char* pTopicName, int32_t vgId) { committed = getCommittedFromServer(tmq, tname, vgId, &epSet); end: - tscInfo("consumer:0x%" PRIx64 " tmq_committed vgId:%d committed:%" PRId64, tmq->consumerId, vgId, committed); + tqInfoC("consumer:0x%" PRIx64 " tmq_committed vgId:%d committed:%" PRId64, tmq->consumerId, vgId, committed); return committed; } int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_assignment** assignment, int32_t* numOfAssignment) { if (tmq == NULL || pTopicName == NULL || assignment == NULL || numOfAssignment == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } *numOfAssignment = 0; @@ -3473,7 +3196,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a SMqClientTopic* pTopic = NULL; int32_t code = getTopicByName(tmq, tname, &pTopic); if (code != 0) { - tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); + tqErrorC("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); goto end; } @@ -3486,7 +3209,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } int32_t type = pClientVg->offsetInfo.beginOffset.type; if (isInSnapshotMode(type, tmq->useSnapshot)) { - tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, assignment not allowed", tmq->consumerId, type); + tqErrorC("consumer:0x%" PRIx64 " offset type:%d not wal version, assignment not allowed", tmq->consumerId, type); code = TSDB_CODE_TMQ_SNAPSHOT_ERROR; goto end; } @@ -3494,7 +3217,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a *assignment = taosMemoryCalloc(*numOfAssignment, sizeof(tmq_topic_assignment)); if (*assignment == NULL) { - tscError("consumer:0x%" PRIx64 " failed to malloc buffer, size:%" PRIzu, tmq->consumerId, + tqErrorC("consumer:0x%" PRIx64 " failed to malloc buffer, size:%" PRIzu, tmq->consumerId, (*numOfAssignment) * sizeof(tmq_topic_assignment)); code = terrno; goto end; @@ -3517,7 +3240,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a pAssignment->begin = pClientVg->offsetInfo.walVerBegin; pAssignment->end = pClientVg->offsetInfo.walVerEnd; pAssignment->vgId = pClientVg->vgId; - tscInfo("consumer:0x%" PRIx64 " get assignment from local:%d->%" PRId64, tmq->consumerId, pAssignment->vgId, + tqInfoC("consumer:0x%" PRIx64 " get assignment from local:%d->%" PRId64, tmq->consumerId, pAssignment->vgId, pAssignment->currentOffset); } @@ -3606,7 +3329,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.beginOffset); - tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s,QID:0x%" PRIx64, tmq->consumerId, + tqInfoC("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s,QID:0x%" PRIx64, tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, NULL, sendInfo); if (code != 0) { @@ -3615,7 +3338,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } if (tsem2_wait(&pCommon->rsp) != 0){ - tscError("consumer:0x%" PRIx64 " failed to wait sem in get assignment", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 " failed to wait sem in get assignment", tmq->consumerId); } code = pCommon->code; @@ -3641,7 +3364,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } SVgOffsetInfo* pOffsetInfo = &pClientVg->offsetInfo; - tscInfo("consumer:0x%" PRIx64 " %s vgId:%d offset is update to:%" PRId64, tmq->consumerId, pTopic->topicName, + tqInfoC("consumer:0x%" PRIx64 " %s vgId:%d offset is update to:%" PRId64, tmq->consumerId, pTopic->topicName, p->vgId, p->currentOffset); pOffsetInfo->walVerBegin = p->begin; @@ -3680,7 +3403,7 @@ static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) { SMqSeekParam* pParam = param; pParam->code = code; if (tsem2_post(&pParam->sem) != 0){ - tscError("failed to post sem in tmqSeekCb"); + tqErrorC("failed to post sem in tmqSeekCb"); } return 0; } @@ -3689,7 +3412,7 @@ static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) { // there is no data to poll int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_t offset) { if (tmq == NULL || pTopicName == NULL) { - tscError("invalid tmq handle, null"); + tqErrorC("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } @@ -3710,7 +3433,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ int32_t type = pOffsetInfo->endOffset.type; if (isInSnapshotMode(type, tmq->useSnapshot)) { - tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, seek not allowed", tmq->consumerId, type); + tqErrorC("consumer:0x%" PRIx64 " offset type:%d not wal version, seek not allowed", tmq->consumerId, type); taosWUnLockLatch(&tmq->lock); return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } @@ -3721,7 +3444,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ return code; } - tscInfo("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, vgId); + tqInfoC("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, vgId); // update the offset, and then commit to vnode pOffsetInfo->endOffset.type = TMQ_OFFSET__LOG; pOffsetInfo->endOffset.version = offset; @@ -3779,22 +3502,22 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); if (code != 0) { if(tsem2_destroy(&pParam->sem) != 0) { - tscError("consumer:0x%" PRIx64 "destroy rsp sem failed in seek offset", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 "destroy rsp sem failed in seek offset", tmq->consumerId); } taosMemoryFree(pParam); return code; } if (tsem2_wait(&pParam->sem) != 0){ - tscError("consumer:0x%" PRIx64 "wait rsp sem failed in seek offset", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 "wait rsp sem failed in seek offset", tmq->consumerId); } code = pParam->code; if(tsem2_destroy(&pParam->sem) != 0) { - tscError("consumer:0x%" PRIx64 "destroy rsp sem failed in seek offset", tmq->consumerId); + tqErrorC("consumer:0x%" PRIx64 "destroy rsp sem failed in seek offset", tmq->consumerId); } taosMemoryFree(pParam); - tscInfo("consumer:0x%" PRIx64 "send seek to vgId:%d, return code:%s", tmq->consumerId, vgId, tstrerror(code)); + tqInfoC("consumer:0x%" PRIx64 "send seek to vgId:%d, return code:%s", tmq->consumerId, vgId, tstrerror(code)); return code; } diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 88f97a498d..c2b9fe34e1 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -63,13 +63,10 @@ int32_t s3Begin() { TAOS_RETURN(TSDB_CODE_SUCCESS); } -void s3End() { (void)S3_deinitialize(); } +void s3End() { S3_deinitialize(); } int32_t s3Init() { TAOS_RETURN(TSDB_CODE_SUCCESS); /*s3Begin();*/ } -void s3CleanUp() { /*s3End();*/ -} - static int32_t s3ListBucket(char const *bucketname); static void s3DumpCfgByEp(int8_t epIndex) { @@ -506,7 +503,9 @@ S3Status initial_multipart_callback(const char *upload_id, void *callbackData) { } S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properties, void *callbackData) { - (void)responsePropertiesCallbackNull(properties, callbackData); + if (S3StatusOK != responsePropertiesCallbackNull(properties, callbackData)) { + uError("%s failed at line %d to process null callback.", __func__, __LINE__); + } MultipartPartData *data = (MultipartPartData *)callbackData; int seq = data->seq; @@ -517,7 +516,9 @@ S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properti } S3Status MultipartResponseProperiesCallbackWithCp(const S3ResponseProperties *properties, void *callbackData) { - (void)responsePropertiesCallbackNull(properties, callbackData); + if (S3StatusOK != responsePropertiesCallbackNull(properties, callbackData)) { + uError("%s failed at line %d to process null callback.", __func__, __LINE__); + } MultipartPartData *data = (MultipartPartData *)callbackData; int seq = data->seq; @@ -897,8 +898,6 @@ upload: if (partData.put_object_data.status != S3StatusOK) { s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); - - //(void)cos_cp_dump(&cp); } if (!manager.etags[seq - 1]) { @@ -952,7 +951,9 @@ _exit: } if (cp.thefile) { - (void)cos_cp_close(cp.thefile); + if (cos_cp_close(cp.thefile)) { + uError("%s failed at line %d to close cp file.", __func__, lino); + } } if (cp.parts) { taosMemoryFree(cp.parts); @@ -1292,7 +1293,10 @@ int32_t s3DeleteObjects(const char *object_name[], int nobject) { void s3DeleteObjectsByPrefix(const char *prefix) { SArray *objectArray = getListByPrefix(prefix); if (objectArray == NULL) return; - (void)s3DeleteObjects(TARRAY_DATA(objectArray), TARRAY_SIZE(objectArray)); + int32_t code = s3DeleteObjects(TARRAY_DATA(objectArray), TARRAY_SIZE(objectArray)); + if (!code) { + uError("%s failed at line %d since %s.", __func__, __LINE__, tstrerror(code)); + } taosArrayDestroyEx(objectArray, s3FreeObjectKey); } @@ -1539,7 +1543,7 @@ int32_t s3Init() { TAOS_RETURN(TSDB_CODE_SUCCESS); } -void s3CleanUp() { cos_http_io_deinitialize(); } +// void s3CleanUp() { cos_http_io_deinitialize(); } static void log_status(cos_status_t *s) { cos_warn_log("status->code: %d", s->code); @@ -1745,20 +1749,20 @@ bool s3Get(const char *object_name, const char *path) { cos_table_t *headers = NULL; int traffic_limit = 0; - //创建内存池 + // 创建内存池 cos_pool_create(&p, NULL); - //初始化请求选项 + // 初始化请求选项 options = cos_request_options_create(p); s3InitRequestOptions(options, is_cname); cos_str_set(&bucket, tsS3BucketName); if (traffic_limit) { - //限速值设置范围为819200 - 838860800，即100KB/s - 100MB/s，如果超出该范围将返回400错误 + // 限速值设置范围为819200 - 838860800，即100KB/s - 100MB/s，如果超出该范围将返回400错误 headers = cos_table_make(p, 1); cos_table_add_int(headers, "x-cos-traffic-limit", 819200); } - //下载对象 + // 下载对象 cos_str_set(&file, path); cos_str_set(&object, object_name); s = cos_get_object_to_file(options, &bucket, &object, headers, NULL, &file, &resp_headers); @@ -1769,7 +1773,7 @@ bool s3Get(const char *object_name, const char *path) { cos_warn_log("get object failed\n"); } - //销毁内存池 + // 销毁内存池 cos_pool_destroy(p); return ret; @@ -1791,10 +1795,10 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_ // int traffic_limit = 0; char range_buf[64]; - //创建内存池 + // 创建内存池 cos_pool_create(&p, NULL); - //初始化请求选项 + // 初始化请求选项 options = cos_request_options_create(p); // init_test_request_options(options, is_cname); s3InitRequestOptions(options, is_cname); @@ -1843,7 +1847,7 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_ // cos_warn_log("Download data=%s", buf); _exit: - //销毁内存池 + // 销毁内存池 cos_pool_destroy(p); *ppBlock = buf; @@ -1932,15 +1936,15 @@ long s3Size(const char *object_name) { cos_string_t object; cos_table_t *resp_headers = NULL; - //创建内存池 + // 创建内存池 cos_pool_create(&p, NULL); - //初始化请求选项 + // 初始化请求选项 options = cos_request_options_create(p); s3InitRequestOptions(options, is_cname); cos_str_set(&bucket, tsS3BucketName); - //获取对象元数据 + // 获取对象元数据 cos_str_set(&object, object_name); s = cos_head_object(options, &bucket, &object, NULL, &resp_headers); // print_headers(resp_headers); @@ -1954,7 +1958,7 @@ long s3Size(const char *object_name) { cos_warn_log("head object failed\n"); } - //销毁内存池 + // 销毁内存池 cos_pool_destroy(p); return size; @@ -1963,7 +1967,6 @@ long s3Size(const char *object_name) { #else int32_t s3Init() { return 0; } -void s3CleanUp() {} int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; } int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; } int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { return 0; } diff --git a/source/common/src/cos_cp.c b/source/common/src/cos_cp.c index adf4160abe..078b14c9e8 100644 --- a/source/common/src/cos_cp.c +++ b/source/common/src/cos_cp.c @@ -309,7 +309,7 @@ int32_t cos_cp_dump(SCheckpoint* cp) { if (!item) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } - (void)cJSON_AddItemToArray(ajson, item); + if (!cJSON_AddItemToArray(ajson, item)) goto _exit; if (NULL == cJSON_AddNumberToObject(item, "index", cp->parts[i].index)) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c index 77787db3ce..47a452eab7 100644 --- a/source/common/src/rsync.c +++ b/source/common/src/rsync.c @@ -32,10 +32,14 @@ static void removeEmptyDir() { empty = false; } if (empty) taosRemoveDir(filename); - (void)taosCloseDir(&pDirTmp); + if (taosCloseDir(&pDirTmp) != 0) { + uError("[rsync] close dir error," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); + } } - (void)taosCloseDir(&pDir); + if (taosCloseDir(&pDir) != 0) { + uError("[rsync] close dir error," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); + } } #ifdef WINDOWS @@ -297,7 +301,7 @@ int32_t downloadByRsync(const char* id, const char* path, int64_t checkpointId) path, el); } - if (code != TSDB_CODE_SUCCESS) { // if failed, try to load it from data directory + if (code != TSDB_CODE_SUCCESS) { // if failed, try to load it from data directory #ifdef WINDOWS memset(pathTransform, 0, PATH_MAX); changeDirFromWindowsToLinux(path, pathTransform); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d8a66f82bf..7a67522231 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -867,6 +867,7 @@ int32_t blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t r code = blockDataEnsureCapacity(pDst, rowCount); if (code) { + blockDataDestroy(pDst); return code; } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 4ef9bf481c..e580ad33bd 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1771,26 +1771,6 @@ _err: return code; } -void tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid) { - uint8_t *p = NULL; - int8_t isLarge = pTag->flags & TD_TAG_LARGE; - int16_t offset = 0; - - if (isLarge) { - p = (uint8_t *)&((int16_t *)pTag->idx)[pTag->nTag]; - } else { - p = (uint8_t *)&pTag->idx[pTag->nTag]; - } - - if (isLarge) { - offset = ((int16_t *)pTag->idx)[iTag]; - } else { - offset = pTag->idx[iTag]; - } - - int32_t nt = tPutI16v(p + offset, cid); -} - // STSchema ======================================== STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols); @@ -4171,12 +4151,12 @@ int32_t tValueColumnInit(SValueColumn *valCol) { return 0; } -int32_t tValueColumnDestroy(SValueColumn *valCol) { +void tValueColumnDestroy(SValueColumn *valCol) { valCol->type = TSDB_DATA_TYPE_NULL; valCol->numOfValues = 0; tBufferDestroy(&valCol->data); tBufferDestroy(&valCol->offsets); - return 0; + return; } void tValueColumnClear(SValueColumn *valCol) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a3220706c7..af1a8ccfbe 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -542,6 +542,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebug", tqClientDebug, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); @@ -1957,7 +1958,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"tdbDebugFlag", &tdbDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag}, {"metaDebugFlag", &metaDebugFlag}, - {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, + {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, {"tqClientDebug", &tqClientDebug}, }; static OptionNameAndVar options[] = {{"audit", &tsEnableAudit}, @@ -2299,11 +2300,14 @@ static int taosLogVarComp(void const *lp, void const *rp) { return strcasecmp(lpVar->name, rpVar->name); } -static int32_t taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag, SArray *noNeedToSetVars) { +static void taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag, SArray *noNeedToSetVars) { if (noNeedToSetVars != NULL && taosArraySearch(noNeedToSetVars, name, taosLogVarComp, TD_EQ) != NULL) { - TAOS_RETURN(TSDB_CODE_SUCCESS); + return; } - return taosSetDebugFlag(pFlagPtr, name, flag); + if (taosSetDebugFlag(pFlagPtr, name, flag) != 0) { + uError("failed to set flag %s to %d", name, flag); + } + return; } int32_t taosSetGlobalDebugFlag(int32_t flag) { return taosSetAllDebugFlag(tsCfg, flag); } @@ -2320,29 +2324,29 @@ static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag) { pItem->i32 = flag; noNeedToSetVars = pItem->array; - (void)taosCheckAndSetDebugFlag(&simDebugFlag, "simDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&uDebugFlag, "uDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&rpcDebugFlag, "rpcDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&qDebugFlag, "qDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&simDebugFlag, "simDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&uDebugFlag, "uDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&rpcDebugFlag, "rpcDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&qDebugFlag, "qDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&jniDebugFlag, "jniDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&cDebugFlag, "cDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&jniDebugFlag, "jniDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&cDebugFlag, "cDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&dDebugFlag, "dDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&vDebugFlag, "vDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&mDebugFlag, "mDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&wDebugFlag, "wDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&sDebugFlag, "sDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&tsdbDebugFlag, "tsdbDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&tqDebugFlag, "tqDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&fsDebugFlag, "fsDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&udfDebugFlag, "udfDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&smaDebugFlag, "smaDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&idxDebugFlag, "idxDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&stDebugFlag, "stDebugFlag", flag, noNeedToSetVars); - (void)taosCheckAndSetDebugFlag(&sndDebugFlag, "sndDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&dDebugFlag, "dDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&vDebugFlag, "vDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&mDebugFlag, "mDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&wDebugFlag, "wDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&sDebugFlag, "sDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&tsdbDebugFlag, "tsdbDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&tqDebugFlag, "tqDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&fsDebugFlag, "fsDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&udfDebugFlag, "udfDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&smaDebugFlag, "smaDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&idxDebugFlag, "idxDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&stDebugFlag, "stDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&sndDebugFlag, "sndDebugFlag", flag, noNeedToSetVars); taosArrayClear(noNeedToSetVars); // reset array diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 557ac07bbd..986747fe58 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -8163,6 +8163,7 @@ int32_t tSerializeSMqHbRsp(void *buf, int32_t bufLen, SMqHbRsp *pRsp) { TAOS_CHECK_EXIT(tEncodeI8(&encoder, privilege->noPrivilege)); } + if (tEncodeI32(&encoder, pRsp->debugFlag) < 0) return -1; tEndEncode(&encoder); _exit: @@ -8196,6 +8197,10 @@ int32_t tDeserializeSMqHbRsp(void *buf, int32_t bufLen, SMqHbRsp *pRsp) { TAOS_CHECK_EXIT(tDecodeI8(&decoder, &data->noPrivilege)); } } + + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI32(&decoder, &pRsp->debugFlag) < 0) return -1; + } tEndDecode(&decoder); _exit: @@ -10681,7 +10686,7 @@ int32_t tDecodeMqMetaRsp(SDecoder *pDecoder, SMqMetaRsp *pRsp) { void tDeleteMqMetaRsp(SMqMetaRsp *pRsp) { taosMemoryFree(pRsp->metaRsp); } -int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRspCommon *pRsp) { +int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRsp *pRsp) { int32_t code = 0; int32_t lino; @@ -10711,19 +10716,20 @@ _exit: return code; } -int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const void *pRsp) { +int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { TAOS_CHECK_RETURN(tEncodeMqDataRspCommon(pEncoder, pRsp)); - TAOS_CHECK_RETURN(tEncodeI64(pEncoder, ((SMqDataRsp *)pRsp)->sleepTime)); + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pRsp->sleepTime)); return 0; } -int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRspCommon *pRsp) { +int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) { int32_t code = 0; int32_t lino; TAOS_CHECK_EXIT(tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset)); TAOS_CHECK_EXIT(tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset)); TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->blockNum)); + if (pRsp->blockNum != 0) { if ((pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void *))) == NULL) { TAOS_CHECK_EXIT(terrno); @@ -10787,17 +10793,16 @@ _exit: return code; } -int32_t tDecodeMqDataRsp(SDecoder *pDecoder, void *pRsp) { +int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { TAOS_CHECK_RETURN(tDecodeMqDataRspCommon(pDecoder, pRsp)); if (!tDecodeIsEnd(pDecoder)) { - TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &((SMqDataRsp *)pRsp)->sleepTime)); + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pRsp->sleepTime)); } return 0; } -static void tDeleteMqDataRspCommon(void *rsp) { - SMqDataRspCommon *pRsp = rsp; +static void tDeleteMqDataRspCommon(SMqDataRsp *pRsp) { taosArrayDestroy(pRsp->blockDataLen); pRsp->blockDataLen = NULL; taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); @@ -10810,15 +10815,13 @@ static void tDeleteMqDataRspCommon(void *rsp) { tOffsetDestroy(&pRsp->rspOffset); } -void tDeleteMqDataRsp(void *rsp) { tDeleteMqDataRspCommon(rsp); } +void tDeleteMqDataRsp(SMqDataRsp *rsp) { tDeleteMqDataRspCommon(rsp); } -int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const void *rsp) { +int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { int32_t code = 0; int32_t lino; - TAOS_CHECK_EXIT(tEncodeMqDataRspCommon(pEncoder, rsp)); - - const STaosxRsp *pRsp = (const STaosxRsp *)rsp; + TAOS_CHECK_EXIT(tEncodeMqDataRspCommon(pEncoder, pRsp)); TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->createTableNum)); if (pRsp->createTableNum) { for (int32_t i = 0; i < pRsp->createTableNum; i++) { @@ -10831,13 +10834,11 @@ _exit: return code; } -int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, void *rsp) { +int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { int32_t code = 0; int32_t lino; - TAOS_CHECK_EXIT(tDecodeMqDataRspCommon(pDecoder, rsp)); - - STaosxRsp *pRsp = (STaosxRsp *)rsp; + TAOS_CHECK_EXIT(tDecodeMqDataRspCommon(pDecoder, pRsp)); TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->createTableNum)); if (pRsp->createTableNum) { if ((pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t))) == NULL) { @@ -10864,10 +10865,9 @@ _exit: return code; } -void tDeleteSTaosxRsp(void *rsp) { - tDeleteMqDataRspCommon(rsp); +void tDeleteSTaosxRsp(SMqDataRsp *pRsp) { + tDeleteMqDataRspCommon(pRsp); - STaosxRsp *pRsp = (STaosxRsp *)rsp; taosArrayDestroy(pRsp->createTableLen); pRsp->createTableLen = NULL; taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree); diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 626d1141e0..2b95e96130 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -354,8 +354,10 @@ bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVa } void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); - (void)tdGetKvRowValOfCol(pVal, pRow, pBitmap, pColIdx->offset, - POINTER_DISTANCE(pColIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx)); + if (tdGetKvRowValOfCol(pVal, pRow, pBitmap, pColIdx->offset, + POINTER_DISTANCE(pColIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx)) != TSDB_CODE_SUCCESS) { + return false; + } return true; } diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 7e8749ef8b..98e46ab672 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -30,7 +30,9 @@ static int64_t m_deltaUtc = 0; void deltaToUtcInitOnce() { struct tm tm = {0}; - (void)taosStrpTime("1970-01-01 00:00:00", (const char*)("%Y-%m-%d %H:%M:%S"), &tm); + if (taosStrpTime("1970-01-01 00:00:00", (const char*)("%Y-%m-%d %H:%M:%S"), &tm) != 0) { + uError("failed to parse time string"); + } m_deltaUtc = (int64_t)taosMktime(&tm); // printf("====delta:%lld\n\n", seconds); } @@ -689,10 +691,10 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { int64_t numOfMonth = (unit == 'y') ? duration * 12 : duration; int64_t fraction = t % TSDB_TICK_PER_SECOND(precision); - struct tm tm; - time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision)); - (void)taosLocalTime(&tt, &tm, NULL); - int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)numOfMonth; + struct tm tm; + time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision)); + struct tm* ptm = taosLocalTime(&tt, &tm, NULL); + int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)numOfMonth; tm.tm_year = mon / 12; tm.tm_mon = mon % 12; int daysOfMonth[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; @@ -750,13 +752,13 @@ int32_t taosTimeCountIntervalForFill(int64_t skey, int64_t ekey, int64_t interva skey /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); ekey /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); - struct tm tm; - time_t t = (time_t)skey; - (void)taosLocalTime(&t, &tm, NULL); - int32_t smon = tm.tm_year * 12 + tm.tm_mon; + struct tm tm; + time_t t = (time_t)skey; + struct tm* ptm = taosLocalTime(&t, &tm, NULL); + int32_t smon = tm.tm_year * 12 + tm.tm_mon; t = (time_t)ekey; - (void)taosLocalTime(&t, &tm, NULL); + ptm = taosLocalTime(&t, &tm, NULL); int32_t emon = tm.tm_year * 12 + tm.tm_mon; if (unit == 'y') { @@ -778,9 +780,9 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { if (IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) { start /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); - struct tm tm; - time_t tt = (time_t)start; - (void)taosLocalTime(&tt, &tm, NULL); + struct tm tm; + time_t tt = (time_t)start; + struct tm* ptm = taosLocalTime(&tt, &tm, NULL); tm.tm_sec = 0; tm.tm_min = 0; tm.tm_hour = 0; diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 1fb5271a0e..89569d69d6 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -81,11 +81,21 @@ static void dmSetAssert(int32_t signum, void *sigInfo, void *context) { tsAssert static void dmStopDnode(int signum, void *sigInfo, void *context) { // taosIgnSignal(SIGUSR1); // taosIgnSignal(SIGUSR2); - (void)taosIgnSignal(SIGTERM); - (void)taosIgnSignal(SIGHUP); - (void)taosIgnSignal(SIGINT); - (void)taosIgnSignal(SIGABRT); - (void)taosIgnSignal(SIGBREAK); + if (taosIgnSignal(SIGTERM) != 0) { + dWarn("failed to ignore signal SIGTERM"); + } + if (taosIgnSignal(SIGHUP) != 0) { + dWarn("failed to ignore signal SIGHUP"); + } + if (taosIgnSignal(SIGINT) != 0) { + dWarn("failed to ignore signal SIGINT"); + } + if (taosIgnSignal(SIGABRT) != 0) { + dWarn("failed to ignore signal SIGABRT"); + } + if (taosIgnSignal(SIGBREAK) != 0) { + dWarn("failed to ignore signal SIGBREAK"); + } dInfo("shut down signal is %d", signum); #ifndef WINDOWS @@ -103,11 +113,19 @@ void dmLogCrash(int signum, void *sigInfo, void *context) { // taosIgnSignal(SIGBREAK); #ifndef WINDOWS - (void)taosIgnSignal(SIGBUS); + if (taosIgnSignal(SIGBUS) != 0) { + dWarn("failed to ignore signal SIGBUS"); + } #endif - (void)taosIgnSignal(SIGABRT); - (void)taosIgnSignal(SIGFPE); - (void)taosIgnSignal(SIGSEGV); + if (taosIgnSignal(SIGABRT) != 0) { + dWarn("failed to ignore signal SIGABRT"); + } + if (taosIgnSignal(SIGFPE) != 0) { + dWarn("failed to ignore signal SIGABRT"); + } + if (taosIgnSignal(SIGSEGV) != 0) { + dWarn("failed to ignore signal SIGABRT"); + } char *pMsg = NULL; const char *flags = "UTL FATAL "; @@ -136,24 +154,31 @@ _return: } static void dmSetSignalHandle() { - (void)taosSetSignal(SIGUSR1, dmSetDebugFlag); - (void)taosSetSignal(SIGUSR2, dmSetAssert); - (void)taosSetSignal(SIGTERM, dmStopDnode); - (void)taosSetSignal(SIGHUP, dmStopDnode); - (void)taosSetSignal(SIGINT, dmStopDnode); - (void)taosSetSignal(SIGBREAK, dmStopDnode); + if (taosSetSignal(SIGUSR1, dmSetDebugFlag) != 0) { + dWarn("failed to set signal SIGUSR1"); + } + if (taosSetSignal(SIGUSR2, dmSetAssert) != 0) { + dWarn("failed to set signal SIGUSR1"); + } + if (taosSetSignal(SIGTERM, dmStopDnode) != 0) { + dWarn("failed to set signal SIGUSR1"); + } + if (taosSetSignal(SIGHUP, dmStopDnode) != 0) { + dWarn("failed to set signal SIGUSR1"); + } + if (taosSetSignal(SIGINT, dmStopDnode) != 0) { + dWarn("failed to set signal SIGUSR1"); + } + if (taosSetSignal(SIGBREAK, dmStopDnode) != 0) { + dWarn("failed to set signal SIGUSR1"); + } #ifndef WINDOWS - (void)taosSetSignal(SIGTSTP, dmStopDnode); - (void)taosSetSignal(SIGQUIT, dmStopDnode); -#endif - -#if 0 -#ifndef WINDOWS - (void)taosSetSignal(SIGBUS, dmLogCrash); -#endif - (void)taosSetSignal(SIGABRT, dmLogCrash); - (void)taosSetSignal(SIGFPE, dmLogCrash); - (void)taosSetSignal(SIGSEGV, dmLogCrash); + if (taosSetSignal(SIGTSTP, dmStopDnode) != 0) { + dWarn("failed to set signal SIGUSR1"); + } + if (taosSetSignal(SIGQUIT, dmStopDnode) != 0) { + dWarn("failed to set signal SIGUSR1"); + } #endif } @@ -164,6 +189,9 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { if (argc < 2) return 0; global.envCmd = taosMemoryMalloc((argc - 1) * sizeof(char *)); + if (global.envCmd == NULL) { + return terrno; + } memset(global.envCmd, 0, (argc - 1) * sizeof(char *)); for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 020a8077b2..419c669103 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -45,7 +45,9 @@ static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) { if (pMgmt->pData->ipWhiteVer == ver) { if (ver == 0) { dDebug("disable ip-white-list on dnode ver: %" PRId64 ", status ver: %" PRId64 "", pMgmt->pData->ipWhiteVer, ver); - (void)rpcSetIpWhite(pMgmt->msgCb.serverRpc, NULL); + if (rpcSetIpWhite(pMgmt->msgCb.serverRpc, NULL) != 0) { + dError("failed to disable ip white list on dnode"); + } } return; } @@ -91,7 +93,9 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { dGInfo("dnode:%d, set to dropped since not exist in mnode, statusSeq:%d", pMgmt->pData->dnodeId, pMgmt->statusSeq); pMgmt->pData->dropped = 1; - (void)dmWriteEps(pMgmt->pData); + if (dmWriteEps(pMgmt->pData) != 0) { + dError("failed to write dnode file"); + } dInfo("dnode will exit since it is in the dropped state"); (void)raise(SIGINT); } @@ -147,7 +151,9 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { req.clusterCfg.monitorParas.tsSlowLogThresholdTest = tsSlowLogThresholdTest; tstrncpy(req.clusterCfg.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN); char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + if (taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0) != 0) { + dError("failed to parse time since %s", tstrerror(code)); + } memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); @@ -243,7 +249,9 @@ void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq) { SEpSet epSet = {0}; dmGetMnodeEpSet(pMgmt->pData, &epSet); - (void)rpcSendRequest(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, NULL); + if (rpcSendRequest(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, NULL) != 0) { + dError("failed to send notify req"); + } } int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 1ed7c9ecd9..58b86b20b1 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -305,11 +305,16 @@ int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) { void dmStopNotifyThread(SDnodeMgmt *pMgmt) { if (taosCheckPthreadValid(pMgmt->notifyThread)) { - (void)tsem_post(&dmNotifyHdl.sem); + if (tsem_post(&dmNotifyHdl.sem) != 0) { + dError("failed to post notify sem"); + } + (void)taosThreadJoin(pMgmt->notifyThread, NULL); taosThreadClear(&pMgmt->notifyThread); } - (void)tsem_destroy(&dmNotifyHdl.sem); + if (tsem_destroy(&dmNotifyHdl.sem) != 0) { + dError("failed to destroy notify sem"); + } } int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) { diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 43c40c65c3..7204cde8f7 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -17,7 +17,9 @@ #include "mmInt.h" void mmGetMonitorInfo(SMnodeMgmt *pMgmt, SMonMmInfo *pInfo) { - (void)mndGetMonitorInfo(pMgmt->pMnode, &pInfo->cluster, &pInfo->vgroup, &pInfo->stb, &pInfo->grant); + if (mndGetMonitorInfo(pMgmt->pMnode, &pInfo->cluster, &pInfo->vgroup, &pInfo->stb, &pInfo->grant) != 0) { + dError("failed to get monitor info"); + } } void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) { diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index 9ae16f7581..65c2bb9bf3 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -23,7 +23,7 @@ static inline void qmSendRsp(SRpcMsg *pMsg, int32_t code) { .contLen = pMsg->info.rspLen, .info = pMsg->info, }; - (void)tmsgSendRsp(&rsp); + tmsgSendRsp(&rsp); } static void qmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index e8594130d6..8c33c5bb4b 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -23,15 +23,15 @@ static inline void smSendRsp(SRpcMsg *pMsg, int32_t code) { .contLen = pMsg->info.rspLen, .info = pMsg->info, }; - (void)tmsgSendRsp(&rsp); + tmsgSendRsp(&rsp); } static void smProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SSnodeMgmt *pMgmt = pInfo->ahandle; for (int32_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pMsg = NULL; - (void)taosGetQitem(qall, (void **)&pMsg); + SRpcMsg *pMsg = NULL; + int32_t num = taosGetQitem(qall, (void **)&pMsg); const STraceId *trace = &pMsg->info.traceId; dTrace("msg:%p, get from snode-write queue", pMsg); diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index 6b01b92445..0e1a4bc98e 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -77,6 +77,7 @@ typedef struct { typedef struct { int32_t vnodeNum; int32_t opened; + int32_t dropped; int32_t failed; bool updateVnodesList; int32_t threadIndex; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 54e2bac66d..70c873e0f5 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -35,10 +35,14 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo, bool isReset) { SVnodeObj *pVnode = *ppVnode; SVnodeLoad vload = {.vgId = pVnode->vgId}; if (!pVnode->failed) { - (void)vnodeGetLoad(pVnode->pImpl, &vload); + if (vnodeGetLoad(pVnode->pImpl, &vload) != 0) { + dError("failed to get vnode load"); + } if (isReset) vnodeResetLoad(pVnode->pImpl, &vload); } - (void)taosArrayPush(pInfo->pVloads, &vload); + if (taosArrayPush(pInfo->pVloads, &vload) == NULL) { + dError("failed to push vnode load"); + } pIter = taosHashIterate(pMgmt->hash, pIter); } @@ -116,7 +120,9 @@ void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) { pMgmt->state.numOfBatchInsertReqs = numOfBatchInsertReqs; pMgmt->state.numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs; - (void)tfsGetMonitorInfo(pMgmt->pTfs, &pInfo->tfs); + if (tfsGetMonitorInfo(pMgmt->pTfs, &pInfo->tfs) != 0) { + dError("failed to get tfs monitor info"); + } taosArrayDestroy(pVloads); } @@ -235,7 +241,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pNode->nodePort = pCreate->replicas[pCfg->syncCfg.replicaNum].port; pNode->nodeRole = TAOS_SYNC_ROLE_VOTER; tstrncpy(pNode->nodeFqdn, pCreate->replicas[pCfg->syncCfg.replicaNum].fqdn, TSDB_FQDN_LEN); - (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + bool ret = tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); pCfg->syncCfg.replicaNum++; } if (pCreate->selfIndex != -1) { @@ -247,7 +253,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pNode->nodePort = pCreate->learnerReplicas[pCfg->syncCfg.totalReplicaNum].port; pNode->nodeRole = TAOS_SYNC_ROLE_LEARNER; tstrncpy(pNode->nodeFqdn, pCreate->learnerReplicas[pCfg->syncCfg.totalReplicaNum].fqdn, TSDB_FQDN_LEN); - (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + bool ret = tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); pCfg->syncCfg.totalReplicaNum++; } pCfg->syncCfg.totalReplicaNum += pCfg->syncCfg.replicaNum; @@ -845,7 +851,9 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } vmCloseVnode(pMgmt, pVnode, false); - (void)vmWriteVnodeListToFile(pMgmt); + if (vmWriteVnodeListToFile(pMgmt) != 0) { + dError("vgId:%d, failed to write vnode list since %s", vgId, terrstr()); + } dInfo("vgId:%d, is dropped", vgId); return 0; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index dd921c615b..3cf0382eba 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -24,7 +24,7 @@ int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { SVnodeObj *pVnode = NULL; (void)taosThreadRwlockRdlock(&pMgmt->lock); - (void)taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); + int32_t r = taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); if (pVnode != NULL) { diskId = pVnode->diskPrimary; } @@ -97,7 +97,7 @@ SVnodeObj *vmAcquireVnodeImpl(SVnodeMgmt *pMgmt, int32_t vgId, bool strict) { SVnodeObj *pVnode = NULL; (void)taosThreadRwlockRdlock(&pMgmt->lock); - (void)taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); + int32_t r = taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); if (pVnode == NULL || strict && (pVnode->dropped || pVnode->failed)) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; pVnode = NULL; @@ -165,7 +165,7 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { (void)taosThreadRwlockWrlock(&pMgmt->lock); SVnodeObj *pOld = NULL; - (void)taosHashGetDup(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), (void *)&pOld); + int32_t r = taosHashGetDup(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), (void *)&pOld); if (pOld) { vmFreeVnodeObj(&pOld); } @@ -184,7 +184,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) } (void)taosThreadRwlockWrlock(&pMgmt->lock); - (void)taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); + int32_t r = taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); (void)taosThreadRwlockUnlock(&pMgmt->lock); vmReleaseVnode(pMgmt, pVnode); @@ -233,8 +233,12 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) if (commitAndRemoveWal) { dInfo("vgId:%d, commit data for vnode split", pVnode->vgId); - (void)vnodeSyncCommit(pVnode->pImpl); - (void)vnodeBegin(pVnode->pImpl); + if (vnodeSyncCommit(pVnode->pImpl) != 0) { + dError("vgId:%d, failed to commit data", pVnode->vgId); + } + if (vnodeBegin(pVnode->pImpl) != 0) { + dError("vgId:%d, failed to begin", pVnode->vgId); + } dInfo("vgId:%d, commit data finished", pVnode->vgId); } @@ -248,8 +252,12 @@ _closed: if (commitAndRemoveWal) { snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d%swal", TD_DIRSEP, pVnode->vgId, TD_DIRSEP); dInfo("vgId:%d, remove all wals, path:%s", pVnode->vgId, path); - (void)tfsRmdir(pMgmt->pTfs, path); - (void)tfsMkdir(pMgmt->pTfs, path); + if (tfsRmdir(pMgmt->pTfs, path) != 0) { + dTrace("vgId:%d, failed to remove wals, path:%s", pVnode->vgId, path); + } + if (tfsMkdir(pMgmt->pTfs, path) != 0) { + dTrace("vgId:%d, failed to create wals, path:%s", pVnode->vgId, path); + } } if (pVnode->dropped) { @@ -289,11 +297,24 @@ static void *vmOpenVnodeInThread(void *param) { SVnodeMgmt *pMgmt = pThread->pMgmt; char path[TSDB_FILENAME_LEN]; - dInfo("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); + dInfo("thread:%d, start to open or destroy %d vnodes", pThread->threadIndex, pThread->vnodeNum); setThreadName("open-vnodes"); for (int32_t v = 0; v < pThread->vnodeNum; ++v) { SWrapperCfg *pCfg = &pThread->pCfgs[v]; + if (pCfg->dropped) { + char stepDesc[TSDB_STEP_DESC_LEN] = {0}; + snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to destroy, %d of %d have been dropped", pCfg->vgId, + pMgmt->state.openVnodes, pMgmt->state.totalVnodes); + tmsgReportStartup("vnode-destroy", stepDesc); + + snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId); + vnodeDestroy(pCfg->vgId, path, pMgmt->pTfs, 0); + pThread->updateVnodesList = true; + pThread->dropped++; + (void)atomic_add_fetch_32(&pMgmt->state.dropVnodes, 1); + continue; + } char stepDesc[TSDB_STEP_DESC_LEN] = {0}; snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId, @@ -333,8 +354,8 @@ static void *vmOpenVnodeInThread(void *param) { (void)atomic_add_fetch_32(&pMgmt->state.openVnodes, 1); } - dInfo("thread:%d, numOfVnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, - pThread->failed); + dInfo("thread:%d, numOfVnodes:%d, opened:%d dropped:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, + pThread->opened, pThread->dropped, pThread->failed); return NULL; } @@ -408,7 +429,7 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) { taosMemoryFree(threads); taosMemoryFree(pCfgs); - if (pMgmt->state.openVnodes != pMgmt->state.totalVnodes) { + if ((pMgmt->state.openVnodes + pMgmt->state.dropVnodes) != pMgmt->state.totalVnodes) { dError("there are total vnodes:%d, opened:%d", pMgmt->state.totalVnodes, pMgmt->state.openVnodes); terrno = TSDB_CODE_VND_INIT_FAILED; return -1; @@ -755,6 +776,7 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) { } pMgmt->state.openVnodes = 0; + pMgmt->state.dropVnodes = 0; dInfo("restore %d vnodes with %d threads", numOfVnodes, threadNum); for (int32_t t = 0; t < threadNum; ++t) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 9829d5ab3a..9c436a3dfa 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -187,7 +187,9 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf static void vmSendResponse(SRpcMsg *pMsg) { if (pMsg->info.handle) { SRpcMsg rsp = {.info = pMsg->info, .code = terrno}; - (void)rpcSendResponse(&rsp); + if (rpcSendResponse(&rsp) != 0) { + dError("failed to send response since %s", terrstr()); + } } } @@ -389,10 +391,28 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { SMultiWorkerCfg scfg = {.max = 1, .name = "vnode-sync", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-rd", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; SMultiWorkerCfg acfg = {.max = 1, .name = "vnode-apply", .fp = (FItems)vnodeApplyWriteMsg, .param = pVnode->pImpl}; - (void)tMultiWorkerInit(&pVnode->pWriteW, &wcfg); - (void)tMultiWorkerInit(&pVnode->pSyncW, &scfg); - (void)tMultiWorkerInit(&pVnode->pSyncRdW, &sccfg); - (void)tMultiWorkerInit(&pVnode->pApplyW, &acfg); + code = tMultiWorkerInit(&pVnode->pWriteW, &wcfg); + if (code) { + return code; + } + code = tMultiWorkerInit(&pVnode->pSyncW, &scfg); + if (code) { + tMultiWorkerCleanup(&pVnode->pWriteW); + return code; + } + code = tMultiWorkerInit(&pVnode->pSyncRdW, &sccfg); + if (code) { + tMultiWorkerCleanup(&pVnode->pWriteW); + tMultiWorkerCleanup(&pVnode->pSyncW); + return code; + } + code = tMultiWorkerInit(&pVnode->pApplyW, &acfg); + if (code) { + tMultiWorkerCleanup(&pVnode->pWriteW); + tMultiWorkerCleanup(&pVnode->pSyncW); + tMultiWorkerCleanup(&pVnode->pSyncRdW); + return code; + } pVnode->pQueryQ = tQueryAutoQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); pVnode->pStreamQ = tAutoQWorkerAllocQueue(&pMgmt->streamPool, pVnode, (FItem)vmProcessStreamQueue); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index f48ca35330..2d0ad70adf 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -47,8 +47,14 @@ static int32_t dmCheckRepeatInit(SDnode *pDnode) { } static int32_t dmInitSystem() { - (void)taosIgnSIGPIPE(); - (void)taosBlockSIGPIPE(); + if (taosIgnSIGPIPE() != 0) { + dError("failed to ignore SIGPIPE"); + } + + if (taosBlockSIGPIPE() != 0) { + dError("failed to block SIGPIPE"); + } + taosResolveCRC(); return 0; } @@ -204,7 +210,9 @@ void dmCleanup() { auditCleanup(); syncCleanUp(); walCleanUp(); - (void)udfcClose(); + if (udfcClose() != 0) { + dError("failed to close udfc"); + } udfStopUdfd(); taosStopCacheRefreshWorker(); (void)dmDiskClose(); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index ba0a40e048..f77571c665 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -47,8 +47,7 @@ int32_t dmInitDnode(SDnode *pDnode) { } // compress module init - (void)tsCompressInit(tsLossyColumns, tsFPrecision, tsDPrecision, tsMaxRange, tsCurRange, (int)tsIfAdtFse, - tsCompressor); + tsCompressInit(tsLossyColumns, tsFPrecision, tsDPrecision, tsMaxRange, tsCurRange, (int)tsIfAdtFse, tsCompressor); pDnode->wrappers[DNODE].func = dmGetMgmtFunc(); pDnode->wrappers[MNODE].func = mmGetMgmtFunc(); @@ -226,7 +225,10 @@ void dmClearVars(SDnode *pDnode) { (void)taosThreadRwlockDestroy(&pWrapper->lock); } if (pDnode->lockfile != NULL) { - (void)taosUnLockFile(pDnode->lockfile); + if (taosUnLockFile(pDnode->lockfile) != 0) { + dError("failed to unlock file"); + } + (void)taosCloseFile(&pDnode->lockfile); pDnode->lockfile = NULL; } @@ -343,7 +345,9 @@ void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg) { rsp.contLen = pMsg->contLen; } - (void)rpcSendResponse(&rsp); + if (rpcSendResponse(&rsp) != 0) { + dError("failed to send response, msg:%p", &rsp); + } rpcFreeCont(pMsg->pCont); } @@ -360,11 +364,16 @@ void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg) { } else { rsp.pCont = rpcMallocCont(contLen); if (rsp.pCont != NULL) { - (void)tSerializeSServerStatusRsp(rsp.pCont, contLen, &statusRsp); - rsp.contLen = contLen; + if (tSerializeSServerStatusRsp(rsp.pCont, contLen, &statusRsp) < 0) { + rsp.code = TSDB_CODE_APP_ERROR; + } else { + rsp.contLen = contLen; + } } } - (void)rpcSendResponse(&rsp); + if (rpcSendResponse(&rsp) != 0) { + dError("failed to send response, msg:%p", &rsp); + } rpcFreeCont(pMsg->pCont); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index e204b5d4aa..b9f4ab54f4 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -18,7 +18,11 @@ #include "qworker.h" #include "tversion.h" -static inline void dmSendRsp(SRpcMsg *pMsg) { (void)rpcSendResponse(pMsg); } +static inline void dmSendRsp(SRpcMsg *pMsg) { + if (rpcSendResponse(pMsg) != 0) { + dError("failed to send response, msg:%p", pMsg); + } +} static inline void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) { SEpSet epSet = {0}; @@ -113,7 +117,11 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { pRpc->info.handle, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId); int32_t svrVer = 0; - (void)taosVersionStrToInt(version, &svrVer); + code = taosVersionStrToInt(version, &svrVer); + if (code != 0) { + dError("failed to convert version string:%s to int, code:%d", version, code); + goto _OVER; + } if ((code = taosCheckVersionCompatible(pRpc->info.cliVer, svrVer, 3)) != 0) { dError("Version not compatible, cli ver: %d, svr ver: %d, ip:0x%x", pRpc->info.cliVer, svrVer, pRpc->info.conn.clientIp); @@ -253,7 +261,9 @@ _OVER: if (pWrapper != NULL) { dmSendRsp(&rsp); } else { - (void)rpcSendResponse(&rsp); + if (rpcSendResponse(&rsp) != 0) { + dError("failed to send response, msg:%p", &rsp); + } } } @@ -310,7 +320,9 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { return code; } else { pMsg->info.handle = 0; - (void)rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pMsg, NULL); + if (rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pMsg, NULL) != 0) { + dError("failed to send rpc msg"); + } return 0; } } @@ -396,7 +408,9 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.timeToGetConn = tsTimeToGetAvailableConn; rpcInit.notWaitAvaliableConn = 0; - (void)taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); + if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) { + dError("failed to convert version string:%s to int", version); + } pTrans->clientRpc = rpcOpen(&rpcInit); if (pTrans->clientRpc == NULL) { @@ -440,7 +454,10 @@ int32_t dmInitStatusClient(SDnode *pDnode) { rpcInit.supportBatch = 1; rpcInit.batchSize = 8 * 1024; rpcInit.timeToGetConn = tsTimeToGetAvailableConn; - (void)taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); + + if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) { + dError("failed to convert version string:%s to int", version); + } pTrans->statusRpc = rpcOpen(&rpcInit); if (pTrans->statusRpc == NULL) { @@ -485,7 +502,9 @@ int32_t dmInitSyncClient(SDnode *pDnode) { rpcInit.supportBatch = 1; rpcInit.batchSize = 8 * 1024; rpcInit.timeToGetConn = tsTimeToGetAvailableConn; - (void)taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); + if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) { + dError("failed to convert version string:%s to int", version); + } pTrans->syncRpc = rpcOpen(&rpcInit); if (pTrans->syncRpc == NULL) { @@ -536,7 +555,11 @@ int32_t dmInitServer(SDnode *pDnode) { rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.parent = pDnode; rpcInit.compressSize = tsCompressMsgSize; - (void)taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); + + if (taosVersionStrToInt(version, &(rpcInit.compatibilityVer)) != 0) { + dError("failed to convert version string:%s to int", version); + } + pTrans->serverRpc = rpcOpen(&rpcInit); if (pTrans->serverRpc == NULL) { dError("failed to init dnode rpc server since:%s", tstrerror(terrno)); diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index f8f2b3dfca..db401375c7 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -150,6 +150,10 @@ int32_t dmReadEps(SDnodeData *pData) { } char *tmp = taosMemoryMalloc(scopeLen + 1); + if (tmp == NULL) { + dError("failed to malloc memory for tsEncryptScope:%s", tsEncryptScope); + goto _OVER; + } memset(tmp, 0, scopeLen + 1); memcpy(tmp, tsEncryptScope, scopeLen); @@ -255,7 +259,9 @@ _OVER: if (taosArrayGetSize(pData->dnodeEps) == 0) { SDnodeEp dnodeEp = {0}; dnodeEp.isMnode = 1; - (void)taosGetFqdnPortFromEp(tsFirst, &dnodeEp.ep); + if (taosGetFqdnPortFromEp(tsFirst, &dnodeEp.ep) != 0) { + dError("failed to get fqdn and port from ep:%s", tsFirst); + } if (taosArrayPush(pData->dnodeEps, &dnodeEp) == NULL) { return terrno; } @@ -366,11 +372,19 @@ int32_t dmGetDnodeSize(SDnodeData *pData) { } void dmUpdateEps(SDnodeData *pData, SArray *eps) { - (void)taosThreadRwlockWrlock(&pData->lock); + if (taosThreadRwlockWrlock(&pData->lock) != 0) { + dError("failed to lock dnode lock"); + } + dDebug("new dnode list get from mnode, dnodeVer:%" PRId64, pData->dnodeVer); dmResetEps(pData, eps); - (void)dmWriteEps(pData); - (void)taosThreadRwlockUnlock(&pData->lock); + if (dmWriteEps(pData) != 0) { + dError("failed to write dnode file"); + } + + if (taosThreadRwlockUnlock(&pData->lock) != 0) { + dError("failed to unlock dnode lock"); + } } static void dmResetEps(SDnodeData *pData, SArray *dnodeEps) { @@ -586,7 +600,9 @@ void dmRemoveDnodePairs(SDnodeData *pData) { snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); snprintf(bak, sizeof(bak), "%s%sdnode%sep.json.bak", tsDataDir, TD_DIRSEP, TD_DIRSEP); dInfo("dnode file:%s is rename to bak file", file); - (void)taosRenameFile(file, bak); + if (taosRenameFile(file, bak) != 0) { + dError("failed to rename dnode file:%s to bak file:%s since %s", file, bak, tstrerror(terrno)); + } } static int32_t dmReadDnodePairs(SDnodeData *pData) { diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 14154d1a23..1da13f72cd 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -342,6 +342,11 @@ static int32_t dmCompareEncryptKey(char *file, char *key, bool toLogFile) { int len = ENCRYPTED_LEN(size); result = taosMemoryMalloc(len); + if (result == NULL) { + code = terrno; + encryptError("failed to alloc memory file:%s since %s", file, tstrerror(code)); + goto _OVER; + } SCryptOpts opts = {0}; strncpy(opts.key, key, ENCRYPT_KEY_LEN); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 99e59662ac..60b732f817 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -106,6 +106,7 @@ typedef enum { // TRN_CONFLICT_TOPIC = 4, // TRN_CONFLICT_TOPIC_INSIDE = 5, TRN_CONFLICT_ARBGROUP = 6, + TRN_CONFLICT_TSMA = 7, } ETrnConflct; typedef enum { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 4365b83f86..a96b8b22f5 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -244,6 +244,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { } storeOffsetRows(pMnode, &req, pConsumer); + rsp.debugFlag = tqClientDebug; code = buildMqHbRsp(pMsg, &rsp); END: @@ -587,8 +588,8 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { SCMSubscribeReq subscribe = {0}; MND_TMQ_RETURN_CHECK(tDeserializeSCMSubscribeReq(msgStr, &subscribe, pMsg->contLen)); - bool ubSubscribe = (taosArrayGetSize(subscribe.topicNames) == 0); - if(ubSubscribe){ + bool unSubscribe = (taosArrayGetSize(subscribe.topicNames) == 0); + if(unSubscribe){ SMqConsumerObj *pConsumerTmp = NULL; MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, subscribe.consumerId, &pConsumerTmp)); if (taosArrayGetSize(pConsumerTmp->assignedTopics) == 0){ @@ -599,7 +600,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { } MND_TMQ_RETURN_CHECK(checkAndSortTopic(pMnode, subscribe.topicNames)); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, - (ubSubscribe ? TRN_CONFLICT_NOTHING :TRN_CONFLICT_DB_INSIDE), + (unSubscribe ? TRN_CONFLICT_NOTHING :TRN_CONFLICT_DB_INSIDE), pMsg, "subscribe"); MND_TMQ_NULL_CHECK(pTrans); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 374dff8d0c..0403029f74 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -2281,6 +2281,10 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, int32_t cols = 0; int32_t bytes = pShow->pMeta->pSchemas[cols].bytes; char *buf = taosMemoryMalloc(bytes); + if (buf == NULL) { + mError("db:%s, failed to malloc buffer", pDb->name); + return; + } int32_t code = 0; int32_t lino = 0; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 4c5a695402..db4d842662 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -184,6 +184,7 @@ static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc) { } static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) { + int32_t code = 0; mTrace("func:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); taosWLockLatch(&pOld->lock); @@ -205,6 +206,11 @@ static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) { if (pNew->commentSize > 0 && pNew->pComment != NULL) { pOld->commentSize = pNew->commentSize; pOld->pComment = taosMemoryMalloc(pOld->commentSize); + if (pOld->pComment == NULL) { + code = terrno; + taosWUnLockLatch(&pOld->lock); + return code; + } (void)memcpy(pOld->pComment, pNew->pComment, pOld->commentSize); } @@ -215,6 +221,11 @@ static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) { if (pNew->codeSize > 0 && pNew->pCode != NULL) { pOld->codeSize = pNew->codeSize; pOld->pCode = taosMemoryMalloc(pOld->codeSize); + if (pOld->pCode == NULL) { + code = terrno; + taosWUnLockLatch(&pOld->lock); + return code; + } (void)memcpy(pOld->pCode, pNew->pCode, pOld->codeSize); } @@ -261,6 +272,10 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre if (NULL != pCreate->pComment) { func.commentSize = strlen(pCreate->pComment) + 1; func.pComment = taosMemoryMalloc(func.commentSize); + if (func.pComment == NULL) { + code = terrno; + goto _OVER; + } } func.codeSize = pCreate->codeLen; func.pCode = taosMemoryMalloc(func.codeSize); @@ -716,6 +731,11 @@ static int32_t mndRetrieveFuncs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl ? TSDB_MAX_BINARY_LEN : pFunc->codeSize + VARSTR_HEADER_SIZE; char *b4 = taosMemoryMalloc(varCodeLen); + if (b4 == NULL) { + code = terrno; + sdbRelease(pSdb, pFunc); + TAOS_RETURN(code); + } (void)memcpy(varDataVal(b4), pFunc->pCode, varCodeLen - VARSTR_HEADER_SIZE); varDataSetLen(b4, varCodeLen - VARSTR_HEADER_SIZE); code = colDataSetVal(pColInfo, numOfRows, (const char *)b4, false); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index c00c88c4f9..6b1c97b399 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -343,6 +343,10 @@ static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *p int32_t code = 0; int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq); void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + code = terrno; + return code; + } code = tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq); if (code < 0) { taosMemoryFree(pReq); @@ -369,6 +373,10 @@ static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, SDAlterMnodeType int32_t code = 0; int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterMnodeTypeReq); void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + code = terrno; + return code; + } code = tSerializeSDCreateMnodeReq(pReq, contLen, pAlterMnodeTypeReq); if (code < 0) { taosMemoryFree(pReq); @@ -395,6 +403,10 @@ static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pA int32_t code = 0; int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq); void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + code = terrno; + return code; + } code = tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq); if (code < 0) { taosMemoryFree(pReq); @@ -420,6 +432,10 @@ static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDrop int32_t code = 0; int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, pDropReq); void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + code = terrno; + return code; + } code = tSerializeSCreateDropMQSNodeReq(pReq, contLen, pDropReq); if (code < 0) { taosMemoryFree(pReq); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 1a76ab2a8b..a258155223 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -1692,7 +1692,7 @@ static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { STransAction dropStbUndoAction = {0}; SMDropStbReq dropStbReq = {0}; STrans *pTrans = - mndTransCreate(pCxt->pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pCxt->pRpcReq, "create-tsma"); + mndTransCreate(pCxt->pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TSMA, pCxt->pRpcReq, "create-tsma"); if (!pTrans) { code = terrno; goto _OVER; @@ -1974,7 +1974,7 @@ _OVER: static int32_t mndDropTSMA(SCreateTSMACxt* pCxt) { int32_t code = -1; STransAction dropStreamRedoAction = {0}; - STrans *pTrans = mndTransCreate(pCxt->pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pCxt->pRpcReq, "drop-tsma"); + STrans *pTrans = mndTransCreate(pCxt->pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TSMA, pCxt->pRpcReq, "drop-tsma"); if (!pTrans) { code = terrno; goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index fa9e4fa8fa..b8cf72cd9e 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1012,10 +1012,10 @@ _OVER: int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { mndTransSetDbName(pTrans, pDb->name, pStb->name); - TAOS_CHECK_RETURN (mndTransCheckConflict(pMnode, pTrans)); - TAOS_CHECK_RETURN (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb)); - TAOS_CHECK_RETURN (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb)); - TAOS_CHECK_RETURN (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb)); + TAOS_CHECK_RETURN(mndTransCheckConflict(pMnode, pTrans)); + TAOS_CHECK_RETURN(mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb)); + TAOS_CHECK_RETURN(mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb)); + TAOS_CHECK_RETURN(mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb)); return 0; } @@ -1051,7 +1051,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) { SRpcMsg rpcMsg = { .msgType = TDMT_VND_FETCH_TTL_EXPIRED_TBS, .pCont = pHead, .contLen = contLen, .info = pReq->info}; - SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); + SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); code = tmsgSendReq(&epSet, &rpcMsg); if (code != 0) { mError("vgId:%d, failed to send drop ttl table request to vnode since 0x%x", pVgroup->vgId, code); @@ -1500,8 +1500,8 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, int64_t suid, col_id_t colId) { int32_t code = 0; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; while (1) { SMqTopicObj *pTopic = NULL; pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); @@ -1562,8 +1562,8 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName, int64_t suid, col_id_t colId) { int32_t code = 0; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; while (1) { SStreamObj *pStream = NULL; pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); @@ -1616,8 +1616,8 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName, int64_t suid, col_id_t colId) { int32_t code = 0; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; while (1) { SSmaObj *pSma = NULL; pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); @@ -2233,7 +2233,7 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion *pStbVer, bool *schema, bool *sma) { int32_t code = 0; - char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; snprintf(tbFName, sizeof(tbFName), "%s.%s", pStbVer->dbFName, pStbVer->stbName); SDbObj *pDb = mndAcquireDb(pMnode, pStbVer->dbFName); @@ -2278,7 +2278,7 @@ static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion *pStbVer, bo static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { int32_t code = 0; - char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); SDbObj *pDb = mndAcquireDb(pMnode, dbFName); @@ -2302,7 +2302,7 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char static int32_t mndBuildStbCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) { int32_t code = 0; - char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); SDbObj *pDb = mndAcquireDb(pMnode, dbFName); @@ -2352,6 +2352,11 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, SStbObj *pObj, void **pCont, i } void *cont = taosMemoryMalloc(contLen); + if (NULL == cont) { + code = terrno; + tFreeSMAlterStbRsp(&alterRsp); + TAOS_RETURN(code); + } tEncoderInit(&ec, cont, contLen); code = tEncodeSMAlterStbRsp(&ec, &alterRsp); tEncoderClear(&ec); @@ -2407,6 +2412,11 @@ int32_t mndBuildSMCreateStbRsp(SMnode *pMnode, char *dbFName, char *stbFName, vo } void *cont = taosMemoryMalloc(contLen); + if (NULL == cont) { + code = terrno; + tFreeSMCreateStbRsp(&stbRsp); + goto _OVER; + } tEncoderInit(&ec, cont, contLen); TAOS_CHECK_GOTO(tEncodeSMCreateStbRsp(&ec, &stbRsp), NULL, _OVER); tEncoderClear(&ec); @@ -2646,7 +2656,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { code = mndAlterStb(pMnode, pReq, &alterReq, pDb, pStb); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; - SName name = {0}; + SName name = {0}; int32_t ret = 0; if ((ret = tNameFromString(&name, alterReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)) != 0) mError("stb:%s, failed to tNameFromString since %s", alterReq.name, tstrerror(ret)); @@ -2769,8 +2779,8 @@ _OVER: static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, int64_t suid) { int32_t code = 0; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; while (1) { SMqTopicObj *pTopic = NULL; pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); @@ -2829,8 +2839,8 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, int64_t suid) { int32_t code = 0; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; while (1) { SStreamObj *pStream = NULL; pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); @@ -2935,7 +2945,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { code = mndDropStb(pMnode, pReq, pDb, pStb); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; - SName name = {0}; + SName name = {0}; int32_t ret = 0; if ((ret = tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)) != 0) mError("stb:%s, failed to tNameFromString since %s", dropReq.name, tstrerror(ret)); @@ -3006,7 +3016,7 @@ _OVER: mndReleaseUser(pMnode, pUser); tFreeSTableMetaRsp(&metaRsp); - //TODO change to TAOS_RETURN + // TODO change to TAOS_RETURN return code; } @@ -3552,7 +3562,7 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc SName name = {0}; - char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; mndExtractTbNameFromStbFullName(pStb->name, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN); varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE])); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -4249,7 +4259,9 @@ static int32_t mndProcessDropTbWithTsma(SRpcMsg *pReq) { code = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, pReq->pTbs, pReq->vgInfo.vgId); if (code) goto _OVER; } - if (mndCreateDropTbsTxnPrepare(pReq, pCtx) == 0) code = 0; + if (mndCreateDropTbsTxnPrepare(pReq, pCtx) == 0) { + code = TSDB_CODE_ACTION_IN_PROGRESS; + } _OVER: tFreeSMDropTbsReq(&dropReq); if (pCtx) mndDestroyDropTbsWithTsmaCtx(pCtx); @@ -4448,7 +4460,7 @@ static int32_t mndProcessFetchTtlExpiredTbs(SRpcMsg *pRsp) { code = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, rsp.pExpiredTbs, rsp.vgId); if (code) goto _end; - if (mndCreateDropTbsTxnPrepare(pRsp, pCtx) == 0) code = 0; + if (mndCreateDropTbsTxnPrepare(pRsp, pCtx) == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _end: if (pCtx) mndDestroyDropTbsWithTsmaCtx(pCtx); tDecoderClear(&decoder); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 3ec99f6e44..69d3de25fc 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -165,68 +165,62 @@ void mndCleanupStream(SMnode *pMnode) { } SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { - int32_t code = 0; - int32_t lino = 0; - terrno = TSDB_CODE_OUT_OF_MEMORY; - - SSdbRow *pRow = NULL; + int32_t code = 0; + int32_t lino = 0; + SSdbRow * pRow = NULL; SStreamObj *pStream = NULL; - void *buf = NULL; + void * buf = NULL; int8_t sver = 0; + int32_t tlen; + int32_t dataPos = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) { - goto STREAM_DECODE_OVER; - } + code = sdbGetRawSoftVer(pRaw, &sver); + TSDB_CHECK_CODE(code, lino, _over); if (sver < 1 || sver > MND_STREAM_VER_NUMBER) { - terrno = 0; mError("stream read invalid ver, data ver: %d, curr ver: %d", sver, MND_STREAM_VER_NUMBER); - goto STREAM_DECODE_OVER; + goto _over; } pRow = sdbAllocRow(sizeof(SStreamObj)); - if (pRow == NULL) { - goto STREAM_DECODE_OVER; - } + TSDB_CHECK_NULL(pRow, code, lino, _over, terrno); pStream = sdbGetRowObj(pRow); - if (pStream == NULL) { - goto STREAM_DECODE_OVER; - } + TSDB_CHECK_NULL(pStream, code, lino, _over, terrno); - int32_t tlen; - int32_t dataPos = 0; - SDB_GET_INT32(pRaw, dataPos, &tlen, STREAM_DECODE_OVER); + SDB_GET_INT32(pRaw, dataPos, &tlen, _over); buf = taosMemoryMalloc(tlen + 1); - if (buf == NULL) { - goto STREAM_DECODE_OVER; - } + TSDB_CHECK_NULL(buf, code, lino, _over, terrno); - SDB_GET_BINARY(pRaw, dataPos, buf, tlen, STREAM_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, buf, tlen, _over); SDecoder decoder; tDecoderInit(&decoder, buf, tlen + 1); - if (tDecodeSStreamObj(&decoder, pStream, sver) < 0) { - tDecoderClear(&decoder); - goto STREAM_DECODE_OVER; - } + code = tDecodeSStreamObj(&decoder, pStream, sver); tDecoderClear(&decoder); - terrno = TSDB_CODE_SUCCESS; - -STREAM_DECODE_OVER: - taosMemoryFreeClear(buf); - if (terrno != TSDB_CODE_SUCCESS) { - char *p = (pStream == NULL) ? "null" : pStream->name; - mError("stream:%s, failed to decode from raw:%p since %s", p, pRaw, tstrerror(terrno)); - taosMemoryFreeClear(pRow); - return NULL; + if (code < 0) { + tFreeStreamObj(pStream); } - mTrace("stream:%s, decode from raw:%p, row:%p, checkpoint:%" PRId64, pStream->name, pRaw, pStream, - pStream->checkpointId); - return pRow; +_over: + taosMemoryFreeClear(buf); + + if (code != TSDB_CODE_SUCCESS) { + char *p = (pStream == NULL) ? "null" : pStream->name; + mError("stream:%s, failed to decode from raw:%p since %s at:%d", p, pRaw, tstrerror(code), lino); + taosMemoryFreeClear(pRow); + + terrno = code; + return NULL; + } else { + mTrace("stream:%s, decode from raw:%p, row:%p, checkpoint:%" PRId64, pStream->name, pRaw, pStream, + pStream->checkpointId); + + terrno = 0; + return pRow; + } } static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream) { diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index e5b4447a39..905a73ad48 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -316,6 +316,7 @@ int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t len) { // kill all trans in the dst DB void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo) { mDebug("start to clear checkpoints in all Dbs"); + char p[128] = {0}; void *pIter = NULL; while ((pIter = taosHashIterate(pChangeInfo->pDBMap, pIter)) != NULL) { @@ -323,15 +324,16 @@ void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo) { size_t len = 0; void *pKey = taosHashGetKey(pDb, &len); - char *p = strndup(pKey, len); + int cpLen = (127 < len) ? 127 : len; + TAOS_STRNCPY(p, pKey, cpLen); + p[cpLen] = '\0'; int32_t code = doKillCheckpointTrans(pMnode, pKey, len); if (code) { - mError("failed to kill trans, transId:%p", pKey) + mError("failed to kill trans, transId:%p", pKey); } else { mDebug("clear checkpoint trans in Db:%s", p); } - taosMemoryFree(p); } mDebug("complete clear checkpoints in all Dbs"); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 347f38193f..40bb99d6b5 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -367,6 +367,7 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, _OVER) if (pTrans->paramLen != 0) { pTrans->param = taosMemoryMalloc(pTrans->paramLen); + if (pTrans->param == NULL) goto _OVER; SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, _OVER); } @@ -902,6 +903,14 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { } } + if (pNew->conflict == TRN_CONFLICT_TSMA) { + if (pTrans->conflict == TRN_CONFLICT_GLOBAL || pTrans->conflict == TRN_CONFLICT_TSMA) { + mndTransLogConflict(pNew, pTrans, true, &conflict); + } else { + mndTransLogConflict(pNew, pTrans, false, &conflict); + } + } + sdbRelease(pMnode->pSdb, pTrans); } @@ -1299,7 +1308,7 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi } else { pAction->errCode = (terrno != 0) ? terrno : code; mError("trans:%d, %s:%d failed to write sdb since %s, type:%s status:%s", pTrans->id, mndTransStr(pAction->stage), - pAction->id, terrstr(), sdbTableName(pAction->pRaw->type), sdbStatusName(pAction->pRaw->status)); + pAction->id, tstrerror(code), sdbTableName(pAction->pRaw->type), sdbStatusName(pAction->pRaw->status)); mndSetTransLastAction(pTrans, pAction); } @@ -1519,7 +1528,13 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr } mndSetTransLastAction(pTrans, pAction); - if (mndCannotExecuteTransAction(pMnode, topHalf)) break; + if (mndCannotExecuteTransAction(pMnode, topHalf)) { + pTrans->lastErrorNo = code; + pTrans->code = code; + mInfo("trans:%d, %s:%d, topHalf:%d, not execute next action, code:%s", pTrans->id, mndTransStr(pAction->stage), + action, topHalf, tstrerror(code)); + break; + } if (code == 0) { pTrans->code = 0; @@ -1617,7 +1632,21 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool code = mndTransExecuteRedoActions(pMnode, pTrans, topHalf); } - if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; + if (mndCannotExecuteTransAction(pMnode, topHalf)) { + pTrans->lastErrorNo = code; + pTrans->code = code; + bool continueExec = true; + if (code != 0 && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { + taosMsleep(100); + continueExec = true; + } else { + continueExec = false; + } + mInfo("trans:%d, cannot execute redo action stage, topHalf:%d, continueExec:%d, code:%s", pTrans->id, topHalf, + continueExec, tstrerror(code)); + + return continueExec; + } terrno = code; if (code == 0) { @@ -1834,13 +1863,13 @@ void mndTransExecuteImp(SMnode *pMnode, STrans *pTrans, bool topHalf) { // start trans, pullup, receive rsp, kill void mndTransExecute(SMnode *pMnode, STrans *pTrans) { bool topHalf = true; - return mndTransExecuteImp(pMnode, pTrans, topHalf); + mndTransExecuteImp(pMnode, pTrans, topHalf); } // update trans void mndTransRefresh(SMnode *pMnode, STrans *pTrans) { bool topHalf = false; - return mndTransExecuteImp(pMnode, pTrans, topHalf); + mndTransExecuteImp(pMnode, pTrans, topHalf); } static int32_t mndProcessTransTimer(SRpcMsg *pReq) { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 2fe1ef4cfb..5a79ac6bc8 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#include "mndVgroup.h" #include "audit.h" #include "mndArbGroup.h" #include "mndDb.h" @@ -26,7 +27,6 @@ #include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" -#include "mndVgroup.h" #include "tmisce.h" #define VGROUP_VER_NUMBER 1 @@ -1670,7 +1670,9 @@ int32_t mndAddNewVgPrepareAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVg) { } TAOS_CHECK_GOTO(mndTransAppendPrepareLog(pTrans, pRaw), NULL, _err); - (void)sdbSetRawStatus(pRaw, SDB_STATUS_CREATING); + if (sdbSetRawStatus(pRaw, SDB_STATUS_CREATING) != 0) { + mError("vgId:%d, failed to set raw status at line:%d", pVg->vgId, __LINE__); + } if (code != 0) { mError("vgId:%d, failed to set raw status since %s at line:%d", pVg->vgId, tstrerror(code), __LINE__); TAOS_RETURN(code); diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index b83554c6f9..ea44a7c549 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -162,13 +162,13 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * pRow->status = pRaw->status; sdbPrintOper(pSdb, pRow, "insert"); - if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { + int32_t code = 0; + if ((code = taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *))) != 0) { sdbUnLock(pSdb, type); sdbFreeRow(pSdb, pRow, false); - return terrno; + return code; } - int32_t code = 0; SdbInsertFp insertFp = pSdb->insertFps[pRow->type]; if (insertFp != NULL) { code = (*insertFp)(pSdb, pRow->pObj); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 7fc079677b..d61f3d80d3 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -38,7 +38,7 @@ int32_t sndBuildStreamTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProce streamTaskOpenAllUpstreamInput(pTask); streamTaskResetUpstreamStageInfo(pTask); - (void)streamSetupScheduleTrigger(pTask); + streamSetupScheduleTrigger(pTask); SCheckpointInfo *pChkInfo = &pTask->chkInfo; tqSetRestoreVersionInfo(pTask); @@ -93,14 +93,18 @@ FAIL: } int32_t sndInit(SSnode *pSnode) { - (void)streamTaskSchedTask(&pSnode->msgCb, pSnode->pMeta->vgId, 0, 0, STREAM_EXEC_T_START_ALL_TASKS); + if (streamTaskSchedTask(&pSnode->msgCb, pSnode->pMeta->vgId, 0, 0, STREAM_EXEC_T_START_ALL_TASKS) != 0) { + sndError("failed to start all tasks"); + } return 0; } void sndClose(SSnode *pSnode) { stopRsync(); streamMetaNotifyClose(pSnode->pMeta); - (void)streamMetaCommit(pSnode->pMeta); + if (streamMetaCommit(pSnode->pMeta) != 0) { + sndError("failed to commit stream meta"); + } streamMetaClose(pSnode->pMeta); taosMemoryFree(pSnode); } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 141aff0337..653b47ff14 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -112,14 +112,14 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle); void tqDestroyTqHandle(void* data); // tqRead -int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* offset); +int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* offset); int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest); int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId); // tqExec -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded); -int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, void* pRsp, int32_t numOfCols, int8_t precision); -int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const void* pRsp, +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded); +int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type, int32_t vgId); void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId); @@ -148,9 +148,9 @@ int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name); // tq util int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); -int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const void* pRsp, int32_t epoch, int64_t consumerId, +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever); -int32_t tqInitDataRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset); +int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset); void tqUpdateNodeStage(STQ* pTq, bool isLeader); int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, SSubmitTbData* pTableData, int64_t earlyTs, const char* id); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 38d6f020c7..30efee42e5 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -292,7 +292,7 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== -int32_t tsdbTakeReadSnap2(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap, const char* id); +int32_t tsdbTakeReadSnap2(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap, const char *id); void tsdbUntakeReadSnap2(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive); int32_t tsdbGetTableSchema(SMeta *pMeta, int64_t uid, STSchema **pSchema, int64_t *suid); @@ -375,7 +375,7 @@ struct STsdb { struct { SVHashTable *ht; SArray *arr; - } *commitInfo; + } * commitInfo; }; struct TSDBKEY { @@ -949,7 +949,7 @@ int32_t tsdbBICacheRelease(SLRUCache *pCache, LRUHandle *h); int32_t tsdbCacheGetBlockS3(SLRUCache *pCache, STsdbFD *pFD, LRUHandle **handle); int32_t tsdbCacheGetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, LRUHandle **handle); -int32_t tsdbCacheSetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, uint8_t *pPage); +void tsdbCacheSetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, uint8_t *pPage); int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); @@ -1069,6 +1069,13 @@ int32_t tsdbSnapPrepDescription(SVnode *pVnode, SSnapshot *pSnap); void tsdbRemoveFile(const char *path); +#define taosCloseFileWithLog(fd) \ + do { \ + if (taosCloseFile(fd) < 0) { \ + tsdbTrace("failed to close file"); \ + } \ + } while (0) + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 4cc4e11329..1bd4317234 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -143,7 +143,7 @@ typedef struct STbUidStore STbUidStore; int metaOpen(SVnode* pVnode, SMeta** ppMeta, int8_t rollback); int metaUpgrade(SVnode* pVnode, SMeta** ppMeta); -int metaClose(SMeta** pMeta); +void metaClose(SMeta** pMeta); int metaBegin(SMeta* pMeta, int8_t fromSys); TXN* metaGetTxn(SMeta* pMeta); int metaCommit(SMeta* pMeta, TXN* txn); @@ -207,7 +207,7 @@ int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pR // tsdb int32_t tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback, bool force); -int32_t tsdbClose(STsdb** pTsdb); +void tsdbClose(STsdb** pTsdb); int32_t tsdbBegin(STsdb* pTsdb); // int32_t tsdbPrepareCommit(STsdb* pTsdb); // int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); @@ -284,7 +284,7 @@ int32_t tqProcessTaskConsenChkptIdReq(STQ* pTq, SRpcMsg* pMsg); int32_t smaInit(); void smaCleanUp(); int32_t smaOpen(SVnode* pVnode, int8_t rollback, bool force); -int32_t smaClose(SSma* pSma); +void smaClose(SSma* pSma); int32_t smaBegin(SSma* pSma); int32_t smaPrepareAsyncCommit(SSma* pSma); int32_t smaCommit(SSma* pSma, SCommitInfo* pInfo); @@ -314,7 +314,7 @@ int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback); // STsdbSnapReader ======================================== int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, void* pRanges, STsdbSnapReader** ppReader); -void tsdbSnapReaderClose(STsdbSnapReader** ppReader); +void tsdbSnapReaderClose(STsdbSnapReader** ppReader); int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData); // STsdbSnapWriter ======================================== int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRanges, STsdbSnapWriter** ppWriter); @@ -323,7 +323,7 @@ int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* pWriter, bool rollback); int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback); // STsdbSnapRAWReader ======================================== int32_t tsdbSnapRAWReaderOpen(STsdb* pTsdb, int64_t ever, int8_t type, STsdbSnapRAWReader** ppReader); -void tsdbSnapRAWReaderClose(STsdbSnapRAWReader** ppReader); +void tsdbSnapRAWReaderClose(STsdbSnapRAWReader** ppReader); int32_t tsdbSnapRAWRead(STsdbSnapRAWReader* pReader, uint8_t** ppData); // STsdbSnapRAWWriter ======================================== int32_t tsdbSnapRAWWriterOpen(STsdb* pTsdb, int64_t ever, STsdbSnapRAWWriter** ppWriter); diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 06576c0671..93347c810f 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -40,7 +40,7 @@ typedef struct SMetaStbStatsEntry { } SMetaStbStatsEntry; typedef struct STagFilterResEntry { - SList list; // the linked list of md5 digest, extracted from the serialized tag query condition + SHashObj *set; // the set of md5 digest, extracted from the serialized tag query condition uint32_t hitTimes; // queried times for current super table } STagFilterResEntry; @@ -112,7 +112,7 @@ static void statsCacheClose(SMeta* pMeta) { static void freeCacheEntryFp(void* param) { STagFilterResEntry** p = param; - tdListEmpty(&(*p)->list); + taosHashCleanup((*p)->set); taosMemoryFreeClear(*p); } @@ -200,10 +200,12 @@ void metaCacheClose(SMeta* pMeta) { entryCacheClose(pMeta); statsCacheClose(pMeta); + taosHashClear(pMeta->pCache->sTagFilterResCache.pTableEntry); taosLRUCacheCleanup(pMeta->pCache->sTagFilterResCache.pUidResCache); (void)taosThreadMutexDestroy(&pMeta->pCache->sTagFilterResCache.lock); taosHashCleanup(pMeta->pCache->sTagFilterResCache.pTableEntry); + taosHashClear(pMeta->pCache->STbGroupResCache.pTableEntry); taosLRUCacheCleanup(pMeta->pCache->STbGroupResCache.pResCache); (void)taosThreadMutexDestroy(&pMeta->pCache->STbGroupResCache.lock); taosHashCleanup(pMeta->pCache->STbGroupResCache.pTableEntry); @@ -216,7 +218,7 @@ void metaCacheClose(SMeta* pMeta) { } } -static int32_t metaRehashCache(SMetaCache* pCache, int8_t expand) { +static void metaRehashCache(SMetaCache* pCache, int8_t expand) { int32_t code = 0; int32_t nBucket; @@ -228,8 +230,7 @@ static int32_t metaRehashCache(SMetaCache* pCache, int8_t expand) { SMetaCacheEntry** aBucket = (SMetaCacheEntry**)taosMemoryCalloc(nBucket, sizeof(SMetaCacheEntry*)); if (aBucket == NULL) { - code = terrno; - goto _exit; + return; } // rehash @@ -250,9 +251,7 @@ static int32_t metaRehashCache(SMetaCache* pCache, int8_t expand) { taosMemoryFree(pCache->sEntryCache.aBucket); pCache->sEntryCache.nBucket = nBucket; pCache->sEntryCache.aBucket = aBucket; - -_exit: - return code; + return; } int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) { @@ -279,7 +278,7 @@ int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) { } } else { // insert if (pCache->sEntryCache.nEntry >= pCache->sEntryCache.nBucket) { - TAOS_UNUSED(metaRehashCache(pCache, 1)); + metaRehashCache(pCache, 1); iBucket = TABS(pInfo->uid) % pCache->sEntryCache.nBucket; } @@ -317,7 +316,7 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid) { pCache->sEntryCache.nEntry--; if (pCache->sEntryCache.nEntry < pCache->sEntryCache.nBucket / 4 && pCache->sEntryCache.nBucket > META_CACHE_BASE_BUCKET) { - TAOS_UNUSED(metaRehashCache(pCache, 0)); + metaRehashCache(pCache, 0); } } else { code = TSDB_CODE_NOT_FOUND; @@ -474,34 +473,6 @@ int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo) { return code; } -static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInvalidRes, int32_t keyLen, - SLRUCache* pCache, uint64_t suid) { - SListIter iter = {0}; - tdListInitIter((SList*)&(pEntry->list), &iter, TD_LIST_FORWARD); - - SListNode* pNode = NULL; - uint64_t buf[3]; - buf[0] = suid; - - int32_t len = sizeof(uint64_t) * tListLen(buf); - - while ((pNode = tdListNext(&iter)) != NULL) { - memcpy(&buf[1], pNode->data, keyLen); - - // check whether it is existed in LRU cache, and remove it from linked list if not. - LRUHandle* pRes = taosLRUCacheLookup(pCache, buf, len); - if (pRes == NULL) { // remove the item in the linked list - if (taosArrayPush(pInvalidRes, &pNode) == NULL) { - return terrno; - } - } else { - (void)taosLRUCacheRelease(pCache, pRes, false); - } - } - - return 0; -} - static FORCE_INLINE void setMD5DigestInKey(uint64_t* pBuf, const char* key, int32_t keyLen) { memcpy(&pBuf[2], key, keyLen); } @@ -562,7 +533,7 @@ int32_t metaGetCachedTableUidList(void* pVnode, tb_uid_t suid, const uint8_t* pK ((double)(*pEntry)->hitTimes) / acc); } - (void)taosLRUCacheRelease(pCache, pHandle, false); + bool ret = taosLRUCacheRelease(pCache, pHandle, false); // unlock meta (void)taosThreadMutexUnlock(pLock); @@ -587,22 +558,11 @@ static void freeUidCachePayload(const void* key, size_t keyLen, void* value, voi if (pEntry != NULL && (*pEntry) != NULL) { int64_t st = taosGetTimestampUs(); - - SListIter iter = {0}; - tdListInitIter((SList*)&((*pEntry)->list), &iter, TD_LIST_FORWARD); - - SListNode* pNode = NULL; - while ((pNode = tdListNext(&iter)) != NULL) { - uint64_t* digest = (uint64_t*)pNode->data; - if (digest[0] == p[2] && digest[1] == p[3]) { - void* tmp = tdListPopNode(&((*pEntry)->list), pNode); - taosMemoryFree(tmp); - - double el = (taosGetTimestampUs() - st) / 1000.0; - metaInfo("clear items in meta-cache, remain cached item:%d, elapsed time:%.2fms", listNEles(&((*pEntry)->list)), - el); - break; - } + int32_t code = taosHashRemove((*pEntry)->set, &p[2], sizeof(uint64_t) * 2); + if (code == TSDB_CODE_SUCCESS) { + double el = (taosGetTimestampUs() - st) / 1000.0; + metaInfo("clear items in meta-cache, remain cached item:%d, elapsed time:%.2fms", taosHashGetSize((*pEntry)->set), + el); } } @@ -610,16 +570,30 @@ static void freeUidCachePayload(const void* key, size_t keyLen, void* value, voi } static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyLen, uint64_t suid) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; STagFilterResEntry* p = taosMemoryMalloc(sizeof(STagFilterResEntry)); - if (p == NULL) { - return terrno; - } + TSDB_CHECK_NULL(p, code, lino, _end, terrno); p->hitTimes = 0; - tdListInit(&p->list, keyLen); - TAOS_CHECK_RETURN(taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES)); - (void)tdListAppend(&p->list, pKey); - return 0; + p->set = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + TSDB_CHECK_NULL(p->set, code, lino, _end, terrno); + code = taosHashPut(p->set, pKey, keyLen, NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); + code = taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + metaError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + if (p != NULL) { + if (p->set != NULL) { + taosHashCleanup(p->set); + } + taosMemoryFree(p); + } + } + return code; } // check both the payload size and selectivity ratio @@ -660,24 +634,19 @@ int32_t metaUidFilterCachePut(void* pVnode, uint64_t suid, const void* pKey, int goto _end; } } else { // check if it exists or not - size_t size = listNEles(&(*pEntry)->list); - if (size == 0) { - (void)tdListAppend(&(*pEntry)->list, pKey); - } else { - SListNode* pNode = listHead(&(*pEntry)->list); - uint64_t* p = (uint64_t*)pNode->data; - if (p[1] == ((uint64_t*)pKey)[1] && p[0] == ((uint64_t*)pKey)[0]) { - // we have already found the existed items, no need to added to cache anymore. - (void)taosThreadMutexUnlock(pLock); - return TSDB_CODE_SUCCESS; - } else { // not equal, append it - (void)tdListAppend(&(*pEntry)->list, pKey); - } + code = taosHashPut((*pEntry)->set, pKey, keyLen, NULL, 0); + if (code == TSDB_CODE_DUP_KEY) { + // we have already found the existed items, no need to added to cache anymore. + (void)taosThreadMutexUnlock(pLock); + return TSDB_CODE_SUCCESS; + } + if (code != TSDB_CODE_SUCCESS) { + goto _end; } } // add to cache. - (void)taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeUidCachePayload, NULL, + (void)taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeUidCachePayload, NULL, NULL, TAOS_LRU_PRIORITY_LOW, NULL); _end: (void)taosThreadMutexUnlock(pLock); @@ -700,23 +669,20 @@ int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) { (void)taosThreadMutexLock(pLock); STagFilterResEntry** pEntry = taosHashGet(pEntryHashMap, &suid, sizeof(uint64_t)); - if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) { + if (pEntry == NULL || taosHashGetSize((*pEntry)->set) == 0) { (void)taosThreadMutexUnlock(pLock); return TSDB_CODE_SUCCESS; } (*pEntry)->hitTimes = 0; - SListIter iter = {0}; - tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD); - - SListNode* pNode = NULL; - while ((pNode = tdListNext(&iter)) != NULL) { - setMD5DigestInKey(p, pNode->data, 2 * sizeof(uint64_t)); + char *iter = taosHashIterate((*pEntry)->set, NULL); + while (iter != NULL) { + setMD5DigestInKey(p, iter, 2 * sizeof(uint64_t)); taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, TAG_FILTER_RES_KEY_LEN); + iter = taosHashIterate((*pEntry)->set, iter); } - - tdListEmpty(&(*pEntry)->list); + taosHashClear((*pEntry)->set); (void)taosThreadMutexUnlock(pLock); metaDebug("vgId:%d suid:%" PRId64 " cached related tag filter uid list cleared", vgId, suid); @@ -761,7 +727,7 @@ int32_t metaGetCachedTbGroup(void* pVnode, tb_uid_t suid, const uint8_t* pKey, i ((double)(*pEntry)->hitTimes) / acc); } - (void)taosLRUCacheRelease(pCache, pHandle, false); + bool ret = taosLRUCacheRelease(pCache, pHandle, false); // unlock meta (void)taosThreadMutexUnlock(pLock); @@ -786,22 +752,11 @@ static void freeTbGroupCachePayload(const void* key, size_t keyLen, void* value, if (pEntry != NULL && (*pEntry) != NULL) { int64_t st = taosGetTimestampUs(); - - SListIter iter = {0}; - tdListInitIter((SList*)&((*pEntry)->list), &iter, TD_LIST_FORWARD); - - SListNode* pNode = NULL; - while ((pNode = tdListNext(&iter)) != NULL) { - uint64_t* digest = (uint64_t*)pNode->data; - if (digest[0] == p[2] && digest[1] == p[3]) { - void* tmp = tdListPopNode(&((*pEntry)->list), pNode); - taosMemoryFree(tmp); - - double el = (taosGetTimestampUs() - st) / 1000.0; - metaDebug("clear one item in tb group cache, remain cached item:%d, elapsed time:%.2fms", - listNEles(&((*pEntry)->list)), el); - break; - } + int32_t code = taosHashRemove((*pEntry)->set, &p[2], sizeof(uint64_t) * 2); + if (code == TSDB_CODE_SUCCESS) { + double el = (taosGetTimestampUs() - st) / 1000.0; + metaDebug("clear one item in tb group cache, remain cached item:%d, elapsed time:%.2fms", + taosHashGetSize((*pEntry)->set), el); } } @@ -837,24 +792,19 @@ int32_t metaPutTbGroupToCache(void* pVnode, uint64_t suid, const void* pKey, int goto _end; } } else { // check if it exists or not - size_t size = listNEles(&(*pEntry)->list); - if (size == 0) { - (void)tdListAppend(&(*pEntry)->list, pKey); - } else { - SListNode* pNode = listHead(&(*pEntry)->list); - uint64_t* p = (uint64_t*)pNode->data; - if (p[1] == ((uint64_t*)pKey)[1] && p[0] == ((uint64_t*)pKey)[0]) { - // we have already found the existed items, no need to added to cache anymore. - (void)taosThreadMutexUnlock(pLock); - return TSDB_CODE_SUCCESS; - } else { // not equal, append it - (void)tdListAppend(&(*pEntry)->list, pKey); - } + code = taosHashPut((*pEntry)->set, pKey, keyLen, NULL, 0); + if (code == TSDB_CODE_DUP_KEY) { + // we have already found the existed items, no need to added to cache anymore. + (void)taosThreadMutexUnlock(pLock); + return TSDB_CODE_SUCCESS; + } + if (code != TSDB_CODE_SUCCESS) { + goto _end; } } // add to cache. - (void)taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeTbGroupCachePayload, NULL, + (void)taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeTbGroupCachePayload, NULL, NULL, TAOS_LRU_PRIORITY_LOW, NULL); _end: (void)taosThreadMutexUnlock(pLock); @@ -877,23 +827,20 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) { (void)taosThreadMutexLock(pLock); STagFilterResEntry** pEntry = taosHashGet(pEntryHashMap, &suid, sizeof(uint64_t)); - if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) { + if (pEntry == NULL || taosHashGetSize((*pEntry)->set) == 0) { (void)taosThreadMutexUnlock(pLock); return TSDB_CODE_SUCCESS; } (*pEntry)->hitTimes = 0; - SListIter iter = {0}; - tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD); - - SListNode* pNode = NULL; - while ((pNode = tdListNext(&iter)) != NULL) { - setMD5DigestInKey(p, pNode->data, 2 * sizeof(uint64_t)); + char *iter = taosHashIterate((*pEntry)->set, NULL); + while (iter != NULL) { + setMD5DigestInKey(p, iter, 2 * sizeof(uint64_t)); taosLRUCacheErase(pMeta->pCache->STbGroupResCache.pResCache, p, TAG_FILTER_RES_KEY_LEN); + iter = taosHashIterate((*pEntry)->set, iter); } - - tdListEmpty(&(*pEntry)->list); + taosHashClear((*pEntry)->set); (void)taosThreadMutexUnlock(pLock); metaDebug("vgId:%d suid:%" PRId64 " cached related tb group cleared", vgId, suid); diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index f2a9f61595..d8afd6aeaf 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -66,7 +66,10 @@ int metaPrepareAsyncCommit(SMeta *pMeta) { int32_t lino; metaWLock(pMeta); - TAOS_UNUSED(ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn)); + int32_t ret = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn); + if (ret < 0) { + metaError("vgId:%d, failed to flush ttl since %s", TD_VID(pMeta->pVnode), tstrerror(ret)); + } metaULock(pMeta); code = tdbCommit(pMeta->pEnv, pMeta->txn); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 591c40332a..f062505ac7 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -39,6 +39,100 @@ static void metaDestroyLock(SMeta *pMeta) { (void)taosThreadRwlockDestroy(&pMeta static void metaCleanup(SMeta **ppMeta); +static void doScan(SMeta *pMeta) { + TBC *cursor = NULL; + int32_t code; + + // open file to write + char path[TSDB_FILENAME_LEN] = {0}; + snprintf(path, TSDB_FILENAME_LEN - 1, "%s%s", pMeta->path, TD_DIRSEP "scan.txt"); + TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (fp == NULL) { + metaError("failed to open file:%s, reason:%s", path, tstrerror(terrno)); + return; + } + + code = tdbTbcOpen(pMeta->pTbDb, &cursor, NULL); + if (code) { + if (taosCloseFile(&fp) != 0) { + metaError("failed to close file:%s, reason:%s", path, tstrerror(terrno)); + } + metaError("failed to open table.db cursor, reason:%s", tstrerror(terrno)); + return; + } + + code = tdbTbcMoveToFirst(cursor); + if (code) { + if (taosCloseFile(&fp) != 0) { + metaError("failed to close file:%s, reason:%s", path, tstrerror(terrno)); + } + tdbTbcClose(cursor); + metaError("failed to move to first, reason:%s", tstrerror(terrno)); + return; + } + + for (;;) { + const void *pKey; + int kLen; + const void *pVal; + int vLen; + if (tdbTbcGet(cursor, &pKey, &kLen, &pVal, &vLen) < 0) { + break; + } + + // decode entry + SDecoder dc = {0}; + SMetaEntry me = {0}; + + tDecoderInit(&dc, (uint8_t *)pVal, vLen); + + if (metaDecodeEntry(&dc, &me) < 0) { + tDecoderClear(&dc); + break; + } + + // skip deleted entry + if (tdbTbGet(pMeta->pUidIdx, &me.uid, sizeof(me.uid), NULL, NULL) == 0) { + // print entry + char buf[1024] = {0}; + if (me.type == TSDB_SUPER_TABLE) { + snprintf(buf, sizeof(buf) - 1, "type: super table, version:%" PRId64 " uid: %" PRId64 " name: %s\n", me.version, + me.uid, me.name); + + } else if (me.type == TSDB_CHILD_TABLE) { + snprintf(buf, sizeof(buf) - 1, + "type: child table, version:%" PRId64 " uid: %" PRId64 " name: %s suid:%" PRId64 "\n", me.version, + me.uid, me.name, me.ctbEntry.suid); + } else { + snprintf(buf, sizeof(buf) - 1, "type: normal table, version:%" PRId64 " uid: %" PRId64 " name: %s\n", + me.version, me.uid, me.name); + } + + if (taosWriteFile(fp, buf, strlen(buf)) < 0) { + metaError("failed to write file:%s, reason:%s", path, tstrerror(terrno)); + tDecoderClear(&dc); + break; + } + } + + tDecoderClear(&dc); + + if (tdbTbcMoveToNext(cursor) < 0) { + break; + } + } + + tdbTbcClose(cursor); + + // close file + if (taosFsyncFile(fp) < 0) { + metaError("failed to fsync file:%s, reason:%s", path, tstrerror(terrno)); + } + if (taosCloseFile(&fp) < 0) { + metaError("failed to close file:%s, reason:%s", path, tstrerror(terrno)); + } +} + int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { SMeta *pMeta = NULL; int32_t code = 0; @@ -60,12 +154,13 @@ int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { pMeta->path = (char *)&pMeta[1]; strcpy(pMeta->path, path); - (void)taosRealPath(pMeta->path, NULL, strlen(path) + 1); + int32_t ret = taosRealPath(pMeta->path, NULL, strlen(path) + 1); pMeta->pVnode = pVnode; // create path if not created yet - (void)taosMkDir(pMeta->path); + code = taosMkDir(pMeta->path); + TSDB_CHECK_CODE(code, lino, _exit); // open env code = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback, @@ -97,7 +192,7 @@ int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { TSDB_CHECK_CODE(code, lino, _exit); sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); - TAOS_UNUSED(taosMkDir(indexFullPath)); + ret = taosMkDir(indexFullPath); SIndexOpts opts = {.cacheSize = 8 * 1024 * 1024}; code = indexOpen(&opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx); @@ -133,6 +228,11 @@ int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { code = metaInitTbFilterCache(pMeta); TSDB_CHECK_CODE(code, lino, _exit); +#if 0 + // Do NOT remove this code, it is used to do debug stuff + doScan(pMeta); +#endif + _exit: if (code) { metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code)); @@ -169,9 +269,9 @@ _exit: return code; } -int metaClose(SMeta **ppMeta) { +void metaClose(SMeta **ppMeta) { metaCleanup(ppMeta); - return 0; + return; } int metaAlterCache(SMeta *pMeta, int32_t nPage) { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 3f6d17a5a7..484c5c0a16 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -62,7 +62,10 @@ int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t u tDecoderInit(&pReader->coder, pReader->pBuf, pReader->szBuf); code = metaDecodeEntry(&pReader->coder, &pReader->me); - if (code) return code; + if (code) { + tDecoderClear(&pReader->coder); + return code; + } // taosMemoryFreeClear(pReader->me.colCmpr.pColCmpr); return 0; @@ -393,6 +396,7 @@ _query: tDecoderInit(&dc, pData, nData); int32_t code = metaDecodeEntry(&dc, &me); if (code) { + tDecoderClear(&dc); goto _err; } if (me.type == TSDB_SUPER_TABLE) { @@ -1277,7 +1281,11 @@ int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) { tDecoderInit(&dc, pData, nData); - TAOS_CHECK_GOTO(metaDecodeEntry(&dc, &oStbEntry), NULL, END); + code = metaDecodeEntry(&dc, &oStbEntry); + if (code) { + tDecoderClear(&dc); + goto END; + } if (oStbEntry.stbEntry.schemaTag.pSchema == NULL || oStbEntry.stbEntry.schemaTag.pSchema == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_INVALID_PARA, NULL, END); @@ -1515,6 +1523,7 @@ int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *pUidTagInfo) { } memcpy(info.pTagVal, pCur->pVal, pCur->vLen); if (taosArrayPush(pUidTagInfo, &info) == NULL) { + taosMemoryFreeClear(info.pTagVal); metaCloseCtbCursor(pCur); taosHashCleanup(pSepecifiedUidMap); return terrno; diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 12ef5088b8..0936d8f092 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -87,7 +87,9 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { if (key.version < pReader->sver // || metaGetInfo(pReader->pMeta, key.uid, &info, NULL) == TSDB_CODE_NOT_FOUND) { - (void)tdbTbcMoveToNext(pReader->pTbc); + if (tdbTbcMoveToNext(pReader->pTbc) != 0) { + metaTrace("vgId:%d, vnode snapshot meta read data done", TD_VID(pReader->pMeta->pVnode)); + } continue; } @@ -110,7 +112,9 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { metaDebug("vgId:%d, vnode snapshot meta read data, version:%" PRId64 " uid:%" PRId64 " blockLen:%d", TD_VID(pReader->pMeta->pVnode), key.version, key.uid, nData); - (void)tdbTbcMoveToNext(pReader->pTbc); + if (tdbTbcMoveToNext(pReader->pTbc) != 0) { + metaTrace("vgId:%d, vnode snapshot meta read data done", TD_VID(pReader->pMeta->pVnode)); + } break; } @@ -233,7 +237,9 @@ static int32_t MoveToSnapShotVersion(SSnapContext* ctx) { return TAOS_GET_TERRNO(code); } if (c < 0) { - (void)tdbTbcMoveToPrev((TBC*)ctx->pCur); + if (tdbTbcMoveToPrev((TBC*)ctx->pCur) != 0) { + metaTrace("vgId:%d, vnode snapshot move to prev failed", TD_VID(ctx->pMeta->pVnode)); + } } return 0; } @@ -599,6 +605,7 @@ int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLe tDecoderInit(&dc, pVal, vLen); ret = metaDecodeEntry(&dc, &me); if (ret < 0) { + tDecoderClear(&dc); ret = TAOS_GET_TERRNO(ret); goto END; } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 0fb8ca3fb1..8814e87140 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -22,7 +22,7 @@ static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, con static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME); +static void metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateChangeTime(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs); static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); @@ -285,7 +285,9 @@ static inline void metaTimeSeriesNotifyCheck(SMeta *pMeta) { int64_t deltaTS = nTimeSeries - pMeta->pVnode->config.vndStats.numOfReportedTimeSeries; if (deltaTS > tsTimeSeriesThreshold) { if (0 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 2)) { - (void)tsem_post(&dmNotifyHdl.sem); + if (tsem_post(&dmNotifyHdl.sem) != 0) { + metaError("vgId:%d, failed to post semaphore, errno:%d", TD_VID(pMeta->pVnode), errno); + } } } #endif @@ -563,6 +565,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { if (ret < 0) { metaError("vgId:%d, failed to decode stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, tstrerror(ret)); + tDecoderClear(&dc); tdbTbcClose(pTbDbc); tdbTbcClose(pUidIdxc); return terrno; @@ -1441,8 +1444,8 @@ static int metaBuildNColIdxKey(SNcolIdxKey *ncolKey, const SMetaEntry *pME) { return 0; } -static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { - if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return 0; +static void metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { + if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return; STtlDelTtlCtx ctx = {.uid = pME->uid, .pTxn = pMeta->txn}; if (pME->type == TSDB_CHILD_TABLE) { @@ -1451,7 +1454,12 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { ctx.ttlDays = pME->ntbEntry.ttlDays; } - return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); + int32_t ret = ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); + if (ret < 0) { + metaError("vgId:%d, failed to delete ttl for table:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pME->name, + pME->uid, tstrerror(ret)); + } + return; } static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl) { @@ -1497,6 +1505,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p tDecoderInit(&tdc, tData, tLen); int32_t ret = metaDecodeEntry(&tdc, &stbEntry); if (ret < 0) { + tDecoderClear(&tdc); metaError("vgId:%d, failed to decode child table:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.ctbEntry.suid, tstrerror(ret)); return ret; @@ -1831,12 +1840,19 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { int16_t cid = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId; int8_t col_type = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type; - (void)tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type); + int32_t ret = tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type); + if (ret < 0) { + terrno = ret; + goto _err; + } } SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1]; uint32_t compress = pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN ? createDefaultColCmprByType(pCol->type) : pAlterTbReq->compress; - (void)updataTableColCmpr(&entry.colCmpr, pCol, 1, compress); + if (updataTableColCmpr(&entry.colCmpr, pCol, 1, compress) != 0) { + metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, + entry.uid); + } freeColCmpr = true; if (entry.colCmpr.nCols != pSchema->nCols) { if (pNewSchema) taosMemoryFree(pNewSchema); @@ -1876,10 +1892,16 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { int16_t cid = pColumn->colId; - (void)tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey); + if (tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey) != 0) { + metaError("vgId:%d, failed to drop ntable column:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, + entry.uid); + } } - (void)updataTableColCmpr(&entry.colCmpr, &tScheam, 0, 0); + if (updataTableColCmpr(&entry.colCmpr, &tScheam, 0, 0) != 0) { + metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, + entry.uid); + } if (entry.colCmpr.nCols != pSchema->nCols) { terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; goto _err; @@ -1928,20 +1950,36 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl // do actual write metaWLock(pMeta); - (void)metaDeleteNcolIdx(pMeta, &oldEntry); - (void)metaUpdateNcolIdx(pMeta, &entry); + if (metaDeleteNcolIdx(pMeta, &oldEntry) < 0) { + metaError("vgId:%d, failed to delete ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + if (metaUpdateNcolIdx(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to update ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + // save to table db - (void)metaSaveToTbDb(pMeta, &entry); + if (metaSaveToTbDb(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to save to tb db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } - (void)metaUpdateUidIdx(pMeta, &entry); + if (metaUpdateUidIdx(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } - (void)metaSaveToSkmDb(pMeta, &entry); + if (metaSaveToSkmDb(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to save to skm db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } - (void)metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs); + if (metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs) < 0) { + metaError("vgId:%d, failed to update change time:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } metaULock(pMeta); - (void)metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp); + if (metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp) < 0) { + metaError("vgId:%d, failed to update meta rsp:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } for (int32_t i = 0; i < entry.colCmpr.nCols; i++) { SColCmpr *p = &entry.colCmpr.pColCmpr[i]; pMetaRsp->pSchemaExt[i].colId = p->id; @@ -1997,14 +2035,18 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA TBC *pUidIdxc = NULL; TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - (void)tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + if (tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c) < 0) { + metaTrace("meta/table: failed to move to uid index, uid:%" PRId64, uid); + } if (c != 0) { tdbTbcClose(pUidIdxc); metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; } - (void)tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + if (tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData) != 0) { + metaError("meta/table: failed to get uid index, uid:%" PRId64, uid); + } oversion = ((SUidIdxVal *)pData)[0].version; // search table.db @@ -2014,7 +2056,9 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA /* get ctbEntry */ TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - (void)tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); + if (tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c) != 0) { + metaError("meta/table: failed to move to tb db, uid:%" PRId64, uid); + } if (c != 0) { tdbTbcClose(pUidIdxc); tdbTbcClose(pTbDbc); @@ -2022,29 +2066,43 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; } - (void)tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); + if (tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData) != 0) { + metaError("meta/table: failed to get tb db, uid:%" PRId64, uid); + } if ((ctbEntry.pBuf = taosMemoryMalloc(nData)) == NULL) { - (void)tdbTbcClose(pUidIdxc); - (void)tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); return terrno; } memcpy(ctbEntry.pBuf, pData, nData); tDecoderInit(&dc1, ctbEntry.pBuf, nData); - (void)metaDecodeEntry(&dc1, &ctbEntry); + ret = metaDecodeEntry(&dc1, &ctbEntry); + if (ret < 0) { + terrno = ret; + goto _err; + } /* get stbEntry*/ - (void)tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal); + if (tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal) != 0) { + metaError("meta/table: failed to get uid index, uid:%" PRId64, ctbEntry.ctbEntry.suid); + } if (!pVal) { terrno = TSDB_CODE_INVALID_MSG; goto _err; } - (void)tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = ((SUidIdxVal *)pVal)[0].version}), - sizeof(STbDbKey), (void **)&stbEntry.pBuf, &nVal); + if (tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = ((SUidIdxVal *)pVal)[0].version}), + sizeof(STbDbKey), (void **)&stbEntry.pBuf, &nVal) != 0) { + metaError("meta/table: failed to get tb db, uid:%" PRId64, ctbEntry.ctbEntry.suid); + } tdbFree(pVal); tDecoderInit(&dc2, stbEntry.pBuf, nVal); - (void)metaDecodeEntry(&dc2, &stbEntry); + ret = metaDecodeEntry(&dc2, &stbEntry); + if (ret < 0) { + terrno = ret; + goto _err; + } SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; SSchema *pColumn = NULL; @@ -2122,12 +2180,18 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaWLock(pMeta); // save to table.db - (void)metaSaveToTbDb(pMeta, &ctbEntry); + if (metaSaveToTbDb(pMeta, &ctbEntry) < 0) { + metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } // save to uid.idx - (void)metaUpdateUidIdx(pMeta, &ctbEntry); + if (metaUpdateUidIdx(pMeta, &ctbEntry) < 0) { + metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } - (void)metaUpdateTagIdx(pMeta, &ctbEntry); + if (metaUpdateTagIdx(pMeta, &ctbEntry) < 0) { + metaError("meta/table: failed to update tag idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } if (NULL == ctbEntry.ctbEntry.pTags) { metaError("meta/table: null tags, update tag val failed."); @@ -2135,13 +2199,22 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA } SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; - (void)tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, - ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn); + if (tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, + ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn) < 0) { + metaError("meta/table: failed to upsert ctb idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } - (void)metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid); - (void)metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid); + if (metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { + metaError("meta/table: failed to clear uid cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } - (void)metaUpdateChangeTime(pMeta, ctbEntry.uid, pAlterTbReq->ctimeMs); + if (metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { + metaError("meta/table: failed to clear group cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } + + if (metaUpdateChangeTime(pMeta, ctbEntry.uid, pAlterTbReq->ctimeMs) < 0) { + metaError("meta/table: failed to update change time:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } metaULock(pMeta); @@ -2189,21 +2262,27 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p TBC *pUidIdxc = NULL; TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - (void)tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + if (tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c) < 0) { + metaError("meta/table: failed to move to uid index, uid:%" PRId64, uid); + } if (c != 0) { tdbTbcClose(pUidIdxc); metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); return TSDB_CODE_FAILED; } - (void)tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + if (tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData) < 0) { + metaError("meta/table: failed to get uid index, uid:%" PRId64, uid); + } oversion = ((SUidIdxVal *)pData)[0].version; // search table.db TBC *pTbDbc = NULL; TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - (void)tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); + if (tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c) < 0) { + metaError("meta/table: failed to move to tb db, uid:%" PRId64, uid); + } if (c != 0) { tdbTbcClose(pUidIdxc); tdbTbcClose(pTbDbc); @@ -2211,13 +2290,15 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p return TSDB_CODE_FAILED; } - (void)tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); + if (tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData) < 0) { + metaError("meta/table: failed to get tb db, uid:%" PRId64, uid); + } // get table entry SDecoder dc = {0}; if ((entry.pBuf = taosMemoryMalloc(nData)) == NULL) { - (void)tdbTbcClose(pUidIdxc); - (void)tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); return terrno; } memcpy(entry.pBuf, pData, nData); @@ -2236,9 +2317,9 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p // build SMetaEntry if (entry.type == TSDB_CHILD_TABLE) { if (pAlterTbReq->updateTTL) { - (void)metaDeleteTtl(pMeta, &entry); + metaDeleteTtl(pMeta, &entry); entry.ctbEntry.ttlDays = pAlterTbReq->newTTL; - (void)metaUpdateTtl(pMeta, &entry); + metaUpdateTtl(pMeta, &entry); } if (pAlterTbReq->newCommentLen >= 0) { entry.ctbEntry.commentLen = pAlterTbReq->newCommentLen; @@ -2246,9 +2327,9 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p } } else { if (pAlterTbReq->updateTTL) { - (void)metaDeleteTtl(pMeta, &entry); + metaDeleteTtl(pMeta, &entry); entry.ntbEntry.ttlDays = pAlterTbReq->newTTL; - (void)metaUpdateTtl(pMeta, &entry); + metaUpdateTtl(pMeta, &entry); } if (pAlterTbReq->newCommentLen >= 0) { entry.ntbEntry.commentLen = pAlterTbReq->newCommentLen; @@ -2257,9 +2338,17 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p } // save to table db - (void)metaSaveToTbDb(pMeta, &entry); - (void)metaUpdateUidIdx(pMeta, &entry); - (void)metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs); + if (metaSaveToTbDb(pMeta, &entry) < 0) { + metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, entry.name, entry.uid); + } + + if (metaUpdateUidIdx(pMeta, &entry) < 0) { + metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, entry.name, entry.uid); + } + + if (metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs) < 0) { + metaError("meta/table: failed to update change time:%s uid:%" PRId64, entry.name, entry.uid); + } metaULock(pMeta); @@ -2305,10 +2394,14 @@ static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTb STbDbKey tbDbKey = {0}; tbDbKey.uid = suid; tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; - (void)tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal); + ret = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal); + if (ret < 0) { + goto _err; + } tDecoderInit(&dc, pVal, nVal); ret = metaDecodeEntry(&dc, &stbEntry); if (ret < 0) { + tDecoderClear(&dc); goto _err; } @@ -2384,7 +2477,10 @@ static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTb tdbTbcClose(pCtbIdxc); goto _err; } - (void)tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); + ret = tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); + if (ret < 0) { + metaError("meta/table: failed to upsert tag idx:%s uid:%" PRId64, stbEntry.name, stbEntry.uid); + } metaDestroyTagIdxKey(pTagIdxKey); pTagIdxKey = NULL; } @@ -2439,11 +2535,15 @@ static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterT STbDbKey tbDbKey = {0}; tbDbKey.uid = suid; tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; - (void)tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal); + ret = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal); + if (ret < 0) { + goto _err; + } tDecoderInit(&dc, pVal, nVal); ret = metaDecodeEntry(&dc, &stbEntry); if (ret < 0) { + tDecoderClear(&dc); goto _err; } @@ -2507,7 +2607,10 @@ static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterT metaWLock(pMeta); for (int i = 0; i < taosArrayGetSize(tagIdxList); i++) { SMetaPair *pair = taosArrayGet(tagIdxList, i); - (void)tdbTbDelete(pMeta->pTagIdx, pair->key, pair->nkey, pMeta->txn); + ret = tdbTbDelete(pMeta->pTagIdx, pair->key, pair->nkey, pMeta->txn); + if (ret < 0) { + metaError("meta/table: failed to delete tag idx:%s uid:%" PRId64, stbEntry.name, stbEntry.uid); + } } metaULock(pMeta); @@ -2594,9 +2697,17 @@ int32_t metaUpdateTableColCompress(SMeta *pMeta, int64_t version, SVAlterTbReq * tbEntry.version = version; metaWLock(pMeta); - (void)metaSaveToTbDb(pMeta, &tbEntry); - (void)metaUpdateUidIdx(pMeta, &tbEntry); - (void)metaUpdateChangeTime(pMeta, suid, pReq->ctimeMs); + if (metaSaveToTbDb(pMeta, &tbEntry) < 0) { + metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, tbEntry.name, tbEntry.uid); + } + + if (metaUpdateUidIdx(pMeta, &tbEntry) < 0) { + metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, tbEntry.name, tbEntry.uid); + } + + if (metaUpdateChangeTime(pMeta, suid, pReq->ctimeMs) < 0) { + metaError("meta/table: failed to update change time:%s uid:%" PRId64, tbEntry.name, tbEntry.uid); + } metaULock(pMeta); @@ -2691,7 +2802,10 @@ static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { // upsert cache SMetaInfo info; metaGetEntryInfo(pME, &info); - (void)metaCacheUpsert(pMeta, &info); + int32_t ret = metaCacheUpsert(pMeta, &info); + if (ret < 0) { + metaError("vgId:%d, failed to upsert cache, uid: %" PRId64 " %s", TD_VID(pMeta->pVnode), pME->uid, tstrerror(ret)); + } SUidIdxVal uidIdxVal = {.suid = info.suid, .version = info.version, .skmVer = info.skmVer}; @@ -2706,8 +2820,8 @@ static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbUpsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn); } -static int metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME) { - if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return 0; +static void metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME) { + if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return; STtlUpdTtlCtx ctx = {.uid = pME->uid, .pTxn = pMeta->txn}; if (pME->type == TSDB_CHILD_TABLE) { @@ -2718,7 +2832,12 @@ static int metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME) { ctx.changeTimeMs = pME->ntbEntry.btime; } - return ttlMgrInsertTtl(pMeta->pTtlMgr, &ctx); + int32_t ret = ttlMgrInsertTtl(pMeta->pTtlMgr, &ctx); + if (ret < 0) { + metaError("vgId:%d, failed to insert ttl, uid: %" PRId64 " %s", TD_VID(pMeta->pVnode), pME->uid, tstrerror(ret)); + } + + return; } static int metaUpdateChangeTime(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs) { @@ -2806,7 +2925,11 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { } tbDbKey.uid = pCtbEntry->ctbEntry.suid; tbDbKey.version = ((SUidIdxVal *)pData)[0].version; - (void)tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); + ret = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); + if (ret < 0) { + metaError("vgId:%d, failed to get stable for update. version:%" PRId64, TD_VID(pMeta->pVnode), pCtbEntry->version); + goto end; + } tDecoderInit(&dc, pData, nData); ret = metaDecodeEntry(&dc, &stbEntry); @@ -2854,7 +2977,9 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { ret = -1; goto end; } - (void)tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); + if (tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn) < 0) { + metaError("vgId:%d, failed to update tag index. version:%" PRId64, TD_VID(pMeta->pVnode), pCtbEntry->version); + } metaDestroyTagIdxKey(pTagIdxKey); pTagIdxKey = NULL; } @@ -2905,7 +3030,11 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { } tEncoderInit(&coder, pVal, vLen); - (void)tEncodeSSchemaWrapper(&coder, pSW); + ret = tEncodeSSchemaWrapper(&coder, pSW); + if (ret < 0) { + rcode = -1; + goto _exit; + } if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, pMeta->txn) < 0) { rcode = -1; @@ -2966,8 +3095,7 @@ int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) { } if (pME->type != TSDB_SUPER_TABLE) { - code = metaUpdateTtl(pMeta, pME); - VND_CHECK_CODE(code, line, _err); + metaUpdateTtl(pMeta, pME); } if (pME->type == TSDB_SUPER_TABLE || pME->type == TSDB_NORMAL_TABLE) { @@ -2985,7 +3113,7 @@ _err: return TSDB_CODE_FAILED; } -int32_t colCompressDebug(SHashObj *pColCmprObj) { +static void colCompressDebug(SHashObj *pColCmprObj) { void *p = taosHashIterate(pColCmprObj, NULL); while (p) { uint32_t cmprAlg = *(uint32_t *)p; @@ -2993,14 +3121,14 @@ int32_t colCompressDebug(SHashObj *pColCmprObj) { p = taosHashIterate(pColCmprObj, p); uint8_t l1, l2, lvl; - (void)tcompressDebug(cmprAlg, &l1, &l2, &lvl); + tcompressDebug(cmprAlg, &l1, &l2, &lvl); const char *l1str = columnEncodeStr(l1); const char *l2str = columnCompressStr(l2); const char *lvlstr = columnLevelStr(lvl); metaDebug("colId: %d, encode:%s, compress:%s,level:%s", colId, l1str, l2str, lvlstr); } - return 0; + return; } int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) { int rc = 0; @@ -3063,7 +3191,7 @@ int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) { metaULock(pMeta); *ppColCmprObj = pColCmprObj; - (void)colCompressDebug(pColCmprObj); + colCompressDebug(pColCmprObj); return 0; } diff --git a/source/dnode/vnode/src/meta/metaTtl.c b/source/dnode/vnode/src/meta/metaTtl.c index e3d6e2cf9b..4077e9fa5d 100644 --- a/source/dnode/vnode/src/meta/metaTtl.c +++ b/source/dnode/vnode/src/meta/metaTtl.c @@ -144,7 +144,7 @@ static void ttlMgrCleanup(STtlManger *pTtlMgr) { taosMemoryFree(pTtlMgr->logPrefix); taosHashCleanup(pTtlMgr->pTtlCache); taosHashCleanup(pTtlMgr->pDirtyUids); - (void)tdbTbClose(pTtlMgr->pTtlIdx); + tdbTbClose(pTtlMgr->pTtlIdx); taosMemoryFree(pTtlMgr); } @@ -302,7 +302,10 @@ int32_t ttlMgrInsertTtl(STtlManger *pTtlMgr, const STtlUpdTtlCtx *updCtx) { } if (ttlMgrNeedFlush(pTtlMgr)) { - (void)ttlMgrFlush(pTtlMgr, updCtx->pTxn); + int32_t ret = ttlMgrFlush(pTtlMgr, updCtx->pTxn); + if (ret < 0) { + metaError("%s, ttlMgr insert failed to flush since %s", pTtlMgr->logPrefix, tstrerror(ret)); + } } code = TSDB_CODE_SUCCESS; @@ -326,7 +329,10 @@ int32_t ttlMgrDeleteTtl(STtlManger *pTtlMgr, const STtlDelTtlCtx *delCtx) { } if (ttlMgrNeedFlush(pTtlMgr)) { - (void)ttlMgrFlush(pTtlMgr, delCtx->pTxn); + int32_t ret = ttlMgrFlush(pTtlMgr, delCtx->pTxn); + if (ret < 0) { + metaError("%s, ttlMgr del failed to flush since %s", pTtlMgr->logPrefix, tstrerror(ret)); + } } code = TSDB_CODE_SUCCESS; @@ -350,7 +356,8 @@ int32_t ttlMgrUpdateChangeTime(STtlManger *pTtlMgr, const STtlUpdCtimeCtx *pUpdC .changeTimeMsDirty = pUpdCtimeCtx->changeTimeMs}; STtlDirtyEntry dirtryEntry = {.type = ENTRY_TYPE_UPSERT}; - code = taosHashPut(pTtlMgr->pTtlCache, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid), &cacheEntry, sizeof(cacheEntry)); + code = + taosHashPut(pTtlMgr->pTtlCache, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid), &cacheEntry, sizeof(cacheEntry)); if (TSDB_CODE_SUCCESS != code) { metaError("%s, ttlMgr update ctime failed to update cache since %s", pTtlMgr->logPrefix, tstrerror(code)); goto _out; @@ -359,13 +366,15 @@ int32_t ttlMgrUpdateChangeTime(STtlManger *pTtlMgr, const STtlUpdCtimeCtx *pUpdC code = taosHashPut(pTtlMgr->pDirtyUids, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid), &dirtryEntry, sizeof(dirtryEntry)); if (TSDB_CODE_SUCCESS != code) { - metaError("%s, ttlMgr update ctime failed to update dirty uids since %s", pTtlMgr->logPrefix, - tstrerror(code)); + metaError("%s, ttlMgr update ctime failed to update dirty uids since %s", pTtlMgr->logPrefix, tstrerror(code)); goto _out; } if (ttlMgrNeedFlush(pTtlMgr)) { - (void)ttlMgrFlush(pTtlMgr, pUpdCtimeCtx->pTxn); + int32_t ret = ttlMgrFlush(pTtlMgr, pUpdCtimeCtx->pTxn); + if (ret < 0) { + metaError("%s, ttlMgr update ctime failed to flush since %s", pTtlMgr->logPrefix, tstrerror(ret)); + } } code = TSDB_CODE_SUCCESS; @@ -420,7 +429,7 @@ int32_t ttlMgrFlush(STtlManger *pTtlMgr, TXN *pTxn) { STtlCacheEntry *cacheEntry = taosHashGet(pTtlMgr->pTtlCache, pUid, sizeof(*pUid)); if (cacheEntry == NULL) { metaError("%s, ttlMgr flush failed to get ttl cache, uid: %" PRId64 ", type: %d", pTtlMgr->logPrefix, *pUid, - pEntry->type); + pEntry->type); continue; } diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index efa248755b..45912b5ae1 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -211,7 +211,10 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS SRSmaStat *pRSmaStat = (SRSmaStat *)(*pSmaStat); pRSmaStat->pSma = (SSma *)pSma; atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INIT); - (void)tsem_init(&pRSmaStat->notEmpty, 0, 0); + if (tsem_init(&pRSmaStat->notEmpty, 0, 0) != 0) { + code = terrno; + TAOS_CHECK_GOTO(code, &lino, _exit); + } if (!(pRSmaStat->blocks = taosArrayInit(1, sizeof(SSDataBlock)))) { code = terrno; TAOS_CHECK_GOTO(code, &lino, _exit); @@ -295,7 +298,10 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { taosHashCleanup(RSMA_INFO_HASH(pStat)); // step 5: free pStat - (void)tsem_destroy(&(pStat->notEmpty)); + if (tsem_destroy(&(pStat->notEmpty)) != 0) { + smaError("vgId:%d, failed to destroy notEmpty semaphore for rsma stat:%p since %s", SMA_VID(pSma), pRSmaStat, + tstrerror(terrno)); + } taosArrayDestroy(pStat->blocks); taosMemoryFreeClear(pStat); } @@ -399,7 +405,7 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { void *tdRSmaExecutorFunc(void *param) { setThreadName("vnode-rsma"); - if(tdRSmaProcessExecImpl((SSma *)param, RSMA_EXEC_OVERFLOW) < 0){ + if (tdRSmaProcessExecImpl((SSma *)param, RSMA_EXEC_OVERFLOW) < 0) { smaError("vgId:%d, failed to process rsma exec", SMA_VID((SSma *)param)); } return NULL; @@ -444,7 +450,9 @@ static int32_t tdRsmaStopExecutor(const SSma *pSma) { pthread = (TdThread *)&pStat->data; for (int32_t i = 0; i < tsNumOfVnodeRsmaThreads; ++i) { - (void)tsem_post(&(pRSmaStat->notEmpty)); + if (tsem_post(&(pRSmaStat->notEmpty)) != 0) { + smaError("vgId:%d, failed to post notEmpty semaphore for rsma since %s", SMA_VID(pSma), tstrerror(terrno)); + } } for (int32_t i = 0; i < tsNumOfVnodeRsmaThreads; ++i) { diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 19df6edead..7bc9237965 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -171,7 +171,7 @@ _exit: TAOS_RETURN(code); } -int32_t smaClose(SSma *pSma) { +void smaClose(SSma *pSma) { if (pSma) { TAOS_UNUSED(smaPreClose(pSma)); (void)taosThreadMutexDestroy(&pSma->mutex); @@ -182,7 +182,7 @@ int32_t smaClose(SSma *pSma) { if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma)); taosMemoryFreeClear(pSma); } - return 0; + return; } /** diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index cf0a9701d9..4fdf299e50 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -89,7 +89,7 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { if (pItem->tmrId) { smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pItem->tmrId, pInfo->suid, i + 1); - if(!taosTmrStopA(&pItem->tmrId)){ + if (!taosTmrStopA(&pItem->tmrId)) { smaError("vgId:%d, failed to stop fetch timer for table %" PRIi64 " level %d", SMA_VID(pSma), pInfo->suid, i + 1); } @@ -404,7 +404,7 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con } STSchema *pTSchema; - code = metaGetTbTSchemaNotNull(SMA_META(pSma), suid, -1, 1, &pTSchema); + code = metaGetTbTSchemaNotNull(SMA_META(pSma), suid, -1, 1, &pTSchema); TAOS_CHECK_EXIT(code); pRSmaInfo->pSma = pSma; pRSmaInfo->pTSchema = pTSchema; @@ -820,7 +820,10 @@ static int32_t tdExecuteRSmaImplAsync(SSma *pSma, int64_t version, const void *p int64_t nItems = atomic_fetch_add_64(&pRSmaStat->nBufItems, 1); if (atomic_load_8(&pInfo->assigned) == 0) { - (void)tsem_post(&(pRSmaStat->notEmpty)); + if (tsem_post(&(pRSmaStat->notEmpty)) != 0) { + smaError("vgId:%d, failed to post notEmpty semaphore for rsma %" PRIi64 " since %s", SMA_VID(pSma), suid, + tstrerror(terrno)); + } } // smoothing consume @@ -1385,7 +1388,8 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { if (rsmaTriggerStat == TASK_TRIGGER_STAT_PAUSED) { bool ret = taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId); if (!ret) { - smaWarn("vgId:%d, rsma fetch task not reset for level %" PRIi8 " since tmr reset failed, rsetId:%d refId:%" PRIi64, + smaWarn("vgId:%d, rsma fetch task not reset for level %" PRIi8 + " since tmr reset failed, rsetId:%d refId:%" PRIi64, SMA_VID(pSma), pItem->level, smaMgmt.rsetId, pRSmaRef->refId); } } @@ -1407,7 +1411,10 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { atomic_store_8(&pItem->fetchLevel, 1); if (atomic_load_8(&pRSmaInfo->assigned) == 0) { - (void)tsem_post(&(pStat->notEmpty)); + if (tsem_post(&(pStat->notEmpty)) != 0) { + smaError("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since sem post failed", + SMA_VID(pSma), pItem->level, pRSmaInfo->suid); + } } } break; case TASK_TRIGGER_STAT_INACTIVE: { @@ -1641,7 +1648,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { batchMax = 100 / batchMax; batchMax = TMAX(batchMax, 4); } - while (occupied || (++batchCnt < batchMax)) { // greedy mode + while (occupied || (++batchCnt < batchMax)) { // greedy mode TAOS_UNUSED(taosReadAllQitems(pInfo->queue, pInfo->qall)); // queue has mutex lock int32_t qallItemSize = taosQallItemSize(pInfo->qall); if (qallItemSize > 0) { @@ -1700,7 +1707,9 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { break; } - (void)tsem_wait(&pRSmaStat->notEmpty); + if (tsem_wait(&pRSmaStat->notEmpty) != 0) { + smaError("vgId:%d, failed to wait for not empty since %s", TD_VID(pVnode), tstrerror(terrno)); + } if ((pEnv->flag & SMA_ENV_FLG_CLOSE) && (atomic_load_64(&pRSmaStat->nBufItems) <= 0)) { smaDebug("vgId:%d, exec task end, flag:%" PRIi8 ", nBufItems:%" PRIi64, SMA_VID(pSma), pEnv->flag, diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d0047b8e3f..a37a9787c9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -166,14 +166,14 @@ void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { } SMqDataRsp dataRsp = {0}; - code = tqInitDataRsp(&dataRsp.common, req.reqOffset); + code = tqInitDataRsp(&dataRsp, req.reqOffset); if (code != 0) { tqError("tqInitDataRsp failed, code:%d", code); return; } - dataRsp.common.blockNum = 0; + dataRsp.blockNum = 0; char buf[TSDB_OFFSET_LEN] = {0}; - (void)tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.reqOffset); + (void)tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.reqOffset); tqInfo("tqPushEmptyDataRsp to consumer:0x%" PRIx64 " vgId:%d, offset:%s,QID:0x%" PRIx64, req.consumerId, vgId, buf, req.reqId); @@ -184,18 +184,18 @@ void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { tDeleteMqDataRsp(&dataRsp); } -int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const void* pRsp, int32_t type, +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type, int32_t vgId) { int64_t sver = 0, ever = 0; walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); char buf1[TSDB_OFFSET_LEN] = {0}; char buf2[TSDB_OFFSET_LEN] = {0}; - (void)tFormatOffset(buf1, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->reqOffset); - (void)tFormatOffset(buf2, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->rspOffset); + (void)tFormatOffset(buf1, TSDB_OFFSET_LEN, &(pRsp->reqOffset)); + (void)tFormatOffset(buf2, TSDB_OFFSET_LEN, &(pRsp->rspOffset)); tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s,QID:0x%" PRIx64, - vgId, pReq->consumerId, pReq->epoch, ((SMqDataRspCommon*)pRsp)->blockNum, buf1, buf2, pReq->reqId); + vgId, pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId); return tqDoSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type, sver, ever); } @@ -518,7 +518,7 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { taosRUnLockLatch(&pTq->lock); SMqDataRsp dataRsp = {0}; - code = tqInitDataRsp(&dataRsp.common, req.reqOffset); + code = tqInitDataRsp(&dataRsp, req.reqOffset); if (code != 0) { return code; } @@ -529,10 +529,10 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { goto END; } - dataRsp.common.rspOffset.type = TMQ_OFFSET__LOG; + dataRsp.rspOffset.type = TMQ_OFFSET__LOG; if (reqOffset.type == TMQ_OFFSET__LOG) { - dataRsp.common.rspOffset.version = reqOffset.version; + dataRsp.rspOffset.version = reqOffset.version; } else if (reqOffset.type < 0) { STqOffset* pOffset = NULL; code = tqMetaGetOffset(pTq, req.subKey, &pOffset); @@ -543,17 +543,17 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { goto END; } - dataRsp.common.rspOffset.version = pOffset->val.version; + dataRsp.rspOffset.version = pOffset->val.version; tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from store:%" PRId64, consumerId, vgId, - req.subKey, dataRsp.common.rspOffset.version); + req.subKey, dataRsp.rspOffset.version); } else { if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEST) { - dataRsp.common.rspOffset.version = sver; // not consume yet, set the earliest position + dataRsp.rspOffset.version = sver; // not consume yet, set the earliest position } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { - dataRsp.common.rspOffset.version = ever; + dataRsp.rspOffset.version = ever; } tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from init:%" PRId64, consumerId, vgId, req.subKey, - dataRsp.common.rspOffset.version); + dataRsp.rspOffset.version); } } else { tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s invalid offset type:%d", consumerId, vgId, req.subKey, diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index b7c842d05c..89350e761f 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -127,7 +127,6 @@ int32_t tqMetaSaveOffset(STQ* pTq, STqOffset* pOffset) { goto END; } - buf = taosMemoryCalloc(1, vlen); if (buf == NULL) { code = terrno; @@ -152,7 +151,8 @@ int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const int32_t code = TDB_CODE_SUCCESS; TXN* txn = NULL; - TQ_ERR_GO_TO_END(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); + TQ_ERR_GO_TO_END( + tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); TQ_ERR_GO_TO_END(tdbTbUpsert(ttb, key, kLen, value, vLen, txn)); TQ_ERR_GO_TO_END(tdbCommit(pTq->pMetaDB, txn)); TQ_ERR_GO_TO_END(tdbPostCommit(pTq->pMetaDB, txn)); @@ -168,7 +168,8 @@ int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen) { int32_t code = TDB_CODE_SUCCESS; TXN* txn = NULL; - TQ_ERR_GO_TO_END(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); + TQ_ERR_GO_TO_END( + tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); TQ_ERR_GO_TO_END(tdbTbDelete(ttb, key, kLen, txn)); TQ_ERR_GO_TO_END(tdbCommit(pTq->pMetaDB, txn)); TQ_ERR_GO_TO_END(tdbPostCommit(pTq->pMetaDB, txn)); @@ -180,7 +181,7 @@ END: return code; } -int32_t tqMetaGetOffset(STQ* pTq, const char* subkey, STqOffset** pOffset){ +int32_t tqMetaGetOffset(STQ* pTq, const char* subkey, STqOffset** pOffset) { void* data = taosHashGet(pTq->pOffset, subkey, strlen(subkey)); if (data == NULL) { int vLen = 0; @@ -203,7 +204,7 @@ int32_t tqMetaGetOffset(STQ* pTq, const char* subkey, STqOffset** pOffset){ tdbFree(data); *pOffset = taosHashGet(pTq->pOffset, subkey, strlen(subkey)); - if(*pOffset == NULL){ + if (*pOffset == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } } else { @@ -266,8 +267,8 @@ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle) { initStorageAPI(&reader.api); if (handle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - handle->execHandle.task = - qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId, &handle->execHandle.numOfCols, handle->consumerId); + handle->execHandle.task = qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId, + &handle->execHandle.numOfCols, handle->consumerId); TQ_NULL_GO_TO_END(handle->execHandle.task); void* scanner = NULL; qExtractStreamScanner(handle->execHandle.task, &scanner); @@ -280,20 +281,21 @@ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle) { handle->execHandle.pTqReader = tqReaderOpen(pVnode); TQ_NULL_GO_TO_END(handle->execHandle.pTqReader); TQ_ERR_GO_TO_END(buildSnapContext(reader.vnode, reader.version, 0, handle->execHandle.subType, handle->fetchMeta, - (SSnapContext**)(&reader.sContext))); + (SSnapContext**)(&reader.sContext))); handle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, handle->consumerId); TQ_NULL_GO_TO_END(handle->execHandle.task); } else if (handle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { handle->pWalReader = walOpenReader(pVnode->pWal, NULL, 0); TQ_NULL_GO_TO_END(handle->pWalReader); - if(handle->execHandle.execTb.qmsg != NULL && strcmp(handle->execHandle.execTb.qmsg, "") != 0) { + if (handle->execHandle.execTb.qmsg != NULL && strcmp(handle->execHandle.execTb.qmsg, "") != 0) { if (nodesStringToNode(handle->execHandle.execTb.qmsg, &handle->execHandle.execTb.node) != 0) { tqError("nodesStringToNode error in sub stable, since %s", terrstr()); return TSDB_CODE_SCH_INTERNAL_ERROR; } } - TQ_ERR_GO_TO_END(buildSnapContext(reader.vnode, reader.version, handle->execHandle.execTb.suid, handle->execHandle.subType, - handle->fetchMeta, (SSnapContext**)(&reader.sContext))); + TQ_ERR_GO_TO_END(buildSnapContext(reader.vnode, reader.version, handle->execHandle.execTb.suid, + handle->execHandle.subType, handle->fetchMeta, + (SSnapContext**)(&reader.sContext))); handle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, handle->consumerId); TQ_NULL_GO_TO_END(handle->execHandle.task); SArray* tbUidList = NULL; @@ -341,7 +343,7 @@ int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle) { handle->execHandle.subType = req->subType; handle->fetchMeta = req->withMeta; if (req->subType == TOPIC_SUB_TYPE__COLUMN) { - void *tmp = taosStrdup(req->qmsg); + void* tmp = taosStrdup(req->qmsg); if (tmp == NULL) { return terrno; } @@ -349,12 +351,12 @@ int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle) { } else if (req->subType == TOPIC_SUB_TYPE__DB) { handle->execHandle.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - if(handle->execHandle.execDb.pFilterOutTbUid == NULL){ + if (handle->execHandle.execDb.pFilterOutTbUid == NULL) { return terrno; } - }else if(req->subType == TOPIC_SUB_TYPE__TABLE){ + } else if (req->subType == TOPIC_SUB_TYPE__TABLE) { handle->execHandle.execTb.suid = req->suid; - void *tmp = taosStrdup(req->qmsg); + void* tmp = taosStrdup(req->qmsg); if (tmp == NULL) { return terrno; } @@ -364,7 +366,7 @@ int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle) { handle->snapshotVer = walGetCommittedVer(pTq->pVnode->pWal); int32_t code = tqMetaInitHandle(pTq, handle); - if (code != 0){ + if (code != 0) { return code; } tqInfo("tqMetaCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey, @@ -437,10 +439,10 @@ END: return code; } -static int32_t replaceTqPath(char** path){ - char* tpath = NULL; +static int32_t replaceTqPath(char** path) { + char* tpath = NULL; int32_t code = tqBuildFName(&tpath, *path, TQ_SUBSCRIBE_NAME); - if (code != 0){ + if (code != 0) { return code; } taosMemoryFree(*path); @@ -475,7 +477,7 @@ END: } int32_t tqMetaOpen(STQ* pTq) { - char* maindb = NULL; + char* maindb = NULL; char* offsetNew = NULL; int32_t code = TDB_CODE_SUCCESS; TQ_ERR_GO_TO_END(tqBuildFName(&maindb, pTq->path, TDB_MAINDB_NAME)); @@ -488,7 +490,7 @@ int32_t tqMetaOpen(STQ* pTq) { } TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME)); - if(taosCheckExistFile(offsetNew)){ + if (taosCheckExistFile(offsetNew)) { TQ_ERR_GO_TO_END(tqOffsetRestoreFromFile(pTq, offsetNew)); TQ_ERR_GO_TO_END(taosRemoveFile(offsetNew)); } @@ -522,7 +524,7 @@ int32_t tqMetaTransform(STQ* pTq) { TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME)); - if(taosCheckExistFile(offset)) { + if (taosCheckExistFile(offset)) { if (taosCopyFile(offset, offsetNew) < 0) { tqError("copy offset file error"); } else { @@ -534,44 +536,22 @@ END: taosMemoryFree(offset); taosMemoryFree(offsetNew); - int32_t ret = tdbTbClose(pExecStore); - if (ret != 0) { - tqError("failed to close tb, ret:%d", ret); - } - ret = tdbTbClose(pCheckStore); - if (ret != 0) { - tqError("failed to close tb, ret:%d", ret); - } - ret = tdbClose(pMetaDB); - if (ret != 0) { - tqError("failed to close tdb, ret:%d", ret); - } - + tdbTbClose(pExecStore); + tdbTbClose(pCheckStore); + tdbClose(pMetaDB); return code; } void tqMetaClose(STQ* pTq) { int32_t ret = 0; if (pTq->pExecStore) { - ret = tdbTbClose(pTq->pExecStore); - if (ret != 0) { - tqError("failed to close tb, ret:%d", ret); - } + tdbTbClose(pTq->pExecStore); } if (pTq->pCheckStore) { - ret = tdbTbClose(pTq->pCheckStore); - if (ret != 0) { - tqError("failed to close tb, ret:%d", ret); - } + tdbTbClose(pTq->pCheckStore); } if (pTq->pOffsetStore) { - ret = tdbTbClose(pTq->pOffsetStore); - if (ret != 0) { - tqError("failed to close tb, ret:%d", ret); - } - } - ret = tdbClose(pTq->pMetaDB); - if (ret != 0) { - tqError("failed to close tdb, ret:%d", ret); + tdbTbClose(pTq->pOffsetStore); } + tdbClose(pTq->pMetaDB); } diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index f71669f158..dbc1b16cf5 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -15,7 +15,7 @@ #include "tq.h" -int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, void* pRsp, int32_t numOfCols, int8_t precision) { +int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) { int32_t dataStrLen = sizeof(SRetrieveTableRspForTmq) + blockGetEncodeSize(pBlock); void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) { @@ -34,11 +34,11 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, void* pRsp, int32_t numOf return terrno; } actualLen += sizeof(SRetrieveTableRspForTmq); - if (taosArrayPush(((SMqDataRspCommon*)pRsp)->blockDataLen, &actualLen) == NULL){ + if (taosArrayPush(pRsp->blockDataLen, &actualLen) == NULL){ taosMemoryFree(buf); return terrno; } - if (taosArrayPush(((SMqDataRspCommon*)pRsp)->blockData, &buf) == NULL) { + if (taosArrayPush(pRsp->blockData, &buf) == NULL) { taosMemoryFree(buf); return terrno; } @@ -46,18 +46,18 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, void* pRsp, int32_t numOf return TSDB_CODE_SUCCESS; } -static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, void* pRsp) { +static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, SMqDataRsp* pRsp) { SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pTqReader->pSchemaWrapper); if (pSW == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } - if (taosArrayPush(((SMqDataRspCommon*)pRsp)->blockSchema, &pSW) == NULL) { + if (taosArrayPush(pRsp->blockSchema, &pSW) == NULL) { return terrno; } return 0; } -static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, void* pRsp, int32_t n) { +static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, int32_t n) { SMetaReader mr = {0}; metaReaderDoInit(&mr, pTq->pVnode->pMeta, META_READER_LOCK); @@ -73,7 +73,8 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, void* pRsp, int32_t metaReaderClear(&mr); return terrno; } - if(taosArrayPush(((SMqDataRspCommon*)pRsp)->blockTbName, &tbName) == NULL){ + if(taosArrayPush(pRsp->blockTbName, &tbName) == NULL){ + tqError("failed to push tbName to blockTbName:%s", tbName); continue; } } @@ -143,7 +144,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* code = tqAddBlockDataToRsp(pHandle->block, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); TSDB_CHECK_CODE(code, line, END); - pRsp->common.blockNum++; + pRsp->blockNum++; if (pDataBlock == NULL) { blockDataDestroy(pHandle->block); pHandle->block = NULL; @@ -167,7 +168,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* code = tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); TSDB_CHECK_CODE(code, line, END); - pRsp->common.blockNum++; + pRsp->blockNum++; totalRows += pDataBlock->info.rows; if (totalRows >= tmqRowSize || (taosGetTimestampMs() - st > 1000)) { break; @@ -176,8 +177,8 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* } tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq task executed finished, total blocks:%d, totalRows:%d", - pHandle->consumerId, vgId, pRsp->common.blockNum, totalRows); - code = qStreamExtractOffset(task, &pRsp->common.rspOffset); + pHandle->consumerId, vgId, pRsp->blockNum, totalRows); + code = qStreamExtractOffset(task, &pRsp->rspOffset); END: if (code != 0) { tqError("consumer:0x%" PRIx64 " vgId:%d tmq task executed error, line:%d code:%d", pHandle->consumerId, vgId, line, @@ -186,7 +187,7 @@ END: return code; } -int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* pOffset) { +int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* pOffset) { const STqExecHandle* pExec = &pHandle->execHandle; qTaskInfo_t task = pExec->task; int code = qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType); @@ -208,7 +209,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatc tqDebug("tmqsnap task execute end, get %p", pDataBlock); if (pDataBlock != NULL && pDataBlock->info.rows > 0) { - if (pRsp->common.withTbName) { + if (pRsp->withTbName) { if (pOffset->type == TMQ_OFFSET__LOG) { int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, 1) < 0) { @@ -221,13 +222,13 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatc tqError("vgId:%d, failed to add tbname to rsp msg, null", pTq->pVnode->config.vgId); return terrno; } - if (taosArrayPush(pRsp->common.blockTbName, &tbName) == NULL){ + if (taosArrayPush(pRsp->blockTbName, &tbName) == NULL){ tqError("vgId:%d, failed to add tbname to rsp msg", pTq->pVnode->config.vgId); continue; } } } - if (pRsp->common.withSchema) { + if (pRsp->withSchema) { if (pOffset->type == TMQ_OFFSET__LOG) { if (tqAddBlockSchemaToRsp(pExec, pRsp) != 0){ tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId); @@ -235,19 +236,19 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatc } } else { SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task)); - if(taosArrayPush(pRsp->common.blockSchema, &pSW) == NULL){ + if(taosArrayPush(pRsp->blockSchema, &pSW) == NULL){ tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId); continue; } } } - if (tqAddBlockDataToRsp(pDataBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pDataBlock->pDataBlock), + if (tqAddBlockDataToRsp(pDataBlock, pRsp, taosArrayGetSize(pDataBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision) != 0) { tqError("vgId:%d, failed to add block to rsp msg", pTq->pVnode->config.vgId); continue; } - pRsp->common.blockNum++; + pRsp->blockNum++; if (pOffset->type == TMQ_OFFSET__LOG) { continue; } else { @@ -281,13 +282,13 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatc tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode), pHandle->snapshotVer + 1); - code = qStreamExtractOffset(task, &pRsp->common.rspOffset); + code = qStreamExtractOffset(task, &pRsp->rspOffset); break; } - if (pRsp->common.blockNum > 0) { + if (pRsp->blockNum > 0) { tqDebug("tmqsnap task exec exited, get data"); - code = qStreamExtractOffset(task, &pRsp->common.rspOffset); + code = qStreamExtractOffset(task, &pRsp->rspOffset); break; } } @@ -296,7 +297,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatc } -static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, STaosxRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded){ +static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded){ int32_t code = 0; STqExecHandle* pExec = &pHandle->execHandle; STqReader* pReader = pExec->pTqReader; @@ -323,7 +324,7 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, STaosxRsp* pRsp, int3 if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) { goto END; } - if (pRsp->common.withTbName) { + if (pRsp->withTbName) { int64_t uid = pExec->pTqReader->lastBlkUid; code = tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)); if (code != 0) { @@ -381,7 +382,7 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, STaosxRsp* pRsp, int3 if (pBlock == NULL) { continue; } - if (tqAddBlockDataToRsp(pBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlock->pDataBlock), + if (tqAddBlockDataToRsp(pBlock, pRsp, taosArrayGetSize(pBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision) != 0){ tqError("vgId:%d, failed to add block to rsp msg", pTq->pVnode->config.vgId); continue; @@ -389,11 +390,11 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, STaosxRsp* pRsp, int3 *totalRows += pBlock->info.rows; blockDataFreeRes(pBlock); SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i); - if (taosArrayPush(pRsp->common.blockSchema, &pSW) == NULL){ + if (taosArrayPush(pRsp->blockSchema, &pSW) == NULL){ tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId); continue; } - pRsp->common.blockNum++; + pRsp->blockNum++; } taosArrayDestroy(pBlocks); @@ -405,7 +406,7 @@ END: taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); } -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows, +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded) { STqExecHandle* pExec = &pHandle->execHandle; int32_t code = 0; diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index b0bf89029e..3ec269ec22 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -16,8 +16,13 @@ #include "tq.h" #include "vnd.h" -#define MAX_REPEAT_SCAN_THRESHOLD 3 -#define SCAN_WAL_IDLE_DURATION 100 +#define MAX_REPEAT_SCAN_THRESHOLD 3 +#define SCAN_WAL_IDLE_DURATION 100 + +typedef struct SBuildScanWalMsgParam { + int64_t metaId; + int32_t numOfTasks; +} SBuildScanWalMsgParam; static int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle); static int32_t setWalReaderStartOffset(SStreamTask* pTask, int32_t vgId); @@ -31,13 +36,12 @@ int32_t tqScanWal(STQ* pTq) { SStreamMeta* pMeta = pTq->pStreamMeta; int32_t vgId = pMeta->vgId; int64_t st = taosGetTimestampMs(); + int32_t numOfTasks = 0; + bool shouldIdle = true; tqDebug("vgId:%d continue to check if data in wal are available, scanCounter:%d", vgId, pMeta->scanInfo.scanCounter); // check all tasks - int32_t numOfTasks = 0; - bool shouldIdle = true; - int32_t code = doScanWalForAllTasks(pMeta, &shouldIdle); if (code) { tqError("vgId:%d failed to start all tasks, try next time, code:%s", vgId, tstrerror(code)); @@ -68,54 +72,61 @@ int32_t tqScanWal(STQ* pTq) { return code; } -typedef struct SBuildScanWalMsgParam { - STQ* pTq; - int32_t numOfTasks; -} SBuildScanWalMsgParam; - static void doStartScanWal(void* param, void* tmrId) { + int32_t vgId = 0; + STQ* pTq = NULL; + int32_t code = 0; + SBuildScanWalMsgParam* pParam = (SBuildScanWalMsgParam*)param; - STQ* pTq = pParam->pTq; - int32_t vgId = pTq->pStreamMeta->vgId; + SStreamMeta* pMeta = taosAcquireRef(streamMetaId, pParam->metaId); + if (pMeta == NULL) { + tqError("metaRid:%" PRId64 " not valid now, stream meta has been freed", pParam->metaId); + taosMemoryFree(pParam); + return; + } + + vgId = pMeta->vgId; + pTq = pMeta->ahandle; + tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d, vnd restored:%d", vgId, pParam->numOfTasks, pTq->pVnode->restored); - int32_t code = streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); - taosMemoryFree(pParam); - + code = streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); if (code) { tqError("vgId:%d failed sched task to scan wal, code:%s", vgId, tstrerror(code)); } + + code = taosReleaseRef(streamMetaId, pParam->metaId); + if (code) { + tqError("vgId:%d failed to release ref for streamMeta, rid:%" PRId64 " code:%s", vgId, pParam->metaId, + tstrerror(code)); + } + + taosMemoryFree(pParam); } int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) { - SStreamMeta* pMeta = pTq->pStreamMeta; - int32_t code = 0; - int32_t vgId = TD_VID(pTq->pVnode); + SStreamMeta* pMeta = pTq->pStreamMeta; + int32_t code = 0; + int32_t vgId = TD_VID(pTq->pVnode); + tmr_h pTimer = NULL; + SBuildScanWalMsgParam* pParam = NULL; - SBuildScanWalMsgParam* pParam = taosMemoryMalloc(sizeof(SBuildScanWalMsgParam)); + pParam = taosMemoryMalloc(sizeof(SBuildScanWalMsgParam)); if (pParam == NULL) { return terrno; } - pParam->pTq = pTq; + pParam->metaId = pMeta->rid; pParam->numOfTasks = numOfTasks; - tmr_h pTimer = NULL; code = streamTimerGetInstance(&pTimer); if (code) { tqError("vgId:%d failed to get tmr ctrl during sched scan wal", vgId); - return code; - } - - if (pMeta->scanInfo.scanTimer == NULL) { - pMeta->scanInfo.scanTimer = taosTmrStart(doStartScanWal, idleDuration, pParam, pTimer); + taosMemoryFree(pParam); } else { - bool ret = taosTmrReset(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer); - if (!ret) { -// tqError("vgId:%d failed to start scan wal in:%dms", vgId, idleDuration); - } + streamTmrStart(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer, vgId, "scan-wal-fut"); } return code; @@ -124,8 +135,8 @@ int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) { int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { int32_t vgId = TD_VID(pTq->pVnode); SStreamMeta* pMeta = pTq->pStreamMeta; - - bool alreadyRestored = pTq->pVnode->restored; + bool alreadyRestored = pTq->pVnode->restored; + int32_t numOfTasks = 0; // do not launch the stream tasks, if it is a follower or not restored vnode. if (!(vnodeIsRoleLeader(pTq->pVnode) && alreadyRestored)) { @@ -134,7 +145,7 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { streamMetaWLock(pMeta); - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + numOfTasks = taosArrayGetSize(pMeta->pTaskList); if (numOfTasks == 0) { tqDebug("vgId:%d no stream tasks existed to run", vgId); streamMetaWUnLock(pMeta); @@ -378,13 +389,13 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { numOfTasks = taosArrayGetSize(pTaskList); for (int32_t i = 0; i < numOfTasks; ++i) { - STaskId* pTaskId = taosArrayGet(pTaskList, i); + STaskId* pTaskId = taosArrayGet(pTaskList, i); if (pTaskId == NULL) { continue; } SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTask(pStreamMeta, pTaskId->streamId, pTaskId->taskId, &pTask); + int32_t code = streamMetaAcquireTask(pStreamMeta, pTaskId->streamId, pTaskId->taskId, &pTask); if (pTask == NULL || code != 0) { continue; } diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index 7c05b405ce..fd46c618a1 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -137,7 +137,7 @@ NextTbl: memcpy(pVal, tVal, tLen); vLen = tLen; } - (void)tdbTbcMoveToNext(pReader->pCur); + TAOS_UNUSED(tdbTbcMoveToNext(pReader->pCur)); break; } if (except == 1) { @@ -147,7 +147,7 @@ NextTbl: pReader->pos += 1; pPair = taosArrayGet(pReader->tdbTbList, pReader->pos); code = tdbTbcOpen(pPair->tbl, &pReader->pCur, NULL); - (void)tdbTbcMoveToFirst(pReader->pCur); + TAOS_UNUSED(tdbTbcMoveToFirst(pReader->pCur)); goto NextTbl; } @@ -210,7 +210,7 @@ int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback, i streamMetaWLock(pTq->pStreamMeta); tqDebug("vgId:%d, vnode stream-task snapshot writer closed", TD_VID(pTq->pVnode)); if (rollback) { - tdbAbort(pTq->pStreamMeta->db, pTq->pStreamMeta->txn); + TAOS_UNUSED(tdbAbort(pTq->pStreamMeta->db, pTq->pStreamMeta->txn)); } else { code = tdbCommit(pTq->pStreamMeta->db, pTq->pStreamMeta->txn); if (code) goto _err; diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 5b83df3217..b4866b8c65 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -20,7 +20,7 @@ static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const static int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqBatchMetaRsp* pRsp, int32_t vgId); -int32_t tqInitDataRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) { +int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) { pRsp->blockData = taosArrayInit(0, sizeof(void*)); pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); @@ -40,7 +40,7 @@ void tqUpdateNodeStage(STQ* pTq, bool isLeader) { streamMetaUpdateStageRole(pTq->pStreamMeta, state.term, isLeader); } -static int32_t tqInitTaosxRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) { +static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) { tOffsetCopy(&pRsp->reqOffset, &pOffset); tOffsetCopy(&pRsp->rspOffset, &pOffset); @@ -116,12 +116,12 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand SMqDataRsp dataRsp = {0}; tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer + 1); - code = tqInitDataRsp(&dataRsp.common, *pOffsetVal); + code = tqInitDataRsp(&dataRsp, *pOffsetVal); if (code != 0) { return code; } tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, - pHandle->subKey, vgId, dataRsp.common.rspOffset.version); + pHandle->subKey, vgId, dataRsp.rspOffset.version); code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); tDeleteMqDataRsp(&dataRsp); @@ -145,24 +145,27 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, terrno = 0; SMqDataRsp dataRsp = {0}; - - int code = tqInitDataRsp(&dataRsp.common, *pOffset); + int code = tqInitDataRsp(&dataRsp, *pOffset); if (code != 0) { goto end; } code = qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); + if (code != 0) { + goto end; + } + code = tqScanData(pTq, pHandle, &dataRsp, pOffset, pRequest); if (code != 0 && terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { goto end; } // till now, all data has been transferred to consumer, new data needs to push client once arrived. - if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.common.blockNum == 0) { + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.blockNum == 0) { // lock taosWLockLatch(&pTq->lock); int64_t ver = walGetCommittedVer(pTq->pVnode->pWal); - if (dataRsp.common.rspOffset.version > ver) { // check if there are data again to avoid lost data + if (dataRsp.rspOffset.version > ver) { // check if there are data again to avoid lost data code = tqRegisterPushHandle(pTq, pHandle, pMsg); taosWUnLockLatch(&pTq->lock); goto end; @@ -170,16 +173,16 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, taosWUnLockLatch(&pTq->lock); } - tOffsetCopy(&dataRsp.common.reqOffset, - pOffset); // reqOffset represents the current date offset, may be changed if wal not exists + // reqOffset represents the current date offset, may be changed if wal not exists + tOffsetCopy(&dataRsp.reqOffset, pOffset); code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); end : { char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.rspOffset); + tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.rspOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s,QID:0x%" PRIx64 " code:%d", - consumerId, pHandle->subKey, vgId, dataRsp.common.blockNum, buf, pRequest->reqId, code); + consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); tDeleteMqDataRsp(&dataRsp); return code; } @@ -208,11 +211,11 @@ static void tDeleteCommon(void* parm) {} static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* offset) { int32_t vgId = TD_VID(pTq->pVnode); - STaosxRsp taosxRsp = {0}; + SMqDataRsp taosxRsp = {0}; SMqBatchMetaRsp btMetaRsp = {0}; int32_t code = 0; - TQ_ERR_GO_TO_END(tqInitTaosxRsp(&taosxRsp.common, *offset)); + TQ_ERR_GO_TO_END(tqInitTaosxRsp(&taosxRsp, *offset)); if (offset->type != TMQ_OFFSET__LOG) { TQ_ERR_GO_TO_END(tqScanTaosx(pTq, pHandle, &taosxRsp, &btMetaRsp, offset)); @@ -227,13 +230,13 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64 ",ts:%" PRId64, - pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.common.blockNum, taosxRsp.common.rspOffset.type, - taosxRsp.common.rspOffset.uid, taosxRsp.common.rspOffset.ts); - if (taosxRsp.common.blockNum > 0) { + pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, + taosxRsp.rspOffset.uid, taosxRsp.rspOffset.ts); + if (taosxRsp.blockNum > 0) { code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto END; } else { - tOffsetCopy(offset, &taosxRsp.common.rspOffset); + tOffsetCopy(offset, &taosxRsp.rspOffset); } } @@ -264,7 +267,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } goto END; } - tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer); + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); code = tqSendDataRsp( pHandle, pMsg, pRequest, &taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); @@ -278,7 +281,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, // process meta if (pHead->msgType != TDMT_VND_SUBMIT) { if (totalRows > 0) { - tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer); + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); code = tqSendDataRsp( pHandle, pMsg, pRequest, &taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); @@ -387,7 +390,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } if (totalRows >= tmqRowSize || (taosGetTimestampMs() - st > 1000)) { - tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer + 1); + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer + 1); code = tqSendDataRsp( pHandle, pMsg, pRequest, &taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); @@ -522,7 +525,7 @@ int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPoll return 0; } -int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const void* pRsp, int32_t epoch, int64_t consumerId, +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever) { int32_t len = 0; int32_t code = 0; diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 30be253b65..6b7e857120 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -84,7 +84,6 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) { code = qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); if (code) { - return code; } } @@ -363,7 +362,7 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { } pRspHead->vgId = htonl(req.upstreamNodeId); - if(pRspHead->vgId == 0) { + if (pRspHead->vgId == 0) { tqError("vgId:%d invalid dispatch msg from upstream to task:0x%x", pMeta->vgId, req.taskId); return TSDB_CODE_INVALID_MSG; } @@ -460,7 +459,7 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { if (code != TSDB_CODE_SUCCESS) { // return error not send rsp manually tqError("s-task:0x%x vgId:%d failed to process retrieve request from 0x%x, code:%s", req.dstTaskId, req.dstNodeId, req.srcTaskId, tstrerror(code)); - } else { // send rsp manually only on success. + } else { // send rsp manually only on success. SRpcMsg rsp = {.info = pMsg->info, .code = 0}; streamTaskSendRetrieveRsp(&req, &rsp); } @@ -515,7 +514,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe } tDecoderClear(&decoder); - tqDebug("tq task:0x%x (vgId:%d) recv check rsp(qid:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId, + tqDebug("tq task:0x%x (vgId:%d) recv check rsp(QID:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); if (!isLeader) { @@ -1272,7 +1271,7 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { streamMutexLock(&pTask->lock); if (pTask->chkInfo.checkpointId < req.checkpointId) { - tqFatal("s-task:%s vgId:%d invalid consensus-checkpointId:%" PRId64 ", greater than existed checkpointId:%"PRId64, + tqFatal("s-task:%s vgId:%d invalid consensus-checkpointId:%" PRId64 ", greater than existed checkpointId:%" PRId64, pTask->id.idStr, vgId, req.checkpointId, pTask->chkInfo.checkpointId); streamMutexUnlock(&pTask->lock); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 482b194a1f..85f74b1672 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -22,6 +22,12 @@ #define ROCKS_BATCH_SIZE (4096) +void tsdbLRUCacheRelease(SLRUCache *cache, LRUHandle *handle, bool eraseIfLastRef) { + if (!taosLRUCacheRelease(cache, handle, eraseIfLastRef)) { + tsdbTrace(" release lru cache failed"); + } +} + static int32_t tsdbOpenBCache(STsdb *pTsdb) { int32_t code = 0, lino = 0; int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; @@ -579,7 +585,7 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud if (pLastCol->dirty) { if (tsdbCacheFlushDirty(key, klen, pLastCol, ud) != 0) { STsdb *pTsdb = (STsdb *)ud; - tsdbError("tsdb/cache: vgId:%d, flush cache %s failed at line %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__); + tsdbTrace("tsdb/cache: vgId:%d, flush cache %s failed at line %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__); } } @@ -597,6 +603,13 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud taosMemoryFree(value); } +static void tsdbCacheOverWriter(const void *key, size_t klen, void *value, void *ud) { + SLastCol *pLastCol = (SLastCol *)value; + pLastCol->dirty = 0; +} + +static int32_t tsdbCachePutToLRU(STsdb *pTsdb, SLastKey *pLastKey, SLastCol *pLastCol, int8_t dirty); + static int32_t tsdbCacheNewTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, int8_t col_type, int8_t lflag) { int32_t code = 0, lino = 0; @@ -606,27 +619,10 @@ static int32_t tsdbCacheNewTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, i SLastCol emptyCol = { .rowKey = emptyRowKey, .colVal = COL_VAL_NONE(cid, col_type), .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_VALID}; - SLastCol *pLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); - if (!pLastCol) { - return terrno; - } - - size_t charge = 0; - *pLastCol = emptyCol; - TAOS_CHECK_EXIT(tsdbCacheReallocSLastCol(pLastCol, &charge)); - SLastKey *pLastKey = &(SLastKey){.lflag = lflag, .uid = uid, .cid = cid}; - LRUStatus status = taosLRUCacheInsert(pCache, pLastKey, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, NULL, - TAOS_LRU_PRIORITY_LOW, pTsdb); - if (status != TAOS_LRU_STATUS_OK) { - tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); - code = TSDB_CODE_FAILED; - pLastCol = NULL; - } - -_exit: - if (TSDB_CODE_SUCCESS != code) { - taosMemoryFree(pLastCol); + code = tsdbCachePutToLRU(pTsdb, pLastKey, &emptyCol, 1); + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); } TAOS_RETURN(code); @@ -723,9 +719,13 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, { SLastCol *pLastCol = NULL; code = tsdbCacheDeserialize(values_list[0], values_list_sizes[0], &pLastCol); - if (code != TSDB_CODE_SUCCESS) { - tsdbWarn("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); + if (code == TSDB_CODE_INVALID_PARA) { + tsdbTrace("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + } else if (code != TSDB_CODE_SUCCESS) { + tsdbError("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + goto _exit; } if (NULL != pLastCol) { rocksdb_writebatch_delete(wb, keys_list[0], klen); @@ -734,9 +734,13 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, pLastCol = NULL; code = tsdbCacheDeserialize(values_list[1], values_list_sizes[1], &pLastCol); - if (code != TSDB_CODE_SUCCESS) { - tsdbWarn("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); + if (code == TSDB_CODE_INVALID_PARA) { + tsdbTrace("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + } else if (code != TSDB_CODE_SUCCESS) { + tsdbError("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + goto _exit; } if (NULL != pLastCol) { rocksdb_writebatch_delete(wb, keys_list[1], klen); @@ -749,9 +753,7 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, for (int i = 0; i < 2; i++) { LRUHandle *h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[i], klen); if (h) { - if (taosLRUCacheRelease(pTsdb->lruCache, h, true)) { - tsdbInfo("vgId:%d, %s release lru cache failed at line %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__); - } + tsdbLRUCacheRelease(pTsdb->lruCache, h, true); taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen); } } @@ -780,17 +782,13 @@ int32_t tsdbCacheNewTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWrap code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, LFLAG_LAST_ROW); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, LFLAG_LAST); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } } } else { @@ -808,17 +806,13 @@ int32_t tsdbCacheNewTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWrap code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, LFLAG_LAST_ROW); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, LFLAG_LAST); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } } @@ -837,10 +831,8 @@ int32_t tsdbCacheDropTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWra code = tsdbCacheCommitNoLock(pTsdb); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } if (pSchemaRow != NULL) { @@ -855,10 +847,8 @@ int32_t tsdbCacheDropTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWra code = tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } } } else { @@ -881,10 +871,8 @@ int32_t tsdbCacheDropTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWra code = tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } } @@ -905,10 +893,8 @@ int32_t tsdbCacheDropSubTables(STsdb *pTsdb, SArray *uids, tb_uid_t suid) { code = tsdbCacheCommitNoLock(pTsdb); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } STSchema *pTSchema = NULL; @@ -934,11 +920,8 @@ int32_t tsdbCacheDropSubTables(STsdb *pTsdb, SArray *uids, tb_uid_t suid) { code = tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - taosMemoryFree(pTSchema); - TAOS_RETURN(code); } } } @@ -959,17 +942,13 @@ int32_t tsdbCacheNewNTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, int8_t code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, 0); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, 1); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } // rocksMayWrite(pTsdb, true, false, false); (void)taosThreadMutexUnlock(&pTsdb->lruMutex); @@ -984,18 +963,14 @@ int32_t tsdbCacheDropNTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, bool h code = tsdbCacheCommitNoLock(pTsdb); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } code = tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } rocksMayWrite(pTsdb, false); @@ -1015,17 +990,13 @@ int32_t tsdbCacheNewSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, int8_t code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, 0); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } code = tsdbCacheNewTableColumn(pTsdb, uid, cid, col_type, 1); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s new table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } } @@ -1041,10 +1012,8 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, bool code = tsdbCacheCommitNoLock(pTsdb); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s commit with no lock failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } for (int i = 0; i < TARRAY_SIZE(uids); ++i) { @@ -1052,10 +1021,8 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, bool code = tsdbCacheDropTableColumn(pTsdb, uid, cid, hasPrimayKey); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s drop table column failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } } @@ -1071,40 +1038,6 @@ typedef struct { SLastKey key; } SIdxKey; -static int32_t tsdbCacheUpdateValue(SValue *pOld, SValue *pNew) { - uint8_t *pFree = NULL; - int nData = 0; - - if (IS_VAR_DATA_TYPE(pOld->type)) { - pFree = pOld->pData; - nData = pOld->nData; - } - - *pOld = *pNew; - if (IS_VAR_DATA_TYPE(pNew->type)) { - if (nData < pNew->nData) { - pOld->pData = taosMemoryCalloc(1, pNew->nData); - if (!pOld->pData) { - return terrno; - } - } else { - pOld->pData = pFree; - pFree = NULL; - } - - if (pNew->nData) { - memcpy(pOld->pData, pNew->pData, pNew->nData); - } else { - pFree = pOld->pData; - pOld->pData = NULL; - } - } - - taosMemoryFreeClear(pFree); - - TAOS_RETURN(TSDB_CODE_SUCCESS); -} - static void tsdbCacheUpdateLastColToNone(SLastCol *pLastCol, ELastCacheStatus cacheStatus) { // update rowkey pLastCol->rowKey.ts = TSKEY_MIN; @@ -1128,11 +1061,7 @@ static void tsdbCacheUpdateLastColToNone(SLastCol *pLastCol, ELastCacheStatus ca } pLastCol->colVal = COL_VAL_NONE(pLastCol->colVal.cid, pLastCol->colVal.value.type); - - if (!pLastCol->dirty) { - pLastCol->dirty = 1; - } - + pLastCol->dirty = 1; pLastCol->cacheStatus = cacheStatus; } @@ -1155,7 +1084,7 @@ static int32_t tsdbCachePutToRocksdb(STsdb *pTsdb, SLastKey *pLastKey, SLastCol TAOS_RETURN(code); } -static int32_t tsdbCachePutToLRU(STsdb *pTsdb, SLastKey *pLastKey, SLastCol *pLastCol) { +static int32_t tsdbCachePutToLRU(STsdb *pTsdb, SLastKey *pLastKey, SLastCol *pLastCol, int8_t dirty) { int32_t code = 0, lino = 0; SLastCol *pLRULastCol = taosMemoryCalloc(1, sizeof(SLastCol)); @@ -1165,11 +1094,11 @@ static int32_t tsdbCachePutToLRU(STsdb *pTsdb, SLastKey *pLastKey, SLastCol *pLa size_t charge = 0; *pLRULastCol = *pLastCol; - pLRULastCol->dirty = 1; + pLRULastCol->dirty = dirty; TAOS_CHECK_EXIT(tsdbCacheReallocSLastCol(pLRULastCol, &charge)); LRUStatus status = taosLRUCacheInsert(pTsdb->lruCache, pLastKey, ROCKS_KEY_LEN, pLRULastCol, charge, tsdbCacheDeleter, - NULL, TAOS_LRU_PRIORITY_LOW, pTsdb); + tsdbCacheOverWriter, NULL, TAOS_LRU_PRIORITY_LOW, pTsdb); if (TAOS_LRU_STATUS_OK != status && TAOS_LRU_STATUS_OK_OVERWRITTEN != status) { tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); code = TSDB_CODE_FAILED; @@ -1216,14 +1145,13 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray if (pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { int32_t cmp_res = tRowKeyCompare(&pLastCol->rowKey, pRowKey); if (cmp_res < 0 || (cmp_res == 0 && !COL_VAL_IS_NONE(pColVal))) { - SLastCol newLastCol = {.rowKey = *pRowKey, .colVal = *pColVal, .cacheStatus = TSDB_LAST_CACHE_VALID}; - code = tsdbCachePutToLRU(pTsdb, key, &newLastCol); + SLastCol newLastCol = { + .rowKey = *pRowKey, .colVal = *pColVal, .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_VALID}; + code = tsdbCachePutToLRU(pTsdb, key, &newLastCol, 1); } } - if (!taosLRUCacheRelease(pCache, h, false)) { - tsdbInfo("vgId:%d, %s release lru cache failed at line %d", TD_VID(pTsdb->pVnode), __func__, __LINE__); - } + tsdbLRUCacheRelease(pCache, h, false); TAOS_CHECK_EXIT(code); } else { if (!remainCols) { @@ -1284,9 +1212,13 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray SLastCol *pLastCol = NULL; code = tsdbCacheDeserialize(values_list[i], values_list_sizes[i], &pLastCol); - if (code != TSDB_CODE_SUCCESS) { - tsdbWarn("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); + if (code == TSDB_CODE_INVALID_PARA) { + tsdbTrace("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + } else if (code != TSDB_CODE_SUCCESS) { + tsdbError("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + goto _exit; } /* if (code) { @@ -1296,7 +1228,7 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray SLastCol *pToFree = pLastCol; if (pLastCol && pLastCol->cacheStatus == TSDB_LAST_CACHE_NO_CACHE) { - if ((code = tsdbCachePutToLRU(pTsdb, &idxKey->key, pLastCol)) != TSDB_CODE_SUCCESS) { + if ((code = tsdbCachePutToLRU(pTsdb, &idxKey->key, pLastCol, 0)) != TSDB_CODE_SUCCESS) { tsdbError("tsdb/cache: vgId:%d, put lru failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, tstrerror(code)); taosMemoryFreeClear(pToFree); @@ -1319,14 +1251,14 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray } if (NULL == pLastCol || cmp_res < 0 || (cmp_res == 0 && !COL_VAL_IS_NONE(pColVal))) { - SLastCol lastColTmp = {.rowKey = *pRowKey, .colVal = *pColVal, .cacheStatus = TSDB_LAST_CACHE_VALID}; + SLastCol lastColTmp = {.rowKey = *pRowKey, .colVal = *pColVal, .dirty = 0, .cacheStatus = TSDB_LAST_CACHE_VALID}; if ((code = tsdbCachePutToRocksdb(pTsdb, &idxKey->key, &lastColTmp)) != TSDB_CODE_SUCCESS) { tsdbError("tsdb/cache: vgId:%d, put rocks failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, tstrerror(code)); taosMemoryFreeClear(pToFree); break; } - if ((code = tsdbCachePutToLRU(pTsdb, &idxKey->key, &lastColTmp)) != TSDB_CODE_SUCCESS) { + if ((code = tsdbCachePutToLRU(pTsdb, &idxKey->key, &lastColTmp, 0)) != TSDB_CODE_SUCCESS) { tsdbError("tsdb/cache: vgId:%d, put lru failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, tstrerror(code)); taosMemoryFreeClear(pToFree); @@ -1438,9 +1370,8 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 } code = tSimpleHashIterateRemove(iColHash, &iCol, sizeof(iCol), &pIte, &iter); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s tSimpleHashIterateRemove failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, + tsdbTrace("vgId:%d, %s tSimpleHashIterateRemove failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - TAOS_CHECK_GOTO(code, &lino, _exit); } } } @@ -1449,9 +1380,8 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 // 3. do update code = tsdbCacheUpdate(pTsdb, suid, uid, ctxArray); if (code < TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s tsdbCacheUpdate failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s tsdbCacheUpdate failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - TAOS_CHECK_GOTO(code, &lino, _exit); } _exit: @@ -1474,6 +1404,9 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo TAOS_CHECK_RETURN(metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema)); ctxArray = taosArrayInit(pBlockData->nColData, sizeof(SLastUpdateCtx)); + if (ctxArray == NULL) { + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } // 1. prepare last STsdbRowKey tsdbRowKey = {0}; @@ -1535,9 +1468,8 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo // 3. do update code = tsdbCacheUpdate(pTsdb, suid, uid, ctxArray); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s tsdbCacheUpdate failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s tsdbCacheUpdate failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - TAOS_CHECK_GOTO(code, &lino, _exit); } _exit: @@ -1678,30 +1610,14 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr continue; } - SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); - if (!pTmpLastCol) { - TAOS_CHECK_EXIT(terrno); - } - - size_t charge = 0; - *pTmpLastCol = *pLastCol; - pLastCol = pTmpLastCol; - code = tsdbCacheReallocSLastCol(pLastCol, &charge); - if (TSDB_CODE_SUCCESS != code) { - taosMemoryFree(pLastCol); + // store result back to rocks cache + code = tsdbCachePutToRocksdb(pTsdb, &idxKey->key, pLastCol); + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s.", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); TAOS_CHECK_EXIT(code); } - LRUStatus status = taosLRUCacheInsert(pCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, NULL, - TAOS_LRU_PRIORITY_LOW, pTsdb); - if (TAOS_LRU_STATUS_OK != status && TAOS_LRU_STATUS_OK_OVERWRITTEN != status) { - tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); - pLastCol = NULL; - TAOS_CHECK_EXIT(TSDB_CODE_FAILED); - } - - // store result back to rocks cache - code = tsdbCachePutToRocksdb(pTsdb, &idxKey->key, pLastCol); + code = tsdbCachePutToLRU(pTsdb, &idxKey->key, pLastCol, 0); if (code) { tsdbError("vgId:%d, %s failed at line %d since %s.", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); TAOS_CHECK_EXIT(code); @@ -1769,25 +1685,21 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA } code = tsdbCacheDeserialize(values_list[i], values_list_sizes[i], &pLastCol); - if (code != TSDB_CODE_SUCCESS) { - tsdbWarn("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); + if (code == TSDB_CODE_INVALID_PARA) { + tsdbTrace("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + } else if (code != TSDB_CODE_SUCCESS) { + tsdbError("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + goto _exit; } SLastCol *pToFree = pLastCol; SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[j]; if (pLastCol && pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { - SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); - if (!pTmpLastCol) { - taosMemoryFreeClear(pToFree); - TAOS_CHECK_EXIT(terrno); - } - - size_t charge = 0; - *pTmpLastCol = *pLastCol; - pLastCol = pTmpLastCol; - code = tsdbCacheReallocSLastCol(pLastCol, &charge); - if (TSDB_CODE_SUCCESS != code) { - taosMemoryFreeClear(pLastCol); + code = tsdbCachePutToLRU(pTsdb, &idxKey->key, pLastCol, 0); + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); taosMemoryFreeClear(pToFree); TAOS_CHECK_EXIT(code); } @@ -1795,20 +1707,10 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA SLastCol lastCol = *pLastCol; code = tsdbCacheReallocSLastCol(&lastCol, NULL); if (TSDB_CODE_SUCCESS != code) { - tsdbCacheFreeSLastColItem(pLastCol); - taosMemoryFreeClear(pLastCol); taosMemoryFreeClear(pToFree); TAOS_CHECK_EXIT(code); } - LRUStatus status = taosLRUCacheInsert(pCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, - NULL, TAOS_LRU_PRIORITY_LOW, pTsdb); - if (TAOS_LRU_STATUS_OK != status && TAOS_LRU_STATUS_OK_OVERWRITTEN != status) { - tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); - taosMemoryFreeClear(pToFree); - TAOS_CHECK_EXIT(TSDB_CODE_FAILED); - } - taosArraySet(pLastArray, idxKey->idx, &lastCol); taosArrayRemove(remainCols, j); taosArrayRemove(ignoreFromRocks, j); @@ -1905,11 +1807,7 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache } if (h) { - code = taosLRUCacheRelease(pCache, h, false); - if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s release lru cache failed at line %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__); - goto _exit; - } + tsdbLRUCacheRelease(pCache, h, false); } } @@ -1936,13 +1834,8 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache // no cache or cache is invalid ++i; } - if (h) { - code = taosLRUCacheRelease(pCache, h, false); - if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s release lru cache failed at line %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__); - goto _exit; - } + tsdbLRUCacheRelease(pCache, h, false); } } @@ -1978,10 +1871,8 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE code = tsdbCacheCommit(pTsdb); if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s commit failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tsdbTrace("vgId:%d, %s commit failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_RETURN(code); } (void)taosThreadMutexLock(&pTsdb->lruMutex); @@ -1996,12 +1887,11 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE if (pLastCol->rowKey.ts <= eKey && pLastCol->rowKey.ts >= sKey) { SLastCol noneCol = {.rowKey.ts = TSKEY_MIN, .colVal = COL_VAL_NONE(cid, pTSchema->columns[i].type), + .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_NO_CACHE}; - code = tsdbCachePutToLRU(pTsdb, &lastKey, &noneCol); - } - if (taosLRUCacheRelease(pTsdb->lruCache, h, false) != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s release lru cache failed at line %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__); + code = tsdbCachePutToLRU(pTsdb, &lastKey, &noneCol, 1); } + tsdbLRUCacheRelease(pTsdb->lruCache, h, false); TAOS_CHECK_EXIT(code); } else { if (!remainCols) { @@ -2053,15 +1943,20 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE for (int i = 0; i < numKeys; ++i) { SLastCol *pLastCol = NULL; code = tsdbCacheDeserialize(values_list[i], values_list_sizes[i], &pLastCol); - if (code != TSDB_CODE_SUCCESS) { - tsdbWarn("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); + if (code == TSDB_CODE_INVALID_PARA) { + tsdbTrace("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + } else if (code != TSDB_CODE_SUCCESS) { + tsdbError("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + goto _exit; } SIdxKey *idxKey = taosArrayGet(remainCols, i); SLastKey *pLastKey = &idxKey->key; if (NULL != pLastCol && (pLastCol->rowKey.ts <= eKey && pLastCol->rowKey.ts >= sKey)) { SLastCol noCacheCol = {.rowKey.ts = TSKEY_MIN, .colVal = COL_VAL_NONE(pLastKey->cid, pTSchema->columns[idxKey->idx].type), + .dirty = 0, .cacheStatus = TSDB_LAST_CACHE_NO_CACHE}; if ((code = tsdbCachePutToRocksdb(pTsdb, pLastKey, &noCacheCol)) != TSDB_CODE_SUCCESS) { @@ -2069,7 +1964,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE tsdbError("tsdb/cache/del: vgId:%d, put to rocks failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); goto _exit; } - if ((code = tsdbCachePutToLRU(pTsdb, pLastKey, &noCacheCol)) != TSDB_CODE_SUCCESS) { + if ((code = tsdbCachePutToLRU(pTsdb, pLastKey, &noCacheCol, 0)) != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pLastCol); tsdbError("tsdb/cache/del: vgId:%d, put to lru failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); goto _exit; @@ -3179,7 +3074,7 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI } if (!taosArrayAddAll(pInfo->pTombData, pIter->pMemDelData)) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err); + TAOS_CHECK_GOTO(terrno, &lino, _err); } size_t delSize = TARRAY_SIZE(pInfo->pTombData); @@ -3559,11 +3454,7 @@ _err: TAOS_RETURN(code); } -void tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h) { - if (taosLRUCacheRelease(pCache, h, false)) { - tsdbError("%s release lru cache failed at line %d.", __func__, __LINE__); - } -} +void tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h) { tsdbLRUCacheRelease(pCache, h, false); } void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity) { taosLRUCacheSetCapacity(pVnode->pTsdb->lruCache, capacity); @@ -3657,7 +3548,7 @@ int32_t tsdbCacheGetBlockS3(SLRUCache *pCache, STsdbFD *pFD, LRUHandle **handle) size_t charge = tsS3BlockSize * pFD->szPage; _taos_lru_deleter_t deleter = deleteBCache; LRUStatus status = - taosLRUCacheInsert(pCache, key, keyLen, pBlock, charge, deleter, &h, TAOS_LRU_PRIORITY_LOW, NULL); + taosLRUCacheInsert(pCache, key, keyLen, pBlock, charge, deleter, NULL, &h, TAOS_LRU_PRIORITY_LOW, NULL); if (status != TAOS_LRU_STATUS_OK) { // code = -1; } @@ -3682,8 +3573,7 @@ int32_t tsdbCacheGetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, LRUHan return code; } -int32_t tsdbCacheSetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, uint8_t *pPage) { - int32_t code = 0; +void tsdbCacheSetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, uint8_t *pPage) { char key[128] = {0}; int keyLen = 0; LRUHandle *handle = NULL; @@ -3696,12 +3586,12 @@ int32_t tsdbCacheSetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, uint8_ _taos_lru_deleter_t deleter = deleteBCache; uint8_t *pPg = taosMemoryMalloc(charge); if (!pPg) { - TAOS_RETURN(terrno); + return; // ignore error with s3 cache and leave error untouched } memcpy(pPg, pPage, charge); LRUStatus status = - taosLRUCacheInsert(pCache, key, keyLen, pPg, charge, deleter, &handle, TAOS_LRU_PRIORITY_LOW, NULL); + taosLRUCacheInsert(pCache, key, keyLen, pPg, charge, deleter, NULL, &handle, TAOS_LRU_PRIORITY_LOW, NULL); if (status != TAOS_LRU_STATUS_OK) { // ignore cache updating if not ok // code = TSDB_CODE_OUT_OF_MEMORY; @@ -3710,6 +3600,4 @@ int32_t tsdbCacheSetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, uint8_ (void)taosThreadMutexUnlock(&pFD->pTsdb->pgMutex); tsdbCacheRelease(pFD->pTsdb->pgCache, handle); - - TAOS_RETURN(code); -} +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 32371dc399..95c5daf842 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -199,9 +199,9 @@ _exit: return code; } -static int32_t tsdbCommitCloseReader(SCommitter2 *committer) { +static void tsdbCommitCloseReader(SCommitter2 *committer) { TARRAY2_CLEAR(committer->sttReaderArray, tsdbSttFileReaderClose); - return 0; + return; } static int32_t tsdbCommitOpenReader(SCommitter2 *committer) { @@ -243,19 +243,19 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) { _exit: if (code) { - TAOS_UNUSED(tsdbCommitCloseReader(committer)); + tsdbCommitCloseReader(committer); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } return code; } -static int32_t tsdbCommitCloseIter(SCommitter2 *committer) { +static void tsdbCommitCloseIter(SCommitter2 *committer) { tsdbIterMergerClose(&committer->tombIterMerger); tsdbIterMergerClose(&committer->dataIterMerger); TARRAY2_CLEAR(committer->tombIterArray, tsdbIterClose); TARRAY2_CLEAR(committer->dataIterArray, tsdbIterClose); - return 0; + return; } static int32_t tsdbCommitOpenIter(SCommitter2 *committer) { @@ -309,7 +309,7 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) { _exit: if (code) { - TAOS_UNUSED(tsdbCommitCloseIter(committer)); + tsdbCommitCloseIter(committer); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } @@ -322,7 +322,7 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { STsdb *tsdb = committer->tsdb; // check if can commit - TAOS_UNUSED(tsdbFSCheckCommit(tsdb, committer->ctx->info->fid)); + tsdbFSCheckCommit(tsdb, committer->ctx->info->fid); committer->ctx->expLevel = tsdbFidLevel(committer->ctx->info->fid, &tsdb->keepCfg, committer->now); tsdbFidKeyRange(committer->ctx->info->fid, committer->minutes, committer->precision, &committer->ctx->minKey, @@ -331,7 +331,9 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { TAOS_CHECK_GOTO(tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, &committer->ctx->did), &lino, _exit); - TAOS_UNUSED(tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did)); + if (tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did) != 0) { + tsdbError("vgId:%d failed to create directory %s", TD_VID(committer->tsdb->pVnode), committer->tsdb->path); + } committer->ctx->tbid->suid = 0; committer->ctx->tbid->uid = 0; @@ -355,8 +357,8 @@ static int32_t tsdbCommitFileSetEnd(SCommitter2 *committer) { int32_t lino = 0; TAOS_CHECK_GOTO(tsdbCommitCloseWriter(committer), &lino, _exit); - TAOS_CHECK_GOTO(tsdbCommitCloseIter(committer), &lino, _exit); - TAOS_CHECK_GOTO(tsdbCommitCloseReader(committer), &lino, _exit); + tsdbCommitCloseIter(committer); + tsdbCommitCloseReader(committer); _exit: if (code) { @@ -409,11 +411,11 @@ static uint32_t tFileSetCommitInfoHash(const void *arg) { return MurmurHash3_32((const char *)&info->fid, sizeof(info->fid)); } -static int32_t tsdbCommitInfoDestroy(STsdb *pTsdb) { +static void tsdbCommitInfoDestroy(STsdb *pTsdb) { if (pTsdb->commitInfo) { for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) { SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i); - TAOS_UNUSED(vHashDrop(pTsdb->commitInfo->ht, info)); + int32_t ret = vHashDrop(pTsdb->commitInfo->ht, info); tsdbTFileSetClear(&info->fset); taosMemoryFree(info); } @@ -423,7 +425,7 @@ static int32_t tsdbCommitInfoDestroy(STsdb *pTsdb) { pTsdb->commitInfo->arr = NULL; taosMemoryFreeClear(pTsdb->commitInfo); } - return 0; + return; } static int32_t tsdbCommitInfoInit(STsdb *pTsdb) { @@ -444,7 +446,7 @@ static int32_t tsdbCommitInfoInit(STsdb *pTsdb) { _exit: if (code) { - TAOS_UNUSED(tsdbCommitInfoDestroy(pTsdb)); + tsdbCommitInfoDestroy(pTsdb); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } return code; @@ -514,7 +516,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) { SFileSetCommitInfo tinfo = { .fid = fid, }; - TAOS_UNUSED(vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info)); + int32_t ret = vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info); if (info == NULL) { TAOS_CHECK_GOTO(tsdbCommitInfoAdd(tsdb, fid), &lino, _exit); } @@ -538,7 +540,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) { }; // check if the file set already on the commit list - TAOS_UNUSED(vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info)); + int32_t ret = vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info); if (info != NULL) { continue; } @@ -586,7 +588,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) { _exit: if (code) { - TAOS_UNUSED(tsdbCommitInfoDestroy(tsdb)); + tsdbCommitInfoDestroy(tsdb); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } return code; @@ -716,7 +718,7 @@ int32_t tsdbCommitCommit(STsdb *tsdb) { (void)taosThreadMutexUnlock(&tsdb->mutex); - TAOS_UNUSED(tsdbCommitInfoDestroy(tsdb)); + tsdbCommitInfoDestroy(tsdb); tsdbUnrefMemTable(pMemTable, NULL, true); } @@ -745,7 +747,7 @@ int32_t tsdbCommitAbort(STsdb *pTsdb) { } } (void)taosThreadMutexUnlock(&pTsdb->mutex); - TAOS_UNUSED(tsdbCommitInfoDestroy(pTsdb)); + tsdbCommitInfoDestroy(pTsdb); _exit: if (code) { diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index 5a2f9628fb..720ba68414 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -1362,7 +1362,7 @@ int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAl }; for (int i = 0; i < TOMB_BLOCK_SIZE(tombBlock); i++) { STombRecord record; - TAOS_UNUSED(tTombBlockGet(tombBlock, i, &record)); + TAOS_CHECK_RETURN(tTombBlockGet(tombBlock, i, &record)); if (i == 0) { tombBlk.minTbid.suid = record.suid; @@ -1519,7 +1519,7 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom while (writer->ctx->hasOldTomb) { for (; writer->ctx->tombBlockIdx < TOMB_BLOCK_SIZE(writer->ctx->tombBlock); writer->ctx->tombBlockIdx++) { STombRecord record1[1]; - TAOS_UNUSED(tTombBlockGet(writer->ctx->tombBlock, writer->ctx->tombBlockIdx, record1)); + TAOS_CHECK_GOTO(tTombBlockGet(writer->ctx->tombBlock, writer->ctx->tombBlockIdx, record1), &lino, _exit); int32_t c = tTombRecordCompare(record, record1); if (c < 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 19e1f42726..a949fd69f0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -130,7 +130,7 @@ _exit: tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); } taosMemoryFree(pData); - (void)taosCloseFile(&pFD); + taosCloseFileWithLog(&pFD); return code; } @@ -300,26 +300,26 @@ static int32_t load_fs(const char *fname, STsdbFS *pFS) { int64_t size; code = taosFStatFile(pFD, &size, NULL); if (code != 0) { - (void)taosCloseFile(&pFD); + taosCloseFileWithLog(&pFD); TSDB_CHECK_CODE(code, lino, _exit); } pData = taosMemoryMalloc(size); if (pData == NULL) { code = terrno; - (void)taosCloseFile(&pFD); + taosCloseFileWithLog(&pFD); TSDB_CHECK_CODE(code, lino, _exit); } if (taosReadFile(pFD, pData, size) < 0) { code = terrno; - (void)taosCloseFile(&pFD); + taosCloseFileWithLog(&pFD); TSDB_CHECK_CODE(code, lino, _exit); } if (!taosCheckChecksumWhole(pData, size)) { code = TSDB_CODE_FILE_CORRUPTED; - (void)taosCloseFile(&pFD); + taosCloseFileWithLog(&pFD); TSDB_CHECK_CODE(code, lino, _exit); } @@ -331,7 +331,7 @@ _exit: tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); } taosMemoryFree(pData); - (void)taosCloseFile(&pFD); + taosCloseFileWithLog(&pFD); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index bcaa70f9a8..d3b783847c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -44,7 +44,12 @@ static int32_t create_fs(STsdb *pTsdb, STFileSystem **fs) { } fs[0]->tsdb = pTsdb; - (void)tsem_init(&fs[0]->canEdit, 0, 1); + int32_t code = tsem_init(&fs[0]->canEdit, 0, 1); + if (code) { + taosMemoryFree(fs[0]); + return code; + } + fs[0]->fsstate = TSDB_FS_STATE_NORMAL; fs[0]->neid = 0; TARRAY2_INIT(fs[0]->fSetArr); @@ -58,7 +63,9 @@ static void destroy_fs(STFileSystem **fs) { TARRAY2_DESTROY(fs[0]->fSetArr, NULL); TARRAY2_DESTROY(fs[0]->fSetArrTmp, NULL); - (void)tsem_destroy(&fs[0]->canEdit); + if (tsem_destroy(&fs[0]->canEdit) != 0) { + tsdbError("failed to destroy semaphore"); + } taosMemoryFree(fs[0]); fs[0] = NULL; } @@ -100,7 +107,7 @@ _exit: tsdbError("%s failed at %s:%d since %s", __func__, fname, __LINE__, tstrerror(code)); } taosMemoryFree(data); - (void)taosCloseFile(&fp); + taosCloseFileWithLog(&fp); return code; } @@ -140,7 +147,7 @@ _exit: tsdbError("%s failed at %s:%d since %s", __func__, fname, __LINE__, tstrerror(code)); json[0] = NULL; } - (void)taosCloseFile(&fp); + taosCloseFileWithLog(&fp); taosMemoryFree(data); return code; } @@ -803,7 +810,11 @@ void tsdbEnableBgTask(STsdb *pTsdb) { void tsdbCloseFS(STFileSystem **fs) { if (fs[0] == NULL) return; - TAOS_UNUSED(tsdbDisableAndCancelAllBgTask((*fs)->tsdb)); + int32_t code = tsdbDisableAndCancelAllBgTask((*fs)->tsdb); + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID((*fs)->tsdb->pVnode), __func__, __LINE__, + tstrerror(code)); + } close_file_system(fs[0]); destroy_fs(fs); return; @@ -833,7 +844,9 @@ int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT e current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M); } - (void)tsem_wait(&fs->canEdit); + if (tsem_wait(&fs->canEdit) != 0) { + tsdbError("vgId:%d failed to wait semaphore", TD_VID(fs->tsdb->pVnode)); + } fs->etype = etype; // edit @@ -865,7 +878,7 @@ static void tsdbFSSetBlockCommit(STFileSet *fset, bool block) { } } -int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid) { +void tsdbFSCheckCommit(STsdb *tsdb, int32_t fid) { (void)taosThreadMutexLock(&tsdb->mutex); STFileSet *fset; tsdbFSGetFSet(tsdb->pFS, fid, &fset); @@ -877,7 +890,7 @@ int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid) { } } (void)taosThreadMutexUnlock(&tsdb->mutex); - return 0; + return; } // IMPORTANT: the caller must hold fs->tsdb->mutex @@ -939,13 +952,17 @@ _exit: } else { tsdbInfo("vgId:%d %s done, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype); } - (void)tsem_post(&fs->canEdit); + if (tsem_post(&fs->canEdit) != 0) { + tsdbError("vgId:%d failed to post semaphore", TD_VID(fs->tsdb->pVnode)); + } return code; } int32_t tsdbFSEditAbort(STFileSystem *fs) { int32_t code = abort_edit(fs); - (void)tsem_post(&fs->canEdit); + if (tsem_post(&fs->canEdit) != 0) { + tsdbError("vgId:%d failed to post semaphore", TD_VID(fs->tsdb->pVnode)); + } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 9993c1e33d..119015636b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -59,10 +59,10 @@ int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT e int32_t tsdbFSEditCommit(STFileSystem *fs); int32_t tsdbFSEditAbort(STFileSystem *fs); // other -void tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset); -int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid); -void tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset); -void tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid); +void tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset); +void tsdbFSCheckCommit(STsdb *tsdb, int32_t fid); +void tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset); +void tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid); // utils int32_t save_fs(const TFileSetArray *arr, const char *fname); void current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index fc78fec2ea..fc681f9753 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -71,6 +71,9 @@ static int32_t tsdbSttLvlInitRef(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl **lv } code = TARRAY2_APPEND(lvl[0]->fobjArr, fobj1); if (code) { + if (tsdbTFileObjUnref(fobj1) != 0) { + tsdbError("failed to unref file obj, fobj:%p", fobj1); + } tsdbSttLvlClear(lvl); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbIter.c b/source/dnode/vnode/src/tsdb/tsdbIter.c index 4c5a803f22..3ed3f72333 100644 --- a/source/dnode/vnode/src/tsdb/tsdbIter.c +++ b/source/dnode/vnode/src/tsdb/tsdbIter.c @@ -225,7 +225,7 @@ static int32_t tsdbMemTableIterNext(STsdbIter *iter, const TABLEID *tbid) { iter->row->row = row[0]; - TAOS_UNUSED(tsdbTbDataIterNext(iter->memtData->tbIter)); + bool r = tsdbTbDataIterNext(iter->memtData->tbIter); goto _exit; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 8b4cdf9367..eb22335311 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -193,7 +193,11 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pMemTable->minVer = TMIN(pMemTable->minVer, version); pMemTable->maxVer = TMAX(pMemTable->maxVer, version); - TAOS_UNUSED(tsdbCacheDel(pTsdb, suid, uid, sKey, eKey)); + if (tsdbCacheDel(pTsdb, suid, uid, sKey, eKey) != 0) { + tsdbError("vgId:%d, failed to delete cache data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 + " eKey:%" PRId64 " at version %" PRId64, + TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, version); + } tsdbTrace("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 " at version %" PRId64, @@ -652,7 +656,10 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, } if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) { - TAOS_UNUSED(tsdbCacheColFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, pBlockData)); + if (tsdbCacheColFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, pBlockData) != 0) { + tsdbError("vgId:%d, failed to update cache data from table suid:%" PRId64 " uid:%" PRId64 " at version %" PRId64, + TD_VID(pMemTable->pTsdb->pVnode), pTbData->suid, pTbData->uid, version); + } } // SMemTable diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 85ed223c6c..3e4208fc54 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -403,7 +403,8 @@ static int32_t loadSttStatisticsBlockData(SSttFileReader *pSttFileReader, SSttBl pBlockLoadInfo->cost.loadStatisBlocks += num; STbStatisBlock block; - TAOS_UNUSED(tStatisBlockInit(&block)); + code = tStatisBlockInit(&block); + QUERY_CHECK_CODE(code, lino, _end); int64_t st = taosGetTimestampUs(); diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 1923c8bafc..c1f8f45d7e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -102,7 +102,7 @@ _exit: return code; } -int32_t tsdbClose(STsdb **pTsdb) { +void tsdbClose(STsdb **pTsdb) { if (*pTsdb) { STsdb *pdb = *pTsdb; tsdbDebug("vgId:%d, tsdb is close at %s, days:%d, keep:%d,%d,%d, keepTimeOffset:%d", TD_VID(pdb->pVnode), pdb->path, @@ -121,5 +121,5 @@ int32_t tsdbClose(STsdb **pTsdb) { (void)taosThreadMutexDestroy(&(*pTsdb)->mutex); taosMemoryFreeClear(*pTsdb); } - return 0; + return; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 0e958b155b..36bfb56120 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -299,7 +299,7 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); if (pReader->status.pLDataIterArray == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } // check file the time range of coverage @@ -556,7 +556,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void pReader->idStr = (idstr != NULL) ? taosStrdup(idstr) : NULL; if (idstr != NULL && pReader->idStr == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _end; } @@ -704,7 +704,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead pReader->cost.headFileLoadTime += (et1 - st) / 1000.0; -_end: +//_end: // tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); return code; } @@ -1872,9 +1872,9 @@ static void doPinSttBlock(SSttBlockReader* pSttBlockReader) { tMergeTreePinSttBl static void doUnpinSttBlock(SSttBlockReader* pSttBlockReader) { tMergeTreeUnpinSttBlock(&pSttBlockReader->mergeTree); } -static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SSttBlockReader* pSttBlockReader, - STableBlockScanInfo* pScanInfo, SRowKey* pSttKey, STsdbReader* pReader, - bool* copied) { +static int32_t tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SSttBlockReader* pSttBlockReader, + STableBlockScanInfo* pScanInfo, SRowKey* pSttKey, STsdbReader* pReader, + bool* copied) { int32_t code = TSDB_CODE_SUCCESS; *copied = false; @@ -2918,7 +2918,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t orde } else { void* p1 = taosArrayAddAll(pSource, pBlockScanInfo->pMemDelData); if (p1 == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 74b7ebc06b..d867318e1c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -436,7 +436,7 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64 code = tsdbCacheGetPageS3(pFD->pTsdb->pgCache, pFD, pgno, &handle); if (code != TSDB_CODE_SUCCESS) { if (handle) { - (void)tsdbCacheRelease(pFD->pTsdb->pgCache, handle); + tsdbCacheRelease(pFD->pTsdb->pgCache, handle); } TSDB_CHECK_CODE(code, lino, _exit); } @@ -447,7 +447,7 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64 uint8_t *pPage = (uint8_t *)taosLRUCacheValue(pFD->pTsdb->pgCache, handle); memcpy(pFD->pBuf, pPage, pFD->szPage); - (void)tsdbCacheRelease(pFD->pTsdb->pgCache, handle); + tsdbCacheRelease(pFD->pTsdb->pgCache, handle); // check if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { @@ -483,7 +483,7 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64 int nPage = pgnoEnd - pgno + 1; for (int i = 0; i < nPage; ++i) { if (pFD->szFile != pgno) { // DONOT cache last volatile page - (void)tsdbCacheSetPageS3(pFD->pTsdb->pgCache, pFD, pgno, pBlock + i * pFD->szPage); + tsdbCacheSetPageS3(pFD->pTsdb->pgCache, pFD, pgno, pBlock + i * pFD->szPage); } if (szHint > 0 && n >= size) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 4b690cf53b..cbe2ab4b8e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -99,8 +99,12 @@ _exit: tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } - (void)taosCloseFile(&fdFrom); - (void)taosCloseFile(&fdTo); + if (taosCloseFile(&fdFrom) != 0) { + tsdbError("vgId:%d, failed to close file %s", TD_VID(rtner->tsdb->pVnode), fname_from); + } + if (taosCloseFile(&fdTo) != 0) { + tsdbError("vgId:%d, failed to close file %s", TD_VID(rtner->tsdb->pVnode), fname_to); + } return code; } @@ -136,8 +140,12 @@ _exit: tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } - (void)taosCloseFile(&fdFrom); - (void)taosCloseFile(&fdTo); + if (taosCloseFile(&fdFrom) != 0) { + tsdbTrace("vgId:%d, failed to close file", TD_VID(rtner->tsdb->pVnode)); + } + if (taosCloseFile(&fdTo) != 0) { + tsdbTrace("vgId:%d, failed to close file", TD_VID(rtner->tsdb->pVnode)); + } return code; } @@ -441,7 +449,9 @@ _exit: tsdbError("vgId:%d %s failed at line %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } - (void)taosCloseFile(&fdFrom); + if (taosCloseFile(&fdFrom) != 0) { + tsdbTrace("vgId:%d, failed to close file", TD_VID(rtner->tsdb->pVnode)); + } return code; } @@ -541,8 +551,13 @@ _exit: tsdbError("vgId:%d %s failed at line %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } - (void)taosCloseFile(&fdFrom); - (void)taosCloseFile(&fdTo); + if (taosCloseFile(&fdFrom) != 0) { + tsdbTrace("vgId:%d, failed to close file", TD_VID(rtner->tsdb->pVnode)); + } + + if (taosCloseFile(&fdTo) != 0) { + tsdbTrace("vgId:%d, failed to close file", TD_VID(rtner->tsdb->pVnode)); + } return code; } @@ -639,8 +654,12 @@ _exit: tsdbError("vgId:%d %s failed at line %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } - (void)taosCloseFile(&fdFrom); - (void)taosCloseFile(&fdTo); + if (taosCloseFile(&fdFrom) != 0) { + tsdbTrace("vgId:%d, failed to close file", TD_VID(rtner->tsdb->pVnode)); + } + if (taosCloseFile(&fdTo) != 0) { + tsdbTrace("vgId:%d, failed to close file", TD_VID(rtner->tsdb->pVnode)); + } return code; } @@ -672,7 +691,7 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) { if (/*lcn < 1 && */ taosCheckExistFile(fobj->fname)) { int32_t mtime = 0; int64_t size = 0; - (void)taosStatFile(fobj->fname, &size, &mtime, NULL); + int32_t r = taosStatFile(fobj->fname, &size, &mtime, NULL); if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) { if (pCfg->s3Compact && lcn < 0) { extern int32_t tsdbAsyncCompact(STsdb * tsdb, const STimeWindow *tw, bool sync); @@ -699,7 +718,9 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) { if (taosCheckExistFile(fname1)) { int32_t mtime = 0; int64_t size = 0; - (void)taosStatFile(fname1, &size, &mtime, NULL); + if (taosStatFile(fname1, &size, &mtime, NULL) != 0) { + tsdbError("vgId:%d, %s failed at %s:%d ", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, __LINE__); + } if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) { TAOS_CHECK_GOTO(tsdbMigrateDataFileLCS3(rtner, fobj, size, chunksize), &lino, _exit); } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index bd22beb52d..d0ea58c28a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -57,10 +57,10 @@ struct STsdbSnapReader { STombBlock tombBlock[1]; }; -static int32_t tsdbSnapReadFileSetCloseReader(STsdbSnapReader* reader) { +static void tsdbSnapReadFileSetCloseReader(STsdbSnapReader* reader) { TARRAY2_CLEAR(reader->sttReaderArr, tsdbSttFileReaderClose); tsdbDataFileReaderClose(&reader->dataReader); - return 0; + return; } static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) { @@ -112,7 +112,7 @@ static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) { _exit: if (code) { - TAOS_UNUSED(tsdbSnapReadFileSetCloseReader(reader)); + tsdbSnapReadFileSetCloseReader(reader); TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } return code; @@ -190,12 +190,12 @@ _exit: return code; } -static int32_t tsdbSnapReadFileSetCloseIter(STsdbSnapReader* reader) { +static void tsdbSnapReadFileSetCloseIter(STsdbSnapReader* reader) { tsdbIterMergerClose(&reader->dataIterMerger); tsdbIterMergerClose(&reader->tombIterMerger); TARRAY2_CLEAR(reader->dataIterArr, tsdbIterClose); TARRAY2_CLEAR(reader->tombIterArr, tsdbIterClose); - return 0; + return; } static int32_t tsdbSnapReadRangeBegin(STsdbSnapReader* reader) { @@ -222,8 +222,8 @@ _exit: } static int32_t tsdbSnapReadRangeEnd(STsdbSnapReader* reader) { - TAOS_UNUSED(tsdbSnapReadFileSetCloseIter(reader)); - TAOS_UNUSED(tsdbSnapReadFileSetCloseReader(reader)); + tsdbSnapReadFileSetCloseIter(reader); + tsdbSnapReadFileSetCloseReader(reader); reader->ctx->fsr = NULL; return 0; } @@ -373,7 +373,7 @@ static int32_t tsdbSnapReadTombData(STsdbSnapReader* reader, uint8_t** data) { int32_t lino = 0; SMetaInfo info; - TAOS_UNUSED(tTombBlockClear(reader->tombBlock)); + tTombBlockClear(reader->tombBlock); TABLEID tbid[1] = {0}; for (STombRecord* record; (record = tsdbIterMergerGetTombRecord(reader->tombIterMerger)) != NULL;) { @@ -463,7 +463,7 @@ void tsdbSnapReaderClose(STsdbSnapReader** reader) { tDestroyTSchema(reader[0]->skmTb->pTSchema); for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->buffers); ++i) { - TAOS_UNUSED(tBufferDestroy(reader[0]->buffers + i)); + tBufferDestroy(reader[0]->buffers + i); } taosMemoryFree(reader[0]); @@ -821,7 +821,9 @@ static int32_t tsdbSnapWriteFileSetBegin(STsdbSnapWriter* writer, int32_t fid) { code = TSDB_CODE_NO_AVAIL_DISK; TSDB_CHECK_CODE(code, lino, _exit); } - TAOS_UNUSED(tfsMkdirRecurAt(writer->tsdb->pVnode->pTfs, writer->tsdb->path, writer->ctx->did)); + if (tfsMkdirRecurAt(writer->tsdb->pVnode->pTfs, writer->tsdb->path, writer->ctx->did) != 0) { + tsdbError("vgId:%d failed to create directory %s", TD_VID(writer->tsdb->pVnode), writer->tsdb->path); + } writer->ctx->hasData = true; writer->ctx->hasTomb = true; @@ -1000,7 +1002,7 @@ static int32_t tsdbSnapWriteDecmprTombBlock(SSnapDataHdr* hdr, STombBlock* tombB int32_t code = 0; int32_t lino = 0; - TAOS_UNUSED(tTombBlockClear(tombBlock)); + tTombBlockClear(tombBlock); int64_t size = hdr->size; size = size / TOMB_RECORD_ELEM_NUM; diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index 1b6639882b..c7f877a51b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -415,7 +415,7 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta &lino, _exit); // decode data - TAOS_UNUSED(tStatisBlockClear(statisBlock)); + tStatisBlockClear(statisBlock); statisBlock->numOfPKs = statisBlk->numOfPKs; statisBlock->numOfRecords = statisBlk->numRec; SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0); @@ -654,7 +654,7 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) { TAOS_CHECK_GOTO(TARRAY2_APPEND_PTR(writer->statisBlkArray, &statisBlk), &lino, _exit); - TAOS_UNUSED(tStatisBlockClear(writer->staticBlock)); + tStatisBlockClear(writer->staticBlock); _exit: if (code) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 34a07333ae..f807ecf2d6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -106,7 +106,7 @@ _exit: #endif void tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32_t (*tGetItemFn)(uint8_t *, void *)) { - TAOS_UNUSED(tGetItemFn(pMapData->pData + pMapData->aOffset[idx], pItem)); + int32_t r = tGetItemFn(pMapData->pData + pMapData->aOffset[idx], pItem); } #ifdef BUILD_NO_CALL @@ -609,7 +609,10 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal * SValue value; if (pRow->type == TSDBROW_ROW_FMT) { - (void)tRowGet(pRow->pTSRow, pTSchema, iCol, pColVal); + int32_t ret = tRowGet(pRow->pTSRow, pTSchema, iCol, pColVal); + if (ret != 0) { + tsdbError("failed to get column value, code:%d", ret); + } } else if (pRow->type == TSDBROW_COL_FMT) { if (iCol == 0) { *pColVal = @@ -809,6 +812,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) if (!COL_VAL_IS_NONE(pColVal)) { if (IS_VAR_DATA_TYPE(pColVal->value.type)) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); + if (!pTColVal) return terrno; if (!COL_VAL_IS_NULL(pColVal)) { code = tRealloc(&pTColVal->value.pData, pColVal->value.nData); if (code) return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil2.c b/source/dnode/vnode/src/tsdb/tsdbUtil2.c index cc9f8ce3ad..ba4ab9386b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil2.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil2.c @@ -93,25 +93,25 @@ void tStatisBlockDestroy(STbStatisBlock *statisBlock) { statisBlock->numOfPKs = 0; statisBlock->numOfRecords = 0; for (int32_t i = 0; i < ARRAY_SIZE(statisBlock->buffers); ++i) { - TAOS_UNUSED(tBufferDestroy(&statisBlock->buffers[i])); + tBufferDestroy(&statisBlock->buffers[i]); } for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) { - TAOS_UNUSED(tValueColumnDestroy(&statisBlock->firstKeyPKs[i])); - TAOS_UNUSED(tValueColumnDestroy(&statisBlock->lastKeyPKs[i])); + tValueColumnDestroy(&statisBlock->firstKeyPKs[i]); + tValueColumnDestroy(&statisBlock->lastKeyPKs[i]); } } -int32_t tStatisBlockClear(STbStatisBlock *statisBlock) { +void tStatisBlockClear(STbStatisBlock *statisBlock) { statisBlock->numOfPKs = 0; statisBlock->numOfRecords = 0; for (int32_t i = 0; i < ARRAY_SIZE(statisBlock->buffers); ++i) { - TAOS_UNUSED(tBufferClear(&statisBlock->buffers[i])); + tBufferClear(&statisBlock->buffers[i]); } for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) { tValueColumnClear(&statisBlock->firstKeyPKs[i]); tValueColumnClear(&statisBlock->lastKeyPKs[i]); } - return 0; + return; } static int32_t tStatisBlockAppend(STbStatisBlock *block, SRowInfo *row) { @@ -252,11 +252,11 @@ void tBrinBlockDestroy(SBrinBlock *brinBlock) { brinBlock->numOfPKs = 0; brinBlock->numOfRecords = 0; for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->buffers); ++i) { - TAOS_UNUSED(tBufferDestroy(&brinBlock->buffers[i])); + tBufferDestroy(&brinBlock->buffers[i]); } for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) { - TAOS_UNUSED(tValueColumnDestroy(&brinBlock->firstKeyPKs[i])); - TAOS_UNUSED(tValueColumnDestroy(&brinBlock->lastKeyPKs[i])); + tValueColumnDestroy(&brinBlock->firstKeyPKs[i]); + tValueColumnDestroy(&brinBlock->lastKeyPKs[i]); } } @@ -264,7 +264,7 @@ void tBrinBlockClear(SBrinBlock *brinBlock) { brinBlock->numOfPKs = 0; brinBlock->numOfRecords = 0; for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->buffers); ++i) { - TAOS_UNUSED(tBufferClear(&brinBlock->buffers[i])); + tBufferClear(&brinBlock->buffers[i]); } for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) { tValueColumnClear(&brinBlock->firstKeyPKs[i]); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil2.h b/source/dnode/vnode/src/tsdb/tsdbUtil2.h index ccc9009fd3..649f317485 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil2.h +++ b/source/dnode/vnode/src/tsdb/tsdbUtil2.h @@ -113,7 +113,7 @@ typedef struct { int32_t tStatisBlockInit(STbStatisBlock *statisBlock); void tStatisBlockDestroy(STbStatisBlock *statisBlock); -int32_t tStatisBlockClear(STbStatisBlock *statisBlock); +void tStatisBlockClear(STbStatisBlock *statisBlock); int32_t tStatisBlockPut(STbStatisBlock *statisBlock, SRowInfo *row, int32_t maxRecords); int32_t tStatisBlockGet(STbStatisBlock *statisBlock, int32_t idx, STbStatisRecord *record); diff --git a/source/dnode/vnode/src/vnd/vnodeAsync.c b/source/dnode/vnode/src/vnd/vnodeAsync.c index 27580f2e76..9e4fbd84a9 100644 --- a/source/dnode/vnode/src/vnd/vnodeAsync.c +++ b/source/dnode/vnode/src/vnd/vnodeAsync.c @@ -187,10 +187,12 @@ static void vnodeAsyncCancelAllTasks(SVAsync *async, SArray *cancelArray) { task->prev->next = task->next; task->next->prev = task->prev; if (task->cancel) { - TAOS_UNUSED(taosArrayPush(cancelArray, &(SVATaskCancelInfo){ - .cancel = task->cancel, - .arg = task->arg, - })); + if (taosArrayPush(cancelArray, &(SVATaskCancelInfo){ + .cancel = task->cancel, + .arg = task->arg, + }) == NULL) { + vError("failed to push cancel task into array"); + }; } vnodeAsyncTaskDone(async, task); } @@ -430,7 +432,7 @@ static void vnodeAsyncLaunchWorker(SVAsync *async) { if (async->workers[i].state == EVA_WORKER_STATE_ACTIVE) { continue; } else if (async->workers[i].state == EVA_WORKER_STATE_STOP) { - TAOS_UNUSED(taosThreadJoin(async->workers[i].thread, NULL)); + int32_t ret = taosThreadJoin(async->workers[i].thread, NULL); async->workers[i].state = EVA_WORKER_STATE_UINIT; } @@ -748,10 +750,12 @@ int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) { task->prev->next = task->next; task->next->prev = task->prev; if (task->cancel) { - TAOS_UNUSED(taosArrayPush(cancelArray, &(SVATaskCancelInfo){ - .cancel = task->cancel, - .arg = task->arg, - })); + if (taosArrayPush(cancelArray, &(SVATaskCancelInfo){ + .cancel = task->cancel, + .arg = task->arg, + }) == NULL) { + vError("failed to push cancel info"); + }; } vnodeAsyncTaskDone(async, task); } @@ -763,10 +767,12 @@ int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) { channel->scheduled->prev->next = channel->scheduled->next; channel->scheduled->next->prev = channel->scheduled->prev; if (channel->scheduled->cancel) { - TAOS_UNUSED(taosArrayPush(cancelArray, &(SVATaskCancelInfo){ - .cancel = channel->scheduled->cancel, - .arg = channel->scheduled->arg, - })); + if (taosArrayPush(cancelArray, &(SVATaskCancelInfo){ + .cancel = channel->scheduled->cancel, + .arg = channel->scheduled->arg, + }) == NULL) { + vError("failed to push cancel info"); + } } vnodeAsyncTaskDone(async, channel->scheduled); } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 0bb16261dd..438083f9b9 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -91,7 +91,9 @@ static int32_t vnodeGetBufPoolToUse(SVnode *pVnode) { struct timeval tv; struct timespec ts; - (void)taosGetTimeOfDay(&tv); + if (taosGetTimeOfDay(&tv) != 0) { + continue; + } ts.tv_nsec = tv.tv_usec * 1000 + WAIT_TIME_MILI_SEC * 1000000; if (ts.tv_nsec > 999999999l) { ts.tv_sec = tv.tv_sec + 1; @@ -199,7 +201,9 @@ _exit: vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d selfIndex:%d changeVersion:%d", pInfo->config.vgId, fname, pInfo->config.syncCfg.replicaNum, pInfo->config.syncCfg.myIndex, pInfo->config.syncCfg.changeVersion); } - (void)taosCloseFile(&pFile); + if (taosCloseFile(&pFile) != 0) { + vError("vgId:%d, failed to close file", pInfo->config.vgId); + } taosMemoryFree(data); return code; } @@ -261,7 +265,9 @@ _exit: } } taosMemoryFree(pData); - (void)taosCloseFile(&pFile); + if (taosCloseFile(&pFile) != 0) { + vError("vgId:%d, failed to close file", pInfo->config.vgId); + } return code; } @@ -494,7 +500,9 @@ void vnodeRollback(SVnode *pVnode) { offset = strlen(tFName); snprintf(tFName + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VND_INFO_FNAME_TMP); - TAOS_UNUSED(taosRemoveFile(tFName)); + if (taosRemoveFile(tFName) != 0) { + vError("vgId:%d, failed to remove file %s since %s", TD_VID(pVnode), tFName, tstrerror(terrno)); + } } static int vnodeEncodeState(const void *pObj, SJson *pJson) { diff --git a/source/dnode/vnode/src/vnd/vnodeHash.c b/source/dnode/vnode/src/vnd/vnodeHash.c index 878f226309..e0faa05c78 100644 --- a/source/dnode/vnode/src/vnd/vnodeHash.c +++ b/source/dnode/vnode/src/vnd/vnodeHash.c @@ -24,10 +24,10 @@ struct SVHashEntry { void* obj; }; -static int32_t vHashRehash(SVHashTable* ht, uint32_t newNumBuckets) { +static void vHashRehash(SVHashTable* ht, uint32_t newNumBuckets) { SVHashEntry** newBuckets = (SVHashEntry**)taosMemoryCalloc(newNumBuckets, sizeof(SVHashEntry*)); if (newBuckets == NULL) { - return terrno; + return; } for (int32_t i = 0; i < ht->numBuckets; i++) { @@ -45,7 +45,7 @@ static int32_t vHashRehash(SVHashTable* ht, uint32_t newNumBuckets) { ht->buckets = newBuckets; ht->numBuckets = newNumBuckets; - return 0; + return; } int32_t vHashInit(SVHashTable** ht, uint32_t (*hash)(const void*), int32_t (*compare)(const void*, const void*)) { @@ -96,7 +96,7 @@ int32_t vHashPut(SVHashTable* ht, void* obj) { } if (ht->numEntries >= ht->numBuckets) { - (void)vHashRehash(ht, ht->numBuckets * 2); + vHashRehash(ht, ht->numBuckets * 2); bucketIndex = ht->hash(obj) % ht->numBuckets; } @@ -142,7 +142,7 @@ int32_t vHashDrop(SVHashTable* ht, const void* obj) { taosMemoryFree(tmp); ht->numEntries--; if (ht->numBuckets > VNODE_HASH_DEFAULT_NUM_BUCKETS && ht->numEntries < ht->numBuckets / 4) { - (void)vHashRehash(ht, ht->numBuckets / 2); + vHashRehash(ht, ht->numBuckets / 2); } return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 1f54fea27c..0d04486925 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -108,7 +108,7 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t pNode->nodePort = pReq->replicas[i].port; tstrncpy(pNode->nodeFqdn, pReq->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); pNode->nodeRole = TAOS_SYNC_ROLE_VOTER; - (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + bool ret = tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); vInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); pCfg->replicaNum++; } @@ -121,7 +121,7 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t pNode->nodePort = pReq->learnerReplicas[pCfg->totalReplicaNum].port; pNode->nodeRole = TAOS_SYNC_ROLE_LEARNER; tstrncpy(pNode->nodeFqdn, pReq->learnerReplicas[pCfg->totalReplicaNum].fqdn, sizeof(pNode->nodeFqdn)); - (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + bool ret = tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); vInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); pCfg->totalReplicaNum++; } @@ -176,8 +176,10 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr int32_t prefixLen = strlen(tsdbFilePrefix); STfsDir *tsdbDir = NULL; - (void)tfsOpendir(pTfs, tsdbPath, &tsdbDir); - if (tsdbDir == NULL) return 0; + int32_t tret = tfsOpendir(pTfs, tsdbPath, &tsdbDir); + if (tsdbDir == NULL) { + return 0; + } while (1) { const STfsFile *tsdbFile = tfsReaddir(tsdbDir); @@ -248,7 +250,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod SNodeInfo *pNode = &pCfg->nodeInfo[0]; pNode->nodePort = tsServerPort; tstrncpy(pNode->nodeFqdn, tsLocalFqdn, TSDB_FQDN_LEN); - (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + bool ret1 = tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); vInfo("vgId:%d, ep:%s:%u dnode:%d", pReq->srcVgId, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); info.config.syncCfg = *pCfg; @@ -317,7 +319,9 @@ int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t s void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs, int32_t nodeId) { vInfo("path:%s is removed while destroy vnode", path); - (void)tfsRmdir(pTfs, path); + if (tfsRmdir(pTfs, path) < 0) { + vError("failed to remove path:%s since %s", path, tstrerror(terrno)); + } // int32_t nlevel = tfsGetLevel(pTfs); if (nodeId > 0 && vgId > 0 /*&& nlevel > 1*/ && tsS3Enabled) { @@ -378,8 +382,13 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } if (updated) { vInfo("vgId:%d, save vnode info since dnode info changed", info.config.vgId); - (void)vnodeSaveInfo(dir, &info); - (void)vnodeCommitInfo(dir); + if (vnodeSaveInfo(dir, &info) < 0) { + vError("vgId:%d, failed to save vnode info since %s", info.config.vgId, tstrerror(terrno)); + } + + if (vnodeCommitInfo(dir) < 0) { + vError("vgId:%d, failed to commit vnode info since %s", info.config.vgId, tstrerror(terrno)); + } } // create handle @@ -405,7 +414,10 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC pVnode->blocked = false; pVnode->disableWrite = false; - (void)tsem_init(&pVnode->syncSem, 0, 0); + if (tsem_init(&pVnode->syncSem, 0, 0) != 0) { + vError("vgId:%d, failed to init semaphore", TD_VID(pVnode)); + goto _err; + } (void)taosThreadMutexInit(&pVnode->mutex, NULL); (void)taosThreadCondInit(&pVnode->poolNotEmpty, NULL); @@ -440,7 +452,8 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC // open wal sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); - (void)taosRealPath(tdir, NULL, sizeof(tdir)); + ret = taosRealPath(tdir, NULL, sizeof(tdir)); + TAOS_UNUSED(ret); pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg)); if (pVnode->pWal == NULL) { @@ -450,7 +463,8 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC // open tq sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); - (void)taosRealPath(tdir, NULL, sizeof(tdir)); + ret = taosRealPath(tdir, NULL, sizeof(tdir)); + TAOS_UNUSED(ret); // open query if (vnodeQueryOpen(pVnode)) { @@ -499,9 +513,9 @@ _err: if (pVnode->pQuery) vnodeQueryClose(pVnode); if (pVnode->pTq) tqClose(pVnode->pTq); if (pVnode->pWal) walClose(pVnode->pWal); - if (pVnode->pTsdb) (void)tsdbClose(&pVnode->pTsdb); - if (pVnode->pSma) (void)smaClose(pVnode->pSma); - if (pVnode->pMeta) (void)metaClose(&pVnode->pMeta); + if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); + if (pVnode->pSma) smaClose(pVnode->pSma); + if (pVnode->pMeta) metaClose(&pVnode->pMeta); if (pVnode->freeList) vnodeCloseBufPool(pVnode); taosMemoryFree(pVnode); @@ -518,18 +532,23 @@ void vnodePostClose(SVnode *pVnode) { vnodeSyncPostClose(pVnode); } void vnodeClose(SVnode *pVnode) { if (pVnode) { vnodeAWait(&pVnode->commitTask); - (void)vnodeAChannelDestroy(&pVnode->commitChannel, true); + if (vnodeAChannelDestroy(&pVnode->commitChannel, true) != 0) { + vError("vgId:%d, failed to destroy commit channel", TD_VID(pVnode)); + } + vnodeSyncClose(pVnode); vnodeQueryClose(pVnode); tqClose(pVnode->pTq); walClose(pVnode->pWal); if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); - (void)smaClose(pVnode->pSma); + smaClose(pVnode->pSma); if (pVnode->pMeta) metaClose(&pVnode->pMeta); vnodeCloseBufPool(pVnode); // destroy handle - (void)tsem_destroy(&pVnode->syncSem); + if (tsem_destroy(&pVnode->syncSem) != 0) { + vError("vgId:%d, failed to destroy semaphore", TD_VID(pVnode)); + } (void)taosThreadCondDestroy(&pVnode->poolNotEmpty); (void)taosThreadMutexDestroy(&pVnode->mutex); (void)taosThreadMutexDestroy(&pVnode->lock); diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index f732596d3b..0c11083367 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -274,13 +274,17 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) int64_t size; code = taosFStatFile(pFile, &size, NULL); if (code != 0) { - (void)taosCloseFile(&pFile); + if (taosCloseFile(&pFile) != 0) { + vError("vgId:%d, failed to close file", vgId); + } TSDB_CHECK_CODE(code, lino, _exit); } *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size + 1); if (*ppData == NULL) { - (void)taosCloseFile(&pFile); + if (taosCloseFile(&pFile) != 0) { + vError("vgId:%d, failed to close file", vgId); + } TSDB_CHECK_CODE(code = terrno, lino, _exit); } ((SSnapDataHdr *)(*ppData))->type = SNAP_DATA_CFG; @@ -289,11 +293,15 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) if (taosReadFile(pFile, ((SSnapDataHdr *)(*ppData))->data, size) < 0) { taosMemoryFree(*ppData); - (void)taosCloseFile(&pFile); + if (taosCloseFile(&pFile) != 0) { + vError("vgId:%d, failed to close file", vgId); + } TSDB_CHECK_CODE(code = terrno, lino, _exit); } - (void)taosCloseFile(&pFile); + if (taosCloseFile(&pFile) != 0) { + vError("vgId:%d, failed to close file", vgId); + } pReader->cfgDone = 1; goto _exit; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 9c0c3136ee..371eaa0774 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -210,7 +210,9 @@ static int32_t vnodePreProcessDropTtlMsg(SVnode *pVnode, SRpcMsg *pMsg) { TSDB_CHECK_CODE(code, lino, _exit); } - (void)tSerializeSVDropTtlTableReq((char *)pContNew + sizeof(SMsgHead), reqLenNew, &ttlReq); + if (tSerializeSVDropTtlTableReq((char *)pContNew + sizeof(SMsgHead), reqLenNew, &ttlReq) != 0) { + vError("vgId:%d %s:%d failed to serialize drop ttl request", TD_VID(pVnode), __func__, lino); + } pContNew->contLen = htonl(reqLenNew); pContNew->vgId = pContOld->vgId; @@ -420,7 +422,9 @@ static int32_t vnodePreProcessDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) { ((SMsgHead *)pCont)->vgId = TD_VID(pVnode); tEncoderInit(pCoder, pCont + sizeof(SMsgHead), size); - (void)tEncodeDeleteRes(pCoder, &res); + if (tEncodeDeleteRes(pCoder, &res) != 0) { + vError("vgId:%d %s failed to encode delete response", TD_VID(pVnode), __func__); + } tEncoderClear(pCoder); rpcFreeCont(pMsg->pCont); @@ -466,7 +470,11 @@ static int32_t vnodePreProcessArbCheckSyncMsg(SVnode *pVnode, SRpcMsg *pMsg) { return TSDB_CODE_INVALID_MSG; } - (void)vnodePreCheckAssignedLogSyncd(pVnode, syncReq.member0Token, syncReq.member1Token); + int32_t ret = vnodePreCheckAssignedLogSyncd(pVnode, syncReq.member0Token, syncReq.member1Token); + if (ret != 0) { + vError("vgId:%d, failed to preprocess arb check sync request since %s", TD_VID(pVnode), tstrerror(ret)); + } + int32_t code = terrno; tFreeSVArbCheckSyncReq(&syncReq); @@ -643,7 +651,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg } break; case TDMT_STREAM_CONSEN_CHKPT: { if (pVnode->restored) { - (void)tqProcessTaskConsenChkptIdReq(pVnode->pTq, pMsg); + if (tqProcessTaskConsenChkptIdReq(pVnode->pTq, pMsg) < 0) { + goto _err; + } } } break; case TDMT_STREAM_TASK_PAUSE: { @@ -660,7 +670,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg } break; case TDMT_VND_STREAM_TASK_RESET: { if (pVnode->restored && vnodeIsLeader(pVnode)) { - (void)tqProcessTaskResetReq(pVnode->pTq, pMsg); + if (tqProcessTaskResetReq(pVnode->pTq, pMsg) < 0) { + goto _err; + } } } break; case TDMT_VND_ALTER_CONFIRM: @@ -710,7 +722,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg vTrace("vgId:%d, process %s request, code:0x%x index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), pRsp->code, ver); - (void)walApplyVer(pVnode->pWal, ver); + walApplyVer(pVnode->pWal, ver); code = tqPushMsg(pVnode->pTq, pMsg->msgType); if (code) { @@ -881,7 +893,10 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) } void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { - (void)tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data); + int32_t code = tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data); + if (code) { + vError("failed to process sma result since %s", tstrerror(code)); + } } void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { @@ -957,7 +972,10 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t code = metaDropTables(pVnode->pMeta, ttlReq.pTbUids); if (code) return code; - (void)tqUpdateTbUidList(pVnode->pTq, ttlReq.pTbUids, false); + code = tqUpdateTbUidList(pVnode->pTq, ttlReq.pTbUids, false); + if (code) { + vError("vgId:%d, failed to update tbUid list since %s", TD_VID(pVnode), tstrerror(code)); + } } end: @@ -1160,7 +1178,9 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, } } else { cRsp.code = TSDB_CODE_SUCCESS; - (void)tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); + if (tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid) < 0) { + vError("vgId:%d, failed to fetch tbUid list", TD_VID(pVnode)); + } if (taosArrayPush(tbUids, &pCreateReq->uid) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; rcode = -1; @@ -1177,11 +1197,13 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, } vDebug("vgId:%d, add %d new created tables into query table list", TD_VID(pVnode), (int32_t)taosArrayGetSize(tbUids)); - (void)tqUpdateTbUidList(pVnode->pTq, tbUids, true); + if (tqUpdateTbUidList(pVnode->pTq, tbUids, true) < 0) { + vError("vgId:%d, failed to update tbUid list since %s", TD_VID(pVnode), tstrerror(terrno)); + } if (tdUpdateTbUidList(pVnode->pSma, pStore, true) < 0) { goto _exit; } - (void)tdUidStoreFree(pStore); + pStore = tdUidStoreFree(pStore); // prepare rsp int32_t ret = 0; @@ -1193,13 +1215,17 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, goto _exit; } tEncoderInit(&encoder, pRsp->pCont, pRsp->contLen); - (void)tEncodeSVCreateTbBatchRsp(&encoder, &rsp); + if (tEncodeSVCreateTbBatchRsp(&encoder, &rsp) != 0) { + vError("vgId:%d, failed to encode create table batch response", TD_VID(pVnode)); + } if (tsEnableAudit && tsEnableAuditCreateTable) { int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; SName name = {0}; - (void)tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); + if (tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB) < 0) { + vError("vgId:%d, failed to get name from string", TD_VID(pVnode)); + } SStringBuilder sb = {0}; for (int32_t i = 0; i < tbNames->size; i++) { @@ -1347,7 +1373,9 @@ _exit: tEncodeSize(tEncodeSVAlterTbRsp, &vAlterTbRsp, pRsp->contLen, ret); pRsp->pCont = rpcMallocCont(pRsp->contLen); tEncoderInit(&ec, pRsp->pCont, pRsp->contLen); - (void)tEncodeSVAlterTbRsp(&ec, &vAlterTbRsp); + if (tEncodeSVAlterTbRsp(&ec, &vAlterTbRsp) != 0) { + vError("vgId:%d, failed to encode alter table response", TD_VID(pVnode)); + } tEncoderClear(&ec); if (vMetaRsp.pSchemas) { taosMemoryFree(vMetaRsp.pSchemas); @@ -1402,7 +1430,11 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in } } else { dropTbRsp.code = TSDB_CODE_SUCCESS; - if (tbUid > 0) (void)tdFetchTbUidList(pVnode->pSma, &pStore, pDropTbReq->suid, tbUid); + if (tbUid > 0) { + if (tdFetchTbUidList(pVnode->pSma, &pStore, pDropTbReq->suid, tbUid) < 0) { + vError("vgId:%d, failed to fetch tbUid list", TD_VID(pVnode)); + } + } } if (taosArrayPush(rsp.pArray, &dropTbRsp) == NULL) { @@ -1426,14 +1458,21 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in } } - (void)tqUpdateTbUidList(pVnode->pTq, tbUids, false); - (void)tdUpdateTbUidList(pVnode->pSma, pStore, false); + if (tqUpdateTbUidList(pVnode->pTq, tbUids, false) < 0) { + vError("vgId:%d, failed to update tbUid list since %s", TD_VID(pVnode), tstrerror(terrno)); + } + + if (tdUpdateTbUidList(pVnode->pSma, pStore, false) < 0) { + goto _exit; + } if (tsEnableAuditCreateTable) { int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; SName name = {0}; - (void)tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); + if (tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB) != 0) { + vError("vgId:%d, failed to get name from string", TD_VID(pVnode)); + } SStringBuilder sb = {0}; for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { @@ -1457,12 +1496,14 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in _exit: taosArrayDestroy(tbUids); - (void)tdUidStoreFree(pStore); + pStore = tdUidStoreFree(pStore); tDecoderClear(&decoder); tEncodeSize(tEncodeSVDropTbBatchRsp, &rsp, pRsp->contLen, ret); pRsp->pCont = rpcMallocCont(pRsp->contLen); tEncoderInit(&encoder, pRsp->pCont, pRsp->contLen); - (void)tEncodeSVDropTbBatchRsp(&encoder, &rsp); + if (tEncodeSVDropTbBatchRsp(&encoder, &rsp) != 0) { + vError("vgId:%d, failed to encode drop table batch response", TD_VID(pVnode)); + } tEncoderClear(&encoder); taosArrayDestroy(rsp.pArray); taosArrayDestroy(tbNames); @@ -1802,7 +1843,9 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in } if (info.suid) { - (void)metaGetInfo(pVnode->pMeta, info.suid, &info, NULL); + if (metaGetInfo(pVnode->pMeta, info.suid, &info, NULL) != 0) { + vWarn("vgId:%d, table uid:%" PRId64 " not exists", TD_VID(pVnode), info.suid); + } } if (pSubmitTbData->sver != info.skmVer) { @@ -1898,7 +1941,9 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in if (taosArrayGetSize(newTbUids) > 0) { vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), (int32_t)taosArrayGetSize(newTbUids)); - (void)tqUpdateTbUidList(pVnode->pTq, newTbUids, true); + if (tqUpdateTbUidList(pVnode->pTq, newTbUids, true) != 0) { + vError("vgId:%d, failed to update tbUid list", TD_VID(pVnode)); + } } _exit: @@ -1907,7 +1952,9 @@ _exit: tEncodeSize(tEncodeSSubmitRsp2, pSubmitRsp, pRsp->contLen, ret); pRsp->pCont = rpcMallocCont(pRsp->contLen); tEncoderInit(&ec, pRsp->pCont, pRsp->contLen); - (void)tEncodeSSubmitRsp2(&ec, pSubmitRsp); + if (tEncodeSSubmitRsp2(&ec, pSubmitRsp) < 0) { + vError("vgId:%d, failed to encode submit response", TD_VID(pVnode)); + } tEncoderClear(&ec); // update statistics @@ -1924,7 +1971,7 @@ _exit: pVnode->monitor.strVgId, pOriginalMsg->info.conn.user, "Success"}; - (void)taos_counter_add(tsInsertCounter, pSubmitRsp->affectedRows, sample_labels); + int tv = taos_counter_add(tsInsertCounter, pSubmitRsp->affectedRows, sample_labels); } if (code == 0) { @@ -2149,7 +2196,12 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe pVnode->config.sttTrigger = req.sttTrigger; } else { vnodeAWait(&pVnode->commitTask); - (void)tsdbDisableAndCancelAllBgTask(pVnode->pTsdb); + + int32_t ret = tsdbDisableAndCancelAllBgTask(pVnode->pTsdb); + if (ret != 0) { + vError("vgId:%d, failed to disable bg task since %s", TD_VID(pVnode), tstrerror(errno)); + } + pVnode->config.sttTrigger = req.sttTrigger; tsdbEnableBgTask(pVnode->pTsdb); } @@ -2167,7 +2219,9 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe } if (walChanged) { - (void)walAlter(pVnode->pWal, &pVnode->config.walCfg); + if (walAlter(pVnode->pWal, &pVnode->config.walCfg) != 0) { + vError("vgId:%d, failed to alter wal config since %s", TD_VID(pVnode), tstrerror(errno)); + } } if (tsdbChanged) { @@ -2181,7 +2235,10 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pRe SBatchDeleteReq deleteReq; SDecoder decoder; tDecoderInit(&decoder, pReq, len); - (void)tDecodeSBatchDeleteReq(&decoder, &deleteReq); + if (tDecodeSBatchDeleteReq(&decoder, &deleteReq) < 0) { + tDecoderClear(&decoder); + return terrno = TSDB_CODE_INVALID_MSG; + } SMetaReader mr = {0}; metaReaderDoInit(&mr, pVnode->pMeta, META_READER_NOLOCK); @@ -2351,7 +2408,9 @@ static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pR } static int32_t vnodeProcessConfigChangeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { - (void)syncCheckMember(pVnode->sync); + if (syncCheckMember(pVnode->sync) != 0) { + vError("vgId:%d, failed to check member", TD_VID(pVnode)); + } pRsp->msgType = TDMT_SYNC_CONFIG_CHANGE_RSP; pRsp->code = TSDB_CODE_SUCCESS; @@ -2411,7 +2470,9 @@ static int32_t vnodeProcessArbCheckSyncReq(SVnode *pVnode, void *pReq, int32_t l syncRsp.member1Token = syncReq.member1Token; syncRsp.vgId = TD_VID(pVnode); - (void)vnodeCheckAssignedLogSyncd(pVnode, syncReq.member0Token, syncReq.member1Token); + if (vnodeCheckAssignedLogSyncd(pVnode, syncReq.member0Token, syncReq.member1Token) != 0) { + vError("vgId:%d, failed to check assigned log syncd", TD_VID(pVnode)); + } syncRsp.errCode = terrno; if (vnodeUpdateArbTerm(pVnode, syncReq.arbTerm) != 0) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index f725fb3809..50bedba75d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -28,7 +28,9 @@ static inline void vnodeWaitBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { const STraceId *trace = &pMsg->info.traceId; vGTrace("vgId:%d, msg:%p wait block, type:%s sec:%d seq:%" PRId64, pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType), pVnode->blockSec, pVnode->blockSeq); - (void)tsem_wait(&pVnode->syncSem); + if (tsem_wait(&pVnode->syncSem) != 0) { + vError("vgId:%d, failed to wait sem", pVnode->config.vgId); + } } static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { @@ -41,7 +43,9 @@ static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { pVnode->blocked = false; pVnode->blockSec = 0; pVnode->blockSeq = 0; - (void)tsem_post(&pVnode->syncSem); + if (tsem_post(&pVnode->syncSem) != 0) { + vError("vgId:%d, failed to post sem", pVnode->config.vgId); + } } (void)taosThreadMutexUnlock(&pVnode->lock); } @@ -69,7 +73,9 @@ void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) { if (rsp.pCont == NULL) { pMsg->code = TSDB_CODE_OUT_OF_MEMORY; } else { - (void)tSerializeSEpSet(rsp.pCont, contLen, &newEpSet); + if (tSerializeSEpSet(rsp.pCont, contLen, &newEpSet) < 0) { + vError("vgId:%d, failed to serialize ep set", pVnode->config.vgId); + } rsp.contLen = contLen; } @@ -163,7 +169,9 @@ void vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit) { rpcFreeCont(rpcMsg.pCont); rpcMsg.pCont = NULL; } else { - (void)tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &rpcMsg); + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &rpcMsg) < 0) { + vTrace("vgId:%d, failed to put vnode commit to queue since %s", pVnode->config.vgId, terrstr()); + } } } @@ -560,7 +568,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) } } while (true); - (void)walApplyVer(pVnode->pWal, commitIdx); + walApplyVer(pVnode->pWal, commitIdx); pVnode->restored = true; SStreamMeta *pMeta = pVnode->pTq->pStreamMeta; @@ -609,13 +617,17 @@ static void vnodeBecomeFollower(const SSyncFSM *pFsm) { if (pVnode->blocked) { pVnode->blocked = false; vDebug("vgId:%d, become follower and post block", pVnode->config.vgId); - (void)tsem_post(&pVnode->syncSem); + if (tsem_post(&pVnode->syncSem) != 0) { + vError("vgId:%d, failed to post sync semaphore", pVnode->config.vgId); + } } (void)taosThreadMutexUnlock(&pVnode->lock); if (pVnode->pTq) { tqUpdateNodeStage(pVnode->pTq, false); - (void)tqStopStreamTasksAsync(pVnode->pTq); + if (tqStopStreamTasksAsync(pVnode->pTq) != 0) { + vError("vgId:%d, failed to stop stream tasks", pVnode->config.vgId); + } } } @@ -627,7 +639,9 @@ static void vnodeBecomeLearner(const SSyncFSM *pFsm) { if (pVnode->blocked) { pVnode->blocked = false; vDebug("vgId:%d, become learner and post block", pVnode->config.vgId); - (void)tsem_post(&pVnode->syncSem); + if (tsem_post(&pVnode->syncSem) != 0) { + vError("vgId:%d, failed to post sync semaphore", pVnode->config.vgId); + } } (void)taosThreadMutexUnlock(&pVnode->lock); } @@ -750,14 +764,19 @@ int32_t vnodeSyncStart(SVnode *pVnode) { void vnodeSyncPreClose(SVnode *pVnode) { vInfo("vgId:%d, sync pre close", pVnode->config.vgId); - (void)syncLeaderTransfer(pVnode->sync); + int32_t code = syncLeaderTransfer(pVnode->sync); + if (code) { + vError("vgId:%d, failed to transfer leader since %s", pVnode->config.vgId, tstrerror(code)); + } syncPreStop(pVnode->sync); (void)taosThreadMutexLock(&pVnode->lock); if (pVnode->blocked) { vInfo("vgId:%d, post block after close sync", pVnode->config.vgId); pVnode->blocked = false; - (void)tsem_post(&pVnode->syncSem); + if (tsem_post(&pVnode->syncSem) != 0) { + vError("vgId:%d, failed to post block", pVnode->config.vgId); + } } (void)taosThreadMutexUnlock(&pVnode->lock); } @@ -792,7 +811,9 @@ void vnodeSyncCheckTimeout(SVnode *pVnode) { pVnode->blocked = false; pVnode->blockSec = 0; pVnode->blockSeq = 0; - (void)tsem_post(&pVnode->syncSem); + if (tsem_post(&pVnode->syncSem) != 0) { + vError("vgId:%d, failed to post block", pVnode->config.vgId); + } } } (void)taosThreadMutexUnlock(&pVnode->lock); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 0882db52a6..e757163ba8 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -334,6 +334,7 @@ typedef struct SCtgViewCache { typedef struct SCtgTSMACache { SRWLatch tsmaLock; SArray* pTsmas; // SArray + bool retryFetch; } SCtgTSMACache; typedef struct SCtgDBCache { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 619f4bc114..0ea1d98312 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1461,7 +1461,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SCatalo pRsp->pTableMeta = taosArrayInit(tbNum, POINTER_BYTES); if (NULL == pRsp->pTableMeta) { ctgError("taosArrayInit %d failed", tbNum); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_JRET(terrno); } for (int32_t i = 0; i < tbNum; ++i) { @@ -1476,7 +1476,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SCatalo if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { ctgError("taosArrayPush failed, idx:%d", i); taosMemoryFreeClear(pTableMeta); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_JRET(terrno); } } } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 46f4f86484..525573ee01 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -72,7 +72,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -94,7 +94,7 @@ int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbMetaNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbMetaNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -105,7 +105,7 @@ int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); return TSDB_CODE_SUCCESS; @@ -133,7 +133,7 @@ int32_t ctgInitGetDbVgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; @@ -161,7 +161,7 @@ int32_t ctgInitGetDbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; @@ -189,7 +189,7 @@ int32_t ctgInitGetDbInfoTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; @@ -223,7 +223,7 @@ int32_t ctgInitGetTbHashTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tableName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -245,7 +245,7 @@ int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbHashNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbHashNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -256,7 +256,7 @@ int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbHashNum); return TSDB_CODE_SUCCESS; @@ -275,7 +275,7 @@ int32_t ctgInitGetQnodeTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -293,7 +293,7 @@ int32_t ctgInitGetDnodeTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -320,7 +320,7 @@ int32_t ctgInitGetIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); return TSDB_CODE_SUCCESS; @@ -348,7 +348,7 @@ int32_t ctgInitGetUdfTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, udfName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, udfName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); return TSDB_CODE_SUCCESS; @@ -376,7 +376,7 @@ int32_t ctgInitGetUserTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, user:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, user:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), user->user); return TSDB_CODE_SUCCESS; @@ -394,7 +394,7 @@ int32_t ctgInitGetSvrVerTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -426,7 +426,7 @@ int32_t ctgInitGetTbIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -459,7 +459,7 @@ int32_t ctgInitGetTbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -492,7 +492,7 @@ int32_t ctgInitGetTbTagTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -514,7 +514,7 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->viewNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->viewNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->viewNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -525,7 +525,7 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, viewNum:%d", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, viewNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->viewNum); return TSDB_CODE_SUCCESS; @@ -546,7 +546,7 @@ int32_t ctgInitGetTbTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tbTsmaNum, sizeof(SMetaRes)); if (NULL == pTaskCtx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbTsmaNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbTsmaNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -574,7 +574,7 @@ int32_t ctgInitGetTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tsmaNum, sizeof(SMetaRes)); if (NULL == pTaskCtx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tsmaNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tsmaNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -603,7 +603,7 @@ static int32_t ctgInitGetTbNamesTask(SCtgJob* pJob, int32_t taskId, void* param) pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tbNameNum, sizeof(SMetaRes)); if (NULL == pTaskCtx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbNameNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbNameNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -1048,7 +1048,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const } double el = (taosGetTimestampUs() - st) / 1000.0; - qDebug("qid:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d, elapsed time:%.2f ms", + qDebug("QID:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d, elapsed time:%.2f ms", pJob->queryId, pJob->refId, taskNum, pReq->forceUpdate, el); return TSDB_CODE_SUCCESS; @@ -1450,16 +1450,17 @@ _return: int32_t ctgCallUserCb(void* param) { SCtgJob* pJob = (SCtgJob*)param; - qDebug("qid:0x%" PRIx64 " ctg start to call user cb with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); + qDebug("QID:0x%" PRIx64 " ctg start to call user cb with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); (*pJob->userFp)(&pJob->jobRes, pJob->userParam, pJob->jobResCode); - qDebug("qid:0x%" PRIx64 " ctg end to call user cb", pJob->queryId); + qDebug("QID:0x%" PRIx64 " ctg end to call user cb", pJob->queryId); int64_t refId = pJob->refId; int32_t code = taosRemoveRef(gCtgMgmt.jobPool, refId); if (code) { - qError("qid:0x%" PRIx64 " remove ctg job %" PRId64 " from jobPool failed, error:%s", pJob->queryId, refId, tstrerror(code)); + qError("QID:0x%" PRIx64 " remove ctg job %" PRId64 " from jobPool failed, error:%s", pJob->queryId, refId, + tstrerror(code)); } return TSDB_CODE_SUCCESS; @@ -1469,7 +1470,7 @@ void ctgUpdateJobErrCode(SCtgJob* pJob, int32_t errCode) { if (!NEED_CLIENT_REFRESH_VG_ERROR(errCode) || errCode == TSDB_CODE_SUCCESS) return; atomic_store_32(&pJob->jobResCode, errCode); - qDebug("qid:0x%" PRIx64 " ctg job errCode updated to %s", pJob->queryId, tstrerror(errCode)); + qDebug("QID:0x%" PRIx64 " ctg job errCode updated to %s", pJob->queryId, tstrerror(errCode)); return; } @@ -1481,7 +1482,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { return TSDB_CODE_SUCCESS; } - qDebug("qid:0x%" PRIx64 " task %d end with res %s", pJob->queryId, pTask->taskId, tstrerror(rspCode)); + qDebug("QID:0x%" PRIx64 " task %d end with res %s", pJob->queryId, pTask->taskId, tstrerror(rspCode)); pTask->code = rspCode; pTask->status = CTG_TASK_DONE; @@ -1490,7 +1491,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1); if (taskDone < taosArrayGetSize(pJob->pTasks)) { - qDebug("qid:0x%" PRIx64 " task done: %d, total: %d", pJob->queryId, taskDone, + qDebug("QID:0x%" PRIx64 " task done: %d, total: %d", pJob->queryId, taskDone, (int32_t)taosArrayGetSize(pJob->pTasks)); ctgUpdateJobErrCode(pJob, rspCode); @@ -4347,7 +4348,7 @@ int32_t ctgLaunchJob(SCtgJob* pJob) { CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("qid:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); + qDebug("QID:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); pTask = taosArrayGet(pJob->pTasks, i); @@ -4360,7 +4361,7 @@ int32_t ctgLaunchJob(SCtgJob* pJob) { } if (taskNum <= 0) { - qDebug("qid:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); + qDebug("QID:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); CTG_ERR_RET(taosAsyncExec(ctgCallUserCb, pJob, NULL)); #if CTG_BATCH_FETCH diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 80650a6095..9db3913375 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -906,7 +906,7 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { if (gCtgMgmt.queue.stopQueue) { ctgFreeQNode(node); CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); - CTG_RET(TSDB_CODE_CTG_EXIT); + CTG_ERR_JRET(TSDB_CODE_CTG_EXIT); } gCtgMgmt.queue.tail->next = node; @@ -924,7 +924,7 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { code = tsem_post(&gCtgMgmt.queue.reqSem); if (TSDB_CODE_SUCCESS != code) { qError("tsem_post failed, code:%x", code); - CTG_RET(code); + CTG_ERR_JRET(code); } if (syncOp) { @@ -935,9 +935,15 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { if (!operation->unLocked) { CTG_LOCK(CTG_READ, &gCtgMgmt.lock); } - taosMemoryFree(operation); + TAOS_UNUSED(tsem_destroy(&operation->rspSem)); + taosMemoryFreeClear(operation); } + return code; +_return: + if (syncOp && operation) { + TAOS_UNUSED(tsem_destroy(&operation->rspSem)); + } return code; } @@ -1583,7 +1589,7 @@ int32_t ctgDropTSMAForTbEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { SCtgTSMACache *pCtgCache = NULL; (void)tNameGetFullDbName(pName, dbFName); - CTG_ERR_JRET(ctgGetDBCache(pCtg, dbFName, &pDbCache)); + CTG_ERR_JRET(ctgAcquireDBCache(pCtg, dbFName, &pDbCache)); if (NULL == pDbCache || !pDbCache->tsmaCache) { goto _return; } @@ -1608,11 +1614,14 @@ int32_t ctgDropTSMAForTbEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { code = createDropAllTbTsmaCtgCacheOp(pCtg, pCache, syncOp, &pOp); } CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); + taosHashRelease(pDbCache->tsmaCache, pCtgCache); + pCtgCache = NULL; + ctgReleaseDBCache(pCtg, pDbCache); + pDbCache = NULL; CTG_ERR_JRET(code); CTG_ERR_JRET(ctgEnqueue(pCtg, pOp)); - taosHashRelease(pDbCache->tsmaCache, pCtgCache); return TSDB_CODE_SUCCESS; @@ -1621,6 +1630,9 @@ _return: if (pCtgCache) { taosHashRelease(pDbCache->tsmaCache, pCtgCache); } + if (pDbCache) { + ctgReleaseDBCache(pCtg, pDbCache); + } if (pOp) { taosMemoryFree(pOp->data); taosMemoryFree(pOp); @@ -2997,6 +3009,7 @@ int32_t ctgOpDropTbTSMA(SCtgCacheOperation *operation) { taosArrayDestroyP(pCtgCache->pTsmas, tFreeAndClearTableTSMAInfo); pCtgCache->pTsmas = NULL; + pCtgCache->retryFetch = true; ctgDebug("all tsmas for table dropped: %s.%s", msg->dbFName, msg->tbName); code = taosHashRemove(dbCache->tsmaCache, msg->tbName, TSDB_TABLE_NAME_LEN); @@ -3975,28 +3988,40 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx // get tsma cache pCache = taosHashAcquire(dbCache->tsmaCache, tsmaSourceTbName.tname, strlen(tsmaSourceTbName.tname)); - if (!pCache || !pCache->pTsmas || pCache->pTsmas->size == 0) { + if (!pCache) { if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { ctgReleaseTSMAToCache(pCtg, dbCache, pCache); CTG_ERR_RET(terrno); } - + continue; + } + CTG_LOCK(CTG_READ, &pCache->tsmaLock); + if ((!pCache->pTsmas || pCache->pTsmas->size == 0) && !pCache->retryFetch) { + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + ctgReleaseTSMAToCache(pCtg, dbCache, pCache); + CTG_ERR_RET(terrno); + } + CTG_UNLOCK(CTG_READ, &pCache->tsmaLock); + taosHashRelease(dbCache->tsmaCache, pCache); continue; } - CTG_LOCK(CTG_READ, &pCache->tsmaLock); - if (hasOutOfDateTSMACache(pCache->pTsmas)) { + if (pCache->retryFetch || hasOutOfDateTSMACache(pCache->pTsmas)) { CTG_UNLOCK(CTG_READ, &pCache->tsmaLock); - taosHashRelease(dbCache->tsmaCache, pCache); ctgDebug("tsma for tb: %s.%s not in cache", tsmaSourceTbName.tname, dbFName); CTG_ERR_JRET(ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TB_TSMA, &tsmaSourceTbName)); if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + taosHashRelease(dbCache->tsmaCache, pCache); CTG_ERR_JRET(terrno); } CTG_CACHE_NHIT_INC(CTG_CI_TBL_TSMA, 1); + CTG_LOCK(CTG_WRITE, &pCache->tsmaLock); + pCache->retryFetch = false; + CTG_UNLOCK(CTG_WRITE, &pCache->tsmaLock); + taosHashRelease(dbCache->tsmaCache, pCache); continue; } diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index d6e941c819..ed9dc81dd7 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -47,7 +47,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu msgNum = 0; } - ctgDebug("qid:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, + ctgDebug("QID:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, TMSG_INFO(cbParam->reqType + 1)); SHashObj* pBatchs = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -114,7 +114,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu pMsgCtx->pBatchs = pBatchs; - ctgDebug("qid:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId, + ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId, pRsp->msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs); (void)(*gCtgAsyncFps[pTask->type].handleRspFp)( @@ -454,7 +454,7 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) { CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("qid:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, + qDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1)); #if CTG_BATCH_FETCH @@ -702,7 +702,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (TDMT_VND_TABLE_CFG == msgType) { SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; pName = ctx->pName; - } else if (TDMT_VND_TABLE_META == msgType || TDMT_VND_TABLE_NAME == msgType) { + } else if (TDMT_VND_TABLE_META == msgType || TDMT_VND_TABLE_NAME == msgType) { if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); @@ -808,7 +808,7 @@ int32_t ctgLaunchBatchs(SCatalog* pCtg, SCtgJob* pJob, SHashObj* pBatchs) { SCtgBatch* pBatch = (SCtgBatch*)p; int32_t msgSize = 0; - ctgDebug("qid:0x%" PRIx64 " ctg start to launch batch %d", pJob->queryId, pBatch->batchId); + ctgDebug("QID:0x%" PRIx64 " ctg start to launch batch %d", pJob->queryId, pBatch->batchId); CTG_ERR_JRET(ctgBuildBatchReqMsg(pBatch, *vgId, &msg, &msgSize)); code = ctgAsyncSendMsg(pCtg, &pBatch->conn, pJob, pBatch->pTaskIds, pBatch->batchId, pBatch->pMsgIdxs, @@ -1124,10 +1124,11 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n int32_t code = tNameExtractFullName(name, tbFName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), name->type, name->dbname, name->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), name->type, + name->dbname, name->tname); CTG_ERR_RET(code); } - + code = queryBuildMsg[TMSG_INDEX(reqType)]((void*)tbFName, &msg, 0, &msgLen, mallocFp); if (code) { ctgError("Build get index msg failed, code:%s, tbFName:%s", tstrerror(code), tbFName); @@ -1450,7 +1451,8 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t code = tNameExtractFullName(pTableName, tbFName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, pTableName->dbname, pTableName->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, + pTableName->dbname, pTableName->tname); CTG_ERR_RET(code); } @@ -1523,7 +1525,8 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t code = tNameExtractFullName(pTableName, tbFName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, pTableName->dbname, pTableName->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, + pTableName->dbname, pTableName->tname); CTG_ERR_RET(code); } @@ -1632,10 +1635,11 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* int32_t reqType = TDMT_MND_VIEW_META; SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pName, fullName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pName->type, pName->dbname, pName->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pName->type, + pName->dbname, pName->tname); CTG_ERR_RET(code); } @@ -1693,10 +1697,11 @@ int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa int32_t msgLen = 0; SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(name, tbFName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), name->type, name->dbname, name->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), name->type, + name->dbname, name->tname); CTG_ERR_RET(code); } @@ -1757,10 +1762,11 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pTbName, tbFName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTbName->type, pTbName->dbname, pTbName->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTbName->type, + pTbName->dbname, pTbName->tname); CTG_ERR_RET(code); } - + SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 96cd783d2f..86a38017bd 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -452,7 +452,8 @@ void ctgClearHandleMeta(SCatalog* pCtg, int64_t* pClearedSize, int64_t* pCleardN code = taosHashRemove(dbCache->tbCache, key, len); if (code) { - qError("taosHashRemove table cache failed, key:%s, len:%d, error:%s", (char*)key, (int32_t)len, tstrerror(code)); + qError("taosHashRemove table cache failed, key:%s, len:%d, error:%s", (char*)key, (int32_t)len, + tstrerror(code)); } cacheSize = @@ -1096,7 +1097,7 @@ void ctgFreeJob(void* job) { taosMemoryFree(job); - qDebug("qid:0x%" PRIx64 ", ctg job 0x%" PRIx64 " freed", qid, rid); + qDebug("QID:0x%" PRIx64 ", ctg job 0x%" PRIx64 " freed", qid, rid); } int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target) { @@ -1241,10 +1242,11 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* d char tbFullName[TSDB_TABLE_FNAME_LEN]; code = tNameExtractFullName(pTableName, tbFullName); if (code) { - ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, pTableName->dbname, pTableName->tname); + ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, + pTableName->dbname, pTableName->tname); CTG_ERR_RET(code); } - + uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); @@ -1704,7 +1706,8 @@ int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) { } int32_t ctgUpdateSendTargetInfo(SMsgSendInfo* pMsgSendInfo, int32_t msgType, char* dbFName, int32_t vgId) { - if (msgType == TDMT_VND_TABLE_META || msgType == TDMT_VND_TABLE_CFG || msgType == TDMT_VND_BATCH_META || msgType == TDMT_VND_TABLE_NAME) { + if (msgType == TDMT_VND_TABLE_META || msgType == TDMT_VND_TABLE_CFG || msgType == TDMT_VND_BATCH_META || + msgType == TDMT_VND_TABLE_NAME) { pMsgSendInfo->target.type = TARGET_TYPE_VNODE; pMsgSendInfo->target.vgId = vgId; pMsgSendInfo->target.dbFName = taosStrdup(dbFName); @@ -2010,7 +2013,8 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { char dbFName[TSDB_DB_FNAME_LEN]; code = tNameExtractFullName(&req->pRawReq->tbName, tbFName); if (code) { - ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), req->pRawReq->tbName.type, req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); + ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), + req->pRawReq->tbName.type, req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); CTG_ERR_RET(code); } @@ -2201,7 +2205,8 @@ int32_t ctgChkSetViewAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) } else { code = tNameExtractFullName(&req->pRawReq->tbName, viewFName); if (code) { - ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), req->pRawReq->tbName.type, req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); + ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), + req->pRawReq->tbName.type, req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); CTG_ERR_RET(code); } } diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 4a2d6bfcd3..e391d274e3 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -553,6 +553,7 @@ typedef struct SIntervalAggOperatorInfo { EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] STimeWindowAggSupp twAggSup; SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation. + struct SOperatorInfo* pOperator; // for limit optimization bool limited; int64_t limit; @@ -621,6 +622,7 @@ typedef struct SStreamIntervalOperatorInfo { int32_t midDelIndex; SSHashObj* pDeletedMap; bool destHasPrimaryKey; + struct SOperatorInfo* pOperator; } SStreamIntervalOperatorInfo; typedef struct SDataGroupInfo { @@ -676,6 +678,7 @@ typedef struct SStreamSessionAggOperatorInfo { bool recvGetAll; bool destHasPrimaryKey; SSHashObj* pPkDeleted; + struct SOperatorInfo* pOperator; } SStreamSessionAggOperatorInfo; typedef struct SStreamStateAggOperatorInfo { @@ -703,6 +706,7 @@ typedef struct SStreamStateAggOperatorInfo { bool recvGetAll; SSHashObj* pPkDeleted; bool destHasPrimaryKey; + struct SOperatorInfo* pOperator; } SStreamStateAggOperatorInfo; typedef struct SStreamEventAggOperatorInfo { @@ -732,6 +736,7 @@ typedef struct SStreamEventAggOperatorInfo { SFilterInfo* pEndCondInfo; SSHashObj* pPkDeleted; bool destHasPrimaryKey; + struct SOperatorInfo* pOperator; } SStreamEventAggOperatorInfo; typedef struct SStreamCountAggOperatorInfo { @@ -756,6 +761,7 @@ typedef struct SStreamCountAggOperatorInfo { SSDataBlock* pCheckpointRes; SSHashObj* pPkDeleted; bool destHasPrimaryKey; + struct SOperatorInfo* pOperator; } SStreamCountAggOperatorInfo; typedef struct SStreamPartitionOperatorInfo { @@ -823,6 +829,10 @@ void cleanupBasicInfo(SOptrBasicInfo* pInfo); int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr, SFunctionStateStore* pStore); void cleanupExprSupp(SExprSupp* pSup); +void cleanupResultInfoInStream(SExecTaskInfo* pTaskInfo, void* pState, SExprSupp* pSup, + SGroupResInfo* pGroupResInfo); +void cleanupResultInfo(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap); int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, const char* pkey, void* pState, SFunctionStateStore* pStore); void cleanupAggSup(SAggSupporter* pAggSup); diff --git a/source/libs/executor/inc/streamexecutorInt.h b/source/libs/executor/inc/streamexecutorInt.h index c6053cc96e..ab00dceb20 100644 --- a/source/libs/executor/inc/streamexecutorInt.h +++ b/source/libs/executor/inc/streamexecutorInt.h @@ -26,6 +26,8 @@ void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type); bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo); void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo); +void reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI); + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index f1c9f1d32b..9e5ad132f7 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -48,6 +48,7 @@ typedef struct SAggOperatorInfo { bool hasValidBlock; SSDataBlock* pNewGroupBlock; bool hasCountFunc; + SOperatorInfo* pOperator; } SAggOperatorInfo; static void destroyAggOperatorInfo(void* param); @@ -119,6 +120,7 @@ int32_t createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pA pInfo->binfo.inputTsOrder = pAggNode->node.inputTsOrder; pInfo->binfo.outputTsOrder = pAggNode->node.outputTsOrder; pInfo->hasCountFunc = pAggNode->hasCountLikeFunc; + pInfo->pOperator = pOperator; setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, !pAggNode->node.forceCreateNonBlockingOptr, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -150,9 +152,17 @@ _error: } void destroyAggOperatorInfo(void* param) { + if (param == NULL) { + return; + } SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); + if (pInfo->pOperator) { + cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, + &pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable); + pInfo->pOperator = NULL; + } cleanupAggSup(&pInfo->aggSup); cleanupExprSupp(&pInfo->scalarExprSup); cleanupGroupResInfo(&pInfo->groupResInfo); @@ -581,6 +591,80 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n return code; } +void cleanupResultInfoInStream(SExecTaskInfo* pTaskInfo, void* pState, SExprSupp* pSup, SGroupResInfo* pGroupResInfo) { + int32_t code = TSDB_CODE_SUCCESS; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + int32_t numOfExprs = pSup->numOfExprs; + int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; + SqlFunctionCtx* pCtx = pSup->pCtx; + int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); + bool needCleanup = false; + + for (int32_t j = 0; j < numOfExprs; ++j) { + needCleanup |= pCtx[j].needCleanup; + } + if (!needCleanup) { + return; + } + + for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { + SResultWindowInfo* pWinInfo = taosArrayGet(pGroupResInfo->pRows, i); + SRowBuffPos* pPos = pWinInfo->pStatePos; + SResultRow* pRow = NULL; + + code = pAPI->stateStore.streamStateGetByPos(pState, pPos, (void**)&pRow); + if (TSDB_CODE_SUCCESS != code) { + qError("failed to get state by pos, code:%s, %s", tstrerror(code), GET_TASKID(pTaskInfo)); + continue; + } + + for (int32_t j = 0; j < numOfExprs; ++j) { + pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); + if (pCtx[j].fpSet.cleanup) { + pCtx[j].fpSet.cleanup(&pCtx[j]); + } + } + } +} + +void cleanupResultInfo(SExecTaskInfo* pTaskInfo, SExprSupp* pSup, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap) { + int32_t numOfExprs = pSup->numOfExprs; + int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; + SqlFunctionCtx* pCtx = pSup->pCtx; + bool needCleanup = false; + for (int32_t j = 0; j < numOfExprs; ++j) { + needCleanup |= pCtx[j].needCleanup; + } + if (!needCleanup) { + return; + } + + // begin from last iter + void* pData = pGroupResInfo->dataPos; + int32_t iter = pGroupResInfo->iter; + while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { + SResultRowPosition* pos = pData; + + SFilePage* page = getBufPage(pBuf, pos->pageId); + if (page == NULL) { + qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); + continue; + } + + SResultRow* pRow = (SResultRow*)((char*)page + pos->offset); + + for (int32_t j = 0; j < numOfExprs; ++j) { + pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); + if (pCtx[j].fpSet.cleanup) { + pCtx[j].fpSet.cleanup(&pCtx[j]); + } + } + + releaseBufPage(pBuf, page); + } +} + void cleanupAggSup(SAggSupporter* pAggSup) { taosMemoryFreeClear(pAggSup->keyBuf); tSimpleHashCleanup(pAggSup->pResultRowHashTable); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 7c1ca294e7..649a7a4524 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -347,11 +347,7 @@ static int32_t doScanCacheNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { SExprSupp* pSup = &pInfo->pseudoExprSup; code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pRes, pRes->info.rows, pTaskInfo, NULL); - if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; - (*ppRes) = NULL; - return code; - } + QUERY_CHECK_CODE(code, lino, _end); pRes->info.id.groupId = tableListGetTableGroupId(pTableList, pRes->info.id.uid); pInfo->indexOfBufferedRes += 1; @@ -414,11 +410,7 @@ static int32_t doScanCacheNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { pInfo->pRes->info.id.uid = *(tb_uid_t*)pUid; code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, pInfo->pRes->info.rows, pTaskInfo, NULL); - if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; - (*ppRes) = NULL; - return code; - } + QUERY_CHECK_CODE(code, lino, _end); } } diff --git a/source/libs/executor/src/dynqueryctrloperator.c b/source/libs/executor/src/dynqueryctrloperator.c index 638536349d..eb49057d89 100644 --- a/source/libs/executor/src/dynqueryctrloperator.c +++ b/source/libs/executor/src/dynqueryctrloperator.c @@ -938,7 +938,9 @@ _return: } if (code) { + qError("%s failed since %s", __func__, tstrerror(code)); pOperator->pTaskInfo->code = code; + T_LONG_JMP(pOperator->pTaskInfo->env, code); } else { seqStableJoinComposeRes(pStbJoin, *pRes); } diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c index e164e07252..f473626953 100644 --- a/source/libs/executor/src/eventwindowoperator.c +++ b/source/libs/executor/src/eventwindowoperator.c @@ -37,6 +37,7 @@ typedef struct SEventWindowOperatorInfo { bool inWindow; SResultRow* pRow; SSDataBlock* pPreDataBlock; + SOperatorInfo* pOperator; } SEventWindowOperatorInfo; static int32_t eventWindowAggregateNext(SOperatorInfo* pOperator, SSDataBlock** pRes); @@ -128,6 +129,7 @@ int32_t createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* phy pInfo->tsSlotId = tsSlotId; pInfo->pPreDataBlock = NULL; + pInfo->pOperator = pOperator; setOperatorInfo(pOperator, "EventWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -152,6 +154,19 @@ _error: return code; } +void cleanupResultInfoInEventWindow(SOperatorInfo* pOperator, SEventWindowOperatorInfo* pInfo) { + if (pInfo == NULL || pInfo->pRow == NULL || pOperator == NULL) { + return; + } + SExprSupp* pSup = &pOperator->exprSupp; + for (int32_t j = 0; j < pSup->numOfExprs; ++j) { + pSup->pCtx[j].resultInfo = getResultEntryInfo(pInfo->pRow, j, pSup->rowEntryInfoOffset); + if (pSup->pCtx[j].fpSet.cleanup) { + pSup->pCtx[j].fpSet.cleanup(&pSup->pCtx[j]); + } + } +} + void destroyEWindowOperatorInfo(void* param) { SEventWindowOperatorInfo* pInfo = (SEventWindowOperatorInfo*)param; if (pInfo == NULL) { @@ -175,6 +190,8 @@ void destroyEWindowOperatorInfo(void* param) { cleanupBasicInfo(&pInfo->binfo); colDataDestroy(&pInfo->twAggSup.timeWindowData); + cleanupResultInfoInEventWindow(pInfo->pOperator, pInfo); + pInfo->pOperator = NULL; cleanupAggSup(&pInfo->aggSup); cleanupExprSupp(&pInfo->scalarSup); taosMemoryFreeClear(param); diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index fa6f61406c..767796977c 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -236,7 +236,7 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { taosArrayRemove(pExchangeInfo->pResultBlockList, 0); void* tmp = taosArrayPush(pExchangeInfo->pRecycledBlocks, &p); if (!tmp) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); pTaskInfo->code = code; T_LONG_JMP(pTaskInfo->env, code); @@ -590,7 +590,7 @@ int32_t buildTableScanOperatorParam(SOperatorParam** ppRes, SArray* pUidList, in if (NULL == pScan->pUidList) { taosMemoryFree(pScan); taosMemoryFreeClear(*ppRes); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pScan->tableSeq = tableSeq; @@ -1094,6 +1094,7 @@ int32_t addDynamicExchangeSource(SOperatorInfo* pOperator) { int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if ((OPTR_IS_OPENED(pOperator) && !pExchangeInfo->dynamicOp) || (pExchangeInfo->dynamicOp && NULL == pOperator->pOperatorGetParam)) { return TSDB_CODE_SUCCESS; @@ -1101,23 +1102,26 @@ int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { if (pExchangeInfo->dynamicOp) { code = addDynamicExchangeSource(pOperator); - if (code) { - return code; - } + QUERY_CHECK_CODE(code, lino, _end); } int64_t st = taosGetTimestampUs(); if (!pExchangeInfo->seqLoadData) { - int32_t code = prepareConcurrentlyLoad(pOperator); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = prepareConcurrentlyLoad(pOperator); + QUERY_CHECK_CODE(code, lino, _end); pExchangeInfo->openedTs = taosGetTimestampUs(); } OPTR_SET_OPENED(pOperator); pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pOperator->pTaskInfo->code = code; + T_LONG_JMP(pOperator->pTaskInfo->env, code); + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 117a30ade2..c74aef3992 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1143,11 +1143,11 @@ SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, S varDataSetLen(tmp, tagVal.nData); memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData); code = colDataSetVal(pColInfo, i, tmp, false); - QUERY_CHECK_CODE(code, lino, _end); #if TAG_FILTER_DEBUG qDebug("tagfilter varch:%s", tmp + 2); #endif taosMemoryFree(tmp); + QUERY_CHECK_CODE(code, lino, _end); } else { code = colDataSetVal(pColInfo, i, (const char*)&tagVal.i64, false); QUERY_CHECK_CODE(code, lino, _end); @@ -2111,6 +2111,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, pCtx->saveHandle.currentPage = -1; pCtx->pStore = pStore; pCtx->hasWindowOrGroup = false; + pCtx->needCleanup = false; } for (int32_t i = 1; i < numOfOutput; ++i) { @@ -2210,10 +2211,8 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi return terrno; } pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t) * pCond->numOfCols); - if (pCond->colList == NULL || pCond->pSlotList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + if (pCond->pSlotList == NULL) { taosMemoryFreeClear(pCond->colList); - taosMemoryFreeClear(pCond->pSlotList); return terrno; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index c3228f59bf..cbf392f67e 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -798,7 +798,6 @@ void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo) { int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; int64_t threadId = taosGetSelfPthreadId(); - int32_t lino = 0; int64_t curOwner = 0; *pRes = NULL; @@ -846,7 +845,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { int32_t code = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot, pRes); if (code) { pTaskInfo->code = code; - qError("%s failed at line %d, code:%s %s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); + qError("%s failed at line %d, code:%s %s", __func__, __LINE__, tstrerror(code), GET_TASKID(pTaskInfo)); } blockDataCheck(*pRes, false); diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 07224961f9..64a07c4653 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -687,10 +687,10 @@ int32_t copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResu code = blockDataEnsureCapacity(pBlock, pBlock->info.rows + pCtx[j].resultInfo->numOfRes); QUERY_CHECK_CODE(code, lino, _end); - int32_t winCode = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); - if (TAOS_FAILED(winCode)) { - qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(winCode)); - QUERY_CHECK_CODE(winCode, lino, _end); + code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + if (TSDB_CODE_SUCCESS != code) { + qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); + QUERY_CHECK_CODE(code, lino, _end); } } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { // do nothing @@ -1020,10 +1020,6 @@ static void destroySqlFunctionCtx(SqlFunctionCtx* pCtx, SExprInfo* pExpr, int32_ } for (int32_t i = 0; i < numOfOutput; ++i) { - if (pCtx[i].fpSet.cleanup != NULL) { - pCtx[i].fpSet.cleanup(&pCtx[i]); - } - if (pExpr != NULL) { SExprInfo* pExprInfo = &pExpr[i]; for (int32_t j = 0; j < pExprInfo->base.numOfParams; ++j) { @@ -1305,10 +1301,17 @@ FORCE_INLINE int32_t getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOpera freeOperatorParam(pOperator->pDownstreamGetParams[idx], OP_GET_PARAM); pOperator->pDownstreamGetParams[idx] = NULL; } + + if (code) { + qError("failed to get next data block from upstream at %s, line:%d code:%s", __func__, __LINE__, tstrerror(code)); + } return code; } code = pOperator->pDownstream[idx]->fpSet.getNextFn(pOperator->pDownstream[idx], pResBlock); + if (code) { + qError("failed to get next data block from upstream at %s, %d code:%s", __func__, __LINE__, tstrerror(code)); + } return code; } diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index e4f6d73b7b..d785a1e619 100644 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -333,14 +333,14 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC continue; } - int32_t ret = taosLSeekFile(pFd->fd, pHead->basic.offset, SEEK_SET); + int64_t ret = taosLSeekFile(pFd->fd, pHead->basic.offset, SEEK_SET); if (ret < 0) { releaseFdToFileCtx(pFd); code = terrno; goto _return; } - ret = (int32_t)taosWriteFile(pFd->fd, pHead->pBuf, pHead->basic.bufSize); + ret = taosWriteFile(pFd->fd, pHead->pBuf, pHead->basic.bufSize); if (ret != pHead->basic.bufSize) { releaseFdToFileCtx(pFd); code = terrno; @@ -578,7 +578,7 @@ static int32_t readBlockFromDisk(SGroupCacheOperatorInfo* pGCache, SGroupCacheDa return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } - int32_t ret = taosLSeekFile(pFileFd->fd, pBasic->offset, SEEK_SET); + int64_t ret = taosLSeekFile(pFileFd->fd, pBasic->offset, SEEK_SET); if (ret < 0) { code = terrno; goto _return; @@ -590,7 +590,7 @@ static int32_t readBlockFromDisk(SGroupCacheOperatorInfo* pGCache, SGroupCacheDa goto _return; } - ret = (int32_t)taosReadFile(pFileFd->fd, *ppBuf, pBasic->bufSize); + ret = taosReadFile(pFileFd->fd, *ppBuf, pBasic->bufSize); if (ret != pBasic->bufSize) { taosMemoryFreeClear(*ppBuf); code = terrno; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index c862a44461..9cf2a3ea17 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -39,6 +39,7 @@ typedef struct SGroupbyOperatorInfo { int32_t groupKeyLen; // total group by column width SGroupResInfo groupResInfo; SExprSupp scalarSup; + SOperatorInfo *pOperator; } SGroupbyOperatorInfo; // The sort in partition may be needed later. @@ -75,10 +76,10 @@ static void freeGroupKey(void* param) { } static void destroyGroupOperatorInfo(void* param) { - SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*)param; - if (pInfo == NULL) { + if (param == NULL) { return; } + SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); taosMemoryFreeClear(pInfo->keyBuf); @@ -86,6 +87,12 @@ static void destroyGroupOperatorInfo(void* param) { taosArrayDestroyEx(pInfo->pGroupColVals, freeGroupKey); cleanupExprSupp(&pInfo->scalarSup); + if (pInfo->pOperator != NULL) { + cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, + &pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable); + pInfo->pOperator = NULL; + } + cleanupGroupResInfo(&pInfo->groupResInfo); cleanupAggSup(&pInfo->aggSup); taosMemoryFreeClear(param); @@ -447,7 +454,7 @@ static int32_t hashGroupbyAggregateNext(SOperatorInfo* pOperator, SSDataBlock** QRY_PARAM_CHECK(ppRes); if (pOperator->status == OP_EXEC_DONE) { - return TSDB_CODE_SUCCESS; + return code; } if (pOperator->status == OP_RES_TO_RETURN) { @@ -499,6 +506,7 @@ _end: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } else { (*ppRes) = buildGroupResultDataBlockByHash(pOperator); } @@ -569,6 +577,8 @@ int32_t createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNo pInfo->binfo.inputTsOrder = pAggNode->node.inputTsOrder; pInfo->binfo.outputTsOrder = pAggNode->node.outputTsOrder; + pInfo->pOperator = pOperator; + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, hashGroupbyAggregateNext, NULL, destroyGroupOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); code = appendDownstream(pOperator, &downstream, 1); @@ -1528,8 +1538,9 @@ static int32_t doStreamHashPartitionNext(SOperatorInfo* pOperator, SSDataBlock** _end: if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } (*ppRes) = NULL; return code; diff --git a/source/libs/executor/src/hashjoinoperator.c b/source/libs/executor/src/hashjoinoperator.c index 347c48b4d1..1f43a429b3 100644 --- a/source/libs/executor/src/hashjoinoperator.c +++ b/source/libs/executor/src/hashjoinoperator.c @@ -993,6 +993,7 @@ static int32_t hJoinMainProcess(struct SOperatorInfo* pOperator, SSDataBlock** p SHJoinOperatorInfo* pJoin = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SSDataBlock* pRes = pJoin->finBlk; int64_t st = 0; @@ -1003,7 +1004,7 @@ static int32_t hJoinMainProcess(struct SOperatorInfo* pOperator, SSDataBlock** p if (pOperator->status == OP_EXEC_DONE) { pRes->info.rows = 0; - goto _return; + goto _end; } if (!pJoin->keyHashBuilt) { @@ -1011,13 +1012,10 @@ static int32_t hJoinMainProcess(struct SOperatorInfo* pOperator, SSDataBlock** p bool queryDone = false; code = hJoinBuildHash(pOperator, &queryDone); - if (code) { - pTaskInfo->code = code; - return code; - } + QUERY_CHECK_CODE(code, lino, _end); if (queryDone) { - goto _return; + goto _end; } } @@ -1025,17 +1023,11 @@ static int32_t hJoinMainProcess(struct SOperatorInfo* pOperator, SSDataBlock** p if (pJoin->ctx.rowRemains) { code = (*pJoin->joinFp)(pOperator); - if (code) { - pTaskInfo->code = code; - return pTaskInfo->code; - } + QUERY_CHECK_CODE(code, lino, _end); if (pRes->info.rows > 0 && pJoin->pFinFilter != NULL) { code = doFilter(pRes, pJoin->pFinFilter, NULL); - if (code) { - pTaskInfo->code = code; - return pTaskInfo->code; - } + QUERY_CHECK_CODE(code, lino, _end); } if (pRes->info.rows > 0) { @@ -1055,10 +1047,7 @@ static int32_t hJoinMainProcess(struct SOperatorInfo* pOperator, SSDataBlock** p pJoin->execInfo.probeBlkRows += pBlock->info.rows; code = hJoinPrepareStart(pOperator, pBlock); - if (code) { - pTaskInfo->code = code; - return pTaskInfo->code; - } + QUERY_CHECK_CODE(code, lino, _end); if (!hJoinBlkReachThreshold(pJoin, pRes->info.rows)) { continue; @@ -1066,10 +1055,7 @@ static int32_t hJoinMainProcess(struct SOperatorInfo* pOperator, SSDataBlock** p if (pRes->info.rows > 0 && pJoin->pFinFilter != NULL) { code = doFilter(pRes, pJoin->pFinFilter, NULL); - if (code) { - pTaskInfo->code = code; - return pTaskInfo->code; - } + QUERY_CHECK_CODE(code, lino, _end); } if (pRes->info.rows > 0) { @@ -1077,11 +1063,15 @@ static int32_t hJoinMainProcess(struct SOperatorInfo* pOperator, SSDataBlock** p } } -_return: +_end: if (pOperator->cost.openCost == 0) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; } - + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } if (pRes->info.rows > 0) { *pResBlock = pRes; } diff --git a/source/libs/executor/src/mergejoinoperator.c b/source/libs/executor/src/mergejoinoperator.c index af5e4ed235..e007504ffb 100644 --- a/source/libs/executor/src/mergejoinoperator.c +++ b/source/libs/executor/src/mergejoinoperator.c @@ -1731,6 +1731,7 @@ int32_t mJoinMainProcess(struct SOperatorInfo* pOperator, SSDataBlock** pResBloc if (pJoin->pFinFilter != NULL) { code = doFilter(pBlock, pJoin->pFinFilter, NULL); if (code) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); pJoin->errCode = code; T_LONG_JMP(pOperator->pTaskInfo->env, pJoin->errCode); } diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c index 49973ac373..7fd6b91e52 100644 --- a/source/libs/executor/src/mergeoperator.c +++ b/source/libs/executor/src/mergeoperator.c @@ -67,6 +67,9 @@ int32_t sortMergeloadNextDataBlock(void* param, SSDataBlock** ppBlock) { SOperatorInfo* pOperator = (SOperatorInfo*)param; int32_t code = pOperator->fpSet.getNextFn(pOperator, ppBlock); blockDataCheck(*ppBlock, false); + if (code) { + qError("failed to get next data block from upstream, %s code:%s", __func__, tstrerror(code)); + } return code; } @@ -492,6 +495,8 @@ int32_t openMultiwayMergeOperator(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + pOperator->pTaskInfo->code = code; T_LONG_JMP(pTaskInfo->env, terrno); } @@ -501,6 +506,8 @@ int32_t openMultiwayMergeOperator(SOperatorInfo* pOperator) { int32_t doMultiwayMerge(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { QRY_PARAM_CHECK(pResBlock); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pOperator->status == OP_EXEC_DONE) { return 0; @@ -509,18 +516,12 @@ int32_t doMultiwayMerge(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SMultiwayMergeOperatorInfo* pInfo = pOperator->info; - int32_t code = pOperator->fpSet._openFn(pOperator); - if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; - return code; - } + code = pOperator->fpSet._openFn(pOperator); + QUERY_CHECK_CODE(code, lino, _end); if (NULL != gMultiwayMergeFps[pInfo->type].getNextFn) { code = (*gMultiwayMergeFps[pInfo->type].getNextFn)(pOperator, pResBlock); - if (code) { - pTaskInfo->code = code; - return code; - } + QUERY_CHECK_CODE(code, lino, _end); } if ((*pResBlock) != NULL) { @@ -530,6 +531,12 @@ int32_t doMultiwayMerge(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { setOperatorCompleted(pOperator); } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } return code; } diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index fe2f3f8dfe..8daf4695db 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -667,13 +667,11 @@ void destroyOperator(SOperatorInfo* pOperator) { pOperator->numOfDownstream = 0; } - cleanupExprSupp(&pOperator->exprSupp); - - // close operator after cleanup exprSupp, since we need to call cleanup of sqlFunctionCtx first to avoid mem leak. if (pOperator->fpSet.closeFn != NULL && pOperator->info != NULL) { pOperator->fpSet.closeFn(pOperator->info); } + cleanupExprSupp(&pOperator->exprSupp); taosMemoryFreeClear(pOperator); } @@ -883,14 +881,17 @@ SSDataBlock* getNextBlockFromDownstreamRemain(struct SOperatorInfo* pOperator, i int32_t optrDefaultGetNextExtFn(struct SOperatorInfo* pOperator, SOperatorParam* pParam, SSDataBlock** pRes) { QRY_PARAM_CHECK(pRes); + int32_t lino = 0; int32_t code = setOperatorParams(pOperator, pParam, OP_GET_PARAM); - if (TSDB_CODE_SUCCESS != code) { + QUERY_CHECK_CODE(code, lino, _end); + code = pOperator->fpSet.getNextFn(pOperator, pRes); + QUERY_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); pOperator->pTaskInfo->code = code; - } else { - code = pOperator->fpSet.getNextFn(pOperator, pRes); - if (code) { - pOperator->pTaskInfo->code = code; - } + T_LONG_JMP(pOperator->pTaskInfo->env, code); } return code; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 790e97b27c..5b9e531679 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -270,6 +270,7 @@ int32_t doProjectOperation(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { SSDataBlock* pRes = pInfo->pRes; SSDataBlock* pFinalRes = pProjectInfo->pFinalRes; int32_t code = 0; + int32_t lino = 0; int64_t st = 0; int32_t order = pInfo->inputTsOrder; int32_t scanFlag = 0; @@ -290,9 +291,7 @@ int32_t doProjectOperation(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { if (downstream == NULL) { code = doGenerateSourceData(pOperator); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); - } + QUERY_CHECK_CODE(code, lino, _end); if (pProjectInfo->outputIgnoreGroup) { pRes->info.id.groupId = 0; @@ -348,20 +347,14 @@ int32_t doProjectOperation(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { } code = setInputDataBlock(pSup, pBlock, order, scanFlag, false); - if (code) { - T_LONG_JMP(pTaskInfo->env, code); - } + QUERY_CHECK_CODE(code, lino, _end); code = blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); - } + QUERY_CHECK_CODE(code, lino, _end); code = projectApplyFunctions(pSup->pExprInfo, pInfo->pRes, pBlock, pSup->pCtx, pSup->numOfExprs, pProjectInfo->pPseudoColInfo); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); - } + QUERY_CHECK_CODE(code, lino, _end); status = doIngroupLimitOffset(pLimitInfo, pBlock->info.id.groupId, pInfo->pRes, pOperator); if (status == PROJECT_RETRIEVE_CONTINUE) { @@ -377,11 +370,8 @@ int32_t doProjectOperation(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { pFinalRes->info.version = pRes->info.version; // continue merge data, ignore the group id - int32_t ret = blockDataMerge(pFinalRes, pRes); - if (ret < 0) { - pTaskInfo->code = code; - return code; - } + code = blockDataMerge(pFinalRes, pRes); + QUERY_CHECK_CODE(code, lino, _end); if (pFinalRes->info.rows + pRes->info.rows <= pOperator->resultInfo.threshold && (pOperator->status != OP_EXEC_DONE)) { continue; @@ -390,10 +380,7 @@ int32_t doProjectOperation(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { // do apply filter code = doFilter(pFinalRes, pOperator->exprSupp.pFilterInfo, NULL); - if (code) { - pTaskInfo->code = code; - return code; - } + QUERY_CHECK_CODE(code, lino, _end); // when apply the limit/offset for each group, pRes->info.rows may be 0, due to limit constraint. if (pFinalRes->info.rows > 0 || (pOperator->status == OP_EXEC_DONE)) { @@ -404,10 +391,7 @@ int32_t doProjectOperation(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { // do apply filter if (pRes->info.rows > 0) { code = doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL); - if (code) { - pTaskInfo->code = code; - return code; - } + QUERY_CHECK_CODE(code, lino, _end); if (pRes->info.rows == 0) { continue; @@ -436,6 +420,13 @@ int32_t doProjectOperation(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { } *pResBlock = (p->info.rows > 0)? p:NULL; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } return code; } @@ -578,14 +569,15 @@ int32_t doApplyIndefinitFunction(SOperatorInfo* pOperator, SSDataBlock** pResBlo SOptrBasicInfo* pInfo = &pIndefInfo->binfo; SExprSupp* pSup = &pOperator->exprSupp; int64_t st = 0; - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SSDataBlock* pRes = pInfo->pRes; blockDataCleanup(pRes); SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (pOperator->status == OP_EXEC_DONE) { - return 0; + return code; } if (pOperator->cost.openCost == 0) { @@ -637,10 +629,7 @@ int32_t doApplyIndefinitFunction(SOperatorInfo* pOperator, SSDataBlock** pResBlo } code = doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); - if (code) { - pTaskInfo->code = code; - return code; - } + QUERY_CHECK_CODE(code, lino, _end); size_t rows = pInfo->pRes->info.rows; if (rows > 0 || pOperator->status == OP_EXEC_DONE) { @@ -658,6 +647,13 @@ int32_t doApplyIndefinitFunction(SOperatorInfo* pOperator, SSDataBlock** pResBlo } *pResBlock = (rows > 0) ? pInfo->pRes : NULL; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } return code; } diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index 4821461ca2..37461382dd 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -73,7 +73,7 @@ int32_t doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC p->schemaInfos = taosArrayInit(1, sizeof(SSchemaInfo)); if (p->id.str == NULL || p->schemaInfos == NULL) { doDestroyTask(p); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } *pTaskInfo = p; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 76e4f8ba56..bae9926f63 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -737,7 +737,7 @@ _end: if (NULL != pVal) { insertRet = taosLRUCacheInsert(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(uint64_t), pVal, - sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW, NULL); + sizeof(STableCachedVal), freeCachedMetaItem, NULL, NULL, TAOS_LRU_PRIORITY_LOW, NULL); if (insertRet != TAOS_LRU_STATUS_OK) { qWarn("failed to put meta into lru cache, code:%d, %s", insertRet, idStr); } @@ -890,7 +890,7 @@ void markGroupProcessed(STableScanInfo* pInfo, uint64_t groupId) { } else { int32_t code = taosHashRemove(pInfo->base.pTableListInfo->remainGroups, &groupId, sizeof(groupId)); if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + qDebug("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); } } } @@ -951,7 +951,8 @@ static int32_t doTableScanImplNext(SOperatorInfo* pOperator, SSDataBlock** ppRes if (isTaskKilled(pTaskInfo)) { pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader); - return pTaskInfo->code; + code = pTaskInfo->code; + goto _end; } if (pOperator->status == OP_EXEC_DONE) { @@ -996,6 +997,7 @@ _end: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } return code; } @@ -1378,8 +1380,7 @@ static int32_t doTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { code = tableListGetSize(pInfo->base.pTableListInfo, &numOfTables); if (code != TSDB_CODE_SUCCESS) { taosRUnLockLatch(&pTaskInfo->lock); - lino = __LINE__; - goto _end; + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->currentTable >= numOfTables) { @@ -1391,11 +1392,11 @@ static int32_t doTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { STableKeyInfo* tmp = (STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); if (!tmp) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); taosRUnLockLatch(&pTaskInfo->lock); (*ppRes) = NULL; - return terrno; + QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); } + tInfo = *tmp; taosRUnLockLatch(&pTaskInfo->lock); @@ -1410,12 +1411,14 @@ static int32_t doTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { } } else { // scan table group by group sequentially code = groupSeqTableScan(pOperator, ppRes); + QUERY_CHECK_CODE(code, lino, _end); } _end: if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + qError("%s %s failed at line %d since %s", GET_TASKID(pTaskInfo), __func__, lino, tstrerror(code)); pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } return code; @@ -2944,8 +2947,9 @@ static int32_t doQueueScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { if (pResult && pResult->info.rows > 0) { bool hasPrimaryKey = pAPI->tqReaderFn.tqGetTablePrimaryKey(pInfo->tqReader); code = processPrimaryKey(pResult, hasPrimaryKey, &pTaskInfo->streamInfo.currentOffset); + QUERY_CHECK_CODE(code, lino, _end); qDebug("tmqsnap doQueueScan get data utid:%" PRId64 "", pResult->info.id.uid); - if (pResult->info.rows > 0 || code != TSDB_CODE_SUCCESS) { + if (pResult->info.rows > 0) { (*ppRes) = pResult; return code; } @@ -3009,8 +3013,9 @@ static int32_t doQueueScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { _end: if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } (*ppRes) = NULL; return code; @@ -3340,9 +3345,7 @@ FETCH_NEXT_BLOCK: if (pBlock->info.parTbName[0]) { code = pAPI->stateStore.streamStatePutParName(pStreamInfo->pState, pBlock->info.id.groupId, pBlock->info.parTbName); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); - } + QUERY_CHECK_CODE(code, lino, _end); } // TODO move into scan @@ -3482,7 +3485,7 @@ FETCH_NEXT_BLOCK: return code; } qError("%s===stream=== %s failed at line %d since pInfo->pUpdateRes is empty", GET_TASKID(pTaskInfo), __func__, - lino); + __LINE__); blockDataCleanup(pInfo->pUpdateDataRes); pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; } break; @@ -3496,7 +3499,7 @@ FETCH_NEXT_BLOCK: return code; } qError("%s===stream=== %s failed at line %d since pInfo->pUpdateRes is empty", GET_TASKID(pTaskInfo), __func__, - lino); + __LINE__); blockDataCleanup(pInfo->pUpdateDataRes); pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; } break; @@ -3658,8 +3661,9 @@ FETCH_NEXT_BLOCK: _end: if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } (*ppRes) = NULL; return code; @@ -3730,6 +3734,7 @@ static int32_t doRawScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { if (pBlock && pBlock->info.rows > 0) { bool hasPrimaryKey = pAPI->snapshotFn.taosXGetTablePrimaryKey(pInfo->sContext); code = processPrimaryKey(pBlock, hasPrimaryKey, &pTaskInfo->streamInfo.currentOffset); + QUERY_CHECK_CODE(code, lino, _end); qDebug("tmqsnap doRawScan get data uid:%" PRId64 "", pBlock->info.id.uid); (*ppRes) = pBlock; return code; @@ -3741,7 +3746,7 @@ static int32_t doRawScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { QUERY_CHECK_CODE(code, lino, _end); if (code != 0) { tDeleteSchemaWrapper(mtInfo.schema); - goto _end; + QUERY_CHECK_CODE(code, lino, _end); } STqOffsetVal offset = {0}; if (mtInfo.uid == 0 || pInfo->sContext->withMeta == ONLY_META) { // read snapshot done, change to get data from wal @@ -3831,6 +3836,7 @@ _end: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } (*ppRes) = NULL; @@ -4288,13 +4294,13 @@ _error: return code; } -static int32_t doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, int32_t count, SMetaReader* mr, - SStorageAPI* pAPI) { +static int32_t doTagScanOneTable(SOperatorInfo* pOperator, SSDataBlock* pRes, SMetaReader* mr, SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STagScanInfo* pInfo = pOperator->info; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; + int32_t count = pRes->info.rows; STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos); if (!item) { @@ -4354,6 +4360,8 @@ _end: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); pTaskInfo->code = code; + } else { + pRes->info.rows++; } return code; @@ -4677,6 +4685,7 @@ _end: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } return code; @@ -4684,6 +4693,7 @@ _end: static int32_t doTagScanFromMetaEntryNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pOperator->status == OP_EXEC_DONE) { (*ppRes) = NULL; return code; @@ -4699,10 +4709,7 @@ static int32_t doTagScanFromMetaEntryNext(SOperatorInfo* pOperator, SSDataBlock* int32_t size = 0; code = tableListGetSize(pInfo->pTableListInfo, &size); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - return code; - } + QUERY_CHECK_CODE(code, lino, _end); if (size == 0) { setTaskStatus(pTaskInfo, TASK_COMPLETED); @@ -4710,26 +4717,23 @@ static int32_t doTagScanFromMetaEntryNext(SOperatorInfo* pOperator, SSDataBlock* return code; } - int32_t count = 0; SMetaReader mr = {0}; pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); + pRes->info.rows = 0; - while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { - code = doTagScanOneTable(pOperator, pRes, count, &mr, &pTaskInfo->storageAPI); - if (code == TSDB_CODE_OUT_OF_MEMORY) { - break; - } else { + while (pInfo->curPos < size && pRes->info.rows < pOperator->resultInfo.capacity) { + code = doTagScanOneTable(pOperator, pRes, &mr, &pTaskInfo->storageAPI); + if (code != TSDB_CODE_OUT_OF_MEMORY) { // ignore other error + code = TSDB_CODE_SUCCESS; } + QUERY_CHECK_CODE(code, lino, _end); - ++count; if (++pInfo->curPos >= size) { setOperatorCompleted(pOperator); } } - pRes->info.rows = count; - pAPI->metaReaderFn.clearReader(&mr); bool bLimitReached = applyLimitOffset(&pInfo->limitInfo, pRes, pTaskInfo); if (bLimitReached) { @@ -4744,6 +4748,13 @@ static int32_t doTagScanFromMetaEntryNext(SOperatorInfo* pOperator, SSDataBlock* pOperator->resultInfo.totalRows += pRes->info.rows; (*ppRes) = (pRes->info.rows == 0) ? NULL : pInfo->pRes; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } return code; } @@ -5429,6 +5440,7 @@ _end: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } else { (*ppRes) = pBlock; } @@ -5822,9 +5834,10 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + STupleHandle* pTupleHandle = NULL; blockDataCleanup(pResBlock); - STupleHandle* pTupleHandle = NULL; + while (1) { while (1) { pTupleHandle = NULL; @@ -5945,6 +5958,7 @@ _end: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } else { (*ppRes) = pBlock; } @@ -6460,7 +6474,12 @@ static int32_t doTableCountScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRe } code = buildVnodeDbTableCount(pOperator, pInfo, pSupp, pRes); - if ((pRes->info.rows > 0) && (code == 0)) { + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed since %s", __func__, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } + if (pRes->info.rows > 0) { *ppRes = pRes; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index d12f6dd94c..1c241dffec 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -204,15 +204,18 @@ int32_t appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) * @brief get next tuple with group id attached, here assume that all tuples are sorted by group keys * @param [in, out] pBlock the output block, the group id will be saved in it * @retval NULL if next group tuple arrived and this new group tuple will be saved in pInfo.pSavedTuple - * @retval NULL if no more tuples */ -static STupleHandle* nextTupleWithGroupId(SSortHandle* pHandle, SSortOperatorInfo* pInfo, SSDataBlock* pBlock) { - int32_t code = 0; +static int32_t nextTupleWithGroupId(SSortHandle* pHandle, SSortOperatorInfo* pInfo, SSDataBlock* pBlock, + STupleHandle** pTupleHandle) { + QRY_PARAM_CHECK(pTupleHandle); + + int32_t code = 0; STupleHandle* retTuple = pInfo->pGroupIdCalc->pSavedTuple; if (!retTuple) { code = tsortNextTuple(pHandle, &retTuple); if (code) { - return NULL; + qError("failed to get next tuple, code:%s", tstrerror(code)); + return code; } } @@ -225,7 +228,8 @@ static STupleHandle* nextTupleWithGroupId(SSortHandle* pHandle, SSortOperatorInf newGroup = tsortCompAndBuildKeys(pInfo->pGroupIdCalc->pSortColsArr, pInfo->pGroupIdCalc->keyBuf, &pInfo->pGroupIdCalc->lastKeysLen, retTuple); } - bool emptyBlock = pBlock->info.rows == 0; + + bool emptyBlock = (pBlock->info.rows == 0); if (newGroup) { if (!emptyBlock) { // new group arrived, and we have already copied some tuples for cur group, save the new group tuple, return @@ -247,17 +251,20 @@ static STupleHandle* nextTupleWithGroupId(SSortHandle* pHandle, SSortOperatorInf } } - return retTuple; + *pTupleHandle = retTuple; + return code; } static int32_t getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo, SSortOperatorInfo* pInfo, SSDataBlock** pResBlock) { QRY_PARAM_CHECK(pResBlock); blockDataCleanup(pDataBlock); - int32_t lino = 0; - int32_t code = 0; - SSDataBlock* p = NULL; + int32_t lino = 0; + int32_t code = 0; + STupleHandle* pTupleHandle = NULL; + SSDataBlock* p = NULL; + code = tsortGetSortedDataBlock(pHandle, &p); if (p == NULL || (code != 0)) { return code; @@ -266,16 +273,15 @@ static int32_t getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, code = blockDataEnsureCapacity(p, capacity); QUERY_CHECK_CODE(code, lino, _error); - STupleHandle* pTupleHandle; while (1) { if (pInfo->pGroupIdCalc) { - pTupleHandle = nextTupleWithGroupId(pHandle, pInfo, p); + code = nextTupleWithGroupId(pHandle, pInfo, p, &pTupleHandle); } else { code = tsortNextTuple(pHandle, &pTupleHandle); } - if (pTupleHandle == NULL || code != 0) { - lino = __LINE__; + TSDB_CHECK_CODE(code, lino, _error); + if (pTupleHandle == NULL) { break; } @@ -320,7 +326,7 @@ static int32_t getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, return code; _error: - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); blockDataDestroy(p); return code; @@ -330,6 +336,9 @@ int32_t loadNextDataBlock(void* param, SSDataBlock** ppBlock) { SOperatorInfo* pOperator = (SOperatorInfo*)param; int32_t code = pOperator->fpSet.getNextFn(pOperator, ppBlock); blockDataCheck(*ppBlock, false); + if (code) { + qError("failed to get next data block from upstream, %s code:%s", __func__, tstrerror(code)); + } return code; } @@ -349,82 +358,84 @@ void applyScalarFunction(SSDataBlock* pBlock, void* param) { int32_t doOpenSortOperator(SOperatorInfo* pOperator) { SSortOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SSortSource* pSource =NULL; if (OPTR_IS_OPENED(pOperator)) { - return TSDB_CODE_SUCCESS; + return code; } pInfo->startTs = taosGetTimestampUs(); // pInfo->binfo.pRes is not equalled to the input datablock. pInfo->pSortHandle = NULL; - int32_t code = + code = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, NULL, pTaskInfo->id.str, pInfo->maxRows, pInfo->maxTupleLength, tsPQSortMemThreshold * 1024 * 1024, &pInfo->pSortHandle); - if (code) { - return code; - } + QUERY_CHECK_CODE(code, lino, _end); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, applyScalarFunction, pOperator); - SSortSource* pSource = taosMemoryCalloc(1, sizeof(SSortSource)); - if (pSource == NULL) { - return terrno; - } + pSource = taosMemoryCalloc(1, sizeof(SSortSource)); + QUERY_CHECK_NULL(pSource, code, lino, _end, terrno); pSource->param = pOperator->pDownstream[0]; pSource->onlyRef = true; code = tsortAddSource(pInfo->pSortHandle, pSource); - if (code) { - taosMemoryFree(pSource); - return code; - } + QUERY_CHECK_CODE(code, lino, _end); + pSource = NULL; code = tsortOpen(pInfo->pSortHandle); - if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; - } else { - pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs) / 1000.0; - pOperator->status = OP_RES_TO_RETURN; - OPTR_SET_OPENED(pOperator); - } + QUERY_CHECK_CODE(code, lino, _end); + pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs) / 1000.0; + pOperator->status = OP_RES_TO_RETURN; + OPTR_SET_OPENED(pOperator); +_end: + if (pSource) { + taosMemoryFree(pSource); + } + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } return code; } int32_t doSort(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { QRY_PARAM_CHECK(pResBlock); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pOperator->status == OP_EXEC_DONE) { - return 0; + return code; } SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSortOperatorInfo* pInfo = pOperator->info; - int32_t code = pOperator->fpSet._openFn(pOperator); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = pOperator->fpSet._openFn(pOperator); + QUERY_CHECK_CODE(code, lino, _end); // multi-group case not handle here SSDataBlock* pBlock = NULL; while (1) { if (tsortIsClosed(pInfo->pSortHandle)) { code = TSDB_CODE_TSC_QUERY_CANCELLED; - T_LONG_JMP(pTaskInfo->env, code); + QUERY_CHECK_CODE(code, lino, _end); } code = getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->matchInfo.pList, pInfo, &pBlock); - if (pBlock == NULL || code != 0) { + QUERY_CHECK_CODE(code, lino, _end); + if (pBlock == NULL) { setOperatorCompleted(pOperator); return code; } code = doFilter(pBlock, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo); - if (code) { - break; - } + QUERY_CHECK_CODE(code, lino, _end); if (blockDataGetNumOfRows(pBlock) == 0) { continue; @@ -443,6 +454,12 @@ int32_t doSort(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { } *pResBlock = blockDataGetNumOfRows(pBlock) > 0 ? pBlock : NULL; +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } return code; } @@ -692,16 +709,16 @@ int32_t doGroupSort(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { QRY_PARAM_CHECK(pResBlock); SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SGroupSortOperatorInfo* pInfo = pOperator->info; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pOperator->status == OP_EXEC_DONE) { - return 0; - } - - int32_t code = pOperator->fpSet._openFn(pOperator); - if (code != TSDB_CODE_SUCCESS) { return code; } + code = pOperator->fpSet._openFn(pOperator); + QUERY_CHECK_CODE(code, lino, _end); + if (!pInfo->hasGroupId) { pInfo->hasGroupId = true; @@ -714,30 +731,25 @@ int32_t doGroupSort(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { pInfo->currGroupId = pInfo->prefetchedSortInput->info.id.groupId; pInfo->childOpStatus = CHILD_OP_NEW_GROUP; code = beginSortGroup(pOperator); - if (code) { - return code; - } + QUERY_CHECK_CODE(code, lino, _end); } SSDataBlock* pBlock = NULL; while (pInfo->pCurrSortHandle != NULL) { if (tsortIsClosed(pInfo->pCurrSortHandle)) { code = TSDB_CODE_TSC_QUERY_CANCELLED; - T_LONG_JMP(pTaskInfo->env, code); + QUERY_CHECK_CODE(code, lino, _end); } // beginSortGroup would fetch all child blocks of pInfo->currGroupId; if (pInfo->childOpStatus == CHILD_OP_SAME_GROUP) { - pTaskInfo->code = code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - return code; + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + QUERY_CHECK_CODE(code, lino, _end); } code = getGroupSortedBlockData(pInfo->pCurrSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->matchInfo.pList, pInfo, &pBlock); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + QUERY_CHECK_CODE(code, lino, _end); if (pBlock != NULL) { pBlock->info.id.groupId = pInfo->currGroupId; pOperator->resultInfo.totalRows += pBlock->info.rows; @@ -748,9 +760,7 @@ int32_t doGroupSort(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { (void) finishSortGroup(pOperator); pInfo->currGroupId = pInfo->prefetchedSortInput->info.id.groupId; code = beginSortGroup(pOperator); - if (code) { - return code; - } + QUERY_CHECK_CODE(code, lino, _end); } else if (pInfo->childOpStatus == CHILD_OP_FINISHED) { (void) finishSortGroup(pOperator); setOperatorCompleted(pOperator); @@ -759,6 +769,12 @@ int32_t doGroupSort(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { } } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } return code; } diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index adf764a8c5..33b3e7748c 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -46,8 +46,16 @@ typedef struct SBuffInfo { } SBuffInfo; void destroyStreamCountAggOperatorInfo(void* param) { + if (param == NULL) { + return; + } SStreamCountAggOperatorInfo* pInfo = (SStreamCountAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); + if (pInfo->pOperator) { + cleanupResultInfoInStream(pInfo->pOperator->pTaskInfo, pInfo->streamAggSup.pState, &pInfo->pOperator->exprSupp, + &pInfo->groupResInfo); + pInfo->pOperator = NULL; + } destroyStreamAggSupporter(&pInfo->streamAggSup); cleanupExprSupp(&pInfo->scalarSupp); clearGroupResInfo(&pInfo->groupResInfo); @@ -82,7 +90,7 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group if (isSlidingCountWindow(pAggSup)) { if (pBuffInfo->winBuffOp == CREATE_NEW_WINDOW) { - code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, + code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size); QUERY_CHECK_CODE(code, lino, _end); @@ -93,9 +101,11 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, (void**)&pCurWin->winInfo.pStatePos, &size); if (winCode == TSDB_CODE_FAILED) { - code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, + code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size); QUERY_CHECK_CODE(code, lino, _end); + } else { + reuseOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->stateStore); } } else { pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, @@ -103,9 +113,11 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, (void**)&pCurWin->winInfo.pStatePos, &size); if (winCode == TSDB_CODE_FAILED) { - code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, + code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size); QUERY_CHECK_CODE(code, lino, _end); + } else { + reuseOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->stateStore); } } if (ts < pCurWin->winInfo.sessionWin.win.ekey) { @@ -741,8 +753,9 @@ static int32_t doStreamCountAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRe _end: if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } setStreamOperatorCompleted(pOperator); (*ppRes) = NULL; @@ -906,6 +919,7 @@ int32_t createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* QUERY_CHECK_CODE(code, lino, _error); taosMemoryFree(buff); } + pInfo->pOperator = pOperator; pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamCountAggNext, NULL, destroyStreamCountAggOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState); diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index a325616bd3..8a706f6d4e 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -48,6 +48,11 @@ void destroyStreamEventOperatorInfo(void* param) { } SStreamEventAggOperatorInfo* pInfo = (SStreamEventAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); + if (pInfo->pOperator) { + cleanupResultInfoInStream(pInfo->pOperator->pTaskInfo, pInfo->streamAggSup.pState, &pInfo->pOperator->exprSupp, + &pInfo->groupResInfo); + pInfo->pOperator = NULL; + } destroyStreamAggSupporter(&pInfo->streamAggSup); clearGroupResInfo(&pInfo->groupResInfo); taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos); @@ -176,7 +181,7 @@ _end: pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin); } pAggSup->stateStore.streamStateFreeCur(pCur); - qDebug("===stream===set event next win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey, + qDebug("===stream===set event cur win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey, pCurWin->winInfo.sessionWin.win.ekey); _error: @@ -230,7 +235,7 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW pWinInfo->pWinFlag->endFlag = ends[i]; } else if (pWin->ekey == pTsData[i]) { pWinInfo->pWinFlag->endFlag |= ends[i]; - } else { + } else if (ends[i] && !pWinInfo->pWinFlag->endFlag) { *pRebuild = true; pWinInfo->pWinFlag->endFlag |= ends[i]; (*pWinRow) = i + 1 - start; @@ -731,8 +736,9 @@ static int32_t doStreamEventAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRe _end: if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } setStreamOperatorCompleted(pOperator); (*ppRes) = NULL; @@ -951,6 +957,7 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno); pInfo->destHasPrimaryKey = pEventNode->window.destHasPrimayKey; + pInfo->pOperator = pOperator; setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED, pInfo, pTaskInfo); // for stream diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index 4d5f597ab6..826220581a 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -1155,8 +1155,9 @@ static int32_t doStreamFillNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { _end: if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } setOperatorCompleted(pOperator); resetStreamFillInfo(pInfo); @@ -1164,12 +1165,12 @@ _end: return code; } -static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { - pFillSup->rowSize = sizeof(SResultCellData) * pFillSup->numOfAllCols; - for (int i = 0; i < pFillSup->numOfAllCols; i++) { - SFillColInfo* pCol = &pFillSup->pAllColInfo[i]; - SResSchema* pSchema = &pCol->pExpr->base.resSchema; - pFillSup->rowSize += pSchema->bytes; +static int32_t initResultBuf(SSDataBlock* pInputRes, SStreamFillSupporter* pFillSup) { + int32_t numOfCols = taosArrayGetSize(pInputRes->pDataBlock); + pFillSup->rowSize = sizeof(SResultCellData) * numOfCols; + for (int i = 0; i < numOfCols; i++) { + SColumnInfoData* pCol = taosArrayGet(pInputRes->pDataBlock, i); + pFillSup->rowSize += pCol->info.bytes; } pFillSup->next.key = INT64_MIN; pFillSup->nextNext.key = INT64_MIN; @@ -1184,7 +1185,7 @@ static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { } static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, - SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) { + SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI, SSDataBlock* pInputRes) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); @@ -1213,7 +1214,7 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod pFillSup->interval = *pInterval; pFillSup->pAPI = pAPI; - code = initResultBuf(pFillSup); + code = initResultBuf(pInputRes, pFillSup); QUERY_CHECK_CODE(code, lino, _end); SExprInfo* noFillExpr = NULL; @@ -1370,7 +1371,11 @@ int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysi code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore); QUERY_CHECK_CODE(code, lino, _error); - pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI); + pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); + QUERY_CHECK_NULL(pInfo->pSrcBlock, code, lino, _error, terrno); + + pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI, + pInfo->pSrcBlock); if (!pInfo->pFillSup) { code = TSDB_CODE_FAILED; QUERY_CHECK_CODE(code, lino, _error); @@ -1379,8 +1384,7 @@ int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysi initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); QUERY_CHECK_NULL(pInfo->pRes, code, lino, _error, terrno); - pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); - QUERY_CHECK_NULL(pInfo->pSrcBlock, code, lino, _error, terrno); + code = blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); QUERY_CHECK_CODE(code, lino, _error); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index fb8d8d654e..e8322d6911 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -473,6 +473,11 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { } SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); + if (pInfo->pOperator) { + cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, + &pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable); + pInfo->pOperator = NULL; + } cleanupAggSup(&pInfo->aggSup); clearGroupResInfo(&pInfo->groupResInfo); taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos); @@ -1799,8 +1804,9 @@ static int32_t doStreamFinalIntervalAggNext(SOperatorInfo* pOperator, SSDataBloc _end: if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } setStreamOperatorCompleted(pOperator); (*ppRes) = NULL; @@ -2024,6 +2030,7 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN pInfo->pDeletedMap = tSimpleHashInit(4096, hashFn); QUERY_CHECK_NULL(pInfo->pDeletedMap, code, lino, _error, terrno); pInfo->destHasPrimaryKey = pIntervalPhyNode->window.destHasPrimayKey; + pInfo->pOperator = pOperator; pOperator->operatorType = pPhyNode->type; if (!IS_FINAL_INTERVAL_OP(pOperator) || numOfChild == 0) { @@ -2088,6 +2095,11 @@ void destroyStreamSessionAggOperatorInfo(void* param) { } SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); + if (pInfo->pOperator) { + cleanupResultInfoInStream(pInfo->pOperator->pTaskInfo, pInfo->streamAggSup.pState, &pInfo->pOperator->exprSupp, + &pInfo->groupResInfo); + pInfo->pOperator = NULL; + } destroyStreamAggSupporter(&pInfo->streamAggSup); cleanupExprSupp(&pInfo->scalarSupp); clearGroupResInfo(&pInfo->groupResInfo); @@ -3564,8 +3576,9 @@ static int32_t doStreamSessionAggNext(SOperatorInfo* pOperator, SSDataBlock** pp _end: if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } setStreamOperatorCompleted(pOperator); (*ppRes) = NULL; @@ -3855,6 +3868,7 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode pInfo->destHasPrimaryKey = pSessionNode->window.destHasPrimayKey; pInfo->pPkDeleted = tSimpleHashInit(64, hashFn); QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno); + pInfo->pOperator = pOperator; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, @@ -4068,8 +4082,9 @@ static int32_t doStreamSessionSemiAggNext(SOperatorInfo* pOperator, SSDataBlock* _end: if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } clearFunctionContext(&pOperator->exprSupp); @@ -4103,6 +4118,7 @@ int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhys SStorageAPI* pAPI = &pTaskInfo->storageAPI; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; pOperator->operatorType = pPhyNode->type; + pInfo->pOperator = pOperator; if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { pOperator->fpSet = @@ -4173,6 +4189,11 @@ void destroyStreamStateOperatorInfo(void* param) { } SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); + if (pInfo->pOperator) { + cleanupResultInfoInStream(pInfo->pOperator->pTaskInfo, pInfo->streamAggSup.pState, &pInfo->pOperator->exprSupp, + &pInfo->groupResInfo); + pInfo->pOperator = NULL; + } destroyStreamAggSupporter(&pInfo->streamAggSup); clearGroupResInfo(&pInfo->groupResInfo); taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos); @@ -4804,8 +4825,9 @@ static int32_t doStreamStateAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRe _end: if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } setStreamOperatorCompleted(pOperator); (*ppRes) = NULL; @@ -5028,6 +5050,7 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pInfo->pPkDeleted = tSimpleHashInit(64, hashFn); QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno); pInfo->destHasPrimaryKey = pStateNode->window.destHasPrimayKey; + pInfo->pOperator = pOperator; setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -5203,6 +5226,7 @@ static int32_t doStreamIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** p code = TSDB_CODE_SUCCESS; break; } + QUERY_CHECK_CODE(code, lino, _end); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey); } @@ -5245,8 +5269,9 @@ static int32_t doStreamIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** p _end: if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } setStreamOperatorCompleted(pOperator); (*ppRes) = NULL; @@ -5361,6 +5386,7 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH, &pInfo->pState->pFileState); QUERY_CHECK_CODE(code, lino, _error); + pInfo->pOperator = pOperator; setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = @@ -5781,8 +5807,9 @@ static int32_t doStreamMidIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock* _end: if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); } (*ppRes) = NULL; return code; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index dcebdf59a9..051a06ba5c 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -131,9 +131,6 @@ const SSTabFltFuncDef filterDict[] = { static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, const char* dbName, int64_t* pRows); -static char* SYSTABLE_IDX_COLUMN[] = {"table_name", "db_name", "create_time", "columns", - "ttl", "stable_name", "vgroup_id', 'uid", "type"}; - static char* SYSTABLE_SPECIAL_COL[] = {"db_name", "vgroup_id"}; static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); @@ -2049,7 +2046,7 @@ static int32_t doSysTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) if (isTaskKilled(pOperator->pTaskInfo)) { setOperatorCompleted(pOperator); (*ppRes) = NULL; - return pTaskInfo->code; + break; } blockDataCleanup(pInfo->pRes); @@ -2092,12 +2089,18 @@ static int32_t doSysTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) continue; } (*ppRes) = pBlock; - return pTaskInfo->code; } else { (*ppRes) = NULL; - return pTaskInfo->code; } + break; } + +_end: + if (pTaskInfo->code) { + qError("%s failed since %s", __func__, tstrerror(pTaskInfo->code)); + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); + } + return pTaskInfo->code; } static void sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo, const char* name, @@ -2822,12 +2825,6 @@ _end: return code; } -static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { - SSDataBlock* pRes = NULL; - int32_t code = doBlockInfoScanNext(pOperator, &pRes); - return pRes; -} - static void destroyBlockDistScanOperatorInfo(void* param) { SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; blockDataDestroy(pDistInfo->pResBlock); @@ -2846,6 +2843,8 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo)); pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t)); if (pCond->colList == NULL || pCond->pSlotList == NULL) { + taosMemoryFree(pCond->colList); + taosMemoryFree(pCond->pSlotList); return terrno; } diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 2ea300ace8..70bf26405e 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -278,7 +278,7 @@ static bool checkNullRow(SExprSupp* pExprSup, SSDataBlock* pSrcBlock, int32_t in } static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, - SSDataBlock* pSrcBlock, int32_t index, bool beforeTs, SExecTaskInfo* pTaskInfo) { + SSDataBlock* pSrcBlock, int32_t index, bool beforeTs, SExecTaskInfo* pTaskInfo, bool genAfterBlock) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; int32_t rows = pResBlock->info.rows; @@ -427,7 +427,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp break; } - if (start.key == INT64_MIN || end.key == INT64_MIN) { + if (start.key == INT64_MIN || end.key == INT64_MIN || genAfterBlock) { colDataSetNULL(pDst, rows); break; } @@ -463,8 +463,13 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp break; } + if (genAfterBlock && rows == 0) { + hasInterp = false; + break; + } + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot); - if (pkey->isNull == false) { + if (pkey->isNull == false && !genAfterBlock) { code = colDataSetVal(pDst, rows, pkey->pData, false); QUERY_CHECK_CODE(code, lino, _end); } else { @@ -836,7 +841,7 @@ static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pS int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, false, pTaskInfo) && + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, false, pTaskInfo, false) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { @@ -864,7 +869,7 @@ static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pS doKeepLinearInfo(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, true, pTaskInfo) && + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i, true, pTaskInfo, false) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { @@ -909,13 +914,12 @@ static void genInterpAfterDataBlock(STimeSliceOperatorInfo* pSliceInfo, SOperato SSDataBlock* pResBlock = pSliceInfo->pRes; SInterval* pInterval = &pSliceInfo->interval; - if (pSliceInfo->fillType == TSDB_FILL_NEXT || pSliceInfo->fillType == TSDB_FILL_LINEAR || - pSliceInfo->pPrevGroupKey == NULL) { + if (pSliceInfo->pPrevGroupKey == NULL) { return; } while (pSliceInfo->current <= pSliceInfo->win.ekey) { - (void)genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, NULL, index, false, pOperator->pTaskInfo); + (void)genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, NULL, index, false, pOperator->pTaskInfo, true); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index fa914d3ee8..8164281871 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -37,6 +37,7 @@ typedef struct SSessionAggOperatorInfo { int64_t gap; // session window gap int32_t tsSlotId; // primary timestamp slot id STimeWindowAggSupp twAggSup; + SOperatorInfo* pOperator; } SSessionAggOperatorInfo; typedef struct SStateWindowOperatorInfo { @@ -50,6 +51,7 @@ typedef struct SStateWindowOperatorInfo { SStateKeys stateKey; int32_t tsSlotId; // primary timestamp column slot id STimeWindowAggSupp twAggSup; + SOperatorInfo* pOperator; } SStateWindowOperatorInfo; typedef enum SResultTsInterpType { @@ -1221,9 +1223,19 @@ _end: } static void destroyStateWindowOperatorInfo(void* param) { + if (param == NULL) { + return; + } SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); taosMemoryFreeClear(pInfo->stateKey.pData); + + if (pInfo->pOperator != NULL) { + cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, + &pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable); + pInfo->pOperator = NULL; + } + cleanupExprSupp(&pInfo->scalarSup); colDataDestroy(&pInfo->twAggSup.timeWindowData); cleanupAggSup(&pInfo->aggSup); @@ -1241,8 +1253,17 @@ void destroyIntervalOperatorInfo(void* param) { if (param == NULL) { return; } + SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param; + cleanupBasicInfo(&pInfo->binfo); + + if (pInfo->pOperator != NULL) { + cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, + &pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable); + pInfo->pOperator = NULL; + } + cleanupAggSup(&pInfo->aggSup); cleanupExprSupp(&pInfo->scalarSupp); @@ -1250,6 +1271,7 @@ void destroyIntervalOperatorInfo(void* param) { taosArrayDestroy(pInfo->pInterpCols); pInfo->pInterpCols = NULL; + taosArrayDestroyEx(pInfo->pPrevValues, freeItem); pInfo->pPrevValues = NULL; @@ -1343,6 +1365,7 @@ int32_t createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { code = terrno; + lino = __LINE__; goto _error; } @@ -1430,6 +1453,7 @@ int32_t createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode } } + pInfo->pOperator = pOperator; initResultRowInfo(&pInfo->binfo.resultRowInfo); setOperatorInfo(pOperator, "TimeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -1449,8 +1473,10 @@ _error: if (pInfo != NULL) { destroyIntervalOperatorInfo(pInfo); } + destroyOperatorAndDownstreams(pOperator, &downstream, 1); pTaskInfo->code = code; + qError("error happens at %s %d, code:%s", __func__, lino, tstrerror(code)); return code; } @@ -1706,7 +1732,7 @@ int32_t createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhy QUERY_CHECK_CODE(code, lino, _error); pInfo->tsSlotId = tsSlotId; - + pInfo->pOperator = pOperator; setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(openStateWindowAggOptr, doStateWindowAggNext, NULL, destroyStateWindowOperatorInfo, @@ -1739,6 +1765,12 @@ void destroySWindowOperatorInfo(void* param) { cleanupBasicInfo(&pInfo->binfo); colDataDestroy(&pInfo->twAggSup.timeWindowData); + if (pInfo->pOperator != NULL) { + cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, pInfo->aggSup.pResultBuf, + &pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable); + pInfo->pOperator = NULL; + } + cleanupAggSup(&pInfo->aggSup); cleanupExprSupp(&pInfo->scalarSupp); @@ -1805,6 +1837,7 @@ int32_t createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPh code = filterInitFromNode((SNode*)pSessionNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); QUERY_CHECK_CODE(code, lino, _error); + pInfo->pOperator = pOperator; setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSessionWindowAggNext, NULL, destroySWindowOperatorInfo, @@ -2121,6 +2154,7 @@ int32_t createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMerge initResultRowInfo(&iaInfo->binfo.resultRowInfo); code = blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity); QUERY_CHECK_CODE(code, lino, _error); + iaInfo->pOperator = pOperator; setOperatorInfo(pOperator, "TimeMergeAlignedIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, false, OP_NOT_OPENED, miaInfo, pTaskInfo); @@ -2460,6 +2494,7 @@ int32_t createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeInterva } } + pIntervalInfo->pOperator = pOperator; initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo); setOperatorInfo(pOperator, "TimeMergeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, false, OP_NOT_OPENED, pMergeIntervalInfo, pTaskInfo); diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 00daafa59d..17c390e239 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -437,41 +437,36 @@ int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource) { static int32_t doAddNewExternalMemSource(SDiskbasedBuf* pBuf, SArray* pAllSources, SSDataBlock* pBlock, int32_t* sourceId, SArray* pPageIdList) { + int32_t code = 0; + int32_t lino = 0; SSortSource* pSource = taosMemoryCalloc(1, sizeof(SSortSource)); - if (pSource == NULL) { - taosArrayDestroy(pPageIdList); - return terrno; - } + QUERY_CHECK_NULL(pSource, code, lino, _err, terrno); pSource->src.pBlock = pBlock; pSource->pageIdList = pPageIdList; - void* p = taosArrayPush(pAllSources, &pSource); - if (p == NULL) { - taosArrayDestroy(pPageIdList); - return terrno; - } + SSortSource** p = taosArrayPush(pAllSources, &pSource); + QUERY_CHECK_NULL(p, code, lino, _err, terrno); + pSource = NULL; (*sourceId) += 1; - int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock); + int32_t rowSize = blockDataGetSerialRowSize((*p)->src.pBlock); // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock))) / rowSize; - if (numOfRows <= 0) { - qError("sort failed at: %s:%d", __func__, __LINE__); - taosArrayDestroy(pPageIdList); - return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; - } + QUERY_CHECK_CONDITION((numOfRows > 0), code, lino, _err, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); - int32_t code = blockDataEnsureCapacity(pSource->src.pBlock, numOfRows); - if (code != 0) { - qError("sort failed at: %s:%d", __func__, __LINE__); - taosArrayDestroy(pPageIdList); - } + code = blockDataEnsureCapacity((*p)->src.pBlock, numOfRows); + QUERY_CHECK_CODE(code, lino, _err); return code; + +_err: + if (pSource) taosMemoryFree(pSource); + qError("sort failed at %s:%d since %s", __func__, lino, tstrerror(code)); + return code; } static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { @@ -554,7 +549,12 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { return code; } - return doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId, pPageIdList); + code = doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId, pPageIdList); + if (code) { + blockDataDestroy(pBlock); + taosArrayDestroy(pPageIdList); + } + return code; } static void setCurrentSourceDone(SSortSource* pSource, SSortHandle* pHandle) { @@ -771,7 +771,7 @@ static int32_t getSortedBlockDataInner(SSortHandle* pHandle, SMsortComparParam* code = adjustMergeTreeForNextTuple(pSource, pHandle->pMergeTree, pHandle, &pHandle->numOfCompletedSources); if (code != TSDB_CODE_SUCCESS) { - return terrno = code; + return code; } if (pHandle->pDataBlock->info.rows >= capacity) { @@ -1023,6 +1023,9 @@ static int32_t doSortForEachGroup(SSortHandle* pHandle, int32_t sortTimes, int32 QUERY_CHECK_CODE(code, lino, _err); code = doAddNewExternalMemSource(pHandle->pBuf, pResList, pBlock, &pHandle->sourceId, pPageIdList); + if (code != TSDB_CODE_SUCCESS) { + blockDataDestroy(pBlock); + } QUERY_CHECK_CODE(code, lino, _err); } @@ -2144,6 +2147,10 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray* if (code) goto _error; code = doAddNewExternalMemSource(pHandle->pBuf, aExtSrc, pMemSrcBlk, &pHandle->sourceId, aPgId); + if (code != TSDB_CODE_SUCCESS) { + blockDataDestroy(pMemSrcBlk); + goto _error; + } cleanupMergeSup(&sup); tMergeTreeDestroy(&pTree); @@ -2306,9 +2313,15 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { } code = tSimpleHashPut(mUidBlk, &pBlk->info.id.uid, sizeof(pBlk->info.id.uid), &tBlk, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS) { + blockDataDestroy(tBlk); + } QUERY_CHECK_CODE(code, lino, _err); void* px = taosArrayPush(aBlkSort, &tBlk); + if (px == NULL) { + blockDataDestroy(tBlk); + } QUERY_CHECK_NULL(px, code, lino, _err, terrno); } } @@ -2378,25 +2391,31 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { return code; } -static void freeSortSource(SSortSource* pSource) { - if (NULL == pSource) { +static void freeSortSource(void* p) { + SSortSource** pSource = (SSortSource**)p; + if (NULL == pSource || NULL == *pSource) { return; } - if (!pSource->onlyRef && pSource->param) { - taosMemoryFree(pSource->param); + if ((*pSource)->pageIdList) { + taosArrayDestroy((*pSource)->pageIdList); } - if (!pSource->onlyRef && pSource->src.pBlock) { - blockDataDestroy(pSource->src.pBlock); - pSource->src.pBlock = NULL; + if (!(*pSource)->onlyRef) { + if ((*pSource)->param) { + taosMemoryFree((*pSource)->param); + } + if ((*pSource)->src.pBlock) { + blockDataDestroy((*pSource)->src.pBlock); + } } - taosMemoryFree(pSource); + taosMemoryFreeClear(*pSource); } static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { int32_t code = 0; + int32_t lino = 0; size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize; SSortSource** p = taosArrayGet(pHandle->pOrderedSource, 0); if (p == NULL) { @@ -2404,17 +2423,12 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { } SSortSource* pSource = *p; - - taosArrayRemove(pHandle->pOrderedSource, 0); - tsortClearOrderedSource(pHandle->pOrderedSource, NULL, NULL); + size_t origSourceCount = taosArrayGetSize(pHandle->pOrderedSource); while (1) { SSDataBlock* pBlock = NULL; code = pHandle->fetchfp(pSource->param, &pBlock); - if (code != 0) { - freeSortSource(pSource); - return code; - } + QUERY_CHECK_CODE(code, lino, _end); if (pBlock == NULL) { break; @@ -2428,10 +2442,7 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { pHandle->numOfPages = 1024; sortBufSize = pHandle->numOfPages * pHandle->pageSize; code = createOneDataBlock(pBlock, false, &pHandle->pDataBlock); - if (code) { - freeSortSource(pSource); - return code; - } + QUERY_CHECK_CODE(code, lino, _end); } if (pHandle->beforeFp != NULL) { @@ -2439,43 +2450,30 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { } code = blockDataMerge(pHandle->pDataBlock, pBlock); - if (code != TSDB_CODE_SUCCESS) { - freeSortSource(pSource); - return code; - } + QUERY_CHECK_CODE(code, lino, _end); size_t size = blockDataGetSize(pHandle->pDataBlock); if (size > sortBufSize) { // Perform the in-memory sort and then flush data in the buffer into disk. int64_t st = taosGetTimestampUs(); code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); - if (code != 0) { - freeSortSource(pSource); - return code; - } + QUERY_CHECK_CODE(code, lino, _end); pHandle->sortElapsed += (taosGetTimestampUs() - st); if (pHandle->pqMaxRows > 0) blockDataKeepFirstNRows(pHandle->pDataBlock, pHandle->pqMaxRows); code = doAddToBuf(pHandle->pDataBlock, pHandle); - if (code != TSDB_CODE_SUCCESS) { - freeSortSource(pSource); - return code; - } + QUERY_CHECK_CODE(code, lino, _end); } } - freeSortSource(pSource); - if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) { size_t size = blockDataGetSize(pHandle->pDataBlock); // Perform the in-memory sort and then flush data in the buffer into disk. int64_t st = taosGetTimestampUs(); code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); - if (code != 0) { - return code; - } + QUERY_CHECK_CODE(code, lino, _end); if (pHandle->pqMaxRows > 0) blockDataKeepFirstNRows(pHandle->pDataBlock, pHandle->pqMaxRows); pHandle->sortElapsed += (taosGetTimestampUs() - st); @@ -2488,12 +2486,16 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { pHandle->loops = 1; pHandle->tupleHandle.rowIndex = -1; pHandle->tupleHandle.pBlock = pHandle->pDataBlock; - return 0; } else { code = doAddToBuf(pHandle->pDataBlock, pHandle); } } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + taosArrayRemoveBatch(pHandle->pOrderedSource, 0, origSourceCount, freeSortSource); return code; } @@ -2867,6 +2869,7 @@ static int32_t tsortSingleTableMergeNextTuple(SSortHandle* pHandle, STupleHandle pHandle->tupleHandle.pBlock = NULL; return code; } + pHandle->tupleHandle.pBlock = pBlock; pHandle->tupleHandle.rowIndex = 0; } @@ -2882,8 +2885,7 @@ int32_t tsortOpen(SSortHandle* pHandle) { } if (pHandle == NULL || pHandle->fetchfp == NULL || pHandle->comparFn == NULL) { - code = TSDB_CODE_INVALID_PARA; - return code; + return TSDB_CODE_INVALID_PARA; } pHandle->opened = true; diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 41e2cadace..0b2fb70eba 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -122,6 +122,7 @@ bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t percentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t percentileFunction(SqlFunctionCtx* pCtx); int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +void percentileFunctionCleanupExt(SqlFunctionCtx* pCtx); bool getApercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); diff --git a/source/libs/function/inc/tpercentile.h b/source/libs/function/inc/tpercentile.h index 1b80c2b1da..09df42d3a3 100644 --- a/source/libs/function/inc/tpercentile.h +++ b/source/libs/function/inc/tpercentile.h @@ -69,7 +69,7 @@ typedef struct tMemBucket { int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup, tMemBucket **pBucket); -void tMemBucketDestroy(tMemBucket *pBucket); +void tMemBucketDestroy(tMemBucket **pBucket); int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 976d15e7d8..604375aed2 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -222,10 +222,10 @@ static int32_t addTimezoneParam(SNodeList* pList) { return code; } - pVal->literal = strndup(buf, len); + pVal->literal = taosStrndup(buf, len); if (pVal->literal == NULL) { nodesDestroyNode((SNode*)pVal); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pVal->translate = true; pVal->node.resType.type = TSDB_DATA_TYPE_BINARY; @@ -3115,6 +3115,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = percentileFunction, .sprocessFunc = percentileScalarFunction, .finalizeFunc = percentileFinalize, + .cleanupFunc = percentileFunctionCleanupExt, #ifdef BUILD_NO_CALL .invertFunc = NULL, #endif diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 7761ec8aa5..157d44b3de 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2009,6 +2009,17 @@ int32_t percentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResu return TSDB_CODE_SUCCESS; } +void percentileFunctionCleanupExt(SqlFunctionCtx* pCtx) { + if (pCtx == NULL || GET_RES_INFO(pCtx) == NULL || GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)) == NULL) { + return; + } + SPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + if (pInfo->pMemBucket != NULL) { + tMemBucketDestroy(&(pInfo->pMemBucket)); + pInfo->pMemBucket = NULL; + } +} + int32_t percentileFunction(SqlFunctionCtx* pCtx) { int32_t code = TSDB_CODE_SUCCESS; int32_t numOfElems = 0; @@ -2095,7 +2106,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { numOfElems += 1; code = tMemBucketPut(pInfo->pMemBucket, data, 1); if (code != TSDB_CODE_SUCCESS) { - tMemBucketDestroy(pInfo->pMemBucket); + tMemBucketDestroy(&(pInfo->pMemBucket)); return code; } } @@ -2103,6 +2114,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { SET_VAL(pResInfo, numOfElems, 1); } + pCtx->needCleanup = true; return TSDB_CODE_SUCCESS; } @@ -2113,8 +2125,8 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t code = 0; double v = 0; - tMemBucket* pMemBucket = ppInfo->pMemBucket; - if (pMemBucket != NULL && pMemBucket->total > 0) { // check for null + tMemBucket** pMemBucket = &ppInfo->pMemBucket; + if ((*pMemBucket) != NULL && (*pMemBucket)->total > 0) { // check for null if (pCtx->numOfParams > 2) { char buf[3200] = {0}; // max length of double num is 317, e.g. use %.6lf to print -1.0e+308, consider the comma and bracket, 3200 is enough. @@ -2126,7 +2138,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { GET_TYPED_DATA(v, double, pVal->nType, &pVal->i); - code = getPercentile(pMemBucket, v, &ppInfo->result); + code = getPercentile((*pMemBucket), v, &ppInfo->result); if (code != TSDB_CODE_SUCCESS) { goto _fin_error; } @@ -2158,7 +2170,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { GET_TYPED_DATA(v, double, pVal->nType, &pVal->i); - code = getPercentile(pMemBucket, v, &ppInfo->result); + code = getPercentile((*pMemBucket), v, &ppInfo->result); if (code != TSDB_CODE_SUCCESS) { goto _fin_error; } @@ -4826,9 +4838,9 @@ int32_t histogramFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResul pInfo->totalCount = 0; pInfo->normalized = 0; - char* binTypeStr = strndup(varDataVal(pCtx->param[1].param.pz), varDataLen(pCtx->param[1].param.pz)); + char* binTypeStr = taosStrndup(varDataVal(pCtx->param[1].param.pz), varDataLen(pCtx->param[1].param.pz)); if (binTypeStr == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } int8_t binType = getHistogramBinType(binTypeStr); taosMemoryFree(binTypeStr); @@ -4836,9 +4848,9 @@ int32_t histogramFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResul if (binType == UNKNOWN_BIN) { return TSDB_CODE_FUNC_FUNTION_PARA_VALUE; } - char* binDesc = strndup(varDataVal(pCtx->param[2].param.pz), varDataLen(pCtx->param[2].param.pz)); + char* binDesc = taosStrndup(varDataVal(pCtx->param[2].param.pz), varDataLen(pCtx->param[2].param.pz)); if (binDesc == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } int64_t normalized = pCtx->param[3].param.i; if (normalized != 0 && normalized != 1) { @@ -6067,7 +6079,7 @@ int32_t modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { pInfo->pHash = NULL; return terrno; } - + pCtx->needCleanup = true; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 92a7c0d669..29c48460c0 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -291,12 +291,12 @@ int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, dou (*pBucket)->maxCapacity = 200000; (*pBucket)->groupPagesMap = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if ((*pBucket)->groupPagesMap == NULL) { - tMemBucketDestroy(*pBucket); + tMemBucketDestroy(pBucket); return terrno; } if (setBoundingBox(&(*pBucket)->range, (*pBucket)->type, minval, maxval) != 0) { // qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval); - tMemBucketDestroy(*pBucket); + tMemBucketDestroy(pBucket); return TSDB_CODE_FUNC_INVALID_VALUE_RANGE; } @@ -306,13 +306,13 @@ int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, dou (*pBucket)->hashFunc = getHashFunc((*pBucket)->type); if ((*pBucket)->hashFunc == NULL) { // qError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type); - tMemBucketDestroy(*pBucket); + tMemBucketDestroy(pBucket); return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; } (*pBucket)->pSlots = (tMemBucketSlot *)taosMemoryCalloc((*pBucket)->numOfSlots, sizeof(tMemBucketSlot)); if ((*pBucket)->pSlots == NULL) { - tMemBucketDestroy(*pBucket); + tMemBucketDestroy(pBucket); return terrno; } @@ -320,13 +320,13 @@ int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, dou if (!osTempSpaceAvailable()) { // qError("MemBucket create disk based Buf failed since %s", terrstr(terrno)); - tMemBucketDestroy(*pBucket); + tMemBucketDestroy(pBucket); return TSDB_CODE_NO_DISKSPACE; } int32_t ret = createDiskbasedBuf(&(*pBucket)->pBuffer, (*pBucket)->bufPageSize, (*pBucket)->bufPageSize * DEFAULT_NUM_OF_SLOT * 4, "1", tsTempDir); if (ret != 0) { - tMemBucketDestroy(*pBucket); + tMemBucketDestroy(pBucket); return ret; } @@ -334,22 +334,22 @@ int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, dou return TSDB_CODE_SUCCESS; } -void tMemBucketDestroy(tMemBucket *pBucket) { - if (pBucket == NULL) { +void tMemBucketDestroy(tMemBucket **pBucket) { + if (*pBucket == NULL) { return; } - void *p = taosHashIterate(pBucket->groupPagesMap, NULL); + void *p = taosHashIterate((*pBucket)->groupPagesMap, NULL); while (p) { SArray **p1 = p; - p = taosHashIterate(pBucket->groupPagesMap, p); + p = taosHashIterate((*pBucket)->groupPagesMap, p); taosArrayDestroy(*p1); } - destroyDiskbasedBuf(pBucket->pBuffer); - taosMemoryFreeClear(pBucket->pSlots); - taosHashCleanup(pBucket->groupPagesMap); - taosMemoryFreeClear(pBucket); + destroyDiskbasedBuf((*pBucket)->pBuffer); + taosMemoryFreeClear((*pBucket)->pSlots); + taosHashCleanup((*pBucket)->groupPagesMap); + taosMemoryFreeClear(*pBucket); } int32_t tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) { diff --git a/source/libs/index/inc/indexFstFile.h b/source/libs/index/inc/indexFstFile.h index b1ff4fee34..145e1040cd 100644 --- a/source/libs/index/inc/indexFstFile.h +++ b/source/libs/index/inc/indexFstFile.h @@ -33,7 +33,7 @@ typedef struct IFileCtx { int (*write)(struct IFileCtx* ctx, uint8_t* buf, int len); int (*read)(struct IFileCtx* ctx, uint8_t* buf, int len); int (*flush)(struct IFileCtx* ctx); - int (*readFrom)(struct IFileCtx* ctx, uint8_t* buf, int len, int32_t offset); + int64_t (*readFrom)(struct IFileCtx* ctx, uint8_t* buf, int len, int32_t offset); int (*size)(struct IFileCtx* ctx); SLRUCache* lru; @@ -64,7 +64,7 @@ typedef struct IFileCtx { static int idxFileCtxDoWrite(IFileCtx* ctx, uint8_t* buf, int len); static int idxFileCtxDoRead(IFileCtx* ctx, uint8_t* buf, int len); -static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t offset); +static int64_t idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t offset); static int idxFileCtxDoFlush(IFileCtx* ctx); IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity); diff --git a/source/libs/index/inc/indexUtil.h b/source/libs/index/inc/indexUtil.h index 61c16e1217..9eb8001d17 100644 --- a/source/libs/index/inc/indexUtil.h +++ b/source/libs/index/inc/indexUtil.h @@ -21,29 +21,29 @@ extern "C" { #endif -#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ - do { \ - TAOS_UNUSED(memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem))); \ - buf += sizeof(key->mem); \ +#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ + do { \ + (void)memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \ + buf += sizeof(key->mem); \ } while (0) -#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ - do { \ - TAOS_UNUSED(memcpy((void *)buf, (void *)key->mem, len)); \ - buf += len; \ +#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ + do { \ + (void)memcpy((void *)buf, (void *)key->mem, len); \ + buf += len; \ } while (0) -#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ - do { \ - type c = var; \ - TAOS_UNUSED(memcpy((void *)buf, (void *)&c, sizeof(c))); \ - buf += sizeof(c); \ +#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ + do { \ + type c = var; \ + (void)memcpy((void *)buf, (void *)&c, sizeof(c)); \ + buf += sizeof(c); \ } while (0) -#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ - do { \ - TAOS_UNUSED(memcpy((void *)buf, (void *)var, len)); \ - buf += len; \ +#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ + do { \ + (void)memcpy((void *)buf, (void *)var, len); \ + buf += len; \ } while (0) #define INDEX_MERGE_ADD_DEL(src, dst, tgt) \ diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 38d4efd1ed..2e08f486ff 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -137,8 +137,12 @@ int32_t indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { TAOS_CHECK_GOTO(terrno, NULL, END); } - TAOS_UNUSED(taosThreadMutexInit(&idx->mtx, NULL)); - TAOS_UNUSED(tsem_init(&idx->sem, 0, 0)); + if (taosThreadMutexInit(&idx->mtx, NULL) != 0) { + TAOS_CHECK_GOTO(terrno, NULL, END); + } + if (tsem_init(&idx->sem, 0, 0) != 0) { + TAOS_CHECK_GOTO(terrno, NULL, END); + } idx->refId = idxAddRef(idx); idx->opts = *opts; @@ -213,7 +217,10 @@ void idxReleaseRef(int64_t ref) { int32_t indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { // TODO(yihao): reduce the lock range int32_t code = 0; - TAOS_UNUSED(taosThreadMutexLock(&index->mtx)); + if (taosThreadMutexLock(&index->mtx) != 0) { + indexError("failed to lock index mutex"); + } + for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); @@ -231,7 +238,9 @@ int32_t indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { } } } - TAOS_UNUSED(taosThreadMutexUnlock(&index->mtx)); + if (taosThreadMutexUnlock(&index->mtx) != 0) { + indexError("failed to unlock index mutex"); + } if (code != 0) { return code; @@ -306,6 +315,10 @@ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { } mtq->opera = opera; mtq->query = taosArrayInit(4, sizeof(SIndexTermQuery)); + if (mtq->query == NULL) { + taosMemoryFree(mtq); + return NULL; + } return mtq; } void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) { @@ -349,11 +362,21 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy if (colVal != NULL && nColVal != 0) { len = idxConvertDataToStr((void*)colVal, IDX_TYPE_GET_TYPE(colType), (void**)&buf); } else if (colVal == NULL) { - buf = strndup(INDEX_DATA_NULL_STR, (int32_t)strlen(INDEX_DATA_NULL_STR)); + buf = taosStrndup(INDEX_DATA_NULL_STR, (int32_t)strlen(INDEX_DATA_NULL_STR)); + if (buf == NULL) { + taosMemoryFree(tm->colName); + taosMemoryFree(tm); + return NULL; + } len = (int32_t)strlen(INDEX_DATA_NULL_STR); } else { static const char* emptyStr = " "; - buf = strndup(emptyStr, (int32_t)strlen(emptyStr)); + buf = taosStrndup(emptyStr, (int32_t)strlen(emptyStr)); + if (buf == NULL) { + taosMemoryFree(tm->colName); + taosMemoryFree(tm); + return NULL; + } len = (int32_t)strlen(emptyStr); } @@ -361,7 +384,6 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy if (tm->colVal == NULL) { taosMemoryFree(tm->colName); taosMemoryFree(tm); - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -463,7 +485,10 @@ static int32_t idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** resu int32_t sz = idxSerialCacheKey(&key, buf); - TAOS_UNUSED(taosThreadMutexLock(&sIdx->mtx)); + if (taosThreadMutexLock(&sIdx->mtx) != 0) { + indexError("failed to lock index mutex"); + } + IndexCache** pCache = taosHashGet(sIdx->colObj, buf, sz); cache = (pCache == NULL) ? NULL : *pCache; TAOS_UNUSED(taosThreadMutexUnlock(&sIdx->mtx)); @@ -757,7 +782,9 @@ static int64_t idxGetAvailableVer(SIndex* sIdx, IndexCache* cache) { IndexTFile* tf = (IndexTFile*)(sIdx->tindex); - TAOS_UNUSED(taosThreadMutexLock(&tf->mtx)); + if (taosThreadMutexLock(&tf->mtx) != 0) { + indexError("failed to lock tfile mutex"); + } TFileReader* rd = tfileCacheGet(tf->cache, &key); TAOS_UNUSED(taosThreadMutexUnlock(&tf->mtx)); @@ -801,9 +828,15 @@ static int32_t idxGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { TFileHeader* header = &reader->header; ICacheKey key = {.suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName)}; - TAOS_UNUSED(taosThreadMutexLock(&tf->mtx)); + if (taosThreadMutexLock(&tf->mtx) != 0) { + indexError("failed to lock tfile mutex"); + } + code = tfileCachePut(tf->cache, &key, reader); - TAOS_UNUSED(taosThreadMutexUnlock(&tf->mtx)); + + if (taosThreadMutexUnlock(&tf->mtx) != 0) { + indexError("failed to unlock tfile mutex"); + } return code; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 613c2430c9..a710bb6c10 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -80,6 +80,10 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STe IndexCache* pCache = mem->pCache; CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); + if (pCt == NULL) { + return terrno; + } + pCt->colVal = term->colVal; pCt->version = atomic_load_64(&pCache->version); @@ -290,6 +294,10 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR IndexCache* pCache = mem->pCache; CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); + if (pCt == NULL) { + return terrno; + } + pCt->colVal = term->colVal; pCt->version = atomic_load_64(&pCache->version); @@ -376,7 +384,7 @@ static IterateValue* idxCacheIteratorGetValue(Iterate* iter); IndexCache* idxCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) { IndexCache* cache = taosMemoryCalloc(1, sizeof(IndexCache)); if (cache == NULL) { - indexError("failed to create index cache"); + indexError("failed to create index cache since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); return NULL; }; @@ -384,14 +392,28 @@ IndexCache* idxCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8 cache->mem->pCache = cache; cache->colName = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? taosStrdup(JSON_COLUMN) : taosStrdup(colName); + if (cache->colName == NULL) { + taosMemoryFree(cache); + indexError("failed to create index cache since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return NULL; + } cache->type = type; cache->index = idx; cache->version = 0; cache->suid = suid; cache->occupiedMem = 0; - TAOS_UNUSED(taosThreadMutexInit(&cache->mtx, NULL)); - TAOS_UNUSED(taosThreadCondInit(&cache->finished, NULL)); + if (taosThreadMutexInit(&cache->mtx, NULL) != 0) { + indexError("failed to create mutex for index cache"); + taosMemoryFree(cache); + return NULL; + } + + if (taosThreadCondInit(&cache->finished, NULL) != 0) { + indexError("failed to create cond for index cache"); + taosMemoryFree(cache); + return NULL; + } idxCacheRef(cache); if (idx != NULL) { @@ -402,10 +424,16 @@ IndexCache* idxCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8 void idxCacheDebug(IndexCache* cache) { MemTable* tbl = NULL; - TAOS_UNUSED(taosThreadMutexLock(&cache->mtx)); + if ((taosThreadMutexLock(&cache->mtx)) != 0) { + indexError("failed to lock cache mutex"); + } + tbl = cache->mem; idxMemRef(tbl); - TAOS_UNUSED(taosThreadMutexUnlock(&cache->mtx)); + + if (taosThreadMutexUnlock(&cache->mtx) != 0) { + indexError("failed to unlock cache mutex"); + } { SSkipList* slt = tbl->mem; @@ -424,7 +452,9 @@ void idxCacheDebug(IndexCache* cache) { } { - TAOS_UNUSED(taosThreadMutexLock(&cache->mtx)); + if (taosThreadMutexLock(&cache->mtx) != 0) { + indexError("failed to lock cache mutex"); + } tbl = cache->imm; idxMemRef(tbl); TAOS_UNUSED(taosThreadMutexUnlock(&cache->mtx)); @@ -472,7 +502,9 @@ void idxCacheDestroyImm(IndexCache* cache) { return; } MemTable* tbl = NULL; - TAOS_UNUSED(taosThreadMutexLock(&cache->mtx)); + if (taosThreadMutexLock(&cache->mtx) != 0) { + indexError("failed to lock cache mutex"); + } tbl = cache->imm; cache->imm = NULL; // or throw int bg thread @@ -509,7 +541,11 @@ Iterate* idxCacheIteratorCreate(IndexCache* cache) { if (iter == NULL) { return NULL; } - TAOS_UNUSED(taosThreadMutexLock(&cache->mtx)); + if (taosThreadMutexLock(&cache->mtx) != 0) { + indexError("failed to lock cache mutex"); + taosMemoryFree(iter); + return NULL; + } idxMemRef(cache->imm); @@ -539,6 +575,10 @@ int idxCacheSchedToMerge(IndexCache* pCache, bool notify) { schedMsg.ahandle = pCache; if (notify) { schedMsg.thandle = taosMemoryMalloc(1); + if (schedMsg.thandle == NULL) { + indexError("fail to schedule merge task"); + return terrno; + } } schedMsg.msg = NULL; idxAcquireRef(pCache->index->refId); @@ -603,7 +643,9 @@ int idxCachePut(void* cache, SIndexTerm* term, uint64_t uid) { // ugly code, refactor later int64_t estimate = sizeof(ct) + strlen(ct->colVal); - TAOS_UNUSED(taosThreadMutexLock(&pCache->mtx)); + if (taosThreadMutexLock(&pCache->mtx) != 0) { + indexError("failed to lock cache mutex"); + } pCache->occupiedMem += estimate; idxCacheMakeRoomForWrite(pCache); MemTable* tbl = pCache->mem; @@ -611,7 +653,9 @@ int idxCachePut(void* cache, SIndexTerm* term, uint64_t uid) { TAOS_UNUSED(tSkipListPut(tbl->mem, (char*)ct)); idxMemUnRef(tbl); - TAOS_UNUSED(taosThreadMutexUnlock(&pCache->mtx)); + if (taosThreadMutexUnlock(&pCache->mtx) != 0) { + indexError("failed to unlock cache mutex"); + } idxCacheUnRef(pCache); return 0; } @@ -619,13 +663,17 @@ void idxCacheForceToMerge(void* cache) { IndexCache* pCache = cache; idxCacheRef(pCache); - TAOS_UNUSED(taosThreadMutexLock(&pCache->mtx)); + if (taosThreadMutexLock(&pCache->mtx) != 0) { + indexError("failed to lock cache mutex"); + } indexInfo("%p is forced to merge into tfile", pCache); pCache->occupiedMem += MEM_SIGNAL_QUIT; idxCacheMakeRoomForWrite(pCache); - TAOS_UNUSED(taosThreadMutexUnlock(&pCache->mtx)); + if (taosThreadMutexUnlock(&pCache->mtx) != 0) { + indexError("failed to unlock cache mutex"); + } idxCacheUnRef(pCache); return; } @@ -656,12 +704,16 @@ int idxCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STerm IndexCache* pCache = cache; MemTable *mem = NULL, *imm = NULL; - TAOS_UNUSED(taosThreadMutexLock(&pCache->mtx)); + if (taosThreadMutexLock(&pCache->mtx) != 0) { + indexError("failed to lock cache mutex"); + } mem = pCache->mem; imm = pCache->imm; idxMemRef(mem); idxMemRef(imm); - TAOS_UNUSED(taosThreadMutexUnlock(&pCache->mtx)); + if (taosThreadMutexUnlock(&pCache->mtx) != 0) { + indexError("failed to unlock cache mutex"); + } int64_t st = taosGetTimestampUs(); @@ -784,6 +836,7 @@ static MemTable* idxInternalCacheCreate(int8_t type) { IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? idxCacheJsonTermCompare : idxCacheTermCompare; MemTable* tbl = taosMemoryCalloc(1, sizeof(MemTable)); + if (tbl == NULL) return NULL; idxMemRef(tbl); // if (ttype == TSDB_DATA_TYPE_BINARY || ttype == TSDB_DATA_TYPE_NCHAR || ttype == TSDB_DATA_TYPE_GEOMETRY) { tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, ttype, MAX_INDEX_KEY_LEN, cmpFn, SL_ALLOW_DUP_KEY, idxCacheTermGet); diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index fee0df5582..b065f0c91e 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -260,7 +260,10 @@ char* idxPackJsonData(SIndexTerm* itm) { int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; char* buf = (char*)taosMemoryCalloc(1, sz); - char* p = buf; + if (buf == NULL) { + return NULL; + } + char* p = buf; memcpy(p, itm->colName, itm->nColName); p += itm->nColName; @@ -288,7 +291,10 @@ char* idxPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) { int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; char* buf = (char*)taosMemoryCalloc(1, sz); - char* p = buf; + if (buf == NULL) { + return NULL; + } + char* p = buf; memcpy(p, itm->colName, itm->nColName); p += itm->nColName; @@ -315,7 +321,11 @@ char* idxPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip) { int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; char* buf = (char*)taosMemoryCalloc(1, sz); - char* p = buf; + if (buf == NULL) { + return NULL; + } + + char* p = buf; memcpy(p, itm->colName, itm->nColName); p += itm->nColName; @@ -335,9 +345,9 @@ int idxUidCompare(const void* a, const void* b) { int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { if (src == NULL) { - *dst = strndup(INDEX_DATA_NULL_STR, (int)strlen(INDEX_DATA_NULL_STR)); + *dst = taosStrndup(INDEX_DATA_NULL_STR, (int)strlen(INDEX_DATA_NULL_STR)); if (*dst == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } return (int32_t)strlen(INDEX_DATA_NULL_STR); } @@ -379,6 +389,9 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { break; case TSDB_DATA_TYPE_USMALLINT: *dst = taosMemoryCalloc(1, bufSize + 1); + if (*dst == NULL) { + return terrno; + } TAOS_UNUSED(idxInt2str(*(uint16_t*)src, *dst, -1)); tlen = strlen(*dst); break; diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index e87126b930..8dafc05255 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -57,9 +57,17 @@ void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) { FstBuilderNode* node = taosMemoryMalloc(sizeof(FstBuilderNode)); + if (node == NULL) { + return; + } + node->isFinal = isFinal; node->finalOutput = 0; node->trans = taosArrayInit(16, sizeof(FstTransition)); + if (node->trans == NULL) { + taosMemoryFree(node); + return; + } FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; if (taosArrayPush(nodes->stack, &un) == NULL) { @@ -103,22 +111,23 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output if (un->last != NULL) return; - // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); - // trn->inp = s->data[s->start]; - // trn->out = out; int32_t len = 0; uint8_t* data = fstSliceData(s, &len); un->last = fstLastTransitionCreate(data[0], out); for (uint64_t i = 1; i < len; i++) { FstBuilderNode* n = taosMemoryMalloc(sizeof(FstBuilderNode)); + if (n == NULL) { + return; + } n->isFinal = false; n->finalOutput = 0; n->trans = taosArrayInit(16, sizeof(FstTransition)); + if (n->trans == NULL) { + taosMemoryFree(n); + return; + } - // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); - // trn->inp = s->data[i]; - // trn->out = out; FstLastTransition* trn = fstLastTransitionCreate(data[i], 0); FstBuilderNodeUnfinished un = {.node = n, .last = trn}; @@ -295,6 +304,9 @@ void fstStateCompileForAnyTrans(IdxFstFile* w, CompiledAddr addr, FstBuilderNode if (sz > TRANS_INDEX_THRESHOLD) { // A value of 255 indicates that no transition exists for the byte at that idx uint8_t* index = (uint8_t*)taosMemoryMalloc(sizeof(uint8_t) * 256); + if (index == NULL) { + return; + } memset(index, 255, sizeof(uint8_t) * 256); for (int32_t i = 0; i < sz; i++) { FstTransition* t = taosArrayGet(node->trans, i); @@ -973,10 +985,16 @@ Fst* fstCreate(FstSlice* slice) { fst->meta->checkSum = checkSum; FstSlice* s = taosMemoryCalloc(1, sizeof(FstSlice)); + if (s == NULL) { + goto FST_CREAT_FAILED; + } + *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice) - 1); fst->data = s; - TAOS_UNUSED(taosThreadMutexInit(&fst->mtx, NULL)); + if (taosThreadMutexInit(&fst->mtx, NULL) != 0) { + goto FST_CREAT_FAILED; + } return fst; FST_CREAT_FAILED: @@ -1139,11 +1157,20 @@ FStmSt* stmStCreate(Fst* fst, FAutoCtx* automation, FstBoundWithData* min, FstBo sws->fst = fst; sws->aut = automation; sws->inp = (SArray*)taosArrayInit(256, sizeof(uint8_t)); + if (sws->inp == NULL) { + taosMemoryFree(sws); + return NULL; + } sws->emptyOutput.null = true; sws->emptyOutput.out = 0; sws->stack = (SArray*)taosArrayInit(256, sizeof(FstStreamState)); + if (sws->stack == NULL) { + taosArrayDestroy(sws->inp); + taosMemoryFree(sws); + return NULL; + } sws->endAt = max; TAOS_UNUSED(stmStSeekMin(sws, min)); @@ -1326,6 +1353,9 @@ FStmStRslt* stmStNextWith(FStmSt* sws, streamCallback__fn callback) { int32_t isz = taosArrayGetSize(sws->inp); uint8_t* buf = (uint8_t*)taosMemoryMalloc(isz * sizeof(uint8_t)); + if (buf == NULL) { + return NULL; + } for (uint32_t i = 0; i < isz; i++) { buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); } diff --git a/source/libs/index/src/indexFstAutomation.c b/source/libs/index/src/indexFstAutomation.c index 0c96d1aa0a..2b1f3395eb 100644 --- a/source/libs/index/src/indexFstAutomation.c +++ b/source/libs/index/src/indexFstAutomation.c @@ -28,6 +28,11 @@ StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueTyp } else if (ty == FST_CHAR) { size_t len = strlen((char*)val); sv->ptr = (char*)taosMemoryCalloc(1, len + 1); + if (sv->ptr == NULL) { + taosMemoryFree(sv); + return NULL; + } + memcpy(sv->ptr, val, len); } else if (ty == FST_ARRAY) { // TODO, @@ -63,6 +68,11 @@ StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { } else if (nsv->type == FST_CHAR) { size_t len = strlen(sv->ptr); nsv->ptr = (char*)taosMemoryCalloc(1, len + 1); + if (nsv->ptr == NULL) { + taosMemoryFree(nsv); + return NULL; + } + memcpy(nsv->ptr, sv->ptr, len); } else if (nsv->type == FST_ARRAY) { // @@ -164,7 +174,18 @@ FAutoCtx* automCtxCreate(void* data, AutomationType atype) { // add more search type } - ctx->data = (data != NULL ? taosStrdup((char*)data) : NULL); + // ctx->data = (data != NULL ? taosStrdup((char*)data) : NULL); + if (data != NULL) { + ctx->data = taosStrdup((char*)data); + if (ctx->data == NULL) { + startWithStateValueDestroy(sv); + taosMemoryFree(ctx); + return NULL; + } + } else { + ctx->data = NULL; + } + ctx->type = atype; ctx->stdata = (void*)sv; return ctx; diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index efe890aa3a..2351273975 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -96,8 +96,8 @@ static FORCE_INLINE int idxFileCtxDoRead(IFileCtx* ctx, uint8_t* buf, int len) { return nRead; } -static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t offset) { - int32_t total = 0, nread = 0; +static int64_t idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t offset) { + int64_t total = 0, nread = 0; int32_t blkId = offset / kBlockSize; int32_t blkOffset = offset % kBlockSize; int32_t blkLeft = kBlockSize - blkOffset; @@ -122,7 +122,7 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of int32_t left = ctx->file.size - offset; if (left < kBlockSize) { nread = TMIN(left, len); - int32_t bytes = taosPReadFile(ctx->file.pFile, buf + total, nread, offset); + int64_t bytes = taosPReadFile(ctx->file.pFile, buf + total, nread, offset); if (bytes != nread) { total = TSDB_CODE_INDEX_INVALID_FILE; break; @@ -134,6 +134,9 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of int32_t cacheMemSize = sizeof(SDataBlock) + kBlockSize; SDataBlock* blk = taosMemoryCalloc(1, cacheMemSize); + if (blk == NULL) { + return terrno; + } blk->blockId = blkId; blk->nread = taosPReadFile(ctx->file.pFile, blk->buf, kBlockSize, blkId * kBlockSize); if (blk->nread < kBlockSize && blk->nread < len) { @@ -145,7 +148,7 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of memcpy(buf + total, blk->buf + blkOffset, nread); LRUStatus s = taosLRUCacheInsert(ctx->lru, key, strlen(key), blk, cacheMemSize, deleteDataBlockFromLRU, NULL, - TAOS_LRU_PRIORITY_LOW, NULL); + NULL, TAOS_LRU_PRIORITY_LOW, NULL); if (s != TAOS_LRU_STATUS_OK) { return -1; } @@ -211,6 +214,10 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int ctx->file.wBufOffset = 0; ctx->file.wBufCap = kBlockSize * 4; ctx->file.wBuf = taosMemoryCalloc(1, ctx->file.wBufCap); + if (ctx->file.wBuf == NULL) { + indexError("failed to allocate memory for write buffer"); + goto END; + } } else { ctx->file.pFile = taosOpenFile(path, TD_FILE_READ); code = taosFStatFile(ctx->file.pFile, &ctx->file.size, NULL); @@ -228,6 +235,11 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int } } else if (ctx->type == TMEMORY) { ctx->mem.buf = taosMemoryCalloc(1, sizeof(char) * capacity); + if (ctx->mem.buf == NULL) { + indexError("failed to allocate memory for memory buffer"); + goto END; + } + ctx->mem.cap = capacity; } @@ -325,6 +337,10 @@ int idxFileFlush(IdxFstFile* write) { void idxFilePackUintIn(IdxFstFile* writer, uint64_t n, uint8_t nBytes) { uint8_t* buf = taosMemoryCalloc(8, sizeof(uint8_t)); + if (buf == NULL) { + indexError("failed to allocate memory for packing uint"); + return; + } for (uint8_t i = 0; i < nBytes; i++) { buf[i] = (uint8_t)n; n = n >> 8; diff --git a/source/libs/index/src/indexFstRegister.c b/source/libs/index/src/indexFstRegister.c index ce34bb50d0..701203e980 100644 --- a/source/libs/index/src/indexFstRegister.c +++ b/source/libs/index/src/indexFstRegister.c @@ -120,6 +120,8 @@ FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNo uint64_t end = start + registry->mruSize; FstRegistryEntry* entry = taosMemoryMalloc(sizeof(FstRegistryEntry)); + if (entry == NULL) return NULL; + if (end - start == 1) { FstRegistryCell* cell = taosArrayGet(registry->table, start); // cell->isNode && diff --git a/source/libs/index/src/indexFstSparse.c b/source/libs/index/src/indexFstSparse.c index 8746b04eab..6fd97af044 100644 --- a/source/libs/index/src/indexFstSparse.c +++ b/source/libs/index/src/indexFstSparse.c @@ -28,6 +28,12 @@ FstSparseSet *sparSetCreate(int32_t sz) { ss->dense = (int32_t *)taosMemoryMalloc(sz * sizeof(int32_t)); ss->sparse = (int32_t *)taosMemoryMalloc(sz * sizeof(int32_t)); + if (ss->dense == NULL || ss->sparse == NULL) { + taosMemoryFree(ss->dense); + taosMemoryFree(ss->sparse); + taosMemoryFree(ss); + return NULL; + } sparSetInitBuf(ss->dense, sz); sparSetInitBuf(ss->sparse, sz); diff --git a/source/libs/index/src/indexFstUtil.c b/source/libs/index/src/indexFstUtil.c index f0df2f7124..8173196860 100644 --- a/source/libs/index/src/indexFstUtil.c +++ b/source/libs/index/src/indexFstUtil.c @@ -77,9 +77,16 @@ CompiledAddr unpackDelta(char* data, uint64_t len, uint64_t nodeAddr) { FstSlice fstSliceCreate(uint8_t* data, uint64_t len) { FstString* str = (FstString*)taosMemoryMalloc(sizeof(FstString)); + if (str == NULL) { + return (FstSlice){.str = NULL, .start = 0, .end = 0}; + } str->ref = 1; str->len = len; str->data = taosMemoryMalloc(len * sizeof(uint8_t)); + if (str->data == NULL) { + taosMemoryFree(str); + return (FstSlice){.str = NULL, .start = 0, .end = 0}; + } if (data != NULL && str->data != NULL) { memcpy(str->data, data, len); @@ -103,9 +110,16 @@ FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) { uint8_t* data = fstSliceData(s, &slen); uint8_t* buf = taosMemoryMalloc(sizeof(uint8_t) * tlen); + if (buf == NULL) { + return (FstSlice){.str = NULL, .start = 0, .end = 0}; + } memcpy(buf, data + start, tlen); FstString* str = taosMemoryMalloc(sizeof(FstString)); + if (str == NULL) { + taosMemoryFree(buf); + return (FstSlice){.str = NULL, .start = 0, .end = 0}; + } str->data = buf; str->len = tlen; str->ref = 1; diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 78fd9da452..e2693ff4a4 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -45,7 +45,7 @@ static int tfileWriteFooter(TFileWriter* write); // handle file corrupt later static int tfileReaderLoadHeader(TFileReader* reader); -static int tfileReaderLoadFst(TFileReader* reader); +static int32_t tfileReaderLoadFst(TFileReader* reader); static int tfileReaderVerify(TFileReader* reader); static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); @@ -739,7 +739,11 @@ IndexTFile* idxTFileCreate(SIndex* idx, const char* path) { tfileCacheDestroy(cache); return NULL; } - TAOS_UNUSED(taosThreadMutexInit(&tfile->mtx, NULL)); + if (taosThreadMutexInit(&tfile->mtx, NULL) != 0) { + taosMemoryFree(tfile); + tfileCacheDestroy(cache); + return NULL; + } tfile->cache = cache; return tfile; } @@ -764,9 +768,16 @@ int idxTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* result) { SIndexTerm* term = query->term; ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; - TAOS_UNUSED(taosThreadMutexLock(&pTfile->mtx)); + if (taosThreadMutexLock(&pTfile->mtx) != 0) { + indexError("failed to lock tfile mutex"); + } + TFileReader* reader = tfileCacheGet(pTfile->cache, &key); - TAOS_UNUSED(taosThreadMutexUnlock(&pTfile->mtx)); + + if (taosThreadMutexUnlock(&pTfile->mtx) != 0) { + indexError("failed to unlock tfile mutex"); + } + if (reader == NULL) { return 0; } @@ -799,6 +810,10 @@ static bool tfileIteratorNext(Iterate* iiter) { int32_t sz = 0; char* ch = (char*)fstSliceData(&rt->data, &sz); colVal = taosMemoryCalloc(1, sz + 1); + if (colVal == NULL) { + return false; + } + memcpy(colVal, ch, sz); offset = (uint64_t)(rt->out.out); @@ -835,6 +850,10 @@ Iterate* tfileIteratorCreate(TFileReader* reader) { } Iterate* iter = taosMemoryCalloc(1, sizeof(Iterate)); + if (iter == NULL) { + return NULL; + } + iter->iter = tfileFstIteratorCreate(reader); if (iter->iter == NULL) { taosMemoryFree(iter); @@ -843,6 +862,11 @@ Iterate* tfileIteratorCreate(TFileReader* reader) { iter->next = tfileIteratorNext; iter->getValue = tifileIterateGetValue; iter->val.val = taosArrayInit(1, sizeof(uint64_t)); + if (iter->val.val == NULL) { + tfileIteratorDestroy(iter); + return NULL; + } + iter->val.colVal = NULL; return iter; } @@ -870,9 +894,13 @@ TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) { TFileReader* rd = NULL; ICacheKey key = {.suid = suid, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; - TAOS_UNUSED(taosThreadMutexLock(&tf->mtx)); + if (taosThreadMutexLock(&tf->mtx) != 0) { + indexError("failed to lock tfile mutex"); + } rd = tfileCacheGet(tf->cache, &key); - TAOS_UNUSED(taosThreadMutexUnlock(&tf->mtx)); + if (taosThreadMutexUnlock(&tf->mtx) != 0) { + indexError("failed to unlock tfile mutex"); + } return rd; } @@ -994,7 +1022,7 @@ static int tfileReaderLoadHeader(TFileReader* reader) { int64_t nread = reader->ctx->readFrom(reader->ctx, (uint8_t*)buf, sizeof(buf), 0); - if (nread == -1) { + if (nread < 0) { indexError("actual Read: %d, to read: %d, code:0x%x, filename: %s", (int)(nread), (int)sizeof(buf), errno, reader->ctx->file.buf); } else { @@ -1004,7 +1032,7 @@ static int tfileReaderLoadHeader(TFileReader* reader) { return 0; } -static int tfileReaderLoadFst(TFileReader* reader) { +static int32_t tfileReaderLoadFst(TFileReader* reader) { IFileCtx* ctx = reader->ctx; int size = ctx->size(ctx); @@ -1012,7 +1040,7 @@ static int tfileReaderLoadFst(TFileReader* reader) { int fstSize = size - reader->header.fstOffset - sizeof(FILE_MAGIC_NUMBER); char* buf = taosMemoryCalloc(1, fstSize); if (buf == NULL) { - return -1; + return terrno; } int64_t ts = taosGetTimestampUs(); @@ -1030,7 +1058,7 @@ static int tfileReaderLoadFst(TFileReader* reader) { taosMemoryFree(buf); fstSliceDestroy(&st); - return reader->fst != NULL ? 0 : -1; + return reader->fst != NULL ? 0 : TSDB_CODE_INDEX_INVALID_FILE; } static int32_t tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) { // TODO(yihao): opt later @@ -1039,7 +1067,7 @@ static int32_t tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArr IFileCtx* ctx = reader->ctx; // add block cache char block[4096] = {0}; - int32_t nread = ctx->readFrom(ctx, (uint8_t*)block, sizeof(block), offset); + int64_t nread = ctx->readFrom(ctx, (uint8_t*)block, sizeof(block), offset); if (nread < sizeof(uint32_t)) { return TSDB_CODE_INDEX_INVALID_FILE; } diff --git a/source/libs/index/src/indexUtil.c b/source/libs/index/src/indexUtil.c index cbecb90d5e..12bff36553 100644 --- a/source/libs/index/src/indexUtil.c +++ b/source/libs/index/src/indexUtil.c @@ -90,6 +90,10 @@ int32_t iUnion(SArray *in, SArray *out) { } MergeIndex *mi = taosMemoryCalloc(sz, sizeof(MergeIndex)); + if (mi == NULL) { + return terrno; + } + for (int i = 0; i < sz; i++) { SArray *t = taosArrayGetP(in, i); mi[i].len = (int32_t)taosArrayGetSize(t); diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c index 76473ccbb1..a2d03bbd6a 100644 --- a/source/libs/monitor/src/monFramework.c +++ b/source/libs/monitor/src/monFramework.c @@ -100,7 +100,7 @@ extern char* tsMonFwUri; #define VNODE_ROLE "taosd_vnodes_info:role" void monInitMonitorFW(){ - (void)taos_collector_registry_default_init(); + taos_collector_registry_default_init(); tsMonitor.metrics = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); taos_gauge_t *gauge = NULL; @@ -115,7 +115,9 @@ void monInitMonitorFW(){ for(int32_t i = 0; i < 25; i++){ gauge= taos_gauge_new(dnodes_gauges[i], "", dnodes_label_count, dnodes_sample_labels); if(taos_collector_registry_register_metric(gauge) == 1){ - (void)taos_counter_destroy(gauge); + if (taos_counter_destroy(gauge) != 0) { + uError("failed to delete metric %s", dnodes_gauges[i]); + } } if (taosHashPut(tsMonitor.metrics, dnodes_gauges[i], strlen(dnodes_gauges[i]), &gauge, sizeof(taos_gauge_t *)) != 0) { @@ -129,7 +131,9 @@ void monInitMonitorFW(){ for(int32_t i = 0; i < 3; i++){ gauge= taos_gauge_new(dnodes_data_gauges[i], "", dnodes_data_label_count, dnodes_data_sample_labels); if(taos_collector_registry_register_metric(gauge) == 1){ - (void)taos_counter_destroy(gauge); + if (taos_counter_destroy(gauge) != 0) { + uError("failed to delete metric %s", dnodes_data_gauges[i]); + } } if (taosHashPut(tsMonitor.metrics, dnodes_data_gauges[i], strlen(dnodes_data_gauges[i]), &gauge, sizeof(taos_gauge_t *)) != 0) { @@ -143,7 +147,9 @@ void monInitMonitorFW(){ for(int32_t i = 0; i < 3; i++){ gauge= taos_gauge_new(dnodes_log_gauges[i], "", dnodes_log_label_count, dnodes_log_sample_labels); if(taos_collector_registry_register_metric(gauge) == 1){ - (void)taos_counter_destroy(gauge); + if (taos_counter_destroy(gauge) != 0) { + uError("failed to delete metric %s", dnodes_log_gauges[i]); + } } if (taosHashPut(tsMonitor.metrics, dnodes_log_gauges[i], strlen(dnodes_log_gauges[i]), &gauge, sizeof(taos_gauge_t *)) != 0) { @@ -154,7 +160,9 @@ void monInitMonitorFW(){ void monCleanupMonitorFW(){ taosHashCleanup(tsMonitor.metrics); - (void)taos_collector_registry_destroy(TAOS_COLLECTOR_REGISTRY_DEFAULT); + if (taos_collector_registry_destroy(TAOS_COLLECTOR_REGISTRY_DEFAULT) != 0) { + uError("failed to destroy default collector registry"); + } TAOS_COLLECTOR_REGISTRY_DEFAULT = NULL; } @@ -174,7 +182,9 @@ void monGenClusterInfoTable(SMonInfo *pMonitor){ uError("failed to delete metric %s", metric_names[i]); } - (void)taosHashRemove(tsMonitor.metrics, metric_names[i], strlen(metric_names[i])); + if (taosHashRemove(tsMonitor.metrics, metric_names[i], strlen(metric_names[i])) != 0) { + uError("failed to remove metric %s", metric_names[i]); + } } if(pBasicInfo->cluster_id == 0) { @@ -191,7 +201,9 @@ void monGenClusterInfoTable(SMonInfo *pMonitor){ for(int32_t i = 0; i < 18; i++){ gauge= taos_gauge_new(metric_names[i], "", label_count, sample_labels1); if(taos_collector_registry_register_metric(gauge) == 1){ - (void)taos_counter_destroy(gauge); + if (taos_counter_destroy(gauge) != 0) { + uError("failed to delete metric %s", metric_names[i]); + } } if (taosHashPut(tsMonitor.metrics, metric_names[i], strlen(metric_names[i]), &gauge, sizeof(taos_gauge_t *)) != 0) { uError("failed to add cluster gauge at%d:%s", i, metric_names[i]); @@ -317,11 +329,15 @@ void monGenVgroupInfoTable(SMonInfo *pMonitor){ const char *vgroup_sample_labels[] = {"cluster_id", "vgroup_id", "database_name"}; taos_gauge_t *tableNumGauge = taos_gauge_new(TABLES_NUM, "", vgroup_label_count, vgroup_sample_labels); if(taos_collector_registry_register_metric(tableNumGauge) == 1){ - (void)taos_counter_destroy(tableNumGauge); + if (taos_counter_destroy(tableNumGauge) != 0) { + uError("failed to delete metric " TABLES_NUM); + } } taos_gauge_t *statusGauge = taos_gauge_new(STATUS, "", vgroup_label_count, vgroup_sample_labels); if(taos_collector_registry_register_metric(statusGauge) == 1){ - (void)taos_counter_destroy(statusGauge); + if (taos_counter_destroy(statusGauge) != 0) { + uError("failed to delete metric " STATUS); + } } char cluster_id[TSDB_CLUSTER_ID_LEN] = {0}; @@ -530,7 +546,9 @@ void monGenDnodeStatusInfoTable(SMonInfo *pMonitor){ gauge= taos_gauge_new(DNODE_STATUS, "", dnodes_label_count, dnodes_sample_labels); if(taos_collector_registry_register_metric(gauge) == 1){ - (void)taos_counter_destroy(gauge); + if (taos_counter_destroy(gauge) != 0) { + uError("failed to delete metric " DNODE_STATUS); + } } char cluster_id[TSDB_CLUSTER_ID_LEN]; @@ -633,7 +651,9 @@ void monGenMnodeRoleTable(SMonInfo *pMonitor){ uError("failed to delete metric %s", mnodes_role_gauges[i]); } - (void)taosHashRemove(tsMonitor.metrics, mnodes_role_gauges[i], strlen(mnodes_role_gauges[i])); + if (taosHashRemove(tsMonitor.metrics, mnodes_role_gauges[i], strlen(mnodes_role_gauges[i])) != 0) { + uError("failed to remove metric %s", mnodes_role_gauges[i]); + } } SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; @@ -647,7 +667,9 @@ void monGenMnodeRoleTable(SMonInfo *pMonitor){ for(int32_t i = 0; i < 1; i++){ gauge= taos_gauge_new(mnodes_role_gauges[i], "", mnodes_role_label_count, mnodes_role_sample_labels); if(taos_collector_registry_register_metric(gauge) == 1){ - (void)taos_counter_destroy(gauge); + if (taos_counter_destroy(gauge) != 0) { + uError("failed to destroy gauge"); + } } if (taosHashPut(tsMonitor.metrics, mnodes_role_gauges[i], strlen(mnodes_role_gauges[i]), &gauge, sizeof(taos_gauge_t *)) != 0) { @@ -702,7 +724,9 @@ void monGenVnodeRoleTable(SMonInfo *pMonitor){ uError("failed to delete metric %s", vnodes_role_gauges[i]); } - (void)taosHashRemove(tsMonitor.metrics, vnodes_role_gauges[i], strlen(vnodes_role_gauges[i])); + if (taosHashRemove(tsMonitor.metrics, vnodes_role_gauges[i], strlen(vnodes_role_gauges[i])) != 0) { + uError("failed to remove metric %s", vnodes_role_gauges[i]); + } } SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup; @@ -716,7 +740,9 @@ void monGenVnodeRoleTable(SMonInfo *pMonitor){ for(int32_t i = 0; i < 1; i++){ gauge= taos_gauge_new(vnodes_role_gauges[i], "", vnodes_role_label_count, vnodes_role_sample_labels); if(taos_collector_registry_register_metric(gauge) == 1){ - (void)taos_counter_destroy(gauge); + if (taos_counter_destroy(gauge) != 0) { + uError("failed to destroy gauge"); + } } if (taosHashPut(tsMonitor.metrics, vnodes_role_gauges[i], strlen(vnodes_role_gauges[i]), &gauge, sizeof(taos_gauge_t *)) != 0) { @@ -774,7 +800,9 @@ void monSendPromReport() { tmp) != 0) { uError("failed to send monitor msg"); } else { - (void)taos_collector_registry_clear_batch(TAOS_COLLECTOR_REGISTRY_DEFAULT); + if (taos_collector_registry_clear_batch(TAOS_COLLECTOR_REGISTRY_DEFAULT) != 0) { + uError("failed to clear batch"); + } } taosMemoryFreeClear(pCont); } diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index f581d8c83d..744177b7a1 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -145,7 +145,9 @@ void monInitVnode() { counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql", label_count, sample_labels); uDebug("new metric:%p", counter); if (taos_collector_registry_register_metric(counter) == 1) { - (void)taos_counter_destroy(counter); + if (taos_counter_destroy(counter) != 0) { + uError("failed to destroy metric:%p", counter); + } uError("failed to register metric:%p", counter); } else { tsInsertCounter = counter; @@ -226,14 +228,17 @@ static void monGenBasicJson(SMonInfo *pMonitor) { SJson *pJson = pMonitor->pJson; char buf[40] = {0}; - (void)taosFormatUtcTime(buf, sizeof(buf), pMonitor->curTime, TSDB_TIME_PRECISION_MILLI); + if (taosFormatUtcTime(buf, sizeof(buf), pMonitor->curTime, TSDB_TIME_PRECISION_MILLI) != 0) { + uError("failed to format time"); + return; + } - (void)tjsonAddStringToObject(pJson, "ts", buf); - (void)tjsonAddDoubleToObject(pJson, "dnode_id", pInfo->dnode_id); - (void)tjsonAddStringToObject(pJson, "dnode_ep", pInfo->dnode_ep); + if (tjsonAddStringToObject(pJson, "ts", buf) != 0) uError("failed to add ts"); + if (tjsonAddDoubleToObject(pJson, "dnode_id", pInfo->dnode_id) != 0) uError("failed to add dnode_id"); + if (tjsonAddStringToObject(pJson, "dnode_ep", pInfo->dnode_ep) != 0) uError("failed to add dnode_ep"); snprintf(buf, sizeof(buf), "%" PRId64, pInfo->cluster_id); - (void)tjsonAddStringToObject(pJson, "cluster_id", buf); - (void)tjsonAddDoubleToObject(pJson, "protocol", pInfo->protocol); + if (tjsonAddStringToObject(pJson, "cluster_id", buf) != 0) uError("failed to add cluster_id"); + if (tjsonAddDoubleToObject(pJson, "protocol", pInfo->protocol) != 0) uError("failed to add protocol"); } static void monGenBasicJsonBasic(SMonInfo *pMonitor) { @@ -244,12 +249,12 @@ static void monGenBasicJsonBasic(SMonInfo *pMonitor) { char buf[40] = {0}; sprintf(buf, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); - (void)tjsonAddStringToObject(pJson, "ts", buf); - (void)tjsonAddDoubleToObject(pJson, "dnode_id", pInfo->dnode_id); - (void)tjsonAddStringToObject(pJson, "dnode_ep", pInfo->dnode_ep); + if (tjsonAddStringToObject(pJson, "ts", buf) != 0) uError("failed to add ts"); + if (tjsonAddDoubleToObject(pJson, "dnode_id", pInfo->dnode_id) != 0) uError("failed to add dnode_id"); + if (tjsonAddStringToObject(pJson, "dnode_ep", pInfo->dnode_ep) != 0) uError("failed to add dnode_ep"); snprintf(buf, sizeof(buf), "%" PRId64, pInfo->cluster_id); - (void)tjsonAddStringToObject(pJson, "cluster_id", buf); - (void)tjsonAddDoubleToObject(pJson, "protocol", pInfo->protocol); + if (tjsonAddStringToObject(pJson, "cluster_id", buf) != 0) uError("failed to add cluster_id"); + if (tjsonAddDoubleToObject(pJson, "protocol", pInfo->protocol) != 0) uError("failed to add protocol"); } static void monGenClusterJson(SMonInfo *pMonitor) { @@ -263,21 +268,24 @@ static void monGenClusterJson(SMonInfo *pMonitor) { return; } - (void)tjsonAddStringToObject(pJson, "first_ep", pInfo->first_ep); - (void)tjsonAddDoubleToObject(pJson, "first_ep_dnode_id", pInfo->first_ep_dnode_id); - (void)tjsonAddStringToObject(pJson, "version", pInfo->version); - (void)tjsonAddDoubleToObject(pJson, "master_uptime", pInfo->master_uptime); - (void)tjsonAddDoubleToObject(pJson, "monitor_interval", pInfo->monitor_interval); - (void)tjsonAddDoubleToObject(pJson, "dbs_total", pInfo->dbs_total); - (void)tjsonAddDoubleToObject(pJson, "tbs_total", pInfo->tbs_total); - (void)tjsonAddDoubleToObject(pJson, "stbs_total", pInfo->stbs_total); - (void)tjsonAddDoubleToObject(pJson, "vgroups_total", pInfo->vgroups_total); - (void)tjsonAddDoubleToObject(pJson, "vgroups_alive", pInfo->vgroups_alive); - (void)tjsonAddDoubleToObject(pJson, "vnodes_total", pInfo->vnodes_total); - (void)tjsonAddDoubleToObject(pJson, "vnodes_alive", pInfo->vnodes_alive); - (void)tjsonAddDoubleToObject(pJson, "connections_total", pInfo->connections_total); - (void)tjsonAddDoubleToObject(pJson, "topics_total", pInfo->topics_toal); - (void)tjsonAddDoubleToObject(pJson, "streams_total", pInfo->streams_total); + if (tjsonAddStringToObject(pJson, "first_ep", pInfo->first_ep) != 0) uError("failed to add first_ep"); + if (tjsonAddDoubleToObject(pJson, "first_ep_dnode_id", pInfo->first_ep_dnode_id) != 0) + uError("failed to add first_ep_dnode_id"); + if (tjsonAddStringToObject(pJson, "version", pInfo->version) != 0) uError("failed to add version"); + if (tjsonAddDoubleToObject(pJson, "master_uptime", pInfo->master_uptime) != 0) uError("failed to add master_uptime"); + if (tjsonAddDoubleToObject(pJson, "monitor_interval", pInfo->monitor_interval) != 0) + uError("failed to add monitor_interval"); + if (tjsonAddDoubleToObject(pJson, "dbs_total", pInfo->dbs_total) != 0) uError("failed to add dbs_total"); + if (tjsonAddDoubleToObject(pJson, "tbs_total", pInfo->tbs_total) != 0) uError("failed to add tbs_total"); + if (tjsonAddDoubleToObject(pJson, "stbs_total", pInfo->stbs_total) != 0) uError("failed to add stbs_total"); + if (tjsonAddDoubleToObject(pJson, "vgroups_total", pInfo->vgroups_total) != 0) uError("failed to add vgroups_total"); + if (tjsonAddDoubleToObject(pJson, "vgroups_alive", pInfo->vgroups_alive) != 0) uError("failed to add vgroups_alive"); + if (tjsonAddDoubleToObject(pJson, "vnodes_total", pInfo->vnodes_total) != 0) uError("failed to add vnodes_total"); + if (tjsonAddDoubleToObject(pJson, "vnodes_alive", pInfo->vnodes_alive) != 0) uError("failed to add vnodes_alive"); + if (tjsonAddDoubleToObject(pJson, "connections_total", pInfo->connections_total) != 0) + uError("failed to add connections_total"); + if (tjsonAddDoubleToObject(pJson, "topics_total", pInfo->topics_toal) != 0) uError("failed to add topics_total"); + if (tjsonAddDoubleToObject(pJson, "streams_total", pInfo->streams_total) != 0) uError("failed to add streams_total"); SJson *pDnodesJson = tjsonAddArrayToObject(pJson, "dnodes"); if (pDnodesJson == NULL) return; @@ -287,9 +295,9 @@ static void monGenClusterJson(SMonInfo *pMonitor) { if (pDnodeJson == NULL) continue; SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i); - (void)tjsonAddDoubleToObject(pDnodeJson, "dnode_id", pDnodeDesc->dnode_id); - (void)tjsonAddStringToObject(pDnodeJson, "dnode_ep", pDnodeDesc->dnode_ep); - (void)tjsonAddStringToObject(pDnodeJson, "status", pDnodeDesc->status); + if (tjsonAddDoubleToObject(pDnodeJson, "dnode_id", pDnodeDesc->dnode_id) != 0) uError("failed to add dnode_id"); + if (tjsonAddStringToObject(pDnodeJson, "dnode_ep", pDnodeDesc->dnode_ep) != 0) uError("failed to add dnode_ep"); + if (tjsonAddStringToObject(pDnodeJson, "status", pDnodeDesc->status) != 0) uError("failed to add status"); if (tjsonAddItemToArray(pDnodesJson, pDnodeJson) != 0) tjsonDelete(pDnodeJson); } @@ -302,9 +310,9 @@ static void monGenClusterJson(SMonInfo *pMonitor) { if (pMnodeJson == NULL) continue; SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->mnodes, i); - (void)tjsonAddDoubleToObject(pMnodeJson, "mnode_id", pMnodeDesc->mnode_id); - (void)tjsonAddStringToObject(pMnodeJson, "mnode_ep", pMnodeDesc->mnode_ep); - (void)tjsonAddStringToObject(pMnodeJson, "role", pMnodeDesc->role); + if (tjsonAddDoubleToObject(pMnodeJson, "mnode_id", pMnodeDesc->mnode_id) != 0) uError("failed to add mnode_id"); + if (tjsonAddStringToObject(pMnodeJson, "mnode_ep", pMnodeDesc->mnode_ep) != 0) uError("failed to add mnode_ep"); + if (tjsonAddStringToObject(pMnodeJson, "role", pMnodeDesc->role) != 0) uError("failed to add role"); if (tjsonAddItemToArray(pMnodesJson, pMnodeJson) != 0) tjsonDelete(pMnodeJson); } @@ -314,11 +322,11 @@ static void monGenClusterJsonBasic(SMonInfo *pMonitor) { SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; - // (void)tjsonAddStringToObject(pMonitor->pJson, "first_ep", pInfo->first_ep); - (void)tjsonAddStringToObject(pMonitor->pJson, "first_ep", tsFirst); - (void)tjsonAddDoubleToObject(pMonitor->pJson, "first_ep_dnode_id", pInfo->first_ep_dnode_id); - (void)tjsonAddStringToObject(pMonitor->pJson, "cluster_version", pInfo->version); - // (void)tjsonAddDoubleToObject(pMonitor->pJson, "monitor_interval", pInfo->monitor_interval); + if (tjsonAddStringToObject(pMonitor->pJson, "first_ep", tsFirst) != 0) uError("failed to add first_ep"); + if (tjsonAddDoubleToObject(pMonitor->pJson, "first_ep_dnode_id", pInfo->first_ep_dnode_id) != 0) + uError("failed to add first_ep_dnode_id"); + if (tjsonAddStringToObject(pMonitor->pJson, "cluster_version", pInfo->version) != 0) + uError("failed to add cluster_version"); } static void monGenVgroupJson(SMonInfo *pMonitor) { @@ -337,10 +345,13 @@ static void monGenVgroupJson(SMonInfo *pMonitor) { } SMonVgroupDesc *pVgroupDesc = taosArrayGet(pInfo->vgroups, i); - (void)tjsonAddDoubleToObject(pVgroupJson, "vgroup_id", pVgroupDesc->vgroup_id); - (void)tjsonAddStringToObject(pVgroupJson, "database_name", pVgroupDesc->database_name); - (void)tjsonAddDoubleToObject(pVgroupJson, "tables_num", pVgroupDesc->tables_num); - (void)tjsonAddStringToObject(pVgroupJson, "status", pVgroupDesc->status); + if (tjsonAddDoubleToObject(pVgroupJson, "vgroup_id", pVgroupDesc->vgroup_id) != 0) + uError("failed to add vgroup_id"); + if (tjsonAddStringToObject(pVgroupJson, "database_name", pVgroupDesc->database_name) != 0) + uError("failed to add database_name"); + if (tjsonAddDoubleToObject(pVgroupJson, "tables_num", pVgroupDesc->tables_num) != 0) + uError("failed to add tables_num"); + if (tjsonAddStringToObject(pVgroupJson, "status", pVgroupDesc->status) != 0) uError("failed to add status"); SJson *pVnodesJson = tjsonAddArrayToObject(pVgroupJson, "vnodes"); if (pVnodesJson == NULL) continue; @@ -352,8 +363,9 @@ static void monGenVgroupJson(SMonInfo *pMonitor) { SJson *pVnodeJson = tjsonCreateObject(); if (pVnodeJson == NULL) continue; - (void)tjsonAddDoubleToObject(pVnodeJson, "dnode_id", pVnodeDesc->dnode_id); - (void)tjsonAddStringToObject(pVnodeJson, "vnode_role", pVnodeDesc->vnode_role); + if (tjsonAddDoubleToObject(pVnodeJson, "dnode_id", pVnodeDesc->dnode_id) != 0) uError("failed to add dnode_id"); + if (tjsonAddStringToObject(pVnodeJson, "vnode_role", pVnodeDesc->vnode_role) != 0) + uError("failed to add vnode_role"); if (tjsonAddItemToArray(pVnodesJson, pVnodeJson) != 0) tjsonDelete(pVnodeJson); } @@ -376,8 +388,9 @@ static void monGenStbJson(SMonInfo *pMonitor) { } SMonStbDesc *pStbDesc = taosArrayGet(pInfo->stbs, i); - (void)tjsonAddStringToObject(pStbJson, "stb_name", pStbDesc->stb_name); - (void)tjsonAddStringToObject(pStbJson, "database_name", pStbDesc->database_name); + if (tjsonAddStringToObject(pStbJson, "stb_name", pStbDesc->stb_name) != 0) uError("failed to add stb_name"); + if (tjsonAddStringToObject(pStbJson, "database_name", pStbDesc->database_name) != 0) + uError("failed to add database_name"); } } @@ -392,9 +405,11 @@ static void monGenGrantJson(SMonInfo *pMonitor) { return; } - (void)tjsonAddDoubleToObject(pJson, "expire_time", pInfo->expire_time); - (void)tjsonAddDoubleToObject(pJson, "timeseries_used", pInfo->timeseries_used); - (void)tjsonAddDoubleToObject(pJson, "timeseries_total", pInfo->timeseries_total); + if (tjsonAddDoubleToObject(pJson, "expire_time", pInfo->expire_time) != 0) uError("failed to add expire_time"); + if (tjsonAddDoubleToObject(pJson, "timeseries_used", pInfo->timeseries_used) != 0) + uError("failed to add timeseries_used"); + if (tjsonAddDoubleToObject(pJson, "timeseries_total", pInfo->timeseries_total) != 0) + uError("failed to add timeseries_total"); } static void monGenDnodeJson(SMonInfo *pMonitor) { @@ -451,36 +466,40 @@ static void monGenDnodeJson(SMonInfo *pMonitor) { double io_read_disk_rate = io_read_disk / interval; double io_write_disk_rate = io_write_disk / interval; - (void)tjsonAddDoubleToObject(pJson, "uptime", pInfo->uptime); - (void)tjsonAddDoubleToObject(pJson, "cpu_engine", cpu_engine); - (void)tjsonAddDoubleToObject(pJson, "cpu_system", pSys->cpu_system); - (void)tjsonAddDoubleToObject(pJson, "cpu_cores", pSys->cpu_cores); - (void)tjsonAddDoubleToObject(pJson, "mem_engine", mem_engine); - (void)tjsonAddDoubleToObject(pJson, "mem_system", pSys->mem_system); - (void)tjsonAddDoubleToObject(pJson, "mem_total", pSys->mem_total); - (void)tjsonAddDoubleToObject(pJson, "disk_engine", pSys->disk_engine); - (void)tjsonAddDoubleToObject(pJson, "disk_used", pSys->disk_used); - (void)tjsonAddDoubleToObject(pJson, "disk_total", pSys->disk_total); - (void)tjsonAddDoubleToObject(pJson, "net_in", net_in_rate); - (void)tjsonAddDoubleToObject(pJson, "net_out", net_out_rate); - (void)tjsonAddDoubleToObject(pJson, "io_read", io_read_rate); - (void)tjsonAddDoubleToObject(pJson, "io_write", io_write_rate); - (void)tjsonAddDoubleToObject(pJson, "io_read_disk", io_read_disk_rate); - (void)tjsonAddDoubleToObject(pJson, "io_write_disk", io_write_disk_rate); - (void)tjsonAddDoubleToObject(pJson, "req_select", pStat->numOfSelectReqs); - (void)tjsonAddDoubleToObject(pJson, "req_select_rate", req_select_rate); - (void)tjsonAddDoubleToObject(pJson, "req_insert", pStat->numOfInsertReqs); - (void)tjsonAddDoubleToObject(pJson, "req_insert_success", pStat->numOfInsertSuccessReqs); - (void)tjsonAddDoubleToObject(pJson, "req_insert_rate", req_insert_rate); - (void)tjsonAddDoubleToObject(pJson, "req_insert_batch", pStat->numOfBatchInsertReqs); - (void)tjsonAddDoubleToObject(pJson, "req_insert_batch_success", pStat->numOfBatchInsertSuccessReqs); - (void)tjsonAddDoubleToObject(pJson, "req_insert_batch_rate", req_insert_batch_rate); - (void)tjsonAddDoubleToObject(pJson, "errors", pStat->errors); - (void)tjsonAddDoubleToObject(pJson, "vnodes_num", pStat->totalVnodes); - (void)tjsonAddDoubleToObject(pJson, "masters", pStat->masterNum); - (void)tjsonAddDoubleToObject(pJson, "has_mnode", pInfo->has_mnode); - (void)tjsonAddDoubleToObject(pJson, "has_qnode", pInfo->has_qnode); - (void)tjsonAddDoubleToObject(pJson, "has_snode", pInfo->has_snode); + if (tjsonAddDoubleToObject(pJson, "uptime", pInfo->uptime) != 0) uError("failed to add uptime"); + if (tjsonAddDoubleToObject(pJson, "cpu_engine", cpu_engine) != 0) uError("failed to add cpu_engine"); + if (tjsonAddDoubleToObject(pJson, "cpu_system", pSys->cpu_system) != 0) uError("failed to add cpu_system"); + if (tjsonAddDoubleToObject(pJson, "cpu_cores", pSys->cpu_cores) != 0) uError("failed to add cpu_cores"); + if (tjsonAddDoubleToObject(pJson, "mem_engine", mem_engine) != 0) uError("failed to add mem_engine"); + if (tjsonAddDoubleToObject(pJson, "mem_system", pSys->mem_system) != 0) uError("failed to add mem_system"); + if (tjsonAddDoubleToObject(pJson, "mem_total", pSys->mem_total) != 0) uError("failed to add mem_total"); + if (tjsonAddDoubleToObject(pJson, "disk_engine", pSys->disk_engine) != 0) uError("failed to add disk_engine"); + if (tjsonAddDoubleToObject(pJson, "disk_used", pSys->disk_used) != 0) uError("failed to add disk_used"); + if (tjsonAddDoubleToObject(pJson, "disk_total", pSys->disk_total) != 0) uError("failed to add disk_total"); + if (tjsonAddDoubleToObject(pJson, "net_in", net_in_rate) != 0) uError("failed to add net_in"); + if (tjsonAddDoubleToObject(pJson, "net_out", net_out_rate) != 0) uError("failed to add net_out"); + if (tjsonAddDoubleToObject(pJson, "io_read", io_read_rate) != 0) uError("failed to add io_read"); + if (tjsonAddDoubleToObject(pJson, "io_write", io_write_rate) != 0) uError("failed to add io_write"); + if (tjsonAddDoubleToObject(pJson, "io_read_disk", io_read_disk_rate) != 0) uError("failed to add io_read_disk"); + if (tjsonAddDoubleToObject(pJson, "io_write_disk", io_write_disk_rate) != 0) uError("failed to add io_write_disk"); + if (tjsonAddDoubleToObject(pJson, "req_select", pStat->numOfSelectReqs) != 0) uError("failed to add req_select"); + if (tjsonAddDoubleToObject(pJson, "req_select_rate", req_select_rate) != 0) uError("failed to add req_select_rate"); + if (tjsonAddDoubleToObject(pJson, "req_insert", pStat->numOfInsertReqs) != 0) uError("failed to add req_insert"); + if (tjsonAddDoubleToObject(pJson, "req_insert_success", pStat->numOfInsertSuccessReqs) != 0) + uError("failed to add req_insert_success"); + if (tjsonAddDoubleToObject(pJson, "req_insert_rate", req_insert_rate) != 0) uError("failed to add req_insert_rate"); + if (tjsonAddDoubleToObject(pJson, "req_insert_batch", pStat->numOfBatchInsertReqs) != 0) + uError("failed to add req_insert_batch"); + if (tjsonAddDoubleToObject(pJson, "req_insert_batch_success", pStat->numOfBatchInsertSuccessReqs) != 0) + uError("failed to add req_insert_batch_success"); + if (tjsonAddDoubleToObject(pJson, "req_insert_batch_rate", req_insert_batch_rate) != 0) + uError("failed to add req_insert_batch_rate"); + if (tjsonAddDoubleToObject(pJson, "errors", pStat->errors) != 0) uError("failed to add errors"); + if (tjsonAddDoubleToObject(pJson, "vnodes_num", pStat->totalVnodes) != 0) uError("failed to add vnodes_num"); + if (tjsonAddDoubleToObject(pJson, "masters", pStat->masterNum) != 0) uError("failed to add masters"); + if (tjsonAddDoubleToObject(pJson, "has_mnode", pInfo->has_mnode) != 0) uError("failed to add has_mnode"); + if (tjsonAddDoubleToObject(pJson, "has_qnode", pInfo->has_qnode) != 0) uError("failed to add has_qnode"); + if (tjsonAddDoubleToObject(pJson, "has_snode", pInfo->has_snode) != 0) uError("failed to add has_snode"); } static void monGenDiskJson(SMonInfo *pMonitor) { @@ -515,18 +534,18 @@ static void monGenDiskJson(SMonInfo *pMonitor) { SJson *pLogdirJson = tjsonCreateObject(); if (pLogdirJson == NULL) return; if (tjsonAddItemToObject(pJson, "logdir", pLogdirJson) != 0) return; - (void)tjsonAddStringToObject(pLogdirJson, "name", pLogDesc->name); - (void)tjsonAddDoubleToObject(pLogdirJson, "avail", pLogDesc->size.avail); - (void)tjsonAddDoubleToObject(pLogdirJson, "used", pLogDesc->size.used); - (void)tjsonAddDoubleToObject(pLogdirJson, "total", pLogDesc->size.total); + if (tjsonAddStringToObject(pLogdirJson, "name", pLogDesc->name) != 0) uError("failed to add string to json"); + if (tjsonAddDoubleToObject(pLogdirJson, "avail", pLogDesc->size.avail) != 0) uError("failed to add double to json"); + if (tjsonAddDoubleToObject(pLogdirJson, "used", pLogDesc->size.used) != 0) uError("failed to add double to json"); + if (tjsonAddDoubleToObject(pLogdirJson, "total", pLogDesc->size.total) != 0) uError("failed to add double to json"); SJson *pTempdirJson = tjsonCreateObject(); if (pTempdirJson == NULL) return; if (tjsonAddItemToObject(pJson, "tempdir", pTempdirJson) != 0) return; - (void)tjsonAddStringToObject(pTempdirJson, "name", pTempDesc->name); - (void)tjsonAddDoubleToObject(pTempdirJson, "avail", pTempDesc->size.avail); - (void)tjsonAddDoubleToObject(pTempdirJson, "used", pTempDesc->size.used); - (void)tjsonAddDoubleToObject(pTempdirJson, "total", pTempDesc->size.total); + if (tjsonAddStringToObject(pTempdirJson, "name", pTempDesc->name) != 0) uError("failed to add string to json"); + if (tjsonAddDoubleToObject(pTempdirJson, "avail", pTempDesc->size.avail) != 0) uError("failed to add double to json"); + if (tjsonAddDoubleToObject(pTempdirJson, "used", pTempDesc->size.used) != 0) uError("failed to add double to json"); + if (tjsonAddDoubleToObject(pTempdirJson, "total", pTempDesc->size.total) != 0) uError("failed to add double to json"); } static const char *monLogLevelStr(ELogLevel level) { @@ -571,26 +590,26 @@ static void monGenLogJson(SMonInfo *pMonitor) { SJson *pLogError = tjsonCreateObject(); if (pLogError == NULL) return; - (void)tjsonAddStringToObject(pLogError, "level", "error"); - (void)tjsonAddDoubleToObject(pLogError, "total", numOfErrorLogs); + if (tjsonAddStringToObject(pLogError, "level", "error") != 0) uError("failed to add string to json"); + if (tjsonAddDoubleToObject(pLogError, "total", numOfErrorLogs) != 0) uError("failed to add double to json"); if (tjsonAddItemToArray(pSummaryJson, pLogError) != 0) tjsonDelete(pLogError); SJson *pLogInfo = tjsonCreateObject(); if (pLogInfo == NULL) return; - (void)tjsonAddStringToObject(pLogInfo, "level", "info"); - (void)tjsonAddDoubleToObject(pLogInfo, "total", numOfInfoLogs); + if (tjsonAddStringToObject(pLogInfo, "level", "info") != 0) uError("failed to add string to json"); + if (tjsonAddDoubleToObject(pLogInfo, "total", numOfInfoLogs) != 0) uError("failed to add double to json"); if (tjsonAddItemToArray(pSummaryJson, pLogInfo) != 0) tjsonDelete(pLogInfo); SJson *pLogDebug = tjsonCreateObject(); if (pLogDebug == NULL) return; - (void)tjsonAddStringToObject(pLogDebug, "level", "debug"); - (void)tjsonAddDoubleToObject(pLogDebug, "total", numOfDebugLogs); + if (tjsonAddStringToObject(pLogDebug, "level", "debug") != 0) uError("failed to add string to json"); + if (tjsonAddDoubleToObject(pLogDebug, "total", numOfDebugLogs) != 0) uError("failed to add double to json"); if (tjsonAddItemToArray(pSummaryJson, pLogDebug) != 0) tjsonDelete(pLogDebug); SJson *pLogTrace = tjsonCreateObject(); if (pLogTrace == NULL) return; - (void)tjsonAddStringToObject(pLogTrace, "level", "trace"); - (void)tjsonAddDoubleToObject(pLogTrace, "total", numOfTraceLogs); + if (tjsonAddStringToObject(pLogTrace, "level", "trace") != 0) uError("failed to add string to json"); + if (tjsonAddDoubleToObject(pLogTrace, "total", numOfTraceLogs) != 0) uError("failed to add double to json"); if (tjsonAddItemToArray(pSummaryJson, pLogTrace) != 0) tjsonDelete(pLogTrace); } diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c index bfdbf92156..53d115fbe9 100644 --- a/source/libs/monitorfw/src/taos_collector_registry.c +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -66,14 +66,14 @@ taos_collector_registry_t *taos_collector_registry_new(const char *name) { return self; } -int taos_collector_registry_default_init(void) { - if (TAOS_COLLECTOR_REGISTRY_DEFAULT != NULL) return 0; +void taos_collector_registry_default_init(void) { + if (TAOS_COLLECTOR_REGISTRY_DEFAULT != NULL) return; TAOS_COLLECTOR_REGISTRY_DEFAULT = taos_collector_registry_new("default"); //if (TAOS_COLLECTOR_REGISTRY_DEFAULT) { // return taos_collector_registry_enable_process_metrics(TAOS_COLLECTOR_REGISTRY_DEFAULT); //} - return 1; + return; } int taos_collector_registry_destroy(taos_collector_registry_t *self) { diff --git a/source/libs/monitorfw/src/taos_metric_formatter_custom.c b/source/libs/monitorfw/src/taos_metric_formatter_custom.c index 6e7ded62bb..5c384c0421 100644 --- a/source/libs/monitorfw/src/taos_metric_formatter_custom.c +++ b/source/libs/monitorfw/src/taos_metric_formatter_custom.c @@ -40,16 +40,26 @@ int taos_metric_formatter_load_sample_new(taos_metric_formatter_t *self, taos_me int32_t len = end -start; char* keyvalues = taosMemoryMalloc(len); + if (keyvalues == NULL) return 1; memset(keyvalues, 0, len); memcpy(keyvalues, start + 1, len - 1); int32_t count = taos_monitor_count_occurrences(keyvalues, ","); char** keyvalue = taosMemoryMalloc(sizeof(char*) * (count + 1)); + if (keyvalue == NULL) { + taosMemoryFreeClear(keyvalues); + return 1; + } memset(keyvalue, 0, sizeof(char*) * (count + 1)); taos_monitor_split_str(keyvalue, keyvalues, ","); char** arr = taosMemoryMalloc(sizeof(char*) * (count + 1) * 2); + if (arr == NULL) { + taosMemoryFreeClear(keyvalue); + taosMemoryFreeClear(keyvalues); + return 1; + } memset(arr, 0, sizeof(char*) * (count + 1) * 2); bool isfound = true; @@ -165,6 +175,7 @@ int taos_metric_formatter_load_metric_new(taos_metric_formatter_t *self, taos_me int32_t size = strlen(metric->name); char* name = taosMemoryMalloc(size + 1); + if (name == NULL) return 1; memset(name, 0, size + 1); memcpy(name, metric->name, size); char* arr[2] = {0}; //arr[0] is table name, arr[1] is metric name diff --git a/source/libs/monitorfw/src/taos_monitor_util.c b/source/libs/monitorfw/src/taos_monitor_util.c index 06ae4993c5..b0eff27507 100644 --- a/source/libs/monitorfw/src/taos_monitor_util.c +++ b/source/libs/monitorfw/src/taos_monitor_util.c @@ -37,6 +37,7 @@ void taos_monitor_split_str(char** arr, char* str, const char* del) { void taos_monitor_split_str_metric(char** arr, taos_metric_t* metric, const char* del, char** buf) { int32_t size = strlen(metric->name); char* name = taosMemoryMalloc(size + 1); + if (name == NULL) return; memset(name, 0, size + 1); memcpy(name, metric->name, size); diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 3ea5ff6611..581c6222d2 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -513,8 +513,8 @@ static int32_t tlvDecodeValueCStr(STlvDecoder* pDecoder, char* pValue) { } static int32_t tlvDecodeCStrP(STlv* pTlv, char** pValue) { - *pValue = strndup(pTlv->value, pTlv->len); - return NULL == *pValue ? TSDB_CODE_OUT_OF_MEMORY : TSDB_CODE_SUCCESS; + *pValue = taosStrndup(pTlv->value, pTlv->len); + return NULL == *pValue ? terrno : TSDB_CODE_SUCCESS; } static int32_t tlvDecodeDynBinary(STlv* pTlv, void** pValue) { diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 927bc6b661..4f30cb12b7 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -204,19 +204,23 @@ static EDealRes walkExprs(SNodeList* pNodeList, ETraversalOrder order, FNodeWalk } void nodesWalkExpr(SNode* pNode, FNodeWalker walker, void* pContext) { - (void)walkExpr(pNode, TRAVERSAL_PREORDER, walker, pContext); + EDealRes res; + res = walkExpr(pNode, TRAVERSAL_PREORDER, walker, pContext); } void nodesWalkExprs(SNodeList* pNodeList, FNodeWalker walker, void* pContext) { - (void)walkExprs(pNodeList, TRAVERSAL_PREORDER, walker, pContext); + EDealRes res; + res = walkExprs(pNodeList, TRAVERSAL_PREORDER, walker, pContext); } void nodesWalkExprPostOrder(SNode* pNode, FNodeWalker walker, void* pContext) { - (void)walkExpr(pNode, TRAVERSAL_POSTORDER, walker, pContext); + EDealRes res; + res = walkExpr(pNode, TRAVERSAL_POSTORDER, walker, pContext); } void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) { - (void)walkExprs(pList, TRAVERSAL_POSTORDER, walker, pContext); + EDealRes res; + res = walkExprs(pList, TRAVERSAL_POSTORDER, walker, pContext); } static void checkParamIsFunc(SFunctionNode* pFunc) { @@ -382,7 +386,7 @@ static EDealRes rewriteExpr(SNode** pRawNode, ETraversalOrder order, FNodeRewrit res = rewriteExpr(&pWin->pEndOffset, order, rewriter, pContext); } break; - } + } case QUERY_NODE_COUNT_WINDOW: { SCountWindowNode* pEvent = (SCountWindowNode*)pNode; res = rewriteExpr(&pEvent->pCol, order, rewriter, pContext); diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index e52c8865c7..cd3095ede8 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -28,12 +28,12 @@ } \ } while (0) -#define CHECK_OUT_OF_MEM(p) \ - do { \ - if (NULL == (p)) { \ - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; \ - goto _err; \ - } \ +#define CHECK_OUT_OF_MEM(p) \ + do { \ + if (NULL == (p)) { \ + pCxt->errCode = terrno; \ + goto _err; \ + } \ } while (0) #define CHECK_PARSER_STATUS(pCxt) \ @@ -404,9 +404,9 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* SValueNode* val = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&val); CHECK_MAKE_NODE(val); - val->literal = strndup(pLiteral->z, pLiteral->n); + val->literal = taosStrndup(pLiteral->z, pLiteral->n); if(!val->literal) { - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + pCxt->errCode = terrno; nodesDestroyNode((SNode*)val); return NULL; } @@ -434,14 +434,22 @@ SNode* createRawValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToke goto _exit; } if (pLiteral) { - val->literal = strndup(pLiteral->z, pLiteral->n); + val->literal = taosStrndup(pLiteral->z, pLiteral->n); + if (!val->literal) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, terrno, "Out of memory"); + goto _exit; + } } else if (pNode) { SRawExprNode* pRawExpr = (SRawExprNode*)pNode; if (!nodesIsExprNode(pRawExpr->pNode)) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, pRawExpr->p); goto _exit; } - val->literal = strndup(pRawExpr->p, pRawExpr->n); + val->literal = taosStrndup(pRawExpr->p, pRawExpr->n); + if (!val->literal) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, terrno, "Out of memory"); + goto _exit; + } } else { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTERNAL_ERROR, "Invalid parameters"); goto _exit; @@ -479,8 +487,8 @@ SNode* createRawValueNodeExt(SAstCreateContext* pCxt, int32_t dataType, const ST goto _exit; } if (pLiteral) { - if (!(val->literal = strndup(pLiteral->z, pLiteral->n))) { - pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_OUT_OF_MEMORY, "Out of memory"); + if (!(val->literal = taosStrndup(pLiteral->z, pLiteral->n))) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, terrno, "Out of memory"); goto _exit; } } else { @@ -579,7 +587,7 @@ SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral) { return NULL; } SNodeList* pHintList = NULL; - char* hint = strndup(pLiteral->z + 3, pLiteral->n - 5); + char* hint = taosStrndup(pLiteral->z + 3, pLiteral->n - 5); if (!hint) return NULL; int32_t i = 0; bool quit = false; @@ -749,13 +757,13 @@ SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral) return NULL; } } - val->literal = strndup(pLiteral->z + 1, pLiteral->n - 2); + val->literal = taosStrndup(pLiteral->z + 1, pLiteral->n - 2); } else { - val->literal = strndup(pLiteral->z, pLiteral->n); + val->literal = taosStrndup(pLiteral->z, pLiteral->n); } if (!val->literal) { nodesDestroyNode((SNode*)val); - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + pCxt->errCode = terrno; return NULL; } val->flag |= VALUE_FLAG_IS_DURATION; @@ -801,13 +809,13 @@ SNode* createTimeOffsetValueNode(SAstCreateContext* pCxt, const SToken* pLiteral return NULL; } } - val->literal = strndup(pLiteral->z + 1, pLiteral->n - 2); + val->literal = taosStrndup(pLiteral->z + 1, pLiteral->n - 2); } else { - val->literal = strndup(pLiteral->z, pLiteral->n); + val->literal = taosStrndup(pLiteral->z, pLiteral->n); } if (!val->literal) { nodesDestroyNode((SNode*)val); - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + pCxt->errCode = terrno; return NULL; } val->flag |= VALUE_FLAG_IS_TIME_OFFSET; @@ -853,9 +861,9 @@ SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLitera SValueNode* val = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&val); CHECK_MAKE_NODE(val); - val->literal = strndup(pLiteral->z, pLiteral->n); + val->literal = taosStrndup(pLiteral->z, pLiteral->n); if (!val->literal) { - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + pCxt->errCode = terrno; nodesDestroyNode((SNode*)val); return NULL; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index f4dd91f392..10d9b19e7f 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -47,6 +47,7 @@ int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) { SAstCreateContext cxt; initAstCreateContext(pParseCxt, &cxt); void* pParser = ParseAlloc((FMalloc)taosMemoryMalloc); + if (!pParser) return terrno; int32_t i = 0; while (1) { SToken t0 = {0}; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index d6841fb492..1c26a7c70e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1079,9 +1079,9 @@ static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifyOpStmt* return buildSyntaxErrMsg(&pCxt->msg, "comment too long", token.z); } int32_t len = trimString(token.z, token.n, pCxt->tmpTokenBuf, TSDB_TB_COMMENT_LEN); - pStmt->pCreateTblReq->comment = strndup(pCxt->tmpTokenBuf, len); + pStmt->pCreateTblReq->comment = taosStrndup(pCxt->tmpTokenBuf, len); if (NULL == pStmt->pCreateTblReq->comment) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pStmt->pCreateTblReq->commentLen = len; } else { diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 06b9c2d9ef..500d0e213e 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -46,7 +46,7 @@ int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData) { } pNew->aCol = taosArrayDup(pDataBlock->pData->aCol, NULL); if (!pNew->aCol) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; taosMemoryFreeClear(*pData); return code; } @@ -951,7 +951,7 @@ int32_t qResetStmtColumns(SArray* pCols, bool deepClear) { SColData* pCol = (SColData*)taosArrayGet(pCols, i); if (pCol == NULL) { qError("qResetStmtColumns column is NULL"); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (deepClear) { tColDataDeepClear(pCol); @@ -971,7 +971,7 @@ int32_t qResetStmtDataBlock(STableDataCxt* block, bool deepClear) { SColData* pCol = (SColData*)taosArrayGet(pBlock->pData->aCol, i); if (pCol == NULL) { qError("qResetStmtDataBlock column is NULL"); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (deepClear) { tColDataDeepClear(pCol); @@ -1033,7 +1033,7 @@ int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool rese pNewTb->aCol = taosArrayDup(pCxt->pData->aCol, NULL); if (NULL == pNewTb->aCol) { insDestroyTableDataCxt(*pDst); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pNewCxt->pData = pNewTb; diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 05373aefb3..838f797394 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -856,7 +856,7 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, code = buildSubmitReq(src->vgId, src->pData, &dst->pData, &dst->size); } if (TSDB_CODE_SUCCESS == code) { - code = (NULL == taosArrayPush(pDataBlocks, &dst) ? TSDB_CODE_OUT_OF_MEMORY : TSDB_CODE_SUCCESS); + code = (NULL == taosArrayPush(pDataBlocks, &dst) ? terrno : TSDB_CODE_SUCCESS); } } @@ -886,17 +886,32 @@ static bool findFileds(SSchema* pSchema, TAOS_FIELD* fields, int numFields) { return false; } -int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD* tFields, +int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD* tFields, int numFields, bool needChangeLength, char* errstr, int32_t errstrLen) { + int ret = 0; if(data == NULL) { uError("rawBlockBindData, data is NULL"); return TSDB_CODE_APP_ERROR; } void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); + SVCreateTbReq *pCreateReqTmp = NULL; + if (tmp == NULL && pCreateTb != NULL){ + ret = cloneSVreateTbReq(pCreateTb, &pCreateReqTmp); + if (ret != TSDB_CODE_SUCCESS){ + uError("cloneSVreateTbReq error"); + goto end; + } + } + STableDataCxt* pTableCxt = NULL; - int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false); + ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, + sizeof(pTableMeta->uid), pTableMeta, &pCreateReqTmp, &pTableCxt, true, false); + if (pCreateReqTmp != NULL) { + tdDestroySVCreateTbReq(pCreateReqTmp); + taosMemoryFree(pCreateReqTmp); + } + if (ret != TSDB_CODE_SUCCESS) { uError("insGetTableDataCxt error"); goto end; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index fe3e9b10da..3ae4583013 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -522,7 +522,7 @@ static int32_t rewriteDropTableWithMetaCache(STranslateContext* pCxt) { if (!pMetaCache->pTableMeta && !(pMetaCache->pTableMeta = taosHashInit(tbMetaExSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK))) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } SMetaRes** ppMetaRes = NULL; @@ -2087,7 +2087,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, pVal->datum.p = taosMemoryCalloc(1, size + VARSTR_HEADER_SIZE); if (NULL == pVal->datum.p) { if (isHexChar) taosMemoryFree(data); - return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); + return generateDealNodeErrMsg(pCxt, terrno); } varDataSetLen(pVal->datum.p, size); memcpy(varDataVal(pVal->datum.p), data, size); @@ -2103,7 +2103,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, int32_t len = TMIN(targetDt.bytes - VARSTR_HEADER_SIZE, vlen); pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1); if (NULL == pVal->datum.p) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); + return generateDealNodeErrMsg(pCxt, terrno); } varDataSetLen(pVal->datum.p, len); strncpy(varDataVal(pVal->datum.p), pVal->literal, len); @@ -2119,7 +2119,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, case TSDB_DATA_TYPE_NCHAR: { pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + 1); if (NULL == pVal->datum.p) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); + return generateDealNodeErrMsg(pCxt, terrno); } int32_t len = 0; @@ -6264,7 +6264,7 @@ static int32_t unionEqualCondTbnamesOfSameTable(SArray* aTableTbnames, SEqCondTb SEqCondTbNameTableInfo* info = taosArrayGet(aTableTbnames, i); if (info->pRealTable == pInfo->pRealTable) { if (NULL == taosArrayAddAll(info->aTbnames, pInfo->aTbnames)) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; break; } taosArrayDestroy(pInfo->aTbnames); @@ -8725,10 +8725,10 @@ static int32_t makeIntervalVal(SRetention* pRetension, int8_t precision, SNode** } char buf[20] = {0}; int32_t len = snprintf(buf, sizeof(buf), "%" PRId64 "%c", timeVal, pRetension->freqUnit); - pVal->literal = strndup(buf, len); + pVal->literal = taosStrndup(buf, len); if (NULL == pVal->literal) { nodesDestroyNode((SNode*)pVal); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pVal->flag |= VALUE_FLAG_IS_DURATION; pVal->node.resType.type = TSDB_DATA_TYPE_BIGINT; @@ -14705,7 +14705,7 @@ static int32_t rewriteDropSuperTablewithOpt(STranslateContext* pCxt, SQuery* pQu break; } if (!isdigit(pStmt->tableName[i])) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, "Table does not exist: `%s`.`%s`", + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, "STable not exist: `%s`.`%s`", pStmt->dbName, pStmt->tableName); } } @@ -14715,8 +14715,11 @@ static int32_t rewriteDropSuperTablewithOpt(STranslateContext* pCxt, SQuery* pQu toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); code = getTargetName(pCxt, &name, pTableName); if (TSDB_CODE_SUCCESS != code) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "%s: db:`%s`, tbuid:`%s`", tstrerror(code), pStmt->dbName, - pStmt->tableName); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "%s: db:`%s`, tbuid:`%s`", + (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_TDB_TABLE_NOT_EXIST) + ? "STable not exist" + : tstrerror(code), + pStmt->dbName, pStmt->tableName); } tstrncpy(pStmt->tableName, pTableName, TSDB_TABLE_NAME_LEN); // rewrite table uid to table name diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 1b90144a2c..1ce8b04324 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -1124,7 +1124,7 @@ int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, if (TSDB_CODE_SUCCESS == code && NULL != pVgList) { *pVgInfo = taosArrayDup(pVgList, NULL); if (NULL == *pVgInfo) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; } } return code; @@ -1439,7 +1439,7 @@ int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes) { *pDnodes = taosArrayDup((SArray*)pRes->pRes, NULL); if (NULL == *pDnodes) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 43159bce20..c3aa95f5b7 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -37,7 +37,7 @@ static int32_t dumpQueryPlan(SQueryPlan* pPlan) { char* pStr = NULL; code = nodesNodeToString((SNode*)pPlan, false, &pStr, NULL); if (TSDB_CODE_SUCCESS == code) { - planDebugL("qid:0x%" PRIx64 " Query Plan, JsonPlan: %s", pPlan->queryId, pStr); + planDebugL("QID:0x%" PRIx64 " Query Plan, JsonPlan: %s", pPlan->queryId, pStr); taosMemoryFree(pStr); } return code; @@ -123,7 +123,7 @@ int32_t qContinuePlanPostQuery(void* pPostPlan) { } int32_t qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstreamSourceNode* pSource) { - planDebug("qid:0x%" PRIx64 " set subplan execution node, groupId:%d", subplan->id.queryId, groupId); + planDebug("QID:0x%" PRIx64 " set subplan execution node, groupId:%d", subplan->id.queryId, groupId); return setSubplanExecutionNode(subplan->pNode, groupId, pSource); } @@ -143,7 +143,7 @@ static void clearSubplanExecutionNode(SPhysiNode* pNode) { } void qClearSubplanExecutionNode(SSubplan* pSubplan) { - planDebug("qid:0x%" PRIx64 " clear subplan execution node, groupId:%d", pSubplan->id.queryId, pSubplan->id.groupId); + planDebug("QID:0x%" PRIx64 " clear subplan execution node, groupId:%d", pSubplan->id.queryId, pSubplan->id.groupId); clearSubplanExecutionNode(pSubplan->pNode); } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index e08868d7c1..7180c58404 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1428,6 +1428,7 @@ void qWorkerDestroy(void **qWorkerMgmt) { return; } + qInfo("wait for destroyed"); while (0 == destroyed) { taosMsleep(2); } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 4f08c93c1e..2a4951d237 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -628,8 +628,8 @@ int32_t sclWalkCaseWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell cell = cell->pNext) { pWhenThen = (SWhenThenNode *)node; - SCL_ERR_RET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); - SCL_ERR_RET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); SCL_ERR_JRET(vectorCompareImpl(pCase, pWhen, pComp, rowIdx, 1, TSDB_ORDER_ASC, OP_TYPE_EQUAL)); @@ -646,6 +646,10 @@ int32_t sclWalkCaseWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell goto _return; } + sclFreeParam(pWhen); + sclFreeParam(pThen); + taosMemoryFreeClear(pWhen); + taosMemoryFreeClear(pThen); } if (pElse) { @@ -672,8 +676,8 @@ _return: sclFreeParam(pWhen); sclFreeParam(pThen); - taosMemoryFree(pWhen); - taosMemoryFree(pThen); + taosMemoryFreeClear(pWhen); + taosMemoryFreeClear(pThen); SCL_RET(code); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 1129b9c28b..377009a07f 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -4352,16 +4352,16 @@ int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP int32_t numOfBins = 0; int32_t totalCount = 0; - char *binTypeStr = strndup(varDataVal(pInput[1].columnData->pData), varDataLen(pInput[1].columnData->pData)); + char *binTypeStr = taosStrndup(varDataVal(pInput[1].columnData->pData), varDataLen(pInput[1].columnData->pData)); if (NULL == binTypeStr) { - SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCL_ERR_RET(terrno); } int8_t binType = getHistogramBinType(binTypeStr); taosMemoryFree(binTypeStr); - char *binDesc = strndup(varDataVal(pInput[2].columnData->pData), varDataLen(pInput[2].columnData->pData)); + char *binDesc = taosStrndup(varDataVal(pInput[2].columnData->pData), varDataLen(pInput[2].columnData->pData)); if (NULL == binDesc) { - SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCL_ERR_RET(terrno); } int64_t normalized = *(int64_t *)(pInput[3].columnData->pData); diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 0f3d1bfa81..8a156e8a06 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -61,6 +61,7 @@ typedef enum { #define SCH_MAX_TASK_TIMEOUT_USEC 300000000 #define SCH_DEFAULT_MAX_RETRY_NUM 6 #define SCH_MIN_AYSNC_EXEC_NUM 3 +#define SCH_DEFAULT_RETRY_TOTAL_ROUND 3 typedef struct SSchDebug { bool lockEnable; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index f475c974cc..03145da939 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -728,7 +728,7 @@ void schFreeJobImpl(void *job) { uint64_t queryId = pJob->queryId; int64_t refId = pJob->refId; - qDebug("qid:0x%" PRIx64 " begin to free sch job, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); + qDebug("QID:0x%" PRIx64 " begin to free sch job, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); schDropJobAllTasks(pJob); @@ -775,7 +775,7 @@ void schFreeJobImpl(void *job) { taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->fetchRes); taosMemoryFreeClear(pJob->sql); - int32_t code = tsem_destroy(&pJob->rspSem); + int32_t code = tsem_destroy(&pJob->rspSem); if (code) { qError("tsem_destroy failed, error:%s", tstrerror(code)); } @@ -786,7 +786,7 @@ void schFreeJobImpl(void *job) { schCloseJobRef(); } - qDebug("qid:0x%" PRIx64 " sch job freed, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); + qDebug("QID:0x%" PRIx64 " sch job freed, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); } int32_t schJobFetchRows(SSchJob *pJob) { @@ -797,7 +797,7 @@ int32_t schJobFetchRows(SSchJob *pJob) { if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - code = tsem_wait(&pJob->rspSem); + code = tsem_wait(&pJob->rspSem); if (code) { qError("tsem_wait for fetch rspSem failed, error:%s", tstrerror(code)); SCH_RET(code); @@ -821,7 +821,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { int64_t refId = -1; SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { - qError("qid:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); + qError("QID:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); SCH_ERR_JRET(terrno); } @@ -831,7 +831,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { if (pReq->sql) { pJob->sql = taosStrdup(pReq->sql); if (NULL == pJob->sql) { - qError("qid:0x%" PRIx64 " strdup sql %s failed", pReq->pDag->queryId, pReq->sql); + qError("QID:0x%" PRIx64 " strdup sql %s failed", pReq->pDag->queryId, pReq->sql); SCH_ERR_JRET(terrno); } } @@ -839,7 +839,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { if (pReq->allocatorRefId > 0) { pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId); if (pJob->allocatorRefId <= 0) { - qError("qid:0x%" PRIx64 " nodesMakeAllocatorWeakRef failed", pReq->pDag->queryId); + qError("QID:0x%" PRIx64 " nodesMakeAllocatorWeakRef failed", pReq->pDag->queryId); SCH_ERR_JRET(terrno); } } @@ -851,11 +851,11 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->pWorkerCb = pReq->pWorkerCb; if (pReq->pNodeList == NULL || taosArrayGetSize(pReq->pNodeList) <= 0) { - qDebug("qid:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); + qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); } else { pJob->nodeList = taosArrayDup(pReq->pNodeList, NULL); if (NULL == pJob->nodeList) { - qError("qid:0x%" PRIx64 " taosArrayDup failed, origNum:%d", pReq->pDag->queryId, + qError("QID:0x%" PRIx64 " taosArrayDup failed, origNum:%d", pReq->pDag->queryId, (int32_t)taosArrayGetSize(pReq->pNodeList)); SCH_ERR_JRET(terrno); } @@ -918,7 +918,7 @@ _return: int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) { int32_t code = 0; - qDebug("qid:0x%" PRIx64 " sch job refId 0x%" PRIx64 " started", pReq->pDag->queryId, pJob->refId); + qDebug("QID:0x%" PRIx64 " sch job refId 0x%" PRIx64 " started", pReq->pDag->queryId, pJob->refId); SCH_ERR_RET(schLaunchJob(pJob)); @@ -926,7 +926,7 @@ int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); code = tsem_wait(&pJob->rspSem); if (code) { - qError("qid:0x%" PRIx64 " tsem_wait sync rspSem failed, error:%s", pReq->pDag->queryId, tstrerror(code)); + qError("QID:0x%" PRIx64 " tsem_wait sync rspSem failed, error:%s", pReq->pDag->queryId, tstrerror(code)); SCH_ERR_RET(code); } } @@ -1191,7 +1191,7 @@ int32_t schProcessOnCbBegin(SSchJob **job, SSchTask **task, uint64_t qId, int64_ (void)schAcquireJob(rId, &pJob); if (NULL == pJob) { - qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, qId, tId, rId); + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, qId, tId, rId); SCH_ERR_RET(TSDB_CODE_QRY_JOB_NOT_EXIST); } diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index fe601c6b86..b3106d8c7e 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -500,7 +500,7 @@ _return: int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); // called if drop task rsp received code (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error @@ -513,7 +513,7 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, pParam->taskId, + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); if (pMsg) { taosMemoryFree(pMsg->pData); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index a0275d9c21..4c609fa5e2 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -366,9 +366,11 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet, if (SCH_IS_DATA_BIND_TASK(pTask)) { if (pEpSet) { pCtx->roundTotal = pEpSet->numOfEps; - } else { + } else if (pTask->candidateAddrs && taosArrayGetSize(pTask->candidateAddrs) > 0) { SQueryNodeAddr *pAddr = taosArrayGet(pTask->candidateAddrs, 0); pCtx->roundTotal = pAddr->epSet.numOfEps; + } else { + pCtx->roundTotal = SCH_DEFAULT_RETRY_TOTAL_ROUND; } } else { pCtx->roundTotal = 1; @@ -994,7 +996,7 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) { int32_t code = 0; - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId, + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId, pStatus->execId, jobTaskStatusStr(pStatus->status)); if (schProcessOnCbBegin(&pJob, &pTask, pStatus->queryId, pStatus->refId, pStatus->taskId)) { @@ -1041,12 +1043,12 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { continue; } - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); pJob = NULL; (void)schAcquireJob(localRsp->rId, &pJob); if (NULL == pJob) { - qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, localRsp->tId, localRsp->rId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_NOT_EXIST); } @@ -1066,7 +1068,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { (void)schReleaseJob(pJob->refId); - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, localRsp->tId, code); SCH_ERR_JRET(code); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index a5c5c1b775..94c196d280 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -164,7 +164,6 @@ extern void* streamTimer; extern int32_t streamBackendId; extern int32_t streamBackendCfWrapperId; extern int32_t taskDbWrapperId; -extern int32_t streamMetaId; int32_t streamTimerInit(); void streamTimerCleanUp(); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 69ccea8e29..a2c9012df5 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -427,7 +427,9 @@ void cleanDir(const char* pPath, const char* id) { if (taosIsDir(pPath)) { taosRemoveDir(pPath); - (void)taosMkDir(pPath); + if (taosMkDir(pPath) != 0) { + stError("%s failed to create dir:%s", id, pPath); + } stInfo("%s clear dir:%s, succ", id, pPath); } } @@ -533,7 +535,7 @@ int32_t rebuildFromRemoteChkp_s3(const char* key, char* chkpPath, int64_t chkpId _EXIT: if (code != 0) { if (rename) { - (void)taosRenameFile(defaultTmp, defaultPath); + TAOS_UNUSED(taosRenameFile(defaultTmp, defaultPath)); } } @@ -653,13 +655,13 @@ int32_t backendFileCopyFilesImpl(const char* src, const char* dst) { taosMemoryFreeClear(srcName); taosMemoryFreeClear(dstName); - (void)taosCloseDir(&pDir); + TAOS_UNUSED(taosCloseDir(&pDir)); return code; _ERROR: taosMemoryFreeClear(srcName); taosMemoryFreeClear(dstName); - (void)taosCloseDir(&pDir); + TAOS_UNUSED(taosCloseDir(&pDir)); return code; } @@ -833,8 +835,11 @@ int32_t streamBackendInit(const char* streamPath, int64_t chkpId, int32_t vgId, pHandle->list = tdListNew(sizeof(SCfComparator)); TSDB_CHECK_NULL(pHandle->list, code, lino, _EXIT, terrno); - (void)taosThreadMutexInit(&pHandle->mutex, NULL); - (void)taosThreadMutexInit(&pHandle->cfMutex, NULL); + code = taosThreadMutexInit(&pHandle->mutex, NULL); + TSDB_CHECK_CODE(code, lino, _EXIT); + + code = taosThreadMutexInit(&pHandle->cfMutex, NULL); + TSDB_CHECK_CODE(code, lino, _EXIT); pHandle->cfInst = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); TSDB_CHECK_NULL(pHandle->cfInst, code, lino, _EXIT, terrno); @@ -948,11 +953,11 @@ void streamBackendCleanup(void* arg) { void streamBackendHandleCleanup(void* arg) { SBackendCfWrapper* wrapper = arg; bool remove = wrapper->remove; - (void)taosThreadRwlockWrlock(&wrapper->rwLock); + TAOS_UNUSED(taosThreadRwlockWrlock(&wrapper->rwLock)); stDebug("start to do-close backendwrapper %p, %s", wrapper, wrapper->idstr); if (wrapper->rocksdb == NULL) { - (void)taosThreadRwlockUnlock(&wrapper->rwLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&wrapper->rwLock)); return; } @@ -1003,9 +1008,9 @@ void streamBackendHandleCleanup(void* arg) { wrapper->readOpts = NULL; taosMemoryFreeClear(wrapper->cfOpts); taosMemoryFreeClear(wrapper->param); - (void)taosThreadRwlockUnlock(&wrapper->rwLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&wrapper->rwLock)); - (void)taosThreadRwlockDestroy(&wrapper->rwLock); + TAOS_UNUSED(taosThreadRwlockDestroy(&wrapper->rwLock)); wrapper->rocksdb = NULL; // taosReleaseRef(streamBackendId, wrapper->backendId); @@ -1099,7 +1104,7 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { int32_t code = 0; STaskDbWrapper* pBackend = arg; SArray * chkpDel = NULL, *chkpDup = NULL; - (void)taosThreadRwlockWrlock(&pBackend->chkpDirLock); + TAOS_UNUSED(taosThreadRwlockWrlock(&pBackend->chkpDirLock)); if (taosArrayPush(pBackend->chkpSaved, &chkpId) == NULL) { TAOS_CHECK_GOTO(terrno, NULL, _exception); @@ -1152,7 +1157,7 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { taosArrayDestroy(pBackend->chkpSaved); pBackend->chkpSaved = chkpDup; - (void)taosThreadRwlockUnlock(&pBackend->chkpDirLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&pBackend->chkpDirLock)); for (int i = 0; i < taosArrayGetSize(chkpDel); i++) { int64_t id = *(int64_t*)taosArrayGet(chkpDel, i); @@ -1169,110 +1174,10 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { _exception: taosArrayDestroy(chkpDup); taosArrayDestroy(chkpDel); - (void)taosThreadRwlockUnlock(&pBackend->chkpDirLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&pBackend->chkpDirLock)); return code; } -#ifdef BUILD_NO_CALL -static int32_t chkpIdComp(const void* a, const void* b) { - int64_t x = *(int64_t*)a; - int64_t y = *(int64_t*)b; - return x < y ? -1 : 1; -} - -int32_t streamBackendLoadCheckpointInfo(void* arg) { - SStreamMeta* pMeta = arg; - int32_t code = 0; - SArray* suffix = NULL; - - int32_t len = strlen(pMeta->path) + 30; - char* chkpPath = taosMemoryCalloc(1, len); - sprintf(chkpPath, "%s%s%s", pMeta->path, TD_DIRSEP, "checkpoints"); - - if (!taosDirExist(chkpPath)) { - // no checkpoint, nothing to load - taosMemoryFree(chkpPath); - return 0; - } - - TdDirPtr pDir = taosOpenDir(chkpPath); - if (pDir == NULL) { - taosMemoryFree(chkpPath); - return 0; - } - - TdDirEntryPtr de = NULL; - suffix = taosArrayInit(4, sizeof(int64_t)); - - while ((de = taosReadDir(pDir)) != NULL) { - if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; - - if (taosDirEntryIsDir(de)) { - char checkpointPrefix[32] = {0}; - int64_t checkpointId = 0; - - int ret = sscanf(taosGetDirEntryName(de), "checkpoint%" PRId64 "", &checkpointId); - if (ret == 1) { - taosArrayPush(suffix, &checkpointId); - } - } else { - continue; - } - } - taosArraySort(suffix, chkpIdComp); - // free previous chkpSaved - taosArrayClear(pMeta->chkpSaved); - for (int i = 0; i < taosArrayGetSize(suffix); i++) { - int64_t id = *(int64_t*)taosArrayGet(suffix, i); - taosArrayPush(pMeta->chkpSaved, &id); - } - - taosArrayDestroy(suffix); - taosCloseDir(&pDir); - taosMemoryFree(chkpPath); - return 0; -} -#endif - -#ifdef BUILD_NO_CALL -int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t*** ppHandle, SArray* refs) { - return 0; - // SArray* pHandle = taosArrayInit(16, POINTER_BYTES); - // void* pIter = taosHashIterate(pMeta->pTaskDbUnique, NULL); - // while (pIter) { - // int64_t id = *(int64_t*)pIter; - - // SBackendCfWrapper* wrapper = taosAcquireRef(streamBackendCfWrapperId, id); - // if (wrapper == NULL) { - // pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter); - // continue; - // } - - // taosThreadRwlockRdlock(&wrapper->rwLock); - // for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { - // if (wrapper->pHandle[i]) { - // rocksdb_column_family_handle_t* p = wrapper->pHandle[i]; - // taosArrayPush(pHandle, &p); - // } - // } - // taosThreadRwlockUnlock(&wrapper->rwLock); - - // taosArrayPush(refs, &id); - // } - - // int32_t nCf = taosArrayGetSize(pHandle); - - // rocksdb_column_family_handle_t** ppCf = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); - // for (int i = 0; i < nCf; i++) { - // ppCf[i] = taosArrayGetP(pHandle, i); - // } - // taosArrayDestroy(pHandle); - - // *ppHandle = ppCf; - // return nCf; -} -#endif - int chkpIdComp(const void* a, const void* b) { int64_t x = *(int64_t*)a; int64_t y = *(int64_t*)b; @@ -1283,6 +1188,9 @@ int chkpIdComp(const void* a, const void* b) { int32_t taskDbLoadChkpInfo(STaskDbWrapper* pBackend) { int32_t code = 0; char* pChkpDir = taosMemoryCalloc(1, 256); + if (pChkpDir == NULL) { + return terrno; + } sprintf(pChkpDir, "%s%s%s", pBackend->path, TD_DIRSEP, "checkpoints"); if (!taosIsDir(pChkpDir)) { @@ -1315,12 +1223,12 @@ int32_t taskDbLoadChkpInfo(STaskDbWrapper* pBackend) { taosArraySort(pBackend->chkpSaved, chkpIdComp); taosMemoryFree(pChkpDir); - (void)taosCloseDir(&pDir); + TAOS_UNUSED(taosCloseDir(&pDir)); return 0; _exception: taosMemoryFree(pChkpDir); - (void)taosCloseDir(&pDir); + TAOS_UNUSED(taosCloseDir(&pDir)); return code; } int32_t chkpGetAllDbCfHandle2(STaskDbWrapper* pBackend, rocksdb_column_family_handle_t*** ppHandle) { @@ -1561,7 +1469,7 @@ void* taskAcquireDb(int64_t refId) { } void taskReleaseDb(int64_t refId) { // release - (void)taosReleaseRef(taskDbWrapperId, refId); + TAOS_UNUSED(taosReleaseRef(taskDbWrapperId, refId)); } int64_t taskGetDBRef(void* arg) { @@ -1622,7 +1530,7 @@ int32_t chkpLoadExtraInfo(char* pChkpIdDir, int64_t* chkpId, int64_t* processId) code = 0; _EXIT: taosMemoryFree(pDst); - (void)taosCloseFile(&pFile); + TAOS_UNUSED(taosCloseFile(&pFile)); return code; } int32_t chkpAddExtraInfo(char* pChkpIdDir, int64_t chkpId, int64_t processId) { @@ -1677,7 +1585,7 @@ int32_t chkpAddExtraInfo(char* pChkpIdDir, int64_t chkpId, int64_t processId) { code = 0; _EXIT: - (void)taosCloseFile(&pFile); + TAOS_UNUSED(taosCloseFile(&pFile)); taosMemoryFree(pDst); return code; } @@ -1735,7 +1643,7 @@ int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId, int64_t processId) { goto _EXIT; } - (void)atomic_store_64(&pTaskDb->dataWritten, 0); + TAOS_UNUSED(atomic_store_64(&pTaskDb->dataWritten, 0)); pTaskDb->chkpId = chkpId; _EXIT: @@ -1743,13 +1651,13 @@ _EXIT: // clear checkpoint dir if failed if (code != 0 && pChkpDir != NULL) { if (taosDirExist(pChkpIdDir)) { - (void)taosRemoveDir(pChkpIdDir); + TAOS_UNUSED(taosRemoveDir(pChkpIdDir)); } } taosMemoryFree(pChkpIdDir); taosMemoryFree(pChkpDir); - (void)taosReleaseRef(taskDbWrapperId, refId); + TAOS_UNUSED(taosReleaseRef(taskDbWrapperId, refId)); taosMemoryFree(ppCf); return code; } @@ -1822,7 +1730,7 @@ int defaultKeyComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, } int streamStateValueIsStale(char* v) { int64_t ts = 0; - (void)taosDecodeFixedI64(v, &ts); + TAOS_UNUSED(taosDecodeFixedI64(v, &ts)); return (ts != 0 && ts < taosGetTimestampMs()) ? 1 : 0; } int iterValueIsStale(rocksdb_iterator_t* iter) { @@ -1865,8 +1773,8 @@ int stateKeyDBComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, p1 = taosDecodeFixedI64(p1, &key1.key.ts); p2 = taosDecodeFixedI64(p2, &key2.key.ts); - (void)taosDecodeFixedI64(p1, &key1.opNum); - (void)taosDecodeFixedI64(p2, &key2.opNum); + TAOS_UNUSED(taosDecodeFixedI64(p1, &key1.opNum)); + TAOS_UNUSED(taosDecodeFixedI64(p2, &key2.opNum)); return stateKeyCmpr(&key1, sizeof(key1), &key2, sizeof(key2)); } @@ -2062,8 +1970,8 @@ int parKeyDBComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, s char* p1 = (char*)aBuf; char* p2 = (char*)bBuf; - (void)taosDecodeFixedI64(p1, &w1); - (void)taosDecodeFixedI64(p2, &w2); + TAOS_UNUSED(taosDecodeFixedI64(p1, &w1)); + TAOS_UNUSED(taosDecodeFixedI64(p2, &w2)); if (w1 == w2) { return 0; } else { @@ -2107,7 +2015,7 @@ int32_t valueIsStale(void* k, int64_t ts) { } void destroyCompare(void* arg) { - (void)arg; + TAOS_UNUSED(arg); return; } @@ -2190,7 +2098,7 @@ int32_t valueDecode(void* value, int32_t vlen, int64_t* ttl, char** dest) { // compatiable with previous data p = taosDecodeBinary(p, (void**)&pOutput, key.len); if (p == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _EXCEPT; } @@ -2205,7 +2113,7 @@ int32_t valueDecode(void* value, int32_t vlen, int64_t* ttl, char** dest) { if (key.compress == 1) { p = taosDecodeBinary(p, (void**)&pCompressData, key.len); if (p == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _EXCEPT; } pOutput = taosMemoryCalloc(1, key.rawLen); @@ -2224,7 +2132,7 @@ int32_t valueDecode(void* value, int32_t vlen, int64_t* ttl, char** dest) { } else { p = taosDecodeBinary(p, (void**)&pOutput, key.len); if (p == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _EXCEPT; } } @@ -2251,31 +2159,28 @@ _EXCEPT: } const char* compareDefaultName(void* arg) { - (void)arg; + TAOS_UNUSED(arg); return ginitDict[0].key; } const char* compareStateName(void* arg) { - (void)arg; + TAOS_UNUSED(arg); return ginitDict[1].key; } -const char* compareWinKeyName(void* arg) { - (void)arg; - return ginitDict[2].key; -} +const char* compareWinKeyName(void* arg) { return ginitDict[2].key; } const char* compareSessionKeyName(void* arg) { - (void)arg; + TAOS_UNUSED(arg); return ginitDict[3].key; } const char* compareFuncKeyName(void* arg) { - (void)arg; + TAOS_UNUSED(arg); return ginitDict[4].key; } const char* compareParKeyName(void* arg) { - (void)arg; + TAOS_UNUSED(arg); return ginitDict[5].key; } const char* comparePartagKeyName(void* arg) { - (void)arg; + TAOS_UNUSED(arg); return ginitDict[6].key; } @@ -2303,7 +2208,7 @@ const char* compactFilteFactoryNameFunc(void* arg) { return "stream_compact_factory_filter_func"; } -void destroyCompactFilte(void* arg) { (void)arg; } +void destroyCompactFilte(void* arg) { TAOS_UNUSED(arg); } unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, char** newval, size_t* newvlen, unsigned char* value_changed) { return streamStateValueIsStale((char*)val) ? 1 : 0; @@ -2374,8 +2279,14 @@ int32_t taskDbOpenCfs(STaskDbWrapper* pTask, char* path, char** pCfNames, int32_ int32_t code = -1; char* err = NULL; - rocksdb_options_t** cfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); + rocksdb_options_t** cfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); + if (cfOpts == NULL) { + return terrno; + } rocksdb_column_family_handle_t** cfHandle = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); + if (cfHandle == NULL) { + return terrno; + } for (int i = 0; i < nCf; i++) { int32_t idx = getCfIdx(pCfNames[i]); @@ -2416,7 +2327,7 @@ void taskDbRemoveRef(void* pTaskDb) { } STaskDbWrapper* pBackend = pTaskDb; - (void)taosReleaseRef(taskDbWrapperId, pBackend->refId); + TAOS_UNUSED(taosReleaseRef(taskDbWrapperId, pBackend->refId)); } void taskDbSetClearFileFlag(void* pTaskDb) { @@ -2459,6 +2370,14 @@ void taskDbInitOpt(STaskDbWrapper* pTaskDb) { pTaskDb->pCfParams = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam)); pTaskDb->pCfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); pTaskDb->pCompares = taosMemoryCalloc(nCf, sizeof(rocksdb_comparator_t*)); + if (pTaskDb->pCf == NULL || pTaskDb->pCfParams == NULL || pTaskDb->pCfOpts == NULL || pTaskDb->pCompares == NULL) { + stError("failed to alloc memory for cf"); + taosMemoryFreeClear(pTaskDb->pCf); + taosMemoryFreeClear(pTaskDb->pCfParams); + taosMemoryFreeClear(pTaskDb->pCfOpts); + taosMemoryFreeClear(pTaskDb->pCompares); + return; + } for (int i = 0; i < nCf; i++) { rocksdb_options_t* opt = rocksdb_options_create_copy(pTaskDb->dbOpt); @@ -2491,24 +2410,24 @@ void taskDbInitChkpOpt(STaskDbWrapper* pTaskDb) { pTaskDb->chkpId = -1; pTaskDb->chkpCap = 4; pTaskDb->chkpSaved = taosArrayInit(4, sizeof(int64_t)); - (void)taskDbLoadChkpInfo(pTaskDb); + TAOS_UNUSED(taskDbLoadChkpInfo(pTaskDb)); pTaskDb->chkpInUse = taosArrayInit(4, sizeof(int64_t)); - (void)taosThreadRwlockInit(&pTaskDb->chkpDirLock, NULL); + TAOS_UNUSED(taosThreadRwlockInit(&pTaskDb->chkpDirLock, NULL)); } void taskDbRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp) { - (void)taosThreadRwlockWrlock(&pTaskDb->chkpDirLock); + TAOS_UNUSED(taosThreadRwlockWrlock(&pTaskDb->chkpDirLock)); if (taosArrayPush(pTaskDb->chkpInUse, &chkp) == NULL) { stError("failed to push chkp: %" PRIi64 " into inuse", chkp); } taosArraySort(pTaskDb->chkpInUse, chkpIdComp); - (void)taosThreadRwlockUnlock(&pTaskDb->chkpDirLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&pTaskDb->chkpDirLock)); } void taskDbUnRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp) { - (void)taosThreadRwlockWrlock(&pTaskDb->chkpDirLock); + TAOS_UNUSED(taosThreadRwlockWrlock(&pTaskDb->chkpDirLock)); int32_t size = taosArrayGetSize(pTaskDb->chkpInUse); for (int i = 0; i < size; i++) { int64_t* p = taosArrayGet(pTaskDb->chkpInUse, i); @@ -2517,13 +2436,13 @@ void taskDbUnRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp) { break; } } - (void)taosThreadRwlockUnlock(&pTaskDb->chkpDirLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&pTaskDb->chkpDirLock)); } void taskDbDestroyChkpOpt(STaskDbWrapper* pTaskDb) { taosArrayDestroy(pTaskDb->chkpSaved); taosArrayDestroy(pTaskDb->chkpInUse); - (void)taosThreadRwlockDestroy(&pTaskDb->chkpDirLock); + TAOS_UNUSED(taosThreadRwlockDestroy(&pTaskDb->chkpDirLock)); } int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** stateFullPath) { @@ -2569,9 +2488,9 @@ int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** sta void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) { STaskDbWrapper* p = pTaskDb; - (void)streamMutexLock(&p->mutex); + TAOS_UNUSED(streamMutexLock(&p->mutex)); p->chkpId = chkpId; - (void)streamMutexUnlock(&p->mutex); + TAOS_UNUSED(streamMutexUnlock(&p->mutex)); } STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) { @@ -2587,7 +2506,9 @@ STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) { pTaskDb->idstr = key ? taosStrdup(key) : NULL; pTaskDb->path = statePath ? taosStrdup(statePath) : NULL; - (void)taosThreadMutexInit(&pTaskDb->mutex, NULL); + code = taosThreadMutexInit(&pTaskDb->mutex, NULL); + TSDB_CHECK_CODE(code, lino, _EXIT); + taskDbInitChkpOpt(pTaskDb); taskDbInitOpt(pTaskDb); @@ -2772,7 +2693,7 @@ int32_t taskDbGenChkpUploadData__rsync(STaskDbWrapper* pDb, int64_t chkpId, char char* buf = taosMemoryCalloc(1, cap); if (buf == NULL) { - (void)taosReleaseRef(taskDbWrapperId, refId); + TAOS_UNUSED(taosReleaseRef(taskDbWrapperId, refId)); return terrno; } @@ -2780,7 +2701,7 @@ int32_t taskDbGenChkpUploadData__rsync(STaskDbWrapper* pDb, int64_t chkpId, char snprintf(buf, cap, "%s%s%s%s%s%" PRId64 "", pDb->path, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkpId); if (nBytes <= 0 || nBytes >= cap) { taosMemoryFree(buf); - (void)taosReleaseRef(taskDbWrapperId, refId); + TAOS_UNUSED(taosReleaseRef(taskDbWrapperId, refId)); return TSDB_CODE_OUT_OF_RANGE; } @@ -2791,7 +2712,7 @@ int32_t taskDbGenChkpUploadData__rsync(STaskDbWrapper* pDb, int64_t chkpId, char taosMemoryFree(buf); } - (void)taosReleaseRef(taskDbWrapperId, refId); + TAOS_UNUSED(taosReleaseRef(taskDbWrapperId, refId)); return code; } @@ -3028,7 +2949,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t inst->dbOpt = handle->dbOpt; rocksdb_writeoptions_disable_WAL(inst->wOpt, 1); - (void)taosHashPut(handle->cfInst, idstr, strlen(idstr) + 1, &inst, sizeof(void*)); + TAOS_UNUSED(taosHashPut(handle->cfInst, idstr, strlen(idstr) + 1, &inst, sizeof(void*))); } else { inst = *pInst; } @@ -3269,9 +3190,9 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe break; \ } \ STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ - (void)atomic_add_fetch_64(&wrapper->dataWritten, 1); \ + TAOS_UNUSED(atomic_add_fetch_64(&wrapper->dataWritten, 1)); \ char toString[128] = {0}; \ - if (stDebugFlag & DEBUG_TRACE) (void)(ginitDict[i].toStrFunc((void*)key, toString)); \ + if (stDebugFlag & DEBUG_TRACE) TAOS_UNUSED((ginitDict[i].toStrFunc((void*)key, toString))); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ rocksdb_writeoptions_t* opts = wrapper->writeOpt; \ @@ -3303,7 +3224,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe } \ STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ char toString[128] = {0}; \ - if (stDebugFlag & DEBUG_TRACE) (void)(ginitDict[i].toStrFunc((void*)key, toString)); \ + if (stDebugFlag & DEBUG_TRACE) TAOS_UNUSED((ginitDict[i].toStrFunc((void*)key, toString))); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ rocksdb_t* db = wrapper->db; \ @@ -3346,9 +3267,9 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe break; \ } \ STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ - (void)atomic_add_fetch_64(&wrapper->dataWritten, 1); \ + TAOS_UNUSED(atomic_add_fetch_64(&wrapper->dataWritten, 1)); \ char toString[128] = {0}; \ - if (stDebugFlag & DEBUG_TRACE) (void)(ginitDict[i].toStrFunc((void*)key, toString)); \ + if (stDebugFlag & DEBUG_TRACE) TAOS_UNUSED(ginitDict[i].toStrFunc((void*)key, toString)); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ rocksdb_t* db = wrapper->db; \ @@ -3387,7 +3308,7 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) { stDebug("streamStateClear_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - (void)atomic_add_fetch_64(&wrapper->dataWritten, 1); + TAOS_UNUSED(atomic_add_fetch_64(&wrapper->dataWritten, 1)); char sKeyStr[128] = {0}; char eKeyStr[128] = {0}; @@ -3403,8 +3324,8 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) { if (err != NULL) { char toStringStart[128] = {0}; char toStringEnd[128] = {0}; - (void)stateKeyToString(&sKey, toStringStart); - (void)stateKeyToString(&eKey, toStringEnd); + TAOS_UNUSED(stateKeyToString(&sKey, toStringStart)); + TAOS_UNUSED(stateKeyToString(&eKey, toStringEnd)); stWarn("failed to delete range cf(state) start: %s, end:%s, reason:%s", toStringStart, toStringEnd, err); taosMemoryFree(err); @@ -3464,7 +3385,7 @@ int32_t streamStateAddIfNotExist_rocksdb(SStreamState* pState, const SWinKey* ke } *pVal = taosMemoryMalloc(size); if (*pVal == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } memset(*pVal, 0, size); return 0; @@ -3482,7 +3403,7 @@ int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, cons if (rocksdb_iter_valid(pCur->iter) && !iterValueIsStale(pCur->iter)) { size_t tlen; char* keyStr = (char*)rocksdb_iter_key(pCur->iter, &tlen); - (void)stateKeyDecode((void*)pKtmp, keyStr); + TAOS_UNUSED(stateKeyDecode((void*)pKtmp, keyStr)); if (pKtmp->opNum != pCur->number) { return -1; } @@ -3535,7 +3456,7 @@ SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWin SStateKey curKey; size_t kLen; char* keyStr = (char*)rocksdb_iter_key(pCur->iter, &kLen); - (void)stateKeyDecode((void*)&curKey, keyStr); + TAOS_UNUSED(stateKeyDecode((void*)&curKey, keyStr)); if (stateKeyCmpr(&sKey, sizeof(sKey), &curKey, sizeof(curKey)) > 0) { return pCur; } @@ -3557,7 +3478,7 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) { { char tbuf[256] = {0}; - (void)stateKeyToString((void*)&maxStateKey, tbuf); + TAOS_UNUSED(stateKeyToString((void*)&maxStateKey, tbuf)); stDebug("seek to last:%s", tbuf); } @@ -3607,7 +3528,7 @@ SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* SStateKey curKey; size_t kLen = 0; char* keyStr = (char*)rocksdb_iter_key(pCur->iter, &kLen); - (void)stateKeyDecode((void*)&curKey, keyStr); + TAOS_UNUSED(stateKeyDecode((void*)&curKey, keyStr)); if (stateKeyCmpr(&sKey, sizeof(sKey), &curKey, sizeof(curKey)) == 0) { pCur->number = pState->number; @@ -3736,6 +3657,10 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta pCur->db = wrapper->db; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); + if (pCur->iter == NULL) { + streamStateFreeCur(pCur); + return NULL; + } char buf[128] = {0}; SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; @@ -3755,9 +3680,14 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta size_t klen; const char* iKey = rocksdb_iter_key(pCur->iter, &klen); SStateSessionKey curKey = {0}; - (void)stateSessionKeyDecode(&curKey, (char*)iKey); + TAOS_UNUSED(stateSessionKeyDecode(&curKey, (char*)iKey)); if (stateSessionKeyCmpr(&sKey, sizeof(sKey), &curKey, sizeof(curKey)) >= 0) return pCur; + if (!rocksdb_iter_valid(pCur->iter)) { + streamStateFreeCur(pCur); + return NULL; + } + rocksdb_iter_prev(pCur->iter); if (!rocksdb_iter_valid(pCur->iter)) { streamStateFreeCur(pCur); @@ -3792,7 +3722,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pSta size_t klen; const char* iKey = rocksdb_iter_key(pCur->iter, &klen); SStateSessionKey curKey = {0}; - (void)stateSessionKeyDecode(&curKey, (char*)iKey); + TAOS_UNUSED(stateSessionKeyDecode(&curKey, (char*)iKey)); if (stateSessionKeyCmpr(&sKey, sizeof(sKey), &curKey, sizeof(curKey)) <= 0) return pCur; rocksdb_iter_next(pCur->iter); @@ -3832,7 +3762,7 @@ SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, con size_t klen; const char* iKey = rocksdb_iter_key(pCur->iter, &klen); SStateSessionKey curKey = {0}; - (void)stateSessionKeyDecode(&curKey, (char*)iKey); + TAOS_UNUSED(stateSessionKeyDecode(&curKey, (char*)iKey)); if (stateSessionKeyCmpr(&sKey, sizeof(sKey), &curKey, sizeof(curKey)) < 0) return pCur; rocksdb_iter_next(pCur->iter); @@ -3872,7 +3802,7 @@ SStreamStateCur* streamStateSessionSeekKeyPrev_rocksdb(SStreamState* pState, con size_t klen; const char* iKey = rocksdb_iter_key(pCur->iter, &klen); SStateSessionKey curKey = {0}; - (void)stateSessionKeyDecode(&curKey, (char*)iKey); + TAOS_UNUSED(stateSessionKeyDecode(&curKey, (char*)iKey)); if (stateSessionKeyCmpr(&sKey, sizeof(sKey), &curKey, sizeof(curKey)) > 0) return pCur; rocksdb_iter_prev(pCur->iter); @@ -3894,7 +3824,7 @@ int32_t streamStateSessionGetKVByCur_rocksdb(SStreamStateCur* pCur, SSessionKey* return -1; } const char* curKey = rocksdb_iter_key(pCur->iter, (size_t*)&kLen); - (void)stateSessionKeyDecode((void*)&ktmp, (char*)curKey); + TAOS_UNUSED(stateSessionKeyDecode((void*)&ktmp, (char*)curKey)); if (pVal != NULL) *pVal = NULL; if (pVLen != NULL) *pVLen = 0; @@ -3973,7 +3903,7 @@ SStreamStateCur* streamStateFillGetCur_rocksdb(SStreamState* pState, const SWinK size_t kLen; SWinKey curKey; char* keyStr = (char*)rocksdb_iter_key(pCur->iter, &kLen); - (void)winKeyDecode((void*)&curKey, keyStr); + TAOS_UNUSED(winKeyDecode((void*)&curKey, keyStr)); if (winKeyCmpr(key, sizeof(*key), &curKey, sizeof(curKey)) == 0) { return pCur; } @@ -3992,7 +3922,7 @@ int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, } size_t klen, vlen; char* keyStr = (char*)rocksdb_iter_key(pCur->iter, &klen); - (void)winKeyDecode(&winKey, keyStr); + TAOS_UNUSED(winKeyDecode(&winKey, keyStr)); const char* valStr = rocksdb_iter_value(pCur->iter, &vlen); int32_t len = valueDecode((void*)valStr, vlen, NULL, (char**)pVal); @@ -4033,7 +3963,7 @@ SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey curKey; size_t kLen = 0; char* keyStr = (char*)rocksdb_iter_key(pCur->iter, &kLen); - (void)winKeyDecode((void*)&curKey, keyStr); + TAOS_UNUSED(winKeyDecode((void*)&curKey, keyStr)); if (winKeyCmpr(key, sizeof(*key), &curKey, sizeof(curKey)) < 0) { return pCur; } @@ -4070,7 +4000,7 @@ SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey curKey; size_t kLen = 0; char* keyStr = (char*)rocksdb_iter_key(pCur->iter, &kLen); - (void)winKeyDecode((void*)&curKey, keyStr); + TAOS_UNUSED(winKeyDecode((void*)&curKey, keyStr)); if (winKeyCmpr(key, sizeof(*key), &curKey, sizeof(curKey)) > 0) { return pCur; } @@ -4154,7 +4084,7 @@ int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKe void* tmp = taosMemoryMalloc(valSize); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev_rocksdb(pState, key); @@ -4162,7 +4092,7 @@ int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKe if (code == 0) { if (sessionRangeKeyCmpr(&searchKey, key) == 0) { - memcpy(tmp, *pVal, valSize); + memcpy(tmp, *pVal, *pVLen); taosMemoryFreeClear(*pVal); goto _end; } @@ -4178,7 +4108,7 @@ int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKe code = streamStateSessionGetKVByCur_rocksdb(pCur, key, pVal, pVLen); if (code == 0) { if (sessionRangeKeyCmpr(&searchKey, key) == 0) { - memcpy(tmp, *pVal, valSize); + memcpy(tmp, *pVal, *pVLen); goto _end; } } @@ -4206,7 +4136,7 @@ void streamStateSessionClear_rocksdb(SStreamState* pState) { if (code == 0 && size > 0) { memset(buf, 0, size); // refactor later - (void)streamStateSessionPut_rocksdb(pState, &delKey, buf, size); + TAOS_UNUSED(streamStateSessionPut_rocksdb(pState, &delKey, buf, size)); } else { taosMemoryFreeClear(buf); break; @@ -4419,7 +4349,7 @@ void streamStateDestroyBatch(void* pBatch) { rocksdb_writebatch_destroy((rock int32_t streamStatePutBatch(SStreamState* pState, const char* cfKeyName, rocksdb_writebatch_t* pBatch, void* key, void* val, int32_t vlen, int64_t ttl) { STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - (void)atomic_add_fetch_64(&wrapper->dataWritten, 1); + TAOS_UNUSED(atomic_add_fetch_64(&wrapper->dataWritten, 1)); int i = streamStateGetCfIdx(pState, cfKeyName); if (i < 0) { @@ -4439,7 +4369,7 @@ int32_t streamStatePutBatch(SStreamState* pState, const char* cfKeyName, rocksdb { char tbuf[256] = {0}; - (void)(ginitDict[i].toStrFunc((void*)key, tbuf)); + TAOS_UNUSED(ginitDict[i].toStrFunc((void*)key, tbuf)); stTrace("streamState str: %s succ to write to %s_%s, len: %d", tbuf, wrapper->idstr, ginitDict[i].key, vlen); } return 0; @@ -4454,7 +4384,7 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - (void)atomic_add_fetch_64(&wrapper->dataWritten, 1); + TAOS_UNUSED(atomic_add_fetch_64(&wrapper->dataWritten, 1)); rocksdb_column_family_handle_t* pCf = wrapper->pCf[ginitDict[cfIdx].idx]; rocksdb_writebatch_put_cf((rocksdb_writebatch_t*)pBatch, pCf, buf, (size_t)klen, ttlV, (size_t)ttlVLen); @@ -4465,7 +4395,7 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb { char tbuf[256] = {0}; - (void)(ginitDict[cfIdx].toStrFunc((void*)key, tbuf)); + TAOS_UNUSED(ginitDict[cfIdx].toStrFunc((void*)key, tbuf)); stTrace("streamState str: %s succ to write to %s_%s", tbuf, wrapper->idstr, ginitDict[cfIdx].key); } return 0; @@ -4473,7 +4403,7 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch) { char* err = NULL; STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - (void)atomic_add_fetch_64(&wrapper->dataWritten, 1); + TAOS_UNUSED(atomic_add_fetch_64(&wrapper->dataWritten, 1)); rocksdb_write(wrapper->db, wrapper->writeOpt, (rocksdb_writebatch_t*)pBatch, &err); if (err != NULL) { stError("streamState failed to write batch, err:%s", err); @@ -4562,7 +4492,7 @@ int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) { if (fname == NULL) { return terrno; } - (void)strncpy(fname, name, len); + TAOS_UNUSED(strncpy(fname, name, len)); if (taosArrayPush(diff, &fname) == NULL) { taosMemoryFree(fname); return terrno; @@ -4642,7 +4572,7 @@ void dbChkpDebugInfo(SDbChkp* pDb) { int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { int32_t code = 0; int32_t nBytes; - (void)taosThreadRwlockWrlock(&p->rwLock); + TAOS_UNUSED(taosThreadRwlockWrlock(&p->rwLock)); p->preCkptId = p->curChkpId; p->curChkpId = chkpId; @@ -4660,7 +4590,7 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { nBytes = snprintf(p->buf, p->len, "%s%s%s%scheckpoint%" PRId64 "", p->path, TD_DIRSEP, "checkpoints", TD_DIRSEP, chkpId); if (nBytes <= 0 || nBytes >= p->len) { - (void)taosThreadRwlockUnlock(&p->rwLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&p->rwLock)); return TSDB_CODE_OUT_OF_RANGE; } @@ -4670,7 +4600,7 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { TdDirPtr pDir = taosOpenDir(p->buf); if (pDir == NULL) { - (void)taosThreadRwlockUnlock(&p->rwLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&p->rwLock)); return terrno; } @@ -4706,9 +4636,9 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { continue; } } - (void)taosCloseDir(&pDir); + TAOS_UNUSED(taosCloseDir(&pDir)); if (code != 0) { - (void)taosThreadRwlockUnlock(&p->rwLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&p->rwLock)); return code; } @@ -4720,14 +4650,14 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { if (name != NULL && !isBkdDataMeta(name, len)) { char* fname = taosMemoryCalloc(1, len + 1); if (fname == NULL) { - (void)taosThreadRwlockUnlock(&p->rwLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&p->rwLock)); return terrno; } - (void)strncpy(fname, name, len); + TAOS_UNUSED(strncpy(fname, name, len)); if (taosArrayPush(p->pAdd, &fname) == NULL) { taosMemoryFree(fname); - (void)taosThreadRwlockUnlock(&p->rwLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&p->rwLock)); return terrno; } } @@ -4761,7 +4691,7 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { p->idx = 1 - p->idx; - (void)taosThreadRwlockUnlock(&p->rwLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&p->rwLock)); return code; } @@ -4819,7 +4749,7 @@ int32_t dbChkpCreate(char* path, int64_t initChkpId, SDbChkp** ppChkp) { } p->update = 0; - (void)taosThreadRwlockInit(&p->rwLock, NULL); + TAOS_UNUSED(taosThreadRwlockInit(&p->rwLock, NULL)); SArray* list = NULL; code = dbChkpGetDelta(p, initChkpId, list); @@ -4860,7 +4790,7 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { static char* chkpMeta = "META"; int32_t code = 0; - (void)taosThreadRwlockRdlock(&p->rwLock); + TAOS_UNUSED(taosThreadRwlockRdlock(&p->rwLock)); int32_t cap = p->len + 128; @@ -5003,7 +4933,7 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { if (nBytes <= 0 || nBytes >= sizeof(content)) { code = TSDB_CODE_OUT_OF_RANGE; stError("chkp failed to format meta file: %s, reason: invalid msg", dstDir); - (void)taosCloseFile(&pFile); + TAOS_UNUSED(taosCloseFile(&pFile)); goto _ERROR; } @@ -5011,10 +4941,10 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { if (nBytes != strlen(content)) { code = terrno; stError("chkp failed to write meta file: %s,reason:%s", dstDir, tstrerror(code)); - (void)taosCloseFile(&pFile); + TAOS_UNUSED(taosCloseFile(&pFile)); goto _ERROR; } - (void)taosCloseFile(&pFile); + TAOS_UNUSED(taosCloseFile(&pFile)); // clear delta data buf taosArrayClearP(p->pAdd, taosMemoryFree); @@ -5023,7 +4953,7 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { _ERROR: taosMemoryFree(buffer); - (void)taosThreadRwlockUnlock(&p->rwLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&p->rwLock)); return code; } @@ -5068,7 +4998,7 @@ void bkdMgtDestroy(SBkdMgt* bm) { pIter = taosHashIterate(bm->pDbChkpTbl, pIter); } - (void)taosThreadRwlockDestroy(&bm->rwLock); + TAOS_UNUSED(taosThreadRwlockDestroy(&bm->rwLock)); taosMemoryFree(bm->path); taosHashCleanup(bm->pDbChkpTbl); @@ -5076,7 +5006,7 @@ void bkdMgtDestroy(SBkdMgt* bm) { } int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, char* dname) { int32_t code = 0; - (void)taosThreadRwlockWrlock(&bm->rwLock); + TAOS_UNUSED(taosThreadRwlockWrlock(&bm->rwLock)); SDbChkp** ppChkp = taosHashGet(bm->pDbChkpTbl, taskId, strlen(taskId)); SDbChkp* pChkp = ppChkp != NULL ? *ppChkp : NULL; @@ -5084,14 +5014,14 @@ int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, int32_t cap = strlen(bm->path) + 64; char* path = taosMemoryCalloc(1, cap); if (path == NULL) { - (void)taosThreadRwlockUnlock(&bm->rwLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&bm->rwLock)); return terrno; } int32_t nBytes = snprintf(path, cap, "%s%s%s", bm->path, TD_DIRSEP, taskId); if (nBytes <= 0 || nBytes >= cap) { taosMemoryFree(path); - (void)taosThreadRwlockUnlock(&bm->rwLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&bm->rwLock)); code = TSDB_CODE_OUT_OF_RANGE; return code; } @@ -5100,20 +5030,20 @@ int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, code = dbChkpCreate(path, chkpId, &p); if (code != 0) { taosMemoryFree(path); - (void)taosThreadRwlockUnlock(&bm->rwLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&bm->rwLock)); return code; } if (taosHashPut(bm->pDbChkpTbl, taskId, strlen(taskId), &p, sizeof(void*)) != 0) { dbChkpDestroy(p); - (void)taosThreadRwlockUnlock(&bm->rwLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&bm->rwLock)); code = terrno; return code; } pChkp = p; code = dbChkpDumpTo(pChkp, dname, list); - (void)taosThreadRwlockUnlock(&bm->rwLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&bm->rwLock)); return code; } else { code = dbChkpGetDelta(pChkp, chkpId, NULL); @@ -5122,7 +5052,7 @@ int32_t bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list, } } - (void)taosThreadRwlockUnlock(&bm->rwLock); + TAOS_UNUSED(taosThreadRwlockUnlock(&bm->rwLock)); return code; } diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 2688617823..75bcc326b3 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -108,7 +108,7 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) { pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId); code = streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, - &pTask->outputInfo.fixedDispatcher.epSet); + &pTask->outputInfo.fixedDispatcher.epSet); } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { streamTaskStartMonitorCheckRsp(pTask); @@ -171,14 +171,14 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage); SStreamTaskState pState = streamTaskGetStatus(pTask); - stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(qid:0x%" PRIx64 + stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(QID:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", pTask->id.idStr, pState.name, pRsp->oldStage, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status); streamMetaReleaseTask(pMeta, pTask); } else { pRsp->status = TASK_DOWNSTREAM_NOT_READY; - stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(qid:0x%" PRIx64 + stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(QID:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp check_status %d", pReq->streamId, taskId, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status); } @@ -259,7 +259,8 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa void* buf = rpcMallocCont(sizeof(SMsgHead) + len); if (buf == NULL) { - stError("s-task:0x%x vgId:%d failed prepare msg, %s at line:%d code:%s", taskId, pMeta->vgId, __func__, __LINE__, tstrerror(code)); + stError("s-task:0x%x vgId:%d failed prepare msg, %s at line:%d code:%s", taskId, pMeta->vgId, __func__, __LINE__, + tstrerror(code)); return terrno; } @@ -332,7 +333,7 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void processDownstreamReadyRsp(SStreamTask* pTask) { EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST; - int32_t code = streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); + int32_t code = streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); if (code) { stError("s-task:%s failed to set event succ, code:%s", pTask->id.idStr, tstrerror(code)); } @@ -354,7 +355,7 @@ void processDownstreamReadyRsp(SStreamTask* pTask) { stDebug("s-task:%s level:%d initial status is %s from mnode, set it to be halt", pTask->id.idStr, pTask->info.taskLevel, streamTaskGetStatusStr(pTask->status.taskStatus)); code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT); - if (code != 0) { // todo: handle error + if (code != 0) { // todo: handle error stError("s-task:%s failed to handle halt event, code:%s", pTask->id.idStr, tstrerror(code)); } } @@ -373,8 +374,9 @@ void processDownstreamReadyRsp(SStreamTask* pTask) { int32_t addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { int32_t vgId = pTask->pMeta->vgId; - int32_t code = 0;; - bool existed = false; + int32_t code = 0; + ; + bool existed = false; streamMutexLock(&pTask->lock); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 47eec2119a..e44bca123b 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -62,7 +62,7 @@ int32_t createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int6 if (pChkpoint->blocks == NULL) { taosMemoryFree(pBlock); taosFreeQitem(pChkpoint); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } void* p = taosArrayPush(pChkpoint->blocks, pBlock); @@ -70,7 +70,7 @@ int32_t createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int6 taosArrayDestroy(pChkpoint->blocks); taosMemoryFree(pBlock); taosFreeQitem(pChkpoint); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } *pRes = pChkpoint; @@ -120,38 +120,39 @@ int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSo } int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTriggerRsp* pRsp) { + SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; + bool unQualified = false; + const char* id = pTask->id.idStr; + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - stError("s-task:%s invalid msg recv, checkpoint-trigger rsp not handled", pTask->id.idStr); + stError("s-task:%s invalid msg recv, checkpoint-trigger rsp not handled", id); return TSDB_CODE_INVALID_MSG; } if (pRsp->rspCode != TSDB_CODE_SUCCESS) { - stDebug("s-task:%s retrieve checkpoint-trgger rsp from upstream:0x%x invalid, code:%s", pTask->id.idStr, - pRsp->upstreamTaskId, tstrerror(pRsp->rspCode)); + stDebug("s-task:%s retrieve checkpoint-trgger rsp from upstream:0x%x invalid, code:%s", id, pRsp->upstreamTaskId, + tstrerror(pRsp->rspCode)); return TSDB_CODE_SUCCESS; } streamMutexLock(&pTask->lock); SStreamTaskState status = streamTaskGetStatus(pTask); - if (status.state != TASK_STATUS__CK) { - stError("s-task:%s status:%s not in checkpoint status, discard the checkpoint-trigger msg", pTask->id.idStr, status.name); - streamMutexUnlock(&pTask->lock); - return TSDB_CODE_STREAM_TASK_IVLD_STATUS; - } - streamMutexUnlock(&pTask->lock); - SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; - streamMutexLock(&pInfo->lock); - if (pInfo->activeId != pRsp->checkpointId || pInfo->transId != pRsp->transId) { - stError("s-task:%s status:%s not in checkpoint status, discard the checkpoint-trigger msg", pTask->id.idStr, status.name); - - streamMutexUnlock(&pInfo->lock); + if (status.state != TASK_STATUS__CK) { + stError("s-task:%s status:%s not in checkpoint status, discard the checkpoint-trigger msg", id, status.name); return TSDB_CODE_STREAM_TASK_IVLD_STATUS; } + streamMutexLock(&pInfo->lock); + unQualified = (pInfo->activeId != pRsp->checkpointId || pInfo->transId != pRsp->transId); streamMutexUnlock(&pInfo->lock); + if (unQualified) { + stError("s-task:%s status:%s not in checkpoint status, discard the checkpoint-trigger msg", id, status.name); + return TSDB_CODE_STREAM_TASK_IVLD_STATUS; + } + // NOTE: here we do not do the duplicated checkpoint-trigger msg check, since it will be done by following functions. int32_t code = appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER, pRsp->checkpointId, pRsp->transId, pRsp->upstreamTaskId); @@ -740,7 +741,7 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d SArray* toDelFiles = taosArrayInit(4, POINTER_BYTES); if (toDelFiles == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if ((code = taskDbGenChkpUploadData(pTask->pBackend, pMeta->bkdChkptMgt, checkpointId, type, &path, toDelFiles, @@ -903,7 +904,7 @@ static int32_t doChkptStatusCheck(SStreamTask* pTask) { return -1; } - if ((pTmrInfo->launchChkptId != pActiveInfo->activeId) || (pActiveInfo->activeId == 0)) { + if (pTmrInfo->launchChkptId != pActiveInfo->activeId) { int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stWarn("s-task:%s vgId:%d checkpoint-trigger retrieve by previous checkpoint procedure, checkpointId:%" PRId64 ", quit, ref:%d", @@ -963,11 +964,44 @@ static int32_t doFindNotSendUpstream(SStreamTask* pTask, SArray* pList, SArray** return 0; } +static int32_t chkptTriggerRecvMonitorHelper(SStreamTask* pTask, SArray* pNotSendList) { + const char* id = pTask->id.idStr; + SArray* pList = pTask->upstreamInfo.pList; // send msg to retrieve checkpoint trigger msg + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr; + int32_t vgId = pTask->pMeta->vgId; + + int32_t code = doChkptStatusCheck(pTask); + if (code) { + return code; + } + + code = doFindNotSendUpstream(pTask, pList, &pNotSendList); + if (code) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stDebug("s-task:%s failed to find not send upstream, code:%s, out of tmr, ref:%d", id, tstrerror(code), ref); + return code; + } + + // do send retrieve checkpoint trigger msg to upstream + code = doSendRetrieveTriggerMsg(pTask, pNotSendList); + if (code) { + stError("s-task:%s vgId:%d failed to retrieve trigger msg, code:%s", pTask->id.idStr, vgId, tstrerror(code)); + code = 0; + } + + return code; +} + void checkpointTriggerMonitorFn(void* param, void* tmrId) { SStreamTask* pTask = param; int32_t vgId = pTask->pMeta->vgId; int64_t now = taosGetTimestampMs(); const char* id = pTask->id.idStr; + SArray* pNotSendList = NULL; + SArray* pList = pTask->upstreamInfo.pList; // send msg to retrieve checkpoint trigger msg + int32_t code = 0; + int32_t numOfNotSend = 0; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr; @@ -1008,42 +1042,21 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { } streamMutexLock(&pActiveInfo->lock); + code = chkptTriggerRecvMonitorHelper(pTask, pNotSendList); + streamMutexUnlock(&pActiveInfo->lock); - int32_t code = doChkptStatusCheck(pTask); - if (code) { - streamMutexUnlock(&pActiveInfo->lock); + if (code != TSDB_CODE_SUCCESS) { streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } - - // send msg to retrieve checkpoint trigger msg - SArray* pList = pTask->upstreamInfo.pList; - SArray* pNotSendList = NULL; - - code = doFindNotSendUpstream(pTask, pList, &pNotSendList); - if (code) { - int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); - stDebug("s-task:%s failed to find not send upstream, code:%s, out of tmr, ref:%d", id, tstrerror(code), ref); - streamMutexUnlock(&pActiveInfo->lock); - streamMetaReleaseTask(pTask->pMeta, pTask); - taosArrayDestroy(pNotSendList); return; } - // do send retrieve checkpoint trigger msg to upstream - int32_t size = taosArrayGetSize(pNotSendList); - code = doSendRetrieveTriggerMsg(pTask, pNotSendList); - if (code) { - stError("s-task:%s vgId:%d failed to retrieve trigger msg, code:%s", pTask->id.idStr, vgId, tstrerror(code)); - } - - streamMutexUnlock(&pActiveInfo->lock); - // check every 100ms - if (size > 0) { + numOfNotSend = taosArrayGetSize(pNotSendList); + if (numOfNotSend > 0) { stDebug("s-task:%s start to monitor checkpoint-trigger in 10s", id); - streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); + streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + "trigger-recv-monitor"); } else { int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stDebug("s-task:%s all checkpoint-trigger recved, quit from monitor checkpoint-trigger tmr, ref:%d", id, ref); @@ -1077,7 +1090,7 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) { SRetrieveChkptTriggerReq* pReq = rpcMallocCont(sizeof(SRetrieveChkptTriggerReq)); if (pReq == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; stError("vgId:%d failed to create msg to retrieve trigger msg for task:%s exec, code:out of memory", vgId, pId); continue; } @@ -1106,19 +1119,13 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) { return code; } -bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) { +static int32_t isAlreadySendTriggerNoLock(SStreamTask* pTask, int32_t downstreamNodeId) { int64_t now = taosGetTimestampMs(); const char* id = pTask->id.idStr; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; SStreamTaskState pStatus = streamTaskGetStatus(pTask); - if (pStatus.state != TASK_STATUS__CK) { - return false; - } - - streamMutexLock(&pInfo->lock); if (!pInfo->dispatchTrigger) { - streamMutexUnlock(&pInfo->lock); return false; } @@ -1146,14 +1153,29 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) id, pSendInfo->sendTs, before, pInfo->activeId, pInfo->transId); } - streamMutexUnlock(&pInfo->lock); return true; } - streamMutexUnlock(&pInfo->lock); return false; } +bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) { + int64_t now = taosGetTimestampMs(); + const char* id = pTask->id.idStr; + SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; + SStreamTaskState pStatus = streamTaskGetStatus(pTask); + + if (pStatus.state != TASK_STATUS__CK) { + return false; + } + + streamMutexLock(&pInfo->lock); + bool send = isAlreadySendTriggerNoLock(pTask, downstreamNodeId); + streamMutexUnlock(&pInfo->lock); + + return send; +} + void streamTaskGetTriggerRecvStatus(SStreamTask* pTask, int32_t* pRecved, int32_t* pTotal) { *pRecved = taosArrayGetSize(pTask->chkInfo.pActiveInfo->pReadyMsgList); @@ -1169,8 +1191,10 @@ void streamTaskGetTriggerRecvStatus(SStreamTask* pTask, int32_t* pRecved, int32_ int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; int64_t now = taosGetTimestampMs(); + int32_t code = 0; streamMutexLock(&pInfo->lock); + pInfo->dispatchTrigger = true; if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { STaskDispatcherFixed* pDispatch = &pTask->outputInfo.fixedDispatcher; @@ -1178,8 +1202,7 @@ int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pDispatch->nodeId, .taskId = pDispatch->taskId}; void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p); if (px == NULL) { // pause the stream task, if memory not enough - streamMutexUnlock(&pInfo->lock); - return terrno; + code = terrno; } } else { for (int32_t i = 0; i < streamTaskGetNumOfDownstream(pTask); ++i) { @@ -1191,14 +1214,15 @@ int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pVgInfo->vgId, .taskId = pVgInfo->taskId}; void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p); if (px == NULL) { // pause the stream task, if memory not enough - streamMutexUnlock(&pInfo->lock); - return terrno; + code = terrno; + break; } } } streamMutexUnlock(&pInfo->lock); - return 0; + + return code; } int32_t streamTaskGetNumOfConfirmed(SActiveCheckpointInfo* pInfo) { diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index d85435d21c..58826b2e99 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -191,7 +191,7 @@ int32_t streamMergedSubmitNew(SStreamMergedSubmit** pSubmit) { if ((*pSubmit)->submits == NULL) { taosFreeQitem(*pSubmit); *pSubmit = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (*pSubmit)->type = STREAM_INPUT__MERGED_SUBMIT; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 111ce37b41..78cbd844a0 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -73,7 +73,7 @@ static int32_t tInitStreamDispatchReq(SStreamDispatchReq* pReq, const SStreamTas if (pReq->data == NULL || pReq->dataLen == NULL) { taosArrayDestroyP(pReq->data, taosMemoryFree); taosArrayDestroy(pReq->dataLen); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } return TSDB_CODE_SUCCESS; @@ -110,7 +110,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r buf = rpcMallocCont(sizeof(SMsgHead) + len); if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } ((SMsgHead*)buf)->vgId = htonl(pEpInfo->nodeId); @@ -209,7 +209,7 @@ int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, buf = rpcMallocCont(sizeof(SMsgHead) + tlen); if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } ((SMsgHead*)buf)->vgId = htonl(nodeId); @@ -526,6 +526,7 @@ static void doMonitorDispatchData(void* param, void* tmrId) { int32_t msgId = pMsgInfo->msgId; int32_t code = 0; int64_t now = taosGetTimestampMs(); + bool inDispatch = true; stDebug("s-task:%s start monitor dispatch data", id); @@ -550,12 +551,15 @@ static void doMonitorDispatchData(void* param, void* tmrId) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s not in dispatch procedure, abort from timer, ref:%d", pTask->id.idStr, ref); - pTask->msgInfo.inMonitor = 0; - streamMutexUnlock(&pMsgInfo->lock); - return; + pMsgInfo->inMonitor = 0; + inDispatch = false; } streamMutexUnlock(&pMsgInfo->lock); + if (!inDispatch) { + return; + } + int32_t numOfFailed = getFailedDispatchInfo(pMsgInfo, now); if (numOfFailed == 0) { stDebug("s-task:%s no error occurs, check again in %dms", id, DISPATCH_RETRY_INTERVAL_MS); @@ -638,15 +642,54 @@ void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) { "dispatch-monitor"); } +static int32_t doAddDispatchBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, + SArray* vgInfo, uint32_t hashValue, int64_t now, bool* pFound) { + size_t numOfVgroups = taosArrayGetSize(vgInfo); + int32_t code = 0; + + *pFound = false; + + for (int32_t j = 0; j < numOfVgroups; j++) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); + if (pVgInfo == NULL) { + continue; + } + + if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { + if ((code = streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j])) < 0) { + stError("s-task:%s failed to add dispatch block, code:%s", pTask->id.idStr, tstrerror(terrno)); + return code; + } + + if (pReqs[j].blockNum == 0) { + SVgroupInfo* pDstVgroupInfo = taosArrayGet(vgInfo, j); + if (pDstVgroupInfo != NULL) { + addDispatchEntry(&pTask->msgInfo, pDstVgroupInfo->vgId, now, false); + } + } + + pReqs[j].blockNum++; + *pFound = true; + break; + } + } + + return code; +} + int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, int64_t groupId, int64_t now) { bool found = false; uint32_t hashValue = 0; - int32_t numOfVgroups = 0; + int32_t code = 0; + SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; - SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; if (pTask->pNameMap == NULL) { pTask->pNameMap = tSimpleHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); + if (pTask->pNameMap == NULL) { + stError("s-task:%s failed to init the name map, code:%s", pTask->id.idStr, tstrerror(terrno)); + return terrno; + } } void* pVal = tSimpleHashGet(pTask->pNameMap, &groupId, sizeof(int64_t)); @@ -669,11 +712,11 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S } } } else { - int32_t code = buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, - pDataBlock->info.parTbName); + code = buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, + pDataBlock->info.parTbName); if (code) { - stError("s-task:%s failed to build child table name for group:%" PRId64 ", code:%s", pTask->id.idStr, - groupId, tstrerror(code)); + stError("s-task:%s failed to build child table name for group:%" PRId64 ", code:%s", pTask->id.idStr, groupId, + tstrerror(code)); } } @@ -688,44 +731,21 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S memcpy(bln.parTbName, pDataBlock->info.parTbName, strlen(pDataBlock->info.parTbName)); // failed to put into name buffer, no need to do anything - if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) { // allow error, and do nothing - int32_t code = tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName)); + if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) { // allow error, and do nothing + code = tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName)); } } - numOfVgroups = taosArrayGetSize(vgInfo); - - // TODO: optimize search streamMutexLock(&pTask->msgInfo.lock); + code = doAddDispatchBlock(pTask, pReqs, pDataBlock, vgInfo, hashValue, now, &found); + streamMutexUnlock(&pTask->msgInfo.lock); - for (int32_t j = 0; j < numOfVgroups; j++) { - SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); - if (pVgInfo == NULL) { - continue; - } - - if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { - if (streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]) < 0) { - streamMutexUnlock(&pTask->msgInfo.lock); - return -1; - } - - if (pReqs[j].blockNum == 0) { - SVgroupInfo* pDstVgroupInfo = taosArrayGet(vgInfo, j); - if (pDstVgroupInfo != NULL) { - addDispatchEntry(&pTask->msgInfo, pDstVgroupInfo->vgId, now, false); - } - } - - pReqs[j].blockNum++; - found = true; - break; - } + if (code) { + return code; } - streamMutexUnlock(&pTask->msgInfo.lock); if (!found) { - stError("s-task:%s not found req hash value:%u", pTask->id.idStr, hashValue); + stError("s-task:%s not found req hash value:%u, failed to add dispatch block", pTask->id.idStr, hashValue); return TSDB_CODE_STREAM_INTERNAL_ERROR; } else { return 0; @@ -919,7 +939,7 @@ static int32_t doTaskChkptStatusCheck(SStreamTask* pTask, int32_t num) { } static int32_t doFindNotConfirmUpstream(SArray** ppNotRspList, SArray* pList, int32_t num, int32_t vgId, int32_t level, - const char* id) { + const char* id) { SArray* pTmp = taosArrayInit(4, sizeof(int32_t)); if (pTmp == NULL) { return terrno; @@ -940,8 +960,8 @@ static int32_t doFindNotConfirmUpstream(SArray** ppNotRspList, SArray* pList, in stError("s-task:%s vgId:%d failed to record not rsp task, code: out of memory", id, vgId); return terrno; } else { - stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, - level, pInfo->upstreamTaskId); + stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, level, + pInfo->upstreamTaskId); } } @@ -987,13 +1007,48 @@ static void doSendChkptReadyMsg(SStreamTask* pTask, SArray* pNotRspList, int64_t } } -static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { +static int32_t chkptReadyMsgSendHelper(SStreamTask* pTask, SArray* pNotRspList) { + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptReadyMsgTmr; + SArray* pList = pActiveInfo->pReadyMsgList; + int32_t num = taosArrayGetSize(pList); + int32_t vgId = pTask->pMeta->vgId; + int32_t checkpointId = pActiveInfo->activeId; + const char* id = pTask->id.idStr; + int32_t notRsp = 0; + + int32_t code = doTaskChkptStatusCheck(pTask, num); + if (code) { + return code; + } + + code = doFindNotConfirmUpstream(&pNotRspList, pList, num, vgId, pTask->info.taskLevel, id); + if (code) { + int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); + stError("s-task:%s failed to find not rsp checkpoint-ready downstream, code:%s, out of tmr, ref:%d", id, + tstrerror(code), ref); + return code; + } + + notRsp = taosArrayGetSize(pNotRspList); + if (notRsp == 0) { + streamClearChkptReadyMsg(pActiveInfo); + } else { + doSendChkptReadyMsg(pTask, pNotRspList, checkpointId, pList); + } + + return code; +} + +static void chkptReadyMsgSendMonitorFn(void* param, void* tmrId) { SStreamTask* pTask = param; int32_t vgId = pTask->pMeta->vgId; const char* id = pTask->id.idStr; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptReadyMsgTmr; SArray* pNotRspList = NULL; + int32_t code = 0; + int32_t notRsp = 0; // check the status every 100ms if (streamTaskShouldStop(pTask)) { @@ -1004,7 +1059,7 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { } if (++pTmrInfo->activeCounter < 50) { - streamTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + streamTmrStart(chkptReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); return; } @@ -1027,45 +1082,26 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { } streamMutexLock(&pActiveInfo->lock); + code = chkptReadyMsgSendHelper(pTask, pNotRspList); + streamMutexUnlock(&pActiveInfo->lock); - SArray* pList = pActiveInfo->pReadyMsgList; - int32_t num = taosArrayGetSize(pList); - int32_t code = doTaskChkptStatusCheck(pTask, num); - if (code) { - streamMutexUnlock(&pActiveInfo->lock); + if (code != TSDB_CODE_SUCCESS) { streamMetaReleaseTask(pTask->pMeta, pTask); - return; - } - - code = doFindNotConfirmUpstream(&pNotRspList, pList, num, vgId, pTask->info.taskLevel, id); - if (code) { - int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); - stError("s-task:%s failed to find not rsp checkpoint-ready downstream, code:%s, out of tmr, ref:%d", id, - tstrerror(code), ref); - streamMutexUnlock(&pActiveInfo->lock); - streamMetaReleaseTask(pTask->pMeta, pTask); - taosArrayDestroy(pNotRspList); return; } - int32_t checkpointId = pActiveInfo->activeId; - int32_t notRsp = taosArrayGetSize(pNotRspList); - doSendChkptReadyMsg(pTask, pNotRspList, checkpointId, pList); - + notRsp = taosArrayGetSize(pNotRspList); if (notRsp > 0) { // send checkpoint-ready msg again - streamTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + stDebug("s-task:%s start to monitor checkpoint-ready msg recv status in 10s", id); + streamTmrStart(chkptReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); - streamMutexUnlock(&pActiveInfo->lock); } else { int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); stDebug( "s-task:%s vgId:%d checkpoint-ready msg confirmed by all upstream task(s), clear checkpoint-ready msg and quit " "from timer, ref:%d", id, vgId, ref); - - streamClearChkptReadyMsg(pActiveInfo); - streamMutexUnlock(&pActiveInfo->lock); // release should be the last execution, since pTask may be destroy after it immidiately. streamMetaReleaseTask(pTask->pMeta, pTask); } @@ -1124,7 +1160,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { stDebug("s-task:%s start checkpoint-ready monitor in 10s, ref:%d ", pTask->id.idStr, ref); streamMetaAcquireOneTask(pTask); - streamTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + streamTmrStart(chkptReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); // mark the timer monitor checkpointId @@ -1190,6 +1226,7 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch taosMemoryFree(buf); return terrno; } + SET_PAYLOAD_LEN(pRetrieve->data, actualLen, actualLen); int32_t payloadLen = actualLen + PAYLOAD_PREFIX_LEN; @@ -1359,29 +1396,11 @@ void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstr pReadyInfo->childId = childId; } -int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, int32_t index, int64_t checkpointId) { - if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - return TSDB_CODE_SUCCESS; - } - - SStreamUpstreamEpInfo* pInfo = NULL; - streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo); - if (pInfo == NULL) { - return TSDB_CODE_STREAM_TASK_NOT_EXIST; - } - - STaskCheckpointReadyInfo info = {0}; - initCheckpointReadyInfo(&info, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId); - - stDebug("s-task:%s (level:%d) prepare checkpoint-ready msg to upstream s-task:0x%" PRIx64 "-0x%x (vgId:%d) idx:%d", - pTask->id.idStr, pTask->info.taskLevel, pTask->id.streamId, pInfo->taskId, pInfo->nodeId, index); - +static int32_t doAddChkptReadyMsg(SStreamTask* pTask, STaskCheckpointReadyInfo* pInfo) { SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; - streamMutexLock(&pActiveInfo->lock); - void* px = taosArrayPush(pActiveInfo->pReadyMsgList, &info); + void* px = taosArrayPush(pActiveInfo->pReadyMsgList, pInfo); if (px == NULL) { - streamMutexUnlock(&pActiveInfo->lock); stError("s-task:%s failed to add readyMsg info, code: out of memory", pTask->id.idStr); return terrno; } @@ -1395,10 +1414,36 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, stDebug("s-task:%s %d/%d checkpoint-trigger recv", pTask->id.idStr, numOfRecv, total); } - streamMutexUnlock(&pActiveInfo->lock); return 0; } +int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, int32_t index, int64_t checkpointId) { + int32_t code = 0; + STaskCheckpointReadyInfo info = {0}; + + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + return TSDB_CODE_SUCCESS; + } + + SStreamUpstreamEpInfo* pInfo = NULL; + streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo); + if (pInfo == NULL) { + return TSDB_CODE_STREAM_TASK_NOT_EXIST; + } + + initCheckpointReadyInfo(&info, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId); + + stDebug("s-task:%s (level:%d) prepare checkpoint-ready msg to upstream s-task:0x%" PRIx64 "-0x%x (vgId:%d) idx:%d", + pTask->id.idStr, pTask->info.taskLevel, pTask->id.streamId, pInfo->taskId, pInfo->nodeId, index); + + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + + streamMutexLock(&pActiveInfo->lock); + code = doAddChkptReadyMsg(pTask, &info); + streamMutexUnlock(&pActiveInfo->lock); + return code; +} + void streamClearChkptReadyMsg(SActiveCheckpointInfo* pActiveInfo) { if (pActiveInfo == NULL) { return; diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 88e40b247b..2e06813071 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -24,7 +24,8 @@ #define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec static int32_t streamTransferStateDoPrepare(SStreamTask* pTask); -static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks); +static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, + int32_t* totalBlocks); bool streamTaskShouldStop(const SStreamTask* pTask) { SStreamTaskState pState = streamTaskGetStatus(pTask); @@ -95,17 +96,53 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* return code; } +static int32_t doAppendPullOverBlock(SStreamTask* pTask, int32_t* pNumOfBlocks, SStreamDataBlock* pRetrieveBlock, + SArray* pRes) { + SSDataBlock block = {0}; + int32_t num = taosArrayGetSize(pRetrieveBlock->blocks); + if (num != 1) { + stError("s-task:%s invalid retrieve block number:%d, ignore", pTask->id.idStr, num); + return TSDB_CODE_INVALID_PARA; + } + + void* p = taosArrayGet(pRetrieveBlock->blocks, 0); + int32_t code = assignOneDataBlock(&block, p); + if (code) { + stError("s-task:%s failed to assign retrieve block, code:%s", pTask->id.idStr, tstrerror(code)); + return code; + } + + block.info.type = STREAM_PULL_OVER; + block.info.childId = pTask->info.selfChildId; + + p = taosArrayPush(pRes, &block); + if (p != NULL) { + (*pNumOfBlocks) += 1; + stDebug("s-task:%s(child %d) retrieve res from upstream completed, QID:0x%" PRIx64, pTask->id.idStr, + pTask->info.selfChildId, pRetrieveBlock->reqId); + } else { + code = terrno; + stError("s-task:%s failed to append pull over block for retrieve data, QID:0x%" PRIx64" code:%s", pTask->id.idStr, + pRetrieveBlock->reqId, tstrerror(code)); + } + + return code; +} + int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) { - int32_t code = TSDB_CODE_SUCCESS; - void* pExecutor = pTask->exec.pExecutor; int32_t size = 0; int32_t numOfBlocks = 0; + int32_t code = TSDB_CODE_SUCCESS; + void* pExecutor = pTask->exec.pExecutor; SArray* pRes = NULL; *totalBlocks = 0; *totalSize = 0; while (1) { + SSDataBlock* output = NULL; + uint64_t ts = 0; + if (pRes == NULL) { pRes = taosArrayInit(4, sizeof(SSDataBlock)); } @@ -115,8 +152,6 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* return code; } - SSDataBlock* output = NULL; - uint64_t ts = 0; if ((code = qExecTask(pExecutor, &output, &ts)) < 0) { if (code == TSDB_CODE_QRY_IN_EXEC) { resetTaskInfo(pExecutor); @@ -124,6 +159,7 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* if (code == TSDB_CODE_OUT_OF_MEMORY || code == TSDB_CODE_INVALID_PARA || code == TSDB_CODE_FILE_CORRUPTED) { stFatal("s-task:%s failed to continue execute since %s", pTask->id.idStr, tstrerror(code)); + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); return code; } else { qResetTaskCode(pExecutor); @@ -133,33 +169,11 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { - SSDataBlock block = {0}; - const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem; - - int32_t num = taosArrayGetSize(pRetrieveBlock->blocks); - if (num != 1) { - stError("s-task:%s invalid retrieve block number:%d, ignore", pTask->id.idStr, num); - continue; - } - - code = assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); - if (code) { - stError("s-task:%s failed to copy datablock, code:%s", pTask->id.idStr, tstrerror(code)); - continue; - } - - block.info.type = STREAM_PULL_OVER; - block.info.childId = pTask->info.selfChildId; - - void* p = taosArrayPush(pRes, &block); - if (p != NULL) { - numOfBlocks += 1; - } else { - stError("s-task:%s failed to add retrieve block", pTask->id.idStr); - } - - stDebug("s-task:%s(child %d) retrieve process completed,QID:0x%" PRIx64 " dump results", pTask->id.idStr, - pTask->info.selfChildId, pRetrieveBlock->reqId); + code = doAppendPullOverBlock(pTask, &numOfBlocks, (SStreamDataBlock*) pItem, pRes); + if (code) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + return code; + } } break; @@ -189,11 +203,11 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* void* p = taosArrayPush(pRes, &block); if (p == NULL) { stError("s-task:%s failed to add computing results, the final res may be incorrect", pTask->id.idStr); + } else { + stDebug("s-task:%s (child %d) executed and get %d result blocks, size:%.2fMiB", pTask->id.idStr, + pTask->info.selfChildId, numOfBlocks, SIZE_IN_MiB(size)); } - stDebug("s-task:%s (child %d) executed and get %d result blocks, size:%.2fMiB", pTask->id.idStr, - pTask->info.selfChildId, numOfBlocks, SIZE_IN_MiB(size)); - // current output should be dispatched to down stream nodes if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || size >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) { code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks); @@ -303,7 +317,7 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { bool finished = false; const char* id = pTask->id.idStr; - if(pTask->info.taskLevel != TASK_LEVEL__SOURCE) { + if (pTask->info.taskLevel != TASK_LEVEL__SOURCE) { stError("s-task:%s not source scan-history task, not exec, quit", pTask->id.idStr); return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0); } @@ -408,7 +422,7 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) { } } else { if (!(status == TASK_STATUS__READY || status == TASK_STATUS__PAUSE || status == TASK_STATUS__DROPPING || - status == TASK_STATUS__STOP)) { + status == TASK_STATUS__STOP)) { stError("s-task:%s invalid task status:%d", id, status); return TSDB_CODE_STREAM_INTERNAL_ERROR; } @@ -718,7 +732,7 @@ int32_t flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpoi // 2. flush data in executor to K/V store, which should be completed before do checkpoint in the K/V. int32_t code = doStreamTaskExecImpl(pTask, pCheckpointBlock, 1); - if(code) { + if (code) { stError("s-task:%s failed to exec stream task before checkpoint, code:%s", id, tstrerror(code)); } @@ -780,7 +794,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { // dispatch checkpoint msg to all downstream tasks int32_t type = pInput->type; if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) { - int32_t code = streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput); + code = streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput); if (code != 0) { stError("s-task:%s failed to process checkpoint-trigger block, code:%s", pTask->id.idStr, tstrerror(code)); } @@ -833,7 +847,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { if (pState.state == TASK_STATUS__CK) { stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name); code = streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue - } else { // todo refactor + } else { // todo refactor if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { code = streamTaskSendCheckpointSourceRsp(pTask); } else { diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index dde3595eb3..19391bf7a0 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -42,7 +42,7 @@ static bool waitForEnoughDuration(SMetaHbInfo* pInfo) { static bool existInHbMsg(SStreamHbMsg* pMsg, SDownstreamTaskEpset* pTaskEpset) { int32_t numOfExisted = taosArrayGetSize(pMsg->pUpdateNodes); - for (int k = 0; k < numOfExisted; ++k) { + for (int32_t k = 0; k < numOfExisted; ++k) { if (pTaskEpset->nodeId == *(int32_t*)taosArrayGet(pMsg->pUpdateNodes, k)) { return true; } @@ -56,7 +56,7 @@ static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) { streamMutexLock(&pTask->lock); int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList); - for (int j = 0; j < num; ++j) { + for (int32_t j = 0; j < num; ++j) { SDownstreamTaskEpset* pTaskEpset = taosArrayGet(pTask->outputInfo.pNodeEpsetUpdateList, j); bool exist = existInHbMsg(pMsg, pTaskEpset); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 7860dd9f0a..44c9e76906 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -476,23 +476,14 @@ _err: if (pMeta->pTasksMap) taosHashCleanup(pMeta->pTasksMap); if (pMeta->pTaskList) taosArrayDestroy(pMeta->pTaskList); if (pMeta->pTaskDb) { - int32_t ret = tdbTbClose(pMeta->pTaskDb); - if (ret) { - stError("vgId:%d tdb failed close task db, code:%s", pMeta->vgId, tstrerror(ret)); - } + tdbTbClose(pMeta->pTaskDb); pMeta->pTaskDb = NULL; } if (pMeta->pCheckpointDb) { - int32_t ret = tdbTbClose(pMeta->pCheckpointDb); - if (ret) { - stError("vgId:%d tdb failed close task checkpointDb, code:%s", pMeta->vgId, tstrerror(ret)); - } + tdbTbClose(pMeta->pCheckpointDb); } if (pMeta->db) { - int32_t ret = tdbClose(pMeta->db); - if (ret) { - stError("vgId:%d tdb failed close meta db, code:%s", pMeta->vgId, tstrerror(ret)); - } + tdbClose(pMeta->db); } if (pMeta->pHbInfo) taosMemoryFreeClear(pMeta->pHbInfo); @@ -598,18 +589,9 @@ void streamMetaCloseImpl(void* arg) { // already log the error, ignore here tdbAbort(pMeta->db, pMeta->txn); - code = tdbTbClose(pMeta->pTaskDb); - if (code) { - stError("vgId:%d failed to close taskDb, code:%s", vgId, tstrerror(code)); - } - code = tdbTbClose(pMeta->pCheckpointDb); - if (code) { - stError("vgId:%d failed to close checkpointDb, code:%s", vgId, tstrerror(code)); - } - code = tdbClose(pMeta->db); - if (code) { - stError("vgId:%d failed to close db, code:%s", vgId, tstrerror(code)); - } + tdbTbClose(pMeta->pTaskDb); + tdbTbClose(pMeta->pCheckpointDb); + tdbClose(pMeta->db); taosArrayDestroy(pMeta->pTaskList); taosArrayDestroy(pMeta->chkpSaved); diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index 63e24b0975..98920e6f70 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -83,12 +83,14 @@ void streamTaskResumeInFuture(SStreamTask* pTask) { ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void streamTaskResumeHelper(void* param, void* tmrId) { - SStreamTask* pTask = (SStreamTask*)param; - SStreamTaskId* pId = &pTask->id; - SStreamTaskState p = streamTaskGetStatus(pTask); + SStreamTask* pTask = (SStreamTask*)param; + SStreamTaskId* pId = &pTask->id; + SStreamTaskState p = streamTaskGetStatus(pTask); + int32_t code = 0; if (p.state == TASK_STATUS__DROPPING || p.state == TASK_STATUS__STOP) { int8_t status = streamTaskSetSchedStatusInactive(pTask); + TAOS_UNUSED(status); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s status:%s not resume task, ref:%d", pId->idStr, p.name, ref); @@ -97,13 +99,12 @@ void streamTaskResumeHelper(void* param, void* tmrId) { return; } - int32_t code = streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pId->streamId, pId->taskId, STREAM_EXEC_T_RESUME_TASK); + code = streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pId->streamId, pId->taskId, STREAM_EXEC_T_RESUME_TASK); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); if (code) { stError("s-task:%s sched task failed, code:%s, ref:%d", pId->idStr, tstrerror(code), ref); } else { - stDebug("trigger to resume s-task:%s after being idled for %dms, ref:%d", pId->idStr, pTask->status.schedIdleTime, - ref); + stDebug("trigger to resume s-task:%s after idled for %dms, ref:%d", pId->idStr, pTask->status.schedIdleTime, ref); // release the task ref count streamTaskClearSchedIdleInfo(pTask); diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 23598cf717..7e3d8d59f9 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -1060,7 +1060,7 @@ _end: return code; } -int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen) { +int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen) { SSessionKey* pWinKey = pKey; const TSKEY gap = 0; int32_t code = TSDB_CODE_SUCCESS; @@ -1082,21 +1082,27 @@ int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey int32_t size = taosArrayGetSize(pWinStates); if (size == 0) { void* pFileStore = getStateFileStore(pFileState); - void* p = NULL; + void* pRockVal = NULL; - int32_t code_file = getCountWinStateFromDisc(pFileStore, pWinKey, &p, pVLen); + int32_t code_file = getCountWinStateFromDisc(pFileStore, pWinKey, &pRockVal, pVLen); if (code_file == TSDB_CODE_SUCCESS && isFlushedState(pFileState, endTs, 0)) { - (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); - if (!(*pVal)) { - code = TSDB_CODE_OUT_OF_MEMORY; + int32_t valSize = *pVLen; + COUNT_TYPE* pWinStateCount = (COUNT_TYPE*)((char*)(pRockVal) + (valSize - sizeof(COUNT_TYPE))); + if ((*pWinStateCount) == winCount) { + code = addNewSessionWindow(pFileState, pWinStates, pWinKey, (SRowBuffPos**)pVal); QUERY_CHECK_CODE(code, lino, _end); - } - - qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, + } else { + (*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen); + if (!(*pVal)) { + code = TSDB_CODE_OUT_OF_MEMORY; + QUERY_CHECK_CODE(code, lino, _end); + } + qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); + } } else { code = addNewSessionWindow(pFileState, pWinStates, pWinKey, (SRowBuffPos**)pVal); - taosMemoryFree(p); + taosMemoryFree(pRockVal); QUERY_CHECK_CODE(code, lino, _end); } } else { diff --git a/source/libs/stream/src/streamStartHistory.c b/source/libs/stream/src/streamStartHistory.c index b376dbd16b..4d7bf2ba87 100644 --- a/source/libs/stream/src/streamStartHistory.c +++ b/source/libs/stream/src/streamStartHistory.c @@ -64,7 +64,6 @@ static int32_t streamTaskSetReady(SStreamTask* pTask) { int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated) { SStreamScanHistoryReq req; - int32_t code = 0; initScanHistoryReq(pTask, &req, igUntreated); int32_t len = sizeof(SStreamScanHistoryReq); @@ -173,7 +172,7 @@ int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask) { code = streamTaskStartScanHistory(pTask); } - // NOTE: there will be an deadlock if launch fill history here. + // NOTE: there will be a deadlock if launch fill history here. // start the related fill-history task, when current task is ready // if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { // streamLaunchFillHistoryTask(pTask); @@ -219,7 +218,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { stDebug("s-task:%s start to launch related fill-history task:0x%" PRIx64 "-0x%x", idStr, hStreamId, hTaskId); - // Set the execute conditions, including the query time window and the version range + // Set the execution conditions, including the query time window and the version range streamMetaRLock(pMeta); SStreamTask** pHTask = taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); streamMetaRUnLock(pMeta); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index eb7254acba..1994c882aa 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -99,8 +99,8 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { } SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState)); stDebug("open stream state %p, %s", pState, path); @@ -170,11 +170,12 @@ int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* int32_t lino = 0; void* pVal = NULL; int32_t len = getRowStateRowSize(pState->pFileState); - code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), &pVal, &len); + int32_t tmpLen = len; + code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), &pVal, &tmpLen); QUERY_CHECK_CODE(code, lino, _end); - char* buf = ((SRowBuffPos*)pVal)->pRowBuff; - uint32_t rowSize = streamFileStateGetSelectRowSize(pState->pFileState); + char* buf = ((SRowBuffPos*)pVal)->pRowBuff; + int32_t rowSize = streamFileStateGetSelectRowSize(pState->pFileState); memcpy(buf + len - rowSize, value, vLen); _end: @@ -188,11 +189,12 @@ int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVa int32_t lino = 0; void* pVal = NULL; int32_t len = getRowStateRowSize(pState->pFileState); - code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), (void**)(&pVal), &len); + int32_t tmpLen = len; + code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), (void**)(&pVal), &tmpLen); QUERY_CHECK_CODE(code, lino, _end); - char* buf = ((SRowBuffPos*)pVal)->pRowBuff; - uint32_t rowSize = streamFileStateGetSelectRowSize(pState->pFileState); + char* buf = ((SRowBuffPos*)pVal)->pRowBuff; + int32_t rowSize = streamFileStateGetSelectRowSize(pState->pFileState); *ppVal = buf + len - rowSize; streamStateReleaseBuf(pState, pVal, false); @@ -543,6 +545,6 @@ int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey return getCountWinResultBuff(pState->pFileState, pKey, winCount, ppVal, pVLen, pWinCode); } -int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen) { - return createCountWinResultBuff(pState->pFileState, pKey, pVal, pVLen); +int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen) { + return createCountWinResultBuff(pState->pFileState, pKey, winCount, pVal, pVLen); } diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 9f8523b040..cf5f1b2b91 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -100,7 +100,7 @@ void* intervalCreateStateKey(SRowBuffPos* pPos, int64_t num) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); return NULL; } - SWinKey* pWinKey = pPos->pKey; + SWinKey* pWinKey = pPos->pKey; pStateKey->key = *pWinKey; pStateKey->opNum = num; return pStateKey; @@ -120,7 +120,7 @@ void* sessionCreateStateKey(SRowBuffPos* pPos, int64_t num) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); return NULL; } - SSessionKey* pWinKey = pPos->pKey; + SSessionKey* pWinKey = pPos->pKey; pStateKey->key = *pWinKey; pStateKey->opNum = num; return pStateKey; @@ -445,7 +445,9 @@ _end: } int32_t clearRowBuff(SStreamFileState* pFileState) { - clearExpiredRowBuff(pFileState, pFileState->maxTs - pFileState->deleteMark, false); + if (pFileState->deleteMark != INT64_MAX) { + clearExpiredRowBuff(pFileState, pFileState->maxTs - pFileState->deleteMark, false); + } if (isListEmpty(pFileState->freeBuffs)) { return flushRowBuff(pFileState); } @@ -734,7 +736,7 @@ void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, boo // todo handle failure memset(buf, 0, len); } - taosMemoryFree(buf); + taosMemoryFreeClear(buf); int32_t numOfElems = streamStateGetBatchSize(batch); if (numOfElems > 0) { @@ -769,6 +771,7 @@ _end: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } + taosMemoryFree(buf); streamStateDestroyBatch(batch); } @@ -850,7 +853,7 @@ int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { if (winRes != TSDB_CODE_SUCCESS) { break; } - + if (vlen != pFileState->rowSize) { code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; QUERY_CHECK_CODE(code, lino, _end); diff --git a/source/libs/stream/test/checkpointTest.cpp b/source/libs/stream/test/checkpointTest.cpp index 34e80fc08b..c14343c92b 100644 --- a/source/libs/stream/test/checkpointTest.cpp +++ b/source/libs/stream/test/checkpointTest.cpp @@ -43,7 +43,6 @@ // } // strcpy(tsSnodeAddress, "127.0.0.1"); // int ret = RUN_ALL_TESTS(); -// s3CleanUp(); // return ret; // } diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index de83c51211..9f6acf6d83 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -728,6 +728,8 @@ int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTe sDebug("vgId:%d, get response info, seqNum:%" PRId64 ", num:%d", pNode->vgId, cbMeta.seqNum, num); code = pFsm->FpCommitCb(pFsm, &rpcMsg, &cbMeta); retry = (code != 0) && (terrno == TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE); + sDebug("vgId:%d, fsm execute, index:%" PRId64 ", term:%" PRId64 ", type:%s, code:%d, retry:%d", pNode->vgId, + pEntry->index, pEntry->term, TMSG_INFO(pEntry->originalRpcType), code, retry); if (retry) { taosMsleep(10); sError("vgId:%d, retry on fsm commit since %s. index:%" PRId64, pNode->vgId, tstrerror(code), pEntry->index); @@ -1139,10 +1141,15 @@ int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncApp pMgr->peerStartTime = pMsg->startTime; } + int32_t code = 0; if (pMgr->restored) { - TAOS_CHECK_RETURN(syncLogReplContinue(pMgr, pNode, pMsg)); + if ((code = syncLogReplContinue(pMgr, pNode, pMsg)) != 0) { + sError("vgId:%d, failed to continue sync log repl since %s", pNode->vgId, tstrerror(code)); + } } else { - TAOS_CHECK_RETURN(syncLogReplRecover(pMgr, pNode, pMsg)); + if ((code = syncLogReplRecover(pMgr, pNode, pMsg)) != 0) { + sError("vgId:%d, failed to recover sync log repl since %s", pNode->vgId, tstrerror(code)); + } } (void)taosThreadMutexUnlock(&pBuf->mutex); return 0; @@ -1439,7 +1446,10 @@ int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode) { if (lastVer != pBuf->matchIndex) return TSDB_CODE_SYN_INTERNAL_ERROR; SyncIndex index = pBuf->endIndex - 1; - TAOS_CHECK_RETURN(syncLogBufferRollback(pBuf, pNode, pBuf->matchIndex + 1)); + int32_t code = 0; + if ((code = syncLogBufferRollback(pBuf, pNode, pBuf->matchIndex + 1)) != 0) { + sError("vgId:%d, failed to rollback sync log buffer since %s", pNode->vgId, tstrerror(code)); + } sInfo("vgId:%d, reset sync log buffer. buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index ad9af57abf..52ff749191 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -34,7 +34,7 @@ typedef struct STxn TXN; // TDB int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb, int8_t rollback, int32_t encryptAlgorithm, char *encryptKey); -int32_t tdbClose(TDB *pDb); +void tdbClose(TDB *pDb); int32_t tdbBegin(TDB *pDb, TXN **pTxn, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, int flags); int32_t tdbCommit(TDB *pDb, TXN *pTxn); @@ -46,7 +46,7 @@ int32_t tdbAlter(TDB *pDb, int pages); // TTB int32_t tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb, int8_t rollback); -int32_t tdbTbClose(TTB *pTb); +void tdbTbClose(TTB *pTb); bool tdbTbExist(const char *tbname, TDB *pEnv); int tdbTbDropByName(const char *tbname, TDB *pEnv, TXN *pTxn); int32_t tdbTbDrop(TTB *pTb); diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index eb2603bdd8..02ab997f69 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -90,7 +90,7 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i return 0; } -int tdbClose(TDB *pDb) { +void tdbClose(TDB *pDb) { SPager *pPager; if (pDb) { @@ -109,7 +109,7 @@ int tdbClose(TDB *pDb) { tdbOsFree(pDb); } - return 0; + return; } int32_t tdbAlter(TDB *pDb, int pages) { return tdbPCacheAlter(pDb->pCache, pages); } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index d41f4c2e49..6dc6aa0940 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -130,12 +130,12 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF return 0; } -int tdbTbClose(TTB *pTb) { +void tdbTbClose(TTB *pTb) { if (pTb) { tdbBtreeClose(pTb->pBt); tdbOsFree(pTb); } - return 0; + return; } bool tdbTbExist(const char *tbname, TDB *pEnv) { diff --git a/source/libs/tdb/test/tdbExOVFLTest.cpp b/source/libs/tdb/test/tdbExOVFLTest.cpp index a2deba4696..388a812ff5 100644 --- a/source/libs/tdb/test/tdbExOVFLTest.cpp +++ b/source/libs/tdb/test/tdbExOVFLTest.cpp @@ -197,8 +197,7 @@ static void insertOfp(void) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); } // TEST(TdbOVFLPagesTest, DISABLED_TbInsertTest) { @@ -247,8 +246,7 @@ TEST(TdbOVFLPagesTest, TbGetTest) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); } // TEST(TdbOVFLPagesTest, DISABLED_TbDeleteTest) { @@ -357,8 +355,7 @@ tdbBegin(pEnv, &txn); tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); } // TEST(tdb_test, DISABLED_simple_insert1) { @@ -492,6 +489,5 @@ TEST(tdb_test, simple_insert1) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); } diff --git a/source/libs/tdb/test/tdbPageDefragmentTest.cpp b/source/libs/tdb/test/tdbPageDefragmentTest.cpp index b64517c787..85b5c6e6b3 100644 --- a/source/libs/tdb/test/tdbPageDefragmentTest.cpp +++ b/source/libs/tdb/test/tdbPageDefragmentTest.cpp @@ -468,8 +468,7 @@ TEST(TdbPageDefragmentTest, DISABLED_simple_insert1) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); } // TEST(TdbPageDefragmentTest, DISABLED_seq_insert) { @@ -551,8 +550,7 @@ TEST(TdbPageDefragmentTest, seq_insert) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); } // TEST(TdbPageDefragmentTest, DISABLED_seq_delete) { @@ -635,8 +633,7 @@ TEST(TdbPageDefragmentTest, seq_delete) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); } // TEST(TdbPageDefragmentTest, DISABLED_defragment_insert) { @@ -717,6 +714,5 @@ TEST(TdbPageDefragmentTest, defragment_insert) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); } diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp index bbbd90e73d..30e1081d0c 100644 --- a/source/libs/tdb/test/tdbPageRecycleTest.cpp +++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp @@ -123,7 +123,7 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) { TDB *pEnv = NULL; - int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0 , 0, NULL); + int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0, 0, NULL); if (ret) { pEnv = NULL; } @@ -187,8 +187,7 @@ static void insertOfp(void) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); } static void clearDb(char const *db) { taosRemoveDir(db); } @@ -471,8 +470,7 @@ TEST(TdbPageRecycleTest, DISABLED_simple_insert1) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); } static void insertDb(int nData) { @@ -537,8 +535,7 @@ static void insertDb(int nData) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); system("ls -l ./tdb"); } @@ -607,8 +604,7 @@ static void deleteDb(int nData) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); system("ls -l ./tdb"); } @@ -675,8 +671,7 @@ static void deleteOfp(void) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); } // TEST(TdbPageRecycleTest, DISABLED_seq_delete_ofp) { @@ -761,8 +756,7 @@ TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_nocommit) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); system("ls -l ./tdb"); } @@ -828,8 +822,7 @@ TEST(TdbPageRecycleTest, recycly_delete_interior_ofp_nocommit) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); system("ls -l ./tdb"); } diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index e358ac0197..a8fdce2c38 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -231,8 +231,7 @@ TEST(tdb_test, DISABLED_simple_insert1) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); } TEST(tdb_test, DISABLED_simple_insert2) { @@ -315,8 +314,7 @@ TEST(tdb_test, DISABLED_simple_insert2) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); } TEST(tdb_test, DISABLED_simple_delete1) { @@ -620,8 +618,7 @@ TEST(tdb_test, multi_thread_query) { tdbTbClose(pDb); // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); } TEST(tdb_test, DISABLED_multi_thread1) { @@ -745,7 +742,6 @@ TEST(tdb_test, DISABLED_multi_thread1) { tdbTbClose(pTb); // Close Env - ret = tdbClose(pDb); - GTEST_ASSERT_EQ(ret, 0); + tdbClose(pEnv); #endif } diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index a353bd7941..51fda85f0a 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -295,7 +295,7 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) { // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dirname.3.html if ((dir = taosStrdup(taosDirName(s))) == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } if (strlen(dir) >= strlen(rname)) { // TODO: check if it is necessary for equal length diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 573b59e30c..7d7868f3cd 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -236,7 +236,7 @@ static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, static void* httpThread(void* arg) { SHttpModule* http = (SHttpModule*)arg; setThreadName("http-cli-send-thread"); - (void)uv_run(http->loop, UV_RUN_DEFAULT); + TAOS_UNUSED(uv_run(http->loop, UV_RUN_DEFAULT)); return NULL; } @@ -266,16 +266,23 @@ static int32_t httpCreateMsg(const char* server, const char* uri, uint16_t port, msg->server = taosStrdup(server); msg->uri = taosStrdup(uri); msg->cont = taosMemoryMalloc(contLen); - if (qid != NULL) - msg->qid = taosStrdup(qid); - else - msg->qid = NULL; if (msg->server == NULL || msg->uri == NULL || msg->cont == NULL) { httpDestroyMsg(msg); *httpMsg = NULL; return terrno; } + if (qid != NULL) { + msg->qid = taosStrdup(qid); + if (msg->qid == NULL) { + httpDestroyMsg(msg); + *httpMsg = NULL; + return terrno; + } + } else { + msg->qid = NULL; + } + memcpy(msg->cont, pCont, contLen); msg->len = contLen; msg->flag = flag; @@ -352,7 +359,9 @@ static void httpAsyncCb(uv_async_t* handle) { static int32_t BATCH_SIZE = 20; int32_t count = 0; - (void)taosThreadMutexLock(&item->mtx); + if ((taosThreadMutexLock(&item->mtx)) != 0) { + tError("http-report failed to lock mutex"); + } httpMayDiscardMsg(http, item); while (!QUEUE_IS_EMPTY(&item->qmsg) && count++ < BATCH_SIZE) { @@ -360,7 +369,9 @@ static void httpAsyncCb(uv_async_t* handle) { QUEUE_REMOVE(h); QUEUE_PUSH(&wq, h); } - (void)taosThreadMutexUnlock(&item->mtx); + if (taosThreadMutexUnlock(&item->mtx) != 0) { + tError("http-report failed to unlock mutex"); + } httpTrace(&wq); @@ -393,7 +404,7 @@ static FORCE_INLINE void clientCloseCb(uv_handle_t* handle) { SHttpModule* http = taosAcquireRef(httpRefMgt, cli->chanId); if (http != NULL) { http->connNum -= 1; - (void)taosReleaseRef(httpRefMgt, chanId); + TAOS_UNUSED(taosReleaseRef(httpRefMgt, chanId)); } destroyHttpClient(cli); @@ -453,7 +464,7 @@ static void clientConnCb(uv_connect_t* req, int32_t status) { if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); } - (void)taosReleaseRef(httpRefMgt, chanId); + TAOS_UNUSED(taosReleaseRef(httpRefMgt, chanId)); return; } http->connNum += 1; @@ -468,7 +479,7 @@ static void clientConnCb(uv_connect_t* req, int32_t status) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); } } - (void)taosReleaseRef(httpRefMgt, chanId); + TAOS_UNUSED(taosReleaseRef(httpRefMgt, chanId)); } int32_t httpSendQuit(SHttpModule* http, int64_t chanId) { @@ -514,7 +525,7 @@ static void httpHandleQuit(SHttpMsg* msg) { SHttpModule* http = taosAcquireRef(httpRefMgt, chanId); if (http == NULL) return; uv_walk(http->loop, httpWalkCb, NULL); - (void)taosReleaseRef(httpRefMgt, chanId); + TAOS_UNUSED(taosReleaseRef(httpRefMgt, chanId)); } static bool httpFailFastShoudIgnoreMsg(SHashObj* pTable, char* server, int16_t port) { @@ -536,15 +547,15 @@ static bool httpFailFastShoudIgnoreMsg(SHashObj* pTable, char* server, int16_t p } static void httpFailFastMayUpdate(SHashObj* pTable, char* server, int16_t port, int8_t succ) { int32_t code = 0; - char buf[256] = {0}; + char buf[256] = {0}; sprintf(buf, "%s:%d", server, port); if (succ) { - (void)taosHashRemove(pTable, buf, strlen(buf)); + TAOS_UNUSED(taosHashRemove(pTable, buf, strlen(buf))); } else { int32_t st = taosGetTimestampSec(); if ((code = taosHashPut(pTable, buf, strlen(buf), &st, sizeof(st))) != 0) { - tError("http-report failed to update conn status, dst:%s, reason:%s", buf, tstrerror(code)); + tError("http-report failed to update conn status, dst:%s, reason:%s", buf, tstrerror(code)); } } return; @@ -628,7 +639,7 @@ static void httpHandleReq(SHttpMsg* msg) { tError("http-report failed to alloc read buf, dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ",reason:%s", cli->addr, cli->port, chanId, cli->seq, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); destroyHttpClient(cli); - (void)taosReleaseRef(httpRefMgt, chanId); + TAOS_UNUSED(taosReleaseRef(httpRefMgt, chanId)); return; } @@ -637,7 +648,7 @@ static void httpHandleReq(SHttpMsg* msg) { tError("http-report failed to init socket handle, dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ", reason:%s", cli->addr, cli->port, chanId, cli->seq, uv_strerror(err)); destroyHttpClient(cli); - (void)taosReleaseRef(httpRefMgt, chanId); + TAOS_UNUSED(taosReleaseRef(httpRefMgt, chanId)); return; } @@ -647,7 +658,7 @@ static void httpHandleReq(SHttpMsg* msg) { tError("http-report failed to open socket, dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ", reason:%s", cli->addr, cli->port, chanId, cli->seq, tstrerror(terrno)); destroyHttpClient(cli); - (void)taosReleaseRef(httpRefMgt, chanId); + TAOS_UNUSED(taosReleaseRef(httpRefMgt, chanId)); return; } @@ -657,7 +668,7 @@ static void httpHandleReq(SHttpMsg* msg) { cli->port, chanId, cli->seq, uv_strerror(ret)); destroyHttpClient(cli); - (void)taosReleaseRef(httpRefMgt, chanId); + TAOS_UNUSED(taosReleaseRef(httpRefMgt, chanId)); return; } @@ -666,9 +677,9 @@ static void httpHandleReq(SHttpMsg* msg) { tError("http-report failed to connect to http-server,dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ", reson:%s", cli->addr, cli->port, chanId, cli->seq, uv_strerror(ret)); httpFailFastMayUpdate(http->connStatusTable, cli->addr, cli->port, 0); - destroyHttpClient(cli); + uv_close((uv_handle_t*)&cli->tcp, httpDestroyClientCb); } - (void)taosReleaseRef(httpRefMgt, chanId); + TAOS_UNUSED(taosReleaseRef(httpRefMgt, chanId)); return; END: @@ -678,7 +689,7 @@ END: } httpDestroyMsg(msg); taosMemoryFree(header); - (void)taosReleaseRef(httpRefMgt, chanId); + TAOS_UNUSED(taosReleaseRef(httpRefMgt, chanId)); } static void httpModuleDestroy(SHttpModule* http) { @@ -689,7 +700,7 @@ static void httpModuleDestroy(SHttpModule* http) { transAsyncPoolDestroy(http->asyncPool); } if (http->loop) { - (void)uv_loop_close(http->loop); + TAOS_UNUSED(uv_loop_close(http->loop)); taosMemoryFree(http->loop); } @@ -754,7 +765,7 @@ int32_t taosSendHttpReport(const char* server, const char* uri, uint16_t port, c int32_t taosSendHttpReportWithQID(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag, const char* qid) { - (void)taosThreadOnce(&transHttpInit, transHttpEnvInit); + TAOS_UNUSED(taosThreadOnce(&transHttpInit, transHttpEnvInit)); return taosSendHttpReportImplByChan(server, uri, port, pCont, contLen, flag, httpDefaultChanId, qid); } @@ -825,7 +836,7 @@ _ERROR: return code; } int64_t taosInitHttpChan() { - (void)taosThreadOnce(&transHttpInit, transHttpEnvInit); + TAOS_UNUSED(taosThreadOnce(&transHttpInit, transHttpEnvInit)); return transInitHttpChanImpl(); } @@ -844,14 +855,16 @@ void taosDestroyHttpChan(int64_t chanId) { ret = httpSendQuit(load, chanId); if (ret != 0) { tDebug("http-report already destroyed, chanId %" PRId64 ",reason:%s", chanId, tstrerror(ret)); - (void)taosReleaseRef(httpRefMgt, chanId); + TAOS_UNUSED(taosReleaseRef(httpRefMgt, chanId)); return; } - (void)taosThreadJoin(load->thread, NULL); + if (taosThreadJoin(load->thread, NULL) != 0) { + tTrace("http-report failed to join thread, chanId %" PRId64 "", chanId); + } httpModuleDestroy(load); - (void)taosReleaseRef(httpRefMgt, chanId); - (void)taosRemoveRef(httpRefMgt, chanId); + TAOS_UNUSED(taosReleaseRef(httpRefMgt, chanId)); + TAOS_UNUSED(taosRemoveRef(httpRefMgt, chanId)); } diff --git a/source/libs/transport/src/tmsgcb.c b/source/libs/transport/src/tmsgcb.c index 619592c82c..4c969003a9 100644 --- a/source/libs/transport/src/tmsgcb.c +++ b/source/libs/transport/src/tmsgcb.c @@ -59,7 +59,9 @@ int32_t tmsgSendSyncReq(const SEpSet* epSet, SRpcMsg* pMsg) { void tmsgSendRsp(SRpcMsg* pMsg) { #if 1 - (void)rpcSendResponse(pMsg); + if (rpcSendResponse(pMsg) != 0) { + tError("failed to send response"); + } #else return (*defaultMsgCb.sendRspFp)(pMsg); #endif @@ -81,6 +83,6 @@ bool tmsgUpdateDnodeInfo(int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint1 void tmsgUpdateDnodeEpSet(SEpSet* epset) { for (int32_t i = 0; i < epset->numOfEps; ++i) { - (void)tmsgUpdateDnodeInfo(NULL, NULL, epset->eps[i].fqdn, &epset->eps[i].port); + bool ret = tmsgUpdateDnodeInfo(NULL, NULL, epset->eps[i].fqdn, &epset->eps[i].port); } } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 9ba1c3d677..9526c0ee9e 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -113,7 +113,7 @@ void* rpcOpen(const SRpcInit* pInit) { } int64_t refId = transAddExHandle(transGetInstMgt(), pRpc); - (void)transAcquireExHandle(transGetInstMgt(), refId); + void* tmp = transAcquireExHandle(transGetInstMgt(), refId); pRpc->refId = refId; return (void*)refId; _end: @@ -127,8 +127,9 @@ void rpcClose(void* arg) { if (arg == NULL) { return; } - (void)transRemoveExHandle(transGetInstMgt(), (int64_t)arg); - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)arg); + TAOS_UNUSED(transRemoveExHandle(transGetInstMgt(), (int64_t)arg)); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)arg)); + tInfo("end to close rpc"); return; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 1dc4f7bfcf..c0453c7759 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -233,11 +233,11 @@ static void destroyThrdObj(SCliThrd* pThrd); int32_t cliSendQuit(SCliThrd* thrd); static void cliWalkCb(uv_handle_t* handle, void* arg); -#define CLI_RELEASE_UV(loop) \ - do { \ - (void)uv_walk(loop, cliWalkCb, NULL); \ - (void)uv_run(loop, UV_RUN_DEFAULT); \ - (void)uv_loop_close(loop); \ +#define CLI_RELEASE_UV(loop) \ + do { \ + TAOS_UNUSED(uv_walk(loop, cliWalkCb, NULL)); \ + TAOS_UNUSED(uv_run(loop, UV_RUN_DEFAULT)); \ + TAOS_UNUSED(uv_loop_close(loop)); \ } while (0); // snprintf may cause performance problem @@ -247,7 +247,7 @@ static void cliWalkCb(uv_handle_t* handle, void* arg); int16_t len = strlen(ip); \ if (ip != NULL) memcpy(t, ip, len); \ t[len] = ':'; \ - (void)titoa(port, 10, &t[len + 1]); \ + TAOS_UNUSED(titoa(port, 10, &t[len + 1])); \ } while (0) #define CONN_PERSIST_TIME(para) ((para) <= 90000 ? 90000 : (para)) @@ -336,7 +336,7 @@ void cliResetTimer(SCliThrd* pThrd, SCliConn* conn) { if (conn->timer) { if (uv_is_active((uv_handle_t*)conn->timer)) { tDebug("%s conn %p stop timer", CONN_GET_INST_LABEL(conn), conn); - (void)uv_timer_stop(conn->timer); + TAOS_UNUSED(uv_timer_stop(conn->timer)); } if (taosArrayPush(pThrd->timerList, &conn->timer) == NULL) { tError("failed to push timer %p to list, reason:%s", conn->timer, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); @@ -375,15 +375,15 @@ bool cliConnSendSeqMsg(int64_t refId, SCliConn* conn) { taosWUnLockLatch(&exh->latch); SCliMsg* t = QUEUE_DATA(h, SCliMsg, seqq); transCtxMerge(&conn->ctx, &t->ctx->appCtx); - (void)transQueuePush(&conn->cliMsgs, t); + TAOS_UNUSED(transQueuePush(&conn->cliMsgs, t)); tDebug("pop from conn %p, refId: %" PRId64 "", conn, refId); - (void)transReleaseExHandle(transGetRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), refId)); cliSend(conn); return true; } taosWUnLockLatch(&exh->latch); tDebug("empty conn %p, refId: %" PRId64 "", conn, refId); - (void)transReleaseExHandle(transGetRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), refId)); return false; } @@ -493,10 +493,16 @@ void cliHandleResp(SCliConn* conn) { } if (CONN_NO_PERSIST_BY_APP(conn)) { + SExHandle* exh = transAcquireExHandle(transGetRefMgt(), refId); + if (exh != NULL) { + exh->handle = NULL; + } + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), refId)); + return addConnToPool(pThrd->pool, conn); } - (void)uv_read_start((uv_stream_t*)conn->stream, cliAllocRecvBufferCb, cliRecvCb); + TAOS_UNUSED(uv_read_start((uv_stream_t*)conn->stream, cliAllocRecvBufferCb, cliRecvCb)); } static void cliDestroyMsgInExhandle(int64_t refId) { if (refId == 0) return; @@ -510,7 +516,7 @@ static void cliDestroyMsgInExhandle(int64_t refId) { destroyCmsg(t); } taosWUnLockLatch(&exh->latch); - (void)transReleaseExHandle(transGetRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), refId)); } } @@ -598,7 +604,7 @@ void cliConnTimeout(uv_timer_t* handle) { tTrace("%s conn %p conn timeout, ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); - (void)uv_timer_stop(handle); + TAOS_UNUSED(uv_timer_stop(handle)); handle->data = NULL; cliResetTimer(pThrd, conn); @@ -610,7 +616,7 @@ void cliReadTimeoutCb(uv_timer_t* handle) { // set up timeout cb SCliConn* conn = handle->data; tTrace("%s conn %p timeout, ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); - (void)uv_read_stop(conn->stream); + TAOS_UNUSED(uv_read_stop(conn->stream)); cliHandleExceptImpl(conn, TSDB_CODE_RPC_TIMEOUT); } @@ -667,6 +673,10 @@ static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) { plist = taosHashGet(pool, key, klen); SMsgList* nList = taosMemoryCalloc(1, sizeof(SMsgList)); + if (nList == NULL) { + tError("failed to alloc memory for msg list, reason:%s", tstrerror(terrno)); + return NULL; + } QUEUE_INIT(&nList->msgQ); nList->numOfConn++; @@ -714,6 +724,11 @@ static SCliConn* getConnFromPool2(SCliThrd* pThrd, char* key, SCliMsg** pMsg) { plist = taosHashGet(pool, key, klen); SMsgList* nList = taosMemoryCalloc(1, sizeof(SMsgList)); + if (nList == NULL) { + tError("failed to alloc memory for msg list, reason:%s", tstrerror(terrno)); + return NULL; + } + QUEUE_INIT(&nList->msgQ); nList->numOfConn++; @@ -847,7 +862,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { pMsg->ctx->task = NULL; transCtxMerge(&conn->ctx, &pMsg->ctx->appCtx); - (void)transQueuePush(&conn->cliMsgs, pMsg); + TAOS_UNUSED(transQueuePush(&conn->cliMsgs, pMsg)); conn->status = ConnNormal; cliSend(conn); @@ -871,8 +886,8 @@ static void addConnToPool(void* pool, SCliConn* conn) { } static int32_t allocConnRef(SCliConn* conn, bool update) { if (update) { - (void)transReleaseExHandle(transGetRefMgt(), conn->refId); - (void)transRemoveExHandle(transGetRefMgt(), conn->refId); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), conn->refId)); + TAOS_UNUSED(transRemoveExHandle(transGetRefMgt(), conn->refId)); conn->refId = -1; } @@ -907,8 +922,8 @@ static int32_t allocConnRef(SCliConn* conn, bool update) { static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) { if (update) { - (void)transReleaseExHandle(transGetRefMgt(), conn->refId); - (void)transRemoveExHandle(transGetRefMgt(), conn->refId); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), conn->refId)); + TAOS_UNUSED(transRemoveExHandle(transGetRefMgt(), conn->refId)); conn->refId = -1; } SExHandle* exh = transAcquireExHandle(transGetRefMgt(), handle); @@ -924,7 +939,7 @@ static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) { tDebug("conn %p specified by %" PRId64 "", conn, handle); - (void)transReleaseExHandle(transGetRefMgt(), handle); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), handle)); return 0; } @@ -1004,7 +1019,7 @@ static int32_t cliCreateConn(SCliThrd* pThrd, SCliConn** pCliConn) { } tDebug("no available timer, create a timer %p", timer); - (void)uv_timer_init(pThrd->loop, timer); + TAOS_UNUSED(uv_timer_init(pThrd->loop, timer)); } timer->data = conn; @@ -1022,7 +1037,7 @@ static int32_t cliCreateConn(SCliThrd* pThrd, SCliConn** pCliConn) { conn->broken = false; transRefCliHandle(conn); - (void)atomic_add_fetch_32(&pThrd->connCount, 1); + TAOS_UNUSED(atomic_add_fetch_32(&pThrd->connCount, 1)); TAOS_CHECK_GOTO(allocConnRef(conn, false), NULL, _failed); @@ -1032,7 +1047,7 @@ _failed: if (conn) { taosMemoryFree(conn->stream); transReqQueueClear(&conn->wreqQueue); - (void)transDestroyBuffer(&conn->readBuf); + TAOS_UNUSED(transDestroyBuffer(&conn->readBuf)); transQueueDestroy(&conn->cliMsgs); } taosMemoryFree(conn); @@ -1059,8 +1074,9 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { } conn->list = NULL; - (void)transReleaseExHandle(transGetRefMgt(), conn->refId); - (void)transRemoveExHandle(transGetRefMgt(), conn->refId); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), conn->refId)); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), conn->refId)); + TAOS_UNUSED(transRemoveExHandle(transGetRefMgt(), conn->refId)); conn->refId = -1; if (conn->task != NULL) { @@ -1071,7 +1087,7 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { if (clear) { if (!uv_is_closing((uv_handle_t*)conn->stream)) { - (void)uv_read_stop(conn->stream); + TAOS_UNUSED(uv_read_stop(conn->stream)); uv_close((uv_handle_t*)conn->stream, cliDestroy); } } @@ -1084,11 +1100,12 @@ static void cliDestroy(uv_handle_t* handle) { SCliThrd* pThrd = conn->hostThrd; cliResetTimer(pThrd, conn); - (void)atomic_sub_fetch_32(&pThrd->connCount, 1); + TAOS_UNUSED(atomic_sub_fetch_32(&pThrd->connCount, 1)); if (conn->refId > 0) { - (void)transReleaseExHandle(transGetRefMgt(), conn->refId); - (void)transRemoveExHandle(transGetRefMgt(), conn->refId); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), conn->refId)); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), conn->refId)); + TAOS_UNUSED(transRemoveExHandle(transGetRefMgt(), conn->refId)); } taosMemoryFree(conn->dstAddr); taosMemoryFree(conn->stream); @@ -1097,7 +1114,7 @@ static void cliDestroy(uv_handle_t* handle) { tTrace("%s conn %p destroy successfully", CONN_GET_INST_LABEL(conn), conn); transReqQueueClear(&conn->wreqQueue); - (void)transDestroyBuffer(&conn->readBuf); + TAOS_UNUSED(transDestroyBuffer(&conn->readBuf)); taosMemoryFree(conn); } @@ -1106,7 +1123,7 @@ static bool cliHandleNoResp(SCliConn* conn) { if (!transQueueEmpty(&conn->cliMsgs)) { SCliMsg* pMsg = transQueueGet(&conn->cliMsgs, 0); if (REQUEST_NO_RESP(&pMsg->msg)) { - (void)transQueuePop(&conn->cliMsgs); + TAOS_UNUSED(transQueuePop(&conn->cliMsgs)); destroyCmsg(pMsg); res = true; } @@ -1153,7 +1170,7 @@ static void cliSendCb(uv_write_t* req, int status) { tTrace("%s conn %p no resp required", CONN_GET_INST_LABEL(pConn), pConn); return; } - (void)uv_read_start((uv_stream_t*)pConn->stream, cliAllocRecvBufferCb, cliRecvCb); + TAOS_UNUSED(uv_read_start((uv_stream_t*)pConn->stream, cliAllocRecvBufferCb, cliRecvCb)); } void cliSendBatch(SCliConn* pConn) { int32_t code = 0; @@ -1295,14 +1312,19 @@ void cliSend(SCliConn* pConn) { uv_timer_t* timer = taosArrayGetSize(pThrd->timerList) > 0 ? *(uv_timer_t**)taosArrayPop(pThrd->timerList) : NULL; if (timer == NULL) { timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); + if (timer == NULL) { + tError("failed to alloc timer since %s", tstrerror(terrno)); + } tDebug("no available timer, create a timer %p", timer); - (void)uv_timer_init(pThrd->loop, timer); + TAOS_UNUSED(uv_timer_init(pThrd->loop, timer)); + } + if (timer != NULL) { + timer->data = pConn; + pConn->timer = timer; } - timer->data = pConn; - pConn->timer = timer; tGTrace("%s conn %p start timer for msg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType)); - (void)uv_timer_start((uv_timer_t*)pConn->timer, cliReadTimeoutCb, TRANS_READ_TIMEOUT, 0); + TAOS_UNUSED(uv_timer_start((uv_timer_t*)pConn->timer, cliReadTimeoutCb, TRANS_READ_TIMEOUT, 0)); } if (pHead->comp == 0 && pMsg->info.compressed == 0 && pConn->clientIp != pConn->serverIp) { @@ -1319,6 +1341,11 @@ void cliSend(SCliConn* pConn) { uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); uv_write_t* req = transReqQueuePush(&pConn->wreqQueue); + if (req == NULL) { + tGError("%s conn %p failed to send msg:%s, errmsg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType), + tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + cliHandleExcept(pConn, -1); + } int status = uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); if (status != 0) { @@ -1402,8 +1429,7 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { tTrace("%s conn %p try to connect to %s", pTransInst->label, conn, pList->dst); int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT * 10); if (fd == -1) { - tError("%s conn %p failed to create socket, reason:%s", transLabel(pTransInst), conn, - tstrerror(terrno)); + tError("%s conn %p failed to create socket, reason:%s", transLabel(pTransInst), conn, tstrerror(terrno)); cliHandleFastFail(conn, -1); return; } @@ -1428,7 +1454,7 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { cliHandleFastFail(conn, -1); return; } - (void)uv_timer_start(conn->timer, cliConnTimeout, TRANS_CONN_TIMEOUT, 0); + TAOS_UNUSED(uv_timer_start(conn->timer, cliConnTimeout, TRANS_CONN_TIMEOUT, 0)); return; } @@ -1541,12 +1567,12 @@ void cliConnCb(uv_connect_t* req, int status) { struct sockaddr peername, sockname; int addrlen = sizeof(peername); - (void)uv_tcp_getpeername((uv_tcp_t*)pConn->stream, &peername, &addrlen); - (void)transSockInfo2Str(&peername, pConn->dst); + TAOS_UNUSED(uv_tcp_getpeername((uv_tcp_t*)pConn->stream, &peername, &addrlen)); + TAOS_UNUSED(transSockInfo2Str(&peername, pConn->dst)); addrlen = sizeof(sockname); - (void)uv_tcp_getsockname((uv_tcp_t*)pConn->stream, &sockname, &addrlen); - (void)transSockInfo2Str(&sockname, pConn->src); + TAOS_UNUSED(uv_tcp_getsockname((uv_tcp_t*)pConn->stream, &sockname, &addrlen)); + TAOS_UNUSED(transSockInfo2Str(&sockname, pConn->src)); struct sockaddr_in addr = *(struct sockaddr_in*)&sockname; struct sockaddr_in saddr = *(struct sockaddr_in*)&peername; @@ -1596,8 +1622,8 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd) { tDebug("cli work thread %p start to quit", pThrd); destroyCmsg(pMsg); - (void)destroyConnPool(pThrd); - (void)uv_walk(pThrd->loop, cliWalkCb, NULL); + TAOS_UNUSED(destroyConnPool(pThrd)); + TAOS_UNUSED(uv_walk(pThrd->loop, cliWalkCb, NULL)); } static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { int64_t refId = (int64_t)(pMsg->msg.info.handle); @@ -1611,8 +1637,7 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { taosRLockLatch(&exh->latch); SCliConn* conn = exh->handle; taosRUnLockLatch(&exh->latch); - - (void)transReleaseExHandle(transGetRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), refId)); tDebug("%s conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn); if (T_REF_VAL_GET(conn) == 2) { @@ -1636,7 +1661,7 @@ static void cliHandleFreeById(SCliMsg* pMsg, SCliThrd* pThrd) { int64_t refId = (int64_t)(pMsg->msg.info.handle); SExHandle* exh = transAcquireExHandle(transGetRefMgt(), refId); if (exh == NULL) { - tDebug("id %" PRId64 " already released", refId); + tDebug("refId %" PRId64 " already released", refId); destroyCmsg(pMsg); return; } @@ -1648,7 +1673,7 @@ static void cliHandleFreeById(SCliMsg* pMsg, SCliThrd* pThrd) { if (conn == NULL || conn->refId != refId) { TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, NULL, _exception); } - tDebug("do free conn %p by id %" PRId64 "", conn, refId); + tDebug("do free conn %p by refId %" PRId64 "", conn, refId); int32_t size = transQueueSize(&conn->cliMsgs); if (size == 0) { @@ -1656,7 +1681,7 @@ static void cliHandleFreeById(SCliMsg* pMsg, SCliThrd* pThrd) { TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, NULL, _exception); } else { destroyCmsg(pMsg); - (void)transReleaseExHandle(transGetRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), refId)); while (T_REF_VAL_GET(conn) >= 1) { transUnrefCliHandle(conn); @@ -1666,9 +1691,9 @@ static void cliHandleFreeById(SCliMsg* pMsg, SCliThrd* pThrd) { _exception: tDebug("already free conn %p by id %" PRId64 "", conn, refId); - (void)transReleaseExHandle(transGetRefMgt(), refId); - (void)transReleaseExHandle(transGetRefMgt(), refId); - (void)transRemoveExHandle(transGetRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), refId)); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), refId)); + TAOS_UNUSED(transRemoveExHandle(transGetRefMgt(), refId)); destroyCmsg(pMsg); } @@ -1691,7 +1716,7 @@ SCliConn* cliGetConn(SCliMsg** pMsg, SCliThrd* pThrd, bool* ignore, char* addr) conn = getConnFromPool2(pThrd, addr, pMsg); if (conn != NULL) specifyConnRef(conn, true, refId); } - (void)transReleaseExHandle(transGetRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), refId)); } return conn; }; @@ -1787,7 +1812,7 @@ static void cliMayUpdateFqdnCache(SHashObj* cache, char* dst) { if (i > 0) { char fqdn[TSDB_FQDN_LEN + 1] = {0}; memcpy(fqdn, dst, i); - (void)cliUpdateFqdnCache(cache, fqdn); + TAOS_UNUSED(cliUpdateFqdnCache(cache, fqdn)); } } @@ -1824,7 +1849,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { if (ignore == true) { // persist conn already release by server STransMsg resp = {0}; - (void)cliBuildExceptResp(pMsg, &resp); + TAOS_UNUSED(cliBuildExceptResp(pMsg, &resp)); // refactorr later resp.info.cliVer = pTransInst->compatibilityVer; @@ -1841,14 +1866,14 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { if (conn != NULL) { transCtxMerge(&conn->ctx, &pMsg->ctx->appCtx); - (void)transQueuePush(&conn->cliMsgs, pMsg); + TAOS_UNUSED(transQueuePush(&conn->cliMsgs, pMsg)); cliSend(conn); } else { code = cliCreateConn(pThrd, &conn); if (code != 0) { tError("%s failed to create conn, reason:%s", pTransInst->label, tstrerror(code)); STransMsg resp = {.code = code}; - (void)cliBuildExceptResp(pMsg, &resp); + TAOS_UNUSED(cliBuildExceptResp(pMsg, &resp)); resp.info.cliVer = pTransInst->compatibilityVer; if (pMsg->type != Release) { @@ -1862,9 +1887,13 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { if (refId != 0) specifyConnRef(conn, true, refId); transCtxMerge(&conn->ctx, &pMsg->ctx->appCtx); - (void)transQueuePush(&conn->cliMsgs, pMsg); + TAOS_UNUSED(transQueuePush(&conn->cliMsgs, pMsg)); conn->dstAddr = taosStrdup(addr); + if (conn->dstAddr == NULL) { + tGError("%s conn %p failed to create socket, reason:%s", transLabel(pTransInst), conn, tstrerror(terrno)); + cliHandleExcept(conn, -1); + } uint32_t ipaddr; int32_t code = cliGetIpFromFqdnCache(pThrd->fqdn2ipCache, fqdn, &ipaddr); @@ -1882,8 +1911,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { tGTrace("%s conn %p try to connect to %s", pTransInst->label, conn, conn->dstAddr); int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT * 10); if (fd == -1) { - tGError("%s conn %p failed to create socket, reason:%s", transLabel(pTransInst), conn, - tstrerror(terrno)); + tGError("%s conn %p failed to create socket, reason:%s", transLabel(pTransInst), conn, tstrerror(terrno)); cliHandleExcept(conn, -1); terrno = 0; return; @@ -1911,7 +1939,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { cliHandleFastFail(conn, ret); return; } - (void)uv_timer_start(conn->timer, cliConnTimeout, TRANS_CONN_TIMEOUT, 0); + TAOS_UNUSED(uv_timer_start(conn->timer, cliConnTimeout, TRANS_CONN_TIMEOUT, 0)); } tGTrace("%s conn %p ready", pTransInst->label, conn); } @@ -2110,9 +2138,15 @@ static void cliAsyncCb(uv_async_t* handle) { // batch process to avoid to lock/unlock frequently queue wq; - (void)taosThreadMutexLock(&item->mtx); + if (taosThreadMutexLock(&item->mtx) != 0) { + tError("failed to lock mutex, reason:%s", tstrerror(terrno)); + } + QUEUE_MOVE(&item->qmsg, &wq); - (void)taosThreadMutexUnlock(&item->mtx); + + if (taosThreadMutexUnlock(&item->mtx) != 0) { + tError("failed to unlock mutex, reason:%s", tstrerror(terrno)); + } int8_t supportBatch = pTransInst->supportBatch; if (supportBatch == 0) { @@ -2162,7 +2196,7 @@ bool cliRecvReleaseReq(SCliConn* conn, STransMsgHead* pHead) { tDebug("%s conn %p receive release request, refId:%" PRId64 ", may ignore", CONN_GET_INST_LABEL(conn), conn, conn->refId); - (void)transClearBuffer(&conn->readBuf); + TAOS_UNUSED(transClearBuffer(&conn->readBuf)); transFreeMsg(transContFromHead((char*)pHead)); for (int i = 0; ahandle == 0 && i < transQueueSize(&conn->cliMsgs); i++) { @@ -2192,10 +2226,10 @@ static void* cliWorkThread(void* arg) { SCliThrd* pThrd = (SCliThrd*)arg; pThrd->pid = taosGetSelfPthreadId(); - (void)strtolower(threadName, pThrd->pTransInst->label); + TAOS_UNUSED(strtolower(threadName, pThrd->pTransInst->label)); setThreadName(threadName); - (void)uv_run(pThrd->loop, UV_RUN_DEFAULT); + TAOS_UNUSED(uv_run(pThrd->loop, UV_RUN_DEFAULT)); tDebug("thread quit-thread:%08" PRId64, pThrd->pid); return NULL; @@ -2239,7 +2273,7 @@ _err: if (cli) { for (int i = 0; i < cli->numOfThreads; i++) { if (cli->pThreadObj[i]) { - (void)cliSendQuit(cli->pThreadObj[i]); + TAOS_UNUSED(cliSendQuit(cli->pThreadObj[i])); destroyThrdObj(cli->pThreadObj[i]); } } @@ -2283,8 +2317,8 @@ static FORCE_INLINE void destroyCmsgAndAhandle(void* param) { } if (pMsg->msg.info.handle != 0) { - (void)transReleaseExHandle(transGetRefMgt(), (int64_t)pMsg->msg.info.handle); - (void)transRemoveExHandle(transGetRefMgt(), (int64_t)pMsg->msg.info.handle); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), (int64_t)pMsg->msg.info.handle)); + TAOS_UNUSED(transRemoveExHandle(transGetRefMgt(), (int64_t)pMsg->msg.info.handle)); } transDestroyConnCtx(pMsg->ctx); @@ -2302,7 +2336,9 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { } QUEUE_INIT(&pThrd->msg); - (void)taosThreadMutexInit(&pThrd->msgMtx, NULL); + if (taosThreadMutexInit(&pThrd->msgMtx, NULL) != 0) { + TAOS_CHECK_GOTO(terrno, NULL, _end); + } pThrd->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); if (pThrd->loop == NULL) { @@ -2324,8 +2360,8 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { pThrd->pool = createConnPool(4); if (pThrd->pool == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + code = terrno; + TAOS_CHECK_GOTO(terrno, NULL, _end); } if ((code = transDQCreate(pThrd->loop, &pThrd->delayQueue)) != 0) { TAOS_CHECK_GOTO(code, NULL, _end); @@ -2366,9 +2402,9 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { if (timer == NULL) { TAOS_CHECK_GOTO(terrno, NULL, _end); } - (void)uv_timer_init(pThrd->loop, timer); + TAOS_UNUSED(uv_timer_init(pThrd->loop, timer)); if (taosArrayPush(pThrd->timerList, &timer) == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + TAOS_CHECK_GOTO(terrno, NULL, _end); } } pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); @@ -2380,19 +2416,19 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { _end: if (pThrd) { - (void)taosThreadMutexDestroy(&pThrd->msgMtx); + TAOS_UNUSED(taosThreadMutexDestroy(&pThrd->msgMtx)); - (void)uv_loop_close(pThrd->loop); + TAOS_UNUSED(uv_loop_close(pThrd->loop)); taosMemoryFree(pThrd->loop); transAsyncPoolDestroy(pThrd->asyncPool); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); - (void)uv_timer_stop(timer); + TAOS_UNUSED(uv_timer_stop(timer)); taosMemoryFree(timer); } taosArrayDestroy(pThrd->timerList); - (void)destroyConnPool(pThrd); + TAOS_UNUSED(destroyConnPool(pThrd)); transDQDestroy(pThrd->delayQueue, NULL); transDQDestroy(pThrd->timeoutQueue, NULL); transDQDestroy(pThrd->waitConnQueue, NULL); @@ -2409,9 +2445,12 @@ static void destroyThrdObj(SCliThrd* pThrd) { return; } - (void)taosThreadJoin(pThrd->thread, NULL); + if (taosThreadJoin(pThrd->thread, NULL) != 0) { + tTrace("failed to join thread, reason:%s", tstrerror(terrno)); + } + CLI_RELEASE_UV(pThrd->loop); - (void)taosThreadMutexDestroy(&pThrd->msgMtx); + TAOS_UNUSED(taosThreadMutexDestroy(&pThrd->msgMtx)); TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsgWrapper, (void*)pThrd); transAsyncPoolDestroy(pThrd->asyncPool); @@ -2478,9 +2517,9 @@ void cliWalkCb(uv_handle_t* handle, void* arg) { if (uv_handle_get_type(handle) == UV_TIMER) { // do nothing } else { - (void)uv_read_stop((uv_stream_t*)handle); + TAOS_UNUSED(uv_read_stop((uv_stream_t*)handle)); } - (void)uv_close(handle, cliDestroy); + TAOS_UNUSED(uv_close(handle, cliDestroy)); } } @@ -2518,7 +2557,7 @@ static void cliSchedMsgToDebug(SCliMsg* pMsg, char* label) { STransConnCtx* pCtx = pMsg->ctx; STraceId* trace = &pMsg->msg.info.traceId; char tbuf[512] = {0}; - (void)epsetToStr(&pCtx->epSet, tbuf, tListLen(tbuf)); + TAOS_UNUSED(epsetToStr(&pCtx->epSet, tbuf, tListLen(tbuf))); tGDebug("%s retry on next node,use:%s, step: %d,timeout:%" PRId64 "", label, tbuf, pCtx->retryStep, pCtx->retryNextInterval); return; @@ -2530,10 +2569,14 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { cliSchedMsgToDebug(pMsg, transLabel(pThrd->pTransInst)); STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + if (arg == NULL) { + tError("failed to malloc memory, reason:%s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return; + } arg->param1 = pMsg; arg->param2 = pThrd; - (void)transDQSched(pThrd->delayQueue, doDelayTask, arg, pCtx->retryNextInterval); + TAOS_UNUSED(transDQSched(pThrd->delayQueue, doDelayTask, arg, pCtx->retryNextInterval)); } FORCE_INLINE bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) { @@ -2751,7 +2794,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (hasEpSet) { if (rpcDebugFlag & DEBUG_TRACE) { char tbuf[512] = {0}; - (void)epsetToStr(&pCtx->epSet, tbuf, tListLen(tbuf)); + TAOS_UNUSED(epsetToStr(&pCtx->epSet, tbuf, tListLen(tbuf))); tGTrace("%s conn %p extract epset from msg", CONN_GET_INST_LABEL(pConn), pConn); } } @@ -2763,7 +2806,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { } else { memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp)); } - (void)tsem_post(pCtx->pSem); + TAOS_UNUSED(tsem_post(pCtx->pSem)); pCtx->pRsp = NULL; } else { STransSyncMsg* pSyncMsg = taosAcquireRef(transGetSyncMsgMgt(), pCtx->syncMsgRef); @@ -2773,8 +2816,8 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pSyncMsg->hasEpSet = 1; epsetAssign(&pSyncMsg->epSet, &pCtx->epSet); } - (void)tsem2_post(pSyncMsg->pSem); - (void)taosReleaseRef(transGetSyncMsgMgt(), pCtx->syncMsgRef); + TAOS_UNUSED(tsem2_post(pSyncMsg->pSem)); + TAOS_UNUSED(taosReleaseRef(transGetSyncMsgMgt(), pCtx->syncMsgRef)); } else { rpcFreeCont(pResp->pCont); } @@ -2843,7 +2886,7 @@ static FORCE_INLINE SCliThrd* transGetWorkThrdFromHandle(STrans* trans, int64_t pThrd = exh->pThrd; taosWUnLockLatch(&exh->latch); - (void)transReleaseExHandle(transGetRefMgt(), handle); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), handle)); return pThrd; } @@ -2949,7 +2992,7 @@ int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, S code = transInitMsg(shandle, pEpSet, pReq, ctx, &pCliMsg); if (code != 0) { taosWUnLockLatch(&exh->latch); - (void)transReleaseExHandle(transGetRefMgt(), handle); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), handle)); TAOS_CHECK_GOTO(code, NULL, _exception); } @@ -2957,13 +3000,13 @@ int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, S taosWUnLockLatch(&exh->latch); tDebug("msg refId: %" PRId64 "", handle); - (void)transReleaseExHandle(transGetRefMgt(), handle); - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), handle)); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); return 0; } else { exh->inited = 1; taosWUnLockLatch(&exh->latch); - (void)transReleaseExHandle(transGetRefMgt(), handle); + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), handle)); } } } @@ -2976,16 +3019,16 @@ int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, S EPSET_GET_INUSE_IP(pEpSet), EPSET_GET_INUSE_PORT(pEpSet), pReq->info.ahandle); if ((code = transAsyncSend(pThrd->asyncPool, &(pCliMsg->q))) != 0) { destroyCmsg(pCliMsg); - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); return (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code); } - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); return 0; _exception: transFreeMsg(pReq->pCont); pReq->pCont = NULL; - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); return code; } int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) { @@ -3021,16 +3064,16 @@ int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* p EPSET_GET_INUSE_IP(pEpSet), EPSET_GET_INUSE_PORT(pEpSet), pReq->info.ahandle); if ((code = transAsyncSend(pThrd->asyncPool, &(pCliMsg->q))) != 0) { destroyCmsg(pCliMsg); - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); return (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code); } - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); return 0; _exception: transFreeMsg(pReq->pCont); pReq->pCont = NULL; - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); return code; } @@ -3068,7 +3111,7 @@ int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); if (pCtx == NULL) { - (void)tsem_destroy(sem); + TAOS_UNUSED(tsem_destroy(sem)); taosMemoryFree(sem); TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); } @@ -3082,7 +3125,7 @@ int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); if (cliMsg == NULL) { - (void)tsem_destroy(sem); + TAOS_UNUSED(tsem_destroy(sem)); taosMemoryFree(sem); taosMemoryFree(pCtx); TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); @@ -3103,18 +3146,18 @@ int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra destroyCmsg(cliMsg); TAOS_CHECK_GOTO((code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code), NULL, _RETURN); } - (void)tsem_wait(sem); + TAOS_UNUSED(tsem_wait(sem)); memcpy(pRsp, pTransRsp, sizeof(STransMsg)); _RETURN: tsem_destroy(sem); taosMemoryFree(sem); - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); taosMemoryFree(pTransRsp); return code; _RETURN1: - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); taosMemoryFree(pTransRsp); taosMemoryFree(pReq->pCont); pReq->pCont = NULL; @@ -3151,7 +3194,7 @@ int32_t transCreateSyncMsg(STransMsg* pTransMsg, int64_t* refId) { return 0; _EXIT: - (void)tsem2_destroy(sem); + TAOS_UNUSED(tsem2_destroy(sem)); taosMemoryFree(sem); taosMemoryFree(pSyncMsg); return code; @@ -3235,15 +3278,15 @@ int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, } } _RETURN: - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); - (void)taosReleaseRef(transGetSyncMsgMgt(), ref); - (void)taosRemoveRef(transGetSyncMsgMgt(), ref); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); + TAOS_UNUSED(taosReleaseRef(transGetSyncMsgMgt(), ref)); + TAOS_UNUSED(taosRemoveRef(transGetSyncMsgMgt(), ref)); return code; _RETURN2: transFreeMsg(pReq->pCont); pReq->pCont = NULL; taosMemoryFree(pTransMsg); - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); return code; } /* @@ -3295,7 +3338,7 @@ int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { } } - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); return code; } @@ -3319,13 +3362,14 @@ int32_t transAllocHandle(int64_t* refId) { QUEUE_INIT(&exh->q); taosInitRWLatch(&exh->latch); - tDebug("pre alloc refId %" PRId64 "", exh->refId); + tDebug("pre alloc refId %" PRId64 ", alloc exhandle %p", exh->refId, exh); *refId = exh->refId; return 0; } int32_t transFreeConnById(void* shandle, int64_t transpointId) { - int32_t code = 0; - STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + int32_t code = 0; + SCliMsg* pCli = NULL; + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); if (pTransInst == NULL) { return TSDB_CODE_RPC_MODULE_QUIT; } @@ -3339,7 +3383,7 @@ int32_t transFreeConnById(void* shandle, int64_t transpointId) { TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, NULL, _exception); } - SCliMsg* pCli = taosMemoryCalloc(1, sizeof(SCliMsg)); + pCli = taosMemoryCalloc(1, sizeof(SCliMsg)); if (pCli == NULL) { TAOS_CHECK_GOTO(terrno, NULL, _exception); } @@ -3352,11 +3396,19 @@ int32_t transFreeConnById(void* shandle, int64_t transpointId) { code = transAsyncSend(pThrd->asyncPool, &pCli->q); if (code != 0) { - taosMemoryFree(pCli); + taosMemoryFreeClear(pCli); TAOS_CHECK_GOTO(code, NULL, _exception); } _exception: - transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); + if (code != 0) { + if (transpointId != 0) { + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), transpointId)); + TAOS_UNUSED(transRemoveExHandle(transGetRefMgt(), transpointId)); + } + taosMemoryFree(pCli); + } + return code; } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 75d118a66b..98b5d907a0 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -272,7 +272,11 @@ int32_t transAsyncPoolCreate(uv_loop_t* loop, int sz, void* arg, AsyncCB cb, SAs } item->pThrd = arg; QUEUE_INIT(&item->qmsg); - (void)taosThreadMutexInit(&item->mtx, NULL); + code = taosThreadMutexInit(&item->mtx, NULL); + if (code) { + taosMemoryFree(item); + break; + } async->data = item; err = uv_async_init(loop, async, cb); @@ -301,7 +305,7 @@ void transAsyncPoolDestroy(SAsyncPool* pool) { SAsyncItem* item = async->data; if (item == NULL) continue; - (void)taosThreadMutexDestroy(&item->mtx); + TAOS_UNUSED(taosThreadMutexDestroy(&item->mtx)); taosMemoryFree(item); } taosMemoryFree(pool->asyncs); @@ -328,9 +332,12 @@ int transAsyncSend(SAsyncPool* pool, queue* q) { uv_async_t* async = &(pool->asyncs[idx]); SAsyncItem* item = async->data; - (void)taosThreadMutexLock(&item->mtx); + if (taosThreadMutexLock(&item->mtx) != 0) { + tError("failed to lock mutex"); + } + QUEUE_PUSH(&item->qmsg, q); - (void)taosThreadMutexUnlock(&item->mtx); + TAOS_UNUSED(taosThreadMutexUnlock(&item->mtx)); int ret = uv_async_send(async); if (ret != 0) { tError("failed to send async,reason:%s", uv_err_name(ret)); @@ -393,7 +400,7 @@ void* transCtxDumpVal(STransCtx* ctx, int32_t key) { return NULL; } void* ret = NULL; - (void)(*cVal->clone)(cVal->val, &ret); + TAOS_UNUSED((*cVal->clone)(cVal->val, &ret)); return ret; } void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType) { @@ -401,7 +408,7 @@ void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType) { if (ctx->brokenVal.clone == NULL) { return ret; } - (void)(*ctx->brokenVal.clone)(ctx->brokenVal.val, &ret); + TAOS_UNUSED((*ctx->brokenVal.clone)(ctx->brokenVal.val, &ret)); *msgType = ctx->brokenVal.msgType; @@ -414,6 +421,9 @@ void transReqQueueInit(queue* q) { } void* transReqQueuePush(queue* q) { STransReq* req = taosMemoryCalloc(1, sizeof(STransReq)); + if (req == NULL) { + return NULL; + } req->wreq.data = req; QUEUE_PUSH(q, &req->q); return &req->wreq; @@ -549,7 +559,7 @@ static void transDQTimeout(uv_timer_t* timer) { } } while (1); if (timeout != 0) { - (void)uv_timer_start(queue->timer, transDQTimeout, timeout, 0); + TAOS_UNUSED(uv_timer_start(queue->timer, transDQTimeout, timeout, 0)); } } int32_t transDQCreate(uv_loop_t* loop, SDelayQueue** queue) { @@ -614,7 +624,7 @@ void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)) { taosMemoryFree(queue); } void transDQCancel(SDelayQueue* queue, SDelayTask* task) { - (void)uv_timer_stop(queue->timer); + TAOS_UNUSED(uv_timer_stop(queue->timer)); if (heapSize(queue->heap) <= 0) { taosMemoryFree(task->arg); @@ -634,7 +644,7 @@ void transDQCancel(SDelayQueue* queue, SDelayTask* task) { SDelayTask* task = container_of(minNode, SDelayTask, node); uint64_t timeout = now > task->execTime ? now - task->execTime : 0; - (void)uv_timer_start(queue->timer, transDQTimeout, timeout, 0); + TAOS_UNUSED(uv_timer_start(queue->timer, transDQTimeout, timeout, 0)); } } @@ -659,7 +669,7 @@ SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, tTrace("timer %p put task into delay queue, timeoutMs:%" PRIu64, queue->timer, timeoutMs); heapInsert(queue->heap, &task->node); - (void)uv_timer_start(queue->timer, transDQTimeout, timeoutMs, 0); + TAOS_UNUSED(uv_timer_start(queue->timer, transDQTimeout, timeoutMs, 0)); return task; } @@ -708,7 +718,7 @@ static void transInitEnv() { svrRefMgt = transOpenRefMgt(50000, transDestroyExHandle); instMgt = taosOpenRef(50, rpcCloseImpl); transSyncMsgMgt = taosOpenRef(50, transDestroySyncMsg); - (void)uv_os_setenv("UV_TCP_SINGLE_ACCEPT", "1"); + TAOS_UNUSED(uv_os_setenv("UV_TCP_SINGLE_ACCEPT", "1")); } static void transDestroyEnv() { transCloseRefMgt(refMgt); @@ -768,6 +778,7 @@ void transDestroyExHandle(void* handle) { if (!QUEUE_IS_EMPTY(&eh->q)) { tDebug("handle %p mem leak", handle); } + tDebug("free exhandle %p", handle); taosMemoryFree(handle); } @@ -775,7 +786,7 @@ void transDestroySyncMsg(void* msg) { if (msg == NULL) return; STransSyncMsg* pSyncMsg = msg; - (void)tsem2_destroy(pSyncMsg->pSem); + TAOS_UNUSED(tsem2_destroy(pSyncMsg->pSem)); taosMemoryFree(pSyncMsg->pSem); transFreeMsg(pSyncMsg->pRsp->pCont); taosMemoryFree(pSyncMsg->pRsp); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 3ef7de6c1d..d8737ef30f 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -188,11 +188,11 @@ static void sendQuitToWorkThrd(SWorkThrd* pThrd); static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static int32_t addHandleToAcceptloop(void* arg); -#define SRV_RELEASE_UV(loop) \ - do { \ - (void)uv_walk(loop, uvWalkCb, NULL); \ - (void)uv_run(loop, UV_RUN_DEFAULT); \ - (void)uv_loop_close(loop); \ +#define SRV_RELEASE_UV(loop) \ + do { \ + TAOS_UNUSED(uv_walk(loop, uvWalkCb, NULL)); \ + TAOS_UNUSED(uv_run(loop, UV_RUN_DEFAULT)); \ + TAOS_UNUSED(uv_loop_close(loop)); \ } while (0); #define ASYNC_ERR_JRET(thrd) \ @@ -594,7 +594,7 @@ void uvOnSendCb(uv_write_t* req, int status) { (pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); memset(&conn->regArg, 0, sizeof(conn->regArg)); } - (void)transQueuePop(&conn->srvMsgs); + TAOS_UNUSED(transQueuePop(&conn->srvMsgs)); taosMemoryFree(msg); msg = (SSvrMsg*)transQueueGet(&conn->srvMsgs, 0); @@ -651,7 +651,7 @@ static int uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { // handle invalid drop_task resp, TD-20098 if (pConn->inType == TDMT_SCH_DROP_TASK && pMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { - (void)transQueuePop(&pConn->srvMsgs); + TAOS_UNUSED(transQueuePop(&pConn->srvMsgs)); destroySmsg(smsg); return TSDB_CODE_INVALID_MSG; } @@ -707,7 +707,11 @@ static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg) { transRefSrvHandle(pConn); uv_write_t* req = transReqQueuePush(&pConn->wreqQueue); - (void)uv_write(req, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnSendCb); + if (req == NULL) { + tError("failed to send resp since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return; + } + TAOS_UNUSED(uv_write(req, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnSendCb)); } static void uvStartSendResp(SSvrMsg* smsg) { // impl @@ -759,9 +763,15 @@ void uvWorkerAsyncCb(uv_async_t* handle) { queue wq; // batch process to avoid to lock/unlock frequently - (void)taosThreadMutexLock(&item->mtx); + if (taosThreadMutexLock(&item->mtx) != 0) { + tError("failed to lock mutex"); + } + QUEUE_MOVE(&item->qmsg, &wq); - (void)taosThreadMutexUnlock(&item->mtx); + + if (taosThreadMutexUnlock(&item->mtx) != 0) { + tError("failed to unlock mutex"); + } while (!QUEUE_IS_EMPTY(&wq)) { queue* head = QUEUE_HEAD(&wq); @@ -784,12 +794,12 @@ void uvWorkerAsyncCb(uv_async_t* handle) { SExHandle* exh2 = transAcquireExHandle(transGetSvrRefMgt(), refId); if (exh2 == NULL || exh1 != exh2) { tTrace("handle except msg %p, ignore it", exh1); - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetSvrRefMgt(), refId)); destroySmsg(msg); continue; } msg->pConn = exh1->handle; - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetSvrRefMgt(), refId)); (*transAsyncHandle[msg->type])(msg, pThrd); } } @@ -827,7 +837,7 @@ static bool uvRecvReleaseReq(SSvrConn* pConn, STransMsgHead* pHead) { tTrace("conn %p received release request", pConn); STraceId traceId = pHead->traceId; - (void)transClearBuffer(&pConn->readBuf); + TAOS_UNUSED(transClearBuffer(&pConn->readBuf)); transFreeMsg(transContFromHead((char*)pHead)); if (pConn->status != ConnAcquire) { return true; @@ -836,6 +846,10 @@ static bool uvRecvReleaseReq(SSvrConn* pConn, STransMsgHead* pHead) { STransMsg tmsg = {.code = 0, .info.handle = (void*)pConn, .info.traceId = traceId, .info.ahandle = (void*)0x9527}; SSvrMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSvrMsg)); + if (srvMsg == NULL) { + tError("failed to alloc buf to send release resp since %s", tstrerror(terrno)); + return true; + } srvMsg->msg = tmsg; srvMsg->type = Release; srvMsg->pConn = pConn; @@ -899,6 +913,11 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { #endif uv_write_t* wr = (uv_write_t*)taosMemoryMalloc(sizeof(uv_write_t)); + if (wr == NULL) { + tError("failed to accept since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return; + } + wr->data = cli; uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify)); @@ -906,7 +925,8 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { tTrace("new connection accepted by main server, dispatch to %dth worker-thread", pObj->workerIdx); - (void)uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnPipeWriteCb); + TAOS_UNUSED( + uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnPipeWriteCb)); } else { if (!uv_is_closing((uv_handle_t*)cli)) { tError("failed to accept tcp: %s", uv_err_name(err)); @@ -971,7 +991,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) { uv_os_fd_t fd; - (void)uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); + TAOS_UNUSED(uv_fileno((const uv_handle_t*)pConn->pTcp, &fd)); tTrace("conn %p created, fd:%d", pConn, fd); struct sockaddr peername, sockname; @@ -981,7 +1001,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { transUnrefSrvHandle(pConn); return; } - (void)transSockInfo2Str(&peername, pConn->dst); + TAOS_UNUSED(transSockInfo2Str(&peername, pConn->dst)); addrlen = sizeof(sockname); if (0 != uv_tcp_getsockname(pConn->pTcp, (struct sockaddr*)&sockname, &addrlen)) { @@ -989,7 +1009,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { transUnrefSrvHandle(pConn); return; } - (void)transSockInfo2Str(&sockname, pConn->src); + TAOS_UNUSED(transSockInfo2Str(&sockname, pConn->src)); struct sockaddr_in addr = *(struct sockaddr_in*)&peername; struct sockaddr_in saddr = *(struct sockaddr_in*)&sockname; @@ -998,7 +1018,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->serverIp = saddr.sin_addr.s_addr; pConn->port = ntohs(addr.sin_port); - (void)uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocRecvBufferCb, uvOnRecvCb); + TAOS_UNUSED(uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocRecvBufferCb, uvOnRecvCb)); } else { tDebug("failed to create new connection"); @@ -1010,7 +1030,7 @@ void* transAcceptThread(void* arg) { // opt setThreadName("trans-accept"); SServerObj* srv = (SServerObj*)arg; - (void)uv_run(srv->loop, UV_RUN_DEFAULT); + TAOS_UNUSED(uv_run(srv->loop, UV_RUN_DEFAULT)); return NULL; } @@ -1021,7 +1041,7 @@ void uvOnPipeConnectionCb(uv_connect_t* connect, int status) { }; SWorkThrd* pThrd = container_of(connect, SWorkThrd, connect_req); - (void)uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); + TAOS_UNUSED(uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb)); } static int32_t addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) { int32_t code = 0; @@ -1124,7 +1144,7 @@ static int32_t addHandleToAcceptloop(void* arg) { void* transWorkerThread(void* arg) { setThreadName("trans-svr-work"); SWorkThrd* pThrd = (SWorkThrd*)arg; - (void)uv_run(pThrd->loop, UV_RUN_DEFAULT); + TAOS_UNUSED(uv_run(pThrd->loop, UV_RUN_DEFAULT)); return NULL; } @@ -1201,7 +1221,7 @@ static FORCE_INLINE SSvrConn* createConn(void* hThrd) { _end: if (pConn) { transQueueDestroy(&pConn->srvMsgs); - (void)transDestroyBuffer(&pConn->readBuf); + TAOS_UNUSED(transDestroyBuffer(&pConn->readBuf)); taosMemoryFree(pConn->pTcp); taosMemoryFree(pConn); pConn = NULL; @@ -1230,8 +1250,8 @@ static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn) { } static int32_t reallocConnRef(SSvrConn* conn) { if (conn->refId > 0) { - (void)transReleaseExHandle(transGetSvrRefMgt(), conn->refId); - (void)transRemoveExHandle(transGetSvrRefMgt(), conn->refId); + TAOS_UNUSED(transReleaseExHandle(transGetSvrRefMgt(), conn->refId)); + TAOS_UNUSED(transRemoveExHandle(transGetSvrRefMgt(), conn->refId)); } // avoid app continue to send msg on invalid handle SExHandle* exh = taosMemoryMalloc(sizeof(SExHandle)); @@ -1267,8 +1287,8 @@ static void uvDestroyConn(uv_handle_t* handle) { } SWorkThrd* thrd = conn->hostThrd; - (void)transReleaseExHandle(transGetSvrRefMgt(), conn->refId); - (void)transRemoveExHandle(transGetSvrRefMgt(), conn->refId); + TAOS_UNUSED(transReleaseExHandle(transGetSvrRefMgt(), conn->refId)); + TAOS_UNUSED(transRemoveExHandle(transGetSvrRefMgt(), conn->refId)); STrans* pTransInst = thrd->pTransInst; tDebug("%s conn %p destroy", transLabel(pTransInst), conn); @@ -1283,7 +1303,7 @@ static void uvDestroyConn(uv_handle_t* handle) { QUEUE_REMOVE(&conn->queue); taosMemoryFree(conn->pTcp); destroyConnRegArg(conn); - (void)transDestroyBuffer(&conn->readBuf); + TAOS_UNUSED(transDestroyBuffer(&conn->readBuf)); taosMemoryFree(conn); if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) { @@ -1394,6 +1414,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, for (int i = 0; i < srv->numOfThreads; i++) { SWorkThrd* thrd = (SWorkThrd*)taosMemoryCalloc(1, sizeof(SWorkThrd)); + if (thrd == NULL) { + code = terrno; + goto End; + } thrd->pTransInst = shandle; thrd->quit = false; thrd->pTransInst = shandle; @@ -1563,7 +1587,7 @@ void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd) { if (!transQueuePush(&conn->srvMsgs, msg)) { return; } - (void)transQueuePop(&conn->srvMsgs); + TAOS_UNUSED(transQueuePop(&conn->srvMsgs)); if (conn->regArg.init) { transFreeMsg(conn->regArg.msg.pCont); @@ -1636,7 +1660,10 @@ void destroyWorkThrd(SWorkThrd* pThrd) { } if (pThrd->inited) { sendQuitToWorkThrd(pThrd); - (void)taosThreadJoin(pThrd->thread, NULL); + if ((taosThreadJoin(pThrd->thread, NULL)) != 0) { + tError("failed to join work-thread"); + } + SRV_RELEASE_UV(pThrd->loop); TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsgWrapper, NULL); } @@ -1644,9 +1671,13 @@ void destroyWorkThrd(SWorkThrd* pThrd) { } void sendQuitToWorkThrd(SWorkThrd* pThrd) { SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg)); + if (msg == NULL) { + tError("failed to send quit msg to work thread since %s", tstrerror(terrno)); + return; + } msg->type = Quit; tDebug("server send quit msg to work thread"); - (void)transAsyncSend(pThrd->asyncPool, &msg->q); + TAOS_UNUSED(transAsyncSend(pThrd->asyncPool, &msg->q)); } void transCloseServer(void* arg) { @@ -1655,8 +1686,10 @@ void transCloseServer(void* arg) { if (srv->inited) { tDebug("send quit msg to accept thread"); - (void)uv_async_send(srv->pAcceptAsync); - (void)taosThreadJoin(srv->thread, NULL); + TAOS_UNUSED(uv_async_send(srv->pAcceptAsync)); + if (taosThreadJoin(srv->thread, NULL) != 0) { + tError("failed to join accept-thread"); + } SRV_RELEASE_UV(srv->loop); for (int i = 0; i < srv->numOfThreads; i++) { @@ -1724,15 +1757,15 @@ int32_t transReleaseSrvHandle(void* handle) { tDebug("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle); if ((code = transAsyncSend(pThrd->asyncPool, &m->q)) != 0) { destroySmsg(m); - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetSvrRefMgt(), refId)); return code; } - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetSvrRefMgt(), refId)); return 0; _return1: tDebug("handle %p failed to send to release handle", exh); - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetSvrRefMgt(), refId)); return code; _return2: tDebug("handle %p failed to send to release handle", exh); @@ -1775,17 +1808,17 @@ int32_t transSendResponse(const STransMsg* msg) { tGDebug("conn %p start to send resp (1/2)", exh->handle); if ((code = transAsyncSend(pThrd->asyncPool, &m->q)) != 0) { destroySmsg(m); - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetSvrRefMgt(), refId)); return code; } - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetSvrRefMgt(), refId)); return 0; _return1: tDebug("handle %p failed to send resp", exh); rpcFreeCont(msg->pCont); - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetSvrRefMgt(), refId)); return code; _return2: tDebug("handle %p failed to send resp", exh); @@ -1820,17 +1853,17 @@ int32_t transRegisterMsg(const STransMsg* msg) { tDebug("%s conn %p start to register brokenlink callback", transLabel(pTransInst), exh->handle); if ((code = transAsyncSend(pThrd->asyncPool, &m->q)) != 0) { destroySmsg(m); - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetSvrRefMgt(), refId)); return code; } - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetSvrRefMgt(), refId)); return 0; _return1: tDebug("handle %p failed to register brokenlink", exh); rpcFreeCont(msg->pCont); - (void)transReleaseExHandle(transGetSvrRefMgt(), refId); + TAOS_UNUSED(transReleaseExHandle(transGetSvrRefMgt(), refId)); return code; _return2: tDebug("handle %p failed to register brokenlink", exh); @@ -1875,7 +1908,7 @@ int32_t transSetIpWhiteList(void* thandle, void* arg, FilteFunc* func) { break; } } - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)thandle); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)thandle)); if (code != 0) { tError("ip-white-list update failed since %s", tstrerror(code)); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 7ea98d648d..9ade5e5638 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -796,7 +796,7 @@ int32_t walMetaSerialize(SWal* pWal, char** serialized) { TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } - if (cJSON_AddItemToObject(pRoot, "meta", pMeta) != 0) { + if (!cJSON_AddItemToObject(pRoot, "meta", pMeta)) { wInfo("vgId:%d, failed to add meta to root", pWal->cfg.vgId); } (void)sprintf(buf, "%" PRId64, pWal->vers.firstVer); @@ -816,13 +816,13 @@ int32_t walMetaSerialize(SWal* pWal, char** serialized) { wInfo("vgId:%d, failed to add lastVer to meta", pWal->cfg.vgId); } - if (cJSON_AddItemToObject(pRoot, "files", pFiles) != 0) { + if (!cJSON_AddItemToObject(pRoot, "files", pFiles)) { wInfo("vgId:%d, failed to add files to root", pWal->cfg.vgId); } SWalFileInfo* pData = pWal->fileInfoSet->pData; for (int i = 0; i < sz; i++) { SWalFileInfo* pInfo = &pData[i]; - if (cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject()) != 0) { + if (!cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject())) { wInfo("vgId:%d, failed to add field to files", pWal->cfg.vgId); } if (pField == NULL) { @@ -937,6 +937,7 @@ static int walFindCurMetaVer(SWal* pWal) { TdDirPtr pDir = taosOpenDir(pWal->path); if (pDir == NULL) { wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, tstrerror(terrno)); + regfree(&walMetaRegexPattern); return terrno; } @@ -956,6 +957,7 @@ static int walFindCurMetaVer(SWal* pWal) { } if (taosCloseDir(&pDir) != 0) { wError("failed to close dir, ret:%s", tstrerror(terrno)); + regfree(&walMetaRegexPattern); return terrno; } regfree(&walMetaRegexPattern); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 81d31f9ecd..3b23a2db80 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -104,7 +104,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { } // set config - TAOS_UNUSED(memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg))); + (void)memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg)); pWal->fsyncSeq = pCfg->fsyncPeriod / 1000; if (pWal->cfg.retentionSize > 0) { @@ -155,7 +155,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->lastRollSeq = -1; // init write buffer - TAOS_UNUSED(memset(&pWal->writeHead, 0, sizeof(SWalCkHead))); + (void)memset(&pWal->writeHead, 0, sizeof(SWalCkHead)); pWal->writeHead.head.protoVer = WAL_PROTO_VER; pWal->writeHead.magic = WAL_MAGIC; @@ -189,6 +189,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { _err: taosArrayDestroy(pWal->fileInfoSet); + taosArrayDestroy(pWal->toDeleteFiles); taosHashCleanup(pWal->pRefHash); TAOS_UNUSED(taosThreadRwlockDestroy(&pWal->mutex)); taosMemoryFreeClear(pWal); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 9cf5bcbf09..610adfb0e1 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -41,7 +41,11 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond, int64_t id) { pReader->cond.enableRef = 0; } - TAOS_UNUSED(taosThreadMutexInit(&pReader->mutex, NULL)); + terrno = taosThreadMutexInit(&pReader->mutex, NULL); + if (terrno) { + taosMemoryFree(pReader); + return NULL; + } pReader->pHead = taosMemoryMalloc(sizeof(SWalCkHead)); if (pReader->pHead == NULL) { @@ -309,8 +313,8 @@ int32_t walSkipFetchBody(SWalReader *pRead) { if (pRead->pWal->cfg.encryptAlgorithm == 1) { cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen); } - int64_t code = taosLSeekFile(pRead->pLogFile, cryptedBodyLen, SEEK_CUR); - if (code < 0) { + int64_t ret = taosLSeekFile(pRead->pLogFile, cryptedBodyLen, SEEK_CUR); + if (ret < 0) { TAOS_RETURN(terrno); } @@ -401,7 +405,9 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { TAOS_RETURN(TSDB_CODE_WAL_LOG_NOT_EXIST); } - TAOS_UNUSED(taosThreadMutexLock(&pReader->mutex)); + if (taosThreadMutexLock(&pReader->mutex) != 0) { + wError("vgId:%d, failed to lock mutex", pReader->pWal->cfg.vgId); + } if (pReader->curVersion != ver) { code = walReaderSeekVer(pReader, ver); @@ -537,7 +543,7 @@ int32_t decryptBody(SWalCfg *cfg, SWalCkHead *pHead, int32_t plainBodyLen, const // wDebug("CBC_Decrypt cryptedBodyLen:%d, plainBodyLen:%d, %s", count, plainBodyLen, func); - TAOS_UNUSED(memcpy(pHead->head.body, newBody, plainBodyLen)); + (void)memcpy(pHead->head.body, newBody, plainBodyLen); taosMemoryFree(newBody); } @@ -546,7 +552,10 @@ int32_t decryptBody(SWalCfg *cfg, SWalCkHead *pHead, int32_t plainBodyLen, const } void walReadReset(SWalReader *pReader) { - TAOS_UNUSED(taosThreadMutexLock(&pReader->mutex)); + if ((taosThreadMutexLock(&pReader->mutex)) != 0) { + wError("vgId:%d, failed to lock mutex", pReader->pWal->cfg.vgId); + } + TAOS_UNUSED(taosCloseFile(&pReader->pIdxFile)); TAOS_UNUSED(taosCloseFile(&pReader->pLogFile)); pReader->curFileFirstVer = -1; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index a5105fc107..c8c37b11bc 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -86,11 +86,9 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { TAOS_RETURN(TSDB_CODE_SUCCESS); } -int32_t walApplyVer(SWal *pWal, int64_t ver) { +void walApplyVer(SWal *pWal, int64_t ver) { // TODO: error check pWal->vers.appliedVer = ver; - - TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t walCommit(SWal *pWal, int64_t ver) { @@ -111,20 +109,20 @@ static int64_t walChangeWrite(SWal *pWal, int64_t ver) { char fnameStr[WAL_FILE_LEN]; if (pWal->pLogFile != NULL) { if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pLogFile)) != 0) { - TAOS_RETURN(terrno); + return -1; } code = taosCloseFile(&pWal->pLogFile); if (code != 0) { - TAOS_RETURN(terrno); + return -1; } } if (pWal->pIdxFile != NULL) { if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pIdxFile)) != 0) { - TAOS_RETURN(terrno); + return -1; } code = taosCloseFile(&pWal->pIdxFile); if (code != 0) { - TAOS_RETURN(terrno); + return -1; } } @@ -141,7 +139,7 @@ static int64_t walChangeWrite(SWal *pWal, int64_t ver) { if (pIdxTFile == NULL) { pWal->pIdxFile = NULL; - TAOS_RETURN(terrno); + return -1; } walBuildLogName(pWal, fileFirstVer, fnameStr); pLogTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); @@ -149,7 +147,7 @@ static int64_t walChangeWrite(SWal *pWal, int64_t ver) { TAOS_UNUSED(taosCloseFile(&pIdxTFile)); pWal->pLogFile = NULL; - TAOS_RETURN(terrno); + return -1; } pWal->pLogFile = pLogTFile; @@ -162,7 +160,7 @@ static int64_t walChangeWrite(SWal *pWal, int64_t ver) { int32_t walRollback(SWal *pWal, int64_t ver) { TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); wInfo("vgId:%d, wal rollback for version %" PRId64, pWal->cfg.vgId, ver); - int64_t code; + int64_t ret; char fnameStr[WAL_FILE_LEN]; if (ver > pWal->vers.lastVer || ver <= pWal->vers.commitVer || ver <= pWal->vers.snapshotVer) { TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); @@ -173,11 +171,11 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // find correct file if (ver < walGetLastFileFirstVer(pWal)) { // change current files - code = walChangeWrite(pWal, ver); - if (code < 0) { + ret = walChangeWrite(pWal, ver); + if (ret < 0) { TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); - TAOS_RETURN(code); + TAOS_RETURN(terrno); } // delete files in descending order @@ -187,10 +185,14 @@ int32_t walRollback(SWal *pWal, int64_t ver) { walBuildLogName(pWal, pInfo->firstVer, fnameStr); wDebug("vgId:%d, wal remove file %s for rollback", pWal->cfg.vgId, fnameStr); - TAOS_UNUSED(taosRemoveFile(fnameStr)); + if (taosRemoveFile(fnameStr) != 0) { + wWarn("vgId:%d, failed to remove file %s for rollback since %s", pWal->cfg.vgId, fnameStr, terrstr()); + } walBuildIdxName(pWal, pInfo->firstVer, fnameStr); wDebug("vgId:%d, wal remove file %s for rollback", pWal->cfg.vgId, fnameStr); - TAOS_UNUSED(taosRemoveFile(fnameStr)); + if (taosRemoveFile(fnameStr) != 0) { + wWarn("vgId:%d, failed to remove file %s for rollback since %s", pWal->cfg.vgId, fnameStr, terrstr()); + } } } @@ -203,8 +205,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) { TAOS_RETURN(terrno); } int64_t idxOff = walGetVerIdxOffset(pWal, ver); - code = taosLSeekFile(pIdxFile, idxOff, SEEK_SET); - if (code < 0) { + ret = taosLSeekFile(pIdxFile, idxOff, SEEK_SET); + if (ret < 0) { TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(terrno); @@ -227,8 +229,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) { TAOS_RETURN(terrno); } - code = taosLSeekFile(pLogFile, entry.offset, SEEK_SET); - if (code < 0) { + ret = taosLSeekFile(pLogFile, entry.offset, SEEK_SET); + if (ret < 0) { // TODO TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); @@ -242,7 +244,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { TAOS_RETURN(terrno); } - code = walValidHeadCksum(&head); + int32_t code = walValidHeadCksum(&head); if (code != 0) { TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); @@ -462,7 +464,9 @@ int32_t walEndSnapshot(SWal *pWal) { } for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter <= pUntil; iter++) { deleteCnt++; - TAOS_UNUSED(taosArrayPush(pWal->toDeleteFiles, iter)); + if (taosArrayPush(pWal->toDeleteFiles, iter) == NULL) { + wError("vgId:%d, failed to push file info to delete list", pWal->cfg.vgId); + } } // make new array, remove files @@ -603,10 +607,10 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } - TAOS_UNUSED(memset(newBody, 0, cyptedBodyLen)); - TAOS_UNUSED(memcpy(newBody, body, plainBodyLen)); + (void)memset(newBody, 0, cyptedBodyLen); + (void)memcpy(newBody, body, plainBodyLen); newBodyEncrypted = taosMemoryMalloc(cyptedBodyLen); if (newBodyEncrypted == NULL) { @@ -615,7 +619,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy if (newBody != NULL) taosMemoryFreeClear(newBody); - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } SCryptOpts opts; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 40f48af266..ef8c1eb860 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -426,16 +426,19 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { return terrno; } + int64_t res = 0; DWORD bytesRead; if (!ReadFile(pFile->hFile, buf, count, &bytesRead, NULL)) { DWORD errCode = GetLastError(); terrno = TAOS_SYSTEM_WINAPI_ERROR(errCode); - bytesRead = -1; + res = -1; + } else { + res = bytesRead; } #if FILE_WITH_LOCK (void)taosThreadRwlockUnlock(&(pFile->rwlock)); #endif - return bytesRead; + return res; } int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index b99017782b..0bf67f96a0 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -65,7 +65,7 @@ uint32_t taosSafeRand(void) { if (pFile == NULL) { seed = (int)taosGetTimestampSec(); } else { - int len = taosReadFile(pFile, &seed, sizeof(seed)); + int64_t len = taosReadFile(pFile, &seed, sizeof(seed)); if (len < 0) { seed = (int)taosGetTimestampSec(); } diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index f6d339c89e..ea9e824947 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -232,7 +232,7 @@ int32_t tsem_init(tsem_t *psem, int flags, unsigned int count) { if(sem_init(psem, flags, count) == 0) { return 0; } else { - return TAOS_SYSTEM_ERROR(errno); + return terrno = TAOS_SYSTEM_ERROR(errno); } } diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 2065ba68fe..09a5579e13 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -55,7 +55,7 @@ typedef struct TdSocket { #endif int refId; SocketFd fd; -} *TdSocketPtr, TdSocket; +} * TdSocketPtr, TdSocket; typedef struct TdSocketServer { #if SOCKET_WITH_LOCK @@ -63,7 +63,7 @@ typedef struct TdSocketServer { #endif int refId; SocketFd fd; -} *TdSocketServerPtr, TdSocketServer; +} * TdSocketServerPtr, TdSocketServer; typedef struct TdEpoll { #if SOCKET_WITH_LOCK @@ -71,52 +71,7 @@ typedef struct TdEpoll { #endif int refId; EpollFd fd; -} *TdEpollPtr, TdEpoll; - -#if 0 -int32_t taosSendto(TdSocketPtr pSocket, void *buf, int len, unsigned int flags, const struct sockaddr *dest_addr, - int addrlen) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen); -#else - return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen); -#endif -} - -int32_t taosWriteSocket(TdSocketPtr pSocket, void *buf, int len) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - return send(pSocket->fd, buf, len, 0); - ; -#else - return write(pSocket->fd, buf, len); -#endif -} -int32_t taosReadSocket(TdSocketPtr pSocket, void *buf, int len) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - return recv(pSocket->fd, buf, len, 0); - ; -#else - return read(pSocket->fd, buf, len); -#endif -} - -int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, - int *addrLen) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } - return recvfrom(pSocket->fd, buf, len, flags, destAddr, addrLen); -} -#endif // endif 0 +} * TdEpollPtr, TdEpoll; int32_t taosCloseSocketNoCheck1(SocketFd fd) { #ifdef WINDOWS @@ -145,136 +100,16 @@ int32_t taosCloseSocket(TdSocketPtr *ppSocket) { code = taosCloseSocketNoCheck1((*ppSocket)->fd); (*ppSocket)->fd = -1; taosMemoryFree(*ppSocket); - + return code; } -#if 0 -int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer) { - int32_t code; - if (ppSocketServer == NULL || *ppSocketServer == NULL || (*ppSocketServer)->fd < 0) { - return -1; - } - code = taosCloseSocketNoCheck1((*ppSocketServer)->fd); - (*ppSocketServer)->fd = -1; - taosMemoryFree(*ppSocketServer); - return code; -} - -int32_t taosShutDownSocketRD(TdSocketPtr pSocket) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - return closesocket(pSocket->fd); -#elif __APPLE__ - return close(pSocket->fd); -#else - return shutdown(pSocket->fd, SHUT_RD); -#endif -} -int32_t taosShutDownSocketServerRD(TdSocketServerPtr pSocketServer) { - if (pSocketServer == NULL || pSocketServer->fd < 0) { - return -1; - } -#ifdef WINDOWS - return closesocket(pSocketServer->fd); -#elif __APPLE__ - return close(pSocketServer->fd); -#else - return shutdown(pSocketServer->fd, SHUT_RD); -#endif -} - -int32_t taosShutDownSocketWR(TdSocketPtr pSocket) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - return closesocket(pSocket->fd); -#elif __APPLE__ - return close(pSocket->fd); -#else - return shutdown(pSocket->fd, SHUT_WR); -#endif -} -int32_t taosShutDownSocketServerWR(TdSocketServerPtr pSocketServer) { - if (pSocketServer == NULL || pSocketServer->fd < 0) { - return -1; - } -#ifdef WINDOWS - return closesocket(pSocketServer->fd); -#elif __APPLE__ - return close(pSocketServer->fd); -#else - return shutdown(pSocketServer->fd, SHUT_WR); -#endif -} -int32_t taosShutDownSocketRDWR(TdSocketPtr pSocket) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - return closesocket(pSocket->fd); -#elif __APPLE__ - return close(pSocket->fd); -#else - return shutdown(pSocket->fd, SHUT_RDWR); -#endif -} -int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer) { - if (pSocketServer == NULL || pSocketServer->fd < 0) { - return -1; - } -#ifdef WINDOWS - return closesocket(pSocketServer->fd); -#elif __APPLE__ - return close(pSocketServer->fd); -#else - return shutdown(pSocketServer->fd, SHUT_RDWR); -#endif -} - -int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - u_long mode; - if (on) { - mode = 1; - ioctlsocket(pSocket->fd, FIONBIO, &mode); - } else { - mode = 0; - ioctlsocket(pSocket->fd, FIONBIO, &mode); - } -#else - int32_t flags = 0; - if ((flags = fcntl(pSocket->fd, F_GETFL, 0)) < 0) { - // printf("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno)); - return 1; - } - - if (on) - flags |= O_NONBLOCK; - else - flags &= ~O_NONBLOCK; - - if ((flags = fcntl(pSocket->fd, F_SETFL, flags)) < 0) { - // printf("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno)); - return 1; - } -#endif - return 0; -} -#endif // endif 0 - int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t optlen) { if (pSocket == NULL || pSocket->fd < 0) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - + #ifdef WINDOWS #ifdef TCP_KEEPCNT if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { @@ -300,11 +135,11 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void } #endif -int ret = setsockopt(pSocket->fd, level, optname, optval, optlen); -if (ret == SOCKET_ERROR) { - int errorCode = WSAGetLastError(); - return terrno = TAOS_SYSTEM_WINSOCKET_ERROR(errorCode); -} + int ret = setsockopt(pSocket->fd, level, optname, optval, optlen); + if (ret == SOCKET_ERROR) { + int errorCode = WSAGetLastError(); + return terrno = TAOS_SYSTEM_WINSOCKET_ERROR(errorCode); + } #else int32_t code = setsockopt(pSocket->fd, level, optname, optval, (int)optlen); if (-1 == code) { @@ -315,22 +150,8 @@ if (ret == SOCKET_ERROR) { #endif } -#if 0 -int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - return -1; -#else - return getsockopt(pSocket->fd, level, optname, optval, (int *)optlen); -#endif -} - -#endif - const char *taosInetNtoa(struct in_addr ipInt, char *dstStr, int32_t len) { - const char* r = inet_ntop(AF_INET, &ipInt, dstStr, len); + const char *r = inet_ntop(AF_INET, &ipInt, dstStr, len); if (NULL == r) { terrno = TAOS_SYSTEM_ERROR(errno); } @@ -344,403 +165,6 @@ const char *taosInetNtoa(struct in_addr ipInt, char *dstStr, int32_t len) { #define TCP_CONN_TIMEOUT 3000 // conn timeout -#if 0 -int32_t taosWriteMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } - int32_t nleft, nwritten; - char *ptr = (char *)buf; - - nleft = nbytes; - - while (nleft > 0) { - nwritten = taosWriteSocket(pSocket, (char *)ptr, (size_t)nleft); - if (nwritten <= 0) { - if (errno == EINTR /* || errno == EAGAIN || errno == EWOULDBLOCK */) - continue; - else - return -1; - } else { - nleft -= nwritten; - ptr += nwritten; - } - - if (errno == SIGPIPE || errno == EPIPE) { - return -1; - } - } - - return (nbytes - nleft); -} - -int32_t taosReadMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } - int32_t nleft, nread; - char *ptr = (char *)buf; - - nleft = nbytes; - - while (nleft > 0) { - nread = taosReadSocket(pSocket, ptr, (size_t)nleft); - if (nread == 0) { - break; - } else if (nread < 0) { - if (errno == EINTR /* || errno == EAGAIN || errno == EWOULDBLOCK*/) { - continue; - } else { - return -1; - } - } else { - nleft -= nread; - ptr += nread; - } - - if (errno == SIGPIPE || errno == EPIPE) { - return -1; - } - } - - return (nbytes - nleft); -} - -int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } - taosSetNonblocking(pSocket, 1); - - int32_t nleft, nwritten, nready; - fd_set fset; - struct timeval tv; - - nleft = nbytes; - while (nleft > 0) { - tv.tv_sec = 30; - tv.tv_usec = 0; - FD_ZERO(&fset); - FD_SET(pSocket->fd, &fset); - if ((nready = select((SocketFd)(pSocket->fd + 1), NULL, &fset, NULL, &tv)) == 0) { - errno = ETIMEDOUT; - // printf("fd %d timeout, no enough space to write", fd); - break; - - } else if (nready < 0) { - if (errno == EINTR) continue; - - // printf("select error, %d (%s)", errno, strerror(errno)); - return -1; - } - - nwritten = (int32_t)send(pSocket->fd, ptr, (size_t)nleft, MSG_NOSIGNAL); - if (nwritten <= 0) { - if (errno == EAGAIN || errno == EINTR) continue; - - // printf("write error, %d (%s)", errno, strerror(errno)); - return -1; - } - - nleft -= nwritten; - ptr += nwritten; - } - - taosSetNonblocking(pSocket, 0); - - return (nbytes - nleft); -} - -TdSocketPtr taosOpenUdpSocket(uint32_t ip, uint16_t port) { - struct sockaddr_in localAddr; - SocketFd fd; - int32_t bufSize = 1024000; - - // printf("open udp socket:0x%x:%hu", ip, port); - - memset((char *)&localAddr, 0, sizeof(localAddr)); - localAddr.sin_family = AF_INET; - localAddr.sin_addr.s_addr = ip; - localAddr.sin_port = (uint16_t)htons(port); - - if ((fd = socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { - // printf("failed to open udp socket: %d (%s)", errno, strerror(errno)); - taosCloseSocketNoCheck1(fd); - return NULL; - } - - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(fd); - return NULL; - } - pSocket->fd = fd; - pSocket->refId = 0; - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - // printf("failed to set the send buffer size for UDP socket\n"); - taosCloseSocket(&pSocket); - return NULL; - } - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - // printf("failed to set the receive buffer size for UDP socket\n"); - taosCloseSocket(&pSocket); - return NULL; - } - - /* bind socket to local address */ - if (bind(pSocket->fd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) { - // printf("failed to bind udp socket: %d (%s), 0x%x:%hu", errno, strerror(errno), ip, port); - taosCloseSocket(&pSocket); - return NULL; - } - - return pSocket; -} - -TdSocketPtr taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { - SocketFd fd = -1; - int32_t ret; - struct sockaddr_in serverAddr, clientAddr; - int32_t bufSize = 1024 * 1024; - - fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - - if (fd <= 2) { - // printf("failed to open the socket: %d (%s)", errno, strerror(errno)); - if (fd >= 0) taosCloseSocketNoCheck1(fd); - return NULL; - } - - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(fd); - return NULL; - } - pSocket->fd = fd; - pSocket->refId = 0; - - /* set REUSEADDR option, so the portnumber can be re-used */ - int32_t reuse = 1; - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - // printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - // printf("failed to set the send buffer size for TCP socket\n"); - taosCloseSocket(&pSocket); - return NULL; - } - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - // printf("failed to set the receive buffer size for TCP socket\n"); - taosCloseSocket(&pSocket); - return NULL; - } - - if (clientIp != 0) { - memset((char *)&clientAddr, 0, sizeof(clientAddr)); - clientAddr.sin_family = AF_INET; - clientAddr.sin_addr.s_addr = clientIp; - clientAddr.sin_port = 0; - - /* bind socket to client address */ - if (bind(pSocket->fd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) { - // printf("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", clientIp, destIp, destPort, - // strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - } - - memset((char *)&serverAddr, 0, sizeof(serverAddr)); - serverAddr.sin_family = AF_INET; - serverAddr.sin_addr.s_addr = destIp; - serverAddr.sin_port = (uint16_t)htons((uint16_t)destPort); - -#ifdef _TD_LINUX - taosSetNonblocking(pSocket, 1); - ret = connect(pSocket->fd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); - if (ret == -1) { - if (errno == EHOSTUNREACH) { - // printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } else if (errno == EINPROGRESS || errno == EAGAIN || errno == EWOULDBLOCK) { - struct pollfd wfd[1]; - - wfd[0].fd = pSocket->fd; - wfd[0].events = POLLOUT; - - int res = poll(wfd, 1, TCP_CONN_TIMEOUT); - if (res == -1 || res == 0) { - // printf("failed to connect socket, ip:0x%x, port:%hu(poll error/conn timeout)", destIp, destPort); - taosCloseSocket(&pSocket); // - return -1; - } - int optVal = -1, optLen = sizeof(int); - if ((0 != taosGetSockOpt(pSocket, SOL_SOCKET, SO_ERROR, &optVal, &optLen)) || (optVal != 0)) { - // printf("failed to connect socket, ip:0x%x, port:%hu(connect host error)", destIp, destPort); - taosCloseSocket(&pSocket); // - return -1; - } - ret = 0; - } else { // Other error - // printf("failed to connect socket, ip:0x%x, port:%hu(target host cannot be reached)", destIp, destPort); - taosCloseSocket(&pSocket); // - return -1; - } - } - taosSetNonblocking(pSocket, 0); - -#else - ret = connect(pSocket->fd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); -#endif - - if (ret != 0) { - // printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } else { - if (taosKeepTcpAlive(pSocket) == -1) { - return NULL; - } - } - - return pSocket; -} - -int32_t taosKeepTcpAlive(TdSocketPtr pSocket) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } - int32_t alive = 1; - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) { - // printf("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - -#ifndef __APPLE__ - // all fails on macosx -#ifdef TCP_KEEPCNT - int32_t probes = 3; - if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { - // printf("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } -#endif - -#ifdef TCP_KEEPIDLE - int32_t alivetime = 10; - if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) { - // printf("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } -#endif - -#ifdef TCP_KEEPINTVL - int32_t interval = 3; - if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) { - // printf("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } -#endif -#endif // __APPLE__ - - int32_t nodelay = 1; - if (taosSetSockOpt(pSocket, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { - // printf("fd:%d setsockopt TCP_NODELAY failed %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - struct linger linger = {0}; - linger.l_onoff = 1; - linger.l_linger = 3; - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_LINGER, (void *)&linger, sizeof(linger)) < 0) { - // printf("setsockopt SO_LINGER failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - return 0; -} - -int taosGetLocalIp(const char *eth, char *ip) { -#if defined(WINDOWS) - // DO NOTHAING - return -1; -#else - int fd; - struct ifreq ifr; - struct sockaddr_in sin; - - fd = socket(AF_INET, SOCK_DGRAM, 0); - if (-1 == fd) { - return -1; - } - strncpy(ifr.ifr_name, eth, IFNAMSIZ); - ifr.ifr_name[IFNAMSIZ - 1] = 0; - - if (ioctl(fd, SIOCGIFADDR, &ifr) < 0) { - taosCloseSocketNoCheck1(fd); - return -1; - } - memcpy(&sin, &ifr.ifr_addr, sizeof(sin)); - taosInetNtoa(sin.sin_addr, ip, 64); - taosCloseSocketNoCheck1(fd); -#endif - return 0; -} -int taosValidIp(uint32_t ip) { -#if defined(WINDOWS) - // DO NOTHAING - return -1; -#else - int ret = -1; - int fd; - - struct ifconf ifconf; - - char buf[512] = {0}; - ifconf.ifc_len = 512; - ifconf.ifc_buf = buf; - - if ((fd = socket(AF_INET, SOCK_DGRAM, 0)) < 0) { - return -1; - } - - ioctl(fd, SIOCGIFCONF, &ifconf); - struct ifreq *ifreq = (struct ifreq *)ifconf.ifc_buf; - for (int i = (ifconf.ifc_len / sizeof(struct ifreq)); i > 0; i--) { - char ip_str[64] = {0}; - if (ifreq->ifr_flags == AF_INET) { - ret = taosGetLocalIp(ifreq->ifr_name, ip_str); - if (ret != 0) { - break; - } - ret = -1; - if (ip == (uint32_t)taosInetAddr(ip_str)) { - ret = 0; - break; - } - ifreq++; - } - } - taosCloseSocketNoCheck1(fd); - return ret; -#endif - return 0; -} -#endif // endif 0 - bool taosValidIpAndPort(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; SocketFd fd; @@ -778,138 +202,19 @@ bool taosValidIpAndPort(uint32_t ip, uint16_t port) { TAOS_SKIP_ERROR(taosCloseSocket(&pSocket)); return false; } - + /* bind socket to server address */ if (-1 == bind(pSocket->fd, (struct sockaddr *)&serverAdd, sizeof(serverAdd))) { terrno = TAOS_SYSTEM_ERROR(errno); TAOS_SKIP_ERROR(taosCloseSocket(&pSocket)); return false; } - + TAOS_SKIP_ERROR(taosCloseSocket(&pSocket)); - + return true; } -#if 0 -TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { - struct sockaddr_in serverAdd; - SocketFd fd; - int32_t reuse; - - // printf("open tcp server socket:0x%x:%hu", ip, port); - - bzero((char *)&serverAdd, sizeof(serverAdd)); - serverAdd.sin_family = AF_INET; - serverAdd.sin_addr.s_addr = ip; - serverAdd.sin_port = (uint16_t)htons(port); - - if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { - // printf("failed to open TCP socket: %d (%s)", errno, strerror(errno)); - taosCloseSocketNoCheck1(fd); - return NULL; - } - - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(fd); - return NULL; - } - pSocket->refId = 0; - pSocket->fd = fd; - - /* set REUSEADDR option, so the portnumber can be re-used */ - reuse = 1; - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - // printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - /* bind socket to server address */ - if (bind(pSocket->fd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) { - // printf("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - if (taosKeepTcpAlive(pSocket) < 0) { - // printf("failed to set tcp server keep-alive option, 0x%x:%hu(%s)", ip, port, strerror(errno)); - return NULL; - } - - if (listen(pSocket->fd, 1024) < 0) { - // printf("listen tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - return (TdSocketServerPtr)pSocket; -} - -TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen) { - if (pServerSocket == NULL || pServerSocket->fd < 0) { - return NULL; - } - SocketFd fd = accept(pServerSocket->fd, destAddr, addrLen); - if (fd == -1) { - // tError("TCP accept failure(%s)", strerror(errno)); - return NULL; - } - - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(fd); - return NULL; - } - pSocket->fd = fd; - pSocket->refId = 0; - return pSocket; -} -#define COPY_SIZE 32768 -// sendfile shall be used - -int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len) { - if (pSrcSocket == NULL || pSrcSocket->fd < 0 || pDestSocket == NULL || pDestSocket->fd < 0) { - return -1; - } - int64_t leftLen; - int64_t readLen, writeLen; - char temp[COPY_SIZE]; - - leftLen = len; - - while (leftLen > 0) { - if (leftLen < COPY_SIZE) - readLen = leftLen; - else - readLen = COPY_SIZE; // 4K - - int64_t retLen = taosReadMsg(pSrcSocket, temp, (int32_t)readLen); - if (readLen != retLen) { - // printf("read error, readLen:%" PRId64 " retLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", - // readLen, retLen, len, leftLen, strerror(errno)); - return -1; - } - - writeLen = taosWriteMsg(pDestSocket, temp, (int32_t)readLen); - - if (readLen != writeLen) { - // printf("copy error, readLen:%" PRId64 " writeLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", - // readLen, writeLen, len, leftLen, strerror(errno)); - return -1; - } - - leftLen -= readLen; - } - - return len; -} - -// Function converting an IP address string to an uint32_t. - -#endif // endif 0 - int32_t taosBlockSIGPIPE() { #ifdef WINDOWS return 0; @@ -1029,7 +334,7 @@ int32_t taosGetFqdn(char *fqdn) { // hints.ai_family = AF_INET; strcpy(fqdn, hostname); strcpy(fqdn + strlen(hostname), ".local"); -#else // linux +#else // linux #endif // linux @@ -1048,7 +353,7 @@ int32_t taosGetFqdn(char *fqdn) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - + terrno = TAOS_SYSTEM_ERROR(ret); return terrno; } @@ -1067,7 +372,7 @@ int32_t taosGetFqdn(char *fqdn) { int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - //fprintf(stderr, "failed to get fqdn, code:%d, hostname:%s, reason:%s\n", ret, hostname, gai_strerror(ret)); + // fprintf(stderr, "failed to get fqdn, code:%d, hostname:%s, reason:%s\n", ret, hostname, gai_strerror(ret)); return TAOS_SYSTEM_WINSOCKET_ERROR(WSAGetLastError()); } strcpy(fqdn, result->ai_canonname); @@ -1082,41 +387,16 @@ void tinet_ntoa(char *ipstr, uint32_t ip) { (void)sprintf(ipstr, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24); } -int32_t taosIgnSIGPIPE() { - sighandler_t h = signal(SIGPIPE, SIG_IGN); +int32_t taosIgnSIGPIPE() { + sighandler_t h = signal(SIGPIPE, SIG_IGN); if (SIG_ERR == h) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - return 0; + return 0; } -#if 0 - -int32_t taosSetMaskSIGPIPE() { -#ifdef WINDOWS - return -1; -#else - sigset_t signal_mask; - (void)sigemptyset(&signal_mask); - (void)sigaddset(&signal_mask, SIGPIPE); - - int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL); - if (rc != 0) { - // printf("failed to setmask SIGPIPE"); - } -#endif -} - -int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *addrLen) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } - return getsockname(pSocket->fd, destAddr, addrLen); -} -#endif // endif 0 - /* * Set TCP connection timeout per-socket level. * ref [https://github.com/libuv/help/issues/54] @@ -1132,7 +412,7 @@ int32_t taosCreateSocketWithTimeout(uint32_t timeout) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - + #if defined(WINDOWS) if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&timeout, sizeof(timeout))) { taosCloseSocketNoCheck1(fd); diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 68392b5050..d265ed510a 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -63,7 +63,8 @@ char *strsep(char **stringp, const char *delim) { /* NOTREACHED */ } /* Duplicate a string, up to at most size characters */ -char *strndup(const char *s, int size) { +char *taosStrndup(const char *s, int size) { + if (s == NULL) return NULL; size_t l; char *s2; l = strlen(s); @@ -72,6 +73,8 @@ char *strndup(const char *s, int size) { if (s2) { strncpy(s2, s, l); s2[l] = '\0'; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; } return s2; } @@ -84,6 +87,17 @@ char *stpncpy(char *dest, const char *src, int n) { if (size == n) return dest; return memset(dest, '\0', n - size); } +#else +char *taosStrndup(const char *s, int size) { + if (s == NULL) { + return NULL; + } + char *p = strndup(s, size); + if (NULL == p) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return p; +} #endif int32_t taosStr2int64(const char *str, int64_t *val) { diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 4d97232806..cb6d3a7736 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -1057,7 +1057,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { int n = snprintf(uid, uidlen, "%.*s", (int)sizeof(buf), buf); // though less performance, much safer return 0; #else - int len = 0; + int64_t len = 0; // fd = open("/proc/sys/kernel/random/uuid", 0); TdFilePtr pFile = taosOpenFile("/proc/sys/kernel/random/uuid", TD_FILE_READ); @@ -1067,7 +1067,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { len = taosReadFile(pFile, uid, uidlen); TAOS_SKIP_ERROR(taosCloseFile(&pFile)); if (len < 0) { - return len; + return terrno; } } diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 89ced69f97..6e7c22c7f1 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -742,6 +742,20 @@ char *tz_win[554][2] = {{"Asia/Shanghai", "China Standard Time"}, static int isdst_now = 0; +void parseTimeStr(char *p, char to[5]) { + for (int i = 0; i < 5; ++i) { + if (strlen(p) > i) { + to[i] = p[i]; + } else { + to[i] = '0'; + } + } + if (strlen(p) == 2) { + to[1] = '0'; + to[2] = p[1]; + } +} + int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8_t *outDaylight, enum TdTimezone *tsTimezone) { if (inTimezoneStr == NULL || inTimezoneStr[0] == 0) { @@ -798,7 +812,9 @@ int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, i memcpy(&winStr[3], pp, ppp - pp); indexStr = ppp - pp + 3; } - sprintf(&winStr[indexStr], "%c%c%c:%c%c:00", (p[0] == '+' ? '+' : '-'), p[1], p[2], p[3], p[4]); + char to[5]; + parseTimeStr(p, to); + sprintf(&winStr[indexStr], "%c%c%c:%c%c:00", (to[0] == '+' ? '+' : '-'), to[1], to[2], to[3], to[4]); *tsTimezone = -taosStr2Int32(p, NULL, 10); } else { *tsTimezone = 0; @@ -806,7 +822,9 @@ int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, i } _putenv(winStr); _tzset(); - strcpy(outTimezoneStr, inTimezoneStr); + if (outTimezoneStr != inTimezoneStr) { + strcpy(outTimezoneStr, inTimezoneStr); + } *outDaylight = 0; #elif defined(_TD_DARWIN_64) diff --git a/source/os/test/osStringTests.cpp b/source/os/test/osStringTests.cpp index 5e07636b1f..9565e568fb 100644 --- a/source/os/test/osStringTests.cpp +++ b/source/os/test/osStringTests.cpp @@ -74,7 +74,7 @@ TEST(osStringTests, strsepNullInput) { TEST(osStringTests, strndupNormalInput) { const char s[] = "This is a test string."; int size = strlen(s) + 1; - char * s2 = strndup(s, size); + char * s2 = taosStrndup(s, size); EXPECT_STREQ(s, s2); diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index f1ee40eccf..21201df309 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -335,7 +335,7 @@ int32_t taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, con if (buf == NULL) { tmp = taosMemoryMalloc(size); if (NULL == tmp) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } else { tmp = buf; @@ -467,7 +467,7 @@ static int32_t taosMergeSortHelper(void *src, int64_t numOfElem, int64_t size, c int32_t currSize; void *tmp = taosMemoryMalloc(numOfElem * size); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (currSize = THRESHOLD_SIZE; currSize <= numOfElem - 1; currSize = 2 * currSize) { diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 59505dc0c1..7989a2468b 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -297,7 +297,7 @@ void taosArrayRemove(SArray* pArray, size_t index) { } if (index == pArray->size - 1) { - (void)taosArrayPop(pArray); + void* t = taosArrayPop(pArray); return; } diff --git a/source/util/src/tbase64.c b/source/util/src/tbase64.c index 86b069b853..f623d34a2f 100644 --- a/source/util/src/tbase64.c +++ b/source/util/src/tbase64.c @@ -22,7 +22,7 @@ int32_t base64_encode(const uint8_t *value, int32_t vlen, char **result) { uint8_t oval = 0; *result = (char *)taosMemoryMalloc((size_t)(vlen * 4) / 3 + 10); if (*result == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } char *out = *result; while (vlen >= 3) { @@ -58,7 +58,7 @@ int32_t base64_decode(const char *value, int32_t inlen, int32_t *outlen, uint8_t int32_t c1, c2, c3, c4; *result = (uint8_t *)taosMemoryMalloc((size_t)(inlen * 3) / 4 + 1); if (*result == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } uint8_t *out = *result; diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 81e50f6a6b..d5001f0264 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -173,7 +173,7 @@ TdThread doRegisterCacheObj(SCacheObj *pCacheObj) { (void)taosThreadOnce(&cacheThreadInit, doInitRefreshThread); (void)taosThreadMutexLock(&guard); - if (taosArrayPush(pCacheArrayList, &pCacheObj) != 0) { + if (taosArrayPush(pCacheArrayList, &pCacheObj) == NULL) { uError("failed to add cache object into array, reason:%s", strerror(errno)); (void)taosThreadMutexUnlock(&guard); return cacheRefreshWorker; @@ -404,7 +404,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLi return NULL; } - (void)doRegisterCacheObj(pCacheObj); + TdThread refreshWorker = doRegisterCacheObj(pCacheObj); return pCacheObj; } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index afbd5304ce..288d440d86 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -324,8 +324,8 @@ bool lossyFloat = false; bool lossyDouble = false; // init call -int32_t tsCompressInit(char *lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals, - uint32_t intervals, int32_t ifAdtFse, const char *compressor) { +void tsCompressInit(char *lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals, uint32_t intervals, + int32_t ifAdtFse, const char *compressor) { // config lossyFloat = strstr(lossyColumns, "float") != NULL; lossyDouble = strstr(lossyColumns, "double") != NULL; @@ -333,7 +333,7 @@ int32_t tsCompressInit(char *lossyColumns, float fPrecision, double dPrecision, tdszInit(fPrecision, dPrecision, maxIntervals, intervals, ifAdtFse, compressor); if (lossyFloat) uTrace("lossy compression float is opened. "); if (lossyDouble) uTrace("lossy compression double is opened. "); - return 0; + return; } // exit call void tsCompressExit() { tdszExit(); } @@ -825,9 +825,9 @@ int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelement return nelements * longBytes; } else if (input[0] == 1) { // Decompress if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) { - (void)tsDecompressTimestampAvx512(input, nelements, output, false); + tsDecompressTimestampAvx512(input, nelements, output, false); } else if (tsSIMDEnable && tsAVX2Supported) { - (void)tsDecompressTimestampAvx2(input, nelements, output, false); + tsDecompressTimestampAvx2(input, nelements, output, false); } else { int64_t *ostream = (int64_t *)output; @@ -1201,9 +1201,9 @@ int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, c } if (tsSIMDEnable && tsAVX2Supported) { - (void)tsDecompressFloatImplAvx2(input, nelements, output); + tsDecompressFloatImplAvx2(input, nelements, output); } else if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) { - (void)tsDecompressFloatImplAvx512(input, nelements, output); + tsDecompressFloatImplAvx512(input, nelements, output); } else { // alternative implementation without SIMD instructions. tsDecompressFloatHelper(input, nelements, (float *)output); } @@ -1766,12 +1766,12 @@ int32_t tsDecompressBigint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, in FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_BIGINT, 0); } -int32_t tcompressDebug(uint32_t cmprAlg, uint8_t *l1Alg, uint8_t *l2Alg, uint8_t *level) { +void tcompressDebug(uint32_t cmprAlg, uint8_t *l1Alg, uint8_t *l2Alg, uint8_t *level) { DEFINE_VAR(cmprAlg) *l1Alg = l1; *l2Alg = l2; *level = lvl; - return 0; + return; } int8_t tUpdateCompress(uint32_t oldCmpr, uint32_t newCmpr, uint8_t l2Disabled, uint8_t lvlDiabled, uint8_t lvlDefault, diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index ee55243415..d84a426e98 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -480,7 +480,7 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p } break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: { - float dval = 0; + float dval = 0; int32_t code = parseCfgReal(pVal, &dval); if (code != TSDB_CODE_SUCCESS) { cfgUnLock(pCfg); @@ -911,7 +911,9 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { strncpy(line, *pEnv, sizeof(line) - 1); pEnv++; - (void)taosEnvToCfg(line, line); + if (taosEnvToCfg(line, line) < 0) { + uTrace("failed to convert env to cfg:%s", line); + } (void)paGetToken(line, &name, &olen); if (olen == 0) continue; @@ -954,7 +956,9 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { while (envCmd[index] != NULL) { strncpy(buf, envCmd[index], sizeof(buf) - 1); buf[sizeof(buf) - 1] = 0; - (void)taosEnvToCfg(buf, buf); + if (taosEnvToCfg(buf, buf) < 0) { + uTrace("failed to convert env to cfg:%s", buf); + } index++; name = value = value2 = value3 = value4 = NULL; @@ -1026,7 +1030,9 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { break; } if (line[_bytes - 1] == '\n') line[_bytes - 1] = 0; - (void)taosEnvToCfg(line, line); + if (taosEnvToCfg(line, line) < 0) { + uTrace("failed to convert env to cfg:%s", line); + } (void)paGetToken(line, &name, &olen); if (olen == 0) continue; @@ -1119,7 +1125,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { code = cfgSetItem(pConfig, name, newValue, CFG_STYPE_CFG_FILE, true); if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) { - (void)printf("cfg:%s, value:%s failed since %s\n", name,newValue, tstrerror(code)); + (void)printf("cfg:%s, value:%s failed since %s\n", name, newValue, tstrerror(code)); break; } } else { @@ -1260,12 +1266,12 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { TAOS_CHECK_EXIT(terrno); } size_t fileSize = taosLSeekFile(pFile, 0, SEEK_END); - if(fileSize <= 0) { + if (fileSize <= 0) { (void)taosCloseFile(&pFile); (void)printf("load json file error: %s\n", filepath); TAOS_CHECK_EXIT(terrno); } - char *buf = taosMemoryMalloc(fileSize + 1); + char *buf = taosMemoryMalloc(fileSize + 1); if (!buf) { (void)taosCloseFile(&pFile); (void)printf("load json file error: %s, failed to alloc memory\n", filepath); @@ -1273,7 +1279,12 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { } buf[fileSize] = 0; - (void)taosLSeekFile(pFile, 0, SEEK_SET); + if (taosLSeekFile(pFile, 0, SEEK_SET) < 0) { + (void)taosCloseFile(&pFile); + (void)printf("load json file error: %s\n", filepath); + taosMemoryFreeClear(buf); + TAOS_RETURN(terrno); + } if (taosReadFile(pFile, buf, fileSize) <= 0) { (void)taosCloseFile(&pFile); (void)printf("load json file error: %s\n", filepath); diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c index a27d2efbcc..598aca730f 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompress.c @@ -313,23 +313,22 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, return nelements * word_length; } -int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output) { +void tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output) { #if __AVX512F__ // todo add it #endif - return 0; + return; } // todo add later -int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output) { +void tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output) { #if __AVX2__ #endif - return 0; + return; } // decode two timestamps in one loop. -int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, - bool bigEndian) { +void tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, bool bigEndian) { int64_t *ostream = (int64_t *)output; int32_t ipos = 1, opos = 0; @@ -466,11 +465,11 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen } } #endif - return 0; + return; } -int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, - bool UNUSED_PARAM(bigEndian)) { +void tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, + bool UNUSED_PARAM(bigEndian)) { int64_t *ostream = (int64_t *)output; int32_t ipos = 1, opos = 0; @@ -581,5 +580,5 @@ int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelem } #endif - return 0; + return; } diff --git a/source/util/src/tdigest.c b/source/util/src/tdigest.c index 636c97dbaa..7ec5531b96 100644 --- a/source/util/src/tdigest.c +++ b/source/util/src/tdigest.c @@ -252,7 +252,9 @@ double tdigestQuantile(TDigest *t, double q) { int64_t weight_so_far; SCentroid *a, *b, tmp; - (void)tdigestCompress(t); + if (tdigestCompress(t) != 0) { + uError("failed to compress t-digest"); + } if (t->num_centroids == 0) return NAN; if (t->num_centroids == 1) return t->centroids[0].mean; if (FLOAT_EQ(q, 0.0)) return t->min; diff --git a/source/util/src/tencode.c b/source/util/src/tencode.c index e962edaa27..db59d19fb5 100644 --- a/source/util/src/tencode.c +++ b/source/util/src/tencode.c @@ -78,7 +78,7 @@ int32_t tStartEncode(SEncoder* pCoder) { pNode = tEncoderMalloc(pCoder, sizeof(*pNode)); if (pNode == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pNode->data = pCoder->data; @@ -126,7 +126,7 @@ int32_t tStartDecode(SDecoder* pCoder) { pNode = tDecoderMalloc(pCoder, sizeof(*pNode)); if (pNode == NULL) { - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_RETURN(terrno); } pNode->data = pCoder->data; diff --git a/source/util/src/thash.c b/source/util/src/thash.c index ab88bef1a0..88fe6618b9 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -837,8 +837,8 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p) { // only add the read lock to disable the resize process taosHashRLock(pHashObj); - int slot; - (void)taosHashReleaseNode(pHashObj, p, &slot); + int slot; + void *tp = taosHashReleaseNode(pHashObj, p, &slot); SHashEntry *pe = pHashObj->hashList[slot]; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index aea6647e4c..2a7ca2c7dd 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -125,6 +125,8 @@ int32_t idxDebugFlag = 131; int32_t sndDebugFlag = 131; int32_t simDebugFlag = 131; +int32_t tqClientDebug = 0; + int64_t dbgEmptyW = 0; int64_t dbgWN = 0; int64_t dbgSmallWN = 0; @@ -153,14 +155,16 @@ static void getDay(char* buf){ time_t t = taosTime(NULL); struct tm tmInfo; if (taosLocalTime(&t, &tmInfo, buf) != NULL) { - (void)strftime(buf, LOG_FILE_DAY_LEN, "%Y-%m-%d", &tmInfo); + TAOS_UNUSED(strftime(buf, LOG_FILE_DAY_LEN, "%Y-%m-%d", &tmInfo)); } } static int64_t getTimestampToday() { time_t t = taosTime(NULL); struct tm tm; - (void) taosLocalTime(&t, &tm, NULL); + if (taosLocalTime(&t, &tm, NULL) == NULL) { + return 0; + } tm.tm_hour = 0; tm.tm_min = 0; tm.tm_sec = 0; @@ -203,7 +207,7 @@ int32_t taosInitSlowLog() { tsLogObj.slowHandle = taosLogBuffNew(LOG_SLOW_BUF_SIZE); if (tsLogObj.slowHandle == NULL) return terrno; - (void)taosUmaskFile(0); + TAOS_UNUSED(taosUmaskFile(0)); tsLogObj.slowHandle->pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (tsLogObj.slowHandle->pFile == NULL) { (void)printf("\nfailed to open slow log file:%s, reason:%s\n", name, strerror(errno)); @@ -281,7 +285,10 @@ static void taosUnLockLogFile(TdFilePtr pFile) { if (pFile == NULL) return; if (tsLogObj.fileNum > 1) { - (void)taosUnLockFile(pFile); + int32_t code = taosUnLockFile(pFile); + if (code != 0) { + TAOS_UNUSED(printf("failed to unlock log file:%p, reason:%s\n", pFile, tstrerror(code))); + } } } @@ -310,7 +317,10 @@ static void taosKeepOldLog(char *oldName) { char compressFileName[PATH_MAX + 20]; snprintf(compressFileName, PATH_MAX + 20, "%s.gz", oldName); if (taosCompressFile(oldName, compressFileName) == 0) { - (void)taosRemoveFile(oldName); + int32_t code = taosRemoveFile(oldName); + if (code != 0) { + TAOS_UNUSED(printf("failed to remove file:%s, reason:%s\n", oldName, tstrerror(code))); + } } } @@ -331,7 +341,7 @@ static OldFileKeeper *taosOpenNewFile() { char name[PATH_MAX + 20]; sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag); - (void)taosUmaskFile(0); + TAOS_UNUSED(taosUmaskFile(0)); TdFilePtr pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { @@ -341,8 +351,10 @@ static OldFileKeeper *taosOpenNewFile() { return NULL; } - (void)taosLockLogFile(pFile); - (void)taosLSeekFile(pFile, 0, SEEK_SET); + TAOS_UNUSED(taosLockLogFile(pFile)); + if (taosLSeekFile(pFile, 0, SEEK_SET) < 0) { + uWarn("failed to seek file:%s, reason:%s", name, tstrerror(terrno)); + } TdFilePtr pOldFile = tsLogObj.logHandle->pFile; tsLogObj.logHandle->pFile = pFile; @@ -384,7 +396,14 @@ static int32_t taosOpenNewLogFile() { (void)taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_DETACHED); OldFileKeeper *oldFileKeeper = taosOpenNewFile(); - (void)taosThreadCreate(&thread, &attr, taosThreadToCloseOldFile, oldFileKeeper); + if (!oldFileKeeper) { + TAOS_UNUSED(taosThreadMutexUnlock(&tsLogObj.logMutex)); + return terrno; + } + if (taosThreadCreate(&thread, &attr, taosThreadToCloseOldFile, oldFileKeeper) != 0) { + uError("failed to create thread to close old log file"); + taosMemoryFreeClear(oldFileKeeper); + } (void)taosThreadAttrDestroy(&attr); } @@ -404,7 +423,7 @@ static void taosOpenNewSlowLogFile() { for (int32_t i = 1; atomic_val_compare_exchange_32(&tsLogObj.slowHandle->lock, 0, 1) == 1; ++i) { if (i % 1000 == 0) { - (void)sched_yield(); + TAOS_UNUSED(sched_yield()); } } tsLogObj.slowHandle->lastDuration = LOG_MAX_WAIT_MSEC; // force write @@ -435,7 +454,10 @@ void taosResetLog() { tsLogObj.lines = tsNumOfLogLines + 10; if (tsLogObj.logHandle) { - (void)taosOpenNewLogFile(); + int32_t code = taosOpenNewLogFile(); + if(code != 0){ + uError("failed to open new log file, reason:%s", tstrerror(code)); + } uInfo("=================================="); uInfo(" reset log file "); } @@ -535,7 +557,7 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { (void)sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag); (void)taosThreadMutexInit(&tsLogObj.logMutex, NULL); - (void)taosUmaskFile(0); + TAOS_UNUSED(taosUmaskFile(0)); tsLogObj.logHandle = taosLogBuffNew(LOG_DEFAULT_BUF_SIZE); if (tsLogObj.logHandle == NULL) return terrno; @@ -544,24 +566,41 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { (void)printf("\nfailed to open log file:%s, reason:%s\n", name, strerror(errno)); return terrno; } - (void)taosLockLogFile(tsLogObj.logHandle->pFile); + TAOS_UNUSED(taosLockLogFile(tsLogObj.logHandle->pFile)); // only an estimate for number of lines int64_t filesize = 0; if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) != 0) { (void)printf("\nfailed to fstat log file:%s, reason:%s\n", name, strerror(errno)); + taosUnLockLogFile(tsLogObj.logHandle->pFile); return terrno; } tsLogObj.lines = (int32_t)(filesize / 60); - (void)taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END); + if (taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END) < 0) { + TAOS_UNUSED(printf("failed to seek to the end of log file:%s, reason:%s\n", name, tstrerror(terrno))); + taosUnLockLogFile(tsLogObj.logHandle->pFile); + return terrno; + } (void)sprintf(name, "==================================================\n"); - (void)taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); + if (taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)) <= 0) { + TAOS_UNUSED(printf("failed to write to log file:%s, reason:%s\n", name, tstrerror(terrno))); + taosUnLockLogFile(tsLogObj.logHandle->pFile); + return terrno; + } (void)sprintf(name, " new log file \n"); - (void)taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); + if (taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)) <= 0) { + TAOS_UNUSED(printf("failed to write to log file:%s, reason:%s\n", name, tstrerror(terrno))); + taosUnLockLogFile(tsLogObj.logHandle->pFile); + return terrno; + } (void)sprintf(name, "==================================================\n"); - (void)taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); + if (taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)) <= 0) { + TAOS_UNUSED(printf("failed to write to log file:%s, reason:%s\n", name, tstrerror(terrno))); + taosUnLockLogFile(tsLogObj.logHandle->pFile); + return terrno; + } return 0; } @@ -569,17 +608,17 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { static void taosUpdateLogNums(ELogLevel level) { switch (level) { case DEBUG_ERROR: - (void)atomic_add_fetch_64(&tsNumOfErrorLogs, 1); + TAOS_UNUSED(atomic_add_fetch_64(&tsNumOfErrorLogs, 1)); break; case DEBUG_INFO: - (void)atomic_add_fetch_64(&tsNumOfInfoLogs, 1); + TAOS_UNUSED(atomic_add_fetch_64(&tsNumOfInfoLogs, 1)); break; case DEBUG_DEBUG: - (void)atomic_add_fetch_64(&tsNumOfDebugLogs, 1); + TAOS_UNUSED(atomic_add_fetch_64(&tsNumOfDebugLogs, 1)); break; case DEBUG_DUMP: case DEBUG_TRACE: - (void)atomic_add_fetch_64(&tsNumOfTraceLogs, 1); + TAOS_UNUSED(atomic_add_fetch_64(&tsNumOfTraceLogs, 1)); break; default: break; @@ -590,7 +629,7 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { struct tm Tm, *ptm; struct timeval timeSecs; - (void)taosGetTimeOfDay(&timeSecs); + TAOS_UNUSED(taosGetTimeOfDay(&timeSecs)); time_t curTime = timeSecs.tv_sec; ptm = taosLocalTime(&curTime, &Tm, NULL); @@ -603,15 +642,18 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL && osLogSpaceSufficient()) { taosUpdateLogNums(level); if (tsAsyncLog) { - (void)taosPushLogBuffer(tsLogObj.logHandle, buffer, len); + TAOS_UNUSED(taosPushLogBuffer(tsLogObj.logHandle, buffer, len)); } else { - (void)taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); + TAOS_UNUSED(taosWriteFile(tsLogObj.logHandle->pFile, buffer, len)); } if (tsNumOfLogLines > 0) { - (void)atomic_add_fetch_32(&tsLogObj.lines, 1); + TAOS_UNUSED(atomic_add_fetch_32(&tsLogObj.lines, 1)); if ((tsLogObj.lines > tsNumOfLogLines) && (tsLogObj.openInProgress == 0)) { - (void)taosOpenNewLogFile(); + int32_t code = taosOpenNewLogFile(); + if (code != 0) { + uError("failed to open new log file, reason:%s", tstrerror(code)); + } } } } @@ -619,7 +661,9 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b if (dflag & DEBUG_SCREEN) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-result" - (void)write(1, buffer, (uint32_t)len); + if (write(1, buffer, (uint32_t)len) < 0) { + TAOS_UNUSED(printf("failed to write log to screen, reason:%s\n", strerror(errno))); + } #pragma GCC diagnostic pop } } @@ -690,12 +734,12 @@ void taosPrintSlowLog(const char *format, ...) { buffer[len++] = '\n'; buffer[len] = 0; - (void)atomic_add_fetch_64(&tsNumOfSlowLogs, 1); + TAOS_UNUSED(atomic_add_fetch_64(&tsNumOfSlowLogs, 1)); if (tsAsyncLog) { - (void)taosPushLogBuffer(tsLogObj.slowHandle, buffer, len); + TAOS_UNUSED(taosPushLogBuffer(tsLogObj.slowHandle, buffer, len)); } else { - (void)taosWriteFile(tsLogObj.slowHandle->pFile, buffer, len); + TAOS_UNUSED(taosWriteFile(tsLogObj.slowHandle->pFile, buffer, len)); } taosMemoryFree(buffer); @@ -714,7 +758,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { pos += 3; if (c >= 16) { temp[pos++] = '\n'; - (void)taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos); + TAOS_UNUSED((taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos) <= 0)); c = 0; pos = 0; } @@ -722,7 +766,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { temp[pos++] = '\n'; - (void)taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos); + TAOS_UNUSED(taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos)); } static void taosCloseLogByFd(TdFilePtr pFile) { @@ -855,12 +899,12 @@ static void taosWriteLog(SLogBuff *pLogBuf) { pLogBuf->lastDuration = 0; if (start < end) { - (void)taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf) + start, pollSize); + TAOS_UNUSED(taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf) + start, pollSize)); } else { int32_t tsize = LOG_BUF_SIZE(pLogBuf) - start; - (void)taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf) + start, tsize); + TAOS_UNUSED(taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf) + start, tsize)); - (void)taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf), end); + TAOS_UNUSED(taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf), end)); } dbgWN++; @@ -981,11 +1025,14 @@ void taosLogCrashInfo(char *nodeType, char *pMsg, int64_t msgLen, int signum, vo goto _return; } - (void)taosLockFile(pFile); + if (taosLockFile(pFile) < 0) { + taosPrintLog(flags, level, dflag, "failed to lock file:%s since %s", filepath, terrstr()); + goto _return; + } int64_t writeSize = taosWriteFile(pFile, &msgLen, sizeof(msgLen)); if (sizeof(msgLen) != writeSize) { - (void)taosUnLockFile(pFile); + TAOS_UNUSED(taosUnLockFile(pFile)); taosPrintLog(flags, level, dflag, "failed to write len to file:%s,%p wlen:%" PRId64 " tlen:%lu since %s", filepath, pFile, writeSize, sizeof(msgLen), terrstr()); goto _return; @@ -993,13 +1040,13 @@ void taosLogCrashInfo(char *nodeType, char *pMsg, int64_t msgLen, int signum, vo writeSize = taosWriteFile(pFile, pMsg, msgLen); if (msgLen != writeSize) { - (void)taosUnLockFile(pFile); + TAOS_UNUSED(taosUnLockFile(pFile)); taosPrintLog(flags, level, dflag, "failed to write file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s", filepath, pFile, writeSize, msgLen, terrstr()); goto _return; } - (void)taosUnLockFile(pFile); + TAOS_UNUSED(taosUnLockFile(pFile)); } _return: @@ -1054,7 +1101,7 @@ void taosReadCrashInfo(char *filepath, char **pMsg, int64_t *pMsgLen, TdFilePtr return; } - (void)taosLockFile(pFile); + TAOS_UNUSED(taosLockFile(pFile)); } else { pFile = *pFd; } @@ -1093,10 +1140,10 @@ void taosReadCrashInfo(char *filepath, char **pMsg, int64_t *pMsgLen, TdFilePtr _return: if (truncateFile) { - (void)taosFtruncateFile(pFile, 0); + TAOS_UNUSED(taosFtruncateFile(pFile, 0)); } - (void)taosUnLockFile(pFile); - (void)taosCloseFile(&pFile); + TAOS_UNUSED(taosUnLockFile(pFile)); + TAOS_UNUSED(taosCloseFile(&pFile)); taosMemoryFree(buf); *pMsg = NULL; @@ -1106,11 +1153,11 @@ _return: void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile) { if (truncateFile) { - (void)taosFtruncateFile(pFile, 0); + TAOS_UNUSED(taosFtruncateFile(pFile, 0)); } - (void)taosUnLockFile(pFile); - (void)taosCloseFile(&pFile); + TAOS_UNUSED(taosUnLockFile(pFile)); + TAOS_UNUSED(taosCloseFile(&pFile)); } #ifdef NDEBUG diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index fbd17dd023..69832cd46c 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -38,18 +38,19 @@ enum { }; struct SLRUEntry { - void *value; - _taos_lru_deleter_t deleter; - void *ud; - SLRUEntry *nextHash; - SLRUEntry *next; - SLRUEntry *prev; - size_t totalCharge; - size_t keyLength; - uint32_t hash; - uint32_t refs; - uint8_t flags; - char keyData[1]; + void *value; + _taos_lru_deleter_t deleter; + _taos_lru_overwriter_t overwriter; + void *ud; + SLRUEntry *nextHash; + SLRUEntry *next; + SLRUEntry *prev; + size_t totalCharge; + size_t keyLength; + uint32_t hash; + uint32_t refs; + uint8_t flags; + char keyData[1]; }; #define TAOS_LRU_ENTRY_IN_CACHE(h) ((h)->flags & TAOS_LRU_IN_CACHE) @@ -403,6 +404,10 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry * if (old != NULL) { status = TAOS_LRU_STATUS_OK_OVERWRITTEN; + if (old->overwriter) { + (*old->overwriter)(old->keyData, old->keyLength, old->value, old->ud); + } + TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); if (!TAOS_LRU_ENTRY_HAS_REFS(old)) { taosLRUCacheShardLRURemove(shard, old); @@ -440,8 +445,9 @@ _exit: } static LRUStatus taosLRUCacheShardInsert(SLRUCacheShard *shard, const void *key, size_t keyLen, uint32_t hash, - void *value, size_t charge, _taos_lru_deleter_t deleter, LRUHandle **handle, - LRUPriority priority, void *ud) { + void *value, size_t charge, _taos_lru_deleter_t deleter, + _taos_lru_overwriter_t overwriter, LRUHandle **handle, LRUPriority priority, + void *ud) { SLRUEntry *e = taosMemoryCalloc(1, sizeof(SLRUEntry) - 1 + keyLen); if (!e) { if (deleter) { @@ -453,6 +459,7 @@ static LRUStatus taosLRUCacheShardInsert(SLRUCacheShard *shard, const void *key, e->value = value; e->flags = 0; e->deleter = deleter; + e->overwriter = overwriter; e->ud = ud; e->keyLength = keyLen; e->hash = hash; @@ -726,12 +733,12 @@ void taosLRUCacheCleanup(SLRUCache *cache) { } LRUStatus taosLRUCacheInsert(SLRUCache *cache, const void *key, size_t keyLen, void *value, size_t charge, - _taos_lru_deleter_t deleter, LRUHandle **handle, LRUPriority priority, void *ud) { + _taos_lru_deleter_t deleter, _taos_lru_overwriter_t overwriter, LRUHandle **handle, LRUPriority priority, void *ud) { uint32_t hash = TAOS_LRU_CACHE_SHARD_HASH32(key, keyLen); uint32_t shardIndex = hash & cache->shardedCache.shardMask; - return taosLRUCacheShardInsert(&cache->shards[shardIndex], key, keyLen, hash, value, charge, deleter, handle, - priority, ud); + return taosLRUCacheShardInsert(&cache->shards[shardIndex], key, keyLen, hash, value, charge, deleter, overwriter, + handle, priority, ud); } LRUHandle *taosLRUCacheLookup(SLRUCache *cache, const void *key, size_t keyLen) { @@ -869,4 +876,4 @@ bool taosLRUCacheIsStrictCapacity(SLRUCache *cache) { (void)taosThreadMutexUnlock(&cache->shardedCache.capacityMutex); return strict; -} \ No newline at end of file +} diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 25e10a17df..fb8e597163 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -134,9 +134,8 @@ static uint64_t allocateNewPositionInFile(SDiskbasedBuf* pBuf, size_t size) { static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { return pageSize + POINTER_BYTES + sizeof(SFilePage); } static int32_t doFlushBufPageImpl(SDiskbasedBuf* pBuf, int64_t offset, const char* pData, int32_t size) { - int32_t ret = taosLSeekFile(pBuf->pFile, offset, SEEK_SET); - if (ret == -1) { - terrno = terrno; + int64_t ret = taosLSeekFile(pBuf->pFile, offset, SEEK_SET); + if (ret < 0) { return terrno; } @@ -246,14 +245,14 @@ static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { return TSDB_CODE_INVALID_PARA; } - int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); - if (ret == -1) { + int64_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); + if (ret < 0) { ret = terrno; return ret; } void* pPage = (void*)GET_PAYLOAD_DATA(pg); - ret = (int32_t)taosReadFile(pBuf->pFile, pPage, pg->length); + ret = taosReadFile(pBuf->pFile, pPage, pg->length); if (ret != pg->length) { ret = terrno; return ret; diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index d067847376..f531d9ad61 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -224,7 +224,9 @@ int32_t taosWriteQitem(STaosQueue *queue, void *pItem) { (void)taosThreadMutexUnlock(&queue->mutex); if (queue->qset) { - (void)tsem_post(&queue->qset->sem); + if (tsem_post(&queue->qset->sem) != 0) { + uError("failed to post semaphore for queue set:%p", queue->qset); + } } return code; } @@ -333,7 +335,10 @@ int32_t taosOpenQset(STaosQset **qset) { } (void)taosThreadMutexInit(&(*qset)->mutex, NULL); - (void)tsem_init(&(*qset)->sem, 0, 0); + if (tsem_init(&(*qset)->sem, 0, 0) != 0) { + taosMemoryFree(*qset); + return terrno; + } uDebug("qset:%p is opened", qset); return 0; @@ -354,7 +359,9 @@ void taosCloseQset(STaosQset *qset) { (void)taosThreadMutexUnlock(&qset->mutex); (void)taosThreadMutexDestroy(&qset->mutex); - (void)tsem_destroy(&qset->sem); + if (tsem_destroy(&qset->sem) != 0) { + uError("failed to destroy semaphore for qset:%p", qset); + } taosMemoryFree(qset); uDebug("qset:%p is closed", qset); } @@ -364,7 +371,9 @@ void taosCloseQset(STaosQset *qset) { // thread to exit. void taosQsetThreadResume(STaosQset *qset) { uDebug("qset:%p, it will exit", qset); - (void)tsem_post(&qset->sem); + if (tsem_post(&qset->sem) != 0) { + uError("failed to post semaphore for qset:%p", qset); + } } int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle) { @@ -432,7 +441,9 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, SQueueInfo *qinfo) STaosQnode *pNode = NULL; int32_t code = 0; - (void)tsem_wait(&qset->sem); + if (tsem_wait(&qset->sem) != 0) { + uError("failed to wait semaphore for qset:%p", qset); + } (void)taosThreadMutexLock(&qset->mutex); @@ -476,7 +487,9 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo * STaosQueue *queue; int32_t code = 0; - (void)tsem_wait(&qset->sem); + if (tsem_wait(&qset->sem) != 0) { + uError("failed to wait semaphore for qset:%p", qset); + } (void)taosThreadMutexLock(&qset->mutex); for (int32_t i = 0; i < qset->numOfQueues; ++i) { @@ -510,7 +523,9 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo * (void)atomic_sub_fetch_32(&qset->numOfItems, qall->numOfItems); for (int32_t j = 1; j < qall->numOfItems; ++j) { - (void)tsem_wait(&qset->sem); + if (tsem_wait(&qset->sem) != 0) { + uError("failed to wait semaphore for qset:%p", qset); + } } } diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index ebc076e02e..a45d157366 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -46,7 +46,7 @@ int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf* pSBf->numBits = 0; pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void*)); if (!pSBf->bfArray) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _error); } @@ -76,7 +76,7 @@ int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t le int32_t size = taosArrayGetSize(pSBf->bfArray); SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); if (!pNormalBf) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _error); } if (tBloomFilterIsFull(pNormalBf)) { @@ -236,7 +236,7 @@ int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) { } pSBf->bfArray = taosArrayInit(size * 2, POINTER_BYTES); if (!pSBf->bfArray) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _error); } @@ -246,7 +246,7 @@ int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) { QUERY_CHECK_CODE(code, lino, _error); void* tmpRes = taosArrayPush(pSBf->bfArray, &pBF); if (!tmpRes) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _error); } } diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 1686b41038..8c708ac6b5 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -178,7 +178,6 @@ void *taosProcessSchedQueue(void *scheduler) { (*(msg.tfp))(msg.ahandle, msg.thandle); } - return NULL; } @@ -230,7 +229,9 @@ void taosCleanUpScheduler(void *param) { for (int32_t i = 0; i < pSched->numOfThreads; ++i) { if (taosCheckPthreadValid(pSched->qthread[i])) { - (void)tsem_post(&pSched->fullSem); + if (tsem_post(&pSched->fullSem) != 0) { + uError("post %s fullSem failed(%s)", pSched->label, strerror(terrno)); + } } } for (int32_t i = 0; i < pSched->numOfThreads; ++i) { @@ -240,12 +241,17 @@ void taosCleanUpScheduler(void *param) { } } - (void)tsem_destroy(&pSched->emptySem); - (void)tsem_destroy(&pSched->fullSem); + if (tsem_destroy(&pSched->emptySem) != 0) { + uError("failed to destroy %s emptySem", pSched->label); + } + if (tsem_destroy(&pSched->fullSem) != 0) { + uError("failed to destroy %s fullSem", pSched->label); + } (void)taosThreadMutexDestroy(&pSched->queueMutex); if (pSched->pTimer) { - (void)taosTmrStop(pSched->pTimer); + bool r = taosTmrStop(pSched->pTimer); + uTrace("stop timer:%p, result:%d", pSched->pTimer, r); pSched->pTimer = NULL; } diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index d5310b2e7e..b11f65da8f 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -314,7 +314,9 @@ static void addToExpired(tmr_obj_t* head) { schedMsg.msg = NULL; schedMsg.ahandle = head; schedMsg.thandle = NULL; - (void)taosScheduleTask(tmrQhandle, &schedMsg); + if (taosScheduleTask(tmrQhandle, &schedMsg) != 0) { + tmrError("%s failed to add expired timer[id=%" PRIuPTR "] to queue.", head->ctrl->label, id); + } tmrDebug("timer[id=%" PRIuPTR "] has been added to queue.", id); head = next; @@ -508,7 +510,7 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* han } } - if (timer->refCount == 1) { + if (timer->refCount != 1) { uError("timer refCount=%d not expected 1", timer->refCount); } memset(timer, 0, sizeof(*timer)); @@ -560,7 +562,9 @@ static int32_t taosTmrModuleInit(void) { } tmrQhandle = taosInitScheduler(10000, taosTmrThreads, "tmr", NULL); - (void)taosInitTimer(taosTimerLoopFunc, MSECONDS_PER_TICK); + if (taosInitTimer(taosTimerLoopFunc, MSECONDS_PER_TICK) != 0) { + tmrError("failed to initialize timer"); + } tmrDebug("timer module is initialized, number of threads: %d", taosTmrThreads); diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 767fb5ce3c..6370e6ca50 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -178,7 +178,7 @@ int32_t tAutoQWorkerInit(SAutoQWorkerPool *pool) { pool->workers = taosArrayInit(2, sizeof(SQueueWorker *)); if (pool->workers == NULL) { taosCloseQset(pool->qset); - return terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)taosThreadMutexInit(&pool->mutex, NULL); @@ -440,7 +440,8 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (taosThreadCreate(&worker->thread, &thAttr, (ThreadFp)tWWorkerThreadFp, worker) != 0) goto _OVER; + code = taosThreadCreate(&worker->thread, &thAttr, (ThreadFp)tWWorkerThreadFp, worker); + if ((code)) goto _OVER; uInfo("worker:%s:%d is launched, max:%d", pool->name, worker->id, pool->max); pool->nextId = (pool->nextId + 1) % pool->max; diff --git a/tests/army/cluster/test_drop_table_by_uid.py b/tests/army/cluster/test_drop_table_by_uid.py new file mode 100644 index 0000000000..f09006b37b --- /dev/null +++ b/tests/army/cluster/test_drop_table_by_uid.py @@ -0,0 +1,368 @@ +import time +import random +import taos +from enum import Enum + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.srvCtl import * + + +class TDTestCase(TBase): + """ + Description: This class is used to verify the feature of 'drop table by uid' for task TS-5111 + FS: https://taosdata.feishu.cn/wiki/JgeDwZkH3iTNv2ksVkWcHenKnTf + TS: https://taosdata.feishu.cn/wiki/DX3FwopwGiXCeRkBNXFcj0MBnnb + create: + 2024-09-23 created by Charles + update: + None + """ + + class TableType(Enum): + STABLE = 0 + CHILD_TABLE = 1 + REGULAR_TABLE = 2 + + def init(self, conn, logSql, replicaVar=1): + """Initialize the TDengine cluster environment + """ + super(TDTestCase, self).init(conn, logSql, replicaVar, db="db") + tdSql.init(conn.cursor(), logSql) + + def get_uid_by_db_table_name(self, db_name, table_name, table_type=TableType.STABLE): + """Get table uid with db name and table name from system table + :param db_name: database name + :param table_name: table name + :param table_type: table type, default is stable + :return: table uid + """ + if table_type == self.TableType.STABLE: + tdSql.query(f"select * from information_schema.ins_stables where db_name='{db_name}' and stable_name like '%{table_name}%';") + elif table_type == self.TableType.CHILD_TABLE: + tdSql.query(f"select * from information_schema.ins_tables where db_name='{db_name}' and table_name like '%{table_name}%' and stable_name is not null;") + else: + tdSql.query(f"select * from information_schema.ins_tables where db_name='{db_name}' and table_name like '%{table_name}%' and stable_name is null;") + # check whether the table uid is empty + if len(tdSql.res) == 0: + tdLog.debug(f"Can't get table uid with db name: {db_name} and table name: {table_name}") + return None + # get table uid list + if table_type == self.TableType.STABLE: + return [item[10] for item in tdSql.res] + else: + return [item[5] for item in tdSql.res] + + def get_uid_by_db_name(self, db_name, table_type=TableType.STABLE): + """Get table uid with db name and table type from system table + :param db_name: database name + :param table_type: table type, default is stable + :return: table uid list + """ + if table_type == self.TableType.STABLE: + tdSql.query(f"select * from information_schema.ins_stables where db_name='{db_name}';") + elif table_type == self.TableType.CHILD_TABLE: + tdSql.query(f"select * from information_schema.ins_tables where db_name='{db_name}' and stable_name is not null;") + else: + tdSql.query(f"select * from information_schema.ins_tables where db_name='{db_name}' and stable_name is null;") + # check whether the table uid is empty + if len(tdSql.res) == 0: + tdLog.debug(f"Can't get table uid with db name: {db_name}") + return None + # get table uid list + if table_type == self.TableType.STABLE: + return [item[10] for item in tdSql.res] + else: + return [item[5] for item in tdSql.res] + + def drop_table_by_uid(self, uid_list, table_type=TableType.STABLE, exist_ops=False): + """Drop the specified tables by uid list + :db_name: database name + :param uid_list: table uid list to be dropped + :param exist_ops: whether to use exist option, default is False + :return: None + """ + # check whether the uid list is empty + if len(uid_list) == 0: + return + # drop table by uid + if exist_ops and table_type == self.TableType.STABLE: + for uid in uid_list: + tdSql.execute(f"drop stable with if exists `{uid}`;") + else: + uids = ','.join(["`" + str(item) + "`" for item in uid_list]) + tdSql.execute(f"drop table with {uids};") + + def test_drop_single_table_by_uid(self): + """Verify the feature of dropping a single stable/child table/regular table by uid with root user + """ + db_name = "test_drop_single_table_by_uid" + tdLog.info("Start test case: test_drop_single_table_by_uid") + # data for case test_drop_single_table_by_uid + tdLog.info("Prepare data for test case test_drop_single_table_by_uid") + tdSql.execute(f"create database {db_name};") + tdSql.execute(f"use {db_name};") + # table with normal characters + tdSql.execute("create stable st1 (ts timestamp, c1 int) tags (t1 int);") + tdSql.execute("create table ct1_1 using st1 tags(1);") + tdSql.execute("create table t1 (ts timestamp, c1 int, c2 float);") + tdLog.info("Finish preparing data for test case test_drop_single_table_by_uid of normal characters") + # get table uid + uid_st1 = self.get_uid_by_db_table_name(db_name, "st1") + tdLog.debug(f"uid_st1: {uid_st1}") + uid_ct1_1 = self.get_uid_by_db_table_name(db_name, "ct1_1", self.TableType.CHILD_TABLE) + tdLog.debug(f"uid_ct1_1: {uid_ct1_1}") + uid_t1 = self.get_uid_by_db_table_name(db_name, "t1", self.TableType.REGULAR_TABLE) + tdLog.debug(f"uid_t1: {uid_t1}") + # drop table by uid + self.drop_table_by_uid(uid_t1, self.TableType.REGULAR_TABLE) + self.drop_table_by_uid(uid_ct1_1, self.TableType.CHILD_TABLE) + self.drop_table_by_uid(uid_st1, self.TableType.STABLE, True) + + # table with special characters + tdSql.execute("create stable `st2\u00bf\u200bfnn1` (ts timestamp, c1 int) tags (t1 int);") + tdSql.execute("create table `ct2_1\u00cf\u00ff` using `st2\u00bf\u200bfnn1` tags(1);") + tdSql.execute("create table `t2\u00ef\u00fa` (ts timestamp, c1 int, c2 float);") + tdLog.info("Finish preparing data for test case test_drop_single_table_by_uid of special characters") + # get table uid + uid_st2 = self.get_uid_by_db_table_name(db_name, "st2") + tdLog.debug(f"uid_st2: {uid_st2}") + uid_ct2_1 = self.get_uid_by_db_table_name(db_name, "ct2_1", self.TableType.CHILD_TABLE) + tdLog.debug(f"uid_ct2_1: {uid_ct2_1}") + uid_t2 = self.get_uid_by_db_table_name(db_name, "t2", self.TableType.REGULAR_TABLE) + tdLog.debug(f"uid_t2: {uid_t2}") + # drop table by uid + self.drop_table_by_uid(uid_t2, self.TableType.REGULAR_TABLE) + self.drop_table_by_uid(uid_ct2_1, self.TableType.CHILD_TABLE) + self.drop_table_by_uid(uid_st2, self.TableType.STABLE, True) + tdSql.execute(f"drop database {db_name};") + tdLog.info("Finish test case: test_drop_single_table_by_uid") + + def test_drop_multiple_tables_by_uid(self): + """Verify the feature of dropping multiple tables by uid with root user + """ + db_name = "test_drop_multiple_tables_by_uid" + table_number = 100 + tdLog.info("Start test case: test_drop_multiple_tables_by_uid") + # data for case test_drop_multiple_tables_by_uid + tdLog.info("Prepare data for test case test_drop_multiple_tables_by_uid") + tdSql.execute(f"create database {db_name};") + tdSql.execute(f"use {db_name};") + # table with normal characters + for i in range(table_number): + tdSql.execute(f"create stable st{i} (ts timestamp, c1 int) tags (t1 int);") + tdSql.execute(f"create table ct{i}_{i} using st{i} tags({i+1});") + tdSql.execute(f"create table t{i} (ts timestamp, c1 int, c2 float);") + tdLog.info("Finish preparing data for test case test_drop_multiple_tables_by_uid of normal characters") + # get table uid + uid_st = self.get_uid_by_db_table_name(db_name, "st") + # tdLog.debug(f"Get multiple stable uid list: {uid_st}") + uid_ct = self.get_uid_by_db_table_name(db_name, "ct", self.TableType.CHILD_TABLE) + # tdLog.debug(f"Get multiple child table uid list: {uid_ct}") + uid_t = self.get_uid_by_db_table_name(db_name, "t", self.TableType.REGULAR_TABLE) + # tdLog.debug(f"Get multiple regular table uid list: {uid_t}") + # drop table by uid + self.drop_table_by_uid(uid_t, self.TableType.REGULAR_TABLE) + self.drop_table_by_uid(uid_ct, self.TableType.CHILD_TABLE) + self.drop_table_by_uid(uid_st, self.TableType.STABLE, True) + + # table with special characters + for i in range(table_number): + tdSql.execute(f"create stable `st{i}\u00bf\u200bfnn1` (ts timestamp, c1 int) tags (t1 int);") + tdSql.execute(f"create table `ct{i}_{i}\u00cf\u00ff` using `st{i}\u00bf\u200bfnn1` tags(1);") + tdSql.execute(f"create table `t{i}\u00ef\u00fa` (ts timestamp, c1 int, c2 float);") + # get table uid + uid_st = self.get_uid_by_db_table_name(db_name, "st") + # tdLog.debug(f"Get multiple stable uid list: {uid_st}") + uid_ct = self.get_uid_by_db_table_name(db_name, "ct", self.TableType.CHILD_TABLE) + # tdLog.debug(f"Get multiple child table uid list: {uid_ct}") + uid_t = self.get_uid_by_db_table_name(db_name, "t", self.TableType.REGULAR_TABLE) + # tdLog.debug(f"Get multiple regular table uid list: {uid_t}") + # drop table by uid + self.drop_table_by_uid(uid_t, self.TableType.REGULAR_TABLE) + self.drop_table_by_uid(uid_ct, self.TableType.CHILD_TABLE) + self.drop_table_by_uid(uid_st, self.TableType.STABLE, True) + tdSql.execute(f"drop database {db_name};") + tdLog.info("Finish test case: test_drop_multiple_tables_by_uid") + + def test_uid_as_table_name(self): + """Verify using uid as table name, drop table with uid doesn't affect other tables + """ + db_name = "test_uid_as_table_name" + tdLog.info("Start test case: test_uid_as_table_name") + # data for case test_uid_as_table_name + tdLog.info("Prepare data for test case test_uid_as_table_name") + tdSql.execute(f"create database {db_name};") + tdSql.execute(f"use {db_name};") + # super table + tdSql.execute(f"create stable `st1\u00bf\u200bfnn1` (ts timestamp, c1 int) tags (t1 int);") + uid_st = self.get_uid_by_db_table_name(db_name, "st") + tdSql.execute(f"create stable `{uid_st[0]}` (ts timestamp, c1 int) tags (t1 int);") + self.drop_table_by_uid(uid_st, self.TableType.STABLE, True) + uid_st = self.get_uid_by_db_table_name(db_name, str(uid_st[0])) + assert uid_st is not None + tdLog.info(f"Drop stable with special characters with uid {uid_st[0]}, stable named as {uid_st[0]} doesn't be affected") + # child table + tdSql.execute(f"create stable `st2\u00bf\u200bfnn1` (ts timestamp, c1 int) tags (t1 int);") + tdSql.execute(f"create table `ct2_1\u00cf\u00ff` using `st2\u00bf\u200bfnn1` tags(1);") + uid_ct = self.get_uid_by_db_table_name(db_name, "ct", self.TableType.CHILD_TABLE) + tdSql.execute(f"create table `{uid_ct[0]}` using `st2\u00bf\u200bfnn1` tags(2);") + self.drop_table_by_uid(uid_ct, self.TableType.CHILD_TABLE) + uid_ct = self.get_uid_by_db_table_name(db_name, str(uid_ct[0]), self.TableType.CHILD_TABLE) + assert uid_ct is not None + tdLog.info(f"Drop child table with special characters with uid {uid_ct[0]}, child table named as {uid_ct[0]} doesn't be affected") + # regular table + tdSql.execute(f"create table `t2\u00bf\u200bfnn1` (ts timestamp, c1 int);") + uid_t = self.get_uid_by_db_table_name(db_name, "t2", self.TableType.REGULAR_TABLE) + tdSql.execute(f"create table `{uid_t[0]}` (ts timestamp, c1 int);") + self.drop_table_by_uid(uid_t, self.TableType.REGULAR_TABLE) + uid_t = self.get_uid_by_db_table_name(db_name, str(uid_t[0]), self.TableType.REGULAR_TABLE) + assert uid_t is not None + tdLog.info(f"Drop regular table with special characters with uid {uid_t[0]}, regular table named as {uid_t[0]} doesn't be affected") + tdSql.execute(f"drop database {db_name};") + tdLog.info("Finish test case: test_uid_as_table_name") + + def test_abnormal_non_exist_uid(self): + """Verify dropping table with non-exist uid + """ + db_name = "test_abnormal_non_exist_uid" + tdLog.info("Start test case: test_abnormal_non_exist_uid") + # data for case test_abnormal_non_exist_uid + tdLog.info("Prepare data for test case test_abnormal_non_exist_uid") + tdSql.execute(f"create database {db_name};") + tdSql.execute(f"use {db_name};") + # drop table with non-exist uid + tdSql.error(f"drop stable with if exists `1234567890`;", expectErrInfo="STable not exist:") + tdSql.error(f"drop table with `1234567890`;", expectErrInfo="Table does not exist:") + tdSql.execute(f"drop database {db_name};") + tdLog.info("Finish test case: test_abnormal_non_exist_uid") + + def test_abnormal_incorrect_table_type(self): + """Verify dropping table with incorrect sql, like drop stable sql with table or child table uid + """ + try: + db_name = "test_abnormal_incorrect_table_type" + tdLog.info("Start test case: test_abnormal_incorrect_table_type") + # data for case test_abnormal_incorrect_table_type + tdLog.info("Prepare data for test case test_abnormal_incorrect_table_type") + tdSql.execute(f"create database {db_name};") + tdSql.execute(f"use {db_name};") + tdSql.execute("create stable `st3\u00bf\u200bfnn1` (ts timestamp, c1 int) tags (t1 int);") + tdSql.execute("create table `ct3_1\u00cf\u00ff` using `st3\u00bf\u200bfnn1` tags(1);") + tdSql.execute("create table `t3\u00ef\u00fa` (ts timestamp, c1 int, c2 float);") + tdLog.info("Finish preparing data for test case test_abnormal_incorrect_table_type of special characters") + # get table uid + uid_st = self.get_uid_by_db_table_name(db_name, "st") + uid_ct = self.get_uid_by_db_table_name(db_name, "ct", self.TableType.CHILD_TABLE) + uid_t = self.get_uid_by_db_table_name(db_name, "t", self.TableType.REGULAR_TABLE) + # drop table with incorrect sql + tdSql.error(f"drop stable with `{uid_ct[0]}`;", expectErrInfo="STable not exist") + tdSql.error(f"drop stable with `{uid_t[0]}`;", expectErrInfo="STable not exist") + tdLog.info("Finish test case: test_abnormal_incorrect_table_type") + except Exception as e: + tdLog.exit("Failed to run test case test_abnormal_incorrect_table_type with msg: %s" % str(e)) + finally: + tdSql.execute(f"drop database {db_name};") + + def test_abnormal_mixed_uid(self): + """Verify dropping table with mixed uid + """ + db_name = "test_abnormal_mixed_uid" + tdLog.info("Start test case: test_abnormal_mixed_uid") + # data for case test_abnormal_mixed_uidF + tdLog.info("Prepare data for test case test_abnormal_mixed_uid") + tdSql.execute(f"create database {db_name};") + tdSql.execute(f"use {db_name};") + tdSql.execute("create stable `st3\u00bf\u200bfnn1` (ts timestamp, c1 int) tags (t1 int);") + tdSql.execute("create table `ct3_1\u00cf\u00ff` using `st3\u00bf\u200bfnn1` tags(1);") + tdSql.execute("create table `t3\u00ef\u00fa` (ts timestamp, c1 int, c2 float);") + tdLog.info("Finish preparing data for test case test_abnormal_mixed_uid of special characters") + # get table uid + uid_st = self.get_uid_by_db_table_name(db_name, "st") + uid_ct = self.get_uid_by_db_table_name(db_name, "ct", self.TableType.CHILD_TABLE) + uid_t = self.get_uid_by_db_table_name(db_name, "t", self.TableType.REGULAR_TABLE) + # drop table with incorrect sql + tdSql.error(f"drop stable with `{uid_st[0]}`,`{uid_ct[0]}`;", expectErrInfo="syntax error") + tdSql.error(f"drop table with `{uid_st[0]}`,`{uid_ct[0]}`,`{uid_t[0]}`;", expectErrInfo="Cannot drop super table in batch") + tdSql.execute(f"drop database {db_name};") + tdLog.info("Finish test case: test_abnormal_mixed_uid") + + def test_abnormal_system_tables(self): + """Verify dropping system tables + """ + try: + uid_list = self.get_uid_by_db_name("information_schema", self.TableType.REGULAR_TABLE) + uid = random.choice(uid_list) + assert uid is None + except Exception as e: + tdLog.exit("Failed to run test case test_abnormal_system_tables with msg: %s" % str(e)) + + def test_abnormal_drop_table_with_non_root_user(self): + """Verify dropping table with non-root user + """ + try: + # create new user and grant create database priviledge + tdSql.execute("create user test pass 'test';") + tdSql.execute("alter user test createdb 1;") + conn = taos.connect(user="test", password="test") + cursor = conn.cursor() + # create database and tables with new user + tdLog.info("Prepare data for test case test_abnormal_drop_table_with_non_root_user") + db_name = "test_abnormal_drop_table_with_non_root_user" + cursor.execute(f"create database {db_name};") + cursor.execute(f"use {db_name};") + time.sleep(3) + cursor.execute("create stable `st4\u00bf\u200bfnn1` (ts timestamp, c1 int) tags (t1 int);") + cursor.execute("create table `ct4_1\u00cf\u00ff` using `st4\u00bf\u200bfnn1` tags(1);") + cursor.execute("create table `t4\u00ef\u00fa` (ts timestamp, c1 int, c2 float);") + tdLog.info("Finish preparing data for test case test_abnormal_drop_table_with_non_root_user of special characters") + # get table uid + uid_st = self.get_uid_by_db_table_name(db_name, "st") + uid_ct = self.get_uid_by_db_table_name(db_name, "ct", self.TableType.CHILD_TABLE) + uid_t = self.get_uid_by_db_table_name(db_name, "t", self.TableType.REGULAR_TABLE) + # drop stable with sql by non-root user + try: + cursor.execute(f"drop stable with `{uid_st[0]}`;") + except Exception as e: + assert "Permission denied or target object not exist" in str(e) + tdLog.info("Drop stable with non-root user failed as expected") + # drop child table with sql by non-root user + try: + cursor.execute(f"drop table with `{uid_ct[0]}`;") + except Exception as e: + assert "Permission denied or target object not exist" in str(e) + tdLog.info("Drop child table with non-root user failed as expected") + # drop regular table with sql by non-root user + try: + cursor.execute(f"drop table with `{uid_t[0]}`;") + except Exception as e: + assert "Permission denied or target object not exist" in str(e) + tdLog.info("Drop regular table with non-root user failed as expected") + tdLog.info("Finish test case: test_abnormal_drop_table_with_non_root_user") + except Exception as e: + tdLog.exit("Failed to run test case test_abnormal_drop_table_with_non_root_user with msg: %s" % str(e)) + finally: + tdSql.execute(f"drop database {db_name};") + tdSql.execute("drop user test;") + + def run(self): + # normal cases + self.test_drop_single_table_by_uid() + self.test_drop_multiple_tables_by_uid() + self.test_uid_as_table_name() + # abnormal cases + self.test_abnormal_non_exist_uid() + self.test_abnormal_incorrect_table_type() + self.test_abnormal_mixed_uid() + self.test_abnormal_system_tables() + self.test_abnormal_drop_table_with_non_root_user() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/ci/check_void.sh b/tests/ci/check_void.sh new file mode 100755 index 0000000000..0937521834 --- /dev/null +++ b/tests/ci/check_void.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# usage: check_void.sh -c func.txt +# Example: check_void.sh -c func.txt -f TDengine/source +# Example: check_void.sh -c func.txt -f TDengine/source/util/src/ttimer.c + +while getopts "c:f:h" arg +do + case "$arg" in + c) + func_list_file=$OPTARG + ;; + f) + source_file=$OPTARG + ;; + h) + echo "Usage: -c [function list file] " + echo " -f [source file or directory] " + echo " -h help" + exit 0 + ;; + *) + echo "Invalid option: -$OPTARG" + exit 1 + ;; + esac +done + +# echo "funclist list configuration file: $func_list_file" + +if [ ! -e "$func_list_file" ] || [ ! -r "$func_list_file" ] +then + echo "$func_list_file doesn't exist or is not readable" +fi + +# cat $func_list_file + +grep_string="" +separator="\|" + +while read -r line +do + # echo "current function: $line" + tmp="$grep_string" + grep_string="$tmp$line$separator" + # echo "grep_string: $grep_string" +done < $func_list_file + +# echo "${grep_string:1:-2}" + +# echo "We are going to check (void) function invocation in $*" + +grep -Rn "${grep_string:1:-2}" $source_file |grep -v "test" > ./check_void_result.txt + +if [ -s ./check_void_result.txt ] +then + failed_number=$(cat ./check_void_result.txt|wc -l ) + echo "Found $failed_number (void) function invocation in $source_file:" + cat ./check_void_result.txt + exit 1 +else + echo "No (void) function invocation in $source_file" + exit 0 +fi \ No newline at end of file diff --git a/tests/ci/func.txt b/tests/ci/func.txt new file mode 100644 index 0000000000..45d4fb1c11 --- /dev/null +++ b/tests/ci/func.txt @@ -0,0 +1,558 @@ +(void)TARRAY2_SORT_INSERT +(void)TDB_INIT_PAGE_LOCK +(void)TDB_UNLOCK_PAGE +(void)addDownstreamFailedStatusResultAsync +(void)addEpIntoEpSet +(void)appendCheckpointIntoInputQ +(void)applyLimitOffset +(void)assignOneDataBlock +(void)catalogGetHandle +(void)catalogRemoveTableMeta +(void)catalogRemoveViewMeta +(void)cliBuildExceptResp +(void)cliUpdateFqdnCache +(void)colDataSetVal +(void)continueDispatchCheckpointTriggerBlock +(void)createStreamTaskIdStr +(void)dBufSetBufPageRecycled +(void)dmWriteEps +(void)doRegisterCacheObj +(void)doSendRetrieveTriggerMsg +(void)epsetToStr +(void)fflush +(void)gzwrite +(void)handleScanhistoryResultBlocks +(void)hashset_add +(void)hashset_add_member +(void)idxFileWrite +(void)idxFlushCacheToTFile +(void)idxTermSearch +(void)indexMultiTermAdd +(void)loadDataTomb +(void)metaCacheUpsert +(void)metaDecodeEntry +(void)metaDelJsonVarFromIdx +(void)metaDeleteNcolIdx +(void)metaDeleteTtl +(void)metaDropTableByUid +(void)metaGetInfo +(void)metaGetStbStats +(void)metaGetTableEntryByVersion +(void)metaInitLock +(void)metaSaveToSkmDb +(void)metaSaveToTbDb +(void)metaStatsCacheDrop +(void)metaStatsCacheUpsert +(void)metaUpdateChangeTime +(void)metaUpdateMetaRsp +(void)metaUpdateNcolIdx +(void)metaUpdateTagIdx +(void)metaUpdateTtl +(void)metaUpdateUidIdx +(void)mndGetMonitorInfo +(void)mndGetOrCreateRebSub +(void)mndPullupArbUpdateGroupBatch +(void)mndRefreshUserIpWhiteList +(void)mndSendArbHeartBeatReq +(void)mndUpdClusterInfo +(void)mndUpdateArbHeartBeat +(void)qSetStreamOpOpen +(void)raftStoreWriteFile +(void)refreshMeta +(void)regcomp +(void)removeMeta +(void)restartStreamTasks +(void)rpcSendRequest +(void)rpcSendResponse +(void)rpcSetIpWhite +(void)rsmaSnapWriterPrepareClose +(void)s3DeleteObjects +(void)sdbSetRawStatus +(void)sdbWriteFile +(void)setenv +(void)snapshotReSend +(void)snapshotSenderCreate +(void)stmtAsyncOutput +(void)streamBuildAndSendDropTaskMsg +(void)streamDispatchStreamBlock +(void)streamLaunchFillHistoryTask +(void)streamMetaAddFailedTask +(void)streamMetaAddTaskLaunchResult +(void)streamMetaCommit +(void)streamMetaRemoveTask +(void)streamMetaSendHbHelper +(void)streamMetaStartAllTasks +(void)streamMetaStartOneTask +(void)streamMetaStopAllTasks +(void)streamMetaUnregisterTask +(void)streamMutexLock +(void)streamMutexUnlock +(void)streamProcessCheckpointTriggerBlock +(void)streamSendCheckMsg +(void)streamSetupScheduleTrigger +(void)streamStateSessionPut_rocksdb +(void)streamTaskBuildCheckpoint +(void)streamTaskBuildCheckpointSourceRsp +(void)streamTaskHandleEventAsync +(void)streamTaskOnHandleEventSuccess +(void)streamTaskReleaseState +(void)streamTaskReloadState +(void)streamTaskResetTimewindowFilter +(void)streamTaskSchedTask +(void)streamTaskSendCheckpointReq +(void)streamTaskSendCheckpointSourceRsp +(void)streamTaskSendCheckpointsourceRsp +(void)streamTaskSetActiveCheckpointInfo +(void)streamTaskSetSchedStatusInactive +(void)streamTaskSnapReaderClose +(void)streamTaskStop +(void)streamTaskUpdateEpsetInfo +(void)syncBeginSnapshot +(void)syncBuildHeartbeat +(void)syncBuildHeartbeatReply +(void)syncBuildLocalCmd +(void)syncCheckMember +(void)syncEndSnapshot +(void)syncEntryDestroy +(void)syncFsmExecute +(void)syncHbTimerInit +(void)syncInit +(void)syncLeaderTransfer +(void)syncLogBufferReset +(void)syncLogBufferRollback +(void)syncLogBufferValidate +(void)syncLogReplAttempt +(void)syncLogReplContinue +(void)syncLogReplGetPrevLogTerm +(void)syncLogReplProbe +(void)syncLogReplRecover +(void)syncLogReplReset +(void)syncLogReplRetryOnNeed +(void)syncLogReplStart +(void)syncNodeElect +(void)syncNodeHeartbeatPeers +(void)syncNodePeerStateInit +(void)syncNodeReplicate +(void)syncNodeReplicateReset +(void)syncNodeReplicateWithoutLock +(void)syncNodeRestartElectTimer +(void)syncNodeSendAppendEntries +(void)syncNodeSendHeartbeat +(void)syncNodeSendMsgById +(void)syncNodeStartHeartbeatTimer +(void)syncNodeStopElectTimer +(void)syncNodeStopHeartbeatTimer +(void)syncNodeStopPingTimer +(void)syncNodeTimerRoutine +(void)syncNodeUpdateAssignedCommitIndex +(void)syncNodeUpdateCommitIndex +(void)syncRespCleanByTTL +(void)syncRespMgrCreate +(void)syncRespMgrDel +(void)syncRespMgrGetAndDel +(void)syncSnapSendRsp +(void)syncUtilNodeInfo2RaftId +(void)syncWriteCfgFile +(void)tBrinBlockGet +(void)tBrinBlockInit +(void)tBufferGet +(void)tBufferGetI32 +(void)tColDataCopyRow +(void)tColDataCopyRowAppend +(void)tColDataSort +(void)tDecodeSBatchDeleteReq +(void)tDecodeSMAlterStbRsp +(void)tDecodeSMCreateStbRsp +(void)tEncodeDeleteRes +(void)tEncodeI32 +(void)tEncodeSBatchDeleteReq +(void)tEncodeSMAlterStbRsp +(void)tEncodeSMCreateStbRsp +(void)tEncodeSSchemaWrapper +(void)tEncodeSSubmitRsp2 +(void)tEncodeSVAlterTbRsp +(void)tEncodeSVCreateTbBatchRsp +(void)tEncodeSVDropStbReq +(void)tEncodeSVDropTbBatchReq +(void)tEncodeSVDropTbBatchRsp +(void)tEncodeStreamCheckpointSourceRsp +(void)tEncodeStreamRetrieveReq +(void)tEncodeStreamTaskCheckRsp +(void)tMultiWorkerInit +(void)tNameExtractFullName +(void)tNameFromString +(void)tRowGet +(void)tSerializeDropOrphanTaskMsg +(void)tSerializeSAlterVnodeConfigReq +(void)tSerializeSAlterVnodeHashRangeReq +(void)tSerializeSAlterVnodeReplicaReq +(void)tSerializeSCMCreateTopicReq +(void)tSerializeSCMSubscribeReq +(void)tSerializeSCompactVnodeReq +(void)tSerializeSCreateDbReq +(void)tSerializeSCreateDropMQSNodeReq +(void)tSerializeSCreateVnodeReq +(void)tSerializeSDCfgDnodeReq +(void)tSerializeSDCreateMnodeReq +(void)tSerializeSDbCfgRsp +(void)tSerializeSDbHbBatchRsp +(void)tSerializeSDisableVnodeWriteReq +(void)tSerializeSDnodeInfoReq +(void)tSerializeSDnodeListRsp +(void)tSerializeSDropDbRsp +(void)tSerializeSDropIdxReq +(void)tSerializeSDropVnodeReq +(void)tSerializeSEpSet +(void)tSerializeSForceBecomeFollowerReq +(void)tSerializeSMDropTopicReq +(void)tSerializeSMTimerMsg +(void)tSerializeSQnodeListRsp +(void)tSerializeSQueryCompactProgressReq +(void)tSerializeSRetrieveFuncRsp +(void)tSerializeSSTbHbRsp +(void)tSerializeSServerStatusRsp +(void)tSerializeSShowVariablesRsp +(void)tSerializeSStatusRsp +(void)tSerializeSTableCfgRsp +(void)tSerializeSTableMetaRsp +(void)tSerializeSUseDbRsp +(void)tSerializeSVDropTtlTableReq +(void)tSerializeSVKillCompactReq +(void)tSerializeSVS3MigrateDbReq +(void)tSerializeSVTrimDbReq +(void)tSimpleHashIterateRemove +(void)tSimpleHashPut +(void)tSimpleHashRemove +(void)tSkipListDestroyIter +(void)tSkipListPut +(void)tSkipListRLock +(void)tSkipListUnlock +(void)tSkipListWLock +(void)tStatisBlockDestroy +(void)tStatisBlockGet +(void)tStatisBlockInit +(void)tTombBlockDestroy +(void)tValueColumnClear +(void)taosAcquireRef +(void)taosAddIntoQset +(void)taosArrayAddAll +(void)taosArrayDestroy +(void)taosArrayInsert +(void)taosArrayPop +(void)taosArrayPush +(void)taosArraySortPWithExt +(void)taosBlockSIGPIPE +(void)taosCalcChecksumAppend +(void)taosCheckAndSetDebugFlag +(void)taosCntrGetCpuCores +(void)taosDecodeFixedI64 +(void)taosDecodeFixedU32 +(void)taosDecodeFixedU64 +(void)taosEncodeFixedU32 +(void)taosEncodeFixedU64 +(void)taosEnvToCfg +(void)taosFormatUtcTime +(void)taosFsyncFile +(void)taosFtruncateFile +(void)taosGetCpuCores +(void)taosGetCpuInstructions +(void)taosGetCpuUsage +(void)taosGetDiskSize +(void)taosGetFqdnPortFromEp +(void)taosGetProcMemory +(void)taosGetQitem +(void)taosGetSysMemory +(void)taosGetTimeOfDay +(void)taosGetTotalMemory +(void)taosHashGetDup +(void)taosHashGetImpl +(void)taosHashPut +(void)taosHashReleaseNode +(void)taosHashRemove +(void)taosIgnSIGPIPE +(void)taosIgnSignal +(void)taosInitTimer +(void)taosLRUCacheRelease +(void)taosLRUEntryTableRemove +(void)taosLSeekFile +(void)taosLocalTime +(void)taosLockFile +(void)taosLockLogFile +(void)taosMemoryFree +(void)taosMkDir +(void)taosOpenNewLogFile +(void)taosParseTime +(void)taosPushLogBuffer +(void)taosReadAllQitems +(void)taosRealPath +(void)taosReleaseRef +(void)taosRemoveDir +(void)taosRemoveRef +(void)taosRenameFile +(void)taosScheduleTask +(void)taosSeekCFile +(void)taosSetSignal +(void)taosStatFile +(void)taosStrpTime +(void)taosTmrReset +(void)taosTmrStop +(void)taosTmrStopA +(void)taosUcs4ToMbs +(void)taosUmaskFile +(void)taosVersionStrToInt +(void)taosWriteFile +(void)taosWriteQitem +(void)taos_collector_destroy +(void)taos_collector_registry_clear_batch +(void)taos_collector_registry_default_init +(void)taos_collector_registry_destroy +(void)taos_collector_registry_register_collector +(void)taos_counter_add +(void)taos_counter_destroy +(void)taos_map_destroy +(void)taos_map_node_destroy +(void)taos_map_set_free_value_fn +(void)taos_metric_destroy +(void)taos_metric_formatter_destroy +(void)taos_metric_sample_destroy +(void)taos_metric_sample_exchange +(void)taos_string_builder_destroy +(void)taskDbLoadChkpInfo +(void)tcompressDebug +(void)tdAppendColValToKvRow +(void)tdAppendColValToTpRow +(void)tdDestroySmaState +(void)tdFetchTbUidList +(void)tdFreeRSmaInfo +(void)tdFreeSmaEnv +(void)tdGetBitmapValTypeII +(void)tdGetKvRowValOfCol +(void)tdGetTpRowValOfCol +(void)tdListAppend +(void)tdListFree +(void)tdListPopNode +(void)tdLockSma +(void)tdProcessTSmaInsert +(void)tdRSmaProcessExecImpl +(void)tdReleaseSmaRef +(void)tdRsmaStopExecutor +(void)tdSKvRowGetVal +(void)tdSRowResetBuf +(void)tdSRowSetInfo +(void)tdSTSRowIterGetKvVal +(void)tdSTSRowIterGetTpVal +(void)tdSTpRowGetVal +(void)tdUidStoreFree +(void)tdUnLockSma +(void)tdUpdateTbUidList +(void)tdbBtcGet +(void)tdbBtcMoveDownward +(void)tdbBtcMoveUpward +(void)tdbBtreeClose +(void)tdbBtreeDecodeCell +(void)tdbBtreeEncodeCell +(void)tdbBtreeInitPage +(void)tdbClose +(void)tdbCloseDir +(void)tdbOsClose +(void)tdbPCacheClose +(void)tdbPCacheCloseImpl +(void)tdbPageAllocate +(void)tdbPageCopy +(void)tdbPageCreate +(void)tdbPageDestroy +(void)tdbPageDropCell +(void)tdbPageFree +(void)tdbPageInsertCell +(void)tdbPagerClose +(void)tdbPagerFlushPage +(void)tdbPagerInsertFreePage +(void)tdbPagerRollback +(void)tdbRefPage +(void)tdbTbClose +(void)tdbTbDelete +(void)tdbTbGet +(void)tdbTbUpsert +(void)tdbTbcClose +(void)tdbTbcGet +(void)tdbTbcMoveTo +(void)tdbTbcMoveToFirst +(void)tdbTbcMoveToNext +(void)tdbTbcMoveToPrev +(void)tdbTxnCloseImpl +(void)tdigestCompress +(void)tfileReaderLoadTableIds +(void)tfileRmExpireFile +(void)tfileWriteFooter +(void)tfileWriteFstOffset +(void)tfsGetMonitorInfo +(void)tfsLock +(void)tfsLockTier +(void)tfsMkdir +(void)tfsMkdirRecurAt +(void)tfsOpendir +(void)tfsRmdir +(void)tfsUnLock +(void)tfsUnLockTier +(void)timer_delete +(void)titoa +(void)tjsonGetStringValue +(void)tmqBuildBatchMetaRspFromWrapper +(void)tmqBuildMetaRspFromWrapper +(void)tmqBuildRspFromWrapper +(void)tmsgPutToQueue +(void)tmsgSendReq +(void)tmsgSendRsp +(void)tmsgUpdateDnodeInfo +(void)toName +(void)tqMetaGetHandle +(void)tqProcessTaskConsenChkptIdReq +(void)tqProcessTaskResetReq +(void)tqScanWalAsync +(void)tqStopStreamTasksAsync +(void)tqUpdateTbUidList +(void)transAcquireExHandle +(void)transAsyncSend +(void)transClearBuffer +(void)transDQSched +(void)transDestroyBuffer +(void)transQueuePop +(void)transQueuePush +(void)transReleaseExHandle +(void)transRemoveExHandle +(void)transSockInfo2Str +(void)tsCompressInit +(void)tsDecompressFloatImplAvx2 +(void)tsDecompressFloatImplAvx512 +(void)tsDecompressTimestampAvx2 +(void)tsDecompressTimestampAvx512 +(void)tsdbAcquireReader +(void)tsdbBeginTaskOnFileSet +(void)tsdbCacheCommit +(void)tsdbCacheCommitNoLock +(void)tsdbCacheDeserialize +(void)tsdbCacheDropNTableColumn +(void)tsdbCacheDropSTableColumn +(void)tsdbCacheDropSubTables +(void)tsdbCacheDropTable +(void)tsdbCacheDropTableColumn +(void)tsdbCacheNewNTableColumn +(void)tsdbCacheNewSTableColumn +(void)tsdbCacheNewTable +(void)tsdbCacheNewTableColumn +(void)tsdbCacheRelease +(void)tsdbCacheSetPageS3 +(void)tsdbCacheUpdate +(void)tsdbCacheUpdateValue +(void)tsdbDisableAndCancelAllBgTask +(void)tsdbEnableBgTask +(void)tsdbFSDestroyCopySnapshot +(void)tsdbFSDestroyRefSnapshot +(void)tsdbFSGetFSet +(void)tsdbFSSetBlockCommit +(void)tsdbFSToBinary +(void)tsdbFinishTaskOnFileSet +(void)tsdbIterMergerClose +(void)tsdbMergeFileSetEndCloseReader +(void)tsdbPreCommit +(void)tsdbRefMemTable +(void)tsdbRowIterOpen +(void)tsdbRowMergerInit +(void)tsdbSetKeepCfg +(void)tsdbSnapRAWReadFileSetCloseIter +(void)tsdbSnapRAWReadFileSetCloseReader +(void)tsdbSnapRAWReaderClose +(void)tsdbSnapRAWWriterPrepareClose +(void)tsdbSnapReaderClose +(void)tsdbSnapWriterPrepareClose +(void)tsdbStopAllCompTask +(void)tsdbSttFileReadStatisBlock +(void)tsdbSttFileReaderClose +(void)tsdbSttLvlClear +(void)tsdbTFileLastChunkName +(void)tsdbTFileName +(void)tsdbTFileObjRef +(void)tsdbTFileObjRemove +(void)tsdbTFileObjRemoveUpdateLC +(void)tsdbTFileObjUnref +(void)tsdbTFileSetRangeArrayDestroy +(void)tsdbTFileSetRangeClear +(void)tsdbTFileSetRemove +(void)tsdbTFileUpdVerRange +(void)tsdbTbDataIterNext +(void)tsdbWriterUpdVerRange +(void)tsem2_destroy +(void)tsem2_post +(void)tsem2_wait +(void)tsem_destroy +(void)tsem_init +(void)tsem_post +(void)tsem_timewait +(void)tsem_wait +(void)ttlMgrFlush +(void)udfcClose +(void)unlink +(void)updataTableColCmpr +(void)usleep +(void)vHashDrop +(void)vHashGet +(void)vHashRehash +(void)varDataCopy +(void)vmWriteVnodeListToFile +(void)vnodeAChannelInit +(void)vnodeAWait +(void)vnodeAsyncCancelAllTasks +(void)vnodeAsyncCommit +(void)vnodeAsyncLaunchWorker +(void)vnodeAsyncSetWorkers +(void)vnodeAsyncTaskDone +(void)vnodeBegin +(void)vnodeCancelAndDisableAllBgTask +(void)vnodeCheckAssignedLogSyncd +(void)vnodeCommitInfo +(void)vnodeEnableBgTask +(void)vnodeGetLoad +(void)vnodeGetPrimaryDir +(void)vnodeGetStreamProgress +(void)vnodeGetTableCfg +(void)vnodeGetTableMeta +(void)vnodePreCheckAssignedLogSyncd +(void)vnodeSaveInfo +(void)vnodeSyncCommit +(void)walInit +(void)walLoadMeta +(void)walSaveMeta +(void)walkExpr +(void)walkExprs +TAOS_UNUSED(metaRehashCache +TAOS_UNUSED(ttlMgrFlush +TAOS_UNUSED(taosMkDir +TAOS_UNUSED(tsdbCommitCloseReader +TAOS_UNUSED(tsdbCommitCloseIter +TAOS_UNUSED(tsdbFSCheckCommit +TAOS_UNUSED(tfsMkdirRecurAt +TAOS_UNUSED(vHashDrop +TAOS_UNUSED(tsdbCommitInfoDestroy +TAOS_UNUSED(vHashGet +TAOS_UNUSED(tTombBlockGet +TAOS_UNUSED(tsdbDisableAndCancelAllBgTask +TAOS_UNUSED(tsdbTbDataIterNext +TAOS_UNUSED(tsdbCacheDel +TAOS_UNUSED(tsdbCacheColFormatUpdate +TAOS_UNUSED(tStatisBlockInit +TAOS_UNUSED(tStatisBlockClear +TAOS_UNUSED(tsdbSnapReadFileSetCloseReader +TAOS_UNUSED(tsdbSnapReadFileSetCloseIter +TAOS_UNUSED(tsdbSnapReadFileSetCloseReader +TAOS_UNUSED(tTombBlockClear +TAOS_UNUSED(tBufferDestroy +TAOS_UNUSED(tfsMkdirRecurAt +TAOS_UNUSED(tGetItemFn +TAOS_UNUSED(tValueColumnDestroy +TAOS_UNUSED(tBufferClear +TAOS_UNUSED(tValueColumnClear +TAOS_UNUSED(taosArrayPush +TAOS_UNUSED(taosThreadJoin +TAOS_UNUSED(tdbOsClose +TAOS_UNUSED(taosThreadMutexInit +TAOS_UNUSED(taosThreadMutexLock diff --git a/tests/docs-examples-test/c.sh b/tests/docs-examples-test/c.sh new file mode 100644 index 0000000000..54e334b22e --- /dev/null +++ b/tests/docs-examples-test/c.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +pgrep taosd || taosd >> /dev/null 2>&1 & +pgrep taosadapter || taosadapter >> /dev/null 2>&1 & + +GREEN='\033[0;32m' +RED='\033[0;31m' +NC='\033[0m' + +TEST_PATH="../../docs/examples/c" +echo "setting TEST_PATH: $TEST_PATH" + +cd "${TEST_PATH}" || { echo -e "${RED}Failed to change directory to ${TEST_PATH}${NC}"; exit 1; } + +LOG_FILE="docs-c-test-out.log" + +> $LOG_FILE + +make > "$LOG_FILE" 2>&1 + +if [ $? -eq 0 ]; then + echo -e "${GREEN}Make completed successfully.${NC}" +else + echo -e "${RED}Make failed. Check log file: $LOG_FILE${NC}" + cat "$LOG_FILE" + exit 1 +fi + + +declare -a TEST_EXES=( + "connect_example" + "create_db_demo" + "insert_data_demo" + "query_data_demo" + "with_reqid_demo" + "stmt_insert_demo" + "tmq_demo" + "sml_insert_demo" +) + +declare -a NEED_CLEAN=( + "true" + "false" + "false" + "false" + "false" + "false" + "false" + "true" +) + +totalCases=0 +totalFailed=0 +totalSuccess=0 + +for i in "${!TEST_EXES[@]}"; do + TEST_EXE="${TEST_EXES[$i]}" + NEED_CLEAN_FLAG="${NEED_CLEAN[$i]}" + + if [ "$NEED_CLEAN_FLAG" = "true" ]; then + echo "Cleaning database before executing $TEST_EXE..." + taos -s "drop database if exists power" >> $LOG_FILE 2>&1 + fi + + echo "Executing $TEST_EXE..." + ./$TEST_EXE >> $LOG_FILE 2>&1 + RESULT=$? + + if [ "$RESULT" -eq 0 ]; then + totalSuccess=$((totalSuccess + 1)) + echo "[$GREEN OK $NC] $TEST_EXE executed successfully." + else + totalFailed=$((totalFailed + 1)) + echo "[$RED FAILED $NC] $TEST_EXE exited with code $RESULT." + fi + + totalCases=$((totalCases + 1)) +done + +tail -n 40 $LOG_FILE + +echo -e "\nTotal number of cases executed: $totalCases" +if [ "$totalSuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalSuccess C case(s) succeed! ### ${NC}" +fi + +if [ "$totalFailed" -ne "0" ]; then + echo -e "\n${RED} ### Total $totalFailed C case(s) failed! ### ${NC}" + exit 1 +fi + +echo "All tests completed." \ No newline at end of file diff --git a/tests/docs-examples-test/node.sh b/tests/docs-examples-test/node.sh index 41acf7c7b4..cbbbd3820e 100644 --- a/tests/docs-examples-test/node.sh +++ b/tests/docs-examples-test/node.sh @@ -2,41 +2,63 @@ set -e +check_transactions() { + for i in {1..30} + do + output=$(taos -s "show transactions;") + if [[ $output == *"Query OK, 0 row(s)"* ]]; then + echo "Success: No transactions are in progress." + return 0 + fi + sleep 1 + done + + echo "Error: Transactions are still in progress after 30 attempts." + return 1 +} + +reset_cache() { + response=$(curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql' --data 'reset query cache') + + if [[ $response == \{\"code\":0* ]]; then + echo "Success: Query cache reset successfully." + else + echo "Error: Failed to reset query cache. Response: $response" + return 1 + fi +} + + pgrep taosd || taosd >> /dev/null 2>&1 & pgrep taosadapter || taosadapter >> /dev/null 2>&1 & +sleep 10 + cd ../../docs/examples/node npm install -cd restexample; -node connect.js +cd websocketexample -cd ../nativeexample +node all_type_query.js -node connect.js +node all_type_stmt.js taos -s "drop database if exists power" -node insert_example.js - -node query_example.js - -node async_query_example.js - -# node subscribe_demo.js - -taos -s "drop topic if exists topic_name_example" -taos -s "drop database if exists power" -node param_bind_example.js +check_transactions || exit 1 +reset_cache || exit 1 +node line_example.js taos -s "drop database if exists power" -node multi_bind_example.js +check_transactions || exit 1 +reset_cache || exit 1 +node nodejsChecker.js -taos -s "drop database if exists test" -node influxdb_line_example.js +node sql_example.js -taos -s "drop database if exists test" -node opentsdb_telnet_example.js +node stmt_example.js -taos -s "drop database if exists test" -node opentsdb_json_example.js + +node tmq_example.js + +node tmq_seek_example.js \ No newline at end of file diff --git a/tests/docs-examples-test/python.sh b/tests/docs-examples-test/python.sh index 84f0771ec5..6a25683b58 100644 --- a/tests/docs-examples-test/python.sh +++ b/tests/docs-examples-test/python.sh @@ -2,6 +2,34 @@ set -e +check_transactions() { + for i in {1..30} + do + output=$(taos -s "show transactions;") + if [[ $output == *"Query OK, 0 row(s)"* ]]; then + echo "Success: No transactions are in progress." + return 0 + fi + sleep 1 + done + + echo "Error: Transactions are still in progress after 30 attempts." + return 1 +} + +reset_cache() { + response=$(curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql' --data 'reset query cache') + + if [[ $response == \{\"code\":0* ]]; then + echo "Success: Query cache reset successfully." + else + echo "Error: Failed to reset query cache. Response: $response" + return 1 + fi +} + + + taosd >>/dev/null 2>&1 & taosadapter >>/dev/null 2>&1 & @@ -11,18 +39,26 @@ cd ../../docs/examples/python # 1 taos -s "create database if not exists log" +check_transactions || exit 1 +reset_cache || exit 1 python3 connect_example.py # 2 taos -s "drop database if exists power" +check_transactions || exit 1 +reset_cache || exit 1 python3 native_insert_example.py # 3 taos -s "drop database power" +check_transactions || exit 1 +reset_cache || exit 1 python3 bind_param_example.py # 4 taos -s "drop database power" +check_transactions || exit 1 +reset_cache || exit 1 python3 multi_bind_example.py # 5 @@ -33,20 +69,28 @@ python3 async_query_example.py # 7 taos -s "drop database if exists test" +check_transactions || exit 1 +reset_cache || exit 1 python3 line_protocol_example.py # 8 taos -s "drop database test" +check_transactions || exit 1 +reset_cache || exit 1 python3 telnet_line_protocol_example.py # 9 taos -s "drop database test" +check_transactions || exit 1 +reset_cache || exit 1 python3 json_protocol_example.py # 10 pip install SQLAlchemy pip install pandas taosBenchmark -y -d power -t 10 -n 10 +check_transactions || exit 1 +reset_cache || exit 1 python3 conn_native_pandas.py python3 conn_rest_pandas.py taos -s "drop database if exists power" @@ -86,8 +130,69 @@ pip3 install kafka-python python3 kafka_example_consumer.py # 21 -pip3 install taos-ws-py==0.3.1 +pip3 install taos-ws-py==0.3.3 python3 conn_websocket_pandas.py # 22 python3 connect_websocket_examples.py + +# 23 +python3 create_db_ws.py + +# 24 +python3 create_db_native.py + +# 25 +python3 create_db_rest.py + +python3 insert_native.py + +python3 insert_rest.py + +python3 insert_ws.py + +python3 query_native.py + +python3 query_rest.py + +python3 query_ws.py + +python3 reqid_native.py + +python3 reqid_rest.py + +python3 reqid_ws.py + +taos -s "drop database power" +check_transactions || exit 1 +reset_cache || exit 1 +python3 schemaless_native.py + +taos -s "drop database power" +check_transactions || exit 1 +reset_cache || exit 1 +python3 schemaless_ws.py + +taos -s "drop database power" +check_transactions || exit 1 +reset_cache || exit 1 +python3 stmt_native.py + +python3 stmt_ws.py + +taos -s "drop topic if exists topic_meters" +check_transactions || exit 1 +reset_cache || exit 1 +taos -s "drop database if exists power" +check_transactions || exit 1 +reset_cache || exit 1 +python3 tmq_native.py + +taos -s "drop topic if exists topic_meters" +check_transactions || exit 1 +reset_cache || exit 1 +taos -s "drop database if exists power" +check_transactions || exit 1 +reset_cache || exit 1 +python3 tmq_websocket_example.py + diff --git a/tests/docs-examples-test/rust.sh b/tests/docs-examples-test/rust.sh new file mode 100644 index 0000000000..91489d8bcd --- /dev/null +++ b/tests/docs-examples-test/rust.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +set -e + + +check_transactions() { + for i in {1..30} + do + output=$(taos -s "show transactions;") + if [[ $output == *"Query OK, 0 row(s)"* ]]; then + echo "Success: No transactions are in progress." + return 0 + fi + sleep 1 + done + + echo "Error: Transactions are still in progress after 30 attempts." + return 1 +} + +reset_cache() { + response=$(curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql' --data 'reset query cache') + + if [[ $response == \{\"code\":0* ]]; then + echo "Success: Query cache reset successfully." + else + echo "Error: Failed to reset query cache. Response: $response" + return 1 + fi +} + +taosd >>/dev/null 2>&1 & +taosadapter >>/dev/null 2>&1 & + +sleep 5 + +cd ../../docs/examples/rust/nativeexample + +cargo run --example connect + +cargo run --example createdb + +cargo run --example insert + +cargo run --example query + +taos -s "drop database if exists power" +check_transactions || exit 1 +reset_cache || exit 1 + +cargo run --example schemaless + +taos -s "drop database if exists power" +check_transactions || exit 1 +reset_cache || exit 1 + + +cargo run --example stmt + +cargo run --example tmq + + + +cd ../restexample + +cargo run --example connect + +cargo run --example createdb + +cargo run --example insert + +cargo run --example query + +taos -s "drop database if exists power" +check_transactions || exit 1 +reset_cache || exit 1 +taos -s "create database if not exists power" +cargo run --example schemaless + +taos -s "drop database if exists power" +check_transactions || exit 1 +reset_cache || exit 1 + + +cargo run --example stmt + +cargo run --example tmq diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 1a00787a6b..6de87efd42 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -23,6 +23,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f insert/test_column_tag_boundary.py ,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_desc.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_null.py +,,y,army,./pytest.sh python3 ./test.py -f cluster/test_drop_table_by_uid.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f cluster/incSnapshot.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f cluster/clusterBasic.py -N 5 ,,y,army,./pytest.sh python3 ./test.py -f query/query_basic.py -N 3 @@ -160,7 +161,6 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxTopic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -R -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqClientConsLog.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqOffset.py @@ -233,6 +233,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/basic5.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/ts-4674.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-30270.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb3.py @@ -280,6 +281,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5466.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5473.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts4563.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_replay.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSeekAndCommit.py @@ -1552,9 +1555,12 @@ ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R #docs-examples test +,,n,docs-examples-test,bash c.sh ,,n,docs-examples-test,bash python.sh -#,,n,docs-examples-test,bash node.sh +,,n,docs-examples-test,bash node.sh ,,n,docs-examples-test,bash csharp.sh ,,n,docs-examples-test,bash jdbc.sh +,,n,docs-examples-test,bash rust.sh ,,n,docs-examples-test,bash go.sh ,,n,docs-examples-test,bash test_R.sh + diff --git a/tests/parallel_test/run_check_assert_container.sh b/tests/parallel_test/run_check_assert_container.sh index e8d78d62ae..5b0f26ffc3 100755 --- a/tests/parallel_test/run_check_assert_container.sh +++ b/tests/parallel_test/run_check_assert_container.sh @@ -31,7 +31,6 @@ fi # enterprise edition INTERNAL_REPDIR=$WORKDIR/TDinternal -REPDIR_DEBUG=$WORKDIR/debugNoSan/ REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal" @@ -46,7 +45,7 @@ docker run \ --rm --ulimit core=-1 taos_test:v1.0 python3 $check_assert_scripts EOF docker run \ - -v $REP_MOUNT_PARAM \ + -v "$REP_MOUNT_PARAM" \ --rm --ulimit core=-1 taos_test:v1.0 python3 $check_assert_scripts ret=$? diff --git a/tests/parallel_test/run_check_void_container.sh b/tests/parallel_test/run_check_void_container.sh new file mode 100755 index 0000000000..3b0b093153 --- /dev/null +++ b/tests/parallel_test/run_check_void_container.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +function usage() { + echo "$0" + echo -e "\t -d work dir" + echo -e "\t -h help" +} + +while getopts "d:h" opt; do + case $opt in + d) + WORKDIR=$OPTARG + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; + esac +done + +if [ -z "$WORKDIR" ]; then + usage + exit 1 +fi + + + # enterprise edition +INTERNAL_REPDIR=$WORKDIR/TDinternal + +REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal" + +CONTAINER_TESTDIR=/home/TDinternal/community + +check_assert_scripts="$CONTAINER_TESTDIR/tests/ci/check_void.sh" +func_list_file="$CONTAINER_TESTDIR/tests/ci/func.txt" +source_file_path="$CONTAINER_TESTDIR/source/" + +ulimit -c unlimited +cat << EOF +docker run \ + -v $REP_MOUNT_PARAM \ + --rm --ulimit core=-1 taos_test:v1.0 "$check_assert_scripts" -c $func_list_file -f $source_file_path +EOF + +docker run \ + -v "$REP_MOUNT_PARAM" \ + --rm --ulimit core=-1 taos_test:v1.0 $check_assert_scripts -c $func_list_file -f $source_file_path + +ret=$? +exit $ret + diff --git a/tests/pytest/util/csv.py b/tests/pytest/util/csv.py index 6e1ee1732c..2d2e929c86 100644 --- a/tests/pytest/util/csv.py +++ b/tests/pytest/util/csv.py @@ -1,5 +1,6 @@ import csv import os +import platform class TDCsv: def __init__(self): @@ -25,7 +26,11 @@ class TDCsv: @property def file(self): if self.file_name and self.file_path: - return os.path.join(self.file_path, self.file_name) + print(f"self.file_path {self.file_path}, self.file_name {self.file_name}") + csv_file = os.path.join(self.file_path, self.file_name) + if platform.system().lower() == 'windows': + csv_file = csv_file.replace("\\", "/") + return csv_file return None diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 90d3f2fe6c..3bc784063e 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -24,26 +24,48 @@ from util.log import * from util.constant import * import ctypes import random -# from datetime import timezone +import datetime import time +from tzlocal import get_localzone def _parse_ns_timestamp(timestr): dt_obj = datetime.datetime.strptime(timestr[:len(timestr)-3], "%Y-%m-%d %H:%M:%S.%f") tz = int(int((dt_obj-datetime.datetime.fromtimestamp(0,dt_obj.tzinfo)).total_seconds())*1e9) + int(dt_obj.microsecond * 1000) + int(timestr[-3:]) return tz - def _parse_datetime(timestr): - try: - return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') - except ValueError: - pass - try: - return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') - except ValueError: - pass + # defined timestr formats + formats = [ + '%Y-%m-%d %H:%M:%S.%f%z', # 包含微秒和时区偏移 + '%Y-%m-%d %H:%M:%S%z', # 不包含微秒但包含时区偏移 + '%Y-%m-%d %H:%M:%S.%f', # 包含微秒 + '%Y-%m-%d %H:%M:%S' # 不包含微秒 + ] + + for fmt in formats: + try: + # try to parse the string with the current format + dt = datetime.datetime.strptime(timestr, fmt) + # 如果字符串包含时区信息，则返回 aware 对象 + # if sting contains timezone info, return aware object + if dt.tzinfo is not None: + return dt + + else: + # if sting does not contain timezone info, assume it is in local timezone + # get local timezone + local_timezone = get_localzone() + # print("Timezone:", local_timezone) + return dt.replace(tzinfo=local_timezone) + except ValueError: + continue # if the current format does not match, try the next format + + # 如果所有格式都不匹配，返回 None + # if none of the formats match, return + raise ValueError(f"input format does not match. correct formats include: '{', '.join(formats)}'") class TDSql: + def __init__(self): self.queryRows = 0 self.queryCols = 0 @@ -408,6 +430,7 @@ class TDSql: if self.queryResult[row][col] != data: if self.cursor.istype(col, "TIMESTAMP"): + # tdLog.debug(f"self.queryResult[row][col]:{self.queryResult[row][col]}, data:{data},len(data):{len(data)}, isinstance(data,str) :{isinstance(data,str)}") # suppose user want to check nanosecond timestamp if a longer data passed`` if isinstance(data,str) : if (len(data) >= 28): @@ -419,8 +442,9 @@ class TDSql: args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) else: + # tdLog.info(f"datetime.timezone.utc:{datetime.timezone.utc},data:{data},_parse_datetime(data).astimezone(datetime.timezone.utc):{_parse_datetime(data).astimezone(datetime.timezone.utc)}") if self.queryResult[row][col].astimezone(datetime.timezone.utc) == _parse_datetime(data).astimezone(datetime.timezone.utc): - # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col].astimezone(datetime.timezone.utc)} == expect:{_parse_datetime(data).astimezone(datetime.timezone.utc)}") if(show): tdLog.info("check successfully") else: diff --git a/tests/system-test/0-others/backquote_check.py b/tests/system-test/0-others/backquote_check.py index 8cb268fb3d..aad2e21e6e 100644 --- a/tests/system-test/0-others/backquote_check.py +++ b/tests/system-test/0-others/backquote_check.py @@ -133,7 +133,7 @@ class TDTestCase: def run(self): self.topic_name_check() self.db_name_check() - if platform.system().lower() == 'windows': + if platform.system().lower() != 'windows': self.stream_name_check() self.table_name_check() self.view_name_check() diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index 35137dfeb1..da218162d4 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -203,7 +203,7 @@ class TDTestCase: assert str(v) == str(value) else: for v in values: - tdLog.debug("Set {} to {}".format(name, v)) + tdLog.debug("Set client {} to {}".format(name, v)) tdSql.error(f'alter local "{name} {v}";') def svr_check(self, item, except_values=False): diff --git a/tests/system-test/1-insert/composite_primary_key_insert.py b/tests/system-test/1-insert/composite_primary_key_insert.py index c888bacadc..311b9cf5b6 100644 --- a/tests/system-test/1-insert/composite_primary_key_insert.py +++ b/tests/system-test/1-insert/composite_primary_key_insert.py @@ -4,6 +4,7 @@ from util.log import * from util.sql import * from util.cases import * from util.csv import * +import platform import os import taos import json @@ -56,7 +57,6 @@ class TDTestCase: tdSql.init(conn.cursor(), True) self.testcasePath = os.path.split(__file__)[0] - self.testcasePath = self.testcasePath.replace('\\', '//') self.testcaseFilename = os.path.split(__file__)[-1] os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) # tdSql.execute(f"insert into db4096.ctb00 file '{self.testcasePath}//tableColumn4096csvLength64k.csv'") diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py index 7e41aad4a8..ad9f6c9be6 100644 --- a/tests/system-test/1-insert/drop.py +++ b/tests/system-test/1-insert/drop.py @@ -54,6 +54,7 @@ class TDTestCase: self.ctb_names = [ f'ctb0', 'ctb1', f'aa\u00bf\u200bctb0', f'aa\u00bf\u200bctb1'] self.ntb_names = [ f'ntb0', f'aa\u00bf\u200bntb0', f'ntb1', f'aa\u00bf\u200bntb1'] self.vgroups_opt = f'vgroups 4' + self.err_dup_cnt = 5 def insert_data(self,column_dict,tbname,row_num): insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) for i in range(row_num): @@ -147,13 +148,31 @@ class TDTestCase: if i == 0: dropTable = f'drop table with `{stb_result[1]}`.`{stb_result[10]}`,' dropStable = f'drop stable with `{stb_result[1]}`.`{stb_result[10]}`,' + dropTableWithSpace = f'drop table with `{stb_result[1]}`.`{stb_result[10]} `,' + dropStableWithSpace = f'drop stable with `{stb_result[1]}`.` {stb_result[10]}`,' + dropStableNotExist = f'drop stable with `{stb_result[1]}`.`{stb_result[10]}_notexist`,' + for _ in range(self.err_dup_cnt): + tdLog.info(dropTableWithSpace[:-1]) + tdSql.error(dropTableWithSpace[:-1], expectErrInfo="Table does not exist", fullMatched=False) + tdLog.info(dropStableWithSpace[:-1]) + tdSql.error(dropStableWithSpace[:-1], expectErrInfo="STable not exist", fullMatched=False) + tdLog.info(dropStableNotExist[:-1]) + tdSql.error(dropStableWithSpace[:-1], expectErrInfo="STable not exist", fullMatched=False) else: dropTable += f'`{stb_result[1]}`.`{stb_result[10]}`,' dropStable += f'`{stb_result[1]}`.`{stb_result[10]}`,' - tdLog.info(dropTable[:-1]) - tdLog.info(dropStable[:-1]) - tdSql.error(dropTable[:-1]) - tdSql.error(dropStable[:-1]) + for _ in range(self.err_dup_cnt): + tdLog.info(dropTable[:-1]) + tdLog.info(dropStable[:-1]) + tdSql.error(dropTable[:-1], expectErrInfo="Cannot drop super table in batch") + tdSql.error(dropStable[:-1], expectErrInfo="syntax error", fullMatched=False) + dropTableWithSpace += f'`{stb_result[1]}`.` {stb_result[10]}`,' + dropStableWithSpace += f'`{stb_result[1]}`.`{stb_result[10]} `,' + for _ in range(self.err_dup_cnt): + tdLog.info(dropTableWithSpace[:-1]) + tdLog.info(dropStableWithSpace[:-1]) + tdSql.error(dropTableWithSpace[:-1], expectErrInfo="Table does not exist", fullMatched=False) + tdSql.error(dropStableWithSpace[:-1], expectErrInfo="syntax error", fullMatched=False) i += 1 i = 0 for stb_result in result: @@ -172,9 +191,10 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select * from information_schema.ins_tables where db_name like "dbtest_%"') tdSql.checkRows(8) - tdSql.error(f'drop stable with information_schema.`ins_tables`;') - tdSql.error(f'drop stable with performance_schema.`perf_connections`;') - self.drop_table_check_end() + for _ in range(self.err_dup_cnt): + tdSql.error(f'drop stable with information_schema.`ins_tables`;', expectErrInfo="Cannot drop table of system database", fullMatched=False) + tdSql.error(f'drop stable with performance_schema.`perf_connections`;', expectErrInfo="Cannot drop table of system database", fullMatched=False) + self.drop_table_check_end() def drop_table_with_check(self): self.drop_table_check_init() tdSql.query(f'select * from information_schema.ins_tables where db_name like "dbtest_%"') @@ -196,8 +216,9 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select * from information_schema.ins_stables where db_name like "dbtest_%"') tdSql.checkRows(2) - tdSql.error(f'drop table with information_schema.`ins_tables`;') - tdSql.error(f'drop table with performance_schema.`perf_connections`;') + for _ in range(self.err_dup_cnt): + tdSql.error(f'drop table with information_schema.`ins_tables`;', expectErrInfo="Cannot drop table of system database", fullMatched=False) + tdSql.error(f'drop table with performance_schema.`perf_connections`;', expectErrInfo="Cannot drop table of system database", fullMatched=False) self.drop_table_check_end() def drop_table_with_check_tsma(self): tdSql.execute(f'create database if not exists {self.dbname} {self.vgroups_opt}') @@ -285,7 +306,7 @@ class TDTestCase: self.drop_table_with_check() self.drop_table_with_check_tsma() self.drop_topic_check() - if platform.system().lower() == 'windows': + if platform.system().lower() != 'windows': self.drop_stream_check() pass def stop(self): diff --git a/tests/system-test/2-query/fill.py b/tests/system-test/2-query/fill.py index 31d2d92563..3a868b7877 100644 --- a/tests/system-test/2-query/fill.py +++ b/tests/system-test/2-query/fill.py @@ -1,6 +1,5 @@ import queue import random -from fabric2.runners import threading from pandas._libs import interval import taos import sys @@ -9,6 +8,7 @@ from util.common import TDCom from util.log import * from util.sql import * from util.cases import * +import threading diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index bcfc389d7b..3cdf52725a 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -907,7 +907,7 @@ class TDTestCase: ## {. . .} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") - tdSql.checkRows(12) + tdSql.checkRows(13) tdSql.checkData(0, 0, 5) tdSql.checkData(1, 0, 5) tdSql.checkData(2, 0, 10) @@ -920,6 +920,7 @@ class TDTestCase: tdSql.checkData(9, 0, 15) tdSql.checkData(10, 0, 15) tdSql.checkData(11, 0, 15) + tdSql.checkData(12, 0, None) ## {} ... tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(next)") @@ -957,10 +958,12 @@ class TDTestCase: ## ..{.} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(next)") - tdSql.checkRows(3) + tdSql.checkRows(5) tdSql.checkData(0, 0, 15) tdSql.checkData(1, 0, 15) tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) ## ... {} tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(next)") @@ -1272,7 +1275,7 @@ class TDTestCase: tdSql.checkData(8, 1, True) tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") - tdSql.checkRows(12) + tdSql.checkRows(13) tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') @@ -1287,6 +1290,7 @@ class TDTestCase: tdSql.checkData(9, 0, '2020-02-01 00:00:13.000') tdSql.checkData(10, 0, '2020-02-01 00:00:14.000') tdSql.checkData(11, 0, '2020-02-01 00:00:15.000') + tdSql.checkData(12, 0, '2020-02-01 00:00:16.000') tdSql.checkData(0, 1, True) tdSql.checkData(1, 1, False) @@ -1300,6 +1304,7 @@ class TDTestCase: tdSql.checkData(9, 1, True) tdSql.checkData(10, 1, True) tdSql.checkData(11, 1, False) + tdSql.checkData(12, 1, True) tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(next)") tdSql.checkRows(6) @@ -1677,9 +1682,13 @@ class TDTestCase: ## | . | { | .} | tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(next)") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 0, 15) tdSql.checkData(1, 0, 15) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) # test fill linear @@ -2732,7 +2741,7 @@ class TDTestCase: tdSql.checkData(4, i, 15) tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(next)") - tdSql.checkRows(3) + tdSql.checkRows(5) tdSql.checkCols(4) for i in range (tdSql.queryCols): @@ -2828,7 +2837,7 @@ class TDTestCase: # test fill next tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(next)") - tdSql.checkRows(18) + tdSql.checkRows(19) tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') @@ -2851,6 +2860,7 @@ class TDTestCase: tdSql.checkData(15, 2, None) tdSql.checkData(16, 2, None) tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, None) tdSql.checkData(17, 0, '2020-02-02 00:00:17.000') @@ -3081,7 +3091,7 @@ class TDTestCase: # test fill linear tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(linear)") - tdSql.checkRows(17) + tdSql.checkRows(18) tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-02 00:00:01.000') @@ -3103,8 +3113,9 @@ class TDTestCase: tdSql.checkData(14, 2, None) tdSql.checkData(15, 2, None) tdSql.checkData(16, 2, None) + tdSql.checkData(17, 2, None) - tdSql.checkData(16, 0, '2020-02-02 00:00:17.000') + tdSql.checkData(17, 0, '2020-02-02 00:00:18.000') tdLog.printNoPrefix("==========step13:test error cases") @@ -3220,7 +3231,7 @@ class TDTestCase: tdSql.checkData(17, 1, True) tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname} range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(next)") - tdSql.checkRows(18) + tdSql.checkRows(19) tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') tdSql.checkData(0, 1, True) @@ -3243,9 +3254,12 @@ class TDTestCase: tdSql.checkData(15, 2, 15) tdSql.checkData(16, 2, 17) tdSql.checkData(17, 2, 17) + tdSql.checkData(18, 2, None) tdSql.checkData(17, 0, '2020-02-01 00:00:17.000') tdSql.checkData(17, 1, False) + tdSql.checkData(18, 0, '2020-02-01 00:00:18.000') + tdSql.checkData(18, 1, True) tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname} range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(linear)") tdSql.checkRows(17) @@ -3362,24 +3376,24 @@ class TDTestCase: tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0) from {dbname}.{stbname} partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(next)") - tdSql.checkRows(48) - for i in range(0, 14): + tdSql.checkRows(57) + for i in range(0, 19): tdSql.checkData(i, 0, 'ctb1') - for i in range(14, 30): + for i in range(19, 38): tdSql.checkData(i, 0, 'ctb2') - for i in range(30, 48): + for i in range(38, 57): tdSql.checkData(i, 0, 'ctb3') tdSql.checkData(0, 1, '2020-02-01 00:00:00.000') - tdSql.checkData(13, 1, '2020-02-01 00:00:13.000') + tdSql.checkData(18, 1, '2020-02-01 00:00:18.000') - tdSql.checkData(14, 1, '2020-02-01 00:00:00.000') - tdSql.checkData(29, 1, '2020-02-01 00:00:15.000') + tdSql.checkData(19, 1, '2020-02-01 00:00:00.000') + tdSql.checkData(37, 1, '2020-02-01 00:00:18.000') - tdSql.checkData(30, 1, '2020-02-01 00:00:00.000') - tdSql.checkData(47, 1, '2020-02-01 00:00:17.000') + tdSql.checkData(38, 1, '2020-02-01 00:00:00.000') + tdSql.checkData(56, 1, '2020-02-01 00:00:18.000') for i in range(0, 2): tdSql.checkData(i, 3, 1) @@ -3390,24 +3404,33 @@ class TDTestCase: for i in range(8, 14): tdSql.checkData(i, 3, 13) - for i in range(14, 18): + for i in range(14, 19): + tdSql.checkData(i, 3, None) + + for i in range(19, 23): tdSql.checkData(i, 3, 3) - for i in range(18, 24): + for i in range(23, 29): tdSql.checkData(i, 3, 9) - for i in range(24, 30): + for i in range(29, 35): tdSql.checkData(i, 3, 15) - for i in range(30, 36): + for i in range(35, 38): + tdSql.checkData(i, 3, None) + + for i in range(38, 44): tdSql.checkData(i, 3, 5) - for i in range(36, 42): + for i in range(44, 50): tdSql.checkData(i, 3, 11) - for i in range(42, 48): + for i in range(50, 56): tdSql.checkData(i, 3, 17) + for i in range(56, 57): + tdSql.checkData(i, 3, None) + tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0) from {dbname}.{stbname} partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(linear)") tdSql.checkRows(39) @@ -3450,7 +3473,7 @@ class TDTestCase: tdSql.checkRows(90) tdSql.query(f"select c0, _irowts, _isfilled, interp(c0) from {dbname}.{stbname} partition by c0 range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(next)") - tdSql.checkRows(90) + tdSql.checkRows(171) tdSql.query(f"select c0, _irowts, _isfilled, interp(c0) from {dbname}.{stbname} partition by c0 range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(linear)") tdSql.checkRows(9) @@ -3467,7 +3490,7 @@ class TDTestCase: tdSql.checkRows(48) tdSql.query(f"select t1, _irowts, _isfilled, interp(c0) from {dbname}.{stbname} partition by t1 range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(next)") - tdSql.checkRows(48) + tdSql.checkRows(57) tdSql.query(f"select t1, _irowts, _isfilled, interp(c0) from {dbname}.{stbname} partition by t1 range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(linear)") tdSql.checkRows(39) @@ -4363,7 +4386,7 @@ class TDTestCase: tdSql.query(f"select _irowts, _isfilled, interp(c0, 1) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(next)") - tdSql.checkRows(9) + tdSql.checkRows(11) tdSql.checkData(0, 1, False) tdSql.checkData(1, 1, True) tdSql.checkData(2, 1, False) @@ -4373,6 +4396,8 @@ class TDTestCase: tdSql.checkData(6, 1, True) tdSql.checkData(7, 1, False) tdSql.checkData(8, 1, False) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, True) tdSql.checkData(0, 2, 1) tdSql.checkData(1, 2, 3) @@ -4383,11 +4408,13 @@ class TDTestCase: tdSql.checkData(6, 2, 8) tdSql.checkData(7, 2, 8) tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 2, None) tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_null} where c0 is not null range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(next)") - tdSql.checkRows(9) + tdSql.checkRows(11) tdSql.checkData(0, 1, False) tdSql.checkData(1, 1, True) tdSql.checkData(2, 1, False) @@ -4397,6 +4424,9 @@ class TDTestCase: tdSql.checkData(6, 1, True) tdSql.checkData(7, 1, False) tdSql.checkData(8, 1, False) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, True) + tdSql.checkData(0, 2, 1) tdSql.checkData(1, 2, 3) @@ -4407,6 +4437,8 @@ class TDTestCase: tdSql.checkData(6, 2, 8) tdSql.checkData(7, 2, 8) tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 2, None) # super table tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") @@ -4443,7 +4475,7 @@ class TDTestCase: tdSql.query(f"select _irowts, _isfilled, interp(c0, 1) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") - tdSql.checkRows(8) + tdSql.checkRows(9) tdSql.checkData(0, 1, False) tdSql.checkData(1, 1, True) tdSql.checkData(2, 1, True) @@ -4452,6 +4484,7 @@ class TDTestCase: tdSql.checkData(5, 1, True) tdSql.checkData(6, 1, False) tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, True) tdSql.checkData(0, 2, 1) tdSql.checkData(1, 2, 9) @@ -4461,11 +4494,12 @@ class TDTestCase: tdSql.checkData(5, 2, 13) tdSql.checkData(6, 2, 13) tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, None) tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} where c0 is not null range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") - tdSql.checkRows(8) + tdSql.checkRows(9) tdSql.checkData(0, 1, False) tdSql.checkData(1, 1, True) tdSql.checkData(2, 1, True) @@ -4474,6 +4508,7 @@ class TDTestCase: tdSql.checkData(5, 1, True) tdSql.checkData(6, 1, False) tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, True) tdSql.checkData(0, 2, 1) tdSql.checkData(1, 2, 9) @@ -4483,36 +4518,37 @@ class TDTestCase: tdSql.checkData(5, 2, 13) tdSql.checkData(6, 2, 13) tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, None) tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0, 1) from {dbname}.{stbname_null} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") - tdSql.checkRows(15) - for i in range(0, 7): + tdSql.checkRows(18) + for i in range(0, 9): tdSql.checkData(i, 0, 'ctb1_null') - for i in range(7, 15): + for i in range(9, 18): tdSql.checkData(i, 0, 'ctb2_null') tdSql.checkData(0, 1, '2020-02-01 00:00:01.000') - tdSql.checkData(6, 1, '2020-02-01 00:00:13.000') + tdSql.checkData(8, 1, '2020-02-01 00:00:17.000') - tdSql.checkData(7, 1, '2020-02-01 00:00:01.000') - tdSql.checkData(14, 1, '2020-02-01 00:00:15.000') + tdSql.checkData(9, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(17, 1, '2020-02-01 00:00:17.000') tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} where c0 is not null partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") - tdSql.checkRows(15) - for i in range(0, 7): + tdSql.checkRows(18) + for i in range(0, 9): tdSql.checkData(i, 0, 'ctb1_null') - for i in range(7, 15): + for i in range(9, 18): tdSql.checkData(i, 0, 'ctb2_null') tdSql.checkData(0, 1, '2020-02-01 00:00:01.000') - tdSql.checkData(6, 1, '2020-02-01 00:00:13.000') + tdSql.checkData(8, 1, '2020-02-01 00:00:17.000') - tdSql.checkData(7, 1, '2020-02-01 00:00:01.000') - tdSql.checkData(14, 1, '2020-02-01 00:00:15.000') + tdSql.checkData(9, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(17, 1, '2020-02-01 00:00:17.000') # fill linear # normal table diff --git a/tests/system-test/2-query/normal.py b/tests/system-test/2-query/normal.py index f4b40f408b..161a9e610d 100644 --- a/tests/system-test/2-query/normal.py +++ b/tests/system-test/2-query/normal.py @@ -140,14 +140,14 @@ class TDTestCase: tdsql2 = tdCom.newTdSqlWithTimezone(timezone="UTC") tdsql2.query(f"select * from {dbname}.tzt") tdsql2.checkRows(1) - tdsql2.checkData(0, 0, "2018-09-17 01:00:00") - - + # checkData:The expected date and time is the local time zone of the machine where the test case is executed. + tdsql2.checkData(0, 0, "2018-09-17 09:00:00") + tdsql2.execute(f'insert into {dbname}.tzt values({self.ts + 1000}, 2)') tdsql2.query(f"select * from {dbname}.tzt order by ts") tdsql2.checkRows(2) - tdsql2.checkData(0, 0, "2018-09-17 01:00:00") - tdsql2.checkData(1, 0, "2018-09-17 01:00:01") + tdsql2.checkData(0, 0, "2018-09-17 09:00:00") + tdsql2.checkData(1, 0, "2018-09-17 09:00:01") tdsql2 = tdCom.newTdSqlWithTimezone(timezone="Asia/Shanghai") tdsql2.query(f"select * from {dbname}.tzt order by ts") @@ -160,7 +160,7 @@ class TDTestCase: tdSql.prepare() self.timeZoneTest() - self.inAndNotinTest() + # self.inAndNotinTest() def stop(self): @@ -168,4 +168,5 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) + tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/tsma.py b/tests/system-test/2-query/tsma.py index 1b52302503..f05398600b 100644 --- a/tests/system-test/2-query/tsma.py +++ b/tests/system-test/2-query/tsma.py @@ -871,6 +871,7 @@ class TDTestCase: .should_query_with_table('meters', '2018-09-17 09:00:00.200', '2018-09-17 09:29:59:999') .should_query_with_tsma('tsma2', '2018-09-17 09:30:00', '2018-09-17 09:59:59.999') .should_query_with_table('meters', '2018-09-17 10:00:00.000', '2018-09-17 10:23:19.800').get_qc()) + tdSql.query('show create table test.meters') self.check(ctxs) if not ignore_some_tests: tdSql.execute('create database db2') diff --git a/tests/system-test/7-tmq/subscribeStb2.py b/tests/system-test/7-tmq/subscribeStb2.py index cdbc41a593..02d1630be7 100644 --- a/tests/system-test/7-tmq/subscribeStb2.py +++ b/tests/system-test/7-tmq/subscribeStb2.py @@ -266,7 +266,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt: + if totalConsumeRows < expectrowcnt: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") @@ -287,7 +287,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt*2: + if totalConsumeRows < expectrowcnt*2: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*2)) tdLog.exit("tmq consume rows error!") diff --git a/tests/system-test/7-tmq/subscribeStb3.py b/tests/system-test/7-tmq/subscribeStb3.py index ed44ab1fb1..5df58d8122 100644 --- a/tests/system-test/7-tmq/subscribeStb3.py +++ b/tests/system-test/7-tmq/subscribeStb3.py @@ -20,12 +20,9 @@ class actionType(Enum): class TDTestCase: hostname = socket.gethostname() - #rpcDebugFlagVal = '143' - #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} - #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal - #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} - #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal - #print ("===================: ", updatecfgDict) + clientCfgDict = {'debugFlag': 135} + updatecfgDict = {'debugFlag': 135, 'asynclog': 0} + updatecfgDict["clientCfg"] = clientCfgDict def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) diff --git a/tests/system-test/7-tmq/td-30270.py b/tests/system-test/7-tmq/td-30270.py new file mode 100644 index 0000000000..73c7a88f65 --- /dev/null +++ b/tests/system-test/7-tmq/td-30270.py @@ -0,0 +1,80 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from taos.tmq import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + clientCfgDict = {'debugFlag': 135} + updatecfgDict = {'debugFlag': 135, 'clientCfg':clientCfgDict} + # updatecfgDict = {'debugFlag': 135, 'clientCfg':clientCfgDict, 'tmqRowSize':1} + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def consume_test(self): + + tdSql.execute(f'create database if not exists d1') + tdSql.execute(f'use d1') + tdSql.execute(f'create table st(ts timestamp, i int) tags(t int)') + tdSql.execute(f'insert into t1 using st tags(1) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into t2 using st tags(2) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into t3 using st tags(3) values(now, 1) (now+1s, 2)') + + + tdSql.execute(f'create topic topic_all as select * from st') + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + } + consumer = Consumer(consumer_dict) + + try: + consumer.unsubscribe() + consumer.unsubscribe() + consumer.subscribe(["topic_all"]) + consumer.subscribe(["topic_all"]) + except TmqError: + tdLog.exit(f"subscribe error") + + cnt = 0 + try: + while True: + res = consumer.poll(2) + if not res: + break + val = res.value() + if val is None: + print(f"null val") + continue + for block in val: + cnt += len(block.fetchall()) + + print(f"block {cnt} rows") + + finally: + consumer.unsubscribe(); + consumer.close() + def run(self): + self.consume_test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqClientConsLog.py b/tests/system-test/7-tmq/tmqClientConsLog.py deleted file mode 100644 index 83d6f93be1..0000000000 --- a/tests/system-test/7-tmq/tmqClientConsLog.py +++ /dev/null @@ -1,231 +0,0 @@ - -import taos -import sys -import time -import socket -import os -import threading -import math - -from util.log import * -from util.sql import * -from util.cases import * -from util.dnodes import * -from util.common import * -sys.path.append("./7-tmq") -from tmqCommon import * - -class TDTestCase: - - clientCfgDict = {'debugFlag': 135} - updatecfgDict = {'debugFlag': 131, 'clientCfg':clientCfgDict} - def __init__(self): - self.vgroups = 3 - self.ctbNum = 10 - self.rowsPerTbl = 1000 - - def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) - tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) - - def prepareTestEnv(self): - tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 2, - 'stbName': 'stb', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 1000, - 'batchNum': 100, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 10, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) - tdLog.info("create stb") - tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) - tdLog.info("create ctb") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], - ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - - # tdLog.info("restart taosd to ensure that the data falls into the disk") - # tdDnodes.stop(1) - # tdDnodes.start(1) - # tdSql.query("flush database %s"%(paraDict['dbName'])) - return - - def updateRowsOfConsumer(self, consumerDict, consumerId, totalRowsOfConsumer): - for key in consumerDict: - if key == consumerId: - consumerDict[key] = totalRowsOfConsumer - return - - consumerDict[consumerId] = totalRowsOfConsumer - return - - def checkClientLog(self, actConsumeTotalRows, numOfConsumer): - # 01931245 TSC consumer:0x5ee20f124420000c process poll rsp, vgId:5, offset:log:3399, blocks:2, rows:6000 vg total:330000 total:654000, reqId:0xa77d2245ae20112 - # 01931245 TSC consumer:0x5ee20f124420000c process poll rsp, vgId:7, offset:log:3384, blocks:1, rows:2000 vg total:326000 total:656000, reqId:0xa77d2245b050113 - # 01931246 TSC consumer:0x5ee20f124420000d process poll rsp, vgId:6, offset:log:3400, blocks:2, rows:6000 vg total:330000 total:330000, reqId:0xa77d2245b380116 - # 01931246 TSC consumer:0x5ee20f124420000d process poll rsp, vgId:6, offset:log:3460, blocks:2, rows:6000 vg total:336000 total:336000, reqId:0xa77d2245b8f011a - # 01931246 TSC consumer:0x5ee20f124420000d process poll rsp, vgId:6, offset:log:3520, blocks:2, rows:6000 vg total:342000 total:342000, reqId:0xa77d2245beb011f - # 01931246 TSC consumer:0x5ee20f124420000d process poll rsp, vgId:6, offset:log:3567, blocks:1, rows:2000 vg total:344000 total:344000, reqId:0xa77d2245c430121 - # filter key: process poll rsp, vgId - - tdLog.printNoPrefix("======== start filter key info from client log file") - - cfgPath = tdCom.getClientCfgPath() - taosLogFile = '%s/../log/taoslog*'%(cfgPath) - filterResultFile = '%s/../log/filter'%(cfgPath) - cmdStr = 'grep -h "process poll rsp, vgId:" %s >> %s'%(taosLogFile, filterResultFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumerDict = {} - for index, line in enumerate(open(filterResultFile,'r')): - - # tdLog.info("row[%d]: %s"%(index, line)) - valueList = line.split(',') - # for i in range(len(valueList)): - # tdLog.info("index[%d]: %s"%(i, valueList[i])) - # get consumer id - list2 = valueList[0].split(':') - list3 = list2[3].split() - consumerId = list3[0] - print("consumerId: %s"%(consumerId)) - - # # get vg id - # list2 = valueList[1].split(':') - # vgId = list2[1] - # print("vgId: %s"%(vgId)) - - # get total rows of a certain consuer - list2 = valueList[6].split(':') - totalRowsOfConsumer = list2[1] - print("totalRowsOfConsumer: %s"%(totalRowsOfConsumer)) - - # update a certain info - self.updateRowsOfConsumer(consumerDict, consumerId, totalRowsOfConsumer) - - # print(consumerDict) - if numOfConsumer != len(consumerDict): - tdLog.info("expect consumer num: %d, act consumer num: %d"%(numOfConsumer, len(consumerDict))) - tdLog.exit("act consumer error!") - - # total rows of all consumers - totalRows = 0 - for key in consumerDict: - totalRows += int(consumerDict[key]) - - if totalRows < actConsumeTotalRows: - tdLog.info("expect consume total rows: %d, act consume total rows: %d"%(actConsumeTotalRows, totalRows)) - tdLog.exit("act consume rows error!") - return - - def tmqCase1(self): - tdLog.printNoPrefix("======== test case 1: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 2, - 'stbName': 'stb', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 1000, - 'batchNum': 100, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 10, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - topicNameList = ['topic1'] - expectRowsList = [] - tmqCom.initConsumerTable() - - tdLog.info("create topics from stb with filter") - queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) - # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - tdSql.query(queryString) - expectRowsList.append(tdSql.getRows()) - totalRowsInserted = expectRowsList[0] - - # init consume info, and start tmq_sim, then check consume result - tdLog.info("insert consume info to consume processor") - consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] - topicList = topicNameList[0] - ifcheckdata = 0 - ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:500, auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - consumerId = 1 - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor 0") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - - expectRows = 2 - resultList = tmqCom.selectConsumeResult(expectRows) - actConsumeTotalRows = resultList[0] + resultList[1] - - tdLog.info("two consumers poll rows: %d， %d"%(resultList[0], resultList[1])) - - tdLog.info("the consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted)) - if not (totalRowsInserted <= actConsumeTotalRows): - tdLog.exit("%d tmq consume rows error!"%consumerId) - - self.checkClientLog(actConsumeTotalRows, 2) - - time.sleep(10) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 1 end ...... ") - - def run(self): - tdSql.prepare() - self.prepareTestEnv() - self.tmqCase1() - - - def stop(self): - tdSql.close() - tdLog.success(f"{__file__} successfully executed") - -event = threading.Event() - -tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index 7b31019572..96352fbe52 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -198,7 +198,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if not (expectrowcnt <= resultList[0] and totalRowsInserted >= resultList[0]): + if expectrowcnt > resultList[0]: tdLog.info("act consume rows: %d, expect consume rows between %d and %d"%(resultList[0], expectrowcnt, totalRowsInserted)) tdLog.exit("%d tmq consume rows error!"%consumerId) @@ -219,7 +219,7 @@ class TDTestCase: actConsumeTotalRows = firstConsumeRows + resultList[0] - if not (expectrowcnt >= resultList[0] and totalRowsInserted == actConsumeTotalRows): + if totalRowsInserted > actConsumeTotalRows: tdLog.info("act consume rows, first: %d, second: %d "%(firstConsumeRows, resultList[0])) tdLog.info("and sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted)) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmq_ts-5473.py b/tests/system-test/7-tmq/tmq_ts-5473.py new file mode 100644 index 0000000000..ad08fa559c --- /dev/null +++ b/tests/system-test/7-tmq/tmq_ts-5473.py @@ -0,0 +1,39 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from taos.tmq import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 135, 'asynclog': 0} + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def run(self): + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/tmq_write_raw_test'%(buildPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + return + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmq_ts5466.py b/tests/system-test/7-tmq/tmq_ts5466.py new file mode 100644 index 0000000000..1afe74c3b4 --- /dev/null +++ b/tests/system-test/7-tmq/tmq_ts5466.py @@ -0,0 +1,51 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from taos.tmq import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 135, 'asynclog': 0} + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def run(self): + tdSql.execute(f'create database if not exists db_taosx') + tdSql.execute(f'create database if not exists db_5466') + tdSql.execute(f'use db_5466') + tdSql.execute(f'create stable if not exists s5466 (ts timestamp, c1 int, c2 int) tags (t binary(32))') + tdSql.execute(f'insert into t1 using s5466 tags("__devicid__") values(1669092069068, 0, 1)') + for i in range(80): + if i < 3: + continue + tdSql.execute(f'alter stable s5466 add column c{i} int') + tdSql.execute(f'insert into t1(ts, c1, c2) values(1669092069067, 0, 1)') + tdSql.execute(f'flush database db_5466') + + tdSql.execute("create topic db_5466_topic with meta as database db_5466") + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/tmq_ts5466'%(buildPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + return + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/8-stream/stream_with_pk_tag.py b/tests/system-test/8-stream/stream_with_pk_tag.py new file mode 100644 index 0000000000..07c24f23e2 --- /dev/null +++ b/tests/system-test/8-stream/stream_with_pk_tag.py @@ -0,0 +1,38 @@ +import threading +from util.log import * +from util.sql import * +from util.cases import * +from util.common import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 135, 'asynclog': 0} + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self.tdCom = tdCom + self.dbname = "stream_test" + + def stream_with_pk_tag(self): + tdSql.execute(f"create database {self.dbname} vgroups 4;") + tdSql.execute(f"use {self.dbname};") + tdSql.execute("create table st(ts timestamp, a int primary key, b int , c int, d double) tags(ta varchar(100),tb int,tc int);") + tdSql.execute('create table t1 using st tags("aa", 1, 2);') + tdSql.execute('create stream streams3_2 trigger at_once ignore expired 0 ignore update 0 into streamt3_2 as select _wstart, a, max(b), count(*), ta from st partition by ta, a session(ts, 10s);;') + sql_list = ["insert into stream_test.t1 values(1648791210001,1,2,3,4.1);", "insert into stream_test.t1 values(1648791210002,2,2,3,1.1);", "insert into stream_test.t1 values(1648791220000,3,2,3,2.1);", "insert into stream_test.t1 values(1648791220001,4,2,3,3.1);"] + for i in range(5): + for sql in sql_list: + tdSql.execute(sql) + time.sleep(2) + + def run(self): + self.stream_with_pk_tag() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index 1f2b3f476c..e86047bca8 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -87,7 +87,6 @@ python3 ./test.py -f 7-tmq/ins_topics_test.py python3 ./test.py -f 7-tmq/tmqMaxTopic.py python3 ./test.py -f 7-tmq/tmqParamsTest.py python3 ./test.py -f 7-tmq/tmqParamsTest.py -R -python3 ./test.py -f 7-tmq/tmqClientConsLog.py python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py python3 ./test.py -f 7-tmq/tmqOffset.py diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 9b1c949807..65ae9fad54 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -1528,7 +1528,7 @@ bool needInsertFrom(char* sql, int len) { // p is string following select keyword bool appendAfterSelect(TAOS* con, SShellCmd* cmd, char* sql, int32_t len) { - char* p = strndup(sql, len); + char* p = taosStrndup(sql, len); // union all char* p1; @@ -1637,7 +1637,7 @@ bool matchSelectQuery(TAOS* con, SShellCmd* cmd) { } // search - char* sql_cp = strndup(p, len); + char* sql_cp = taosStrndup(p, len); int32_t n = searchAfterSelect(sql_cp, len); taosMemoryFree(sql_cp); if (n == -1 || n > len) return false; @@ -1712,7 +1712,7 @@ bool matchCreateTable(TAOS* con, SShellCmd* cmd) { p += 13; len -= 13; - char* ps = strndup(p, len); + char* ps = taosStrndup(p, len); bool ret = false; char* last = lastWord(ps); @@ -1765,7 +1765,7 @@ bool matchOther(TAOS* con, SShellCmd* cmd) { if (len < 8) return false; // like 'from ( ' - char* sql = strndup(p, len); + char* sql = taosStrndup(p, len); char* last = lastWord(sql); if (strcmp(last, "from(") == 0) { @@ -1828,7 +1828,7 @@ bool matchOther(TAOS* con, SShellCmd* cmd) { bool matchEnd(TAOS* con, SShellCmd* cmd) { // str dump bool ret = false; - char* ps = strndup(cmd->command, cmd->commandSize); + char* ps = taosStrndup(cmd->command, cmd->commandSize); char* last = lastWord(ps); char* elast = strrchr(last, '.'); // find end last if (elast) { diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index fc7c914c21..1ec1697c3a 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -34,10 +34,10 @@ int shell_conn_ws_server(bool first) { size_t len = strlen(shell.args.dsn); char * dsn = taosMemoryMalloc(len + 32); sprintf(dsn, "%s&conn_mode=1", shell.args.dsn); - shell.ws_conn = ws_connect_with_dsn(dsn); + shell.ws_conn = ws_connect(dsn); taosMemoryFree(dsn); } else { - shell.ws_conn = ws_connect_with_dsn(shell.args.dsn); + shell.ws_conn = ws_connect(shell.args.dsn); } if (NULL == shell.ws_conn) { @@ -95,7 +95,7 @@ int shell_conn_ws_server(bool first) { static int horizontalPrintWebsocket(WS_RES* wres, double* execute_time) { const void* data = NULL; int rows; - ws_fetch_block(wres, &data, &rows); + ws_fetch_raw_block(wres, &data, &rows); if (wres) { *execute_time += (double)(ws_take_timing(wres)/1E6); } @@ -129,7 +129,7 @@ static int horizontalPrintWebsocket(WS_RES* wres, double* execute_time) { putchar('\n'); } numOfRows += rows; - ws_fetch_block(wres, &data, &rows); + ws_fetch_raw_block(wres, &data, &rows); } while (rows && !shell.stop_query); return numOfRows; } @@ -137,7 +137,7 @@ static int horizontalPrintWebsocket(WS_RES* wres, double* execute_time) { static int verticalPrintWebsocket(WS_RES* wres, double* pexecute_time) { int rows = 0; const void* data = NULL; - ws_fetch_block(wres, &data, &rows); + ws_fetch_raw_block(wres, &data, &rows); if (wres) { *pexecute_time += (double)(ws_take_timing(wres)/1E6); } @@ -172,7 +172,7 @@ static int verticalPrintWebsocket(WS_RES* wres, double* pexecute_time) { } numOfRows++; } - ws_fetch_block(wres, &data, &rows); + ws_fetch_raw_block(wres, &data, &rows); } while (rows && !shell.stop_query); return numOfRows; } @@ -192,7 +192,7 @@ static int dumpWebsocketToFile(const char* fname, WS_RES* wres, } int rows = 0; const void* data = NULL; - ws_fetch_block(wres, &data, &rows); + ws_fetch_raw_block(wres, &data, &rows); if (wres) { *pexecute_time += (double)(ws_take_timing(wres)/1E6); } @@ -226,7 +226,7 @@ static int dumpWebsocketToFile(const char* fname, WS_RES* wres, } taosFprintfFile(pFile, "\r\n"); } - ws_fetch_block(wres, &data, &rows); + ws_fetch_raw_block(wres, &data, &rows); } while (rows && !shell.stop_query); taosCloseFile(&pFile); return numOfRows; diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index e5902856e6..991a004a74 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -3,6 +3,8 @@ add_dependencies(tmq_demo taos) add_executable(tmq_sim tmqSim.c) add_executable(create_table createTable.c) add_executable(tmq_taosx_ci tmq_taosx_ci.c) +add_executable(tmq_ts5466 tmq_ts5466.c) +add_executable(tmq_write_raw_test tmq_write_raw_test.c) add_executable(write_raw_block_test write_raw_block_test.c) add_executable(sml_test sml_test.c) add_executable(get_db_name_test get_db_name_test.c) @@ -53,6 +55,13 @@ target_link_libraries( PUBLIC common PUBLIC os ) +target_link_libraries( + tmq_ts5466 + PUBLIC taos + PUBLIC util + PUBLIC common + PUBLIC os +) target_link_libraries( tmq_taosx_ci PUBLIC taos @@ -69,11 +78,11 @@ target_link_libraries( ) target_link_libraries( - replay_test - PUBLIC taos - PUBLIC util - PUBLIC common - PUBLIC os + replay_test + PUBLIC taos + PUBLIC util + PUBLIC common + PUBLIC os ) target_link_libraries( @@ -84,6 +93,14 @@ target_link_libraries( PUBLIC os ) +target_link_libraries( + tmq_write_raw_test + PUBLIC taos + PUBLIC util + PUBLIC common + PUBLIC os +) + target_link_libraries( sml_test PUBLIC taos diff --git a/utils/test/c/tmq_ts5466.c b/utils/test/c/tmq_ts5466.c new file mode 100644 index 0000000000..86a247a0ed --- /dev/null +++ b/utils/test/c/tmq_ts5466.c @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "cJSON.h" +#include "taos.h" +#include "tmsg.h" +#include "types.h" + +static TAOS* use_db() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return NULL; + } + + TAOS_RES* pRes = taos_query(pConn, "use db_taosx"); + if (taos_errno(pRes) != 0) { + printf("error in use db_taosx, reason:%s\n", taos_errstr(pRes)); + return NULL; + } + taos_free_result(pRes); + return pConn; +} + +static void msg_process(TAOS_RES* msg) { + printf("-----------topic-------------: %s\n", tmq_get_topic_name(msg)); + printf("db: %s\n", tmq_get_db_name(msg)); + printf("vg: %d\n", tmq_get_vgroup_id(msg)); + TAOS* pConn = use_db(); + if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META || tmq_get_res_type(msg) == TMQ_RES_METADATA) { + char* result = tmq_get_json_meta(msg); + printf("meta result: %s\n", result); + tmq_free_json_meta(result); + } + + tmq_raw_data raw = {0}; + tmq_get_raw(msg, &raw); + printf("write raw data type: %d\n", raw.raw_type); + int32_t ret = tmq_write_raw(pConn, raw); + printf("write raw data: %s\n", tmq_err2str(ret)); + ASSERT(ret == 0); + + tmq_free_raw(raw); + taos_close(pConn); +} + +void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + printf("commit %d tmq %p param %p\n", code, tmq, param); +} + +tmq_t* build_consumer() { + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set(conf, "client.id", "my app 1"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "msg.consume.excluded", "1"); + tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + assert(tmq); + tmq_conf_destroy(conf); + return tmq; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "db_5466_topic"); + return topic_list; +} + +void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { + int32_t code; + + if ((code = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); + printf("subscribe err\n"); + return; + } + int32_t cnt = 0; + while (1) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 5000); + if (tmqmessage) { + cnt++; + msg_process(tmqmessage); + taos_free_result(tmqmessage); + } else { + break; + } + } + + code = tmq_consumer_close(tmq); + if (code) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +int main(int argc, char* argv[]) { + tmq_t* tmq = build_consumer(); + tmq_list_t* topic_list = build_topic_list(); + basic_consume_loop(tmq, topic_list); + tmq_list_destroy(topic_list); +} \ No newline at end of file diff --git a/utils/test/c/tmq_write_raw_test.c b/utils/test/c/tmq_write_raw_test.c new file mode 100644 index 0000000000..f33fac9a0a --- /dev/null +++ b/utils/test/c/tmq_write_raw_test.c @@ -0,0 +1,281 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "cJSON.h" +#include "taos.h" +#include "tmsg.h" +#include "types.h" + +static TAOS* use_db() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return NULL; + } + + TAOS_RES* pRes = taos_query(pConn, "use db_taosx"); + if (taos_errno(pRes) != 0) { + printf("error in use db_taosx, reason:%s\n", taos_errstr(pRes)); + return NULL; + } + taos_free_result(pRes); + return pConn; +} + +static void msg_process(TAOS_RES* msg) { + printf("-----------topic-------------: %s\n", tmq_get_topic_name(msg)); + printf("db: %s\n", tmq_get_db_name(msg)); + printf("vg: %d\n", tmq_get_vgroup_id(msg)); + TAOS* pConn = use_db(); + if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META || tmq_get_res_type(msg) == TMQ_RES_METADATA) { + char* result = tmq_get_json_meta(msg); + printf("meta result: %s\n", result); + tmq_free_json_meta(result); + } + + tmq_raw_data raw = {0}; + tmq_get_raw(msg, &raw); + printf("write raw data type: %d\n", raw.raw_type); + int32_t ret = tmq_write_raw(pConn, raw); + printf("write raw data: %s\n", tmq_err2str(ret)); + ASSERT(ret == 0); + + tmq_free_raw(raw); + taos_close(pConn); +} + +int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000, \"ttt\", true)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct0 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct1 using st1(t1) tags(2000)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct2 using st1(t1) tags(NULL)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct1 using st1(t1) tags(2000) values(1626006833600, 3, 4, 'b')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct3 using st1(t1) tags(3000)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct0 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct1 using st1(t1) tags(2000) values(1626006833600, 3, 4, 'b')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query( + pConn, + "insert into ct3 using st1(t1) tags(3000) values(1626006833600, 5, 6, 'c') ct1 using st1(t1) tags(2000) values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, " + "'ddd') ct0 using st1 tags(1000, \"ttt\", true) values(1626006833603, 4, 3, 'hwj') ct1 using st1(t1) tags(2000) values(now+5s, 23, 32, 's21ds')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + return 0; +} + +int32_t init_env() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_taosx"); + if (taos_errno(pRes) != 0) { + printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists db_taosx vgroups 1 wal_retention_period 3600"); + if (taos_errno(pRes) != 0) { + printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop topic if exists topic_db"); + if (taos_errno(pRes) != 0) { + printf("error in drop topic, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists abc1"); + if (taos_errno(pRes) != 0) { + printf("error in drop db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1 wal_retention_period 3600"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + buildDatabase(pConn, pRes); + + taos_close(pConn); + return 0; +} + +int32_t create_topic() { + printf("create topic\n"); + TAOS_RES* pRes; + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + pRes = taos_query(pConn, "create topic topic_db with meta as database abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic topic_db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + taos_close(pConn); + return 0; +} + +void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + printf("commit %d tmq %p param %p\n", code, tmq, param); +} + +tmq_t* build_consumer() { + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set(conf, "client.id", "my app 1"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "msg.consume.excluded", "1"); +// tmq_conf_set(conf, "max.poll.interval.ms", "20000"); + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + assert(tmq); + tmq_conf_destroy(conf); + return tmq; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "topic_db"); + return topic_list; +} + +void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { + int32_t code; + + if ((code = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); + printf("subscribe err\n"); + return; + } + int32_t cnt = 0; + while (1) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 5000); + if (tmqmessage) { + cnt++; + msg_process(tmqmessage); + taos_free_result(tmqmessage); + } else { + break; + } + } + + code = tmq_consumer_close(tmq); + if (code) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +int main(int argc, char* argv[]) { + if (init_env() < 0) { + return -1; + } + create_topic(); + + tmq_t* tmq = build_consumer(); + tmq_list_t* topic_list = build_topic_list(); + basic_consume_loop(tmq, topic_list); + tmq_list_destroy(topic_list); +}