diff --git a/docs/examples/c-ws/.gitignore b/docs/examples/c-ws/.gitignore new file mode 100644 index 0000000000..afe9743149 --- /dev/null +++ b/docs/examples/c-ws/.gitignore @@ -0,0 +1,3 @@ +* +!*.c +!.gitignore diff --git a/docs/examples/c-ws/connect_example.c b/docs/examples/c-ws/connect_example.c new file mode 100644 index 0000000000..5a9cd64292 --- /dev/null +++ b/docs/examples/c-ws/connect_example.c @@ -0,0 +1,21 @@ +// compile with +// gcc connect_example.c -o connect_example -ltaos +#include +#include +#include "taosws.h" + +int main() { + ws_enable_log("debug"); + char *dsn = "ws://localhost:6041"; + WS_TAOS *taos = ws_connect(dsn); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return -1; + } + fprintf(stdout, "Connected to %s successfully.\n", dsn); + + /* put your code here for read and write */ + + // close & clean + ws_close(taos); +} diff --git a/docs/examples/c-ws/create_db_demo.c b/docs/examples/c-ws/create_db_demo.c new file mode 100755 index 0000000000..13f038f1df --- /dev/null +++ b/docs/examples/c-ws/create_db_demo.c @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o create_db_demo create_db_demo.c -ltaos + +#include +#include +#include +#include +#include "taosws.h" + +static int DemoCreateDB() { + ws_enable_log("debug"); + // ANCHOR: create_db_and_table + int code = 0; + char *dsn = "ws://localhost:6041"; + // connect + WS_TAOS *taos = ws_connect(dsn); + + if (taos == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return -1; + } + + // create database + WS_RES *result = ws_query(taos, "CREATE DATABASE IF NOT EXISTS power"); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to create database power, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(result)); + ws_close(taos); + return -1; + } + ws_free_result(result); + fprintf(stdout, "Create database power successfully.\n"); + + // create table + const char *sql = + "CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId " + "INT, location BINARY(24))"; + result = ws_query(taos, sql); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to create stable power.meters, ErrCode: 0x%x, ErrMessage: %s\n.", code, ws_errstr(result)); + ws_close(taos); + return -1; + } + ws_free_result(result); + fprintf(stdout, "Create stable power.meters successfully.\n"); + + // close & clean + ws_close(taos); + return 0; + // ANCHOR_END: create_db_and_table +} + +int main(int argc, char *argv[]) { return DemoCreateDB(); } diff --git a/docs/examples/c-ws/insert_data_demo.c b/docs/examples/c-ws/insert_data_demo.c new file mode 100644 index 0000000000..4abaea4c5e --- /dev/null +++ b/docs/examples/c-ws/insert_data_demo.c @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o insert_data_demo insert_data_demo.c -ltaos + +#include +#include +#include +#include +#include "taosws.h" + +static int DemoInsertData() { + // ANCHOR: insert_data + int code = 0; + char *dsn = "ws://localhost:6041"; + // connect + WS_TAOS *taos = ws_connect(dsn); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return -1; + } + + // insert data, please make sure the database and table are already created + const char *sql = + "INSERT INTO " + "power.d1001 USING power.meters TAGS(2,'California.SanFrancisco') " + "VALUES " + "(NOW + 1a, 10.30000, 219, 0.31000) " + "(NOW + 2a, 12.60000, 218, 0.33000) " + "(NOW + 3a, 12.30000, 221, 0.31000) " + "power.d1002 USING power.meters TAGS(3, 'California.SanFrancisco') " + "VALUES " + "(NOW + 1a, 10.30000, 218, 0.25000) "; + WS_RES *result = ws_query(taos, sql); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to insert data to power.meters, sql: %s, ErrCode: 0x%x, ErrMessage: %s\n.", sql, code, + ws_errstr(result)); + ws_close(taos); + return -1; + } + ws_free_result(result); + + // you can check affectedRows here + int rows = ws_affected_rows(result); + fprintf(stdout, "Successfully inserted %d rows into power.meters.\n", rows); + + // close & clean + ws_close(taos); + return 0; + // ANCHOR_END: insert_data +} + +int main(int argc, char *argv[]) { return DemoInsertData(); } diff --git a/docs/examples/c-ws/query_data_demo.c b/docs/examples/c-ws/query_data_demo.c new file mode 100644 index 0000000000..e70e15458d --- /dev/null +++ b/docs/examples/c-ws/query_data_demo.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o query_data_demo query_data_demo.c -ltaos + +#include +#include +#include +#include +#include "taosws.h" + +static int DemoQueryData() { + // ANCHOR: query_data + int code = 0; + char *dsn = "ws://localhost:6041"; + + // connect + WS_TAOS *taos = ws_connect(dsn); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return -1; + } + + // query data, please make sure the database and table are already created + const char *sql = "SELECT ts, current, location FROM power.meters limit 100"; + WS_RES *result = ws_query(taos, sql); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to query data from power.meters, sql: %s, ErrCode: 0x%x, ErrMessage: %s\n.", sql, code, + ws_errstr(result)); + ws_close(taos); + return -1; + } + + WS_ROW row = NULL; + int rows = 0; + int num_fields = ws_field_count(result); + const WS_FIELD *fields = ws_fetch_fields(result); + + fprintf(stdout, "query successfully, got %d fields, the sql is: %s.\n", num_fields, sql); + + // fetch the records row by row + while ((row = ws_fetch_row(result))) { + // Add your data processing logic here + + rows++; + } + fprintf(stdout, "total rows: %d\n", rows); + ws_free_result(result); + + // close & clean + ws_close(taos); + return 0; + // ANCHOR_END: query_data +} + +int main(int argc, char *argv[]) { return DemoQueryData(); } diff --git a/docs/examples/c-ws/sml_insert_demo.c b/docs/examples/c-ws/sml_insert_demo.c new file mode 100644 index 0000000000..1149aa2543 --- /dev/null +++ b/docs/examples/c-ws/sml_insert_demo.c @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o sml_insert_demo sml_insert_demo.c -ltaos + +#include +#include +#include +#include "taosws.h" + +static int DemoSmlInsert() { + // ANCHOR: schemaless + int code = 0; + char *dsn = "ws://localhost:6041"; + + // connect + WS_TAOS *taos = ws_connect(dsn); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return -1; + } + + // create database + WS_RES *result = ws_query(taos, "CREATE DATABASE IF NOT EXISTS power"); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to create database power, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(result)); + ws_close(taos); + return -1; + } + ws_free_result(result); + + // use database + result = ws_query(taos, "USE power"); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to execute use power, ErrCode: 0x%x, ErrMessage: %s\n.", code, ws_errstr(result)); + ws_close(taos); + return -1; + } + ws_free_result(result); + + // schemaless demo data + char *line_demo = + "meters,groupid=2,location=California.SanFrancisco current=10.3000002f64,voltage=219i32,phase=0.31f64 " + "1626006833639"; + char *telnet_demo = "metric_telnet 1707095283260 4 host=host0 interface=eth0"; + char *json_demo = + "{\"metric\": \"metric_json\",\"timestamp\": 1626846400,\"value\": 10.3, \"tags\": {\"groupid\": 2, " + "\"location\": \"California.SanFrancisco\", \"id\": \"d1001\"}}"; + + // influxdb line protocol + char *lines[] = {line_demo}; + int totalLines = 0; + result = ws_schemaless_insert_raw(taos, line_demo, strlen(line_demo), &totalLines, WS_TSDB_SML_LINE_PROTOCOL, + WS_TSDB_SML_TIMESTAMP_MILLI_SECONDS); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to insert schemaless line data, data: %s, ErrCode: 0x%x, ErrMessage: %s\n.", line_demo, + code, ws_errstr(result)); + ws_close(taos); + return -1; + } + + fprintf(stdout, "Insert %d rows of schemaless line data successfully.\n", totalLines); + ws_free_result(result); + + // opentsdb telnet protocol + totalLines = 0; + result = ws_schemaless_insert_raw(taos, telnet_demo, strlen(telnet_demo), &totalLines, WS_TSDB_SML_TELNET_PROTOCOL, + WS_TSDB_SML_TIMESTAMP_MILLI_SECONDS); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to insert schemaless telnet data, data: %s, ErrCode: 0x%x, ErrMessage: %s\n.", telnet_demo, + code, ws_errstr(result)); + ws_close(taos); + return -1; + } + + fprintf(stdout, "Insert %d rows of schemaless telnet data successfully.\n", totalLines); + ws_free_result(result); + + // opentsdb json protocol + char *jsons[1] = {0}; + // allocate memory for json data. can not use static memory. + totalLines = 0; + result = ws_schemaless_insert_raw(taos, json_demo, strlen(json_demo), &totalLines, WS_TSDB_SML_JSON_PROTOCOL, + WS_TSDB_SML_TIMESTAMP_MILLI_SECONDS); + code = ws_errno(result); + if (code != 0) { + free(jsons[0]); + fprintf(stderr, "Failed to insert schemaless json data, Server: %s, ErrCode: 0x%x, ErrMessage: %s\n.", json_demo, + code, ws_errstr(result)); + ws_close(taos); + return -1; + } + free(jsons[0]); + + fprintf(stdout, "Insert %d rows of schemaless json data successfully.\n", totalLines); + ws_free_result(result); + + // close & clean + ws_close(taos); + return 0; + // ANCHOR_END: schemaless +} + +int main(int argc, char *argv[]) { return DemoSmlInsert(); } diff --git a/docs/examples/c-ws/stmt_insert_demo.c b/docs/examples/c-ws/stmt_insert_demo.c new file mode 100644 index 0000000000..0ddfb53303 --- /dev/null +++ b/docs/examples/c-ws/stmt_insert_demo.c @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o stmt_insert_demo stmt_insert_demo.c -ltaos + +#include +#include +#include +#include +#include "taosws.h" + +/** + * @brief execute sql only. + * + * @param taos + * @param sql + */ +void executeSQL(WS_TAOS *taos, const char *sql) { + WS_RES *res = ws_query(taos, sql); + int code = ws_errno(res); + if (code != 0) { + fprintf(stderr, "%s\n", ws_errstr(res)); + ws_free_result(res); + ws_close(taos); + exit(EXIT_FAILURE); + } + ws_free_result(res); +} + +/** + * @brief check return status and exit program when error occur. + * + * @param stmt + * @param code + * @param msg + */ +void checkErrorCode(WS_STMT *stmt, int code, const char *msg) { + if (code != 0) { + fprintf(stderr, "%s. code: %d, error: %s\n", msg, code, ws_stmt_errstr(stmt)); + ws_stmt_close(stmt); + exit(EXIT_FAILURE); + } +} + +typedef struct { + int64_t ts; + float current; + int voltage; + float phase; +} Row; + +int num_of_sub_table = 10; +int num_of_row = 10; +int total_affected = 0; +/** + * @brief insert data using stmt API + * + * @param taos + */ +void insertData(WS_TAOS *taos) { + // init + WS_STMT *stmt = ws_stmt_init(taos); + if (stmt == NULL) { + fprintf(stderr, "Failed to init ws_stmt, error: %s\n", ws_stmt_errstr(NULL)); + exit(EXIT_FAILURE); + } + // prepare + const char *sql = "INSERT INTO ? USING meters TAGS(?,?) VALUES (?,?,?,?)"; + int code = ws_stmt_prepare(stmt, sql, 0); + checkErrorCode(stmt, code, "Failed to execute ws_stmt_prepare"); + for (int i = 1; i <= num_of_sub_table; i++) { + char table_name[20]; + sprintf(table_name, "d_bind_%d", i); + char location[20]; + sprintf(location, "location_%d", i); + + // set table name and tags + WS_MULTI_BIND tags[2]; + // groupId + tags[0].buffer_type = TSDB_DATA_TYPE_INT; + tags[0].buffer_length = sizeof(int); + tags[0].length = (int32_t *)&tags[0].buffer_length; + tags[0].buffer = &i; + tags[0].is_null = NULL; + tags[0].num = 1; + // location + tags[1].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[1].buffer_length = strlen(location); + tags[1].length = (int32_t *)&tags[1].buffer_length; + tags[1].buffer = location; + tags[1].is_null = NULL; + tags[1].num = 1; + code = ws_stmt_set_tbname_tags(stmt, table_name, tags, 2); + checkErrorCode(stmt, code, "Failed to set table name and tags\n"); + + // insert rows + WS_MULTI_BIND params[4]; + // ts + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(int64_t); + params[0].length = (int32_t *)¶ms[0].buffer_length; + params[0].is_null = NULL; + params[0].num = 1; + // current + params[1].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[1].buffer_length = sizeof(float); + params[1].length = (int32_t *)¶ms[1].buffer_length; + params[1].is_null = NULL; + params[1].num = 1; + // voltage + params[2].buffer_type = TSDB_DATA_TYPE_INT; + params[2].buffer_length = sizeof(int); + params[2].length = (int32_t *)¶ms[2].buffer_length; + params[2].is_null = NULL; + params[2].num = 1; + // phase + params[3].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[3].buffer_length = sizeof(float); + params[3].length = (int32_t *)¶ms[3].buffer_length; + params[3].is_null = NULL; + params[3].num = 1; + + for (int j = 0; j < num_of_row; j++) { + struct timeval tv; + gettimeofday(&tv, NULL); + long long milliseconds = tv.tv_sec * 1000LL + tv.tv_usec / 1000; // current timestamp in milliseconds + int64_t ts = milliseconds + j; + float current = (float)rand() / RAND_MAX * 30; + int voltage = rand() % 300; + float phase = (float)rand() / RAND_MAX; + params[0].buffer = &ts; + params[1].buffer = ¤t; + params[2].buffer = &voltage; + params[3].buffer = &phase; + // bind param + code = ws_stmt_bind_param_batch(stmt, params, 4); + checkErrorCode(stmt, code, "Failed to bind param"); + } + // add batch + code = ws_stmt_add_batch(stmt); + checkErrorCode(stmt, code, "Failed to add batch"); + // execute batch + int affected_rows = 0; + code = ws_stmt_execute(stmt, &affected_rows); + checkErrorCode(stmt, code, "Failed to exec stmt"); + // get affected rows + int affected = ws_stmt_affected_rows_once(stmt); + total_affected += affected; + } + fprintf(stdout, "Successfully inserted %d rows to power.meters.\n", total_affected); + ws_stmt_close(stmt); +} + +int main() { + int code = 0; + char *dsn = "ws://localhost:6041"; + WS_TAOS *taos = ws_connect(dsn); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + exit(EXIT_FAILURE); + } + // create database and table + executeSQL(taos, "CREATE DATABASE IF NOT EXISTS power"); + executeSQL(taos, "USE power"); + executeSQL(taos, + "CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS " + "(groupId INT, location BINARY(24))"); + insertData(taos); + ws_close(taos); +} diff --git a/docs/examples/c-ws/tmq_demo.c b/docs/examples/c-ws/tmq_demo.c new file mode 100644 index 0000000000..e84c75fa3a --- /dev/null +++ b/docs/examples/c-ws/tmq_demo.c @@ -0,0 +1,488 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// to compile: gcc -o tmq_demo tmq_demo.c -ltaos -lpthread + +#include +#include +#include +#include +#include +#include +#include +#include "taosws.h" + +volatile int thread_stop = 0; +static int running = 1; +static int count = 0; +const char* topic_name = "topic_meters"; + +typedef struct { + const char* enable_auto_commit; + const char* auto_commit_interval_ms; + const char* group_id; + const char* client_id; + const char* td_connect_host; + const char* td_connect_port; + const char* td_connect_user; + const char* td_connect_pass; + const char* auto_offset_reset; +} ConsumerConfig; + +ConsumerConfig config = {.enable_auto_commit = "true", + .auto_commit_interval_ms = "1000", + .group_id = "group1", + .client_id = "client1", + .td_connect_host = "localhost", + .td_connect_port = "6030", + .td_connect_user = "root", + .td_connect_pass = "taosdata", + .auto_offset_reset = "latest"}; + +void* prepare_data(void* arg) { + int code = 0; + char* dsn = "ws://localhost:6041"; + WS_TAOS* pConn = ws_connect(dsn); + if (pConn == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return NULL; + } + + WS_RES* pRes; + int i = 1; + + while (!thread_stop) { + char buf[200] = {0}; + i++; + snprintf( + buf, sizeof(buf), + "INSERT INTO power.d1001 USING power.meters TAGS(2,'California.SanFrancisco') VALUES (NOW + %da, 10.30000, " + "219, 0.31000)", + i); + + pRes = ws_query(pConn, buf); + code = ws_errno(pRes); + if (code != 0) { + fprintf(stderr, "Failed to insert data to power.meters, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + } + ws_free_result(pRes); + sleep(1); + } + fprintf(stdout, "Prepare data thread exit\n"); + return NULL; +} + +// ANCHOR: msg_process +int32_t msg_process(WS_RES* msg) { + int32_t rows = 0; + const char* topicName = ws_tmq_get_topic_name(msg); + const char* dbName = ws_tmq_get_db_name(msg); + int32_t vgroupId = ws_tmq_get_vgroup_id(msg); + + while (true) { + // get one row data from message + WS_ROW row = ws_fetch_row(msg); + if (row == NULL) break; + + // Add your data processing logic here + + rows++; + } + + return rows; +} +// ANCHOR_END: msg_process + +WS_TAOS* init_env() { + int code = 0; + char* dsn = "ws://localhost:6041"; + WS_TAOS* pConn = ws_connect(dsn); + if (pConn == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return NULL; + } + + WS_RES* pRes; + // drop database if exists + pRes = ws_query(pConn, "DROP TOPIC IF EXISTS topic_meters"); + code = ws_errno(pRes); + if (code != 0) { + fprintf(stderr, "Failed to drop topic_meters, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + goto END; + } + ws_free_result(pRes); + + pRes = ws_query(pConn, "DROP DATABASE IF EXISTS power"); + code = ws_errno(pRes); + if (code != 0) { + fprintf(stderr, "Failed to drop database power, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + goto END; + } + ws_free_result(pRes); + + // create database + pRes = ws_query(pConn, "CREATE DATABASE power PRECISION 'ms' WAL_RETENTION_PERIOD 3600"); + code = ws_errno(pRes); + if (code != 0) { + fprintf(stderr, "Failed to create power, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + goto END; + } + ws_free_result(pRes); + + // create super table + pRes = + ws_query(pConn, + "CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS " + "(groupId INT, location BINARY(24))"); + code = ws_errno(pRes); + if (code != 0) { + fprintf(stderr, "Failed to create super table meters, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + goto END; + } + ws_free_result(pRes); + + return pConn; + +END: + ws_free_result(pRes); + ws_close(pConn); + return NULL; +} + +void deinit_env(WS_TAOS* pConn) { + if (pConn) ws_close(pConn); +} + +int32_t create_topic(WS_TAOS* pConn) { + WS_RES* pRes; + int code = 0; + + if (!pConn) { + fprintf(stderr, "Invalid input parameter.\n"); + return -1; + } + + pRes = ws_query(pConn, "USE power"); + code = ws_errno(pRes); + if (ws_errno(pRes) != 0) { + fprintf(stderr, "Failed to use power, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + return -1; + } + ws_free_result(pRes); + + pRes = ws_query( + pConn, + "CREATE TOPIC IF NOT EXISTS topic_meters AS SELECT ts, current, voltage, phase, groupid, location FROM meters"); + code = ws_errno(pRes); + if (code != 0) { + fprintf(stderr, "Failed to create topic topic_meters, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + return -1; + } + ws_free_result(pRes); + return 0; +} + +int32_t drop_topic(WS_TAOS* pConn) { + WS_RES* pRes; + int code = 0; + + if (!pConn) { + fprintf(stderr, "Invalid input parameter.\n"); + return -1; + } + + pRes = ws_query(pConn, "USE power"); + code = ws_errno(pRes); + if (ws_errno(pRes) != 0) { + fprintf(stderr, "Failed to use power, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + return -1; + } + ws_free_result(pRes); + + pRes = ws_query(pConn, "DROP TOPIC IF EXISTS topic_meters"); + code = ws_errno(pRes); + if (code != 0) { + fprintf(stderr, "Failed to drop topic topic_meters, ErrCode: 0x%x, ErrMessage: %s.\n", code, ws_errstr(pRes)); + return -1; + } + ws_free_result(pRes); + return 0; +} + +void tmq_commit_cb_print(ws_tmq_t* tmq, int32_t code, void* param) { + count += 1; + fprintf(stdout, "tmq_commit_cb_print() code: %d, tmq: %p, param: %p, count: %d.\n", code, tmq, param, count); +} + +// ANCHOR: create_consumer_1 +ws_tmq_t* build_consumer(const ConsumerConfig* config) { + ws_tmq_conf_res_t code; + ws_tmq_t* tmq = NULL; + + // create a configuration object + ws_tmq_conf_t* conf = ws_tmq_conf_new(); + + // set the configuration parameters + code = ws_tmq_conf_set(conf, "enable.auto.commit", config->enable_auto_commit); + if (WS_TMQ_CONF_OK != code) { + ws_tmq_conf_destroy(conf); + return NULL; + } + code = ws_tmq_conf_set(conf, "auto.commit.interval.ms", config->auto_commit_interval_ms); + if (WS_TMQ_CONF_OK != code) { + ws_tmq_conf_destroy(conf); + return NULL; + } + code = ws_tmq_conf_set(conf, "group.id", config->group_id); + if (WS_TMQ_CONF_OK != code) { + ws_tmq_conf_destroy(conf); + return NULL; + } + code = ws_tmq_conf_set(conf, "client.id", config->client_id); + if (WS_TMQ_CONF_OK != code) { + ws_tmq_conf_destroy(conf); + return NULL; + } + + code = ws_tmq_conf_set(conf, "auto.offset.reset", config->auto_offset_reset); + if (WS_TMQ_CONF_OK != code) { + ws_tmq_conf_destroy(conf); + return NULL; + } + + // create a consumer object + tmq = ws_tmq_consumer_new(conf, "taos://localhost:6041", NULL, 0); + +_end: + // destroy the configuration object + ws_tmq_conf_destroy(conf); + return tmq; +} +// ANCHOR_END: create_consumer_1 + +// ANCHOR: build_topic_list +// build a topic list used to subscribe +ws_tmq_list_t* build_topic_list() { + // create a empty topic list + ws_tmq_list_t* topicList = ws_tmq_list_new(); + + // append topic name to the list + int32_t code = ws_tmq_list_append(topicList, topic_name); + if (code) { + // if failed, destroy the list and return NULL + ws_tmq_list_destroy(topicList); + fprintf(stderr, + "Failed to create topic_list, topic: %s, groupId: %s, clientId: %s, ErrCode: 0x%x, ErrMessage: %s.\n", + topic_name, config.group_id, config.client_id, code, ws_tmq_errstr(NULL)); + return NULL; + } + // if success, return the list + return topicList; +} +// ANCHOR_END: build_topic_list + +// ANCHOR: basic_consume_loop +void basic_consume_loop(ws_tmq_t* tmq) { + int32_t totalRows = 0; // total rows consumed + int32_t msgCnt = 0; // total messages consumed + int32_t timeout = 5000; // poll timeout + + while (running) { + // poll message from TDengine + WS_RES* tmqmsg = ws_tmq_consumer_poll(tmq, timeout); + if (tmqmsg) { + msgCnt++; + + // Add your data processing logic here + totalRows += msg_process(tmqmsg); + + // free the message + ws_free_result(tmqmsg); + } + if (msgCnt > 50) { + // consume 50 messages and break + break; + } + } + + // print the result: total messages and total rows consumed + fprintf(stdout, "%d msg consumed, include %d rows\n", msgCnt, totalRows); +} +// ANCHOR_END: basic_consume_loop + +// ANCHOR: consume_repeatly +void consume_repeatly(ws_tmq_t* tmq) { + int32_t numOfAssignment = 0; + ws_tmq_topic_assignment* pAssign = NULL; + + // get the topic assignment + int32_t code = ws_tmq_get_topic_assignment(tmq, topic_name, &pAssign, &numOfAssignment); + if (code != 0 || pAssign == NULL || numOfAssignment == 0) { + fprintf(stderr, "Failed to get assignment, topic: %s, groupId: %s, clientId: %s, ErrCode: 0x%x, ErrMessage: %s.\n", + topic_name, config.group_id, config.client_id, code, ws_tmq_errstr(tmq)); + return; + } + + // seek to the earliest offset + for (int32_t i = 0; i < numOfAssignment; ++i) { + ws_tmq_topic_assignment* p = &pAssign[i]; + + code = ws_tmq_offset_seek(tmq, topic_name, p->vgId, p->begin); + if (code != 0) { + fprintf(stderr, + "Failed to seek offset, topic: %s, groupId: %s, clientId: %s, vgId: %d, ErrCode: 0x%x, ErrMessage: %s.\n", + topic_name, config.group_id, config.client_id, p->vgId, code, ws_tmq_errstr(tmq)); + break; + } + } + if (code == 0) fprintf(stdout, "Assignment seek to beginning successfully.\n"); + + // free the assignment array + ws_tmq_free_assignment(pAssign, numOfAssignment); + + // let's consume the messages again + basic_consume_loop(tmq); +} +// ANCHOR_END: consume_repeatly + +// ANCHOR: manual_commit +void manual_commit(ws_tmq_t* tmq) { + int32_t totalRows = 0; // total rows consumed + int32_t msgCnt = 0; // total messages consumed + int32_t timeout = 5000; // poll timeout + + while (running) { + // poll message from TDengine + WS_RES* tmqmsg = ws_tmq_consumer_poll(tmq, timeout); + if (tmqmsg) { + msgCnt++; + // process the message + totalRows += msg_process(tmqmsg); + // commit the message + int32_t code = ws_tmq_commit_sync(tmq, tmqmsg); + if (code) { + fprintf(stderr, + "Failed to commit offset, topic: %s, groupId: %s, clientId: %s, ErrCode: 0x%x, ErrMessage: %s.\n", + topic_name, config.group_id, config.client_id, code, ws_tmq_errstr(tmq)); + // free the message + ws_free_result(tmqmsg); + break; + } else { + fprintf(stdout, "Commit offset manually successfully.\n"); + } + // free the message + ws_free_result(tmqmsg); + } + if (msgCnt > 50) { + // consume 50 messages and break + break; + } + } + + // print the result: total messages and total rows consumed + fprintf(stdout, "%d msg consumed, include %d rows.\n", msgCnt, totalRows); +} +// ANCHOR_END: manual_commit + +int main(int argc, char* argv[]) { + int32_t code; + pthread_t thread_id; + + WS_TAOS* pConn = init_env(); + if (pConn == NULL) { + fprintf(stderr, "Failed to init env.\n"); + return -1; + } + + if (create_topic(pConn) < 0) { + fprintf(stderr, "Failed to create topic.\n"); + return -1; + } + + if (pthread_create(&thread_id, NULL, &prepare_data, NULL)) { + fprintf(stderr, "Failed to create thread.\n"); + return -1; + } + + // ANCHOR: create_consumer_2 + ws_tmq_t* tmq = build_consumer(&config); + if (NULL == tmq) { + fprintf(stderr, "Failed to create native consumer, host: %s, groupId: %s, , clientId: %s.\n", + config.td_connect_host, config.group_id, config.client_id); + return -1; + } else { + fprintf(stdout, "Create consumer successfully, host: %s, groupId: %s, clientId: %s.\n", config.td_connect_host, + config.group_id, config.client_id); + } + + // ANCHOR_END: create_consumer_2 + + // ANCHOR: subscribe_3 + ws_tmq_list_t* topic_list = build_topic_list(); + if (NULL == topic_list) { + fprintf(stderr, "Failed to create topic_list, topic: %s, groupId: %s, clientId: %s.\n", topic_name, config.group_id, + config.client_id); + return -1; + } + + if ((code = ws_tmq_subscribe(tmq, topic_list))) { + fprintf(stderr, + "Failed to subscribe topic_list, topic: %s, groupId: %s, clientId: %s, ErrCode: 0x%x, ErrMessage: %s.\n", + topic_name, config.group_id, config.client_id, code, ws_tmq_errstr(tmq)); + } else { + fprintf(stdout, "Subscribe topics successfully.\n"); + } + + ws_tmq_list_destroy(topic_list); + + basic_consume_loop(tmq); + // ANCHOR_END: subscribe_3 + + consume_repeatly(tmq); + + manual_commit(tmq); + + // ANCHOR: unsubscribe_and_close + // unsubscribe the topic + code = ws_tmq_unsubscribe(tmq); + if (code) { + fprintf(stderr, + "Failed to unsubscribe consumer, topic: %s, groupId: %s, clientId: %s, ErrCode: 0x%x, ErrMessage: %s.\n", + topic_name, config.group_id, config.client_id, code, ws_tmq_errstr(tmq)); + } else { + fprintf(stdout, "Consumer unsubscribed successfully.\n"); + } + + // close the consumer + code = ws_tmq_consumer_close(tmq); + if (code) { + fprintf(stderr, "Failed to close consumer, topic: %s, groupId: %s, clientId: %s, ErrCode: 0x%x, ErrMessage: %s.\n", + topic_name, config.group_id, config.client_id, code, ws_tmq_errstr(tmq)); + } else { + fprintf(stdout, "Consumer closed successfully.\n"); + } + // ANCHOR_END: unsubscribe_and_close + + thread_stop = 1; + pthread_join(thread_id, NULL); + + if (drop_topic(pConn) < 0) { + fprintf(stderr, "Failed to drop topic.\n"); + return -1; + } + + deinit_env(pConn); + return 0; +} diff --git a/docs/examples/c-ws/with_reqid_demo.c b/docs/examples/c-ws/with_reqid_demo.c new file mode 100644 index 0000000000..7dae526a9e --- /dev/null +++ b/docs/examples/c-ws/with_reqid_demo.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o with_reqid_demo with_reqid_demo.c -ltaos + +#include +#include +#include +#include +#include "taosws.h" + +static int DemoWithReqId() { + // ANCHOR: with_reqid + int code = 0; + char *dsn = "ws://localhost:6041"; + + // connect + WS_TAOS *taos = ws_connect(dsn); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to %s, ErrCode: 0x%x, ErrMessage: %s.\n", dsn, ws_errno(NULL), ws_errstr(NULL)); + return -1; + } + + const char *sql = "SELECT ts, current, location FROM power.meters limit 1"; + // query data with reqid + long reqid = 3L; + WS_RES *result = ws_query_with_reqid(taos, sql, reqid); + code = ws_errno(result); + if (code != 0) { + fprintf(stderr, "Failed to execute sql withQID: %ld, ErrCode: 0x%x, ErrMessage: %s\n.", reqid, code, + ws_errstr(result)); + ws_close(taos); + return -1; + } + + WS_ROW row = NULL; + int rows = 0; + int num_fields = ws_field_count(result); + const WS_FIELD *fields = ws_fetch_fields(result); + + fprintf(stdout, "query successfully, got %d fields, the sql is: %s.\n", num_fields, sql); + + // fetch the records row by row + while ((row = ws_fetch_row(result))) { + // Add your data processing logic here + + rows++; + } + fprintf(stdout, "total rows: %d\n", rows); + ws_free_result(result); + + // close & clean + ws_close(taos); + return 0; + // ANCHOR_END: with_reqid +} + +int main(int argc, char *argv[]) { return DemoWithReqId(); } diff --git a/docs/examples/node/package.json b/docs/examples/node/package.json index d00d71d99f..3f5f54e9d7 100644 --- a/docs/examples/node/package.json +++ b/docs/examples/node/package.json @@ -4,7 +4,6 @@ "main": "index.js", "license": "MIT", "dependencies": { - "@tdengine/client": "^3.0.1", - "@tdengine/rest": "^3.0.0" + "@tdengine/websocket": "^3.1.0" } } diff --git a/docs/examples/node/websocketexample/all_type_query.js b/docs/examples/node/websocketexample/all_type_query.js index 266d110d24..6524d35718 100644 --- a/docs/examples/node/websocketexample/all_type_query.js +++ b/docs/examples/node/websocketexample/all_type_query.js @@ -34,6 +34,7 @@ async function json_tag_example() { } catch (err) { console.error(`Failed to create database example_json_tag or stable stb, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (wsSql) { await wsSql.close(); @@ -78,9 +79,10 @@ async function all_type_example() { let row = wsRows.getData(); console.log(row); } - + } catch (err) { console.error(`Failed to create database all_type_example or stable stb, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (wsSql) { await wsSql.close(); @@ -91,7 +93,7 @@ async function all_type_example() { async function test() { await json_tag_example() - await all_type_example() + await all_type_example() taos.destroy(); } diff --git a/docs/examples/node/websocketexample/all_type_stmt.js b/docs/examples/node/websocketexample/all_type_stmt.js index 8a0dcf21e1..f095bee090 100644 --- a/docs/examples/node/websocketexample/all_type_stmt.js +++ b/docs/examples/node/websocketexample/all_type_stmt.js @@ -46,6 +46,7 @@ async function json_tag_example() { } catch (err) { console.error(`Failed to create database example_json_tag or stable stb, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err } finally { if (wsSql) { await wsSql.close(); @@ -125,6 +126,7 @@ async function all_type_example() { } catch (err) { console.error(`Failed to create database all_type_example or stable stb, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (stmt) { await stmt.close(); @@ -136,10 +138,7 @@ async function all_type_example() { } - - -async function test() { - taos.setLevel("debug") +async function test() { await json_tag_example() await all_type_example() taos.destroy(); diff --git a/docs/examples/node/websocketexample/json_line_example.js b/docs/examples/node/websocketexample/json_line_example.js deleted file mode 100644 index e6587eaa45..0000000000 --- a/docs/examples/node/websocketexample/json_line_example.js +++ /dev/null @@ -1,53 +0,0 @@ -const taos = require("@tdengine/websocket"); - -var host = null; -for(var i = 2; i < global.process.argv.length; i++){ - var key = global.process.argv[i].split("=")[0]; - var value = global.process.argv[i].split("=")[1]; - if("host" == key){ - host = value; - } -} - -if(host == null){ - console.log("Usage: node nodejsChecker.js host= port="); - process.exit(0); - } - -let dbData = ["{\"metric\": \"meter_current\",\"timestamp\": 1626846402,\"value\": 10.3, \"tags\": {\"groupid\": 2, \"location\": \"California.SanFrancisco\", \"id\": \"d1001\"}}", - "{\"metric\": \"meter_current\",\"timestamp\": 1626846403,\"value\": 10.3, \"tags\": {\"groupid\": 2, \"location\": \"California.SanFrancisco\", \"id\": \"d1002\"}}", - "{\"metric\": \"meter_current\",\"timestamp\": 1626846404,\"value\": 10.3, \"tags\": {\"groupid\": 2, \"location\": \"California.SanFrancisco\", \"id\": \"d1003\"}}"] - -async function createConnect() { - let dsn = 'ws://' + host + ':6041' - let conf = new taos.WSConfig(dsn); - conf.setUser('root'); - conf.setPwd('taosdata'); - conf.setDb('power'); - return await taos.sqlConnect(conf); -} - -async function test() { - let wsSql = null; - let wsRows = null; - let reqId = 0; - try { - wsSql = await createConnect() - await wsSql.exec('CREATE DATABASE IF NOT EXISTS power KEEP 3650 DURATION 10 BUFFER 16 WAL_LEVEL 1;', reqId++); - await wsSql.schemalessInsert([dbData], taos.SchemalessProto.OpenTSDBJsonFormatProtocol, taos.Precision.SECONDS, 0); - } - catch (err) { - console.error(err.code, err.message); - } - finally { - if (wsRows) { - await wsRows.close(); - } - if (wsSql) { - await wsSql.close(); - } - taos.destroy(); - } -} - -test() \ No newline at end of file diff --git a/docs/examples/node/websocketexample/line_example.js b/docs/examples/node/websocketexample/line_example.js index ac3083d358..97570475b0 100644 --- a/docs/examples/node/websocketexample/line_example.js +++ b/docs/examples/node/websocketexample/line_example.js @@ -15,8 +15,8 @@ async function createConnect() { return wsSql; } + async function test() { - let dsn = 'ws://localhost:6041' let wsSql = null; let wsRows = null; let ttl = 0; @@ -29,6 +29,7 @@ async function test() { } catch (err) { console.error(`Failed to insert data with schemaless, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (wsRows) { @@ -40,4 +41,5 @@ async function test() { taos.destroy(); } } + test() diff --git a/docs/examples/node/websocketexample/nodejsChecker.js b/docs/examples/node/websocketexample/nodejsChecker.js index d81aeb585f..9f1d40f0cc 100644 --- a/docs/examples/node/websocketexample/nodejsChecker.js +++ b/docs/examples/node/websocketexample/nodejsChecker.js @@ -10,11 +10,9 @@ for(var i = 2; i < global.process.argv.length; i++){ } if(host == null){ - console.log("Usage: node nodejsChecker.js host= port="); - process.exit(0); + host = 'localhost'; } - async function createConnect() { let dsn = 'ws://' + host + ':6041' console.log(dsn) @@ -41,7 +39,7 @@ async function test() { taosResult = await wsSql.exec('USE power', reqId++); console.log(taosResult); - taosResult = await wsSql.exec('CREATE STABLE IF NOT EXISTS meters (_ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);', reqId++); + taosResult = await wsSql.exec('CREATE STABLE IF NOT EXISTS meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);', reqId++); console.log(taosResult); taosResult = await wsSql.exec('DESCRIBE meters', reqId++); @@ -62,6 +60,7 @@ async function test() { } catch (err) { console.error(err.code, err.message); + throw err; } finally { if (wsRows) { diff --git a/docs/examples/node/websocketexample/sql_example.js b/docs/examples/node/websocketexample/sql_example.js index 8eb8af989d..0a09228d97 100644 --- a/docs/examples/node/websocketexample/sql_example.js +++ b/docs/examples/node/websocketexample/sql_example.js @@ -41,6 +41,7 @@ async function createDbAndTable() { console.log("Create stable power.meters successfully"); } catch (err) { console.error(`Failed to create database power or stable meters, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (wsSql) { await wsSql.close(); @@ -53,21 +54,23 @@ async function createDbAndTable() { // ANCHOR: insertData async function insertData() { let wsSql = null + let insertQuery = "INSERT INTO " + + "power.d1001 USING power.meters (location, groupId) TAGS('California.SanFrancisco', 2) " + + "VALUES " + + "(NOW + 1a, 10.30000, 219, 0.31000) " + + "(NOW + 2a, 12.60000, 218, 0.33000) " + + "(NOW + 3a, 12.30000, 221, 0.31000) " + + "power.d1002 USING power.meters (location, groupId) TAGS('California.SanFrancisco', 3) " + + "VALUES " + + "(NOW + 1a, 10.30000, 218, 0.25000) "; + try { wsSql = await createConnect(); - let insertQuery = "INSERT INTO " + - "power.d1001 USING power.meters (location, groupId) TAGS('California.SanFrancisco', 2) " + - "VALUES " + - "(NOW + 1a, 10.30000, 219, 0.31000) " + - "(NOW + 2a, 12.60000, 218, 0.33000) " + - "(NOW + 3a, 12.30000, 221, 0.31000) " + - "power.d1002 USING power.meters TAGS('California.SanFrancisco', 3) " + - "VALUES " + - "(NOW + 1a, 10.30000, 218, 0.25000) "; taosResult = await wsSql.exec(insertQuery); console.log("Successfully inserted " + taosResult.getAffectRows() + " rows to power.meters."); } catch (err) { console.error(`Failed to insert data to power.meters, sql: ${insertQuery}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (wsSql) { await wsSql.close(); @@ -91,6 +94,7 @@ async function queryData() { } catch (err) { console.error(`Failed to query data from power.meters, sql: ${sql}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (wsRows) { @@ -118,6 +122,7 @@ async function sqlWithReqid() { } catch (err) { console.error(`Failed to query data from power.meters, reqId: ${reqId}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (wsRows) { @@ -135,7 +140,7 @@ async function test() { await insertData(); await queryData(); await sqlWithReqid(); - taos.destroy(); + taos.destroy(); } test() diff --git a/docs/examples/node/websocketexample/stmt_example.js b/docs/examples/node/websocketexample/stmt_example.js index e3bb3c4dda..6f98bb7974 100644 --- a/docs/examples/node/websocketexample/stmt_example.js +++ b/docs/examples/node/websocketexample/stmt_example.js @@ -23,7 +23,7 @@ async function prepare() { return wsSql } -(async () => { +async function test() { let stmt = null; let connector = null; try { @@ -60,6 +60,7 @@ async function prepare() { } catch (err) { console.error(`Failed to insert to table meters using stmt, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (stmt) { @@ -70,4 +71,6 @@ async function prepare() { } taos.destroy(); } -})(); +} + +test() \ No newline at end of file diff --git a/docs/examples/node/websocketexample/telnet_line_example.js b/docs/examples/node/websocketexample/telnet_line_example.js deleted file mode 100644 index 924137e162..0000000000 --- a/docs/examples/node/websocketexample/telnet_line_example.js +++ /dev/null @@ -1,58 +0,0 @@ -const taos = require("@tdengine/websocket"); - -var host = null; -for(var i = 2; i < global.process.argv.length; i++){ - var key = global.process.argv[i].split("=")[0]; - var value = global.process.argv[i].split("=")[1]; - if("host" == key){ - host = value; - } -} - -if(host == null){ - console.log("Usage: node nodejsChecker.js host= port="); - process.exit(0); - } - - let dbData = ["meters.current 1648432611249 10.3 location=California.SanFrancisco groupid=2", - "meters.current 1648432611250 12.6 location=California.SanFrancisco groupid=2", - "meters.current 1648432611249 10.8 location=California.LosAngeles groupid=3", - "meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3", - "meters.voltage 1648432611249 219 location=California.SanFrancisco groupid=2", - "meters.voltage 1648432611250 218 location=California.SanFrancisco groupid=2", - "meters.voltage 1648432611249 221 location=California.LosAngeles groupid=3", - "meters.voltage 1648432611250 217 location=California.LosAngeles groupid=3",]; - -async function createConnect() { - let dsn = 'ws://' + host + ':6041' - let conf = new taos.WSConfig(dsn); - conf.setUser('root'); - conf.setPwd('taosdata'); - - return await taos.sqlConnect(conf); -} - -async function test() { - let wsSql = null; - let wsRows = null; - let reqId = 0; - try { - wsSql = await createConnect() - await wsSql.exec('create database if not exists power KEEP 3650 DURATION 10 BUFFER 16 WAL_LEVEL 1;', reqId++); - await wsSql.exec('use power', reqId++); - await wsSql.schemalessInsert(dbData, taos.SchemalessProto.OpenTSDBTelnetLineProtocol, taos.Precision.MILLI_SECONDS, 0); - } - catch (err) { - console.error(err.code, err.message); - } - finally { - if (wsRows) { - await wsRows.close(); - } - if (wsSql) { - await wsSql.close(); - } - taos.destroy(); - } -} -test() \ No newline at end of file diff --git a/docs/examples/node/websocketexample/tmq_example.js b/docs/examples/node/websocketexample/tmq_example.js index 5097402e6a..4f7f099c31 100644 --- a/docs/examples/node/websocketexample/tmq_example.js +++ b/docs/examples/node/websocketexample/tmq_example.js @@ -1,3 +1,4 @@ +const { sleep } = require("@tdengine/websocket"); const taos = require("@tdengine/websocket"); // ANCHOR: create_consumer @@ -49,12 +50,20 @@ async function prepare() { let createTopic = `CREATE TOPIC IF NOT EXISTS ${topics[0]} AS SELECT * FROM ${db}.${stable}`; await wsSql.exec(createTopic); + await wsSql.close(); +} - - for (let i = 0; i < 10; i++) { +async function insert() { + let conf = new taos.WSConfig('ws://localhost:6041'); + conf.setUser('root'); + conf.setPwd('taosdata'); + conf.setDb('power'); + let wsSql = await taos.sqlConnect(conf); + for (let i = 0; i < 50; i++) { await wsSql.exec(`INSERT INTO d1001 USING ${stable} (location, groupId) TAGS ("California.SanFrancisco", 3) VALUES (NOW, ${10 + i}, ${200 + i}, ${0.32 + i})`); + await sleep(100); } - wsSql.close(); + await wsSql.close(); } async function subscribe(consumer) { @@ -82,13 +91,17 @@ async function test() { let consumer = null; try { await prepare(); - consumer = await createConsumer() - await subscribe(consumer) + consumer = await createConsumer(); + const allPromises = []; + allPromises.push(subscribe(consumer)); + allPromises.push(insert()); + await Promise.all(allPromises); await consumer.unsubscribe(); console.log("Consumer unsubscribed successfully."); } catch (err) { console.error(`Failed to unsubscribe consumer, topic: ${topic}, groupId: ${groupId}, clientId: ${clientId}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (consumer) { diff --git a/docs/examples/node/websocketexample/tmq_seek_example.js b/docs/examples/node/websocketexample/tmq_seek_example.js index b2bd569d92..f676efe36f 100644 --- a/docs/examples/node/websocketexample/tmq_seek_example.js +++ b/docs/examples/node/websocketexample/tmq_seek_example.js @@ -1,41 +1,45 @@ +const { sleep } = require("@tdengine/websocket"); const taos = require("@tdengine/websocket"); const db = 'power'; const stable = 'meters'; +const url = 'ws://localhost:6041'; const topic = 'topic_meters' const topics = [topic]; const groupId = "group1"; const clientId = "client1"; - -// ANCHOR: create_consumer async function createConsumer() { + + let groupId = "group1"; + let clientId = "client1"; let configMap = new Map([ - [taos.TMQConstants.GROUP_ID, "group1"], - [taos.TMQConstants.CLIENT_ID, 'client1'], + [taos.TMQConstants.GROUP_ID, groupId], + [taos.TMQConstants.CLIENT_ID, clientId], [taos.TMQConstants.CONNECT_USER, "root"], [taos.TMQConstants.CONNECT_PASS, "taosdata"], [taos.TMQConstants.AUTO_OFFSET_RESET, "latest"], - [taos.TMQConstants.WS_URL, 'ws://localhost:6041'], + [taos.TMQConstants.WS_URL, url], [taos.TMQConstants.ENABLE_AUTO_COMMIT, 'true'], [taos.TMQConstants.AUTO_COMMIT_INTERVAL_MS, '1000'] ]); try { - return await taos.tmqConnect(configMap); + conn = await taos.tmqConnect(configMap); + console.log(`Create consumer successfully, host: ${url}, groupId: ${groupId}, clientId: ${clientId}`) + return conn; } catch (err) { - console.error(err); + console.error(`Failed to create websocket consumer, topic: ${topic}, groupId: ${groupId}, clientId: ${clientId}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); throw err; } } -// ANCHOR_END: create_consumer async function prepare() { - let conf = new taos.WSConfig('ws://localhost:6041'); + let conf = new taos.WSConfig('ws://192.168.1.98:6041'); conf.setUser('root'); conf.setPwd('taosdata'); conf.setDb('power'); - const createDB = `CREATE DATABASE IF NOT EXISTS ${db} KEEP 3650 DURATION 10 BUFFER 16 WAL_LEVEL 1;`; + const createDB = `CREATE DATABASE IF NOT EXISTS ${db}`; const createStable = `CREATE STABLE IF NOT EXISTS ${db}.${stable} (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);`; let wsSql = await taos.sqlConnect(conf); @@ -44,58 +48,63 @@ async function prepare() { let createTopic = `CREATE TOPIC IF NOT EXISTS ${topics[0]} AS SELECT * FROM ${db}.${stable}`; await wsSql.exec(createTopic); + await wsSql.close(); +} - - for (let i = 0; i < 10; i++) { +async function insert() { + let conf = new taos.WSConfig('ws://localhost:6041'); + conf.setUser('root'); + conf.setPwd('taosdata'); + conf.setDb('power'); + let wsSql = await taos.sqlConnect(conf); + for (let i = 0; i < 1; i++) { await wsSql.exec(`INSERT INTO d1001 USING ${stable} (location, groupId) TAGS ("California.SanFrancisco", 3) VALUES (NOW, ${10 + i}, ${200 + i}, ${0.32 + i})`); } await wsSql.close(); } -// ANCHOR: subscribe +// ANCHOR: offset async function subscribe(consumer) { try { - await consumer.subscribe(['topic_meters']); - for (let i = 0; i < 50; i++) { - let res = await consumer.poll(100); - for (let [key, value] of res) { - // Add your data processing logic here - console.log(`data: ${key} ${value}`); - } - } - } catch (err) { - console.error(`Failed to poll data, topic: ${topic}, groupId: ${groupId}, clientId: ${clientId}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); - throw err; - } - -} -// ANCHOR_END: subscribe - -// ANCHOR: offset -async function test() { - let consumer = null; - try { - await prepare(); - let consumer = await createConsumer() await consumer.subscribe(['topic_meters']); let res = new Map(); while (res.size == 0) { res = await consumer.poll(100); + await consumer.commit(); } - let assignment = await consumer.assignment(); await consumer.seekToBeginning(assignment); console.log("Assignment seek to beginning successfully"); + } catch (err) { + console.error(`Failed to seek offset, topic: ${topic}, groupId: ${groupId}, clientId: ${clientId}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; + } +} +// ANCHOR_END: offset + +async function test() { + let consumer = null; + try { + await prepare(); + consumer = await createConsumer(); + const allPromises = []; + allPromises.push(subscribe(consumer)); + allPromises.push(insert()); + await Promise.all(allPromises); + await consumer.unsubscribe(); + console.log("Consumer unsubscribed successfully."); } catch (err) { - console.error(`Failed to seek offset, topic: ${topic}, groupId: ${groupId}, clientId: ${clientId}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + console.error(`Failed to consumer, topic: ${topic}, groupId: ${groupId}, clientId: ${clientId}, ErrCode: ${err.code}, ErrMessage: ${err.message}`); + throw err; } finally { if (consumer) { await consumer.close(); + console.log("Consumer closed successfully."); } taos.destroy(); } } -// ANCHOR_END: offset + test() diff --git a/docs/examples/python/connect_example.py b/docs/examples/python/connect_example.py index ce8b306024..e1f1a63d57 100644 --- a/docs/examples/python/connect_example.py +++ b/docs/examples/python/connect_example.py @@ -15,6 +15,7 @@ def create_connection(): print(f"Connected to {host}:{port} successfully."); except Exception as err: print(f"Failed to connect to {host}:{port} , ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/connect_websocket_examples.py b/docs/examples/python/connect_websocket_examples.py index 56d208f5db..fe8f407865 100644 --- a/docs/examples/python/connect_websocket_examples.py +++ b/docs/examples/python/connect_websocket_examples.py @@ -15,7 +15,7 @@ def create_connection(): print(f"Connected to {host}:{port} successfully."); except Exception as err: print(f"Failed to connect to {host}:{port} , ErrMessage:{err}") - + raise err return conn # ANCHOR_END: connect @@ -28,6 +28,7 @@ def create_db_table(conn): conn.execute("CREATE TABLE IF NOT EXISTS `d0` USING `meters` (groupId, location) TAGS(0, 'Los Angles')") except Exception as err: print(f'Exception {err}') + raise err # ANCHOR_END: create_db def insert(conn): @@ -42,9 +43,10 @@ def insert(conn): """ try: inserted = conn.execute(sql) - assert inserted == 8 + assert inserted == 4 except Exception as err: print(f'Exception111 {err}') + raise err # ANCHOR_END: insert def query(conn): @@ -58,6 +60,7 @@ def query(conn): print(row) except Exception as err: print(f'Exception {err}') + raise err # ANCHOR_END: query if __name__ == "__main__": diff --git a/docs/examples/python/create_db_native.py b/docs/examples/python/create_db_native.py index 34dabfabe2..cf263b7b27 100644 --- a/docs/examples/python/create_db_native.py +++ b/docs/examples/python/create_db_native.py @@ -21,6 +21,7 @@ try: except Exception as err: print(f"Failed to create database power or stable meters, ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/create_db_rest.py b/docs/examples/python/create_db_rest.py index 4b98c991a3..4a247dfaec 100644 --- a/docs/examples/python/create_db_rest.py +++ b/docs/examples/python/create_db_rest.py @@ -20,6 +20,7 @@ try: except Exception as err: print(f"Failed to create database power or stable meters, ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/create_db_ws.py b/docs/examples/python/create_db_ws.py index ddbacb4b1f..29ee95dc65 100644 --- a/docs/examples/python/create_db_ws.py +++ b/docs/examples/python/create_db_ws.py @@ -21,6 +21,7 @@ try: except Exception as err: print(f"Failed to create database power or stable meters, ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/insert_native.py b/docs/examples/python/insert_native.py index 19dafa3f23..b49860dfb1 100644 --- a/docs/examples/python/insert_native.py +++ b/docs/examples/python/insert_native.py @@ -22,6 +22,7 @@ try: except Exception as err: print(f"Failed to insert data to power.meters, sql: {sql}, ErrMessage: {err}.") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/insert_rest.py b/docs/examples/python/insert_rest.py index 526c3a6a69..115ec1a702 100644 --- a/docs/examples/python/insert_rest.py +++ b/docs/examples/python/insert_rest.py @@ -21,6 +21,7 @@ try: except Exception as err: print(f"Failed to insert data to power.meters, sql:{sql}, ErrMessage:{err}.") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/insert_ws.py b/docs/examples/python/insert_ws.py index 886dda1c10..9fec00e02b 100644 --- a/docs/examples/python/insert_ws.py +++ b/docs/examples/python/insert_ws.py @@ -22,6 +22,7 @@ try: except Exception as err: print(f"Failed to insert data to power.meters, sql: {sql}, ErrMessage: {err}.") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/query_native.py b/docs/examples/python/query_native.py index 072807986e..7c2015fe04 100644 --- a/docs/examples/python/query_native.py +++ b/docs/examples/python/query_native.py @@ -16,6 +16,7 @@ try: except Exception as err: print(f"Failed to query data from power.meters, sql: {sql}, ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/query_rest.py b/docs/examples/python/query_rest.py index 85a70fd382..b864a4ccd6 100644 --- a/docs/examples/python/query_rest.py +++ b/docs/examples/python/query_rest.py @@ -15,3 +15,4 @@ try: except Exception as err: print(f"Failed to query data from power.meters, sql: {sql}, ErrMessage:{err}") + raise err diff --git a/docs/examples/python/query_ws.py b/docs/examples/python/query_ws.py index afab438ad9..52484b5308 100644 --- a/docs/examples/python/query_ws.py +++ b/docs/examples/python/query_ws.py @@ -15,6 +15,7 @@ try: except Exception as err: print(f"Failed to query data from power.meters, sql: {sql}, ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/reqid_native.py b/docs/examples/python/reqid_native.py index 7f16093835..57f438d3f2 100644 --- a/docs/examples/python/reqid_native.py +++ b/docs/examples/python/reqid_native.py @@ -18,7 +18,7 @@ try: except Exception as err: print(f"Failed to execute sql with reqId:{reqId}, ErrMessage:{err}") - + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/reqid_rest.py b/docs/examples/python/reqid_rest.py index 570e671092..a5f752113b 100644 --- a/docs/examples/python/reqid_rest.py +++ b/docs/examples/python/reqid_rest.py @@ -16,3 +16,4 @@ try: except Exception as err: print(f"Failed to execute sql with reqId:{reqId}, ErrMessage:{err}") + raise err diff --git a/docs/examples/python/reqid_ws.py b/docs/examples/python/reqid_ws.py index 7c74104169..217ff0b54b 100644 --- a/docs/examples/python/reqid_ws.py +++ b/docs/examples/python/reqid_ws.py @@ -19,6 +19,7 @@ try: except Exception as err: print(f"Failed to execute sql with reqId:{reqId}, ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/schemaless_native.py b/docs/examples/python/schemaless_native.py index 96d8f3177f..9654ee02dd 100644 --- a/docs/examples/python/schemaless_native.py +++ b/docs/examples/python/schemaless_native.py @@ -35,6 +35,7 @@ try: print("Inserted data with schemaless successfully."); except Exception as err: print(f"Failed to insert data with schemaless, ErrMessage:{err}") + raise err finally: if conn: conn.close() diff --git a/docs/examples/python/schemaless_ws.py b/docs/examples/python/schemaless_ws.py index 39de55393d..3033e9b670 100644 --- a/docs/examples/python/schemaless_ws.py +++ b/docs/examples/python/schemaless_ws.py @@ -75,8 +75,6 @@ def schemaless_insert(): conn.close() if __name__ == "__main__": - try: - prepare() - schemaless_insert() - except Exception as err: - print(f"Failed to insert data with schemaless, err:{err}") + prepare() + schemaless_insert() + diff --git a/docs/examples/python/stmt_native.py b/docs/examples/python/stmt_native.py index a1af7d1dd7..3e4475f1f7 100644 --- a/docs/examples/python/stmt_native.py +++ b/docs/examples/python/stmt_native.py @@ -57,6 +57,7 @@ try: except Exception as err: print(f"Failed to insert to table meters using stmt, ErrMessage:{err}") + raise err finally: if stmt: stmt.close() diff --git a/docs/examples/python/stmt_ws.py b/docs/examples/python/stmt_ws.py index 45d9222315..82686abda4 100644 --- a/docs/examples/python/stmt_ws.py +++ b/docs/examples/python/stmt_ws.py @@ -62,6 +62,7 @@ try: except Exception as err: print(f"Failed to insert to table meters using stmt, ErrMessage:{err}") + raise err finally: if stmt: stmt.close() diff --git a/docs/examples/python/tmq_native.py b/docs/examples/python/tmq_native.py index d4ccfda138..34bac26d95 100644 --- a/docs/examples/python/tmq_native.py +++ b/docs/examples/python/tmq_native.py @@ -152,6 +152,7 @@ def unsubscribe(consumer): print("Consumer unsubscribed successfully."); except Exception as err: print(f"Failed to unsubscribe consumer. topic: {topic}, groupId: {groupId}, clientId: {clientId}, ErrMessage:{err}.") + raise err finally: if consumer: consumer.close() @@ -166,7 +167,6 @@ if __name__ == "__main__": subscribe(consumer) seek_offset(consumer) commit_offset(consumer) - except Exception as err: - print(f"Failed to execute consumer example, topic: {topic}, groupId: {groupId}, clientId: {clientId}, ErrMessage:{err}.") finally: - unsubscribe(consumer); + if consumer: + unsubscribe(consumer); diff --git a/docs/examples/python/tmq_websocket_example.py b/docs/examples/python/tmq_websocket_example.py index c9c7924661..7bd3f57da6 100644 --- a/docs/examples/python/tmq_websocket_example.py +++ b/docs/examples/python/tmq_websocket_example.py @@ -31,7 +31,7 @@ def prepareMeta(): # create super table rowsAffected = conn.execute( - "CREATE TABLE IF NOT EXISTS `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` BINARY(16))" + "CREATE TABLE IF NOT EXISTS `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` BINARY(64))" ) assert rowsAffected == 0 @@ -155,6 +155,7 @@ def unsubscribe(consumer): print("Consumer unsubscribed successfully."); except Exception as err: print(f"Failed to unsubscribe consumer. topic: {topic}, groupId: {groupId}, clientId: {clientId}, ErrMessage:{err}.") + raise err finally: if consumer: consumer.close() @@ -170,7 +171,6 @@ if __name__ == "__main__": subscribe(consumer) seek_offset(consumer) commit_offset(consumer) - except Exception as err: - print(f"Failed to execute consumer example, topic: {topic}, groupId: {groupId}, clientId: {clientId}, ErrMessage:{err}.") finally: - unsubscribe(consumer) + if consumer: + unsubscribe(consumer) diff --git a/docs/zh/06-advanced/05-data-in/08-kafka.md b/docs/zh/06-advanced/05-data-in/08-kafka.md index e05c205f6e..837aa8d8fb 100644 --- a/docs/zh/06-advanced/05-data-in/08-kafka.md +++ b/docs/zh/06-advanced/05-data-in/08-kafka.md @@ -44,8 +44,50 @@ TDengine 可以高效地从 Kafka 读取数据并将其写入 TDengine，以实 如果服务端开启了 SASL 认证机制，此处需要启用 SASL 并配置相关内容，目前支持 PLAIN/SCRAM-SHA-256/GSSAPI 三种认证机制，请按实际情况进行选择。 +#### 4.1. PLAIN 认证 + +选择 `PLAIN` 认证机制，输入用户名和密码： + +![kafka-04-sasl-plain.png](./kafka-04-sasl-plain.png) + +#### 4.1. SCRAM(SCRAM-SHA-256) 认证 + +选择 `SCRAM-SHA-256` 认证机制，输入用户名和密码： + ![kafka-04.png](./kafka-04.png) +#### 4.3. GSSAPI 认证 + +选择 `GSSAPI` ，将通过 [RDkafka 客户端](https://github.com/confluentinc/librdkafka) 调用 GSSAPI 应用 Kerberos 认证机制： + +![kafka-04-sasl-gssapi.png](./kafka-04-sasl-gssapi.png) + +需要输入的信息有： + +- Kerberos 服务名，一般是 `kafka`； +- Kerberos 认证主体，即认证用户名，例如 `kafkaclient`； +- Kerberos 初始化命令（可选，一般不用填写）； +- Kerberos 密钥表，需提供文件并上传； + +以上信息均需由 Kafka 服务管理者提供。 + +除此之外，在服务器上需要配置 [Kerberos](https://web.mit.edu/kerberos/) 认证服务。在 Ubuntu 下使用 `apt install krb5-user` ；在 CentOS 下，使用 `yum install krb5-workstation`；即可。 + +配置完成后，可以使用 [kcat](https://github.com/edenhill/kcat) 工具进行 Kafka 主题消费验证： + +```bash +kcat \ + -b \ + -G kcat \ + -X security.protocol=SASL_PLAINTEXT \ + -X sasl.mechanism=GSSAPI \ + -X sasl.kerberos.keytab= \ + -X sasl.kerberos.principal= \ + -X sasl.kerberos.service.name=kafka +``` + +如果出现错误：“Server xxxx not found in kerberos database”，则需要配置 Kafka 节点对应的域名并在 Kerberos 客户端配置文件 `/etc/krb5.conf` 中配置反向域名解析 `rdns = true`。 + ### 5. 配置 SSL 证书 如果服务端开启了 SSL 加密认证,此处需要启用 SSL 并配置相关内容。 @@ -160,4 +202,4 @@ json 数据支持 JSONObject 或者 JSONArray，使用 json 解析器可以解 ### 9. 创建完成 -点击 **提交** 按钮，完成创建 Kafka 到 TDengine 的数据同步任务，回到**数据源列表**页面可查看任务执行情况。 \ No newline at end of file +点击 **提交** 按钮，完成创建 Kafka 到 TDengine 的数据同步任务，回到**数据源列表**页面可查看任务执行情况。 diff --git a/docs/zh/06-advanced/05-data-in/kafka-04-sasl-gssapi.png b/docs/zh/06-advanced/05-data-in/kafka-04-sasl-gssapi.png new file mode 100644 index 0000000000..051efda8ab Binary files /dev/null and b/docs/zh/06-advanced/05-data-in/kafka-04-sasl-gssapi.png differ diff --git a/docs/zh/06-advanced/05-data-in/kafka-04-sasl-plain.png b/docs/zh/06-advanced/05-data-in/kafka-04-sasl-plain.png new file mode 100644 index 0000000000..3c11dcf7e1 Binary files /dev/null and b/docs/zh/06-advanced/05-data-in/kafka-04-sasl-plain.png differ diff --git a/docs/zh/07-develop/01-connect/index.md b/docs/zh/07-develop/01-connect/index.md index d15f481b05..1dfb95d169 100644 --- a/docs/zh/07-develop/01-connect/index.md +++ b/docs/zh/07-develop/01-connect/index.md @@ -387,7 +387,19 @@ DSN 的详细说明和如何使用详见 [连接功能](../../reference/connecto - `reconnectIntervalMs`：重连间隔毫秒时间，默认为 2000。 -C/C++ 语言连接器使用 `taos_connect()` 函数用于建立与 TDengine 数据库的连接。其参数详细说明如下： +**Websocket 连接** +C/C++ 语言连接器 Websocket 连接方式使用 `ws_connect()` 函数用于建立与 TDengine 数据库的连接。其参数为 DSN 描述字符串，其基本结构如下： + +```text +[+]://[[:@]:][/][?=[&=]] +|------|------------|---|-----------|-----------|------|------|------------|-----------------------| +|driver| protocol | | username | password | host | port | database | params | +``` + +DSN 的详细说明和如何使用详见 [连接功能](../../reference/connector/cpp/#dsn) + +**原生连接** +C/C++ 语言连接器原生连接方式使用 `taos_connect()` 函数用于建立与 TDengine 数据库的连接。其参数详细说明如下： - `host`：要连接的数据库服务器的主机名或IP地址。如果是本地数据库，可以使用 `"localhost"`。 - `user`：用于登录数据库的用户名。 @@ -440,7 +452,10 @@ C/C++ 语言连接器使用 `taos_connect()` 函数用于建立与 TDengine 数 ``` -不支持 +```c +{{#include docs/examples/c-ws/connect_example.c}} +``` + 不支持 diff --git a/docs/zh/07-develop/02-sql.md b/docs/zh/07-develop/02-sql.md index be44458c5b..5461c975dd 100644 --- a/docs/zh/07-develop/02-sql.md +++ b/docs/zh/07-develop/02-sql.md @@ -68,9 +68,15 @@ REST API：直接调用 `taosadapter` 提供的 REST API 接口，进行数据 ``` -```c + +```c title="Websocket 连接" +{{#include docs/examples/c-ws/create_db_demo.c:create_db_and_table}} +``` + +```c title="原生连接" {{#include docs/examples/c/create_db_demo.c:create_db_and_table}} ``` + @@ -144,7 +150,12 @@ NOW 为系统内部函数，默认为客户端所在计算机当前时间。 NOW ``` -```c + +```c title="Websocket 连接" +{{#include docs/examples/c-ws/insert_data_demo.c:insert_data}} +``` + +```c title="原生连接" {{#include docs/examples/c/insert_data_demo.c:insert_data}} ``` @@ -218,7 +229,12 @@ rust 连接器还支持使用 **serde** 进行反序列化行为结构体的结 ``` -```c + +```c title="Websocket 连接" +{{#include docs/examples/c-ws/query_data_demo.c:query_data}} +``` + +```c title="原生连接" {{#include docs/examples/c/query_data_demo.c:query_data}} ``` @@ -293,9 +309,15 @@ reqId 可用于请求链路追踪，reqId 就像分布式系统中的 traceId ``` -```c + +```c "Websocket 连接" +{{#include docs/examples/c-ws/with_reqid_demo.c:with_reqid}} +``` + +```c "原生连接" {{#include docs/examples/c/with_reqid_demo.c:with_reqid}} ``` + diff --git a/docs/zh/07-develop/04-schemaless.md b/docs/zh/07-develop/04-schemaless.md index 17a377950c..a865b58b28 100644 --- a/docs/zh/07-develop/04-schemaless.md +++ b/docs/zh/07-develop/04-schemaless.md @@ -237,7 +237,10 @@ writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO ``` -不支持 + +```c +{{#include docs/examples/c-ws/sml_insert_demo.c:schemaless}} +``` 不支持 diff --git a/docs/zh/07-develop/05-stmt.md b/docs/zh/07-develop/05-stmt.md index e659177c94..624600ba4d 100644 --- a/docs/zh/07-develop/05-stmt.md +++ b/docs/zh/07-develop/05-stmt.md @@ -64,7 +64,9 @@ import TabItem from "@theme/TabItem"; ``` -不支持 +```c +{{#include docs/examples/c-ws/stmt_insert_demo.c}} +``` 不支持 diff --git a/docs/zh/07-develop/07-tmq.md b/docs/zh/07-develop/07-tmq.md index 96f34c6d5d..c668203259 100644 --- a/docs/zh/07-develop/07-tmq.md +++ b/docs/zh/07-develop/07-tmq.md @@ -28,21 +28,21 @@ TDengine 消费者的概念跟 Kafka 类似，消费者通过订阅主题来接 ### 创建参数 创建消费者的参数较多，非常灵活的支持了各种连接类型、 Offset 提交方式、压缩、重连、反序列化等特性。各语言连接器都适用的通用基础配置项如下表所示： -| 参数名称 | 类型 | 参数说明 | 备注 | -| :-----------------------: | :-----: | ----------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `td.connect.ip` | string | 服务端的 IP 地址 | | -| `td.connect.user` | string | 用户名 | | -| `td.connect.pass` | string | 密码 | | -| `td.connect.port` | integer | 服务端的端口号 | | -| `group.id` | string | 消费组 ID，同一消费组共享消费进度 |
**必填项**。最大长度：192。
每个topic最多可建立 100 个 consumer group | -| `client.id` | string | 客户端 ID | 最大长度：192 | -| `auto.offset.reset` | enum | 消费组订阅的初始位置 |
`earliest`: default(version < 3.2.0.0);从头开始订阅;
`latest`: default(version >= 3.2.0.0);仅从最新数据开始订阅;
`none`: 没有提交的 offset 无法订阅 | -| `enable.auto.commit` | boolean | 是否启用消费位点自动提交，true: 自动提交，客户端应用无需commit；false：客户端应用需要自行commit | 默认值为 true | -| `auto.commit.interval.ms` | integer | 消费记录自动提交消费位点时间间隔，单位为毫秒 | 默认值为 5000 | -| `msg.with.table.name` | boolean | 是否允许从消息中解析表名, 不适用于列订阅（列订阅时可将 tbname 作为列写入 subquery 语句）（从3.2.0.0版本该参数废弃，恒为true） | 默认关闭 | -| `enable.replay` | boolean | 是否开启数据回放功能 | 默认关闭 | -| `session.timeout.ms` | integer | consumer 心跳丢失后超时时间，超时后会触发 rebalance 逻辑，成功后该 consumer 会被删除（从3.3.3.0版本开始支持） | 默认值为 12000，取值范围 [6000， 1800000] | -| `max.poll.interval.ms` | integer | consumer poll 拉取数据间隔的最长时间，超过该时间，会认为该 consumer 离线，触发rebalance 逻辑，成功后该 consumer 会被删除（从3.3.3.0版本开始支持） | 默认值为 300000，[1000，INT32_MAX] | +| 参数名称 | 类型 | 参数说明 | 备注 | +| :-----------------------: | :-----: | ------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `td.connect.ip` | string | 服务端的 IP 地址 | | +| `td.connect.user` | string | 用户名 | | +| `td.connect.pass` | string | 密码 | | +| `td.connect.port` | integer | 服务端的端口号 | | +| `group.id` | string | 消费组 ID，同一消费组共享消费进度 |
**必填项**。最大长度：192。
每个topic最多可建立 100 个 consumer group | +| `client.id` | string | 客户端 ID | 最大长度：192 | +| `auto.offset.reset` | enum | 消费组订阅的初始位置 |
`earliest`: default(version < 3.2.0.0);从头开始订阅;
`latest`: default(version >= 3.2.0.0);仅从最新数据开始订阅;
`none`: 没有提交的 offset 无法订阅 | +| `enable.auto.commit` | boolean | 是否启用消费位点自动提交，true: 自动提交，客户端应用无需commit；false：客户端应用需要自行commit | 默认值为 true | +| `auto.commit.interval.ms` | integer | 消费记录自动提交消费位点时间间隔，单位为毫秒 | 默认值为 5000 | +| `msg.with.table.name` | boolean | 是否允许从消息中解析表名, 不适用于列订阅（列订阅时可将 tbname 作为列写入 subquery 语句）（从3.2.0.0版本该参数废弃，恒为true） | 默认关闭 | +| `enable.replay` | boolean | 是否开启数据回放功能 | 默认关闭 | +| `session.timeout.ms` | integer | consumer 心跳丢失后超时时间，超时后会触发 rebalance 逻辑，成功后该 consumer 会被删除（从3.3.3.0版本开始支持） | 默认值为 12000，取值范围 [6000， 1800000] | +| `max.poll.interval.ms` | integer | consumer poll 拉取数据间隔的最长时间，超过该时间，会认为该 consumer 离线，触发rebalance 逻辑，成功后该 consumer 会被删除（从3.3.3.0版本开始支持） | 默认值为 300000，[1000，INT32_MAX] | 下面是各语言连接器创建参数： @@ -94,8 +94,8 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 - -同通用基础配置项。 +- Websocket 连接: 因为使用 dsn，不需要 `td.connect.ip`，`td.connect.port`，`td.connect.user` 和 `td.connect.pass` 四个配置项，其余同通用配置项。 +- 原生连接: 同通用基础配置项。 @@ -154,7 +154,15 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 ``` -不支持 +```c +{{#include docs/examples/c-ws/tmq_demo.c:create_consumer_1}} +``` + +```c +{{#include docs/examples/c-ws/tmq_demo.c:create_consumer_2}} +``` + +调用 `build_consumer` 函数尝试获取消费者实例 `tmq`。成功则打印成功日志，失败则打印失败日志。 不支持 @@ -283,7 +291,29 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 ``` -不支持 +```c +{{#include docs/examples/c-ws/tmq_demo.c:build_topic_list}} +``` + +```c +{{#include docs/examples/c-ws/tmq_demo.c:basic_consume_loop}} +``` + +```c +{{#include docs/examples/c-ws/tmq_demo.c:msg_process}} +``` + +```c +{{#include docs/examples/c-ws/tmq_demo.c:subscribe_3}} +``` + +订阅消费数据步骤： + 1. 调用 `ws_build_topic_list` 函数创建一个主题列表 `topic_list`。 + 2. 如果 `topic_list` 为 `NULL`，表示创建失败，函数返回 `-1`。 + 3. 使用 `ws_tmq_subscribe` 函数订阅 `tmq` 指定的主题列表。如果订阅失败，打印错误信息。 + 4. 销毁主题列表 `topic_list` 以释放资源。 + 5. 调用 `basic_consume_loop` 函数开始基本的消费循环，处理订阅的消息。 + 不支持 @@ -427,7 +457,17 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 ``` -不支持 +```c +{{#include docs/examples/c-ws/tmq_demo.c:consume_repeatly}} +``` + +1. 通过 `ws_tmq_get_topic_assignment` 函数获取特定主题的分配信息，包括分配的数量和具体分配详情。 +2. 如果获取分配信息失败，则打印错误信息并返回。 +3. 对于每个分配，使用 `ws_tmq_offset_seek` 函数将消费者的偏移量设置到最早的偏移量。 +4. 如果设置偏移量失败，则打印错误信息。 +5. 释放分配信息数组以释放资源。 +6. 调用 `basic_consume_loop` 函数开始新的消费循环，处理消息。 + 不支持 @@ -554,7 +594,12 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 ``` -不支持 +```c +{{#include docs/examples/c-ws/tmq_demo.c:manual_commit}} +``` + +可以通过 `ws_tmq_commit_sync` 函数来手工提交消费进度。 + 不支持 @@ -662,7 +707,9 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 ``` -不支持 +```c +{{#include docs/examples/c-ws/tmq_demo.c:unsubscribe_and_close}} +``` 不支持 @@ -777,7 +824,13 @@ Rust 连接器创建消费者的参数为 DSN， 可以设置的参数列表请 -不支持 +
+完整代码示例 +```c +{{#include docs/examples/c-ws/tmq_demo.c}} +``` +
+ 不支持 diff --git a/docs/zh/14-reference/05-connector/10-cpp.mdx b/docs/zh/14-reference/05-connector/10-cpp.mdx index 7c0da088a6..0df6ed924c 100644 --- a/docs/zh/14-reference/05-connector/10-cpp.mdx +++ b/docs/zh/14-reference/05-connector/10-cpp.mdx @@ -4,8 +4,615 @@ title: C/C++ Connector toc_max_heading_level: 4 --- -C/C++ 开发人员可以使用 TDengine 的客户端驱动，即 C/C++连接器 （以下都用 TDengine 客户端驱动表示），开发自己的应用来连接 TDengine 集群完成数据存储、查询以及其他功能。TDengine 客户端驱动的 API 类似于 MySQL 的 C API。应用程序使用时，需要包含 TDengine 头文件 _taos.h_，里面列出了提供的 API 的函数原型；应用程序还要链接到所在平台上对应的动态库。 +C/C++ 开发人员可以使用 TDengine 的客户端驱动，即 C/C++连接器 （以下都用 TDengine 客户端驱动表示），开发自己的应用来连接 TDengine 集群完成数据存储、查询以及其他功能。TDengine 客户端驱动的 API 类似于 MySQL 的 C API。应用程序使用时，需要包含 TDengine 头文件，里面列出了提供的 API 的函数原型；应用程序还要链接到所在平台上对应的动态库。 +TDengine 的客户端驱动提供了 taosws 和 taos 两个动态库，分别支持 Websocket 连接和原生连接。 Websocket 连接和原生连接的区别是 Websocket 连接方式不要求客户端和服务端版本完全匹配，而原生连接要求，在性能上 Websocket 连接方式也接近于原生连接，一般我们推荐使用 Websocket 连接方式。 +下面我们分开介绍两种连接方式的使用方法。 + + +## Websocket 连接方式 + +Websocket 连接方式需要使用 taosws.h 头文件和 taosws 动态库。 + +```c +#include +``` + +TDengine 服务端或客户端安装后，`taosws.h` 位于： + +- Linux：`/usr/local/taos/include` +- Windows：`C:\TDengine\include` +- macOS：`/usr/local/include` + +TDengine 客户端驱动的动态库位于： + +- Linux: `/usr/local/taos/driver/libtaosws.so` +- Windows: `C:\TDengine\taosws.dll` +- macOS: `/usr/local/lib/libtaosws.dylib` + +### 支持的平台 + +请参考[支持的平台列表](../#支持的平台) + +### 版本历史 + +| TDengine 客户端版本 | 主要变化 | TDengine 版本 | +| :------------------: | :---------------------------: | :----------------: | +| 3.3.3.0 | 首次发布，提供了 SQL执行，参数绑定，无模式写入和数据订阅等全面功能支持。 | 3.3.2.0及更高版本 | + + +### 错误码 + +在 C 接口的设计中，错误码采用整数类型表示，每个错误码都对应一个特定的错误状态。如未特别说明，当 API 的返回值是整数时，_0_ 代表成功，其它是代表失败原因的错误码，当返回值是指针时， _NULL_ 表示失败。 +Websocket 连接方式单独的错误码在 `taosws.h` 中， + + +| 错误码 | 错误描述 | 可能的出错场景或者可能的原因 | 建议用户采取的措施 | +| ------- | -------- | ---------------------------- | ------------------ | +| 0xE000 | DSN 错误 | DSN 不符合规范 | 检查 dsn 字符串是否符合规范 | +| 0xE001 | 内部错误 | 不确定 | 保留现场和日志，github上报issue | +| 0xE002 | 连接关闭 | 网络断开 | 请检查网络状况，查看 `taosadapter` 日志。 | +| 0xE003 | 发送超时 | 网络断开 | 请检查网络状况 | +| 0xE004 | 接收超时 | 慢查询，或者网络断开 | 排查 `taosadapter` 日志 | + +其余错误码请参考同目录下 `taoserror.h` 文件，详细的原生连接错误码说明参考：[错误码](../../../reference/error-code)。 +:::info +WebSocket 连接方式错误码只保留了原生连接错误码的后两个字节。 +::: + +### 示例程序 + +本节展示了使用客户端驱动访问 TDengine 集群的常见访问方式的示例代码。 + +- 同步查询示例：[同步查询](https://github.com/taosdata/TDengine/tree/main/docs/examples/c-ws/query_data_demo.c) + +- 参数绑定示例：[参数绑定](https://github.com/taosdata/TDengine/tree/main/docs/examples/c-ws/stmt_insert_demo.c) + +- 无模式写入示例：[无模式写入](https://github.com/taosdata/TDengine/tree/main/docs/examples/c-ws/sml_insert_demo.c) + +- 订阅和消费示例：[订阅和消费](https://github.com/taosdata/TDengine/tree/main/docs/examples/c-ws/tmq_demo.c) + +:::info +更多示例代码及下载请见 [GitHub](https://github.com/taosdata/TDengine/tree/main/docs/examples/c-ws)。 +::: + +### API 参考 + +以下分别介绍 TDengine 客户端驱动的 DSN、基础 API、同步查询 API、参数绑定 API、无模式写入 API 和 数据订阅订阅 API。 + +#### DSN + +C/C++ Websocket 连接器通过 DSN 连接描述字符串来表示连接信息。 +DSN 描述字符串基本结构如下： + +```text +[+]://[[:@]:][/][?=[&=]] +|------|------------|---|-----------|-----------|------|------|------------|-----------------------| +|driver| protocol | | username | password | host | port | database | params | +``` + +各部分意义见下表： + +- **driver**: 必须指定驱动名以便连接器选择何种方式创建连接，支持如下驱动名： + - **taos**: 默认驱动，支持 SQL 执行，参数绑定，无模式写入。 + - **tmq**: 使用 TMQ 订阅数据。 +- **protocol**: 显示指定以何种方式建立连接，例如：`taos+ws://localhost:6041` 指定以 Websocket 方式建立连接。 + - **http/ws**: 使用 Websocket 协议。 + - **https/wss**: 在 Websocket 连接方式下显示启用 SSL/TLS 协议。 + +- **username/password**: 用于创建连接的用户名及密码。 +- **host/port**: 指定创建连接的服务器及端口，当不指定服务器地址及端口时 Websocket 连接默认为 `localhost:6041` 。 +- **database**: 指定默认连接的数据库名，可选参数。 +- **params**：其他可选参数。 + +一个完整的 DSN 描述字符串示例如下：`taos+ws://localhost:6041/test`， 表示使用 Websocket（`ws`）方式通过 `6041` 端口连接服务器 `localhost`，并指定默认数据库为 `test`。 + +#### 基础 API + +基础 API 用于完成创建数据库连接等工作，为其它 API 的执行提供运行时环境。 + +- `char *ws_get_client_info()` + - **接口说明**：获取客户端版本信息。 + - **返回值**：返回客户端版本信息。 + +- `WS_TAOS *ws_connect(const char *dsn)` + - **接口说明**：创建数据库连接，初始化连接上下文。 + - **参数说明**： + - dsn：[入参] 连接信息，见上文 DSN 章节。 + - **返回值**：返回数据库连接，返回值为空表示失败。应用程序需要保存返回的参数，以便后续使用。 + :::info + 同一进程可以根据不同的 dsn 连接多个 TDengine 集群 + ::: + +- `const char *ws_get_server_info(WS_TAOS *taos)` + - **接口说明**：获取服务端版本信息。 + - **参数说明**： + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - **返回值**：返回获取服务端版本信息。 + +- `int32_t ws_select_db(WS_TAOS *taos, const char *db)` + - **接口说明**：将当前的缺省数据库设置为 `db`。 + - **参数说明**： + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - db：[入参] 数据库名称。 + - **返回值**：`0`：成功，非 `0`：失败，详情请参考错误码页面。 + +- `int32_t ws_get_current_db(WS_TAOS *taos, char *database, int len, int *required)` + - **接口说明**：获取当前数据库名称。 + - **参数说明**： + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - database：[出参] 存储当前数据库名称。 + - len：[入参] database 的空间大小。 + - required：[出参] 存储当前数据库名称所需的空间（包含最后的'\0'）。 + - **返回值**：`0`：成功，`-1`：失败，可调用函数 ws_errstr(NULL) 获取更详细的错误信息。 + - 如果，database == NULL 或者 len\<=0 返回失败。 + - 如果，len 小于 存储数据库名称所需的空间（包含最后的'\0'），返回失败，database 里赋值截断的数据，以'\0'结尾。 + - 如果，len 大于等于 存储数据库名称所需的空间（包含最后的'\0'），返回成功，database 里赋值以'\0‘结尾数据库名称。 + +- `int32_t ws_close(WS_TAOS *taos);` + - **接口说明**：关闭连接。 + - **参数说明**： + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - **返回值**：`0`：成功，非 `0`：失败，详情请参考错误码页面。 + +#### 同步查询 + +本小节介绍 API 均属于同步接口。应用调用后，会阻塞等待响应，直到获得返回结果或错误信息。 + +- `WS_RES *ws_query(WS_TAOS *taos, const char *sql)` + - **接口说明**：执行 SQL 语句，可以是 DQL、DML 或 DDL 语句。 + - **参数说明**： + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - sql：[入参] 需要执行 SQL 语句。 + - **返回值**：不能通过返回值是否是 `NULL` 来判断执行结果是否失败，而是需要调用 `ws_errno()` 函数解析结果集中的错误代码来进行判断。 + - ws_errno 返回值：`0`：成功，`-1`：失败，详情请调用 ws_errstr 函数来获取错误提示。 + +- `int32_t ws_result_precision(const WS_RES *rs)` + - **接口说明**：返回结果集时间戳字段的精度类别。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：`0`：毫秒，`1`：微秒，`2`：纳秒。 + +- `WS_ROW ws_fetch_row(WS_RES *rs)` + - **接口说明**：按行获取查询结果集中的数据。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：非 `NULL`：成功，`NULL`：失败，可调用函数 ws_errstr(NULL) 获取更详细的错误信息。 + +- `int32_t ws_fetch_raw_block(WS_RES *rs, const void **pData, int32_t *numOfRows)` + - **接口说明**：批量获取查询结果集中的数据。 + - **参数说明**： + - res：[入参] 结果集。 + - pData：[出参] 用于存储从结果集中获取一个数据块。 + - numOfRows：[出参] 用于存储从结果集中获取数据块包含的行数。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int32_t ws_num_fields(const WS_RES *rs)` 和 `int32_t ws_field_count(const WS_RES *rs)` + - **接口说明**：这两个 API 等价，用于获取查询结果集中的列数。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：返回值为结果集中列的数量。 + +- `int32_t ws_affected_rows(const WS_RES *rs)` + - **接口说明**：获取被所执行的 SQL 语句影响的行数。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：返回值表示受影响的行数。 + +- `int64_t ws_affected_rows64(const WS_RES *rs)` + - **接口说明**：获取被所执行的 SQL 语句影响的行数。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：返回值表示受影响的行数。 + +- `const struct WS_FIELD *ws_fetch_fields(WS_RES *rs)` + - **接口说明**：获取查询结果集每列数据的属性（列的名称、列的数据类型、列的长度），与 `ws_num_fields()` 配合使用，可用来解析 `ws_fetch_row()` 返回的一个元组(一行)的数据。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：非 `NULL`：成功，返回一个指向 WS_FIELD 结构体的指针，每个元素代表一列的元数据。`NULL`：失败。 + +- `int32_t ws_stop_query(WS_RES *rs)` + - **接口说明**：停止当前查询的执行。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int32_t ws_free_result(WS_RES *rs)` + - **接口说明**：释放查询结果集以及相关的资源。查询完成后，务必调用该 API 释放资源，否则可能导致应用内存泄露。但也需注意，释放资源后，如果再调用 `ws_fetch_fields()` 等获取查询结果的函数，将导致应用崩溃。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `const char *ws_errstr(WS_RES *rs)` + - **接口说明**：获取最近一次 API 调用失败的原因，返回值为字符串标识的错误提示信息。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：字符串标识的错误提示信息。 + +- `int32_t ws_errno(WS_RES *rs)` + - **接口说明**：获取最近一次 API 调用失败的原因，返回值为错误代码。 + - **参数说明**： + - res：[入参] 结果集。 + - **返回值**：字符串标识的错误提示信息。 + +:::note +TDengine 推荐数据库应用的每个线程都建立一个独立的连接，或基于线程建立连接池。不要在应用中将该连接 (WS_TAOS\*) 结构体传递到不同的线程共享使用。 +另一个需要注意的是，在上述同步 API 执行过程中，不能调用类似 pthread_cancel 之类的 API 来强制结束线程，因为涉及一些模块的同步操作，如果强制结束线程有可能造成包括但不限于死锁等异常状况。 + +::: + +#### 参数绑定 + +除了直接调用 `ws_query()` 通过执行 SQL 进行数据写入，TDengine 也提供了支持参数绑定的 Prepare API，风格与 MySQL 类似，目前也仅支持用问号 `?` 来代表待绑定的参数。 + +通过参数绑定接口写入数据时，可以避免 SQL 语法解析的资源消耗，从而在绝大多数情况下显著提升写入性能。此时的典型操作步骤如下： + +1. 调用 `ws_stmt_init()` 创建参数绑定对象； +2. 调用 `ws_stmt_prepare()` 解析 INSERT 语句； +3. 如果 INSERT 语句中预留了表名但没有预留 TAGS，那么调用 `ws_stmt_set_tbname()` 来设置表名； +4. 如果 INSERT 语句中既预留了表名又预留了 TAGS（例如 INSERT 语句采取的是自动建表的方式），那么调用 `ws_stmt_set_tbname_tags()` 来设置表名和 TAGS 的值； +5. 调用 `ws_stmt_bind_param_batch()` 以多行的方式设置 VALUES 的值； +6. 调用 `ws_stmt_add_batch()` 把当前绑定的参数加入批处理； +7. 可以重复第 3 ～ 6 步，为批处理加入更多的数据行； +8. 调用 `ws_stmt_execute()` 执行已经准备好的批处理指令； +9. 执行完毕，调用 `ws_stmt_close()` 释放所有资源。 + +说明：如果 `ws_stmt_execute()` 执行成功，假如不需要改变 SQL 语句的话，那么是可以复用 `ws_stmt_prepare()` 的解析结果，直接进行第 3 ～ 6 步绑定新数据的。但如果执行出错，那么并不建议继续在当前的环境上下文下继续工作，而是建议释放资源，然后从 `ws_stmt_init()` 步骤重新开始。 + +接口相关的具体函数如下（也可以参考 [stmt_insert_demo.c](https://github.com/taosdata/TDengine/blob/develop/docs/examples/c-ws/stmt_insert_demo.c) 文件中使用对应函数的方式）： + +- `WS_STMT *ws_stmt_init(const WS_TAOS *taos)` + - **接口说明**：初始化一个预编译的 SQL 语句对象。 + - **参数说明**： + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - **返回值**：非 `NULL`：成功，返回一个指向 WS_STMT 结构体的指针，该结构体表示预编译的 SQL 语句对象。`NULL`：失败，详情请调用 ws_stmt_errstr() 函数来获取错误提示。 + +- `int ws_stmt_prepare(WS_STMT *stmt, const char *sql, unsigned long len)` + - **接口说明**：解析一条预编译的 SQL 语句，将解析结果和参数信息绑定到 stmt 上。 + - **参数说明**： + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - sql：[入参] 需要解析的 SQL 语句。 + - len：[入参] 参数 sql 的长度。如果参数 len 大于 0，将使用此参数作为 SQL 语句的长度，如等于 0，将自动判断 SQL 语句的长度。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int ws_stmt_bind_param_batch(WS_STMT *stmt, const WS_MULTI_BIND *bind, uint32_t len)` + - **接口说明**：以多列的方式传递待绑定的数据，需要保证这里传递的数据列的顺序、列的数量与 SQL 语句中的 VALUES 参数完全一致。 + - **参数说明**： + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - bind：[入参] 指向一个有效的 WS_MULTI_BIND 结构体指针，该结构体包含了要批量绑定到 SQL 语句中的参数列表。 + - len: [入参] bind 数组的元素个数。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int ws_stmt_set_tbname(WS_STMT *stmt, const char *name)` + - **接口说明**：（仅支持用于替换 INSERT 语句中的参数值）当 SQL 语句中的表名使用了 `?` 占位时，可以使用此函数绑定一个具体的表名。 + - **参数说明**： + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - name：[入参] 指向一个包含子表名称的字符串常量。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int ws_stmt_set_tbname_tags(WS_STMT *stmt, + const char *name, + const WS_MULTI_BIND *bind, + uint32_t len);` + - **接口说明**：（仅支持用于替换 INSERT 语句中的参数值）当 SQL 语句中的表名和 TAGS 都使用了 `?` 占位时，可以使用此函数绑定具体的表名和具体的 TAGS 取值。最典型的使用场景是使用了自动建表功能的 INSERT 语句（目前版本不支持指定具体的 TAGS 列）。TAGS 参数中的列数量需要与 SQL 语句中要求的 TAGS 数量完全一致。 + - **参数说明**： + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - name：[入参] 指向一个包含子表名称的字符串常量。 + - tags：[入参] 指向一个有效的 WS_MULTI_BIND 结构体指针，该结构体包含了子表标签的值。 + - len：[入参] bind 数组的元素个数。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int ws_stmt_add_batch(WS_STMT *stmt)` + - **接口说明**：将当前绑定的参数加入批处理中，调用此函数后，可以再次调用 `ws_stmt_bind_param_batch()` 绑定新的参数。需要注意，此函数仅支持 INSERT/IMPORT 语句，如果是 SELECT 等其他 SQL 语句，将返回错误。 + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int ws_stmt_execute(WS_STMT *stmt, int32_t *affected_rows)` + - **接口说明**：执行准备好的语句。目前，一条语句只能执行一次。 + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - affected_rows：[出参] 成功写入的行数。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `int ws_stmt_affected_rows(WS_STMT *stmt)` + - **接口说明**：获取执行预编译 SQL 语句后受影响的行数。 + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - **返回值**：返回受影响的行数。 + +- `int ws_stmt_affected_rows_once(WS_STMT *stmt)` + - **接口说明**：获取执行一次绑定语句影响的行数。 + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - **返回值**：返回受影响的行数。 + +- `int32_t ws_stmt_close(WS_STMT *stmt)` + - **接口说明**：执行完毕，释放所有资源。 + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。 + +- `const char *ws_stmt_errstr(WS_STMT *stmt)` + - **接口说明**：用于在其他 STMT API 返回错误（返回错误码或空指针）时获取错误信息。 + - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 + - **返回值**：返回一个指向包含错误信息的字符串的指针。 + +#### 无模式写入 + +除了使用 SQL 方式或者使用参数绑定 API 写入数据外，还可以使用 Schemaless 的方式完成写入。Schemaless 可以免于预先创建超级表/数据子表的数据结构，而是可以直接写入数据，TDengine 系统会根据写入的数据内容自动创建和维护所需要的表结构。Schemaless 的使用方式详见 [Schemaless 写入](../../../develop/schemaless/) 章节，这里介绍与之配套使用的 C/C++ API。 +- `WS_RES *ws_schemaless_insert_raw(WS_TAOS *taos, + const char *lines, + int len, + int32_t *totalRows, + int protocal, + int precision)` + - **接口说明**：执行无模式的批量插入操作，将行协议的文本数据写入到 TDengine 中。通过传递的参数lines指针和长度len来表示数据，为了解决原始接口数据包含'\0'而被截断的问题。 + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - lines：[入参] 文本数据。满足解析格式要求的无模式文本字符串。 + - len：[入参] 数据缓冲区 lines 的总长度（字节数）。 + - totalRows：[出参] 指向一个整数指针，用于返回成功插入的记录总数。 + - protocol：[入参] 行协议类型，用于标识文本数据格式。 + - precision：[入参] 文本数据中的时间戳精度字符串。 + - **返回值**：返回一个指向 WS_RES 结构体的指针，该结构体包含了插入操作的结果。应用可以通过使用 `ws_errstr()` 获得错误信息，也可以使用 `ws_errno()` 获得错误码。在某些情况下，返回的 WS_RES 为 `NULL`，此时仍然可以调用 `ws_errno()` 来安全地获得错误码信息。 + 返回的 WS_RES 需要调用方来负责释放，否则会出现内存泄漏。 + + **说明** + 协议类型是枚举类型，包含以下三种格式： + + - WS_TSDB_SML_LINE_PROTOCOL：InfluxDB 行协议（Line Protocol) + - WS_TSDB_SML_TELNET_PROTOCOL: OpenTSDB Telnet 文本行协议 + - WS_TSDB_SML_JSON_PROTOCOL: OpenTSDB Json 协议格式 + + 时间戳分辨率的定义，定义在 `taosws.h` 文件中，具体内容如下： + + - WS_TSDB_SML_TIMESTAMP_NOT_CONFIGURED = 0, + - WS_TSDB_SML_TIMESTAMP_HOURS, + - WS_TSDB_SML_TIMESTAMP_MINUTES, + - WS_TSDB_SML_TIMESTAMP_SECONDS, + - WS_TSDB_SML_TIMESTAMP_MILLI_SECONDS, + - WS_TSDB_SML_TIMESTAMP_MICRO_SECONDS, + - WS_TSDB_SML_TIMESTAMP_NANO_SECONDS + + 需要注意的是，时间戳分辨率参数只在协议类型为 `WS_SML_LINE_PROTOCOL` 的时候生效。 + 对于 OpenTSDB 的文本协议，时间戳的解析遵循其官方解析规则 — 按照时间戳包含的字符的数量来确认时间精度。 + + **schemaless 其他相关的接口** + +- `WS_RES *ws_schemaless_insert_raw_with_reqid(WS_TAOS *taos, + const char *lines, + int len, + int32_t *totalRows, + int protocal, + int precision, + uint64_t reqid)` + - **接口说明**：执行无模式的批量插入操作，将行协议的文本数据写入到 TDengine 中。通过传递的参数lines指针和长度len来表示数据，为了解决原始接口数据包含'\0'而被截断的问题。通过传递参数reqid来跟踪整个的函数调用链情况。 + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - lines：[入参] 文本数据。满足解析格式要求的无模式文本字符串。 + - len：[入参] 数据缓冲区 lines 的总长度（字节数）。 + - totalRows：[出参] 指向一个整数指针，用于返回成功插入的记录总数。 + - protocol：[入参] 行协议类型，用于标识文本数据格式。 + - precision：[入参] 文本数据中的时间戳精度字符串。 + - reqid：[入参] 指定的请求 ID，用于跟踪调用请求。请求 ID (reqid) 可以用于在客户端和服务器端之间建立请求和响应之间的关联，对于分布式系统中的跟踪和调试非常有用。 + - **返回值**：返回一个指向 WS_RES 结构体的指针，该结构体包含了插入操作的结果。应用可以通过使用 `ws_errstr()` 获得错误信息，也可以使用 `ws_errno()` 获得错误码。在某些情况下，返回的 WS_RES 为 `NULL`，此时仍然可以调用 `ws_errno()` 来安全地获得错误码信息。 + 返回的 WS_RES 需要调用方来负责释放，否则会出现内存泄漏。 + +- `WS_RES *ws_schemaless_insert_raw_ttl(WS_TAOS *taos, + const char *lines, + int len, + int32_t *totalRows, + int protocal, + int precision, + int ttl)` + - **接口说明**：执行无模式的批量插入操作，将行协议的文本数据写入到 TDengine 中。通过传递的参数lines指针和长度len来表示数据，为了解决原始接口数据包含'\0'而被截断的问题。通过传递ttl参数来控制建表的ttl到期时间。 + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - lines：[入参] 文本数据。满足解析格式要求的无模式文本字符串。 + - len：[入参] 数据缓冲区 lines 的总长度（字节数）。 + - totalRows：[出参] 指向一个整数指针，用于返回成功插入的记录总数。 + - protocol：[入参] 行协议类型，用于标识文本数据格式。 + - precision：[入参] 文本数据中的时间戳精度字符串。 + - ttl：[入参] 指定的生存时间（TTL），单位为天。记录在超过这个生存时间后会被自动删除。 + - **返回值**：返回一个指向 WS_RES 结构体的指针，该结构体包含了插入操作的结果。应用可以通过使用 `ws_errstr()` 获得错误信息，也可以使用 `ws_errno()` 获得错误码。在某些情况下，返回的 WS_RES 为 `NULL`，此时仍然可以调用 `ws_errno()` 来安全地获得错误码信息。 + 返回的 WS_RES 需要调用方来负责释放，否则会出现内存泄漏。 + +- `WS_RES *ws_schemaless_insert_raw_ttl_with_reqid(WS_TAOS *taos, + const char *lines, + int len, + int32_t *totalRows, + int protocal, + int precision, + int ttl, + uint64_t reqid)` + - **接口说明**：执行无模式的批量插入操作，将行协议的文本数据写入到 TDengine 中。通过传递的参数lines指针和长度len来表示数据，为了解决原始接口数据包含'\0'而被截断的问题。通过传递ttl参数来控制建表的ttl到期时间。通过传递参数reqid来跟踪整个的函数调用链情况。 + - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `ws_connect()` 函数建立。 + - lines：[入参] 文本数据。满足解析格式要求的无模式文本字符串。 + - len：[入参] 数据缓冲区 lines 的总长度（字节数）。 + - totalRows：[出参] 指向一个整数指针，用于返回成功插入的记录总数。 + - protocol：[入参] 行协议类型，用于标识文本数据格式。 + - precision：[入参] 文本数据中的时间戳精度字符串。 + - ttl：[入参] 指定的生存时间（TTL），单位为天。记录在超过这个生存时间后会被自动删除。 + - reqid：[入参] 指定的请求 ID，用于跟踪调用请求。请求 ID (reqid) 可以用于在客户端和服务器端之间建立请求和响应之间的关联，对于分布式系统中的跟踪和调试非常有用。 + - **返回值**：返回一个指向 WS_RES 结构体的指针，该结构体包含了插入操作的结果。应用可以通过使用 `ws_errstr()` 获得错误信息，也可以使用 `ws_errno()` 获得错误码。在某些情况下，返回的 WS_RES 为 `NULL`，此时仍然可以调用 `ws_errno()` 来安全地获得错误码信息。 + 返回的 WS_RES 需要调用方来负责释放，否则会出现内存泄漏。 + + **说明** + - 上面这3个接口是扩展接口，主要用于在 schemaless 写入时传递 ttl、reqid 参数，可以根据需要使用。 + - 带 ttl 的接口可以传递 ttl 参数来控制建表的ttl到期时间。 + - 带 reqid 的接口可以通过传递 reqid 参数来追踪整个的调用链。 + +#### 数据订阅 +- `const char *ws_tmq_errstr(ws_tmq_t *tmq)` + - **接口说明**：用于获取数据订阅的错误信息。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - **返回值**：返回一个指向包含错误信息字符串的指针，返回值非NULL，但是错误信息可能为空字符串。 + +- `ws_tmq_conf_t *ws_tmq_conf_new(void);` + - **接口说明**：创建一个新的 TMQ 配置对象。 + - **返回值**：非 `NULL`：成功，返回一个指向 ws_tmq_conf_t 结构体的指针，该结构体用于配置 TMQ 的行为和特性。`NULL`：失败，可调用函数 ws_errstr(NULL) 获取更详细的错误信息。 + +- `enum ws_tmq_conf_res_t ws_tmq_conf_set(ws_tmq_conf_t *conf, const char *key, const char *value)` + - **接口说明**：设置 TMQ 配置对象中的配置项，用于配置消费参数。 + - conf：[入参] 指向一个有效的 ws_tmq_conf_t 结构体指针，该结构体代表一个 TMQ 配置对象。 + - key：[入参] 数配置项的键名。 + - value：[入参] 配置项的值。 + - **返回值**：返回一个 ws_tmq_conf_res_t 枚举值，表示配置设置的结果。 + - WS_TMQ_CONF_OK：成功设置配置项。 + - WS_TMQ_CONF_INVALID_KEY：键值无效。 + - WS_TMQ_CONF_UNKNOWN：键名无效。 + +- `int32_t ws_tmq_conf_destroy(ws_tmq_conf_t *conf)` + - **接口说明**：销毁一个 TMQ 配置对象并释放相关资源。 + - conf：[入参] 指向一个有效的 ws_tmq_conf_t 结构体指针，该结构体代表一个 TMQ 配置对象。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(NULL)` 获取更详细的错误信息。 + +- `ws_tmq_list_t *ws_tmq_list_new(void)` + - **接口说明**：用于创建一个 ws_tmq_list_t 结构体，用于存储订阅的 topic。 + - **返回值**：非 `NULL`：成功，返回一个指向 ws_tmq_list_t 结构体的指针。`NULL`：失败，可调用函数 `ws_tmq_errstr(NULL)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_list_append(ws_tmq_list_t *list, const char *topic)` + - **接口说明**：用于向 ws_tmq_list_t 结构体中添加一个 topic。 + - list：[入参] 指向一个有效的 ws_tmq_list_t 结构体指针，该结构体代表一个 TMQ 列表对象。 + - topic：[入参] topic 名称。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(NULL)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_list_destroy(ws_tmq_list_t *list);` + - **接口说明**：用于销毁 ws_tmq_list_t 结构体，ws_tmq_list_new 的结果需要通过该接口销毁。 + - list：[入参] 指向一个有效的 ws_tmq_list_t 结构体指针，该结构体代表一个 TMQ 列表对象。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(NULL)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_list_get_size(ws_tmq_list_t *list);` + - **接口说明**：用于获取 ws_tmq_list_t 结构体中 topic 的个数。 + - list：[入参] 指向一个有效的 ws_tmq_list_t 结构体指针，该结构体代表一个 TMQ 列表对象。 + - **返回值**：`>=0`：成功，返回 ws_tmq_list_t 结构体中 topic 的个数。`-1`：失败，表示输入参数 list 为 NULL 。 + +- `char **ws_tmq_list_to_c_array(const ws_tmq_list_t *list, uint32_t *topic_num);` + - **接口说明**：用于将 ws_tmq_list_t 结构体转换为 C 数组，数组每个元素为字符串指针。 + - list：[入参] 指向一个有效的 ws_tmq_list_t 结构体指针，该结构体代表一个 TMQ 列表对象。 + - topic_num：[入参] list 的元素个数。 + - **返回值**：非 `NULL`：成功，返回 c 数组, 每个元素是字符串指针，代表一个 topic 名称。`NULL`：失败，表示输入参数 list 为 NULL 。 + +- `ws_tmq_t *ws_tmq_consumer_new(ws_tmq_conf_t *conf, const char *dsn, char *errstr, int errstr_len)` + - **接口说明**：用于创建一个 ws_tmq_t 结构体，用于消费数据，消费完数据后需调用 tmq_consumer_close 关闭消费者。 + - conf：[入参] 指向一个有效的 ws_tmq_conf_t 结构体指针，该结构体代表一个 TMQ 配置对象。 + - dsn：[入参] dsn 信息字符串，具体可参考上面 DSN 章节。一个常见的合法 dsn 为 "tmq+ws://root:taosdata@localhost:6041"。 + - errstr：[出参] 指向一个有效的字符缓冲区指针，用于接收创建过程中可能产生的错误信息。内存的申请/释放由调用者负责。 + - errstrLen：[入参] 指定 errstr 缓冲区的大小（以字节为单位）。 + - **返回值**：非 `NULL`：成功，返回一个指向 ws_tmq_t 结构体的指针，该结构体代表一个 TMQ 消费者对象。。`NULL`：失败，错误信息存储在参数 errstr 中 。 + +- `int32_t ws_tmq_subscribe(ws_tmq_t *tmq, const ws_tmq_list_t *topic_list)` + - **接口说明**：用于订阅 topic 列表，消费完数据后，需调用 ws_tmq_subscribe 取消订阅。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - topic_list：[入参] 指向一个有效的 ws_tmq_list_t 结构体指针，该结构体包含一个或多个主题名称，目前仅支持一个主题名称。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_unsubscribe(ws_tmq_t *tmq)` + - **接口说明**：用于取消订阅的 topic 列表。需与 ws_tmq_subscribe 配合使用。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `WS_RES *ws_tmq_consumer_poll(ws_tmq_t *tmq, int64_t timeout)` + - **接口说明**：用于轮询消费数据，每一个消费者，只能单线程调用该接口。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - timeout：[入参] 轮询的超时时间，单位为毫秒，负数表示默认超时1秒。 + - **返回值**：非 `NULL`：成功，返回一个指向 WS_RES 结构体的指针，该结构体包含了接收到的消息。`NULL`：失败，表示没有数据。WS_RES 结果和 taos_query 返回结果一致，可通过查询的各种接口获取 WS_RES 里的信息，比如 schema 等。 + +- `int32_t ws_tmq_consumer_close(ws_tmq_t *tmq)` + - **接口说明**：用于关闭 ws_tmq_t 结构体。需与 ws_tmq_consumer_new 配合使用。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_get_topic_assignment(ws_tmq_t *tmq, + const char *pTopicName, + struct ws_tmq_topic_assignment **assignment, + int32_t *numOfAssignment)` + - **接口说明**：返回当前 consumer 分配的 vgroup 的信息，每个 vgroup 的信息包括 vgId，wal 的最大最小 offset，以及当前消费到的 offset。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - pTopicName：[入参] 要查询分配信息的主题名称。 + - assignment：[出参] 指向一个 tmq_topic_assignment 结构体指针的指针，用于接收分配信息。数据大小为 numOfAssignment，需要通过 tmq_free_assignment 接口释放。 + - numOfAssignment：[出参] 指向一个整数指针，用于接收分配给该 consumer 有效的 vgroup 个数。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_free_assignment(struct ws_tmq_topic_assignment *pAssignment, int32_t numOfAssignment)` + - **接口说明**：返回当前consumer分配的vgroup的信息，每个vgroup的信息包括vgId，wal的最大最小offset，以及当前消费到的offset。 + - pAssignment：[入参] 指向一个有效的 ws_tmq_topic_assignment 结构体数组的指针，该数组包含了 vgroup 分配信息。 + - numOfAssignment：[入参] pAssignment 指向的数组元素个数。 + - **返回值**：`0`：成功。非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int64_t ws_tmq_committed(ws_tmq_t *tmq, const char *pTopicName, int32_t vgId)` + - **接口说明**：获取 TMQ 消费者对象对特定 topic 和 vgroup 的已提交偏移量。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - pTopicName：[入参] 要查询已提交偏移量的主题名称。 + - vgId：[入参] vgroup 的 ID。 + - **返回值**：`>=0`：成功，返回一个 int64_t 类型的值，表示已提交的偏移量。`<0`：失败，返回值就是错误码，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_commit_sync(ws_tmq_t *tmq, const WS_RES *rs)` + - **接口说明**：同步提交 TMQ 消费者对象处理的消息偏移量。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - rs：[入参] 指向一个有效的 WS_RES 结构体指针，该结构体包含了已处理的消息。如果为 NULL，提交当前 consumer 所有消费的 vgroup 的当前进度。 + - **返回值**：`0`：成功，已经成功提交偏移量。非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_commit_offset_sync(ws_tmq_t *tmq, + const char *pTopicName, + int32_t vgId, + int64_t offset)` + - **接口说明**：同步提交 TMQ 消费者对象的特定主题和 vgroup 的偏移量。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - pTopicName：[入参] 要提交偏移量的主题名称。 + - vgId：[入参] 虚拟组 vgroup 的 ID。 + - offset：[入参] 要提交的偏移量。 + - **返回值**：`0`：成功，已经成功提交偏移量。非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int64_t ws_tmq_position(ws_tmq_t *tmq, const char *pTopicName, int32_t vgId)` + - **接口说明**：获取当前消费位置，即已消费到的数据位置的下一个位置. + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - pTopicName：[入参] 要查询当前位置的主题名称。 + - vgId：[入参] 虚拟组 vgroup 的 ID。 + - **返回值**：`>=0`：成功，返回一个 int64_t 类型的值，表示当前位置的偏移量。`<0`：失败，返回值就是错误码，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + + - `int32_t ws_tmq_offset_seek(ws_tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset)` + - **接口说明**：将 TMQ 消费者对象在某个特定 topic 和 vgroup 的偏移量设置到指定的位置。 + - tmq：[入参] 指向一个有效的 ws_tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 + - pTopicName：[入参] 要查询当前位置的主题名称。 + - vgId：[入参] 虚拟组 vgroup 的 ID。 + - offset：[入参] 虚拟组 vgroup 的 ID。 + - **返回值**：`0`：成功，非 `0`：失败，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int64_t ws_tmq_get_vgroup_offset(const WS_RES *rs)` + - **接口说明**：从 TMQ 消费者获取的消息结果中提取虚拟组（vgroup）的当前消费数据位置的偏移量。 + - res：[入参] 指向一个有效的 WS_RES 结构体指针，该结构体包含了从 TMQ 消费者轮询得到的消息。 + - **返回值**：`>=0`：成功，返回一个 int64_t 类型的值，表示当前消费位置的偏移量。`<0`：失败，返回值就是错误码，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `int32_t ws_tmq_get_vgroup_id(const WS_RES *rs)` + - **接口说明**：从 TMQ 消费者获取的消息结果中提取所属虚拟组（vgroup）的 ID。 + - res：[入参] 指向一个有效的 WS_RES 结构体指针，该结构体包含了从 TMQ 消费者轮询得到的消息。 + - **返回值**：`>=0`：成功，返回一个 int32_t 类型的值，表示虚拟组（vgroup）的 ID。`<0`：失败，返回值就是错误码，可调用函数 `ws_tmq_errstr(tmq)` 获取更详细的错误信息。 + +- `const char *ws_tmq_get_table_name(const WS_RES *rs)` + - **接口说明**：从 TMQ 消费者获取的消息结果中获取所属的表名。 + - res：[入参] 指向一个有效的 WS_RES 结构体指针，该结构体包含了从 TMQ 消费者轮询得到的消息。 + - **返回值**：非 `NULL`：成功，返回一个 const char * 类型的指针，指向表名字符串。`NULL`：失败，非法的输入参数。 + +- `enum ws_tmq_res_t ws_tmq_get_res_type(const WS_RES *rs)` + - **接口说明**：从 TMQ 消费者获取的消息结果中获取消息类型。 + - res：[入参] 指向一个有效的 WS_RES 结构体指针，该结构体包含了从 TMQ 消费者轮询得到的消息。 + - **返回值**：返回一个 ws_tmq_res_t 类型的枚举值，表示消息类型。 + - ws_tmq_res_t 表示消费到的数据类型，定义如下： + ``` + typedef enum ws_tmq_res_t { + WS_TMQ_RES_INVALID = -1, // 无效 + WS_TMQ_RES_DATA = 1, // 数据类型 + WS_TMQ_RES_TABLE_META = 2, // 元数据类型 + WS_TMQ_RES_METADATA = 3 // 既有元数据类型又有数据类型，即自动建表 + } tmq_res_t; + ``` + +- `const char *ws_tmq_get_topic_name(const WS_RES *rs)` + - **接口说明**：从 TMQ 消费者获取的消息结果中获取所属的 topic 名称。 + - res：[入参] 指向一个有效的 WS_RES 结构体指针，该结构体包含了从 TMQ 消费者轮询得到的消息。 + - **返回值**：非 `NULL`：成功，返回一个 const char * 类型的指针，指向 topic 名称字符串。`NULL`：失败，非法的输入参数。 + +- `const char *ws_tmq_get_db_name(const WS_RES *rs)` + - **接口说明**：从 TMQ 消费者获取的消息结果中获取所属的数据库名称。 + - res：[入参] 指向一个有效的 WS_RES 结构体指针，该结构体包含了从 TMQ 消费者轮询得到的消息。 + - **返回值**：非 `NULL`：成功，返回一个 const char * 类型的指针，指向数据库名称字符串。`NULL`：失败，非法的输入参数。 + +## 原生连接方式 +原生连接方式需要使用 taos.h 头文件和 taos 动态库。 ```c #include ``` @@ -22,21 +629,21 @@ TDengine 客户端驱动的动态库位于： - Windows: `C:\TDengine\taos.dll` - macOS: `/usr/local/lib/libtaos.dylib` -## 支持的平台 +### 支持的平台 请参考[支持的平台列表](../#支持的平台) -## 支持的版本 +### 支持的版本 TDengine 客户端驱动的版本号与 TDengine 服务端的版本号是一一对应的强对应关系，建议使用与 TDengine 服务端完全相同的客户端驱动。虽然低版本的客户端驱动在前三段版本号一致（即仅第四段版本号不同）的情况下也能够与高版本的服务端相兼容，但这并非推荐用法。强烈不建议使用高版本的客户端驱动访问低版本的服务端。 -## 错误码 +### 错误码 在 C 接口的设计中，错误码采用整数类型表示，每个错误码都对应一个特定的错误状态。如未特别说明，当 API 的返回值是整数时，_0_ 代表成功，其它是代表失败原因的错误码，当返回值是指针时， _NULL_ 表示失败。 所有的错误码以及对应的原因描述在 `taoserror.h` 文件中。 详细的错误码说明参考：[错误码](../../../reference/error-code) -## 示例程序 +### 示例程序 本节展示了使用客户端驱动访问 TDengine 集群的常见访问方式的示例代码。 @@ -57,11 +664,11 @@ TDengine 客户端驱动的版本号与 TDengine 服务端的版本号是一一 ::: -## API 参考 +### API 参考 -以下分别介绍 TDengine 客户端驱动的基础 API、同步 API、异步 API、订阅 API 和无模式写入 API。 +以下分别介绍 TDengine 客户端驱动的基础 API、同步 API、异步 API、参数绑定 API，无模式写入 API 和数据订阅 API。 -### 基础 API +#### 基础 API 基础 API 用于完成创建数据库连接等工作，为其它 API 的执行提供运行时环境。 @@ -145,7 +752,7 @@ TDengine 客户端驱动的版本号与 TDengine 服务端的版本号是一一 - **参数说明**： - taos：[入参] 指向数据库连接的指针，数据库连接是通过 `taos_connect()` 函数建立。 -### 同步查询 +#### 同步查询 本小节介绍 API 均属于同步接口。应用调用后，会阻塞等待响应，直到获得返回结果或错误信息。 @@ -228,7 +835,7 @@ TDengine 客户端驱动的版本号与 TDengine 服务端的版本号是一一 ::: -### 异步查询 +#### 异步查询 TDengine 还提供性能更高的异步 API 处理数据插入、查询操作。在软硬件环境相同的情况下，异步 API 处理数据插入的速度比同步 API 快 2 ～ 4 倍。异步 API 采用非阻塞式的调用方式，在系统真正完成某个具体数据库操作前，立即返回。调用的线程可以去处理其他工作，从而可以提升整个应用的性能。异步 API 在网络延迟严重的情况下，优势尤为突出。 @@ -252,7 +859,7 @@ TDengine 还提供性能更高的异步 API 处理数据插入、查询操作。 TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多线程同时打开多张表，并可以同时对每张打开的表进行查询或者插入操作。需要指出的是，**客户端应用必须确保对同一张表的操作完全串行化**，即对同一个表的插入或查询操作未完成时（未返回时），不能够执行第二个插入或查询操作。 -### 参数绑定 +#### 参数绑定 除了直接调用 `taos_query()` 进行查询，TDengine 也提供了支持参数绑定的 Prepare API，风格与 MySQL 类似，目前也仅支持用问号 `?` 来代表待绑定的参数。 @@ -350,7 +957,7 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多 - stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。 - **返回值**：返回一个指向包含错误信息的字符串的指针。 -### 无模式写入 +#### 无模式写入 除了使用 SQL 方式或者使用参数绑定 API 写入数据外，还可以使用 Schemaless 的方式完成写入。Schemaless 可以免于预先创建超级表/数据子表的数据结构，而是可以直接写入数据，TDengine 系统会根据写入的数据内容自动创建和维护所需要的表结构。Schemaless 的使用方式详见 [Schemaless 写入](../../../develop/schemaless/) 章节，这里介绍与之配套使用的 C/C++ API。 @@ -474,7 +1081,7 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多 - 带_ttl的接口可以传递ttl参数来控制建表的ttl到期时间。 - 带_reqid的接口可以通过传递reqid参数来追踪整个的调用链。 -### 数据订阅 +#### 数据订阅 - `const char *tmq_err2str(int32_t code)` - **接口说明**：用于将数据订阅的错误码转换为错误信息。 - code：[入参] 数据订阅的错误码。 @@ -584,7 +1191,7 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多 - **接口说明**：获取 TMQ 消费者对象对特定 topic 和 vgroup 的已提交偏移量。 - tmq：[入参] 指向一个有效的 tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 - pTopicName：[入参] 要查询已提交偏移量的主题名称。 - - assignment：[入参] vgroup 的 ID。 + - vgId：[入参] vgroup 的 ID。 - **返回值**：`>=0`：成功，返回一个 int64_t 类型的值，表示已提交的偏移量。`<0`：失败，返回值就是错误码，可调用函数 `char *tmq_err2str(int32_t code)` 获取更详细的错误信息。 - `int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg)` @@ -604,6 +1211,8 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多 - **接口说明**：同步提交 TMQ 消费者对象的特定主题和 vgroup 的偏移量。 - tmq：[入参] 指向一个有效的 tmq_t 结构体指针，该结构体代表一个 TMQ 消费者对象。 - pTopicName：[入参] 要提交偏移量的主题名称。 + - vgId：[入参] 虚拟组 vgroup 的 ID。 + - offset：[入参] 要提交的偏移量。 - **返回值**：`0`：成功，已经成功提交偏移量。非 `0`：失败，可调用函数 `char *tmq_err2str(int32_t code)` 获取更详细的错误信息。 - `void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param)` diff --git a/docs/zh/14-reference/05-connector/26-rust.mdx b/docs/zh/14-reference/05-connector/26-rust.mdx index c5d2a165d4..62d58fd416 100644 --- a/docs/zh/14-reference/05-connector/26-rust.mdx +++ b/docs/zh/14-reference/05-connector/26-rust.mdx @@ -113,11 +113,11 @@ DSN 描述字符串基本结构如下： 各部分意义见下表： - **driver**: 必须指定驱动名以便连接器选择何种方式创建连接，支持如下驱动名： - - **taos**: 表名使用 TDengine 连接器驱动。 + - **taos**: 使用 TDengine 连接器驱动，默认是使用 taos 驱动。 - **tmq**: 使用 TMQ 订阅数据。 +- **protocol**: 显示指定以何种方式建立连接，例如：`taos+ws://localhost:6041` 指定以 Websocket 方式建立连接。 - **http/ws**: 使用 Websocket 创建连接。 - **https/wss**: 在 Websocket 连接方式下显示启用 SSL/TLS 连接。 -- **protocol**: 显示指定以何种方式建立连接，例如：`taos+ws://localhost:6041` 指定以 Websocket 方式建立连接。 - **username/password**: 用于创建连接的用户名及密码。 - **host/port**: 指定创建连接的服务器及端口，当不指定服务器地址及端口时（`taos://`），原生连接默认为 `localhost:6030`，Websocket 连接默认为 `localhost:6041` 。 - **database**: 指定默认连接的数据库名，可选参数。 diff --git a/docs/zh/14-reference/05-connector/35-node.mdx b/docs/zh/14-reference/05-connector/35-node.mdx index 3d8e82086e..bd2ca537e3 100644 --- a/docs/zh/14-reference/05-connector/35-node.mdx +++ b/docs/zh/14-reference/05-connector/35-node.mdx @@ -85,8 +85,6 @@ Node.js 连接器目前仅支持 Websocket 连接器， 其通过 taosAdapter | [sql_example](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/sql_example.js) | 基本的使用如如建立连接，执行 SQL 等操作。 | | [stmt_example](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/stmt_example.js) | 绑定参数插入的示例。 | | | [line_example](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/line_example.js) | 行协议写入示例。 | -| [telnet_line_example](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/telnet_line_example.js) | OpenTSDB Telnet 行协议写入示例。 | -| [json_line_example](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/json_line_example.js) | OpenTSDB JSON 行协议写入示例。 | | [tmq_example](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/tmq_example.js) | 订阅的使用示例。 | | [all_type_query](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/all_type_query.js) | 支持全部类型示例。 | | [all_type_stmt](https://github.com/taosdata/TDengine/tree/main/docs/examples/node/websocketexample/all_type_stmt.js) | 参数绑定支持全部类型示例。 | diff --git a/docs/zh/14-reference/05-connector/index.md b/docs/zh/14-reference/05-connector/index.md index c4e5628686..04a2ef6c1f 100644 --- a/docs/zh/14-reference/05-connector/index.md +++ b/docs/zh/14-reference/05-connector/index.md @@ -28,14 +28,14 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 TDengine 版本更新往往会增加新的功能特性，列表中的连接器版本为连接器最佳适配版本。 -| **TDengine 版本** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | -| ---------------------- | ----------- | ------------------------------------------- | ------------ | ------------- | --------------- | -------- | -| **3.3.0.0 及以上** | 3.3.0及以上 | taospy 2.7.15及以上，taos-ws-py 0.3.2及以上 | 3.5.5及以上 | 3.1.3及以上 | 3.1.0及以上 | 当前版本 | -| **3.0.0.0 及以上** | 3.0.2以上 | 当前版本 | 3.0 分支 | 3.0.0 | 3.1.0 | 当前版本 | -| **2.4.0.14 及以上** | 2.0.38 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | -| **2.4.0.4 - 2.4.0.13** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | -| **2.2.x.x ** | 2.0.36 | 当前版本 | master 分支 | n/a | 2.0.7 - 2.0.9 | 当前版本 | -| **2.0.x.x ** | 2.0.34 | 当前版本 | master 分支 | n/a | 2.0.1 - 2.0.6 | 当前版本 | +| **TDengine 版本** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | **C/C++** | +| ---------------------- | ----------- | ------------------------------------------- | ------------ | ------------- | --------------- | -------- | -------------------- | +| **3.3.0.0 及以上** | 3.3.0及以上 | taospy 2.7.15及以上，taos-ws-py 0.3.2及以上 | 3.5.5及以上 | 3.1.3及以上 | 3.1.0及以上 | 当前版本 | 与 TDengine 相同版本 | +| **3.0.0.0 及以上** | 3.0.2以上 | 当前版本 | 3.0 分支 | 3.0.0 | 3.1.0 | 当前版本 | 与 TDengine 相同版本 | +| **2.4.0.14 及以上** | 2.0.38 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | 与 TDengine 相同版本 | +| **2.4.0.4 - 2.4.0.13** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | 与 TDengine 相同版本 | +| **2.2.x.x ** | 2.0.36 | 当前版本 | master 分支 | n/a | 2.0.7 - 2.0.9 | 当前版本 | 与 TDengine 相同版本 | +| **2.0.x.x ** | 2.0.34 | 当前版本 | master 分支 | n/a | 2.0.1 - 2.0.6 | 当前版本 | 与 TDengine 相同版本 | ## 功能特性 @@ -43,13 +43,13 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 ### 使用原生接口（taosc） -| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Rust** | -| ------------------- | -------- | ---------- | ------ | ------ | -------- | -| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | -| **执行 SQL** | 支持 | 支持 | 支持 | 支持 | 支持 | -| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | -| **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 支持 | 支持 | -| **无模式写入** | 支持 | 支持 | 支持 | 支持 | 支持 | +| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Rust** | **C/C++** | +| ------------------- | -------- | ---------- | ------ | ------ | -------- | --------- | +| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **执行 SQL** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **无模式写入** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | :::info 由于不同编程语言数据库框架规范不同，并不意味着所有 C/C++ 接口都需要对应封装支持。 @@ -64,13 +64,13 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 ### 使用 Websocket 接口 -| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | -| ------------------- | -------- | ---------- | ------ | ------ | ----------- | -------- | -| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **执行 SQL** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **无模式写入** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | **C/C++** | +| ------------------- | -------- | ---------- | ------ | ------ | ----------- | -------- | --------- | +| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **执行 SQL** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **无模式写入** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | :::warning - 无论选用何种编程语言的连接器，2.0 及以上版本的 TDengine 推荐数据库应用的每个线程都建立一个独立的连接，或基于线程建立连接池，以避免连接内的“USE statement”状态量在线程之间相互干扰（但连接的查询和写入操作都是线程安全的）。 diff --git a/docs/zh/14-reference/07-supported.md b/docs/zh/14-reference/07-supported.md index 47c874c907..10ca237653 100644 --- a/docs/zh/14-reference/07-supported.md +++ b/docs/zh/14-reference/07-supported.md @@ -7,11 +7,11 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" ## TDengine 服务端支持的平台列表 | | **Windows server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18 以上** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **macOS** | -| ------------ | ---------------------------- | ----------------- | ---------------- | ---------------- | ------------ | ----------------- | ---------------- | --------- | -| X64 | ●/E | ●/E | ● | ● | ●/E | ●/E | ●/E | ● | -| 树莓派 ARM64 | | | ● | | | | | | -| 华为云 ARM64 | | | | ● | | | | | -| M1 | | | | | | | | ● | +| ------------ | ---------------------------- | ----------------- | ---------------- | ------------------ | ------------ | ----------------- | ---------------- | --------- | +| X64 | ●/E | ●/E | ● | ● | ●/E | ●/E | ●/E | ● | +| 树莓派 ARM64 | | | ● | | | | | | +| 华为云 ARM64 | | | | ● | | | | | +| M1 | | | | | | | | ● | 注：1) ● 表示经过官方测试验证， ○ 表示非官方测试验证，E 表示仅企业版支持。 2) 社区版仅支持主流操作系统的较新版本，包括 Ubuntu 18+/CentOS 7+/RedHat/Debian/CoreOS/FreeBSD/OpenSUSE/SUSE Linux/Fedora/macOS 等。如果有其他操作系统及版本的需求，请联系企业版支持。 @@ -31,6 +31,7 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" | **Go** | ● | ● | ● | ● | ● | | **NodeJs** | ● | ● | ● | ○ | ○ | | **C#** | ● | ● | ○ | ○ | ○ | +| **Rust** | ● | ● | ○ | ● | ● | | **RESTful** | ● | ● | ● | ● | ● | 注：● 表示官方测试验证通过，○ 表示非官方测试验证通过，-- 表示未经验证。 diff --git a/include/libs/monitorfw/taos_collector_registry.h b/include/libs/monitorfw/taos_collector_registry.h index 81d1f8050c..5ac849530f 100644 --- a/include/libs/monitorfw/taos_collector_registry.h +++ b/include/libs/monitorfw/taos_collector_registry.h @@ -37,9 +37,9 @@ extern taos_collector_registry_t *TAOS_COLLECTOR_REGISTRY_DEFAULT; /** * @brief Initializes the default collector registry and enables metric collection on the executing process - * @return A non-zero integer value upon failure + * @return */ -int taos_collector_registry_default_init(void); +void taos_collector_registry_default_init(void); /** * @brief Constructs a taos_collector_registry_t* diff --git a/include/util/tcompression.h b/include/util/tcompression.h index c8821234b6..fef6c0713c 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -153,11 +153,10 @@ int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int int32_t getWordLength(char type); int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type); -int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output); -int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output); -int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, - bool bigEndian); -int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, bool bigEndian); +void tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output); +void tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output); +void tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, bool bigEndian); +void tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, bool bigEndian); /************************************************************************* * REGULAR COMPRESSION 2 diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 6957874642..1b5f95ce16 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -1316,9 +1316,9 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d,QID:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); code = refreshMeta(pRequest->pTscObj, pRequest); - if (code != 0){ - tscWarn("0x%" PRIx64 " refresh meta failed, code:%d - %s,QID:0x%" PRIx64, pRequest->self, code, - tstrerror(code), pRequest->requestId); + if (code != 0) { + tscWarn("0x%" PRIx64 " refresh meta failed, code:%d - %s,QID:0x%" PRIx64, pRequest->self, code, tstrerror(code), + pRequest->requestId); } pRequest->prevCode = code; doAsyncQuery(pRequest, true); @@ -1985,7 +1985,9 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col STscStmt2 *pStmt = (STscStmt2 *)stmt; if (pStmt->options.asyncExecFn && !pStmt->semWaited) { - (void)tsem_wait(&pStmt->asyncQuerySem); + if (tsem_wait(&pStmt->asyncQuerySem) != 0) { + tscError("wait async query sem failed"); + } pStmt->semWaited = true; } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 48049f0baf..17787ac7c0 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -26,7 +26,7 @@ #define RAW_NULL_CHECK(c) \ do { \ if (c == NULL) { \ - code = TSDB_CODE_OUT_OF_MEMORY; \ + code = terrno; \ goto end; \ } \ } while (0) @@ -1780,6 +1780,42 @@ end: return code; } +static int32_t buildCreateTbMap(STaosxRsp* rsp, SHashObj* pHashObj) { + // find schema data info + int32_t code = 0; + SVCreateTbReq pCreateReq = {0}; + SDecoder decoderTmp = {0}; + + for (int j = 0; j < rsp->createTableNum; j++) { + void** dataTmp = taosArrayGet(rsp->createTableReq, j); + RAW_NULL_CHECK(dataTmp); + int32_t* lenTmp = taosArrayGet(rsp->createTableLen, j); + RAW_NULL_CHECK(lenTmp); + + tDecoderInit(&decoderTmp, *dataTmp, *lenTmp); + RAW_RETURN_CHECK (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq)); + + if (pCreateReq.type != TSDB_CHILD_TABLE) { + code = TSDB_CODE_INVALID_MSG; + goto end; + } + if (taosHashGet(pHashObj, pCreateReq.name, strlen(pCreateReq.name)) == NULL){ + RAW_RETURN_CHECK(taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), &pCreateReq, sizeof(SVCreateTbReq))); + } else{ + tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); + pCreateReq = (SVCreateTbReq){0}; + } + + tDecoderClear(&decoderTmp); + } + return 0; + +end: + tDecoderClear(&decoderTmp); + tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); + return code; +} + static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) { if (taos == NULL || data == NULL) { SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data); @@ -1791,7 +1827,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) SMqTaosxRspObj rspObj = {0}; SDecoder decoder = {0}; STableMeta* pTableMeta = NULL; - SVCreateTbReq* pCreateReqDst = NULL; + SHashObj* pCreateTbHash = NULL; SRequestObj* pRequest = NULL; RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest)); @@ -1832,6 +1868,9 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) RAW_RETURN_CHECK(smlInitHandle(&pQuery)); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); RAW_NULL_CHECK(pVgHash); + pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + RAW_NULL_CHECK(pCreateTbHash); + RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.rsp, pCreateTbHash)); uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum); while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) { @@ -1854,40 +1893,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) (void)strcpy(pName.tname, tbName); // find schema data info - for (int j = 0; j < rspObj.rsp.createTableNum; j++) { - void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j); - RAW_NULL_CHECK(dataTmp); - int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j); - RAW_NULL_CHECK(dataTmp); - - SDecoder decoderTmp = {0}; - SVCreateTbReq pCreateReq = {0}; - tDecoderInit(&decoderTmp, *dataTmp, *lenTmp); - if (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq) < 0) { - tDecoderClear(&decoderTmp); - tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); - code = TSDB_CODE_TMQ_INVALID_MSG; - SET_ERROR_MSG("decode create table:%s req failed", tbName); - goto end; - } - - if (pCreateReq.type != TSDB_CHILD_TABLE) { - code = TSDB_CODE_TSC_INVALID_VALUE; - tDecoderClear(&decoderTmp); - tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); - SET_ERROR_MSG("create table req type is not child table: %s, type: %d", tbName, pCreateReq.type); - goto end; - } - if (strcmp(tbName, pCreateReq.name) == 0) { - RAW_RETURN_CHECK(cloneSVreateTbReq(&pCreateReq, &pCreateReqDst)); - tDecoderClear(&decoderTmp); - tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); - break; - } - tDecoderClear(&decoderTmp); - tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); - } - + SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName)); SVgroupInfo vg = {0}; RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg)); if (pCreateReqDst) { // change stable name to get meta @@ -1920,13 +1926,17 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) } void* rawData = getRawDataFromRes(pRetrieve); char err[ERR_MSG_LEN] = {0}; - code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqDst, fields, pSW->nCols, true, err, ERR_MSG_LEN); + SVCreateTbReq* pCreateReqTmp = NULL; + if (pCreateReqDst){ + RAW_RETURN_CHECK(cloneSVreateTbReq(pCreateReqDst, &pCreateReqTmp)); + } + code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqTmp, fields, pSW->nCols, true, err, ERR_MSG_LEN); + if (pCreateReqTmp != NULL) { + tdDestroySVCreateTbReq(pCreateReqTmp); + taosMemoryFree(pCreateReqTmp); + } taosMemoryFree(fields); taosMemoryFreeClear(pTableMeta); - if (pCreateReqDst) { - tdDestroySVCreateTbReq(pCreateReqDst); - taosMemoryFreeClear(pCreateReqDst); - } if (code != TSDB_CODE_SUCCESS) { SET_ERROR_MSG("table:%s, err:%s", tbName, err); goto end; @@ -1940,16 +1950,18 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) end: uDebug(LOG_ID_TAG " write raw metadata return, msg:%s", LOG_ID_VALUE, tstrerror(code)); + void* pIter = taosHashIterate(pCreateTbHash, NULL); + while (pIter) { + tDestroySVCreateTbReq(pIter, TSDB_MSG_FLG_DECODE); + pIter = taosHashIterate(pCreateTbHash, pIter); + } + taosHashCleanup(pCreateTbHash); tDeleteSTaosxRsp(&rspObj.rsp); tDecoderClear(&decoder); qDestroyQuery(pQuery); destroyRequest(pRequest); taosHashCleanup(pVgHash); taosMemoryFreeClear(pTableMeta); - if (pCreateReqDst) { - tdDestroySVCreateTbReq(pCreateReqDst); - taosMemoryFreeClear(pCreateReqDst); - } return code; } diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index c26293e587..a0fd49ac86 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -698,7 +698,9 @@ static void* stmtBindThreadFunc(void* param) { continue; } - (void)stmtAsyncOutput(pStmt, asyncParam); + if (stmtAsyncOutput(pStmt, asyncParam) != 0) { + qError("stmt async output failed"); + } } qInfo("stmt bind thread stopped"); @@ -822,7 +824,11 @@ TAOS_STMT2* stmtInit2(STscObj* taos, TAOS_STMT2_OPTION* pOptions) { pStmt->sql.siInfo.tableColsReady = true; if (pStmt->options.asyncExecFn) { - (void)tsem_init(&pStmt->asyncQuerySem, 0, 1); + if (tsem_init(&pStmt->asyncQuerySem, 0, 1) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + (void)stmtClose(pStmt); + return NULL; + } } pStmt->semWaited = false; @@ -1603,7 +1609,9 @@ static void asyncQueryCb(void* userdata, TAOS_RES* res, int code) { (void)stmtCleanExecInfo(pStmt, (code ? false : true), false); ++pStmt->sql.runTimes; - (void)tsem_post(&pStmt->asyncQuerySem); + if (tsem_post(&pStmt->asyncQuerySem) != 0) { + tscError("failed to post asyncQuerySem"); + } } int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) { @@ -1710,7 +1718,9 @@ int stmtClose2(TAOS_STMT2* stmt) { } if (pStmt->options.asyncExecFn && !pStmt->semWaited) { - (void)tsem_wait(&pStmt->asyncQuerySem); + if (tsem_wait(&pStmt->asyncQuerySem) != 0) { + tscError("failed to wait asyncQuerySem"); + } } STMT_DLOG("stmt %p closed, stbInterlaceMode: %d, statInfo: ctgGetTbMetaNum=>%" PRId64 ", getCacheTbInfo=>%" PRId64 @@ -1727,7 +1737,9 @@ int stmtClose2(TAOS_STMT2* stmt) { STMT_ERR_RET(stmtCleanSQLInfo(pStmt)); if (pStmt->options.asyncExecFn) { - (void)tsem_destroy(&pStmt->asyncQuerySem); + if (tsem_destroy(&pStmt->asyncQuerySem) != 0) { + tscError("failed to destroy asyncQuerySem"); + } } taosMemoryFree(stmt); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index bb73f43ebb..b5aff49232 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -24,7 +24,7 @@ int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { SVnodeObj *pVnode = NULL; (void)taosThreadRwlockRdlock(&pMgmt->lock); - (void)taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); + int32_t r = taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); if (pVnode != NULL) { diskId = pVnode->diskPrimary; } @@ -97,7 +97,7 @@ SVnodeObj *vmAcquireVnodeImpl(SVnodeMgmt *pMgmt, int32_t vgId, bool strict) { SVnodeObj *pVnode = NULL; (void)taosThreadRwlockRdlock(&pMgmt->lock); - (void)taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); + int32_t r = taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); if (pVnode == NULL || strict && (pVnode->dropped || pVnode->failed)) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; pVnode = NULL; @@ -165,7 +165,7 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { (void)taosThreadRwlockWrlock(&pMgmt->lock); SVnodeObj *pOld = NULL; - (void)taosHashGetDup(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), (void *)&pOld); + int32_t r = taosHashGetDup(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), (void *)&pOld); if (pOld) { vmFreeVnodeObj(&pOld); } @@ -184,7 +184,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) } (void)taosThreadRwlockWrlock(&pMgmt->lock); - (void)taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); + int32_t r = taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); (void)taosThreadRwlockUnlock(&pMgmt->lock); vmReleaseVnode(pMgmt, pVnode); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 99e59662ac..60b732f817 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -106,6 +106,7 @@ typedef enum { // TRN_CONFLICT_TOPIC = 4, // TRN_CONFLICT_TOPIC_INSIDE = 5, TRN_CONFLICT_ARBGROUP = 6, + TRN_CONFLICT_TSMA = 7, } ETrnConflct; typedef enum { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 374dff8d0c..0403029f74 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -2281,6 +2281,10 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, int32_t cols = 0; int32_t bytes = pShow->pMeta->pSchemas[cols].bytes; char *buf = taosMemoryMalloc(bytes); + if (buf == NULL) { + mError("db:%s, failed to malloc buffer", pDb->name); + return; + } int32_t code = 0; int32_t lino = 0; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 4c5a695402..db4d842662 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -184,6 +184,7 @@ static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc) { } static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) { + int32_t code = 0; mTrace("func:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); taosWLockLatch(&pOld->lock); @@ -205,6 +206,11 @@ static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) { if (pNew->commentSize > 0 && pNew->pComment != NULL) { pOld->commentSize = pNew->commentSize; pOld->pComment = taosMemoryMalloc(pOld->commentSize); + if (pOld->pComment == NULL) { + code = terrno; + taosWUnLockLatch(&pOld->lock); + return code; + } (void)memcpy(pOld->pComment, pNew->pComment, pOld->commentSize); } @@ -215,6 +221,11 @@ static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) { if (pNew->codeSize > 0 && pNew->pCode != NULL) { pOld->codeSize = pNew->codeSize; pOld->pCode = taosMemoryMalloc(pOld->codeSize); + if (pOld->pCode == NULL) { + code = terrno; + taosWUnLockLatch(&pOld->lock); + return code; + } (void)memcpy(pOld->pCode, pNew->pCode, pOld->codeSize); } @@ -261,6 +272,10 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre if (NULL != pCreate->pComment) { func.commentSize = strlen(pCreate->pComment) + 1; func.pComment = taosMemoryMalloc(func.commentSize); + if (func.pComment == NULL) { + code = terrno; + goto _OVER; + } } func.codeSize = pCreate->codeLen; func.pCode = taosMemoryMalloc(func.codeSize); @@ -716,6 +731,11 @@ static int32_t mndRetrieveFuncs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl ? TSDB_MAX_BINARY_LEN : pFunc->codeSize + VARSTR_HEADER_SIZE; char *b4 = taosMemoryMalloc(varCodeLen); + if (b4 == NULL) { + code = terrno; + sdbRelease(pSdb, pFunc); + TAOS_RETURN(code); + } (void)memcpy(varDataVal(b4), pFunc->pCode, varCodeLen - VARSTR_HEADER_SIZE); varDataSetLen(b4, varCodeLen - VARSTR_HEADER_SIZE); code = colDataSetVal(pColInfo, numOfRows, (const char *)b4, false); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index c00c88c4f9..6b1c97b399 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -343,6 +343,10 @@ static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *p int32_t code = 0; int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq); void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + code = terrno; + return code; + } code = tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq); if (code < 0) { taosMemoryFree(pReq); @@ -369,6 +373,10 @@ static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, SDAlterMnodeType int32_t code = 0; int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterMnodeTypeReq); void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + code = terrno; + return code; + } code = tSerializeSDCreateMnodeReq(pReq, contLen, pAlterMnodeTypeReq); if (code < 0) { taosMemoryFree(pReq); @@ -395,6 +403,10 @@ static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pA int32_t code = 0; int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq); void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + code = terrno; + return code; + } code = tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq); if (code < 0) { taosMemoryFree(pReq); @@ -420,6 +432,10 @@ static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDrop int32_t code = 0; int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, pDropReq); void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + code = terrno; + return code; + } code = tSerializeSCreateDropMQSNodeReq(pReq, contLen, pDropReq); if (code < 0) { taosMemoryFree(pReq); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 1a76ab2a8b..a258155223 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -1692,7 +1692,7 @@ static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { STransAction dropStbUndoAction = {0}; SMDropStbReq dropStbReq = {0}; STrans *pTrans = - mndTransCreate(pCxt->pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pCxt->pRpcReq, "create-tsma"); + mndTransCreate(pCxt->pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TSMA, pCxt->pRpcReq, "create-tsma"); if (!pTrans) { code = terrno; goto _OVER; @@ -1974,7 +1974,7 @@ _OVER: static int32_t mndDropTSMA(SCreateTSMACxt* pCxt) { int32_t code = -1; STransAction dropStreamRedoAction = {0}; - STrans *pTrans = mndTransCreate(pCxt->pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pCxt->pRpcReq, "drop-tsma"); + STrans *pTrans = mndTransCreate(pCxt->pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TSMA, pCxt->pRpcReq, "drop-tsma"); if (!pTrans) { code = terrno; goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index fa9e4fa8fa..56461e9cfd 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2352,6 +2352,11 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, SStbObj *pObj, void **pCont, i } void *cont = taosMemoryMalloc(contLen); + if (NULL == cont) { + code = terrno; + tFreeSMAlterStbRsp(&alterRsp); + TAOS_RETURN(code); + } tEncoderInit(&ec, cont, contLen); code = tEncodeSMAlterStbRsp(&ec, &alterRsp); tEncoderClear(&ec); @@ -2407,6 +2412,11 @@ int32_t mndBuildSMCreateStbRsp(SMnode *pMnode, char *dbFName, char *stbFName, vo } void *cont = taosMemoryMalloc(contLen); + if (NULL == cont) { + code = terrno; + tFreeSMCreateStbRsp(&stbRsp); + goto _OVER; + } tEncoderInit(&ec, cont, contLen); TAOS_CHECK_GOTO(tEncodeSMCreateStbRsp(&ec, &stbRsp), NULL, _OVER); tEncoderClear(&ec); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 347f38193f..6f7b24eab2 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -367,6 +367,7 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, _OVER) if (pTrans->paramLen != 0) { pTrans->param = taosMemoryMalloc(pTrans->paramLen); + if (pTrans->param == NULL) goto _OVER; SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, _OVER); } @@ -902,6 +903,14 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { } } + if (pNew->conflict == TRN_CONFLICT_TSMA) { + if (pTrans->conflict == TRN_CONFLICT_GLOBAL || pTrans->conflict == TRN_CONFLICT_TSMA) { + mndTransLogConflict(pNew, pTrans, true, &conflict); + } else { + mndTransLogConflict(pNew, pTrans, false, &conflict); + } + } + sdbRelease(pMnode->pSdb, pTrans); } @@ -1299,7 +1308,7 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi } else { pAction->errCode = (terrno != 0) ? terrno : code; mError("trans:%d, %s:%d failed to write sdb since %s, type:%s status:%s", pTrans->id, mndTransStr(pAction->stage), - pAction->id, terrstr(), sdbTableName(pAction->pRaw->type), sdbStatusName(pAction->pRaw->status)); + pAction->id, tstrerror(code), sdbTableName(pAction->pRaw->type), sdbStatusName(pAction->pRaw->status)); mndSetTransLastAction(pTrans, pAction); } @@ -1519,7 +1528,13 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr } mndSetTransLastAction(pTrans, pAction); - if (mndCannotExecuteTransAction(pMnode, topHalf)) break; + if (mndCannotExecuteTransAction(pMnode, topHalf)) { + pTrans->lastErrorNo = code; + pTrans->code = code; + mInfo("trans:%d, %s:%d, topHalf:%d, not execute next action, code:%s", pTrans->id, mndTransStr(pAction->stage), + action, topHalf, tstrerror(code)); + break; + } if (code == 0) { pTrans->code = 0; @@ -1617,7 +1632,20 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool code = mndTransExecuteRedoActions(pMnode, pTrans, topHalf); } - if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; + if (mndCannotExecuteTransAction(pMnode, topHalf)) { + pTrans->lastErrorNo = code; + pTrans->code = code; + bool continueExec = true; + if (code != 0 && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { + continueExec = true; + } else { + continueExec = false; + } + mInfo("trans:%d, cannot execute redo action stage, topHalf:%d, continueExec:%d, code:%s", pTrans->id, topHalf, + continueExec, tstrerror(code)); + + return continueExec; + } terrno = code; if (code == 0) { @@ -1834,13 +1862,13 @@ void mndTransExecuteImp(SMnode *pMnode, STrans *pTrans, bool topHalf) { // start trans, pullup, receive rsp, kill void mndTransExecute(SMnode *pMnode, STrans *pTrans) { bool topHalf = true; - return mndTransExecuteImp(pMnode, pTrans, topHalf); + mndTransExecuteImp(pMnode, pTrans, topHalf); } // update trans void mndTransRefresh(SMnode *pMnode, STrans *pTrans) { bool topHalf = false; - return mndTransExecuteImp(pMnode, pTrans, topHalf); + mndTransExecuteImp(pMnode, pTrans, topHalf); } static int32_t mndProcessTransTimer(SRpcMsg *pReq) { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 2fe1ef4cfb..5a79ac6bc8 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#include "mndVgroup.h" #include "audit.h" #include "mndArbGroup.h" #include "mndDb.h" @@ -26,7 +27,6 @@ #include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" -#include "mndVgroup.h" #include "tmisce.h" #define VGROUP_VER_NUMBER 1 @@ -1670,7 +1670,9 @@ int32_t mndAddNewVgPrepareAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVg) { } TAOS_CHECK_GOTO(mndTransAppendPrepareLog(pTrans, pRaw), NULL, _err); - (void)sdbSetRawStatus(pRaw, SDB_STATUS_CREATING); + if (sdbSetRawStatus(pRaw, SDB_STATUS_CREATING) != 0) { + mError("vgId:%d, failed to set raw status at line:%d", pVg->vgId, __LINE__); + } if (code != 0) { mError("vgId:%d, failed to set raw status since %s at line:%d", pVg->vgId, tstrerror(code), __LINE__); TAOS_RETURN(code); diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index b83554c6f9..ea44a7c549 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -162,13 +162,13 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * pRow->status = pRaw->status; sdbPrintOper(pSdb, pRow, "insert"); - if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { + int32_t code = 0; + if ((code = taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *))) != 0) { sdbUnLock(pSdb, type); sdbFreeRow(pSdb, pRow, false); - return terrno; + return code; } - int32_t code = 0; SdbInsertFp insertFp = pSdb->insertFps[pRow->type]; if (insertFp != NULL) { code = (*insertFp)(pSdb, pRow->pObj); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 7fc079677b..d61f3d80d3 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -38,7 +38,7 @@ int32_t sndBuildStreamTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProce streamTaskOpenAllUpstreamInput(pTask); streamTaskResetUpstreamStageInfo(pTask); - (void)streamSetupScheduleTrigger(pTask); + streamSetupScheduleTrigger(pTask); SCheckpointInfo *pChkInfo = &pTask->chkInfo; tqSetRestoreVersionInfo(pTask); @@ -93,14 +93,18 @@ FAIL: } int32_t sndInit(SSnode *pSnode) { - (void)streamTaskSchedTask(&pSnode->msgCb, pSnode->pMeta->vgId, 0, 0, STREAM_EXEC_T_START_ALL_TASKS); + if (streamTaskSchedTask(&pSnode->msgCb, pSnode->pMeta->vgId, 0, 0, STREAM_EXEC_T_START_ALL_TASKS) != 0) { + sndError("failed to start all tasks"); + } return 0; } void sndClose(SSnode *pSnode) { stopRsync(); streamMetaNotifyClose(pSnode->pMeta); - (void)streamMetaCommit(pSnode->pMeta); + if (streamMetaCommit(pSnode->pMeta) != 0) { + sndError("failed to commit stream meta"); + } streamMetaClose(pSnode->pMeta); taosMemoryFree(pSnode); } diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 91aa513aa6..36068d1447 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -40,7 +40,7 @@ typedef struct SMetaStbStatsEntry { } SMetaStbStatsEntry; typedef struct STagFilterResEntry { - SList list; // the linked list of md5 digest, extracted from the serialized tag query condition + SHashObj *set; // the set of md5 digest, extracted from the serialized tag query condition uint32_t hitTimes; // queried times for current super table } STagFilterResEntry; @@ -112,7 +112,7 @@ static void statsCacheClose(SMeta* pMeta) { static void freeCacheEntryFp(void* param) { STagFilterResEntry** p = param; - tdListEmpty(&(*p)->list); + taosHashCleanup((*p)->set); taosMemoryFreeClear(*p); } @@ -200,10 +200,12 @@ void metaCacheClose(SMeta* pMeta) { entryCacheClose(pMeta); statsCacheClose(pMeta); + taosHashClear(pMeta->pCache->sTagFilterResCache.pTableEntry); taosLRUCacheCleanup(pMeta->pCache->sTagFilterResCache.pUidResCache); (void)taosThreadMutexDestroy(&pMeta->pCache->sTagFilterResCache.lock); taosHashCleanup(pMeta->pCache->sTagFilterResCache.pTableEntry); + taosHashClear(pMeta->pCache->STbGroupResCache.pTableEntry); taosLRUCacheCleanup(pMeta->pCache->STbGroupResCache.pResCache); (void)taosThreadMutexDestroy(&pMeta->pCache->STbGroupResCache.lock); taosHashCleanup(pMeta->pCache->STbGroupResCache.pTableEntry); @@ -471,34 +473,6 @@ int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo) { return code; } -static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInvalidRes, int32_t keyLen, - SLRUCache* pCache, uint64_t suid) { - SListIter iter = {0}; - tdListInitIter((SList*)&(pEntry->list), &iter, TD_LIST_FORWARD); - - SListNode* pNode = NULL; - uint64_t buf[3]; - buf[0] = suid; - - int32_t len = sizeof(uint64_t) * tListLen(buf); - - while ((pNode = tdListNext(&iter)) != NULL) { - memcpy(&buf[1], pNode->data, keyLen); - - // check whether it is existed in LRU cache, and remove it from linked list if not. - LRUHandle* pRes = taosLRUCacheLookup(pCache, buf, len); - if (pRes == NULL) { // remove the item in the linked list - if (taosArrayPush(pInvalidRes, &pNode) == NULL) { - return terrno; - } - } else { - bool ret = taosLRUCacheRelease(pCache, pRes, false); - } - } - - return 0; -} - static FORCE_INLINE void setMD5DigestInKey(uint64_t* pBuf, const char* key, int32_t keyLen) { memcpy(&pBuf[2], key, keyLen); } @@ -584,22 +558,11 @@ static void freeUidCachePayload(const void* key, size_t keyLen, void* value, voi if (pEntry != NULL && (*pEntry) != NULL) { int64_t st = taosGetTimestampUs(); - - SListIter iter = {0}; - tdListInitIter((SList*)&((*pEntry)->list), &iter, TD_LIST_FORWARD); - - SListNode* pNode = NULL; - while ((pNode = tdListNext(&iter)) != NULL) { - uint64_t* digest = (uint64_t*)pNode->data; - if (digest[0] == p[2] && digest[1] == p[3]) { - void* tmp = tdListPopNode(&((*pEntry)->list), pNode); - taosMemoryFree(tmp); - - double el = (taosGetTimestampUs() - st) / 1000.0; - metaInfo("clear items in meta-cache, remain cached item:%d, elapsed time:%.2fms", listNEles(&((*pEntry)->list)), - el); - break; - } + int32_t code = taosHashRemove((*pEntry)->set, &p[2], sizeof(uint64_t) * 2); + if (code == TSDB_CODE_SUCCESS) { + double el = (taosGetTimestampUs() - st) / 1000.0; + metaInfo("clear items in meta-cache, remain cached item:%d, elapsed time:%.2fms", taosHashGetSize((*pEntry)->set), + el); } } @@ -607,16 +570,30 @@ static void freeUidCachePayload(const void* key, size_t keyLen, void* value, voi } static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyLen, uint64_t suid) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; STagFilterResEntry* p = taosMemoryMalloc(sizeof(STagFilterResEntry)); - if (p == NULL) { - return terrno; - } + TSDB_CHECK_NULL(p, code, lino, _end, terrno); p->hitTimes = 0; - tdListInit(&p->list, keyLen); - TAOS_CHECK_RETURN(taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES)); - TAOS_CHECK_RETURN(tdListAppend(&p->list, pKey)); - return 0; + p->set = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + TSDB_CHECK_NULL(p->set, code, lino, _end, terrno); + code = taosHashPut(p->set, pKey, keyLen, NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); + code = taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + metaError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + if (p != NULL) { + if (p->set != NULL) { + taosHashCleanup(p->set); + } + taosMemoryFree(p); + } + } + return code; } // check both the payload size and selectivity ratio @@ -657,25 +634,14 @@ int32_t metaUidFilterCachePut(void* pVnode, uint64_t suid, const void* pKey, int goto _end; } } else { // check if it exists or not - size_t size = listNEles(&(*pEntry)->list); - if (size == 0) { - code = tdListAppend(&(*pEntry)->list, pKey); - if (code) { - goto _end; - } - } else { - SListNode* pNode = listHead(&(*pEntry)->list); - uint64_t* p = (uint64_t*)pNode->data; - if (p[1] == ((uint64_t*)pKey)[1] && p[0] == ((uint64_t*)pKey)[0]) { - // we have already found the existed items, no need to added to cache anymore. - (void)taosThreadMutexUnlock(pLock); - return TSDB_CODE_SUCCESS; - } else { // not equal, append it - code = tdListAppend(&(*pEntry)->list, pKey); - if (code) { - goto _end; - } - } + code = taosHashPut((*pEntry)->set, pKey, keyLen, NULL, 0); + if (code == TSDB_CODE_DUP_KEY) { + // we have already found the existed items, no need to added to cache anymore. + (void)taosThreadMutexUnlock(pLock); + return TSDB_CODE_SUCCESS; + } + if (code != TSDB_CODE_SUCCESS) { + goto _end; } } @@ -703,23 +669,20 @@ int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) { (void)taosThreadMutexLock(pLock); STagFilterResEntry** pEntry = taosHashGet(pEntryHashMap, &suid, sizeof(uint64_t)); - if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) { + if (pEntry == NULL || taosHashGetSize((*pEntry)->set) == 0) { (void)taosThreadMutexUnlock(pLock); return TSDB_CODE_SUCCESS; } (*pEntry)->hitTimes = 0; - SListIter iter = {0}; - tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD); - - SListNode* pNode = NULL; - while ((pNode = tdListNext(&iter)) != NULL) { - setMD5DigestInKey(p, pNode->data, 2 * sizeof(uint64_t)); + char *iter = taosHashIterate((*pEntry)->set, NULL); + while (iter != NULL) { + setMD5DigestInKey(p, iter, 2 * sizeof(uint64_t)); taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, TAG_FILTER_RES_KEY_LEN); + iter = taosHashIterate((*pEntry)->set, iter); } - - tdListEmpty(&(*pEntry)->list); + taosHashClear((*pEntry)->set); (void)taosThreadMutexUnlock(pLock); metaDebug("vgId:%d suid:%" PRId64 " cached related tag filter uid list cleared", vgId, suid); @@ -789,22 +752,11 @@ static void freeTbGroupCachePayload(const void* key, size_t keyLen, void* value, if (pEntry != NULL && (*pEntry) != NULL) { int64_t st = taosGetTimestampUs(); - - SListIter iter = {0}; - tdListInitIter((SList*)&((*pEntry)->list), &iter, TD_LIST_FORWARD); - - SListNode* pNode = NULL; - while ((pNode = tdListNext(&iter)) != NULL) { - uint64_t* digest = (uint64_t*)pNode->data; - if (digest[0] == p[2] && digest[1] == p[3]) { - void* tmp = tdListPopNode(&((*pEntry)->list), pNode); - taosMemoryFree(tmp); - - double el = (taosGetTimestampUs() - st) / 1000.0; - metaDebug("clear one item in tb group cache, remain cached item:%d, elapsed time:%.2fms", - listNEles(&((*pEntry)->list)), el); - break; - } + int32_t code = taosHashRemove((*pEntry)->set, &p[2], sizeof(uint64_t) * 2); + if (code == TSDB_CODE_SUCCESS) { + double el = (taosGetTimestampUs() - st) / 1000.0; + metaDebug("clear one item in tb group cache, remain cached item:%d, elapsed time:%.2fms", + taosHashGetSize((*pEntry)->set), el); } } @@ -840,25 +792,14 @@ int32_t metaPutTbGroupToCache(void* pVnode, uint64_t suid, const void* pKey, int goto _end; } } else { // check if it exists or not - size_t size = listNEles(&(*pEntry)->list); - if (size == 0) { - code = tdListAppend(&(*pEntry)->list, pKey); - if (code) { - goto _end; - } - } else { - SListNode* pNode = listHead(&(*pEntry)->list); - uint64_t* p = (uint64_t*)pNode->data; - if (p[1] == ((uint64_t*)pKey)[1] && p[0] == ((uint64_t*)pKey)[0]) { - // we have already found the existed items, no need to added to cache anymore. - (void)taosThreadMutexUnlock(pLock); - return TSDB_CODE_SUCCESS; - } else { // not equal, append it - code = tdListAppend(&(*pEntry)->list, pKey); - if (code) { - goto _end; - } - } + code = taosHashPut((*pEntry)->set, pKey, keyLen, NULL, 0); + if (code == TSDB_CODE_DUP_KEY) { + // we have already found the existed items, no need to added to cache anymore. + (void)taosThreadMutexUnlock(pLock); + return TSDB_CODE_SUCCESS; + } + if (code != TSDB_CODE_SUCCESS) { + goto _end; } } @@ -886,23 +827,20 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) { (void)taosThreadMutexLock(pLock); STagFilterResEntry** pEntry = taosHashGet(pEntryHashMap, &suid, sizeof(uint64_t)); - if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) { + if (pEntry == NULL || taosHashGetSize((*pEntry)->set) == 0) { (void)taosThreadMutexUnlock(pLock); return TSDB_CODE_SUCCESS; } (*pEntry)->hitTimes = 0; - SListIter iter = {0}; - tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD); - - SListNode* pNode = NULL; - while ((pNode = tdListNext(&iter)) != NULL) { - setMD5DigestInKey(p, pNode->data, 2 * sizeof(uint64_t)); + char *iter = taosHashIterate((*pEntry)->set, NULL); + while (iter != NULL) { + setMD5DigestInKey(p, iter, 2 * sizeof(uint64_t)); taosLRUCacheErase(pMeta->pCache->STbGroupResCache.pResCache, p, TAG_FILTER_RES_KEY_LEN); + iter = taosHashIterate((*pEntry)->set, iter); } - - tdListEmpty(&(*pEntry)->list); + taosHashClear((*pEntry)->set); (void)taosThreadMutexUnlock(pLock); metaDebug("vgId:%d suid:%" PRId64 " cached related tb group cleared", vgId, suid); diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index efa248755b..45912b5ae1 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -211,7 +211,10 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS SRSmaStat *pRSmaStat = (SRSmaStat *)(*pSmaStat); pRSmaStat->pSma = (SSma *)pSma; atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INIT); - (void)tsem_init(&pRSmaStat->notEmpty, 0, 0); + if (tsem_init(&pRSmaStat->notEmpty, 0, 0) != 0) { + code = terrno; + TAOS_CHECK_GOTO(code, &lino, _exit); + } if (!(pRSmaStat->blocks = taosArrayInit(1, sizeof(SSDataBlock)))) { code = terrno; TAOS_CHECK_GOTO(code, &lino, _exit); @@ -295,7 +298,10 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { taosHashCleanup(RSMA_INFO_HASH(pStat)); // step 5: free pStat - (void)tsem_destroy(&(pStat->notEmpty)); + if (tsem_destroy(&(pStat->notEmpty)) != 0) { + smaError("vgId:%d, failed to destroy notEmpty semaphore for rsma stat:%p since %s", SMA_VID(pSma), pRSmaStat, + tstrerror(terrno)); + } taosArrayDestroy(pStat->blocks); taosMemoryFreeClear(pStat); } @@ -399,7 +405,7 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { void *tdRSmaExecutorFunc(void *param) { setThreadName("vnode-rsma"); - if(tdRSmaProcessExecImpl((SSma *)param, RSMA_EXEC_OVERFLOW) < 0){ + if (tdRSmaProcessExecImpl((SSma *)param, RSMA_EXEC_OVERFLOW) < 0) { smaError("vgId:%d, failed to process rsma exec", SMA_VID((SSma *)param)); } return NULL; @@ -444,7 +450,9 @@ static int32_t tdRsmaStopExecutor(const SSma *pSma) { pthread = (TdThread *)&pStat->data; for (int32_t i = 0; i < tsNumOfVnodeRsmaThreads; ++i) { - (void)tsem_post(&(pRSmaStat->notEmpty)); + if (tsem_post(&(pRSmaStat->notEmpty)) != 0) { + smaError("vgId:%d, failed to post notEmpty semaphore for rsma since %s", SMA_VID(pSma), tstrerror(terrno)); + } } for (int32_t i = 0; i < tsNumOfVnodeRsmaThreads; ++i) { diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ee8ee962e9..4fdf299e50 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -1707,7 +1707,9 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { break; } - (void)tsem_wait(&pRSmaStat->notEmpty); + if (tsem_wait(&pRSmaStat->notEmpty) != 0) { + smaError("vgId:%d, failed to wait for not empty since %s", TD_VID(pVnode), tstrerror(terrno)); + } if ((pEnv->flag & SMA_ENV_FLG_CLOSE) && (atomic_load_64(&pRSmaStat->nBufItems) <= 0)) { smaDebug("vgId:%d, exec task end, flag:%" PRIi8 ", nBufItems:%" PRIi64, SMA_VID(pSma), pEnv->flag, diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 6da705423a..e3382cde32 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1474,6 +1474,9 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo TAOS_CHECK_RETURN(metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema)); ctxArray = taosArrayInit(pBlockData->nColData, sizeof(SLastUpdateCtx)); + if (ctxArray == NULL) { + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } // 1. prepare last STsdbRowKey tsdbRowKey = {0}; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 57f6aa3592..95c5daf842 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -331,7 +331,9 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { TAOS_CHECK_GOTO(tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, &committer->ctx->did), &lino, _exit); - TAOS_UNUSED(tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did)); + if (tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did) != 0) { + tsdbError("vgId:%d failed to create directory %s", TD_VID(committer->tsdb->pVnode), committer->tsdb->path); + } committer->ctx->tbid->suid = 0; committer->ctx->tbid->uid = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index 5a2f9628fb..720ba68414 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -1362,7 +1362,7 @@ int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAl }; for (int i = 0; i < TOMB_BLOCK_SIZE(tombBlock); i++) { STombRecord record; - TAOS_UNUSED(tTombBlockGet(tombBlock, i, &record)); + TAOS_CHECK_RETURN(tTombBlockGet(tombBlock, i, &record)); if (i == 0) { tombBlk.minTbid.suid = record.suid; @@ -1519,7 +1519,7 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom while (writer->ctx->hasOldTomb) { for (; writer->ctx->tombBlockIdx < TOMB_BLOCK_SIZE(writer->ctx->tombBlock); writer->ctx->tombBlockIdx++) { STombRecord record1[1]; - TAOS_UNUSED(tTombBlockGet(writer->ctx->tombBlock, writer->ctx->tombBlockIdx, record1)); + TAOS_CHECK_GOTO(tTombBlockGet(writer->ctx->tombBlock, writer->ctx->tombBlockIdx, record1), &lino, _exit); int32_t c = tTombRecordCompare(record, record1); if (c < 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbIter.c b/source/dnode/vnode/src/tsdb/tsdbIter.c index 4c5a803f22..3ed3f72333 100644 --- a/source/dnode/vnode/src/tsdb/tsdbIter.c +++ b/source/dnode/vnode/src/tsdb/tsdbIter.c @@ -225,7 +225,7 @@ static int32_t tsdbMemTableIterNext(STsdbIter *iter, const TABLEID *tbid) { iter->row->row = row[0]; - TAOS_UNUSED(tsdbTbDataIterNext(iter->memtData->tbIter)); + bool r = tsdbTbDataIterNext(iter->memtData->tbIter); goto _exit; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 8b4cdf9367..eb22335311 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -193,7 +193,11 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pMemTable->minVer = TMIN(pMemTable->minVer, version); pMemTable->maxVer = TMAX(pMemTable->maxVer, version); - TAOS_UNUSED(tsdbCacheDel(pTsdb, suid, uid, sKey, eKey)); + if (tsdbCacheDel(pTsdb, suid, uid, sKey, eKey) != 0) { + tsdbError("vgId:%d, failed to delete cache data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 + " eKey:%" PRId64 " at version %" PRId64, + TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, version); + } tsdbTrace("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 " at version %" PRId64, @@ -652,7 +656,10 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, } if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) { - TAOS_UNUSED(tsdbCacheColFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, pBlockData)); + if (tsdbCacheColFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, pBlockData) != 0) { + tsdbError("vgId:%d, failed to update cache data from table suid:%" PRId64 " uid:%" PRId64 " at version %" PRId64, + TD_VID(pMemTable->pTsdb->pVnode), pTbData->suid, pTbData->uid, version); + } } // SMemTable diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 85ed223c6c..3e4208fc54 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -403,7 +403,8 @@ static int32_t loadSttStatisticsBlockData(SSttFileReader *pSttFileReader, SSttBl pBlockLoadInfo->cost.loadStatisBlocks += num; STbStatisBlock block; - TAOS_UNUSED(tStatisBlockInit(&block)); + code = tStatisBlockInit(&block); + QUERY_CHECK_CODE(code, lino, _end); int64_t st = taosGetTimestampUs(); diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index af42a0e592..cbe2ab4b8e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -691,7 +691,7 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) { if (/*lcn < 1 && */ taosCheckExistFile(fobj->fname)) { int32_t mtime = 0; int64_t size = 0; - (void)taosStatFile(fobj->fname, &size, &mtime, NULL); + int32_t r = taosStatFile(fobj->fname, &size, &mtime, NULL); if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) { if (pCfg->s3Compact && lcn < 0) { extern int32_t tsdbAsyncCompact(STsdb * tsdb, const STimeWindow *tw, bool sync); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 94ca8d96a1..d0ea58c28a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -821,7 +821,9 @@ static int32_t tsdbSnapWriteFileSetBegin(STsdbSnapWriter* writer, int32_t fid) { code = TSDB_CODE_NO_AVAIL_DISK; TSDB_CHECK_CODE(code, lino, _exit); } - TAOS_UNUSED(tfsMkdirRecurAt(writer->tsdb->pVnode->pTfs, writer->tsdb->path, writer->ctx->did)); + if (tfsMkdirRecurAt(writer->tsdb->pVnode->pTfs, writer->tsdb->path, writer->ctx->did) != 0) { + tsdbError("vgId:%d failed to create directory %s", TD_VID(writer->tsdb->pVnode), writer->tsdb->path); + } writer->ctx->hasData = true; writer->ctx->hasTomb = true; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 5b8a062361..00806885ef 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -106,7 +106,7 @@ _exit: #endif void tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32_t (*tGetItemFn)(uint8_t *, void *)) { - TAOS_UNUSED(tGetItemFn(pMapData->pData + pMapData->aOffset[idx], pItem)); + int32_t r = tGetItemFn(pMapData->pData + pMapData->aOffset[idx], pItem); } #ifdef BUILD_NO_CALL diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index b857cdeb42..0d04486925 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -452,7 +452,8 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC // open wal sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); - (void)taosRealPath(tdir, NULL, sizeof(tdir)); + ret = taosRealPath(tdir, NULL, sizeof(tdir)); + TAOS_UNUSED(ret); pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg)); if (pVnode->pWal == NULL) { @@ -462,7 +463,8 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC // open tq sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); - (void)taosRealPath(tdir, NULL, sizeof(tdir)); + ret = taosRealPath(tdir, NULL, sizeof(tdir)); + TAOS_UNUSED(ret); // open query if (vnodeQueryOpen(pVnode)) { @@ -544,7 +546,9 @@ void vnodeClose(SVnode *pVnode) { vnodeCloseBufPool(pVnode); // destroy handle - (void)tsem_destroy(&pVnode->syncSem); + if (tsem_destroy(&pVnode->syncSem) != 0) { + vError("vgId:%d, failed to destroy semaphore", TD_VID(pVnode)); + } (void)taosThreadCondDestroy(&pVnode->poolNotEmpty); (void)taosThreadMutexDestroy(&pVnode->mutex); (void)taosThreadMutexDestroy(&pVnode->lock); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 9dbf16cb48..371eaa0774 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -210,7 +210,9 @@ static int32_t vnodePreProcessDropTtlMsg(SVnode *pVnode, SRpcMsg *pMsg) { TSDB_CHECK_CODE(code, lino, _exit); } - (void)tSerializeSVDropTtlTableReq((char *)pContNew + sizeof(SMsgHead), reqLenNew, &ttlReq); + if (tSerializeSVDropTtlTableReq((char *)pContNew + sizeof(SMsgHead), reqLenNew, &ttlReq) != 0) { + vError("vgId:%d %s:%d failed to serialize drop ttl request", TD_VID(pVnode), __func__, lino); + } pContNew->contLen = htonl(reqLenNew); pContNew->vgId = pContOld->vgId; @@ -420,7 +422,9 @@ static int32_t vnodePreProcessDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) { ((SMsgHead *)pCont)->vgId = TD_VID(pVnode); tEncoderInit(pCoder, pCont + sizeof(SMsgHead), size); - (void)tEncodeDeleteRes(pCoder, &res); + if (tEncodeDeleteRes(pCoder, &res) != 0) { + vError("vgId:%d %s failed to encode delete response", TD_VID(pVnode), __func__); + } tEncoderClear(pCoder); rpcFreeCont(pMsg->pCont); @@ -647,7 +651,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg } break; case TDMT_STREAM_CONSEN_CHKPT: { if (pVnode->restored) { - (void)tqProcessTaskConsenChkptIdReq(pVnode->pTq, pMsg); + if (tqProcessTaskConsenChkptIdReq(pVnode->pTq, pMsg) < 0) { + goto _err; + } } } break; case TDMT_STREAM_TASK_PAUSE: { @@ -664,7 +670,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg } break; case TDMT_VND_STREAM_TASK_RESET: { if (pVnode->restored && vnodeIsLeader(pVnode)) { - (void)tqProcessTaskResetReq(pVnode->pTq, pMsg); + if (tqProcessTaskResetReq(pVnode->pTq, pMsg) < 0) { + goto _err; + } } } break; case TDMT_VND_ALTER_CONFIRM: @@ -1215,7 +1223,9 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; SName name = {0}; - (void)tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); + if (tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB) < 0) { + vError("vgId:%d, failed to get name from string", TD_VID(pVnode)); + } SStringBuilder sb = {0}; for (int32_t i = 0; i < tbNames->size; i++) { @@ -1942,7 +1952,9 @@ _exit: tEncodeSize(tEncodeSSubmitRsp2, pSubmitRsp, pRsp->contLen, ret); pRsp->pCont = rpcMallocCont(pRsp->contLen); tEncoderInit(&ec, pRsp->pCont, pRsp->contLen); - (void)tEncodeSSubmitRsp2(&ec, pSubmitRsp); + if (tEncodeSSubmitRsp2(&ec, pSubmitRsp) < 0) { + vError("vgId:%d, failed to encode submit response", TD_VID(pVnode)); + } tEncoderClear(&ec); // update statistics @@ -2223,7 +2235,10 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pRe SBatchDeleteReq deleteReq; SDecoder decoder; tDecoderInit(&decoder, pReq, len); - (void)tDecodeSBatchDeleteReq(&decoder, &deleteReq); + if (tDecodeSBatchDeleteReq(&decoder, &deleteReq) < 0) { + tDecoderClear(&decoder); + return terrno = TSDB_CODE_INVALID_MSG; + } SMetaReader mr = {0}; metaReaderDoInit(&mr, pVnode->pMeta, META_READER_NOLOCK); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 0882db52a6..e757163ba8 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -334,6 +334,7 @@ typedef struct SCtgViewCache { typedef struct SCtgTSMACache { SRWLatch tsmaLock; SArray* pTsmas; // SArray + bool retryFetch; } SCtgTSMACache; typedef struct SCtgDBCache { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 80650a6095..eafd85a504 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2997,6 +2997,7 @@ int32_t ctgOpDropTbTSMA(SCtgCacheOperation *operation) { taosArrayDestroyP(pCtgCache->pTsmas, tFreeAndClearTableTSMAInfo); pCtgCache->pTsmas = NULL; + pCtgCache->retryFetch = true; ctgDebug("all tsmas for table dropped: %s.%s", msg->dbFName, msg->tbName); code = taosHashRemove(dbCache->tsmaCache, msg->tbName, TSDB_TABLE_NAME_LEN); @@ -3975,17 +3976,25 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx // get tsma cache pCache = taosHashAcquire(dbCache->tsmaCache, tsmaSourceTbName.tname, strlen(tsmaSourceTbName.tname)); - if (!pCache || !pCache->pTsmas || pCache->pTsmas->size == 0) { + if (!pCache) { if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { ctgReleaseTSMAToCache(pCtg, dbCache, pCache); CTG_ERR_RET(terrno); } - + continue; + } + CTG_LOCK(CTG_READ, &pCache->tsmaLock); + if ((!pCache->pTsmas || pCache->pTsmas->size == 0) && !pCache->retryFetch) { + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + ctgReleaseTSMAToCache(pCtg, dbCache, pCache); + CTG_ERR_RET(terrno); + } + CTG_UNLOCK(CTG_READ, &pCache->tsmaLock); + taosHashRelease(dbCache->tsmaCache, pCache); continue; } - CTG_LOCK(CTG_READ, &pCache->tsmaLock); - if (hasOutOfDateTSMACache(pCache->pTsmas)) { + if (pCache->retryFetch || hasOutOfDateTSMACache(pCache->pTsmas)) { CTG_UNLOCK(CTG_READ, &pCache->tsmaLock); taosHashRelease(dbCache->tsmaCache, pCache); @@ -3997,6 +4006,7 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx } CTG_CACHE_NHIT_INC(CTG_CI_TBL_TSMA, 1); + pCache->retryFetch = false; continue; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index e935b43c00..04b3a83264 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2211,10 +2211,8 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi return terrno; } pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t) * pCond->numOfCols); - if (pCond->colList == NULL || pCond->pSlotList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + if (pCond->pSlotList == NULL) { taosMemoryFreeClear(pCond->colList); - taosMemoryFreeClear(pCond->pSlotList); return terrno; } diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index f22205786f..2e08f486ff 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -315,6 +315,10 @@ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { } mtq->opera = opera; mtq->query = taosArrayInit(4, sizeof(SIndexTermQuery)); + if (mtq->query == NULL) { + taosMemoryFree(mtq); + return NULL; + } return mtq; } void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) { @@ -359,10 +363,20 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy len = idxConvertDataToStr((void*)colVal, IDX_TYPE_GET_TYPE(colType), (void**)&buf); } else if (colVal == NULL) { buf = taosStrndup(INDEX_DATA_NULL_STR, (int32_t)strlen(INDEX_DATA_NULL_STR)); + if (buf == NULL) { + taosMemoryFree(tm->colName); + taosMemoryFree(tm); + return NULL; + } len = (int32_t)strlen(INDEX_DATA_NULL_STR); } else { static const char* emptyStr = " "; buf = taosStrndup(emptyStr, (int32_t)strlen(emptyStr)); + if (buf == NULL) { + taosMemoryFree(tm->colName); + taosMemoryFree(tm); + return NULL; + } len = (int32_t)strlen(emptyStr); } diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index eff2922119..a710bb6c10 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -384,7 +384,7 @@ static IterateValue* idxCacheIteratorGetValue(Iterate* iter); IndexCache* idxCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) { IndexCache* cache = taosMemoryCalloc(1, sizeof(IndexCache)); if (cache == NULL) { - indexError("failed to create index cache"); + indexError("failed to create index cache since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); return NULL; }; @@ -392,6 +392,11 @@ IndexCache* idxCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8 cache->mem->pCache = cache; cache->colName = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? taosStrdup(JSON_COLUMN) : taosStrdup(colName); + if (cache->colName == NULL) { + taosMemoryFree(cache); + indexError("failed to create index cache since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return NULL; + } cache->type = type; cache->index = idx; cache->version = 0; @@ -831,6 +836,7 @@ static MemTable* idxInternalCacheCreate(int8_t type) { IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? idxCacheJsonTermCompare : idxCacheTermCompare; MemTable* tbl = taosMemoryCalloc(1, sizeof(MemTable)); + if (tbl == NULL) return NULL; idxMemRef(tbl); // if (ttype == TSDB_DATA_TYPE_BINARY || ttype == TSDB_DATA_TYPE_NCHAR || ttype == TSDB_DATA_TYPE_GEOMETRY) { tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, ttype, MAX_INDEX_KEY_LEN, cmpFn, SL_ALLOW_DUP_KEY, idxCacheTermGet); diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index ae6309f292..b065f0c91e 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -389,6 +389,9 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { break; case TSDB_DATA_TYPE_USMALLINT: *dst = taosMemoryCalloc(1, bufSize + 1); + if (*dst == NULL) { + return terrno; + } TAOS_UNUSED(idxInt2str(*(uint16_t*)src, *dst, -1)); tlen = strlen(*dst); break; diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index 4f1cc61719..8dafc05255 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -111,9 +111,6 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output if (un->last != NULL) return; - // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); - // trn->inp = s->data[s->start]; - // trn->out = out; int32_t len = 0; uint8_t* data = fstSliceData(s, &len); un->last = fstLastTransitionCreate(data[0], out); @@ -126,10 +123,11 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output n->isFinal = false; n->finalOutput = 0; n->trans = taosArrayInit(16, sizeof(FstTransition)); + if (n->trans == NULL) { + taosMemoryFree(n); + return; + } - // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); - // trn->inp = s->data[i]; - // trn->out = out; FstLastTransition* trn = fstLastTransitionCreate(data[i], 0); FstBuilderNodeUnfinished un = {.node = n, .last = trn}; @@ -1159,11 +1157,20 @@ FStmSt* stmStCreate(Fst* fst, FAutoCtx* automation, FstBoundWithData* min, FstBo sws->fst = fst; sws->aut = automation; sws->inp = (SArray*)taosArrayInit(256, sizeof(uint8_t)); + if (sws->inp == NULL) { + taosMemoryFree(sws); + return NULL; + } sws->emptyOutput.null = true; sws->emptyOutput.out = 0; sws->stack = (SArray*)taosArrayInit(256, sizeof(FstStreamState)); + if (sws->stack == NULL) { + taosArrayDestroy(sws->inp); + taosMemoryFree(sws); + return NULL; + } sws->endAt = max; TAOS_UNUSED(stmStSeekMin(sws, min)); diff --git a/source/libs/index/src/indexFstAutomation.c b/source/libs/index/src/indexFstAutomation.c index d0ea30997e..2b1f3395eb 100644 --- a/source/libs/index/src/indexFstAutomation.c +++ b/source/libs/index/src/indexFstAutomation.c @@ -174,7 +174,18 @@ FAutoCtx* automCtxCreate(void* data, AutomationType atype) { // add more search type } - ctx->data = (data != NULL ? taosStrdup((char*)data) : NULL); + // ctx->data = (data != NULL ? taosStrdup((char*)data) : NULL); + if (data != NULL) { + ctx->data = taosStrdup((char*)data); + if (ctx->data == NULL) { + startWithStateValueDestroy(sv); + taosMemoryFree(ctx); + return NULL; + } + } else { + ctx->data = NULL; + } + ctx->type = atype; ctx->stdata = (void*)sv; return ctx; diff --git a/source/libs/index/src/indexFstRegister.c b/source/libs/index/src/indexFstRegister.c index ce34bb50d0..701203e980 100644 --- a/source/libs/index/src/indexFstRegister.c +++ b/source/libs/index/src/indexFstRegister.c @@ -120,6 +120,8 @@ FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNo uint64_t end = start + registry->mruSize; FstRegistryEntry* entry = taosMemoryMalloc(sizeof(FstRegistryEntry)); + if (entry == NULL) return NULL; + if (end - start == 1) { FstRegistryCell* cell = taosArrayGet(registry->table, start); // cell->isNode && diff --git a/source/libs/index/src/indexFstSparse.c b/source/libs/index/src/indexFstSparse.c index 8746b04eab..6fd97af044 100644 --- a/source/libs/index/src/indexFstSparse.c +++ b/source/libs/index/src/indexFstSparse.c @@ -28,6 +28,12 @@ FstSparseSet *sparSetCreate(int32_t sz) { ss->dense = (int32_t *)taosMemoryMalloc(sz * sizeof(int32_t)); ss->sparse = (int32_t *)taosMemoryMalloc(sz * sizeof(int32_t)); + if (ss->dense == NULL || ss->sparse == NULL) { + taosMemoryFree(ss->dense); + taosMemoryFree(ss->sparse); + taosMemoryFree(ss); + return NULL; + } sparSetInitBuf(ss->dense, sz); sparSetInitBuf(ss->sparse, sz); diff --git a/source/libs/index/src/indexFstUtil.c b/source/libs/index/src/indexFstUtil.c index 8c1095fb1d..8173196860 100644 --- a/source/libs/index/src/indexFstUtil.c +++ b/source/libs/index/src/indexFstUtil.c @@ -77,10 +77,13 @@ CompiledAddr unpackDelta(char* data, uint64_t len, uint64_t nodeAddr) { FstSlice fstSliceCreate(uint8_t* data, uint64_t len) { FstString* str = (FstString*)taosMemoryMalloc(sizeof(FstString)); + if (str == NULL) { + return (FstSlice){.str = NULL, .start = 0, .end = 0}; + } str->ref = 1; str->len = len; str->data = taosMemoryMalloc(len * sizeof(uint8_t)); - if (str == NULL || str->data == NULL) { + if (str->data == NULL) { taosMemoryFree(str); return (FstSlice){.str = NULL, .start = 0, .end = 0}; } @@ -107,9 +110,16 @@ FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) { uint8_t* data = fstSliceData(s, &slen); uint8_t* buf = taosMemoryMalloc(sizeof(uint8_t) * tlen); + if (buf == NULL) { + return (FstSlice){.str = NULL, .start = 0, .end = 0}; + } memcpy(buf, data + start, tlen); FstString* str = taosMemoryMalloc(sizeof(FstString)); + if (str == NULL) { + taosMemoryFree(buf); + return (FstSlice){.str = NULL, .start = 0, .end = 0}; + } str->data = buf; str->len = tlen; str->ref = 1; diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c index 1e358ac8d4..a2d03bbd6a 100644 --- a/source/libs/monitor/src/monFramework.c +++ b/source/libs/monitor/src/monFramework.c @@ -100,9 +100,7 @@ extern char* tsMonFwUri; #define VNODE_ROLE "taosd_vnodes_info:role" void monInitMonitorFW(){ - if (taos_collector_registry_default_init() != 0) { - uError("failed to init default collector registry"); - } + taos_collector_registry_default_init(); tsMonitor.metrics = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); taos_gauge_t *gauge = NULL; diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c index bfdbf92156..53d115fbe9 100644 --- a/source/libs/monitorfw/src/taos_collector_registry.c +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -66,14 +66,14 @@ taos_collector_registry_t *taos_collector_registry_new(const char *name) { return self; } -int taos_collector_registry_default_init(void) { - if (TAOS_COLLECTOR_REGISTRY_DEFAULT != NULL) return 0; +void taos_collector_registry_default_init(void) { + if (TAOS_COLLECTOR_REGISTRY_DEFAULT != NULL) return; TAOS_COLLECTOR_REGISTRY_DEFAULT = taos_collector_registry_new("default"); //if (TAOS_COLLECTOR_REGISTRY_DEFAULT) { // return taos_collector_registry_enable_process_metrics(TAOS_COLLECTOR_REGISTRY_DEFAULT); //} - return 1; + return; } int taos_collector_registry_destroy(taos_collector_registry_t *self) { diff --git a/source/libs/monitorfw/src/taos_metric_formatter_custom.c b/source/libs/monitorfw/src/taos_metric_formatter_custom.c index 6e7ded62bb..5c384c0421 100644 --- a/source/libs/monitorfw/src/taos_metric_formatter_custom.c +++ b/source/libs/monitorfw/src/taos_metric_formatter_custom.c @@ -40,16 +40,26 @@ int taos_metric_formatter_load_sample_new(taos_metric_formatter_t *self, taos_me int32_t len = end -start; char* keyvalues = taosMemoryMalloc(len); + if (keyvalues == NULL) return 1; memset(keyvalues, 0, len); memcpy(keyvalues, start + 1, len - 1); int32_t count = taos_monitor_count_occurrences(keyvalues, ","); char** keyvalue = taosMemoryMalloc(sizeof(char*) * (count + 1)); + if (keyvalue == NULL) { + taosMemoryFreeClear(keyvalues); + return 1; + } memset(keyvalue, 0, sizeof(char*) * (count + 1)); taos_monitor_split_str(keyvalue, keyvalues, ","); char** arr = taosMemoryMalloc(sizeof(char*) * (count + 1) * 2); + if (arr == NULL) { + taosMemoryFreeClear(keyvalue); + taosMemoryFreeClear(keyvalues); + return 1; + } memset(arr, 0, sizeof(char*) * (count + 1) * 2); bool isfound = true; @@ -165,6 +175,7 @@ int taos_metric_formatter_load_metric_new(taos_metric_formatter_t *self, taos_me int32_t size = strlen(metric->name); char* name = taosMemoryMalloc(size + 1); + if (name == NULL) return 1; memset(name, 0, size + 1); memcpy(name, metric->name, size); char* arr[2] = {0}; //arr[0] is table name, arr[1] is metric name diff --git a/source/libs/monitorfw/src/taos_monitor_util.c b/source/libs/monitorfw/src/taos_monitor_util.c index 06ae4993c5..b0eff27507 100644 --- a/source/libs/monitorfw/src/taos_monitor_util.c +++ b/source/libs/monitorfw/src/taos_monitor_util.c @@ -37,6 +37,7 @@ void taos_monitor_split_str(char** arr, char* str, const char* del) { void taos_monitor_split_str_metric(char** arr, taos_metric_t* metric, const char* del, char** buf) { int32_t size = strlen(metric->name); char* name = taosMemoryMalloc(size + 1); + if (name == NULL) return; memset(name, 0, size + 1); memcpy(name, metric->name, size); diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 927bc6b661..4f30cb12b7 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -204,19 +204,23 @@ static EDealRes walkExprs(SNodeList* pNodeList, ETraversalOrder order, FNodeWalk } void nodesWalkExpr(SNode* pNode, FNodeWalker walker, void* pContext) { - (void)walkExpr(pNode, TRAVERSAL_PREORDER, walker, pContext); + EDealRes res; + res = walkExpr(pNode, TRAVERSAL_PREORDER, walker, pContext); } void nodesWalkExprs(SNodeList* pNodeList, FNodeWalker walker, void* pContext) { - (void)walkExprs(pNodeList, TRAVERSAL_PREORDER, walker, pContext); + EDealRes res; + res = walkExprs(pNodeList, TRAVERSAL_PREORDER, walker, pContext); } void nodesWalkExprPostOrder(SNode* pNode, FNodeWalker walker, void* pContext) { - (void)walkExpr(pNode, TRAVERSAL_POSTORDER, walker, pContext); + EDealRes res; + res = walkExpr(pNode, TRAVERSAL_POSTORDER, walker, pContext); } void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) { - (void)walkExprs(pList, TRAVERSAL_POSTORDER, walker, pContext); + EDealRes res; + res = walkExprs(pList, TRAVERSAL_POSTORDER, walker, pContext); } static void checkParamIsFunc(SFunctionNode* pFunc) { @@ -382,7 +386,7 @@ static EDealRes rewriteExpr(SNode** pRawNode, ETraversalOrder order, FNodeRewrit res = rewriteExpr(&pWin->pEndOffset, order, rewriter, pContext); } break; - } + } case QUERY_NODE_COUNT_WINDOW: { SCountWindowNode* pEvent = (SCountWindowNode*)pNode; res = rewriteExpr(&pEvent->pCol, order, rewriter, pContext); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index e08868d7c1..7180c58404 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1428,6 +1428,7 @@ void qWorkerDestroy(void **qWorkerMgmt) { return; } + qInfo("wait for destroyed"); while (0 == destroyed) { taosMsleep(2); } diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index d55bf19c87..7a8be34781 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1178,106 +1178,6 @@ _exception: return code; } -#ifdef BUILD_NO_CALL -static int32_t chkpIdComp(const void* a, const void* b) { - int64_t x = *(int64_t*)a; - int64_t y = *(int64_t*)b; - return x < y ? -1 : 1; -} - -int32_t streamBackendLoadCheckpointInfo(void* arg) { - SStreamMeta* pMeta = arg; - int32_t code = 0; - SArray* suffix = NULL; - - int32_t len = strlen(pMeta->path) + 30; - char* chkpPath = taosMemoryCalloc(1, len); - sprintf(chkpPath, "%s%s%s", pMeta->path, TD_DIRSEP, "checkpoints"); - - if (!taosDirExist(chkpPath)) { - // no checkpoint, nothing to load - taosMemoryFree(chkpPath); - return 0; - } - - TdDirPtr pDir = taosOpenDir(chkpPath); - if (pDir == NULL) { - taosMemoryFree(chkpPath); - return 0; - } - - TdDirEntryPtr de = NULL; - suffix = taosArrayInit(4, sizeof(int64_t)); - - while ((de = taosReadDir(pDir)) != NULL) { - if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; - - if (taosDirEntryIsDir(de)) { - char checkpointPrefix[32] = {0}; - int64_t checkpointId = 0; - - int ret = sscanf(taosGetDirEntryName(de), "checkpoint%" PRId64 "", &checkpointId); - if (ret == 1) { - taosArrayPush(suffix, &checkpointId); - } - } else { - continue; - } - } - taosArraySort(suffix, chkpIdComp); - // free previous chkpSaved - taosArrayClear(pMeta->chkpSaved); - for (int i = 0; i < taosArrayGetSize(suffix); i++) { - int64_t id = *(int64_t*)taosArrayGet(suffix, i); - taosArrayPush(pMeta->chkpSaved, &id); - } - - taosArrayDestroy(suffix); - taosCloseDir(&pDir); - taosMemoryFree(chkpPath); - return 0; -} -#endif - -#ifdef BUILD_NO_CALL -int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t*** ppHandle, SArray* refs) { - return 0; - // SArray* pHandle = taosArrayInit(16, POINTER_BYTES); - // void* pIter = taosHashIterate(pMeta->pTaskDbUnique, NULL); - // while (pIter) { - // int64_t id = *(int64_t*)pIter; - - // SBackendCfWrapper* wrapper = taosAcquireRef(streamBackendCfWrapperId, id); - // if (wrapper == NULL) { - // pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter); - // continue; - // } - - // taosThreadRwlockRdlock(&wrapper->rwLock); - // for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { - // if (wrapper->pHandle[i]) { - // rocksdb_column_family_handle_t* p = wrapper->pHandle[i]; - // taosArrayPush(pHandle, &p); - // } - // } - // taosThreadRwlockUnlock(&wrapper->rwLock); - - // taosArrayPush(refs, &id); - // } - - // int32_t nCf = taosArrayGetSize(pHandle); - - // rocksdb_column_family_handle_t** ppCf = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); - // for (int i = 0; i < nCf; i++) { - // ppCf[i] = taosArrayGetP(pHandle, i); - // } - // taosArrayDestroy(pHandle); - - // *ppHandle = ppCf; - // return nCf; -} -#endif - int chkpIdComp(const void* a, const void* b) { int64_t x = *(int64_t*)a; int64_t y = *(int64_t*)b; @@ -1288,6 +1188,9 @@ int chkpIdComp(const void* a, const void* b) { int32_t taskDbLoadChkpInfo(STaskDbWrapper* pBackend) { int32_t code = 0; char* pChkpDir = taosMemoryCalloc(1, 256); + if (pChkpDir == NULL) { + return terrno; + } sprintf(pChkpDir, "%s%s%s", pBackend->path, TD_DIRSEP, "checkpoints"); if (!taosIsDir(pChkpDir)) { @@ -2376,8 +2279,14 @@ int32_t taskDbOpenCfs(STaskDbWrapper* pTask, char* path, char** pCfNames, int32_ int32_t code = -1; char* err = NULL; - rocksdb_options_t** cfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); + rocksdb_options_t** cfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); + if (cfOpts == NULL) { + return terrno; + } rocksdb_column_family_handle_t** cfHandle = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); + if (cfHandle == NULL) { + return terrno; + } for (int i = 0; i < nCf; i++) { int32_t idx = getCfIdx(pCfNames[i]); @@ -2461,6 +2370,14 @@ void taskDbInitOpt(STaskDbWrapper* pTaskDb) { pTaskDb->pCfParams = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam)); pTaskDb->pCfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); pTaskDb->pCompares = taosMemoryCalloc(nCf, sizeof(rocksdb_comparator_t*)); + if (pTaskDb->pCf == NULL || pTaskDb->pCfParams == NULL || pTaskDb->pCfOpts == NULL || pTaskDb->pCompares == NULL) { + stError("failed to alloc memory for cf"); + taosMemoryFreeClear(pTaskDb->pCf); + taosMemoryFreeClear(pTaskDb->pCfParams); + taosMemoryFreeClear(pTaskDb->pCfOpts); + taosMemoryFreeClear(pTaskDb->pCompares); + return; + } for (int i = 0; i < nCf; i++) { rocksdb_options_t* opt = rocksdb_options_create_copy(pTaskDb->dbOpt); @@ -4166,7 +4083,7 @@ int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKe if (code == 0) { if (sessionRangeKeyCmpr(&searchKey, key) == 0) { - memcpy(tmp, *pVal, valSize); + memcpy(tmp, *pVal, *pVLen); taosMemoryFreeClear(*pVal); goto _end; } @@ -4182,7 +4099,7 @@ int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKe code = streamStateSessionGetKVByCur_rocksdb(pCur, key, pVal, pVLen); if (code == 0) { if (sessionRangeKeyCmpr(&searchKey, key) == 0) { - memcpy(tmp, *pVal, valSize); + memcpy(tmp, *pVal, *pVLen); goto _end; } } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index eb7254acba..41ff8f3c24 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -170,11 +170,12 @@ int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* int32_t lino = 0; void* pVal = NULL; int32_t len = getRowStateRowSize(pState->pFileState); - code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), &pVal, &len); + int32_t tmpLen = len; + code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), &pVal, &tmpLen); QUERY_CHECK_CODE(code, lino, _end); char* buf = ((SRowBuffPos*)pVal)->pRowBuff; - uint32_t rowSize = streamFileStateGetSelectRowSize(pState->pFileState); + int32_t rowSize = streamFileStateGetSelectRowSize(pState->pFileState); memcpy(buf + len - rowSize, value, vLen); _end: @@ -188,11 +189,12 @@ int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVa int32_t lino = 0; void* pVal = NULL; int32_t len = getRowStateRowSize(pState->pFileState); - code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), (void**)(&pVal), &len); + int32_t tmpLen = len; + code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), (void**)(&pVal), &tmpLen); QUERY_CHECK_CODE(code, lino, _end); char* buf = ((SRowBuffPos*)pVal)->pRowBuff; - uint32_t rowSize = streamFileStateGetSelectRowSize(pState->pFileState); + int32_t rowSize = streamFileStateGetSelectRowSize(pState->pFileState); *ppVal = buf + len - rowSize; streamStateReleaseBuf(pState, pVal, false); diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 9f8523b040..f7a88746ec 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -100,7 +100,7 @@ void* intervalCreateStateKey(SRowBuffPos* pPos, int64_t num) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); return NULL; } - SWinKey* pWinKey = pPos->pKey; + SWinKey* pWinKey = pPos->pKey; pStateKey->key = *pWinKey; pStateKey->opNum = num; return pStateKey; @@ -120,7 +120,7 @@ void* sessionCreateStateKey(SRowBuffPos* pPos, int64_t num) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); return NULL; } - SSessionKey* pWinKey = pPos->pKey; + SSessionKey* pWinKey = pPos->pKey; pStateKey->key = *pWinKey; pStateKey->opNum = num; return pStateKey; @@ -734,7 +734,7 @@ void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, boo // todo handle failure memset(buf, 0, len); } - taosMemoryFree(buf); + taosMemoryFreeClear(buf); int32_t numOfElems = streamStateGetBatchSize(batch); if (numOfElems > 0) { @@ -769,6 +769,7 @@ _end: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } + taosMemoryFree(buf); streamStateDestroyBatch(batch); } @@ -850,7 +851,7 @@ int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { if (winRes != TSDB_CODE_SUCCESS) { break; } - + if (vlen != pFileState->rowSize) { code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; QUERY_CHECK_CODE(code, lino, _end); diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 016ee5cf20..9f6acf6d83 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -728,6 +728,8 @@ int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTe sDebug("vgId:%d, get response info, seqNum:%" PRId64 ", num:%d", pNode->vgId, cbMeta.seqNum, num); code = pFsm->FpCommitCb(pFsm, &rpcMsg, &cbMeta); retry = (code != 0) && (terrno == TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE); + sDebug("vgId:%d, fsm execute, index:%" PRId64 ", term:%" PRId64 ", type:%s, code:%d, retry:%d", pNode->vgId, + pEntry->index, pEntry->term, TMSG_INFO(pEntry->originalRpcType), code, retry); if (retry) { taosMsleep(10); sError("vgId:%d, retry on fsm commit since %s. index:%" PRId64, pNode->vgId, tstrerror(code), pEntry->index); diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 3a90fae1ab..a4cfa69459 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -266,16 +266,23 @@ static int32_t httpCreateMsg(const char* server, const char* uri, uint16_t port, msg->server = taosStrdup(server); msg->uri = taosStrdup(uri); msg->cont = taosMemoryMalloc(contLen); - if (qid != NULL) - msg->qid = taosStrdup(qid); - else - msg->qid = NULL; if (msg->server == NULL || msg->uri == NULL || msg->cont == NULL) { httpDestroyMsg(msg); *httpMsg = NULL; return terrno; } + if (qid != NULL) { + msg->qid = taosStrdup(qid); + if (msg->qid == NULL) { + httpDestroyMsg(msg); + *httpMsg = NULL; + return terrno; + } + } else { + msg->qid = NULL; + } + memcpy(msg->cont, pCont, contLen); msg->len = contLen; msg->flag = flag; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index ea77dbf432..7b14e14799 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -493,6 +493,12 @@ void cliHandleResp(SCliConn* conn) { } if (CONN_NO_PERSIST_BY_APP(conn)) { + SExHandle* exh = transAcquireExHandle(transGetRefMgt(), refId); + if (exh != NULL) { + exh->handle = NULL; + } + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), refId)); + return addConnToPool(pThrd->pool, conn); } @@ -667,6 +673,10 @@ static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) { plist = taosHashGet(pool, key, klen); SMsgList* nList = taosMemoryCalloc(1, sizeof(SMsgList)); + if (nList == NULL) { + tError("failed to alloc memory for msg list, reason:%s", tstrerror(terrno)); + return NULL; + } QUEUE_INIT(&nList->msgQ); nList->numOfConn++; @@ -714,6 +724,11 @@ static SCliConn* getConnFromPool2(SCliThrd* pThrd, char* key, SCliMsg** pMsg) { plist = taosHashGet(pool, key, klen); SMsgList* nList = taosMemoryCalloc(1, sizeof(SMsgList)); + if (nList == NULL) { + tError("failed to alloc memory for msg list, reason:%s", tstrerror(terrno)); + return NULL; + } + QUEUE_INIT(&nList->msgQ); nList->numOfConn++; @@ -1059,6 +1074,7 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { } conn->list = NULL; + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), conn->refId)); TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), conn->refId)); TAOS_UNUSED(transRemoveExHandle(transGetRefMgt(), conn->refId)); conn->refId = -1; @@ -1087,6 +1103,7 @@ static void cliDestroy(uv_handle_t* handle) { TAOS_UNUSED(atomic_sub_fetch_32(&pThrd->connCount, 1)); if (conn->refId > 0) { + TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), conn->refId)); TAOS_UNUSED(transReleaseExHandle(transGetRefMgt(), conn->refId)); TAOS_UNUSED(transRemoveExHandle(transGetRefMgt(), conn->refId)); } @@ -1295,11 +1312,16 @@ void cliSend(SCliConn* pConn) { uv_timer_t* timer = taosArrayGetSize(pThrd->timerList) > 0 ? *(uv_timer_t**)taosArrayPop(pThrd->timerList) : NULL; if (timer == NULL) { timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); + if (timer == NULL) { + tError("failed to alloc timer since %s", tstrerror(terrno)); + } tDebug("no available timer, create a timer %p", timer); TAOS_UNUSED(uv_timer_init(pThrd->loop, timer)); } - timer->data = pConn; - pConn->timer = timer; + if (timer != NULL) { + timer->data = pConn; + pConn->timer = timer; + } tGTrace("%s conn %p start timer for msg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType)); TAOS_UNUSED(uv_timer_start((uv_timer_t*)pConn->timer, cliReadTimeoutCb, TRANS_READ_TIMEOUT, 0)); @@ -1319,6 +1341,11 @@ void cliSend(SCliConn* pConn) { uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); uv_write_t* req = transReqQueuePush(&pConn->wreqQueue); + if (req == NULL) { + tGError("%s conn %p failed to send msg:%s, errmsg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType), + tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + cliHandleExcept(pConn, -1); + } int status = uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); if (status != 0) { @@ -1863,6 +1890,10 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { TAOS_UNUSED(transQueuePush(&conn->cliMsgs, pMsg)); conn->dstAddr = taosStrdup(addr); + if (conn->dstAddr == NULL) { + tGError("%s conn %p failed to create socket, reason:%s", transLabel(pTransInst), conn, tstrerror(terrno)); + cliHandleExcept(conn, -1); + } uint32_t ipaddr; int32_t code = cliGetIpFromFqdnCache(pThrd->fqdn2ipCache, fqdn, &ipaddr); @@ -2538,6 +2569,10 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { cliSchedMsgToDebug(pMsg, transLabel(pThrd->pTransInst)); STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + if (arg == NULL) { + tError("failed to malloc memory, reason:%s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return; + } arg->param1 = pMsg; arg->param2 = pThrd; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 77f7765627..98b5d907a0 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -421,6 +421,9 @@ void transReqQueueInit(queue* q) { } void* transReqQueuePush(queue* q) { STransReq* req = taosMemoryCalloc(1, sizeof(STransReq)); + if (req == NULL) { + return NULL; + } req->wreq.data = req; QUEUE_PUSH(q, &req->q); return &req->wreq; diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 1f1cc7bb35..d8737ef30f 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -707,6 +707,10 @@ static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg) { transRefSrvHandle(pConn); uv_write_t* req = transReqQueuePush(&pConn->wreqQueue); + if (req == NULL) { + tError("failed to send resp since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return; + } TAOS_UNUSED(uv_write(req, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnSendCb)); } static void uvStartSendResp(SSvrMsg* smsg) { @@ -842,6 +846,10 @@ static bool uvRecvReleaseReq(SSvrConn* pConn, STransMsgHead* pHead) { STransMsg tmsg = {.code = 0, .info.handle = (void*)pConn, .info.traceId = traceId, .info.ahandle = (void*)0x9527}; SSvrMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSvrMsg)); + if (srvMsg == NULL) { + tError("failed to alloc buf to send release resp since %s", tstrerror(terrno)); + return true; + } srvMsg->msg = tmsg; srvMsg->type = Release; srvMsg->pConn = pConn; @@ -905,6 +913,11 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { #endif uv_write_t* wr = (uv_write_t*)taosMemoryMalloc(sizeof(uv_write_t)); + if (wr == NULL) { + tError("failed to accept since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return; + } + wr->data = cli; uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify)); @@ -1401,6 +1414,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, for (int i = 0; i < srv->numOfThreads; i++) { SWorkThrd* thrd = (SWorkThrd*)taosMemoryCalloc(1, sizeof(SWorkThrd)); + if (thrd == NULL) { + code = terrno; + goto End; + } thrd->pTransInst = shandle; thrd->quit = false; thrd->pTransInst = shandle; @@ -1654,6 +1671,10 @@ void destroyWorkThrd(SWorkThrd* pThrd) { } void sendQuitToWorkThrd(SWorkThrd* pThrd) { SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg)); + if (msg == NULL) { + tError("failed to send quit msg to work thread since %s", tstrerror(terrno)); + return; + } msg->type = Quit; tDebug("server send quit msg to work thread"); TAOS_UNUSED(transAsyncSend(pThrd->asyncPool, &msg->q)); diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 2065ba68fe..09a5579e13 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -55,7 +55,7 @@ typedef struct TdSocket { #endif int refId; SocketFd fd; -} *TdSocketPtr, TdSocket; +} * TdSocketPtr, TdSocket; typedef struct TdSocketServer { #if SOCKET_WITH_LOCK @@ -63,7 +63,7 @@ typedef struct TdSocketServer { #endif int refId; SocketFd fd; -} *TdSocketServerPtr, TdSocketServer; +} * TdSocketServerPtr, TdSocketServer; typedef struct TdEpoll { #if SOCKET_WITH_LOCK @@ -71,52 +71,7 @@ typedef struct TdEpoll { #endif int refId; EpollFd fd; -} *TdEpollPtr, TdEpoll; - -#if 0 -int32_t taosSendto(TdSocketPtr pSocket, void *buf, int len, unsigned int flags, const struct sockaddr *dest_addr, - int addrlen) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen); -#else - return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen); -#endif -} - -int32_t taosWriteSocket(TdSocketPtr pSocket, void *buf, int len) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - return send(pSocket->fd, buf, len, 0); - ; -#else - return write(pSocket->fd, buf, len); -#endif -} -int32_t taosReadSocket(TdSocketPtr pSocket, void *buf, int len) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - return recv(pSocket->fd, buf, len, 0); - ; -#else - return read(pSocket->fd, buf, len); -#endif -} - -int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, - int *addrLen) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } - return recvfrom(pSocket->fd, buf, len, flags, destAddr, addrLen); -} -#endif // endif 0 +} * TdEpollPtr, TdEpoll; int32_t taosCloseSocketNoCheck1(SocketFd fd) { #ifdef WINDOWS @@ -145,136 +100,16 @@ int32_t taosCloseSocket(TdSocketPtr *ppSocket) { code = taosCloseSocketNoCheck1((*ppSocket)->fd); (*ppSocket)->fd = -1; taosMemoryFree(*ppSocket); - + return code; } -#if 0 -int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer) { - int32_t code; - if (ppSocketServer == NULL || *ppSocketServer == NULL || (*ppSocketServer)->fd < 0) { - return -1; - } - code = taosCloseSocketNoCheck1((*ppSocketServer)->fd); - (*ppSocketServer)->fd = -1; - taosMemoryFree(*ppSocketServer); - return code; -} - -int32_t taosShutDownSocketRD(TdSocketPtr pSocket) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - return closesocket(pSocket->fd); -#elif __APPLE__ - return close(pSocket->fd); -#else - return shutdown(pSocket->fd, SHUT_RD); -#endif -} -int32_t taosShutDownSocketServerRD(TdSocketServerPtr pSocketServer) { - if (pSocketServer == NULL || pSocketServer->fd < 0) { - return -1; - } -#ifdef WINDOWS - return closesocket(pSocketServer->fd); -#elif __APPLE__ - return close(pSocketServer->fd); -#else - return shutdown(pSocketServer->fd, SHUT_RD); -#endif -} - -int32_t taosShutDownSocketWR(TdSocketPtr pSocket) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - return closesocket(pSocket->fd); -#elif __APPLE__ - return close(pSocket->fd); -#else - return shutdown(pSocket->fd, SHUT_WR); -#endif -} -int32_t taosShutDownSocketServerWR(TdSocketServerPtr pSocketServer) { - if (pSocketServer == NULL || pSocketServer->fd < 0) { - return -1; - } -#ifdef WINDOWS - return closesocket(pSocketServer->fd); -#elif __APPLE__ - return close(pSocketServer->fd); -#else - return shutdown(pSocketServer->fd, SHUT_WR); -#endif -} -int32_t taosShutDownSocketRDWR(TdSocketPtr pSocket) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - return closesocket(pSocket->fd); -#elif __APPLE__ - return close(pSocket->fd); -#else - return shutdown(pSocket->fd, SHUT_RDWR); -#endif -} -int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer) { - if (pSocketServer == NULL || pSocketServer->fd < 0) { - return -1; - } -#ifdef WINDOWS - return closesocket(pSocketServer->fd); -#elif __APPLE__ - return close(pSocketServer->fd); -#else - return shutdown(pSocketServer->fd, SHUT_RDWR); -#endif -} - -int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - u_long mode; - if (on) { - mode = 1; - ioctlsocket(pSocket->fd, FIONBIO, &mode); - } else { - mode = 0; - ioctlsocket(pSocket->fd, FIONBIO, &mode); - } -#else - int32_t flags = 0; - if ((flags = fcntl(pSocket->fd, F_GETFL, 0)) < 0) { - // printf("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno)); - return 1; - } - - if (on) - flags |= O_NONBLOCK; - else - flags &= ~O_NONBLOCK; - - if ((flags = fcntl(pSocket->fd, F_SETFL, flags)) < 0) { - // printf("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno)); - return 1; - } -#endif - return 0; -} -#endif // endif 0 - int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t optlen) { if (pSocket == NULL || pSocket->fd < 0) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - + #ifdef WINDOWS #ifdef TCP_KEEPCNT if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { @@ -300,11 +135,11 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void } #endif -int ret = setsockopt(pSocket->fd, level, optname, optval, optlen); -if (ret == SOCKET_ERROR) { - int errorCode = WSAGetLastError(); - return terrno = TAOS_SYSTEM_WINSOCKET_ERROR(errorCode); -} + int ret = setsockopt(pSocket->fd, level, optname, optval, optlen); + if (ret == SOCKET_ERROR) { + int errorCode = WSAGetLastError(); + return terrno = TAOS_SYSTEM_WINSOCKET_ERROR(errorCode); + } #else int32_t code = setsockopt(pSocket->fd, level, optname, optval, (int)optlen); if (-1 == code) { @@ -315,22 +150,8 @@ if (ret == SOCKET_ERROR) { #endif } -#if 0 -int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } -#ifdef WINDOWS - return -1; -#else - return getsockopt(pSocket->fd, level, optname, optval, (int *)optlen); -#endif -} - -#endif - const char *taosInetNtoa(struct in_addr ipInt, char *dstStr, int32_t len) { - const char* r = inet_ntop(AF_INET, &ipInt, dstStr, len); + const char *r = inet_ntop(AF_INET, &ipInt, dstStr, len); if (NULL == r) { terrno = TAOS_SYSTEM_ERROR(errno); } @@ -344,403 +165,6 @@ const char *taosInetNtoa(struct in_addr ipInt, char *dstStr, int32_t len) { #define TCP_CONN_TIMEOUT 3000 // conn timeout -#if 0 -int32_t taosWriteMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } - int32_t nleft, nwritten; - char *ptr = (char *)buf; - - nleft = nbytes; - - while (nleft > 0) { - nwritten = taosWriteSocket(pSocket, (char *)ptr, (size_t)nleft); - if (nwritten <= 0) { - if (errno == EINTR /* || errno == EAGAIN || errno == EWOULDBLOCK */) - continue; - else - return -1; - } else { - nleft -= nwritten; - ptr += nwritten; - } - - if (errno == SIGPIPE || errno == EPIPE) { - return -1; - } - } - - return (nbytes - nleft); -} - -int32_t taosReadMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } - int32_t nleft, nread; - char *ptr = (char *)buf; - - nleft = nbytes; - - while (nleft > 0) { - nread = taosReadSocket(pSocket, ptr, (size_t)nleft); - if (nread == 0) { - break; - } else if (nread < 0) { - if (errno == EINTR /* || errno == EAGAIN || errno == EWOULDBLOCK*/) { - continue; - } else { - return -1; - } - } else { - nleft -= nread; - ptr += nread; - } - - if (errno == SIGPIPE || errno == EPIPE) { - return -1; - } - } - - return (nbytes - nleft); -} - -int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } - taosSetNonblocking(pSocket, 1); - - int32_t nleft, nwritten, nready; - fd_set fset; - struct timeval tv; - - nleft = nbytes; - while (nleft > 0) { - tv.tv_sec = 30; - tv.tv_usec = 0; - FD_ZERO(&fset); - FD_SET(pSocket->fd, &fset); - if ((nready = select((SocketFd)(pSocket->fd + 1), NULL, &fset, NULL, &tv)) == 0) { - errno = ETIMEDOUT; - // printf("fd %d timeout, no enough space to write", fd); - break; - - } else if (nready < 0) { - if (errno == EINTR) continue; - - // printf("select error, %d (%s)", errno, strerror(errno)); - return -1; - } - - nwritten = (int32_t)send(pSocket->fd, ptr, (size_t)nleft, MSG_NOSIGNAL); - if (nwritten <= 0) { - if (errno == EAGAIN || errno == EINTR) continue; - - // printf("write error, %d (%s)", errno, strerror(errno)); - return -1; - } - - nleft -= nwritten; - ptr += nwritten; - } - - taosSetNonblocking(pSocket, 0); - - return (nbytes - nleft); -} - -TdSocketPtr taosOpenUdpSocket(uint32_t ip, uint16_t port) { - struct sockaddr_in localAddr; - SocketFd fd; - int32_t bufSize = 1024000; - - // printf("open udp socket:0x%x:%hu", ip, port); - - memset((char *)&localAddr, 0, sizeof(localAddr)); - localAddr.sin_family = AF_INET; - localAddr.sin_addr.s_addr = ip; - localAddr.sin_port = (uint16_t)htons(port); - - if ((fd = socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { - // printf("failed to open udp socket: %d (%s)", errno, strerror(errno)); - taosCloseSocketNoCheck1(fd); - return NULL; - } - - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(fd); - return NULL; - } - pSocket->fd = fd; - pSocket->refId = 0; - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - // printf("failed to set the send buffer size for UDP socket\n"); - taosCloseSocket(&pSocket); - return NULL; - } - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - // printf("failed to set the receive buffer size for UDP socket\n"); - taosCloseSocket(&pSocket); - return NULL; - } - - /* bind socket to local address */ - if (bind(pSocket->fd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) { - // printf("failed to bind udp socket: %d (%s), 0x%x:%hu", errno, strerror(errno), ip, port); - taosCloseSocket(&pSocket); - return NULL; - } - - return pSocket; -} - -TdSocketPtr taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { - SocketFd fd = -1; - int32_t ret; - struct sockaddr_in serverAddr, clientAddr; - int32_t bufSize = 1024 * 1024; - - fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - - if (fd <= 2) { - // printf("failed to open the socket: %d (%s)", errno, strerror(errno)); - if (fd >= 0) taosCloseSocketNoCheck1(fd); - return NULL; - } - - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(fd); - return NULL; - } - pSocket->fd = fd; - pSocket->refId = 0; - - /* set REUSEADDR option, so the portnumber can be re-used */ - int32_t reuse = 1; - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - // printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - // printf("failed to set the send buffer size for TCP socket\n"); - taosCloseSocket(&pSocket); - return NULL; - } - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - // printf("failed to set the receive buffer size for TCP socket\n"); - taosCloseSocket(&pSocket); - return NULL; - } - - if (clientIp != 0) { - memset((char *)&clientAddr, 0, sizeof(clientAddr)); - clientAddr.sin_family = AF_INET; - clientAddr.sin_addr.s_addr = clientIp; - clientAddr.sin_port = 0; - - /* bind socket to client address */ - if (bind(pSocket->fd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) { - // printf("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", clientIp, destIp, destPort, - // strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - } - - memset((char *)&serverAddr, 0, sizeof(serverAddr)); - serverAddr.sin_family = AF_INET; - serverAddr.sin_addr.s_addr = destIp; - serverAddr.sin_port = (uint16_t)htons((uint16_t)destPort); - -#ifdef _TD_LINUX - taosSetNonblocking(pSocket, 1); - ret = connect(pSocket->fd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); - if (ret == -1) { - if (errno == EHOSTUNREACH) { - // printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } else if (errno == EINPROGRESS || errno == EAGAIN || errno == EWOULDBLOCK) { - struct pollfd wfd[1]; - - wfd[0].fd = pSocket->fd; - wfd[0].events = POLLOUT; - - int res = poll(wfd, 1, TCP_CONN_TIMEOUT); - if (res == -1 || res == 0) { - // printf("failed to connect socket, ip:0x%x, port:%hu(poll error/conn timeout)", destIp, destPort); - taosCloseSocket(&pSocket); // - return -1; - } - int optVal = -1, optLen = sizeof(int); - if ((0 != taosGetSockOpt(pSocket, SOL_SOCKET, SO_ERROR, &optVal, &optLen)) || (optVal != 0)) { - // printf("failed to connect socket, ip:0x%x, port:%hu(connect host error)", destIp, destPort); - taosCloseSocket(&pSocket); // - return -1; - } - ret = 0; - } else { // Other error - // printf("failed to connect socket, ip:0x%x, port:%hu(target host cannot be reached)", destIp, destPort); - taosCloseSocket(&pSocket); // - return -1; - } - } - taosSetNonblocking(pSocket, 0); - -#else - ret = connect(pSocket->fd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); -#endif - - if (ret != 0) { - // printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } else { - if (taosKeepTcpAlive(pSocket) == -1) { - return NULL; - } - } - - return pSocket; -} - -int32_t taosKeepTcpAlive(TdSocketPtr pSocket) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } - int32_t alive = 1; - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) { - // printf("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - -#ifndef __APPLE__ - // all fails on macosx -#ifdef TCP_KEEPCNT - int32_t probes = 3; - if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { - // printf("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } -#endif - -#ifdef TCP_KEEPIDLE - int32_t alivetime = 10; - if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) { - // printf("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } -#endif - -#ifdef TCP_KEEPINTVL - int32_t interval = 3; - if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) { - // printf("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } -#endif -#endif // __APPLE__ - - int32_t nodelay = 1; - if (taosSetSockOpt(pSocket, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { - // printf("fd:%d setsockopt TCP_NODELAY failed %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - struct linger linger = {0}; - linger.l_onoff = 1; - linger.l_linger = 3; - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_LINGER, (void *)&linger, sizeof(linger)) < 0) { - // printf("setsockopt SO_LINGER failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - return 0; -} - -int taosGetLocalIp(const char *eth, char *ip) { -#if defined(WINDOWS) - // DO NOTHAING - return -1; -#else - int fd; - struct ifreq ifr; - struct sockaddr_in sin; - - fd = socket(AF_INET, SOCK_DGRAM, 0); - if (-1 == fd) { - return -1; - } - strncpy(ifr.ifr_name, eth, IFNAMSIZ); - ifr.ifr_name[IFNAMSIZ - 1] = 0; - - if (ioctl(fd, SIOCGIFADDR, &ifr) < 0) { - taosCloseSocketNoCheck1(fd); - return -1; - } - memcpy(&sin, &ifr.ifr_addr, sizeof(sin)); - taosInetNtoa(sin.sin_addr, ip, 64); - taosCloseSocketNoCheck1(fd); -#endif - return 0; -} -int taosValidIp(uint32_t ip) { -#if defined(WINDOWS) - // DO NOTHAING - return -1; -#else - int ret = -1; - int fd; - - struct ifconf ifconf; - - char buf[512] = {0}; - ifconf.ifc_len = 512; - ifconf.ifc_buf = buf; - - if ((fd = socket(AF_INET, SOCK_DGRAM, 0)) < 0) { - return -1; - } - - ioctl(fd, SIOCGIFCONF, &ifconf); - struct ifreq *ifreq = (struct ifreq *)ifconf.ifc_buf; - for (int i = (ifconf.ifc_len / sizeof(struct ifreq)); i > 0; i--) { - char ip_str[64] = {0}; - if (ifreq->ifr_flags == AF_INET) { - ret = taosGetLocalIp(ifreq->ifr_name, ip_str); - if (ret != 0) { - break; - } - ret = -1; - if (ip == (uint32_t)taosInetAddr(ip_str)) { - ret = 0; - break; - } - ifreq++; - } - } - taosCloseSocketNoCheck1(fd); - return ret; -#endif - return 0; -} -#endif // endif 0 - bool taosValidIpAndPort(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; SocketFd fd; @@ -778,138 +202,19 @@ bool taosValidIpAndPort(uint32_t ip, uint16_t port) { TAOS_SKIP_ERROR(taosCloseSocket(&pSocket)); return false; } - + /* bind socket to server address */ if (-1 == bind(pSocket->fd, (struct sockaddr *)&serverAdd, sizeof(serverAdd))) { terrno = TAOS_SYSTEM_ERROR(errno); TAOS_SKIP_ERROR(taosCloseSocket(&pSocket)); return false; } - + TAOS_SKIP_ERROR(taosCloseSocket(&pSocket)); - + return true; } -#if 0 -TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { - struct sockaddr_in serverAdd; - SocketFd fd; - int32_t reuse; - - // printf("open tcp server socket:0x%x:%hu", ip, port); - - bzero((char *)&serverAdd, sizeof(serverAdd)); - serverAdd.sin_family = AF_INET; - serverAdd.sin_addr.s_addr = ip; - serverAdd.sin_port = (uint16_t)htons(port); - - if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { - // printf("failed to open TCP socket: %d (%s)", errno, strerror(errno)); - taosCloseSocketNoCheck1(fd); - return NULL; - } - - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(fd); - return NULL; - } - pSocket->refId = 0; - pSocket->fd = fd; - - /* set REUSEADDR option, so the portnumber can be re-used */ - reuse = 1; - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - // printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - /* bind socket to server address */ - if (bind(pSocket->fd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) { - // printf("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - if (taosKeepTcpAlive(pSocket) < 0) { - // printf("failed to set tcp server keep-alive option, 0x%x:%hu(%s)", ip, port, strerror(errno)); - return NULL; - } - - if (listen(pSocket->fd, 1024) < 0) { - // printf("listen tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - return (TdSocketServerPtr)pSocket; -} - -TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen) { - if (pServerSocket == NULL || pServerSocket->fd < 0) { - return NULL; - } - SocketFd fd = accept(pServerSocket->fd, destAddr, addrLen); - if (fd == -1) { - // tError("TCP accept failure(%s)", strerror(errno)); - return NULL; - } - - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(fd); - return NULL; - } - pSocket->fd = fd; - pSocket->refId = 0; - return pSocket; -} -#define COPY_SIZE 32768 -// sendfile shall be used - -int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len) { - if (pSrcSocket == NULL || pSrcSocket->fd < 0 || pDestSocket == NULL || pDestSocket->fd < 0) { - return -1; - } - int64_t leftLen; - int64_t readLen, writeLen; - char temp[COPY_SIZE]; - - leftLen = len; - - while (leftLen > 0) { - if (leftLen < COPY_SIZE) - readLen = leftLen; - else - readLen = COPY_SIZE; // 4K - - int64_t retLen = taosReadMsg(pSrcSocket, temp, (int32_t)readLen); - if (readLen != retLen) { - // printf("read error, readLen:%" PRId64 " retLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", - // readLen, retLen, len, leftLen, strerror(errno)); - return -1; - } - - writeLen = taosWriteMsg(pDestSocket, temp, (int32_t)readLen); - - if (readLen != writeLen) { - // printf("copy error, readLen:%" PRId64 " writeLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", - // readLen, writeLen, len, leftLen, strerror(errno)); - return -1; - } - - leftLen -= readLen; - } - - return len; -} - -// Function converting an IP address string to an uint32_t. - -#endif // endif 0 - int32_t taosBlockSIGPIPE() { #ifdef WINDOWS return 0; @@ -1029,7 +334,7 @@ int32_t taosGetFqdn(char *fqdn) { // hints.ai_family = AF_INET; strcpy(fqdn, hostname); strcpy(fqdn + strlen(hostname), ".local"); -#else // linux +#else // linux #endif // linux @@ -1048,7 +353,7 @@ int32_t taosGetFqdn(char *fqdn) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - + terrno = TAOS_SYSTEM_ERROR(ret); return terrno; } @@ -1067,7 +372,7 @@ int32_t taosGetFqdn(char *fqdn) { int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - //fprintf(stderr, "failed to get fqdn, code:%d, hostname:%s, reason:%s\n", ret, hostname, gai_strerror(ret)); + // fprintf(stderr, "failed to get fqdn, code:%d, hostname:%s, reason:%s\n", ret, hostname, gai_strerror(ret)); return TAOS_SYSTEM_WINSOCKET_ERROR(WSAGetLastError()); } strcpy(fqdn, result->ai_canonname); @@ -1082,41 +387,16 @@ void tinet_ntoa(char *ipstr, uint32_t ip) { (void)sprintf(ipstr, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24); } -int32_t taosIgnSIGPIPE() { - sighandler_t h = signal(SIGPIPE, SIG_IGN); +int32_t taosIgnSIGPIPE() { + sighandler_t h = signal(SIGPIPE, SIG_IGN); if (SIG_ERR == h) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - return 0; + return 0; } -#if 0 - -int32_t taosSetMaskSIGPIPE() { -#ifdef WINDOWS - return -1; -#else - sigset_t signal_mask; - (void)sigemptyset(&signal_mask); - (void)sigaddset(&signal_mask, SIGPIPE); - - int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL); - if (rc != 0) { - // printf("failed to setmask SIGPIPE"); - } -#endif -} - -int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *addrLen) { - if (pSocket == NULL || pSocket->fd < 0) { - return -1; - } - return getsockname(pSocket->fd, destAddr, addrLen); -} -#endif // endif 0 - /* * Set TCP connection timeout per-socket level. * ref [https://github.com/libuv/help/issues/54] @@ -1132,7 +412,7 @@ int32_t taosCreateSocketWithTimeout(uint32_t timeout) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - + #if defined(WINDOWS) if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&timeout, sizeof(timeout))) { taosCloseSocketNoCheck1(fd); diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 59505dc0c1..7989a2468b 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -297,7 +297,7 @@ void taosArrayRemove(SArray* pArray, size_t index) { } if (index == pArray->size - 1) { - (void)taosArrayPop(pArray); + void* t = taosArrayPop(pArray); return; } diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 81e50f6a6b..d5001f0264 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -173,7 +173,7 @@ TdThread doRegisterCacheObj(SCacheObj *pCacheObj) { (void)taosThreadOnce(&cacheThreadInit, doInitRefreshThread); (void)taosThreadMutexLock(&guard); - if (taosArrayPush(pCacheArrayList, &pCacheObj) != 0) { + if (taosArrayPush(pCacheArrayList, &pCacheObj) == NULL) { uError("failed to add cache object into array, reason:%s", strerror(errno)); (void)taosThreadMutexUnlock(&guard); return cacheRefreshWorker; @@ -404,7 +404,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLi return NULL; } - (void)doRegisterCacheObj(pCacheObj); + TdThread refreshWorker = doRegisterCacheObj(pCacheObj); return pCacheObj; } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 2ced73a3f0..288d440d86 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -825,9 +825,9 @@ int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelement return nelements * longBytes; } else if (input[0] == 1) { // Decompress if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) { - (void)tsDecompressTimestampAvx512(input, nelements, output, false); + tsDecompressTimestampAvx512(input, nelements, output, false); } else if (tsSIMDEnable && tsAVX2Supported) { - (void)tsDecompressTimestampAvx2(input, nelements, output, false); + tsDecompressTimestampAvx2(input, nelements, output, false); } else { int64_t *ostream = (int64_t *)output; @@ -1201,9 +1201,9 @@ int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, c } if (tsSIMDEnable && tsAVX2Supported) { - (void)tsDecompressFloatImplAvx2(input, nelements, output); + tsDecompressFloatImplAvx2(input, nelements, output); } else if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) { - (void)tsDecompressFloatImplAvx512(input, nelements, output); + tsDecompressFloatImplAvx512(input, nelements, output); } else { // alternative implementation without SIMD instructions. tsDecompressFloatHelper(input, nelements, (float *)output); } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index ee55243415..d84a426e98 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -480,7 +480,7 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p } break; case CFG_DTYPE_FLOAT: case CFG_DTYPE_DOUBLE: { - float dval = 0; + float dval = 0; int32_t code = parseCfgReal(pVal, &dval); if (code != TSDB_CODE_SUCCESS) { cfgUnLock(pCfg); @@ -911,7 +911,9 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { strncpy(line, *pEnv, sizeof(line) - 1); pEnv++; - (void)taosEnvToCfg(line, line); + if (taosEnvToCfg(line, line) < 0) { + uTrace("failed to convert env to cfg:%s", line); + } (void)paGetToken(line, &name, &olen); if (olen == 0) continue; @@ -954,7 +956,9 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { while (envCmd[index] != NULL) { strncpy(buf, envCmd[index], sizeof(buf) - 1); buf[sizeof(buf) - 1] = 0; - (void)taosEnvToCfg(buf, buf); + if (taosEnvToCfg(buf, buf) < 0) { + uTrace("failed to convert env to cfg:%s", buf); + } index++; name = value = value2 = value3 = value4 = NULL; @@ -1026,7 +1030,9 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { break; } if (line[_bytes - 1] == '\n') line[_bytes - 1] = 0; - (void)taosEnvToCfg(line, line); + if (taosEnvToCfg(line, line) < 0) { + uTrace("failed to convert env to cfg:%s", line); + } (void)paGetToken(line, &name, &olen); if (olen == 0) continue; @@ -1119,7 +1125,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { code = cfgSetItem(pConfig, name, newValue, CFG_STYPE_CFG_FILE, true); if (TSDB_CODE_SUCCESS != code && TSDB_CODE_CFG_NOT_FOUND != code) { - (void)printf("cfg:%s, value:%s failed since %s\n", name,newValue, tstrerror(code)); + (void)printf("cfg:%s, value:%s failed since %s\n", name, newValue, tstrerror(code)); break; } } else { @@ -1260,12 +1266,12 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { TAOS_CHECK_EXIT(terrno); } size_t fileSize = taosLSeekFile(pFile, 0, SEEK_END); - if(fileSize <= 0) { + if (fileSize <= 0) { (void)taosCloseFile(&pFile); (void)printf("load json file error: %s\n", filepath); TAOS_CHECK_EXIT(terrno); } - char *buf = taosMemoryMalloc(fileSize + 1); + char *buf = taosMemoryMalloc(fileSize + 1); if (!buf) { (void)taosCloseFile(&pFile); (void)printf("load json file error: %s, failed to alloc memory\n", filepath); @@ -1273,7 +1279,12 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { } buf[fileSize] = 0; - (void)taosLSeekFile(pFile, 0, SEEK_SET); + if (taosLSeekFile(pFile, 0, SEEK_SET) < 0) { + (void)taosCloseFile(&pFile); + (void)printf("load json file error: %s\n", filepath); + taosMemoryFreeClear(buf); + TAOS_RETURN(terrno); + } if (taosReadFile(pFile, buf, fileSize) <= 0) { (void)taosCloseFile(&pFile); (void)printf("load json file error: %s\n", filepath); diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c index a27d2efbcc..598aca730f 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompress.c @@ -313,23 +313,22 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, return nelements * word_length; } -int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output) { +void tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output) { #if __AVX512F__ // todo add it #endif - return 0; + return; } // todo add later -int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output) { +void tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output) { #if __AVX2__ #endif - return 0; + return; } // decode two timestamps in one loop. -int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, - bool bigEndian) { +void tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, bool bigEndian) { int64_t *ostream = (int64_t *)output; int32_t ipos = 1, opos = 0; @@ -466,11 +465,11 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen } } #endif - return 0; + return; } -int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, - bool UNUSED_PARAM(bigEndian)) { +void tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, + bool UNUSED_PARAM(bigEndian)) { int64_t *ostream = (int64_t *)output; int32_t ipos = 1, opos = 0; @@ -581,5 +580,5 @@ int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelem } #endif - return 0; + return; } diff --git a/source/util/src/tdigest.c b/source/util/src/tdigest.c index 636c97dbaa..7ec5531b96 100644 --- a/source/util/src/tdigest.c +++ b/source/util/src/tdigest.c @@ -252,7 +252,9 @@ double tdigestQuantile(TDigest *t, double q) { int64_t weight_so_far; SCentroid *a, *b, tmp; - (void)tdigestCompress(t); + if (tdigestCompress(t) != 0) { + uError("failed to compress t-digest"); + } if (t->num_centroids == 0) return NAN; if (t->num_centroids == 1) return t->centroids[0].mean; if (FLOAT_EQ(q, 0.0)) return t->min; diff --git a/source/util/src/thash.c b/source/util/src/thash.c index ab88bef1a0..88fe6618b9 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -837,8 +837,8 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p) { // only add the read lock to disable the resize process taosHashRLock(pHashObj); - int slot; - (void)taosHashReleaseNode(pHashObj, p, &slot); + int slot; + void *tp = taosHashReleaseNode(pHashObj, p, &slot); SHashEntry *pe = pHashObj->hashList[slot]; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 274edeaa90..6a25c30704 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -541,7 +541,6 @@ static void processLogFileName(const char* logName , int32_t maxFileNum){ } static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { - int32_t code = 0; #ifdef WINDOWS_STASH /* * always set maxFileNum to 1 @@ -576,10 +575,10 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { } tsLogObj.lines = (int32_t)(filesize / 60); - if ((code = taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END)) < 0) { - TAOS_UNUSED(printf("failed to seek to the end of log file:%s, reason:%s\n", name, tstrerror(code))); + if (taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END) < 0) { + TAOS_UNUSED(printf("failed to seek to the end of log file:%s, reason:%s\n", name, tstrerror(terrno))); taosUnLockLogFile(tsLogObj.logHandle->pFile); - return code; + return terrno; } (void)sprintf(name, "==================================================\n"); diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 1686b41038..8c708ac6b5 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -178,7 +178,6 @@ void *taosProcessSchedQueue(void *scheduler) { (*(msg.tfp))(msg.ahandle, msg.thandle); } - return NULL; } @@ -230,7 +229,9 @@ void taosCleanUpScheduler(void *param) { for (int32_t i = 0; i < pSched->numOfThreads; ++i) { if (taosCheckPthreadValid(pSched->qthread[i])) { - (void)tsem_post(&pSched->fullSem); + if (tsem_post(&pSched->fullSem) != 0) { + uError("post %s fullSem failed(%s)", pSched->label, strerror(terrno)); + } } } for (int32_t i = 0; i < pSched->numOfThreads; ++i) { @@ -240,12 +241,17 @@ void taosCleanUpScheduler(void *param) { } } - (void)tsem_destroy(&pSched->emptySem); - (void)tsem_destroy(&pSched->fullSem); + if (tsem_destroy(&pSched->emptySem) != 0) { + uError("failed to destroy %s emptySem", pSched->label); + } + if (tsem_destroy(&pSched->fullSem) != 0) { + uError("failed to destroy %s fullSem", pSched->label); + } (void)taosThreadMutexDestroy(&pSched->queueMutex); if (pSched->pTimer) { - (void)taosTmrStop(pSched->pTimer); + bool r = taosTmrStop(pSched->pTimer); + uTrace("stop timer:%p, result:%d", pSched->pTimer, r); pSched->pTimer = NULL; } diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index d5310b2e7e..b11f65da8f 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -314,7 +314,9 @@ static void addToExpired(tmr_obj_t* head) { schedMsg.msg = NULL; schedMsg.ahandle = head; schedMsg.thandle = NULL; - (void)taosScheduleTask(tmrQhandle, &schedMsg); + if (taosScheduleTask(tmrQhandle, &schedMsg) != 0) { + tmrError("%s failed to add expired timer[id=%" PRIuPTR "] to queue.", head->ctrl->label, id); + } tmrDebug("timer[id=%" PRIuPTR "] has been added to queue.", id); head = next; @@ -508,7 +510,7 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* han } } - if (timer->refCount == 1) { + if (timer->refCount != 1) { uError("timer refCount=%d not expected 1", timer->refCount); } memset(timer, 0, sizeof(*timer)); @@ -560,7 +562,9 @@ static int32_t taosTmrModuleInit(void) { } tmrQhandle = taosInitScheduler(10000, taosTmrThreads, "tmr", NULL); - (void)taosInitTimer(taosTimerLoopFunc, MSECONDS_PER_TICK); + if (taosInitTimer(taosTimerLoopFunc, MSECONDS_PER_TICK) != 0) { + tmrError("failed to initialize timer"); + } tmrDebug("timer module is initialized, number of threads: %d", taosTmrThreads); diff --git a/tests/army/cluster/test_drop_table_by_uid.py b/tests/army/cluster/test_drop_table_by_uid.py new file mode 100644 index 0000000000..f09006b37b --- /dev/null +++ b/tests/army/cluster/test_drop_table_by_uid.py @@ -0,0 +1,368 @@ +import time +import random +import taos +from enum import Enum + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.srvCtl import * + + +class TDTestCase(TBase): + """ + Description: This class is used to verify the feature of 'drop table by uid' for task TS-5111 + FS: https://taosdata.feishu.cn/wiki/JgeDwZkH3iTNv2ksVkWcHenKnTf + TS: https://taosdata.feishu.cn/wiki/DX3FwopwGiXCeRkBNXFcj0MBnnb + create: + 2024-09-23 created by Charles + update: + None + """ + + class TableType(Enum): + STABLE = 0 + CHILD_TABLE = 1 + REGULAR_TABLE = 2 + + def init(self, conn, logSql, replicaVar=1): + """Initialize the TDengine cluster environment + """ + super(TDTestCase, self).init(conn, logSql, replicaVar, db="db") + tdSql.init(conn.cursor(), logSql) + + def get_uid_by_db_table_name(self, db_name, table_name, table_type=TableType.STABLE): + """Get table uid with db name and table name from system table + :param db_name: database name + :param table_name: table name + :param table_type: table type, default is stable + :return: table uid + """ + if table_type == self.TableType.STABLE: + tdSql.query(f"select * from information_schema.ins_stables where db_name='{db_name}' and stable_name like '%{table_name}%';") + elif table_type == self.TableType.CHILD_TABLE: + tdSql.query(f"select * from information_schema.ins_tables where db_name='{db_name}' and table_name like '%{table_name}%' and stable_name is not null;") + else: + tdSql.query(f"select * from information_schema.ins_tables where db_name='{db_name}' and table_name like '%{table_name}%' and stable_name is null;") + # check whether the table uid is empty + if len(tdSql.res) == 0: + tdLog.debug(f"Can't get table uid with db name: {db_name} and table name: {table_name}") + return None + # get table uid list + if table_type == self.TableType.STABLE: + return [item[10] for item in tdSql.res] + else: + return [item[5] for item in tdSql.res] + + def get_uid_by_db_name(self, db_name, table_type=TableType.STABLE): + """Get table uid with db name and table type from system table + :param db_name: database name + :param table_type: table type, default is stable + :return: table uid list + """ + if table_type == self.TableType.STABLE: + tdSql.query(f"select * from information_schema.ins_stables where db_name='{db_name}';") + elif table_type == self.TableType.CHILD_TABLE: + tdSql.query(f"select * from information_schema.ins_tables where db_name='{db_name}' and stable_name is not null;") + else: + tdSql.query(f"select * from information_schema.ins_tables where db_name='{db_name}' and stable_name is null;") + # check whether the table uid is empty + if len(tdSql.res) == 0: + tdLog.debug(f"Can't get table uid with db name: {db_name}") + return None + # get table uid list + if table_type == self.TableType.STABLE: + return [item[10] for item in tdSql.res] + else: + return [item[5] for item in tdSql.res] + + def drop_table_by_uid(self, uid_list, table_type=TableType.STABLE, exist_ops=False): + """Drop the specified tables by uid list + :db_name: database name + :param uid_list: table uid list to be dropped + :param exist_ops: whether to use exist option, default is False + :return: None + """ + # check whether the uid list is empty + if len(uid_list) == 0: + return + # drop table by uid + if exist_ops and table_type == self.TableType.STABLE: + for uid in uid_list: + tdSql.execute(f"drop stable with if exists `{uid}`;") + else: + uids = ','.join(["`" + str(item) + "`" for item in uid_list]) + tdSql.execute(f"drop table with {uids};") + + def test_drop_single_table_by_uid(self): + """Verify the feature of dropping a single stable/child table/regular table by uid with root user + """ + db_name = "test_drop_single_table_by_uid" + tdLog.info("Start test case: test_drop_single_table_by_uid") + # data for case test_drop_single_table_by_uid + tdLog.info("Prepare data for test case test_drop_single_table_by_uid") + tdSql.execute(f"create database {db_name};") + tdSql.execute(f"use {db_name};") + # table with normal characters + tdSql.execute("create stable st1 (ts timestamp, c1 int) tags (t1 int);") + tdSql.execute("create table ct1_1 using st1 tags(1);") + tdSql.execute("create table t1 (ts timestamp, c1 int, c2 float);") + tdLog.info("Finish preparing data for test case test_drop_single_table_by_uid of normal characters") + # get table uid + uid_st1 = self.get_uid_by_db_table_name(db_name, "st1") + tdLog.debug(f"uid_st1: {uid_st1}") + uid_ct1_1 = self.get_uid_by_db_table_name(db_name, "ct1_1", self.TableType.CHILD_TABLE) + tdLog.debug(f"uid_ct1_1: {uid_ct1_1}") + uid_t1 = self.get_uid_by_db_table_name(db_name, "t1", self.TableType.REGULAR_TABLE) + tdLog.debug(f"uid_t1: {uid_t1}") + # drop table by uid + self.drop_table_by_uid(uid_t1, self.TableType.REGULAR_TABLE) + self.drop_table_by_uid(uid_ct1_1, self.TableType.CHILD_TABLE) + self.drop_table_by_uid(uid_st1, self.TableType.STABLE, True) + + # table with special characters + tdSql.execute("create stable `st2\u00bf\u200bfnn1` (ts timestamp, c1 int) tags (t1 int);") + tdSql.execute("create table `ct2_1\u00cf\u00ff` using `st2\u00bf\u200bfnn1` tags(1);") + tdSql.execute("create table `t2\u00ef\u00fa` (ts timestamp, c1 int, c2 float);") + tdLog.info("Finish preparing data for test case test_drop_single_table_by_uid of special characters") + # get table uid + uid_st2 = self.get_uid_by_db_table_name(db_name, "st2") + tdLog.debug(f"uid_st2: {uid_st2}") + uid_ct2_1 = self.get_uid_by_db_table_name(db_name, "ct2_1", self.TableType.CHILD_TABLE) + tdLog.debug(f"uid_ct2_1: {uid_ct2_1}") + uid_t2 = self.get_uid_by_db_table_name(db_name, "t2", self.TableType.REGULAR_TABLE) + tdLog.debug(f"uid_t2: {uid_t2}") + # drop table by uid + self.drop_table_by_uid(uid_t2, self.TableType.REGULAR_TABLE) + self.drop_table_by_uid(uid_ct2_1, self.TableType.CHILD_TABLE) + self.drop_table_by_uid(uid_st2, self.TableType.STABLE, True) + tdSql.execute(f"drop database {db_name};") + tdLog.info("Finish test case: test_drop_single_table_by_uid") + + def test_drop_multiple_tables_by_uid(self): + """Verify the feature of dropping multiple tables by uid with root user + """ + db_name = "test_drop_multiple_tables_by_uid" + table_number = 100 + tdLog.info("Start test case: test_drop_multiple_tables_by_uid") + # data for case test_drop_multiple_tables_by_uid + tdLog.info("Prepare data for test case test_drop_multiple_tables_by_uid") + tdSql.execute(f"create database {db_name};") + tdSql.execute(f"use {db_name};") + # table with normal characters + for i in range(table_number): + tdSql.execute(f"create stable st{i} (ts timestamp, c1 int) tags (t1 int);") + tdSql.execute(f"create table ct{i}_{i} using st{i} tags({i+1});") + tdSql.execute(f"create table t{i} (ts timestamp, c1 int, c2 float);") + tdLog.info("Finish preparing data for test case test_drop_multiple_tables_by_uid of normal characters") + # get table uid + uid_st = self.get_uid_by_db_table_name(db_name, "st") + # tdLog.debug(f"Get multiple stable uid list: {uid_st}") + uid_ct = self.get_uid_by_db_table_name(db_name, "ct", self.TableType.CHILD_TABLE) + # tdLog.debug(f"Get multiple child table uid list: {uid_ct}") + uid_t = self.get_uid_by_db_table_name(db_name, "t", self.TableType.REGULAR_TABLE) + # tdLog.debug(f"Get multiple regular table uid list: {uid_t}") + # drop table by uid + self.drop_table_by_uid(uid_t, self.TableType.REGULAR_TABLE) + self.drop_table_by_uid(uid_ct, self.TableType.CHILD_TABLE) + self.drop_table_by_uid(uid_st, self.TableType.STABLE, True) + + # table with special characters + for i in range(table_number): + tdSql.execute(f"create stable `st{i}\u00bf\u200bfnn1` (ts timestamp, c1 int) tags (t1 int);") + tdSql.execute(f"create table `ct{i}_{i}\u00cf\u00ff` using `st{i}\u00bf\u200bfnn1` tags(1);") + tdSql.execute(f"create table `t{i}\u00ef\u00fa` (ts timestamp, c1 int, c2 float);") + # get table uid + uid_st = self.get_uid_by_db_table_name(db_name, "st") + # tdLog.debug(f"Get multiple stable uid list: {uid_st}") + uid_ct = self.get_uid_by_db_table_name(db_name, "ct", self.TableType.CHILD_TABLE) + # tdLog.debug(f"Get multiple child table uid list: {uid_ct}") + uid_t = self.get_uid_by_db_table_name(db_name, "t", self.TableType.REGULAR_TABLE) + # tdLog.debug(f"Get multiple regular table uid list: {uid_t}") + # drop table by uid + self.drop_table_by_uid(uid_t, self.TableType.REGULAR_TABLE) + self.drop_table_by_uid(uid_ct, self.TableType.CHILD_TABLE) + self.drop_table_by_uid(uid_st, self.TableType.STABLE, True) + tdSql.execute(f"drop database {db_name};") + tdLog.info("Finish test case: test_drop_multiple_tables_by_uid") + + def test_uid_as_table_name(self): + """Verify using uid as table name, drop table with uid doesn't affect other tables + """ + db_name = "test_uid_as_table_name" + tdLog.info("Start test case: test_uid_as_table_name") + # data for case test_uid_as_table_name + tdLog.info("Prepare data for test case test_uid_as_table_name") + tdSql.execute(f"create database {db_name};") + tdSql.execute(f"use {db_name};") + # super table + tdSql.execute(f"create stable `st1\u00bf\u200bfnn1` (ts timestamp, c1 int) tags (t1 int);") + uid_st = self.get_uid_by_db_table_name(db_name, "st") + tdSql.execute(f"create stable `{uid_st[0]}` (ts timestamp, c1 int) tags (t1 int);") + self.drop_table_by_uid(uid_st, self.TableType.STABLE, True) + uid_st = self.get_uid_by_db_table_name(db_name, str(uid_st[0])) + assert uid_st is not None + tdLog.info(f"Drop stable with special characters with uid {uid_st[0]}, stable named as {uid_st[0]} doesn't be affected") + # child table + tdSql.execute(f"create stable `st2\u00bf\u200bfnn1` (ts timestamp, c1 int) tags (t1 int);") + tdSql.execute(f"create table `ct2_1\u00cf\u00ff` using `st2\u00bf\u200bfnn1` tags(1);") + uid_ct = self.get_uid_by_db_table_name(db_name, "ct", self.TableType.CHILD_TABLE) + tdSql.execute(f"create table `{uid_ct[0]}` using `st2\u00bf\u200bfnn1` tags(2);") + self.drop_table_by_uid(uid_ct, self.TableType.CHILD_TABLE) + uid_ct = self.get_uid_by_db_table_name(db_name, str(uid_ct[0]), self.TableType.CHILD_TABLE) + assert uid_ct is not None + tdLog.info(f"Drop child table with special characters with uid {uid_ct[0]}, child table named as {uid_ct[0]} doesn't be affected") + # regular table + tdSql.execute(f"create table `t2\u00bf\u200bfnn1` (ts timestamp, c1 int);") + uid_t = self.get_uid_by_db_table_name(db_name, "t2", self.TableType.REGULAR_TABLE) + tdSql.execute(f"create table `{uid_t[0]}` (ts timestamp, c1 int);") + self.drop_table_by_uid(uid_t, self.TableType.REGULAR_TABLE) + uid_t = self.get_uid_by_db_table_name(db_name, str(uid_t[0]), self.TableType.REGULAR_TABLE) + assert uid_t is not None + tdLog.info(f"Drop regular table with special characters with uid {uid_t[0]}, regular table named as {uid_t[0]} doesn't be affected") + tdSql.execute(f"drop database {db_name};") + tdLog.info("Finish test case: test_uid_as_table_name") + + def test_abnormal_non_exist_uid(self): + """Verify dropping table with non-exist uid + """ + db_name = "test_abnormal_non_exist_uid" + tdLog.info("Start test case: test_abnormal_non_exist_uid") + # data for case test_abnormal_non_exist_uid + tdLog.info("Prepare data for test case test_abnormal_non_exist_uid") + tdSql.execute(f"create database {db_name};") + tdSql.execute(f"use {db_name};") + # drop table with non-exist uid + tdSql.error(f"drop stable with if exists `1234567890`;", expectErrInfo="STable not exist:") + tdSql.error(f"drop table with `1234567890`;", expectErrInfo="Table does not exist:") + tdSql.execute(f"drop database {db_name};") + tdLog.info("Finish test case: test_abnormal_non_exist_uid") + + def test_abnormal_incorrect_table_type(self): + """Verify dropping table with incorrect sql, like drop stable sql with table or child table uid + """ + try: + db_name = "test_abnormal_incorrect_table_type" + tdLog.info("Start test case: test_abnormal_incorrect_table_type") + # data for case test_abnormal_incorrect_table_type + tdLog.info("Prepare data for test case test_abnormal_incorrect_table_type") + tdSql.execute(f"create database {db_name};") + tdSql.execute(f"use {db_name};") + tdSql.execute("create stable `st3\u00bf\u200bfnn1` (ts timestamp, c1 int) tags (t1 int);") + tdSql.execute("create table `ct3_1\u00cf\u00ff` using `st3\u00bf\u200bfnn1` tags(1);") + tdSql.execute("create table `t3\u00ef\u00fa` (ts timestamp, c1 int, c2 float);") + tdLog.info("Finish preparing data for test case test_abnormal_incorrect_table_type of special characters") + # get table uid + uid_st = self.get_uid_by_db_table_name(db_name, "st") + uid_ct = self.get_uid_by_db_table_name(db_name, "ct", self.TableType.CHILD_TABLE) + uid_t = self.get_uid_by_db_table_name(db_name, "t", self.TableType.REGULAR_TABLE) + # drop table with incorrect sql + tdSql.error(f"drop stable with `{uid_ct[0]}`;", expectErrInfo="STable not exist") + tdSql.error(f"drop stable with `{uid_t[0]}`;", expectErrInfo="STable not exist") + tdLog.info("Finish test case: test_abnormal_incorrect_table_type") + except Exception as e: + tdLog.exit("Failed to run test case test_abnormal_incorrect_table_type with msg: %s" % str(e)) + finally: + tdSql.execute(f"drop database {db_name};") + + def test_abnormal_mixed_uid(self): + """Verify dropping table with mixed uid + """ + db_name = "test_abnormal_mixed_uid" + tdLog.info("Start test case: test_abnormal_mixed_uid") + # data for case test_abnormal_mixed_uidF + tdLog.info("Prepare data for test case test_abnormal_mixed_uid") + tdSql.execute(f"create database {db_name};") + tdSql.execute(f"use {db_name};") + tdSql.execute("create stable `st3\u00bf\u200bfnn1` (ts timestamp, c1 int) tags (t1 int);") + tdSql.execute("create table `ct3_1\u00cf\u00ff` using `st3\u00bf\u200bfnn1` tags(1);") + tdSql.execute("create table `t3\u00ef\u00fa` (ts timestamp, c1 int, c2 float);") + tdLog.info("Finish preparing data for test case test_abnormal_mixed_uid of special characters") + # get table uid + uid_st = self.get_uid_by_db_table_name(db_name, "st") + uid_ct = self.get_uid_by_db_table_name(db_name, "ct", self.TableType.CHILD_TABLE) + uid_t = self.get_uid_by_db_table_name(db_name, "t", self.TableType.REGULAR_TABLE) + # drop table with incorrect sql + tdSql.error(f"drop stable with `{uid_st[0]}`,`{uid_ct[0]}`;", expectErrInfo="syntax error") + tdSql.error(f"drop table with `{uid_st[0]}`,`{uid_ct[0]}`,`{uid_t[0]}`;", expectErrInfo="Cannot drop super table in batch") + tdSql.execute(f"drop database {db_name};") + tdLog.info("Finish test case: test_abnormal_mixed_uid") + + def test_abnormal_system_tables(self): + """Verify dropping system tables + """ + try: + uid_list = self.get_uid_by_db_name("information_schema", self.TableType.REGULAR_TABLE) + uid = random.choice(uid_list) + assert uid is None + except Exception as e: + tdLog.exit("Failed to run test case test_abnormal_system_tables with msg: %s" % str(e)) + + def test_abnormal_drop_table_with_non_root_user(self): + """Verify dropping table with non-root user + """ + try: + # create new user and grant create database priviledge + tdSql.execute("create user test pass 'test';") + tdSql.execute("alter user test createdb 1;") + conn = taos.connect(user="test", password="test") + cursor = conn.cursor() + # create database and tables with new user + tdLog.info("Prepare data for test case test_abnormal_drop_table_with_non_root_user") + db_name = "test_abnormal_drop_table_with_non_root_user" + cursor.execute(f"create database {db_name};") + cursor.execute(f"use {db_name};") + time.sleep(3) + cursor.execute("create stable `st4\u00bf\u200bfnn1` (ts timestamp, c1 int) tags (t1 int);") + cursor.execute("create table `ct4_1\u00cf\u00ff` using `st4\u00bf\u200bfnn1` tags(1);") + cursor.execute("create table `t4\u00ef\u00fa` (ts timestamp, c1 int, c2 float);") + tdLog.info("Finish preparing data for test case test_abnormal_drop_table_with_non_root_user of special characters") + # get table uid + uid_st = self.get_uid_by_db_table_name(db_name, "st") + uid_ct = self.get_uid_by_db_table_name(db_name, "ct", self.TableType.CHILD_TABLE) + uid_t = self.get_uid_by_db_table_name(db_name, "t", self.TableType.REGULAR_TABLE) + # drop stable with sql by non-root user + try: + cursor.execute(f"drop stable with `{uid_st[0]}`;") + except Exception as e: + assert "Permission denied or target object not exist" in str(e) + tdLog.info("Drop stable with non-root user failed as expected") + # drop child table with sql by non-root user + try: + cursor.execute(f"drop table with `{uid_ct[0]}`;") + except Exception as e: + assert "Permission denied or target object not exist" in str(e) + tdLog.info("Drop child table with non-root user failed as expected") + # drop regular table with sql by non-root user + try: + cursor.execute(f"drop table with `{uid_t[0]}`;") + except Exception as e: + assert "Permission denied or target object not exist" in str(e) + tdLog.info("Drop regular table with non-root user failed as expected") + tdLog.info("Finish test case: test_abnormal_drop_table_with_non_root_user") + except Exception as e: + tdLog.exit("Failed to run test case test_abnormal_drop_table_with_non_root_user with msg: %s" % str(e)) + finally: + tdSql.execute(f"drop database {db_name};") + tdSql.execute("drop user test;") + + def run(self): + # normal cases + self.test_drop_single_table_by_uid() + self.test_drop_multiple_tables_by_uid() + self.test_uid_as_table_name() + # abnormal cases + self.test_abnormal_non_exist_uid() + self.test_abnormal_incorrect_table_type() + self.test_abnormal_mixed_uid() + self.test_abnormal_system_tables() + self.test_abnormal_drop_table_with_non_root_user() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/docs-examples-test/node.sh b/tests/docs-examples-test/node.sh index 41acf7c7b4..cbbbd3820e 100644 --- a/tests/docs-examples-test/node.sh +++ b/tests/docs-examples-test/node.sh @@ -2,41 +2,63 @@ set -e +check_transactions() { + for i in {1..30} + do + output=$(taos -s "show transactions;") + if [[ $output == *"Query OK, 0 row(s)"* ]]; then + echo "Success: No transactions are in progress." + return 0 + fi + sleep 1 + done + + echo "Error: Transactions are still in progress after 30 attempts." + return 1 +} + +reset_cache() { + response=$(curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql' --data 'reset query cache') + + if [[ $response == \{\"code\":0* ]]; then + echo "Success: Query cache reset successfully." + else + echo "Error: Failed to reset query cache. Response: $response" + return 1 + fi +} + + pgrep taosd || taosd >> /dev/null 2>&1 & pgrep taosadapter || taosadapter >> /dev/null 2>&1 & +sleep 10 + cd ../../docs/examples/node npm install -cd restexample; -node connect.js +cd websocketexample -cd ../nativeexample +node all_type_query.js -node connect.js +node all_type_stmt.js taos -s "drop database if exists power" -node insert_example.js - -node query_example.js - -node async_query_example.js - -# node subscribe_demo.js - -taos -s "drop topic if exists topic_name_example" -taos -s "drop database if exists power" -node param_bind_example.js +check_transactions || exit 1 +reset_cache || exit 1 +node line_example.js taos -s "drop database if exists power" -node multi_bind_example.js +check_transactions || exit 1 +reset_cache || exit 1 +node nodejsChecker.js -taos -s "drop database if exists test" -node influxdb_line_example.js +node sql_example.js -taos -s "drop database if exists test" -node opentsdb_telnet_example.js +node stmt_example.js -taos -s "drop database if exists test" -node opentsdb_json_example.js + +node tmq_example.js + +node tmq_seek_example.js \ No newline at end of file diff --git a/tests/docs-examples-test/python.sh b/tests/docs-examples-test/python.sh index 84f0771ec5..6a25683b58 100644 --- a/tests/docs-examples-test/python.sh +++ b/tests/docs-examples-test/python.sh @@ -2,6 +2,34 @@ set -e +check_transactions() { + for i in {1..30} + do + output=$(taos -s "show transactions;") + if [[ $output == *"Query OK, 0 row(s)"* ]]; then + echo "Success: No transactions are in progress." + return 0 + fi + sleep 1 + done + + echo "Error: Transactions are still in progress after 30 attempts." + return 1 +} + +reset_cache() { + response=$(curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql' --data 'reset query cache') + + if [[ $response == \{\"code\":0* ]]; then + echo "Success: Query cache reset successfully." + else + echo "Error: Failed to reset query cache. Response: $response" + return 1 + fi +} + + + taosd >>/dev/null 2>&1 & taosadapter >>/dev/null 2>&1 & @@ -11,18 +39,26 @@ cd ../../docs/examples/python # 1 taos -s "create database if not exists log" +check_transactions || exit 1 +reset_cache || exit 1 python3 connect_example.py # 2 taos -s "drop database if exists power" +check_transactions || exit 1 +reset_cache || exit 1 python3 native_insert_example.py # 3 taos -s "drop database power" +check_transactions || exit 1 +reset_cache || exit 1 python3 bind_param_example.py # 4 taos -s "drop database power" +check_transactions || exit 1 +reset_cache || exit 1 python3 multi_bind_example.py # 5 @@ -33,20 +69,28 @@ python3 async_query_example.py # 7 taos -s "drop database if exists test" +check_transactions || exit 1 +reset_cache || exit 1 python3 line_protocol_example.py # 8 taos -s "drop database test" +check_transactions || exit 1 +reset_cache || exit 1 python3 telnet_line_protocol_example.py # 9 taos -s "drop database test" +check_transactions || exit 1 +reset_cache || exit 1 python3 json_protocol_example.py # 10 pip install SQLAlchemy pip install pandas taosBenchmark -y -d power -t 10 -n 10 +check_transactions || exit 1 +reset_cache || exit 1 python3 conn_native_pandas.py python3 conn_rest_pandas.py taos -s "drop database if exists power" @@ -86,8 +130,69 @@ pip3 install kafka-python python3 kafka_example_consumer.py # 21 -pip3 install taos-ws-py==0.3.1 +pip3 install taos-ws-py==0.3.3 python3 conn_websocket_pandas.py # 22 python3 connect_websocket_examples.py + +# 23 +python3 create_db_ws.py + +# 24 +python3 create_db_native.py + +# 25 +python3 create_db_rest.py + +python3 insert_native.py + +python3 insert_rest.py + +python3 insert_ws.py + +python3 query_native.py + +python3 query_rest.py + +python3 query_ws.py + +python3 reqid_native.py + +python3 reqid_rest.py + +python3 reqid_ws.py + +taos -s "drop database power" +check_transactions || exit 1 +reset_cache || exit 1 +python3 schemaless_native.py + +taos -s "drop database power" +check_transactions || exit 1 +reset_cache || exit 1 +python3 schemaless_ws.py + +taos -s "drop database power" +check_transactions || exit 1 +reset_cache || exit 1 +python3 stmt_native.py + +python3 stmt_ws.py + +taos -s "drop topic if exists topic_meters" +check_transactions || exit 1 +reset_cache || exit 1 +taos -s "drop database if exists power" +check_transactions || exit 1 +reset_cache || exit 1 +python3 tmq_native.py + +taos -s "drop topic if exists topic_meters" +check_transactions || exit 1 +reset_cache || exit 1 +taos -s "drop database if exists power" +check_transactions || exit 1 +reset_cache || exit 1 +python3 tmq_websocket_example.py + diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 602ac9ad66..a6d65b254f 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -23,6 +23,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f insert/test_column_tag_boundary.py ,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_desc.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_null.py +,,y,army,./pytest.sh python3 ./test.py -f cluster/test_drop_table_by_uid.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f cluster/incSnapshot.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f cluster/clusterBasic.py -N 5 ,,y,army,./pytest.sh python3 ./test.py -f query/query_basic.py -N 3 @@ -1554,9 +1555,10 @@ #docs-examples test ,,n,docs-examples-test,bash python.sh -#,,n,docs-examples-test,bash node.sh +,,n,docs-examples-test,bash node.sh ,,n,docs-examples-test,bash csharp.sh ,,n,docs-examples-test,bash jdbc.sh ,,n,docs-examples-test,bash rust.sh ,,n,docs-examples-test,bash go.sh ,,n,docs-examples-test,bash test_R.sh + diff --git a/tests/system-test/2-query/tsma.py b/tests/system-test/2-query/tsma.py index 1b52302503..f05398600b 100644 --- a/tests/system-test/2-query/tsma.py +++ b/tests/system-test/2-query/tsma.py @@ -871,6 +871,7 @@ class TDTestCase: .should_query_with_table('meters', '2018-09-17 09:00:00.200', '2018-09-17 09:29:59:999') .should_query_with_tsma('tsma2', '2018-09-17 09:30:00', '2018-09-17 09:59:59.999') .should_query_with_table('meters', '2018-09-17 10:00:00.000', '2018-09-17 10:23:19.800').get_qc()) + tdSql.query('show create table test.meters') self.check(ctxs) if not ignore_some_tests: tdSql.execute('create database db2') diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 3ba6f8521e..a16a03d30a 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -18,7 +18,7 @@ IF (TD_WEBSOCKET) COMMAND git clean -f -d BUILD_COMMAND COMMAND cargo update - COMMAND export CFLAGS=-fno-builtin && RUSTFLAGS=-Ctarget-feature=-crt-static cargo build --release -p taos-ws-sys --features rustls + COMMAND RUSTFLAGS=-Ctarget-feature=-crt-static cargo build --release -p taos-ws-sys --features rustls INSTALL_COMMAND COMMAND cp target/release/${websocket_lib_file} ${CMAKE_BINARY_DIR}/build/lib COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include @@ -37,7 +37,7 @@ IF (TD_WEBSOCKET) COMMAND git clean -f -d BUILD_COMMAND COMMAND cargo update - COMMAND export CFLAGS=-fno-builtin && cargo build --release -p taos-ws-sys --features rustls + COMMAND cargo build --release -p taos-ws-sys --features rustls INSTALL_COMMAND COMMAND cp target/release/taosws.dll ${CMAKE_BINARY_DIR}/build/lib COMMAND cp target/release/taosws.dll.lib ${CMAKE_BINARY_DIR}/build/lib/taosws.lib @@ -57,7 +57,7 @@ IF (TD_WEBSOCKET) COMMAND git clean -f -d BUILD_COMMAND COMMAND cargo update - COMMAND export CFLAGS=-fno-builtin && cargo build --release -p taos-ws-sys --features rustls + COMMAND cargo build --release -p taos-ws-sys --features rustls INSTALL_COMMAND COMMAND cp target/release/${websocket_lib_file} ${CMAKE_BINARY_DIR}/build/lib COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include