From dec257ff599d78ec12e58c0527531e8498a15a0a Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 23 Oct 2023 08:59:54 +0000 Subject: [PATCH 001/181] TD-26529 --- include/common/tmsg.h | 9 + include/common/tmsgdef.h | 1 + include/libs/monitor/monitor.h | 2 + include/libs/monitorfw/taos_alloc.h | 47 ++ include/libs/monitorfw/taos_collector.h | 89 ++++ .../libs/monitorfw/taos_collector_registry.h | 123 ++++++ include/libs/monitorfw/taos_counter.h | 102 +++++ include/libs/monitorfw/taos_linked_list.h | 27 ++ include/libs/monitorfw/taos_map.h | 25 ++ include/libs/monitorfw/taos_metric.h | 49 +++ include/libs/monitorfw/taos_metric_sample.h | 59 +++ include/libs/monitorfw/taos_monitor.h | 130 ++++++ include/util/tlog.h | 1 + source/common/src/tmsg.c | 38 ++ source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 3 +- source/dnode/mgmt/node_mgmt/CMakeLists.txt | 2 +- source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 2 + source/dnode/mgmt/node_util/CMakeLists.txt | 2 +- source/dnode/mgmt/node_util/inc/dmUtil.h | 1 + source/dnode/mnode/impl/CMakeLists.txt | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 19 + source/dnode/mnode/impl/src/mndMnode.c | 1 + source/dnode/vnode/src/vnd/vnodeSvr.c | 16 + source/libs/CMakeLists.txt | 1 + source/libs/monitor/CMakeLists.txt | 2 +- source/libs/monitor/src/monMain.c | 28 ++ source/libs/monitorfw/CMakeLists.txt | 9 + source/libs/monitorfw/inc/taos_assert.h | 27 ++ .../monitorfw/inc/taos_collector_registry_i.h | 25 ++ .../monitorfw/inc/taos_collector_registry_t.h | 40 ++ source/libs/monitorfw/inc/taos_collector_t.h | 32 ++ source/libs/monitorfw/inc/taos_errors.h | 25 ++ .../libs/monitorfw/inc/taos_linked_list_i.h | 93 ++++ .../libs/monitorfw/inc/taos_linked_list_t.h | 53 +++ source/libs/monitorfw/inc/taos_log.h | 27 ++ source/libs/monitorfw/inc/taos_map_i.h | 37 ++ source/libs/monitorfw/inc/taos_map_t.h | 44 ++ .../monitorfw/inc/taos_metric_formatter_i.h | 82 ++++ .../monitorfw/inc/taos_metric_formatter_t.h | 26 ++ source/libs/monitorfw/inc/taos_metric_i.h | 43 ++ .../libs/monitorfw/inc/taos_metric_sample_i.h | 48 ++ .../libs/monitorfw/inc/taos_metric_sample_t.h | 28 ++ source/libs/monitorfw/inc/taos_metric_t.h | 54 +++ .../monitorfw/inc/taos_string_builder_i.h | 77 ++++ .../monitorfw/inc/taos_string_builder_t.h | 25 ++ source/libs/monitorfw/src/taos_collector.c | 110 +++++ .../monitorfw/src/taos_collector_registry.c | 200 +++++++++ source/libs/monitorfw/src/taos_counter.c | 65 +++ source/libs/monitorfw/src/taos_linked_list.c | 220 ++++++++++ source/libs/monitorfw/src/taos_map.c | 414 ++++++++++++++++++ source/libs/monitorfw/src/taos_metric.c | 173 ++++++++ .../monitorfw/src/taos_metric_formatter.c | 258 +++++++++++ .../libs/monitorfw/src/taos_metric_sample.c | 96 ++++ .../libs/monitorfw/src/taos_string_builder.c | 152 +++++++ 54 files changed, 3259 insertions(+), 5 deletions(-) create mode 100644 include/libs/monitorfw/taos_alloc.h create mode 100644 include/libs/monitorfw/taos_collector.h create mode 100644 include/libs/monitorfw/taos_collector_registry.h create mode 100644 include/libs/monitorfw/taos_counter.h create mode 100644 include/libs/monitorfw/taos_linked_list.h create mode 100644 include/libs/monitorfw/taos_map.h create mode 100644 include/libs/monitorfw/taos_metric.h create mode 100644 include/libs/monitorfw/taos_metric_sample.h create mode 100644 include/libs/monitorfw/taos_monitor.h create mode 100644 source/libs/monitorfw/CMakeLists.txt create mode 100644 source/libs/monitorfw/inc/taos_assert.h create mode 100644 source/libs/monitorfw/inc/taos_collector_registry_i.h create mode 100644 source/libs/monitorfw/inc/taos_collector_registry_t.h create mode 100644 source/libs/monitorfw/inc/taos_collector_t.h create mode 100644 source/libs/monitorfw/inc/taos_errors.h create mode 100644 source/libs/monitorfw/inc/taos_linked_list_i.h create mode 100644 source/libs/monitorfw/inc/taos_linked_list_t.h create mode 100644 source/libs/monitorfw/inc/taos_log.h create mode 100644 source/libs/monitorfw/inc/taos_map_i.h create mode 100644 source/libs/monitorfw/inc/taos_map_t.h create mode 100644 source/libs/monitorfw/inc/taos_metric_formatter_i.h create mode 100644 source/libs/monitorfw/inc/taos_metric_formatter_t.h create mode 100644 source/libs/monitorfw/inc/taos_metric_i.h create mode 100644 source/libs/monitorfw/inc/taos_metric_sample_i.h create mode 100644 source/libs/monitorfw/inc/taos_metric_sample_t.h create mode 100644 source/libs/monitorfw/inc/taos_metric_t.h create mode 100644 source/libs/monitorfw/inc/taos_string_builder_i.h create mode 100644 source/libs/monitorfw/inc/taos_string_builder_t.h create mode 100644 source/libs/monitorfw/src/taos_collector.c create mode 100644 source/libs/monitorfw/src/taos_collector_registry.c create mode 100644 source/libs/monitorfw/src/taos_counter.c create mode 100644 source/libs/monitorfw/src/taos_linked_list.c create mode 100644 source/libs/monitorfw/src/taos_map.c create mode 100644 source/libs/monitorfw/src/taos_metric.c create mode 100644 source/libs/monitorfw/src/taos_metric_formatter.c create mode 100644 source/libs/monitorfw/src/taos_metric_sample.c create mode 100644 source/libs/monitorfw/src/taos_string_builder.c diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 18b10a1749..6c6bacb6ad 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1544,6 +1544,15 @@ int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); void tFreeSStatusReq(SStatusReq* pReq); +typedef struct { + int32_t contLen; + char* pCont; +} SStatisReq; + +int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); +int32_t tDeserializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); +void tFreeSStatisReq(SStatisReq *pReq); + typedef struct { int32_t dnodeId; int64_t clusterId; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index b92bba831c..2fe7657b7f 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -189,6 +189,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_MND_STREAM_NODECHANGE_CHECK, "stream-nodechange-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB_TIMER, "trim-db-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_STATIS, "statis", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_NEW_MSG_SEG(TDMT_VND_MSG) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 91b3a54ea1..1f3d21777d 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -222,6 +222,8 @@ void monSetQmInfo(SMonQmInfo *pInfo); void monSetSmInfo(SMonSmInfo *pInfo); void monSetBmInfo(SMonBmInfo *pInfo); void monSendReport(); +void monSendPromReport(); +void monSendContent(char *pCont); void tFreeSMonMmInfo(SMonMmInfo *pInfo); void tFreeSMonVmInfo(SMonVmInfo *pInfo); diff --git a/include/libs/monitorfw/taos_alloc.h b/include/libs/monitorfw/taos_alloc.h new file mode 100644 index 0000000000..893ffc7e9b --- /dev/null +++ b/include/libs/monitorfw/taos_alloc.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_alloc.h + * @brief memory management + */ + +#ifndef TAOS_ALLOC_H +#define TAOS_ALLOC_H + +#include +#include + +/** + * @brief Redefine this macro if you wish to override it. The default value is malloc. + */ +#define taos_malloc malloc + +/** + * @brief Redefine this macro if you wish to override it. The default value is realloc. + */ +#define taos_realloc realloc + +/** + * @brief Redefine this macro if you wish to override it. The default value is strdup. + */ +#define taos_strdup strdup + +/** + * @brief Redefine this macro if you wish to override it. The default value is free. + */ +#define taos_free free + +#endif // TAOS_ALLOC_H diff --git a/include/libs/monitorfw/taos_collector.h b/include/libs/monitorfw/taos_collector.h new file mode 100644 index 0000000000..918395ae72 --- /dev/null +++ b/include/libs/monitorfw/taos_collector.h @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_COLLECTOR_H +#define TAOS_COLLECTOR_H + +#include "taos_map.h" +#include "taos_metric.h" + +/** + * @file taos_collector.h + * @brief A Prometheus collector returns a collection of metrics + */ + +/** + * @brief A prometheus collector calls collect to prepare metrics and return them to the registry to which it is + * registered. + */ +typedef struct taos_collector taos_collector_t; + +/** + * @brief The function responsible for preparing metric data and returning metrics for a given collector. + * + * If you use the default collector registry, this should not concern you. If you are using a custom collector, you may + * set this function on your collector to do additional work before returning the contained metrics. + * + * @param self The target taos_collector_t* + * @return The taos_map_t* containing the collected metrics + */ +typedef taos_map_t *taos_collect_fn(taos_collector_t *self); + +/** + * @brief Create a collector + * @param name The name of the collector. The name MUST NOT be default or process. + * @return The constructed taos_collector_t* + */ +taos_collector_t *taos_collector_new(const char *name); + +/** + * @brief Destroy a collector. You MUST set self to NULL after destruction. + * @param self The target taos_collector_t* + * @return A non-zero integer value upon failure. + */ +int taos_collector_destroy(taos_collector_t *self); + +/** + * @brief Frees a collector passed as a void pointer. You MUST set self to NULL after destruction. + * @param gen The target taos_collector_t* represented as a void* + */ +void taos_collector_free_generic(void *gen); + +/** + * @brief Destroys a collector passed as a void pointer. You MUST set self to NULL after destruction. + * @param gen The target taos_collector_t* represented as a void* + * @return A non-zero integer value upon failure. + */ +int taos_collector_destroy_generic(void *gen); + +/** + * @brief Add a metric to a collector + * @param self The target taos_collector_t* + * @param metric the taos_metric_t* to add to the taos_collector_t* passed as self. + * @return A non-zero integer value upon failure. + */ +int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric); + +/** + * @brief The collect function is responsible for doing any work involving a set of metrics and then returning them + * for metric exposition. + * @param self The target taos_collector_t* + * @param fn The taos_collect_fn* which will be responsible for handling any metric collection operations before + * returning the collected metrics for exposition. + * @return A non-zero integer value upon failure. + */ +int taos_collector_set_collect_fn(taos_collector_t *self, taos_collect_fn *fn); + +#endif // TAOS_COLLECTOR_H diff --git a/include/libs/monitorfw/taos_collector_registry.h b/include/libs/monitorfw/taos_collector_registry.h new file mode 100644 index 0000000000..f67deb074b --- /dev/null +++ b/include/libs/monitorfw/taos_collector_registry.h @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_collector_registry.h + * @brief The collector registry registers collectors for metric exposition. + */ + +#ifndef TAOS_REGISTRY_H +#define TAOS_REGISTRY_H + +#include "taos_collector.h" +#include "taos_metric.h" + +/** + * @brief A taos_registry_t is responsible for registering metrics and briding them to the string exposition format + */ +typedef struct taos_collector_registry taos_collector_registry_t; + +/** + * @brief Initialize the default registry by calling taos_collector_registry_init within your program. You MUST NOT + * modify this value. + */ +extern taos_collector_registry_t *TAOS_COLLECTOR_REGISTRY_DEFAULT; + +/** + * @brief Initializes the default collector registry and enables metric collection on the executing process + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_default_init(void); + +/** + * @brief Constructs a taos_collector_registry_t* + * @param name The name of the collector registry. It MUST NOT be default. + * @return The constructed taos_collector_registry_t* + */ +taos_collector_registry_t *taos_collector_registry_new(const char *name); + +/** + * @brief Destroy a collector registry. You MUST set self to NULL after destruction. + * @param self The target taos_collector_registry_t* + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_destroy(taos_collector_registry_t *self); + +/** + * @brief Enable process metrics on the given collector registry + * @param self The target taos_collector_registry_t* + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_enable_process_metrics(taos_collector_registry_t *self); + +/** + * @brief Registers a metric with the default collector on TAOS_DEFAULT_COLLECTOR_REGISTRY + * + * The metric to be registered MUST NOT already be registered with the given . If so, the program will + * halt. It returns a taos_metric_t* to simplify metric creation and registration. Furthermore, + * TAOS_DEFAULT_COLLECTOR_REGISTRY must be registered via taos_collector_registry_default_init() prior to calling this + * function. The metric will be added to the default registry's default collector. + * + * @param metric The metric to register on TAOS_DEFAULT_COLLECTOR_REGISTRY* + * @return The registered taos_metric_t* + */ +taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metric); + +/** + * @brief Registers a metric with the default collector on TAOS_DEFAULT_COLLECTOR_REGISTRY. Returns an non-zero integer + * value on failure. + * + * See taos_collector_registry_must_register_metric. + * + * @param metric The metric to register on TAOS_DEFAULT_COLLECTOR_REGISTRY* + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_register_metric(taos_metric_t *metric); + +/** + * @brief Register a collector with the given registry. Returns a non-zero integer value on failure. + * @param self The target taos_collector_registry_t* + * @param collector The taos_collector_t* to register onto the taos_collector_registry_t* as self + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_register_collector(taos_collector_registry_t *self, taos_collector_t *collector); + +/** + * @brief Returns a string in the default metric exposition format. The string MUST be freed to avoid unnecessary heap + * memory growth. + * + * Reference: https://prometheus.io/docs/instrumenting/exposition_formats/ + * + * @param self The target taos_collector_registry_t* + * @return The string int he default metric exposition format. + */ +const char *taos_collector_registry_bridge(taos_collector_registry_t *self, int64_t ts); + +int taos_collector_registry_clear_out(taos_collector_registry_t *self); + +/** + *@brief Validates that the given metric name complies with the specification: + * + * Reference: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels + * + * Returns a non-zero integer value on failure. + * + * @param self The target taos_collector_registry_t* + * @param metric_name The metric name to validate + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_validate_metric_name(taos_collector_registry_t *self, const char *metric_name); + +#endif // TAOS_H diff --git a/include/libs/monitorfw/taos_counter.h b/include/libs/monitorfw/taos_counter.h new file mode 100644 index 0000000000..28a9eed41c --- /dev/null +++ b/include/libs/monitorfw/taos_counter.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_COUNTER_H +#define TAOS_COUNTER_H + +#include + +#include "taos_metric.h" + +/** + * @file taos_counter.h + * @brief https://prometheus.io/docs/concepts/metric_types/#counter + */ + +/** + * @brief A prometheus counter. + * + * References + * * See https://prometheus.io/docs/concepts/metric_types/#counter + */ +typedef taos_metric_t taos_counter_t; + +/** + * @brief Construct a taos_counter_t* + * @param name The name of the metric + * @param help The metric description + * @param label_key_count The number of labels associated with the given metric. Pass 0 if the metric does not + * require labels. + * @param label_keys A collection of label keys. The number of keys MUST match the value passed as label_key_count. If + * no labels are required, pass NULL. Otherwise, it may be convenient to pass this value as a + * literal. + * @return The constructed taos_counter_t* + * + * *Example* + * + * // An example with labels + * taos_counter_new("foo", "foo is a counter with labels", 2, (const char**) { "one", "two" }); + * + * // An example without labels + * taos_counter_new("foo", "foo is a counter without labels", 0, NULL); + */ +taos_counter_t *taos_counter_new(const char *name, const char *help, size_t label_key_count, const char **label_keys); + +/** + * @brief Destroys a taos_counter_t*. You must set self to NULL after destruction. A non-zero integer value will be + * returned on failure. + * @param self A taos_counter_t* + * @return A non-zero integer value upon failure. + */ +int taos_counter_destroy(taos_counter_t *self); + +/** + * @brief Increment the taos_counter_t by 1. A non-zero integer value will be returned on failure. + * @param self The target taos_counter_t* + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the counter's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_counter_inc(foo_counter, (const char**) { "bar", "bang" }); + ** + * // An example without labels + * taos_counter_inc(foo_counter, NULL); + */ +int taos_counter_inc(taos_counter_t *self, const char **label_values); + +/** + * @brief Add the value to the taos_counter_t*. A non-zero integer value will be returned on failure. + * @param self The target taos_counter_t* + * @param r_value The double to add to the taos_counter_t passed as self. The value MUST be greater than or equal to 0. + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the counter's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_counter_add(foo_counter, 22, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_counter_add(foo_counter, 22, NULL); + */ +int taos_counter_add(taos_counter_t *self, double r_value, const char **label_values); + +#endif // TAOS_COUNTER_H diff --git a/include/libs/monitorfw/taos_linked_list.h b/include/libs/monitorfw/taos_linked_list.h new file mode 100644 index 0000000000..1fff5e65a6 --- /dev/null +++ b/include/libs/monitorfw/taos_linked_list.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_LIST_H +#define TAOS_LIST_H + +#include + +struct taos_linked_list; +/** + * @brief Provides a generic linked list + */ +typedef struct taos_linked_list taos_linked_list_t; + +#endif // TAOS_LIST_H diff --git a/include/libs/monitorfw/taos_map.h b/include/libs/monitorfw/taos_map.h new file mode 100644 index 0000000000..fa554d569c --- /dev/null +++ b/include/libs/monitorfw/taos_map.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_MAP_H +#define TAOS_MAP_H + +struct taos_map; +typedef struct taos_map taos_map_t; + +struct taos_map_node; +typedef struct taos_map_node taos_map_node_t; + +#endif // TAOS_MAP_H diff --git a/include/libs/monitorfw/taos_metric.h b/include/libs/monitorfw/taos_metric.h new file mode 100644 index 0000000000..a377311fa0 --- /dev/null +++ b/include/libs/monitorfw/taos_metric.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_metric.h + * @brief Functions for retrieving metric samples from metrics given an ordered set of labels + */ + +#ifndef TAOS_METRIC_H +#define TAOS_METRIC_H + +#include "taos_metric_sample.h" + +struct taos_metric; +/** + * @brief A prometheus metric. + * + * Reference: https://prometheus.io/docs/concepts/data_model + */ +typedef struct taos_metric taos_metric_t; + +/** + * @brief Returns a taos_metric_sample_t*. The order of label_values is significant. + * + * You may use this function to cache metric samples to avoid sample lookup. Metric samples are stored in a hash map + * with O(1) lookups in average case; nonethless, caching metric samples and updating them directly might be + * preferrable in performance-sensitive situations. + * + * @param self The target taos_metric_t* + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the counter's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A taos_metric_sample_t* + */ +taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const char **label_values); + +#endif // TAOS_METRIC_H diff --git a/include/libs/monitorfw/taos_metric_sample.h b/include/libs/monitorfw/taos_metric_sample.h new file mode 100644 index 0000000000..1c37f59e32 --- /dev/null +++ b/include/libs/monitorfw/taos_metric_sample.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_metric_sample.h + * @brief Functions for interfacting with metric samples directly + */ + +#ifndef TAOS_METRIC_SAMPLE_H +#define TAOS_METRIC_SAMPLE_H + +struct taos_metric_sample; +/** + * @brief Contains the specific metric and value given the name and label set + * Reference: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels + */ +typedef struct taos_metric_sample taos_metric_sample_t; + +/** + * @brief Add the r_value to the sample. The value must be greater than or equal to zero. + * @param self The target taos_metric_sample_t* + * @param r_value The double to add to taos_metric_sample_t* provided by self + * @return Non-zero integer value upon failure + */ +int taos_metric_sample_add(taos_metric_sample_t *self, double r_value); + +/** + * @brief Subtract the r_value from the sample. + * + * This operation MUST be called a sample derived from a gauge metric. + * @param self The target taos_metric_sample_t* + * @param r_value The double to subtract from the taos_metric_sample_t* provided by self + * @return Non-zero integer value upon failure + */ +int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value); + +/** + * @brief Set the r_value of the sample. + * + * This operation MUST be called on a sample derived from a gauge metric. + * @param self The target taos_metric_sample_t* + * @param r_value The double which will be set to the taos_metric_sample_t* provided by self + * @return Non-zero integer value upon failure + */ +int taos_metric_sample_set(taos_metric_sample_t *self, double r_value); + +#endif // TAOS_METRIC_SAMPLE_H diff --git a/include/libs/monitorfw/taos_monitor.h b/include/libs/monitorfw/taos_monitor.h new file mode 100644 index 0000000000..ec4f007c45 --- /dev/null +++ b/include/libs/monitorfw/taos_monitor.h @@ -0,0 +1,130 @@ +/* +Copyright 2019 DigitalOcean Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/** + * @file taos_monitor.h + * @brief Include taos_monitor.h to include the entire public API + * @mainpage Welcome to the documentation site for prometheus-client-c! + * @tableofcontents + * @section Introduction + * + * prometheus-client-c is a small suite of Prometheus client libraries targeted for the C programming language. + * In this brief tutorial you will learn how to create and register metrics, update metric samples, and expose metrics + * over HTTP. + * + * @section Creating-and-Registering-Metrics Creating and Registering Metrics + * + * prometheus-client-c supports the following metric types: + * + * * [Counter](https://prometheus.io/docs/concepts/metric_types/#counter) + * * [Gauge](https://prometheus.io/docs/concepts/metric_types/#gauge) + * * [Histogram](https://prometheus.io/docs/concepts/metric_types/#histogram) + * + * To get started using one of the metric types, declare the metric at file scope. For example: + * + * @code{.c} + * + * #incldue "taos_monitor.h" + * + * taos_counter_t *my_counter; + * + * @endcode + * + * Next, create a metric initialization function. You can create the metric and register it with the default metric + * collector registry in one chain of functions. A metric collector is responsible for collecting metrics and returning + * them. A metric collector registry is declared in global scope and contains metric collectors. More on this later... + * + * To create a metric and register it with the default metric collector registry in one shot, you may chain the metric + * constructor into the taos_collector_registry_must_register_metric function. For example: + * + * @code{.c} + * + * void foo_metric_init(void) { + * my_counter = taos_collector_registry_must_register_metric(taos_counter_new("my_counter", "counts things", 0, NULL)); + * } + * + * @endcode + * + * The first argument to taos_counter_new is the counter name. The second argument is the counter description. The third + * argument is the number of metric labels. In this case, we will only have one metric sample for this metric so we pass + * 0 to specify that no labels will be used. The 4th argument is an array of strings storing the metric labels. Since we + * have none, we pass NULL. A call to foo_metric_init within the program's main function will initialize the metrics + * for the file we just created to the default prometheus metric collector registery called + * TAOS_COLLECTOR_REGISTRY_DEFAULT + * + * @section Updating-Metric-Sample-Values Updating Metric Sample Values + * + * Now that we have a metric configured for creation and registration, we can update our metric within any of the + * functions of the file in which it was declared. For example: + * + * @code{.c} + * + * void my_lib_do_something(void) { + * printf("I did a really important thing!\n"); + * taos_counter_inc(my_counter, NULL); + * } + * @endcode + * + * This function will increment the default metric sample for my_counter. Since we are not using metric labels, we pass + * NULL as the second argument. + * + * @section Program-Initialization Program Initialization + * + * At the start of the program's main function you need to do two things: + * + * * Initialize the default metric collector registry: + * + * @code{.c} + * + * taos_collector_registry_default_init(); + * + * @endcode + * + * * For each file containing prometheus metrics, call its corresponding metric initialization function + * + * @code{.c} + * + * foo_metric_init() + * + * @endcode + * + * After initialization is complete, you may proceed to do work and update your metrics. + * + * @section Metric-Exposition-Over-HTTP Metric Exposition Over HTTP + * + * @todo Describe how to use libpromhttp to expose metrics over HTTP + * + * @section Where-To-Go-From-Here Where to Go From Here? + * + * Take a look at the [Files](https://github.internal.digitalocean.com/pages/timeseries/prometheus-client-c/files.html) + * tab in this documentation site for more information about the public API available to you. Also, you can take a look + * at the examples directory at the + * [Github repository](https://github.internal.digitalocean.com/timeseries/prometheus-client-c) for inspiration. + */ + +#ifndef TAOS_INCLUDED +#define TAOS_INCLUDED + +#include "taos_alloc.h" +#include "taos_collector.h" +#include "taos_collector_registry.h" +#include "taos_counter.h" +#include "taos_linked_list.h" +#include "taos_map.h" +#include "taos_metric.h" +#include "taos_metric_sample.h" + +#endif // TAOS_INCLUDED \ No newline at end of file diff --git a/include/util/tlog.h b/include/util/tlog.h index a6d146a79e..1d795045ef 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -118,6 +118,7 @@ void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile); #define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }} #define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", DEBUG_TRACE, uDebugFlag, __VA_ARGS__); }} #define uDebugL(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLongString("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }} +#define uInfoL(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLongString("UTL ", DEBUG_INFO, uDebugFlag, __VA_ARGS__); }} #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); } #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index bd6eb46bad..d22d4c4cf5 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1423,6 +1423,44 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) { void tFreeSStatusRsp(SStatusRsp *pRsp) { taosArrayDestroy(pRsp->pDnodeEps); } +int32_t tSerializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + + if (tEncodeI32(&encoder, pReq->contLen) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->pCont) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeI32(&decoder, &pReq->contLen) < 0) return -1; + if (pReq->contLen > 0) { + pReq->pCont = taosMemoryMalloc(pReq->contLen + 1); + if (pReq->pCont == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->pCont) < 0) return -1; + } + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} + +void tFreeSStatisReq(SStatisReq *pReq) { + taosMemoryFreeClear(pReq->pCont); +} + int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index d5488da770..8f60a3dc85 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -189,7 +189,8 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_INDEX, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_INDEX, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RESTORE_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - + if (dmSetMgmtHandle(pArray, TDMT_MND_STATIS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_CONTINUE, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/node_mgmt/CMakeLists.txt b/source/dnode/mgmt/node_mgmt/CMakeLists.txt index f1be20289a..3761020560 100644 --- a/source/dnode/mgmt/node_mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/node_mgmt/CMakeLists.txt @@ -1,7 +1,7 @@ aux_source_directory(src IMPLEMENT_SRC) add_library(dnode STATIC ${IMPLEMENT_SRC}) target_link_libraries( - dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode + dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw ) target_include_directories( dnode diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index b3db7c3058..c237ce73c0 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -106,6 +106,8 @@ void dmSendMonitorReport() { dmGetQmMonitorInfo(pDnode); dmGetSmMonitorInfo(pDnode); monSendReport(); + + monSendPromReport(); } void dmGetVnodeLoads(SMonVloadInfo *pInfo) { diff --git a/source/dnode/mgmt/node_util/CMakeLists.txt b/source/dnode/mgmt/node_util/CMakeLists.txt index 5c670cbdd3..d882d784de 100644 --- a/source/dnode/mgmt/node_util/CMakeLists.txt +++ b/source/dnode/mgmt/node_util/CMakeLists.txt @@ -6,5 +6,5 @@ target_include_directories( PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor + node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor monitorfw ) \ No newline at end of file diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 0a52c578a5..4c00a48796 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -39,6 +39,7 @@ #include "sync.h" #include "tfs.h" #include "wal.h" +#include "taos_monitor.h" #ifdef __cplusplus extern "C" { diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 48dc71a12b..8f9564b072 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -16,7 +16,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser audit + mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser audit monitorfw ) IF (TD_GRANT) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index b53dee7bff..01332472a1 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -73,6 +73,7 @@ static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessStatusReq(SRpcMsg *pReq); static int32_t mndProcessNotifyReq(SRpcMsg *pReq); static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq); +static int32_t mndProcessStatisReq(SRpcMsg *pReq); static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); @@ -108,6 +109,7 @@ int32_t mndInitDnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq); mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq); mndSetMsgHandle(pMnode, TDMT_MND_RESTORE_DNODE, mndProcessRestoreDnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_STATIS, mndProcessStatisReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndRetrieveConfigs); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndCancelGetNextConfig); @@ -489,6 +491,23 @@ static bool mndUpdateMnodeState(SMnodeObj *pObj, SMnodeLoad *pMload) { return stateChanged; } +static int32_t mndProcessStatisReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SStatisReq statisReq = {0}; + int32_t code = -1; + + if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + monSendContent(pReq->pCont); + +_OVER: + tFreeSStatisReq(&statisReq); + return code; +} + static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStatusReq statusReq = {0}; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 22b2fec857..a93973fd13 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -23,6 +23,7 @@ #include "mndTrans.h" #include "tmisce.h" #include "audit.h" +#include "taos_monitor.h" #define MNODE_VER_NUMBER 2 #define MNODE_RESERVE_SIZE 64 diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index efa722d41a..c4aa5c6428 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -20,6 +20,7 @@ #include "vnode.h" #include "vnodeInt.h" #include "audit.h" +#include "taos_monitor.h" static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -40,6 +41,8 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessConfigChangeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +taos_counter_t *insert_counter = NULL; + static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t btime, int64_t *pUid) { int32_t code = 0; int32_t lino = 0; @@ -1613,6 +1616,19 @@ _exit: tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT); } + if(insert_counter == NULL){ + int32_t label_count =1; + const char *sample_labels[] = {"vgid"}; + insert_counter = taos_counter_new("insert_counter", "counter for insert sql", label_count, sample_labels); + insert_counter = taos_collector_registry_must_register_metric(insert_counter); + } + + char vgId[50]; + sprintf(vgId, "%"PRId32, TD_VID(pVnode)); + const char *sample_labels[] = {vgId}; + + taos_counter_inc(insert_counter, sample_labels); + // clear taosArrayDestroy(newTbUids); tDestroySubmitReq(pSubmitReq, 0 == pMsg->version ? TSDB_MSG_FLG_CMPT : TSDB_MSG_FLG_DECODE); diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 9f812517c1..6196d30a58 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -8,6 +8,7 @@ add_subdirectory(qcom) add_subdirectory(nodes) add_subdirectory(catalog) add_subdirectory(audit) +add_subdirectory(monitorfw) add_subdirectory(scalar) add_subdirectory(function) diff --git a/source/libs/monitor/CMakeLists.txt b/source/libs/monitor/CMakeLists.txt index 30dce7aaef..13523fd3cc 100644 --- a/source/libs/monitor/CMakeLists.txt +++ b/source/libs/monitor/CMakeLists.txt @@ -6,7 +6,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries(monitor os util common transport) +target_link_libraries(monitor os util common transport monitorfw) if(${BUILD_TEST}) add_subdirectory(test) diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 56cf0a2b51..93abda1281 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -18,6 +18,7 @@ #include "taoserror.h" #include "thttp.h" #include "ttime.h" +#include "taos_monitor.h" static SMonitor tsMonitor = {0}; static char* tsMonUri = "/report"; @@ -108,6 +109,9 @@ int32_t monInit(const SMonCfg *pCfg) { tsLogFp = monRecordLog; tsMonitor.lastTime = taosGetTimestampMs(); taosThreadMutexInit(&tsMonitor.lock, NULL); + + taos_collector_registry_default_init(); + return 0; } @@ -542,3 +546,27 @@ void monSendReport() { monCleanupMonitorInfo(pMonitor); } + +void monSendPromReport() { + char *pCont = (char *)taos_collector_registry_bridge( + TAOS_COLLECTOR_REGISTRY_DEFAULT, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + uInfoL("report cont:\n%s\n", pCont); + if (pCont != NULL) { + EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + uError("failed to send monitor msg"); + }else{ + taos_collector_registry_clear_out(TAOS_COLLECTOR_REGISTRY_DEFAULT); + } + } +} + +void monSendContent(char *pCont) { + uInfoL("report cont:\n%s\n", pCont); + if (pCont != NULL) { + EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + uError("failed to send monitor msg"); + } + } +} \ No newline at end of file diff --git a/source/libs/monitorfw/CMakeLists.txt b/source/libs/monitorfw/CMakeLists.txt new file mode 100644 index 0000000000..610cd63985 --- /dev/null +++ b/source/libs/monitorfw/CMakeLists.txt @@ -0,0 +1,9 @@ +aux_source_directory(src MONITOR2_SRC) +add_library(monitorfw STATIC ${MONITOR2_SRC}) +target_include_directories( + monitorfw + PUBLIC "${TD_SOURCE_DIR}/include/libs/monitorfw" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries(monitorfw os util common transport) diff --git a/source/libs/monitorfw/inc/taos_assert.h b/source/libs/monitorfw/inc/taos_assert.h new file mode 100644 index 0000000000..d3226eed26 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_assert.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#ifndef TAOS_ASSERT_H +#define TAOS_ASSERT_H + +#ifdef TAOS_ASSERT_ENABLE +#define TAOS_ASSERT(i) assert(i); +#else +#define TAOS_ASSERT(i) +#endif // TAOS_TEST + +#endif // TAOS_ASSERT_H diff --git a/source/libs/monitorfw/inc/taos_collector_registry_i.h b/source/libs/monitorfw/inc/taos_collector_registry_i.h new file mode 100644 index 0000000000..ed5bb1fe19 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_collector_registry_i.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "taos_collector_registry_t.h" + +#ifndef TAOS_COLLECTOR_REGISTRY_I_INCLUDED +#define TAOS_COLLECTOR_REGISTRY_I_INCLUDED + +int taos_collector_registry_enable_custom_process_metrics(taos_collector_registry_t *self, + const char *process_limits_path, + const char *process_stats_path); + +#endif // TAOS_COLLECTOR_REGISTRY_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_collector_registry_t.h b/source/libs/monitorfw/inc/taos_collector_registry_t.h new file mode 100644 index 0000000000..2264d18081 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_collector_registry_t.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_REGISTRY_T_H +#define TAOS_REGISTRY_T_H + +#include +#include + +// Public +#include "taos_collector_registry.h" + +// Private +#include "taos_map_t.h" +#include "taos_metric_formatter_t.h" +#include "taos_string_builder_t.h" + +struct taos_collector_registry { + const char *name; + bool disable_process_metrics; /**< Disables the collection of process metrics */ + taos_map_t *collectors; /**< Map of collectors keyed by name */ + taos_string_builder_t *string_builder; /**< Enables string building */ + taos_metric_formatter_t *metric_formatter; /**< metric formatter for metric exposition on bridge call */ + pthread_rwlock_t *lock; /**< mutex for safety against concurrent registration */ + taos_string_builder_t *out; +}; + +#endif // TAOS_REGISTRY_T_H diff --git a/source/libs/monitorfw/inc/taos_collector_t.h b/source/libs/monitorfw/inc/taos_collector_t.h new file mode 100644 index 0000000000..264e8e9ad9 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_collector_t.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_COLLECTOR_T_H +#define TAOS_COLLECTOR_T_H + +#include "taos_collector.h" +#include "taos_map_t.h" +#include "taos_string_builder_t.h" + +struct taos_collector { + const char *name; + taos_map_t *metrics; + taos_collect_fn *collect_fn; + taos_string_builder_t *string_builder; + const char *proc_limits_file_path; + const char *proc_stat_file_path; +}; + +#endif // TAOS_COLLECTOR_T_H diff --git a/source/libs/monitorfw/inc/taos_errors.h b/source/libs/monitorfw/inc/taos_errors.h new file mode 100644 index 0000000000..ee2a894df3 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_errors.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define TAOS_STDIO_CLOSE_DIR_ERROR "failed to close dir" +#define TAOS_STDIO_OPEN_DIR_ERROR "failed to open dir" +#define TAOS_METRIC_INCORRECT_TYPE "incorrect metric type" +#define TAOS_METRIC_INVALID_LABEL_NAME "invalid label name" +#define TAOS_PTHREAD_RWLOCK_DESTROY_ERROR "failed to destroy the pthread_rwlock_t*" +#define TAOS_PTHREAD_RWLOCK_INIT_ERROR "failed to initialize the pthread_rwlock_t*" +#define TAOS_PTHREAD_RWLOCK_LOCK_ERROR "failed to lock the pthread_rwlock_t*" +#define TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR "failed to unlock the pthread_rwlock_t*" +#define TAOS_REGEX_REGCOMP_ERROR "failed to compile the regular expression" +#define TAOS_REGEX_REGEXEC_ERROR "failed to execute the regular expression" diff --git a/source/libs/monitorfw/inc/taos_linked_list_i.h b/source/libs/monitorfw/inc/taos_linked_list_i.h new file mode 100644 index 0000000000..ed11a76427 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_linked_list_i.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_LIST_I_INCLUDED +#define TAOS_LIST_I_INCLUDED + +// Private +#include "taos_linked_list_t.h" + +/** + * @brief API PRIVATE Returns a pointer to a taos_linked_list + */ +taos_linked_list_t *taos_linked_list_new(void); + +/** + * @brief API PRIVATE removes all nodes from the given taos_linked_list * + */ +int taos_linked_list_purge(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Destroys a taos_linked_list + */ +int taos_linked_list_destroy(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Append an item to the back of the list + */ +int taos_linked_list_append(taos_linked_list_t *self, void *item); + +/** + * @brief API PRIVATE Push an item onto the front of the list + */ +int taos_linked_list_push(taos_linked_list_t *self, void *item); + +/** + * @brief API PRIVATE Pop the first item off of the list + */ +void *taos_linked_list_pop(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Returns the item at the head of the list or NULL if not present + */ +void *taos_linked_list_first(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Returns the item at the tail of the list or NULL if not present + */ +void *taos_linked_list_last(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Removes an item from the linked list + */ +int taos_linked_list_remove(taos_linked_list_t *self, void *item); + +/** + * @brief API PRIVATE Compares two items within a linked list + */ +taos_linked_list_compare_t taos_linked_list_compare(taos_linked_list_t *self, void *item_a, void *node_b); + +/** + * @brief API PRIVATE Get the size + */ +size_t taos_linked_list_size(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Set the free_fn member on taos_linked_list + */ +int taos_linked_list_set_free_fn(taos_linked_list_t *self, taos_linked_list_free_item_fn free_fn); + +/** + * @brief API PRIVATE Set the compare_fn member on the taos_linked_list + */ +int taos_linked_list_set_compare_fn(taos_linked_list_t *self, taos_linked_list_compare_item_fn compare_fn); + +/** + * API PRIVATE + * @brief does nothing + */ +void taos_linked_list_no_op_free(void *item); + +#endif // TAOS_LIST_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_linked_list_t.h b/source/libs/monitorfw/inc/taos_linked_list_t.h new file mode 100644 index 0000000000..ccd82ffb61 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_linked_list_t.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_LIST_T_H +#define TAOS_LIST_T_H + +#include "taos_linked_list.h" + +typedef enum { TAOS_LESS = -1, TAOS_EQUAL = 0, TAOS_GREATER = 1 } taos_linked_list_compare_t; + +/** + * @brief API PRIVATE Frees an item in a taos_linked_list_node + */ +typedef void (*taos_linked_list_free_item_fn)(void *); + +/** + * @brief API PRIVATE Compares two items within a taos_linked_list + */ +typedef taos_linked_list_compare_t (*taos_linked_list_compare_item_fn)(void *item_a, void *item_b); + +/** + * @brief API PRIVATE A struct containing a generic item, represented as a void pointer, and next, a pointer to the + * next taos_linked_list_node* + */ +typedef struct taos_linked_list_node { + struct taos_linked_list_node *next; + void *item; +} taos_linked_list_node_t; + +/** + * @brief API PRIVATE A linked list comprised of taos_linked_list_node* instances + */ +struct taos_linked_list { + taos_linked_list_node_t *head; + taos_linked_list_node_t *tail; + size_t size; + taos_linked_list_free_item_fn free_fn; + taos_linked_list_compare_item_fn compare_fn; +}; + +#endif // TAOS_LIST_T_H diff --git a/source/libs/monitorfw/inc/taos_log.h b/source/libs/monitorfw/inc/taos_log.h new file mode 100644 index 0000000000..6d6c9e1fb3 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_log.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#ifndef TAOS_LOG_H +#define TAOS_LOG_H + +#ifdef TAOS_LOG_ENABLE +#define TAOS_LOG(msg) printf("%s %s %s %s %d %s\n", __DATE__, __TIME__, __FILE__, __FUNCTION__, __LINE__, msg); +#else +#define TAOS_LOG(msg) +#endif // TAOS_LOG_ENABLE + +#endif // TAOS_LOG_H diff --git a/source/libs/monitorfw/inc/taos_map_i.h b/source/libs/monitorfw/inc/taos_map_i.h new file mode 100644 index 0000000000..808548e96a --- /dev/null +++ b/source/libs/monitorfw/inc/taos_map_i.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_MAP_I_INCLUDED +#define TAOS_MAP_I_INCLUDED + +#include "taos_map_t.h" + +taos_map_t *taos_map_new(void); + +int taos_map_set_free_value_fn(taos_map_t *self, taos_map_node_free_value_fn free_value_fn); + +void *taos_map_get(taos_map_t *self, const char *key); + +int taos_map_set(taos_map_t *self, const char *key, void *value); + +int taos_map_delete(taos_map_t *self, const char *key); + +int taos_map_destroy(taos_map_t *self); + +size_t taos_map_size(taos_map_t *self); + +taos_map_node_t *taos_map_node_new(const char *key, void *value, taos_map_node_free_value_fn free_value_fn); + +#endif // TAOS_MAP_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_map_t.h b/source/libs/monitorfw/inc/taos_map_t.h new file mode 100644 index 0000000000..6fcbda1366 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_map_t.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_MAP_T_H +#define TAOS_MAP_T_H + +#include + +// Public +#include "taos_map.h" + +// Private +#include "taos_linked_list_t.h" + +typedef void (*taos_map_node_free_value_fn)(void *); + +struct taos_map_node { + const char *key; + void *value; + taos_map_node_free_value_fn free_value_fn; +}; + +struct taos_map { + size_t size; /**< contains the size of the map */ + size_t max_size; /**< stores the current max_size */ + taos_linked_list_t *keys; /**< linked list containing containing all keys present */ + taos_linked_list_t **addrs; /**< Sequence of linked lists. Each list contains nodes with the same index */ + pthread_rwlock_t *rwlock; + taos_map_node_free_value_fn free_value_fn; +}; + +#endif // TAOS_MAP_T_H diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_i.h b/source/libs/monitorfw/inc/taos_metric_formatter_i.h new file mode 100644 index 0000000000..9a63850ea4 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_formatter_i.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_FORMATTER_I_H +#define TAOS_METRIC_FORMATTER_I_H + +// Private +#include "taos_metric_formatter_t.h" +#include "taos_metric_t.h" + +/** + * @brief API PRIVATE taos_metric_formatter constructor + */ +taos_metric_formatter_t *taos_metric_formatter_new(); + +/** + * @brief API PRIVATE taos_metric_formatter destructor + */ +int taos_metric_formatter_destroy(taos_metric_formatter_t *self); + +/** + * @brief API PRIVATE Loads the help text + */ +int taos_metric_formatter_load_help(taos_metric_formatter_t *self, const char *name, const char *help); + +/** + * @brief API PRIVATE Loads the type text + */ +int taos_metric_formatter_load_type(taos_metric_formatter_t *self, const char *name, taos_metric_type_t metric_type); + +/** + * @brief API PRIVATE Loads the formatter with a metric sample L-value + * @param name The metric name + * @param suffix The metric suffix. This is applicable to Summary and Histogram metric types. + * @param label_count The number of labels for the given metric. + * @param label_keys An array of constant strings. + * @param label_values An array of constant strings. + * + * The number of const char **and taos_label_value must be the same. + */ +int taos_metric_formatter_load_l_value(taos_metric_formatter_t *metric_formatter, const char *name, const char *suffix, + size_t label_count, const char **label_keys, const char **label_values); + +/** + * @brief API PRIVATE Loads the formatter with a metric sample + */ +int taos_metric_formatter_load_sample( + taos_metric_formatter_t *metric_formatter, taos_metric_sample_t *sample, int64_t ts); + +/** + * @brief API PRIVATE Loads a metric in the string exposition format + */ +int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, int64_t ts); + +/** + * @brief API PRIVATE Loads the given metrics + */ +int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, int64_t ts); + +/** + * @brief API PRIVATE Clear the underlying string_builder + */ +int taos_metric_formatter_clear(taos_metric_formatter_t *self); + +/** + * @brief API PRIVATE Returns the string built by taos_metric_formatter + */ +char *taos_metric_formatter_dump(taos_metric_formatter_t *metric_formatter); + +#endif // TAOS_METRIC_FORMATTER_I_H diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_t.h b/source/libs/monitorfw/inc/taos_metric_formatter_t.h new file mode 100644 index 0000000000..0d7425aa59 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_formatter_t.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_FORMATTER_T_H +#define TAOS_METRIC_FORMATTER_T_H + +#include "taos_string_builder_t.h" + +typedef struct taos_metric_formatter { + taos_string_builder_t *string_builder; + taos_string_builder_t *err_builder; +} taos_metric_formatter_t; + +#endif // TAOS_METRIC_FORMATTER_T_H diff --git a/source/libs/monitorfw/inc/taos_metric_i.h b/source/libs/monitorfw/inc/taos_metric_i.h new file mode 100644 index 0000000000..e8ae799547 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_i.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Private +#include "taos_metric_t.h" + +#ifndef TAOS_METRIC_I_INCLUDED +#define TAOS_METRIC_I_INCLUDED + +/** + * @brief API PRIVATE Returns a *taos_metric + */ +taos_metric_t *taos_metric_new(taos_metric_type_t type, const char *name, const char *help, size_t label_key_count, + const char **label_keys); + +/** + * @brief API PRIVATE Destroys a *taos_metric + */ +int taos_metric_destroy(taos_metric_t *self); + +/** + * @brief API PRIVATE takes a generic item, casts to a *taos_metric_t and destroys it + */ +int taos_metric_destroy_generic(void *item); + +/** + * @brief API Private takes a generic item, casts to a *taos_metric_t and destroys it. Discards any errors. + */ +void taos_metric_free_generic(void *item); + +#endif // TAOS_METRIC_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_metric_sample_i.h b/source/libs/monitorfw/inc/taos_metric_sample_i.h new file mode 100644 index 0000000000..b5e90f1933 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_sample_i.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "taos_metric_sample_t.h" +#include "taos_metric_t.h" + +#ifndef TAOS_METRIC_SAMPLE_I_H +#define TAOS_METRIC_SAMPLE_I_H + +/** + * @brief API PRIVATE Return a taos_metric_sample_t* + * + * @param type The type of metric sample + * @param l_value The entire left value of the metric e.g metric_name{foo="bar"} + * @param r_value A double representing the value of the sample + */ +taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value); + +/** + * @brief API PRIVATE Destroy the taos_metric_sample** + */ +int taos_metric_sample_destroy(taos_metric_sample_t *self); + +/** + * @brief API PRIVATE A taos_linked_list_free_item_fn to enable item destruction within a linked list's destructor + */ +int taos_metric_sample_destroy_generic(void *); + +/** + * @brief API PRIVATE A taos_linked_list_free_item_fn to enable item destruction within a linked list's destructor. + * + * This function ignores any errors. + */ +void taos_metric_sample_free_generic(void *gen); + +#endif // TAOS_METRIC_SAMPLE_I_H diff --git a/source/libs/monitorfw/inc/taos_metric_sample_t.h b/source/libs/monitorfw/inc/taos_metric_sample_t.h new file mode 100644 index 0000000000..59a398b938 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_sample_t.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_SAMPLE_T_H +#define TAOS_METRIC_SAMPLE_T_H + +#include "taos_metric_sample.h" +#include "taos_metric_t.h" + +struct taos_metric_sample { + taos_metric_type_t type; /**< type is the metric type for the sample */ + char *l_value; /**< l_value is the full metric name and label set represeted as a string */ + _Atomic double r_value; /**< r_value is the value of the metric sample */ +}; + +#endif // TAOS_METRIC_SAMPLE_T_H diff --git a/source/libs/monitorfw/inc/taos_metric_t.h b/source/libs/monitorfw/inc/taos_metric_t.h new file mode 100644 index 0000000000..806466528d --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_t.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_T_H +#define TAOS_METRIC_T_H + +#include + +// Public +#include "taos_metric.h" + +// Private +#include "taos_map_i.h" +#include "taos_map_t.h" +#include "taos_metric_formatter_t.h" + +/** + * @brief API PRIVATE Contains metric type constants + */ +typedef enum taos_metric_type { TAOS_COUNTER, TAOS_GAUGE, TAOS_HISTOGRAM, TAOS_SUMMARY } taos_metric_type_t; + +/** + * @brief API PRIVATE Maps metric type constants to human readable string values + */ +extern char *taos_metric_type_map[4]; + +/** + * @brief API PRIVATE An opaque struct to users containing metric metadata; one or more metric samples; and a metric + * formatter for locating metric samples and exporting metric data + */ +struct taos_metric { + taos_metric_type_t type; /**< metric_type The type of metric */ + const char *name; /**< name The name of the metric */ + const char *help; /**< help The help output for the metric */ + taos_map_t *samples; /**< samples Map comprised of samples for the given metric */ + size_t label_key_count; /**< label_keys_count The count of labe_keys*/ + taos_metric_formatter_t *formatter; /**< formatter The metric formatter */ + pthread_rwlock_t *rwlock; /**< rwlock Required for locking on certain non-atomic operations */ + const char **label_keys; /**< labels Array comprised of const char **/ +}; + +#endif // TAOS_METRIC_T_H diff --git a/source/libs/monitorfw/inc/taos_string_builder_i.h b/source/libs/monitorfw/inc/taos_string_builder_i.h new file mode 100644 index 0000000000..142ca020ba --- /dev/null +++ b/source/libs/monitorfw/inc/taos_string_builder_i.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_STRING_BUILDER_I_H +#define TAOS_STRING_BUILDER_I_H + +#include + +#include "taos_string_builder_t.h" + +/** + * API PRIVATE + * @brief Constructor for taos_string_builder + */ +taos_string_builder_t *taos_string_builder_new(void); + +/** + * API PRIVATE + * @brief Destroys a taos_string_builder* + */ +int taos_string_builder_destroy(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Adds a string + */ +int taos_string_builder_add_str(taos_string_builder_t *self, const char *str); + +/** + * API PRIVATE + * @brief Adds a char + */ +int taos_string_builder_add_char(taos_string_builder_t *self, char c); + +/** + * API PRIVATE + * @brief Clear the string + */ +int taos_string_builder_clear(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Remove data from the end + */ +int taos_string_buillder_truncate(taos_string_builder_t *self, size_t len); + +/** + * API PRIVATE + * @brief Returns the length of the string + */ +size_t taos_string_builder_len(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Returns a copy of the string. The returned string must be deallocated when no longer needed. + */ +char *taos_string_builder_dump(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Getter for str member + */ +char *taos_string_builder_str(taos_string_builder_t *self); + +#endif // TAOS_STRING_BUILDER_I_H diff --git a/source/libs/monitorfw/inc/taos_string_builder_t.h b/source/libs/monitorfw/inc/taos_string_builder_t.h new file mode 100644 index 0000000000..edd3d574fa --- /dev/null +++ b/source/libs/monitorfw/inc/taos_string_builder_t.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_STRING_BUILDER_T_H +#define TAOS_STRING_BUILDER_T_H + +struct taos_string_builder; +/** + * @brief API PRIVATE A structure with functions responsible for building a string + */ +typedef struct taos_string_builder taos_string_builder_t; + +#endif // TAOS_STRING_BUILDER_T_H diff --git a/source/libs/monitorfw/src/taos_collector.c b/source/libs/monitorfw/src/taos_collector.c new file mode 100644 index 0000000000..9d96f61b4b --- /dev/null +++ b/source/libs/monitorfw/src/taos_collector.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +// Public +#include "taos_alloc.h" +#include "taos_collector.h" +#include "taos_collector_registry.h" + +// Private +#include "taos_assert.h" +#include "taos_collector_t.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_metric_i.h" +#include "taos_string_builder_i.h" + +taos_map_t *taos_collector_default_collect(taos_collector_t *self) { return self->metrics; } + +taos_collector_t *taos_collector_new(const char *name) { + int r = 0; + taos_collector_t *self = (taos_collector_t *)taos_malloc(sizeof(taos_collector_t)); + self->name = taos_strdup(name); + self->metrics = taos_map_new(); + if (self->metrics == NULL) { + taos_collector_destroy(self); + return NULL; + } + r = taos_map_set_free_value_fn(self->metrics, &taos_metric_free_generic); + if (r) { + taos_collector_destroy(self); + return NULL; + } + self->collect_fn = &taos_collector_default_collect; + self->string_builder = taos_string_builder_new(); + if (self->string_builder == NULL) { + taos_collector_destroy(self); + return NULL; + } + self->proc_limits_file_path = NULL; + self->proc_stat_file_path = NULL; + return self; +} + +int taos_collector_destroy(taos_collector_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_map_destroy(self->metrics); + if (r) ret = r; + self->metrics = NULL; + + r = taos_string_builder_destroy(self->string_builder); + if (r) ret = r; + self->string_builder = NULL; + + taos_free((char *)self->name); + self->name = NULL; + taos_free(self); + self = NULL; + + return ret; +} + +int taos_collector_destroy_generic(void *gen) { + int r = 0; + taos_collector_t *self = (taos_collector_t *)gen; + r = taos_collector_destroy(self); + self = NULL; + return r; +} + +void taos_collector_free_generic(void *gen) { + taos_collector_t *self = (taos_collector_t *)gen; + taos_collector_destroy(self); +} + +int taos_collector_set_collect_fn(taos_collector_t *self, taos_collect_fn *fn) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->collect_fn = fn; + return 0; +} + +int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (taos_map_get(self->metrics, metric->name) != NULL) { + TAOS_LOG("metric already found in collector"); + return 1; + } + return taos_map_set(self->metrics, metric->name, metric); +} diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c new file mode 100644 index 0000000000..7efe82ad23 --- /dev/null +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +// Public +#include "taos_alloc.h" +#include "taos_collector.h" +#include "taos_collector_registry.h" + +// Private +#include "taos_assert.h" +#include "taos_collector_registry_t.h" +#include "taos_collector_t.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_metric_formatter_i.h" +#include "taos_metric_i.h" +#include "taos_metric_t.h" +#include "taos_string_builder_i.h" + +taos_collector_registry_t *TAOS_COLLECTOR_REGISTRY_DEFAULT; + +taos_collector_registry_t *taos_collector_registry_new(const char *name) { + int r = 0; + + taos_collector_registry_t *self = (taos_collector_registry_t *)taos_malloc(sizeof(taos_collector_registry_t)); + + self->disable_process_metrics = false; + + self->name = taos_strdup(name); + self->collectors = taos_map_new(); + taos_map_set_free_value_fn(self->collectors, &taos_collector_free_generic); + taos_map_set(self->collectors, "default", taos_collector_new("default")); + + self->metric_formatter = taos_metric_formatter_new(); + self->string_builder = taos_string_builder_new(); + self->out = taos_string_builder_new(); + self->lock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t)); + r = pthread_rwlock_init(self->lock, NULL); + if (r) { + TAOS_LOG("failed to initialize rwlock"); + return NULL; + } + return self; +} + +int taos_collector_registry_default_init(void) { + if (TAOS_COLLECTOR_REGISTRY_DEFAULT != NULL) return 0; + + TAOS_COLLECTOR_REGISTRY_DEFAULT = taos_collector_registry_new("default"); + //if (TAOS_COLLECTOR_REGISTRY_DEFAULT) { + // return taos_collector_registry_enable_process_metrics(TAOS_COLLECTOR_REGISTRY_DEFAULT); + //} + return 1; +} + +int taos_collector_registry_destroy(taos_collector_registry_t *self) { + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_map_destroy(self->collectors); + self->collectors = NULL; + if (r) ret = r; + + r = taos_metric_formatter_destroy(self->metric_formatter); + self->metric_formatter = NULL; + if (r) ret = r; + + r = taos_string_builder_destroy(self->string_builder); + self->string_builder = NULL; + if (r) ret = r; + + r = pthread_rwlock_destroy(self->lock); + taos_free(self->lock); + self->lock = NULL; + if (r) ret = r; + + taos_free((char *)self->name); + self->name = NULL; + + taos_free(self); + self = NULL; + + return ret; +} + +int taos_collector_registry_register_metric(taos_metric_t *metric) { + TAOS_ASSERT(metric != NULL); + + taos_collector_t *default_collector = + (taos_collector_t *)taos_map_get(TAOS_COLLECTOR_REGISTRY_DEFAULT->collectors, "default"); + + if (default_collector == NULL) { + return 1; + } + + return taos_collector_add_metric(default_collector, metric); +} + +taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metric) { + int err = taos_collector_registry_register_metric(metric); + if (err != 0) { + exit(err); + } + return metric; +} + +int taos_collector_registry_register_collector(taos_collector_registry_t *self, taos_collector_t *collector) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = pthread_rwlock_wrlock(self->lock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + return 1; + } + if (taos_map_get(self->collectors, collector->name) != NULL) { + TAOS_LOG("the given taos_collector_t* is already registered"); + int rr = pthread_rwlock_unlock(self->lock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return 1; + } + } + r = taos_map_set(self->collectors, collector->name, collector); + if (r) { + int rr = pthread_rwlock_unlock(self->lock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return r; + } + } + r = pthread_rwlock_unlock(self->lock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return 1; + } + return 0; +} + +int taos_collector_registry_validate_metric_name(taos_collector_registry_t *self, const char *metric_name) { + regex_t r; + int ret = 0; + ret = regcomp(&r, "^[a-zA-Z_:][a-zA-Z0-9_:]*$", REG_EXTENDED); + if (ret) { + TAOS_LOG(TAOS_REGEX_REGCOMP_ERROR); + regfree(&r); + return ret; + } + + ret = regexec(&r, metric_name, 0, NULL, 0); + if (ret) { + TAOS_LOG(TAOS_REGEX_REGEXEC_ERROR); + regfree(&r); + return ret; + } + regfree(&r); + return 0; +} + +const char *taos_collector_registry_bridge(taos_collector_registry_t *self, int64_t ts) { + taos_metric_formatter_clear(self->metric_formatter); + taos_metric_formatter_load_metrics(self->metric_formatter, self->collectors, ts); + char *out = taos_metric_formatter_dump(self->metric_formatter); + + int r = 0; + r = taos_string_builder_add_str(self->out, out); + if (r) return NULL; + taos_free(out); + + return taos_string_builder_str(self->out); +} + +int taos_collector_registry_clear_out(taos_collector_registry_t *self){ + return taos_string_builder_clear(self->out); +} diff --git a/source/libs/monitorfw/src/taos_counter.c b/source/libs/monitorfw/src/taos_counter.c new file mode 100644 index 0000000000..d522411b2b --- /dev/null +++ b/source/libs/monitorfw/src/taos_counter.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Public +#include "taos_counter.h" + +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_metric_i.h" +#include "taos_metric_sample_i.h" +#include "taos_metric_sample_t.h" +#include "taos_metric_t.h" + +taos_counter_t *taos_counter_new(const char *name, const char *help, size_t label_key_count, const char **label_keys) { + return (taos_counter_t *)taos_metric_new(TAOS_COUNTER, name, help, label_key_count, label_keys); +} + +int taos_counter_destroy(taos_counter_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + int r = 0; + r = taos_metric_destroy(self); + self = NULL; + return r; +} + +int taos_counter_inc(taos_counter_t *self, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_COUNTER) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_add(sample, 1.0); +} + +int taos_counter_add(taos_counter_t *self, double r_value, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_COUNTER) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_add(sample, r_value); +} diff --git a/source/libs/monitorfw/src/taos_linked_list.c b/source/libs/monitorfw/src/taos_linked_list.c new file mode 100644 index 0000000000..ab4e23ff29 --- /dev/null +++ b/source/libs/monitorfw/src/taos_linked_list.c @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_linked_list_i.h" +#include "taos_linked_list_t.h" +#include "taos_log.h" + +taos_linked_list_t *taos_linked_list_new(void) { + taos_linked_list_t *self = (taos_linked_list_t *)taos_malloc(sizeof(taos_linked_list_t)); + self->head = NULL; + self->tail = NULL; + self->free_fn = NULL; + self->compare_fn = NULL; + self->size = 0; + return self; +} + +int taos_linked_list_purge(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node = self->head; + while (node != NULL) { + taos_linked_list_node_t *next = node->next; + if (node->item != NULL) { + if (self->free_fn) { + (*self->free_fn)(node->item); + } else { + taos_free(node->item); + } + } + taos_free(node); + node = NULL; + node = next; + } + self->head = NULL; + self->tail = NULL; + self->size = 0; + return 0; +} + +int taos_linked_list_destroy(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + int ret = 0; + + r = taos_linked_list_purge(self); + if (r) ret = r; + taos_free(self); + self = NULL; + return ret; +} + +void *taos_linked_list_first(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self->head) { + return self->head->item; + } else { + return NULL; + } +} + +void *taos_linked_list_last(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self->tail) { + return self->tail->item; + } else { + return NULL; + } +} + +int taos_linked_list_append(taos_linked_list_t *self, void *item) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node = (taos_linked_list_node_t *)taos_malloc(sizeof(taos_linked_list_node_t)); + + node->item = item; + if (self->tail) { + self->tail->next = node; + } else { + self->head = node; + } + self->tail = node; + node->next = NULL; + self->size++; + return 0; +} + +int taos_linked_list_push(taos_linked_list_t *self, void *item) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node = (taos_linked_list_node_t *)taos_malloc(sizeof(taos_linked_list_node_t)); + + node->item = item; + node->next = self->head; + self->head = node; + if (self->tail == NULL) { + self->tail = node; + } + self->size++; + return 0; +} + +void *taos_linked_list_pop(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return NULL; + taos_linked_list_node_t *node = self->head; + void *item = NULL; + if (node != NULL) { + item = node->item; + self->head = node->next; + if (self->tail == node) { + self->tail = NULL; + } + if (node->item != NULL) { + if (self->free_fn) { + (*self->free_fn)(node->item); + } else { + taos_free(node->item); + } + } + node->item = NULL; + node = NULL; + self->size--; + } + return item; +} + +int taos_linked_list_remove(taos_linked_list_t *self, void *item) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node; + taos_linked_list_node_t *prev_node = NULL; + + // Locate the node + for (node = self->head; node != NULL; node = node->next) { + if (self->compare_fn) { + if ((*self->compare_fn)(node->item, item) == TAOS_EQUAL) { + break; + } + } else { + if (node->item == item) { + break; + } + } + prev_node = node; + } + + if (node == NULL) return 0; + + if (prev_node) { + prev_node->next = node->next; + } else { + self->head = node->next; + } + if (node->next == NULL) { + self->tail = prev_node; + } + + if (node->item != NULL) { + if (self->free_fn) { + (*self->free_fn)(node->item); + } else { + taos_free(node->item); + } + } + + node->item = NULL; + taos_free(node); + node = NULL; + self->size--; + return 0; +} + +taos_linked_list_compare_t taos_linked_list_compare(taos_linked_list_t *self, void *item_a, void *item_b) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->compare_fn) { + return (*self->compare_fn)(item_a, item_b); + } else { + return strcmp(item_a, item_b); + } +} + +size_t taos_linked_list_size(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + return self->size; +} + +int taos_linked_list_set_free_fn(taos_linked_list_t *self, taos_linked_list_free_item_fn free_fn) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->free_fn = free_fn; + return 0; +} + +int taos_linked_list_set_compare_fn(taos_linked_list_t *self, taos_linked_list_compare_item_fn compare_fn) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->compare_fn = compare_fn; + return 0; +} + +void taos_linked_list_no_op_free(void *item) {} diff --git a/source/libs/monitorfw/src/taos_map.c b/source/libs/monitorfw/src/taos_map.c new file mode 100644 index 0000000000..fce308f11d --- /dev/null +++ b/source/libs/monitorfw/src/taos_map.c @@ -0,0 +1,414 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_linked_list_i.h" +#include "taos_linked_list_t.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_map_t.h" + +#define TAOS_MAP_INITIAL_SIZE 32 + +static void destroy_map_node_value_no_op(void *value) {} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// taos_map_node +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +taos_map_node_t *taos_map_node_new(const char *key, void *value, taos_map_node_free_value_fn free_value_fn) { + taos_map_node_t *self = taos_malloc(sizeof(taos_map_node_t)); + self->key = taos_strdup(key); + self->value = value; + self->free_value_fn = free_value_fn; + return self; +} + +int taos_map_node_destroy(taos_map_node_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + taos_free((void *)self->key); + self->key = NULL; + if (self->value != NULL) (*self->free_value_fn)(self->value); + self->value = NULL; + taos_free(self); + self = NULL; + return 0; +} + +void taos_map_node_free(void *item) { + taos_map_node_t *map_node = (taos_map_node_t *)item; + taos_map_node_destroy(map_node); +} + +taos_linked_list_compare_t taos_map_node_compare(void *item_a, void *item_b) { + taos_map_node_t *map_node_a = (taos_map_node_t *)item_a; + taos_map_node_t *map_node_b = (taos_map_node_t *)item_b; + + return strcmp(map_node_a->key, map_node_b->key); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// taos_map +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +taos_map_t *taos_map_new() { + int r = 0; + + taos_map_t *self = (taos_map_t *)taos_malloc(sizeof(taos_map_t)); + self->size = 0; + self->max_size = TAOS_MAP_INITIAL_SIZE; + + self->keys = taos_linked_list_new(); + if (self->keys == NULL) return NULL; + + // These each key will be allocated once by taos_map_node_new and used here as well to save memory. With that said + // we will only have to deallocate each key once. That will happen on taos_map_node_destroy. + r = taos_linked_list_set_free_fn(self->keys, taos_linked_list_no_op_free); + if (r) { + taos_map_destroy(self); + return NULL; + } + + self->addrs = taos_malloc(sizeof(taos_linked_list_t) * self->max_size); + self->free_value_fn = destroy_map_node_value_no_op; + + for (int i = 0; i < self->max_size; i++) { + self->addrs[i] = taos_linked_list_new(); + r = taos_linked_list_set_free_fn(self->addrs[i], taos_map_node_free); + if (r) { + taos_map_destroy(self); + return NULL; + } + r = taos_linked_list_set_compare_fn(self->addrs[i], taos_map_node_compare); + if (r) { + taos_map_destroy(self); + return NULL; + } + } + + self->rwlock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t)); + r = pthread_rwlock_init(self->rwlock, NULL); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_INIT_ERROR); + taos_map_destroy(self); + return NULL; + } + + return self; +} + +int taos_map_destroy(taos_map_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + int ret = 0; + + r = taos_linked_list_destroy(self->keys); + if (r) ret = r; + self->keys = NULL; + + for (size_t i = 0; i < self->max_size; i++) { + r = taos_linked_list_destroy(self->addrs[i]); + if (r) ret = r; + self->addrs[i] = NULL; + } + taos_free(self->addrs); + self->addrs = NULL; + + r = pthread_rwlock_destroy(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_DESTROY_ERROR) + ret = r; + } + + taos_free(self->rwlock); + self->rwlock = NULL; + taos_free(self); + self = NULL; + + return ret; +} + +static size_t taos_map_get_index_internal(const char *key, size_t *size, size_t *max_size) { + size_t index; + size_t a = 31415, b = 27183; + for (index = 0; *key != '\0'; key++, a = a * b % (*max_size - 1)) { + index = (a * index + *key) % *max_size; + } + return index; +} + +/** + * @brief API PRIVATE hash function that returns an array index from the given key and taos_map. + * + * The algorithm is based off of Horner's method. In a simpler version, you set the return value to 0. Next, for each + * character in the string, you add the integer value of the current character to the product of the prime number and + * the current return value, set the result to the return value, then finally return the return value. + * + * In this version of the algorithm, we attempt to achieve a probabily of key to index conversion collisions to + * 1/M (with M being the max_size of the map). This optimizes dispersion and consequently, evens out the performance + * for gets and sets for each item. Instead of using a fixed prime number, we generate a coefficient for each iteration + * through the loop. + * + * Reference: + * * Algorithms in C: Third Edition by Robert Sedgewick, p579 + */ +size_t taos_map_get_index(taos_map_t *self, const char *key) { + return taos_map_get_index_internal(key, &self->size, &self->max_size); +} + +static void *taos_map_get_internal(const char *key, size_t *size, size_t *max_size, taos_linked_list_t *keys, + taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn) { + size_t index = taos_map_get_index_internal(key, size, max_size); + taos_linked_list_t *list = addrs[index]; + taos_map_node_t *temp_map_node = taos_map_node_new(key, NULL, free_value_fn); + + for (taos_linked_list_node_t *current_node = list->head; current_node != NULL; current_node = current_node->next) { + taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item; + taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, temp_map_node); + if (result == TAOS_EQUAL) { + taos_map_node_destroy(temp_map_node); + temp_map_node = NULL; + return current_map_node->value; + } + } + taos_map_node_destroy(temp_map_node); + temp_map_node = NULL; + return NULL; +} + +void *taos_map_get(taos_map_t *self, const char *key) { + TAOS_ASSERT(self != NULL); + int r = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + NULL; + } + void *payload = + taos_map_get_internal(key, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn); + r = pthread_rwlock_unlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return NULL; + } + return payload; +} + +static int taos_map_set_internal(const char *key, void *value, size_t *size, size_t *max_size, taos_linked_list_t *keys, + taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn, + bool destroy_current_value) { + taos_map_node_t *map_node = taos_map_node_new(key, value, free_value_fn); + if (map_node == NULL) return 1; + + size_t index = taos_map_get_index_internal(key, size, max_size); + taos_linked_list_t *list = addrs[index]; + for (taos_linked_list_node_t *current_node = list->head; current_node != NULL; current_node = current_node->next) { + taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item; + taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, map_node); + if (result == TAOS_EQUAL) { + if (destroy_current_value) { + free_value_fn(current_map_node->value); + current_map_node->value = NULL; + } + taos_free((char *)current_map_node->key); + current_map_node->key = NULL; + taos_free(current_map_node); + current_map_node = NULL; + current_node->item = map_node; + return 0; + } + } + taos_linked_list_append(list, map_node); + taos_linked_list_append(keys, (char *)map_node->key); + (*size)++; + return 0; +} + +int taos_map_ensure_space(taos_map_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + + if (self->size <= self->max_size / 2) { + return 0; + } + + // Increase the max size + size_t new_max = self->max_size * 2; + size_t new_size = 0; + + // Create a new list of keys + taos_linked_list_t *new_keys = taos_linked_list_new(); + if (new_keys == NULL) return 1; + + r = taos_linked_list_set_free_fn(new_keys, taos_linked_list_no_op_free); + if (r) return r; + + // Create a new array of addrs + taos_linked_list_t **new_addrs = taos_malloc(sizeof(taos_linked_list_t) * new_max); + + // Initialize the new array + for (int i = 0; i < new_max; i++) { + new_addrs[i] = taos_linked_list_new(); + r = taos_linked_list_set_free_fn(new_addrs[i], taos_map_node_free); + if (r) return r; + r = taos_linked_list_set_compare_fn(new_addrs[i], taos_map_node_compare); + if (r) return r; + } + + // Iterate through each linked-list at each memory region in the map's backbone + for (int i = 0; i < self->max_size; i++) { + // Create a new map node for each node in the linked list and insert it into the new map. Afterwards, deallocate + // the old map node + taos_linked_list_t *list = self->addrs[i]; + taos_linked_list_node_t *current_node = list->head; + while (current_node != NULL) { + taos_map_node_t *map_node = (taos_map_node_t *)current_node->item; + r = taos_map_set_internal(map_node->key, map_node->value, &new_size, &new_max, new_keys, new_addrs, + self->free_value_fn, false); + if (r) return r; + + taos_linked_list_node_t *next = current_node->next; + taos_free(current_node); + current_node = NULL; + taos_free((void *)map_node->key); + map_node->key = NULL; + taos_free(map_node); + map_node = NULL; + current_node = next; + } + // We're done deallocating each map node in the linked list, so deallocate the linked-list object + taos_free(self->addrs[i]); + self->addrs[i] = NULL; + } + // Destroy the collection of keys in the map + taos_linked_list_destroy(self->keys); + self->keys = NULL; + + // Deallocate the backbone of the map + taos_free(self->addrs); + self->addrs = NULL; + + // Update the members of the current map + self->size = new_size; + self->max_size = new_max; + self->keys = new_keys; + self->addrs = new_addrs; + + return 0; +} + +int taos_map_set(taos_map_t *self, const char *key, void *value) { + TAOS_ASSERT(self != NULL); + int r = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + return r; + } + + r = taos_map_ensure_space(self); + if (r) { + int rr = 0; + rr = pthread_rwlock_unlock(self->rwlock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return r; + } + } + r = taos_map_set_internal(key, value, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn, + true); + if (r) { + int rr = 0; + rr = pthread_rwlock_unlock(self->rwlock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return r; + } + } + r = pthread_rwlock_unlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + } + return r; +} + +static int taos_map_delete_internal(const char *key, size_t *size, size_t *max_size, taos_linked_list_t *keys, + taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn) { + int r = 0; + size_t index = taos_map_get_index_internal(key, size, max_size); + taos_linked_list_t *list = addrs[index]; + taos_map_node_t *temp_map_node = taos_map_node_new(key, NULL, free_value_fn); + + for (taos_linked_list_node_t *current_node = list->head; current_node != NULL; current_node = current_node->next) { + taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item; + taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, temp_map_node); + if (result == TAOS_EQUAL) { + r = taos_linked_list_remove(list, current_node); + if (r) return r; + + r = taos_linked_list_remove(keys, (char *)current_map_node->key); + if (r) return r; + + (*size)--; + break; + } + } + r = taos_map_node_destroy(temp_map_node); + temp_map_node = NULL; + return r; +} + +int taos_map_delete(taos_map_t *self, const char *key) { + TAOS_ASSERT(self != NULL); + int r = 0; + int ret = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + ret = r; + } + r = taos_map_delete_internal(key, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn); + if (r) ret = r; + r = pthread_rwlock_unlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + ret = r; + } + return ret; +} + +int taos_map_set_free_value_fn(taos_map_t *self, taos_map_node_free_value_fn free_value_fn) { + TAOS_ASSERT(self != NULL); + self->free_value_fn = free_value_fn; + return 0; +} + +size_t taos_map_size(taos_map_t *self) { + TAOS_ASSERT(self != NULL); + return self->size; +} diff --git a/source/libs/monitorfw/src/taos_metric.c b/source/libs/monitorfw/src/taos_metric.c new file mode 100644 index 0000000000..a9bb6d45a4 --- /dev/null +++ b/source/libs/monitorfw/src/taos_metric.c @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_metric_formatter_i.h" +#include "taos_metric_i.h" +#include "taos_metric_sample_i.h" + +char *taos_metric_type_map[4] = {"counter", "gauge", "histogram", "summary"}; + +taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name, const char *help, + size_t label_key_count, const char **label_keys) { + int r = 0; + taos_metric_t *self = (taos_metric_t *)taos_malloc(sizeof(taos_metric_t)); + self->type = metric_type; + self->name = name; + self->help = help; + + const char **k = (const char **)taos_malloc(sizeof(const char *) * label_key_count); + + for (int i = 0; i < label_key_count; i++) { + if (strcmp(label_keys[i], "le") == 0) { + TAOS_LOG(TAOS_METRIC_INVALID_LABEL_NAME); + taos_metric_destroy(self); + return NULL; + } + if (strcmp(label_keys[i], "quantile") == 0) { + TAOS_LOG(TAOS_METRIC_INVALID_LABEL_NAME); + taos_metric_destroy(self); + return NULL; + } + k[i] = taos_strdup(label_keys[i]); + } + self->label_keys = k; + self->label_key_count = label_key_count; + self->samples = taos_map_new(); + + if (metric_type == TAOS_HISTOGRAM) { + + } else { + r = taos_map_set_free_value_fn(self->samples, &taos_metric_sample_free_generic); + if (r) { + taos_metric_destroy(self); + return NULL; + } + } + + self->formatter = taos_metric_formatter_new(); + if (self->formatter == NULL) { + taos_metric_destroy(self); + return NULL; + } + self->rwlock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t)); + r = pthread_rwlock_init(self->rwlock, NULL); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_INIT_ERROR); + return NULL; + } + return self; +} + +int taos_metric_destroy(taos_metric_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_map_destroy(self->samples); + self->samples = NULL; + if (r) ret = r; + + r = taos_metric_formatter_destroy(self->formatter); + self->formatter = NULL; + if (r) ret = r; + + r = pthread_rwlock_destroy(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_DESTROY_ERROR); + ret = r; + } + + taos_free(self->rwlock); + self->rwlock = NULL; + + for (int i = 0; i < self->label_key_count; i++) { + taos_free((void *)self->label_keys[i]); + self->label_keys[i] = NULL; + } + taos_free(self->label_keys); + self->label_keys = NULL; + + taos_free(self); + self = NULL; + + return ret; +} + +int taos_metric_destroy_generic(void *item) { + int r = 0; + taos_metric_t *self = (taos_metric_t *)item; + r = taos_metric_destroy(self); + self = NULL; + return r; +} + +void taos_metric_free_generic(void *item) { + taos_metric_t *self = (taos_metric_t *)item; + taos_metric_destroy(self); +} + +taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const char **label_values) { + TAOS_ASSERT(self != NULL); + int r = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + return NULL; + } + +#define TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK() \ + r = pthread_rwlock_unlock(self->rwlock); \ + if (r) TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); \ + return NULL; + + // Get l_value + r = taos_metric_formatter_load_l_value(self->formatter, self->name, NULL, self->label_key_count, self->label_keys, + label_values); + if (r) { + TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); + } + + // This must be freed before returning + const char *l_value = taos_metric_formatter_dump(self->formatter); + if (l_value == NULL) { + TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); + } + + // Get sample + taos_metric_sample_t *sample = (taos_metric_sample_t *)taos_map_get(self->samples, l_value); + if (sample == NULL) { + sample = taos_metric_sample_new(self->type, l_value, 0.0); + r = taos_map_set(self->samples, l_value, sample); + if (r) { + TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); + } + } + pthread_rwlock_unlock(self->rwlock); + taos_free((void *)l_value); + return sample; +} + diff --git a/source/libs/monitorfw/src/taos_metric_formatter.c b/source/libs/monitorfw/src/taos_metric_formatter.c new file mode 100644 index 0000000000..235ecb2bce --- /dev/null +++ b/source/libs/monitorfw/src/taos_metric_formatter.c @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_collector_t.h" +#include "taos_linked_list_t.h" +#include "taos_map_i.h" +#include "taos_metric_formatter_i.h" +#include "taos_metric_sample_t.h" +#include "taos_metric_t.h" +#include "taos_string_builder_i.h" + +#include + +taos_metric_formatter_t *taos_metric_formatter_new() { + taos_metric_formatter_t *self = (taos_metric_formatter_t *)taos_malloc(sizeof(taos_metric_formatter_t)); + self->string_builder = taos_string_builder_new(); + if (self->string_builder == NULL) { + taos_metric_formatter_destroy(self); + return NULL; + } + self->err_builder = taos_string_builder_new(); + if (self->err_builder == NULL) { + taos_metric_formatter_destroy(self); + return NULL; + } + return self; +} + +int taos_metric_formatter_destroy(taos_metric_formatter_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_string_builder_destroy(self->string_builder); + self->string_builder = NULL; + if (r) ret = r; + + r = taos_string_builder_destroy(self->err_builder); + self->err_builder = NULL; + if (r) ret = r; + + taos_free(self); + self = NULL; + return ret; +} + +int taos_metric_formatter_load_help(taos_metric_formatter_t *self, const char *name, const char *help) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, "# HELP "); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, name); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, help); + if (r) return r; + + return taos_string_builder_add_char(self->string_builder, '\n'); +} + +int taos_metric_formatter_load_type(taos_metric_formatter_t *self, const char *name, taos_metric_type_t metric_type) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, "# TYPE "); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, name); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, taos_metric_type_map[metric_type]); + if (r) return r; + + return taos_string_builder_add_char(self->string_builder, '\n'); +} + +int taos_metric_formatter_load_l_value(taos_metric_formatter_t *self, const char *name, const char *suffix, + size_t label_count, const char **label_keys, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, name); + if (r) return r; + + if (suffix != NULL) { + r = taos_string_builder_add_char(self->string_builder, '_'); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, suffix); + if (r) return r; + } + + if (label_count == 0) return 0; + + for (int i = 0; i < label_count; i++) { + if (i == 0) { + r = taos_string_builder_add_char(self->string_builder, '{'); + if (r) return r; + } + r = taos_string_builder_add_str(self->string_builder, (const char *)label_keys[i]); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, '='); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, '"'); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, (const char *)label_values[i]); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, '"'); + if (r) return r; + + if (i == label_count - 1) { + r = taos_string_builder_add_char(self->string_builder, '}'); + if (r) return r; + } else { + r = taos_string_builder_add_char(self->string_builder, ','); + if (r) return r; + } + } + return 0; +} + +int taos_metric_formatter_load_sample(taos_metric_formatter_t *self, taos_metric_sample_t *sample, int64_t ts) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, sample->l_value); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + char buffer[50]; + sprintf(buffer, "%.17g", sample->r_value); + r = taos_string_builder_add_str(self->string_builder, buffer); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + sprintf(buffer, "%ld", ts); + r = taos_string_builder_add_str(self->string_builder, buffer); + if (r) return r; + + taos_metric_sample_set(sample, 0); + + return taos_string_builder_add_char(self->string_builder, '\n'); +} + +int taos_metric_formatter_clear(taos_metric_formatter_t *self) { + TAOS_ASSERT(self != NULL); + return taos_string_builder_clear(self->string_builder); +} + +char *taos_metric_formatter_dump(taos_metric_formatter_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + if (self == NULL) return NULL; + char *data = taos_string_builder_dump(self->string_builder); + if (data == NULL) return NULL; + r = taos_string_builder_clear(self->string_builder); + if (r) { + taos_free(data); + return NULL; + } + return data; +} + +int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, int64_t ts) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_metric_formatter_load_help(self, metric->name, metric->help); + if (r) return r; + + r = taos_metric_formatter_load_type(self, metric->name, metric->type); + if (r) return r; + + for (taos_linked_list_node_t *current_node = metric->samples->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *key = (const char *)current_node->item; + if (metric->type == TAOS_HISTOGRAM) { + + } else { + taos_metric_sample_t *sample = (taos_metric_sample_t *)taos_map_get(metric->samples, key); + if (sample == NULL) return 1; + r = taos_metric_formatter_load_sample(self, sample, ts); + if (r) return r; + } + } + return taos_string_builder_add_char(self->string_builder, '\n'); +} + +int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, int64_t ts) { + TAOS_ASSERT(self != NULL); + int r = 0; + for (taos_linked_list_node_t *current_node = collectors->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *collector_name = (const char *)current_node->item; + taos_collector_t *collector = (taos_collector_t *)taos_map_get(collectors, collector_name); + if (collector == NULL) return 1; + + taos_map_t *metrics = collector->collect_fn(collector); + if (metrics == NULL) return 1; + + for (taos_linked_list_node_t *current_node = metrics->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *metric_name = (const char *)current_node->item; + taos_metric_t *metric = (taos_metric_t *)taos_map_get(metrics, metric_name); + if (metric == NULL) return 1; + r = taos_metric_formatter_load_metric(self, metric, ts); + if (r) return r; + } + } + return r; +} diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c new file mode 100644 index 0000000000..747eebc87c --- /dev/null +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_metric_sample_i.h" +#include "taos_metric_sample_t.h" + +taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value) { + taos_metric_sample_t *self = (taos_metric_sample_t *)taos_malloc(sizeof(taos_metric_sample_t)); + self->type = type; + self->l_value = taos_strdup(l_value); + self->r_value = ATOMIC_VAR_INIT(r_value); + return self; +} + +int taos_metric_sample_destroy(taos_metric_sample_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + taos_free((void *)self->l_value); + self->l_value = NULL; + taos_free((void *)self); + self = NULL; + return 0; +} + +int taos_metric_sample_destroy_generic(void *gen) { + int r = 0; + + taos_metric_sample_t *self = (taos_metric_sample_t *)gen; + r = taos_metric_sample_destroy(self); + self = NULL; + return r; +} + +void taos_metric_sample_free_generic(void *gen) { + taos_metric_sample_t *self = (taos_metric_sample_t *)gen; + taos_metric_sample_destroy(self); +} + +int taos_metric_sample_add(taos_metric_sample_t *self, double r_value) { + TAOS_ASSERT(self != NULL); + if (r_value < 0) { + return 1; + } + _Atomic double old = atomic_load(&self->r_value); + for (;;) { + _Atomic double new = ATOMIC_VAR_INIT(old + r_value); + if (atomic_compare_exchange_weak(&self->r_value, &old, new)) { + return 0; + } + } +} + +int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { + TAOS_ASSERT(self != NULL); + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + _Atomic double old = atomic_load(&self->r_value); + for (;;) { + _Atomic double new = ATOMIC_VAR_INIT(old - r_value); + if (atomic_compare_exchange_weak(&self->r_value, &old, new)) { + return 0; + } + } +} + +int taos_metric_sample_set(taos_metric_sample_t *self, double r_value) { + if (self->type != TAOS_GAUGE && self->type != TAOS_COUNTER) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + atomic_store(&self->r_value, r_value); + return 0; +} diff --git a/source/libs/monitorfw/src/taos_string_builder.c b/source/libs/monitorfw/src/taos_string_builder.c new file mode 100644 index 0000000000..0f3940cdab --- /dev/null +++ b/source/libs/monitorfw/src/taos_string_builder.c @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_string_builder_i.h" +#include "taos_string_builder_t.h" + +// The initial size of a string created via taos_string_builder +#define TAOS_STRING_BUILDER_INIT_SIZE 32 + +// taos_string_builder_init prototype declaration +int taos_string_builder_init(taos_string_builder_t *self); + +struct taos_string_builder { + char *str; /**< the target string */ + size_t allocated; /**< the size allocated to the string in bytes */ + size_t len; /**< the length of str */ + size_t init_size; /**< the initialize size of space to allocate */ +}; + +taos_string_builder_t *taos_string_builder_new(void) { + int r = 0; + + taos_string_builder_t *self = (taos_string_builder_t *)taos_malloc(sizeof(taos_string_builder_t)); + self->init_size = TAOS_STRING_BUILDER_INIT_SIZE; + r = taos_string_builder_init(self); + if (r) { + taos_string_builder_destroy(self); + return NULL; + } + + return self; +} + +int taos_string_builder_init(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->str = (char *)taos_malloc(self->init_size); + *self->str = '\0'; + self->allocated = self->init_size; + self->len = 0; + return 0; +} + +int taos_string_builder_destroy(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + taos_free(self->str); + self->str = NULL; + taos_free(self); + self = NULL; + return 0; +} + +/** + * @brief API PRIVATE Grows the size of the string given the value we want to add + * + * The method continuously shifts left until the new size is large enough to accommodate add_len. This private method + * is called in methods that need to add one or more characters to the underlying string. + */ +static int taos_string_builder_ensure_space(taos_string_builder_t *self, size_t add_len) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (add_len == 0 || self->allocated >= self->len + add_len + 1) return 0; + while (self->allocated < self->len + add_len + 1) self->allocated <<= 1; + self->str = (char *)taos_realloc(self->str, self->allocated); + return 0; +} + +int taos_string_builder_add_str(taos_string_builder_t *self, const char *str) { + TAOS_ASSERT(self != NULL); + int r = 0; + + if (self == NULL) return 1; + if (str == NULL || *str == '\0') return 0; + + size_t len = strlen(str); + r = taos_string_builder_ensure_space(self, len); + if (r) return r; + + memcpy(self->str + self->len, str, len); + self->len += len; + self->str[self->len] = '\0'; + return 0; +} + +int taos_string_builder_add_char(taos_string_builder_t *self, char c) { + TAOS_ASSERT(self != NULL); + int r = 0; + + if (self == NULL) return 1; + r = taos_string_builder_ensure_space(self, 1); + if (r) return r; + + self->str[self->len] = c; + self->len++; + self->str[self->len] = '\0'; + return 0; +} + +int taos_string_builder_truncate(taos_string_builder_t *self, size_t len) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (len >= self->len) return 0; + + self->len = len; + self->str[self->len] = '\0'; + return 0; +} + +int taos_string_builder_clear(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + taos_free(self->str); + self->str = NULL; + return taos_string_builder_init(self); +} + +size_t taos_string_builder_len(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + return self->len; +} + +char *taos_string_builder_dump(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + // +1 to accommodate \0 + char *out = (char *)taos_malloc((self->len + 1) * sizeof(char)); + memcpy(out, self->str, self->len + 1); + return out; +} + +char *taos_string_builder_str(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + return self->str; +} From 43cd6049114b3480b7a4d0ffda4186bf60c09d3e Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 23 Oct 2023 11:19:53 +0000 Subject: [PATCH 002/181] remove reference --- source/dnode/mgmt/node_util/inc/dmUtil.h | 1 - source/dnode/mnode/impl/src/mndMnode.c | 1 - 2 files changed, 2 deletions(-) diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 4c00a48796..0a52c578a5 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -39,7 +39,6 @@ #include "sync.h" #include "tfs.h" #include "wal.h" -#include "taos_monitor.h" #ifdef __cplusplus extern "C" { diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index a93973fd13..22b2fec857 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -23,7 +23,6 @@ #include "mndTrans.h" #include "tmisce.h" #include "audit.h" -#include "taos_monitor.h" #define MNODE_VER_NUMBER 2 #define MNODE_RESERVE_SIZE 64 From a006c4634c1cb7cd7e307af30dbd15de61d59035 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 23 Oct 2023 12:47:49 +0000 Subject: [PATCH 003/181] destroy registry --- source/libs/monitor/src/monMain.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 93abda1281..20bd064d69 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -125,6 +125,9 @@ void monCleanup() { tFreeSMonQmInfo(&tsMonitor.qmInfo); tFreeSMonBmInfo(&tsMonitor.bmInfo); taosThreadMutexDestroy(&tsMonitor.lock); + + taos_collector_registry_destroy(TAOS_COLLECTOR_REGISTRY_DEFAULT); + TAOS_COLLECTOR_REGISTRY_DEFAULT = NULL; } static void monCleanupMonitorInfo(SMonInfo *pMonitor) { From 9756016d22ab54692ee3c133a1bd0975fddbb700 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 24 Oct 2023 01:07:45 +0000 Subject: [PATCH 004/181] memory leak --- source/libs/monitorfw/src/taos_collector_registry.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c index 7efe82ad23..6df1653393 100644 --- a/source/libs/monitorfw/src/taos_collector_registry.c +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -88,6 +88,10 @@ int taos_collector_registry_destroy(taos_collector_registry_t *self) { self->string_builder = NULL; if (r) ret = r; + r = taos_string_builder_destroy(self->out); + self->out = NULL; + if (r) ret = r; + r = pthread_rwlock_destroy(self->lock); taos_free(self->lock); self->lock = NULL; From b82cd97b80d8abfa2a71baf061806e7807a66b67 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 24 Oct 2023 04:10:56 +0000 Subject: [PATCH 005/181] memory leak and enable --- source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 2 +- source/libs/monitor/src/monMain.c | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index c237ce73c0..14732cb509 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -107,7 +107,7 @@ void dmSendMonitorReport() { dmGetSmMonitorInfo(pDnode); monSendReport(); - monSendPromReport(); + //monSendPromReport(); } void dmGetVnodeLoads(SMonVloadInfo *pInfo) { diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 20bd064d69..b62cde5c5b 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -19,6 +19,7 @@ #include "thttp.h" #include "ttime.h" #include "taos_monitor.h" +#include "tglobal.h" static SMonitor tsMonitor = {0}; static char* tsMonUri = "/report"; @@ -553,7 +554,7 @@ void monSendReport() { void monSendPromReport() { char *pCont = (char *)taos_collector_registry_bridge( TAOS_COLLECTOR_REGISTRY_DEFAULT, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); - uInfoL("report cont:\n%s\n", pCont); + //uInfoL("report cont:\n%s\n", pCont); if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; if (taosSendHttpReport(tsMonitor.cfg.server, tsMonUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { @@ -565,7 +566,8 @@ void monSendPromReport() { } void monSendContent(char *pCont) { - uInfoL("report cont:\n%s\n", pCont); + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; + //uInfoL("report cont:\n%s\n", pCont); if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; if (taosSendHttpReport(tsMonitor.cfg.server, tsMonUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { From 7c83d190a42aa83b989a59cd95ab0e87346fba3f Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 24 Oct 2023 05:29:45 +0000 Subject: [PATCH 006/181] ts format --- include/libs/monitorfw/taos_collector_registry.h | 2 +- source/libs/monitor/src/monMain.c | 5 +++-- source/libs/monitorfw/inc/taos_metric_formatter_i.h | 6 +++--- source/libs/monitorfw/src/taos_collector_registry.c | 2 +- source/libs/monitorfw/src/taos_metric_formatter.c | 9 ++++----- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/libs/monitorfw/taos_collector_registry.h b/include/libs/monitorfw/taos_collector_registry.h index f67deb074b..f3c89ad543 100644 --- a/include/libs/monitorfw/taos_collector_registry.h +++ b/include/libs/monitorfw/taos_collector_registry.h @@ -103,7 +103,7 @@ int taos_collector_registry_register_collector(taos_collector_registry_t *self, * @param self The target taos_collector_registry_t* * @return The string int he default metric exposition format. */ -const char *taos_collector_registry_bridge(taos_collector_registry_t *self, int64_t ts); +const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts); int taos_collector_registry_clear_out(taos_collector_registry_t *self); diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index b62cde5c5b..4670334565 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -552,8 +552,9 @@ void monSendReport() { } void monSendPromReport() { - char *pCont = (char *)taos_collector_registry_bridge( - TAOS_COLLECTOR_REGISTRY_DEFAULT, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + char ts[50]; + sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + char *pCont = (char *)taos_collector_registry_bridge(TAOS_COLLECTOR_REGISTRY_DEFAULT, ts); //uInfoL("report cont:\n%s\n", pCont); if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_i.h b/source/libs/monitorfw/inc/taos_metric_formatter_i.h index 9a63850ea4..a5dd23410f 100644 --- a/source/libs/monitorfw/inc/taos_metric_formatter_i.h +++ b/source/libs/monitorfw/inc/taos_metric_formatter_i.h @@ -57,17 +57,17 @@ int taos_metric_formatter_load_l_value(taos_metric_formatter_t *metric_formatter * @brief API PRIVATE Loads the formatter with a metric sample */ int taos_metric_formatter_load_sample( - taos_metric_formatter_t *metric_formatter, taos_metric_sample_t *sample, int64_t ts); + taos_metric_formatter_t *metric_formatter, taos_metric_sample_t *sample, char *ts); /** * @brief API PRIVATE Loads a metric in the string exposition format */ -int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, int64_t ts); +int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts); /** * @brief API PRIVATE Loads the given metrics */ -int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, int64_t ts); +int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts); /** * @brief API PRIVATE Clear the underlying string_builder diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c index 6df1653393..f94c8fdcd1 100644 --- a/source/libs/monitorfw/src/taos_collector_registry.c +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -186,7 +186,7 @@ int taos_collector_registry_validate_metric_name(taos_collector_registry_t *self return 0; } -const char *taos_collector_registry_bridge(taos_collector_registry_t *self, int64_t ts) { +const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts) { taos_metric_formatter_clear(self->metric_formatter); taos_metric_formatter_load_metrics(self->metric_formatter, self->collectors, ts); char *out = taos_metric_formatter_dump(self->metric_formatter); diff --git a/source/libs/monitorfw/src/taos_metric_formatter.c b/source/libs/monitorfw/src/taos_metric_formatter.c index 235ecb2bce..3529d9dda6 100644 --- a/source/libs/monitorfw/src/taos_metric_formatter.c +++ b/source/libs/monitorfw/src/taos_metric_formatter.c @@ -158,7 +158,7 @@ int taos_metric_formatter_load_l_value(taos_metric_formatter_t *self, const char return 0; } -int taos_metric_formatter_load_sample(taos_metric_formatter_t *self, taos_metric_sample_t *sample, int64_t ts) { +int taos_metric_formatter_load_sample(taos_metric_formatter_t *self, taos_metric_sample_t *sample, char *ts) { TAOS_ASSERT(self != NULL); if (self == NULL) return 1; @@ -178,8 +178,7 @@ int taos_metric_formatter_load_sample(taos_metric_formatter_t *self, taos_metric r = taos_string_builder_add_char(self->string_builder, ' '); if (r) return r; - sprintf(buffer, "%ld", ts); - r = taos_string_builder_add_str(self->string_builder, buffer); + r = taos_string_builder_add_str(self->string_builder, ts); if (r) return r; taos_metric_sample_set(sample, 0); @@ -206,7 +205,7 @@ char *taos_metric_formatter_dump(taos_metric_formatter_t *self) { return data; } -int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, int64_t ts) { +int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts) { TAOS_ASSERT(self != NULL); if (self == NULL) return 1; @@ -233,7 +232,7 @@ int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric return taos_string_builder_add_char(self->string_builder, '\n'); } -int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, int64_t ts) { +int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts) { TAOS_ASSERT(self != NULL); int r = 0; for (taos_linked_list_node_t *current_node = collectors->keys->head; current_node != NULL; From ff46bdb17a8415097461c8f90f971c8eba7bc851 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 24 Oct 2023 06:51:00 +0000 Subject: [PATCH 007/181] ci memory leak test --- source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index 14732cb509..bf88d0014e 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -105,9 +105,9 @@ void dmSendMonitorReport() { dmGetVmMonitorInfo(pDnode); dmGetQmMonitorInfo(pDnode); dmGetSmMonitorInfo(pDnode); - monSendReport(); + //monSendReport(); - //monSendPromReport(); + monSendPromReport(); } void dmGetVnodeLoads(SMonVloadInfo *pInfo) { From 4c401c39fbf4ca9bd85a0d0dc0e56c840c6089d7 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 24 Oct 2023 09:52:34 +0000 Subject: [PATCH 008/181] atomic --- source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 12 ++++++------ source/libs/monitorfw/inc/taos_metric_sample_t.h | 2 +- source/libs/monitorfw/src/taos_collector.c | 1 - source/libs/monitorfw/src/taos_metric_formatter.c | 2 +- source/libs/monitorfw/src/taos_metric_sample.c | 13 +++++++++++++ 5 files changed, 21 insertions(+), 9 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index bf88d0014e..02d8bb43ea 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -99,12 +99,12 @@ void dmSendMonitorReport() { if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); - SDnode *pDnode = dmInstance(); - dmGetDmMonitorInfo(pDnode); - dmGetMmMonitorInfo(pDnode); - dmGetVmMonitorInfo(pDnode); - dmGetQmMonitorInfo(pDnode); - dmGetSmMonitorInfo(pDnode); + //SDnode *pDnode = dmInstance(); + //dmGetDmMonitorInfo(pDnode); + //dmGetMmMonitorInfo(pDnode); + //dmGetVmMonitorInfo(pDnode); + //dmGetQmMonitorInfo(pDnode); + //dmGetSmMonitorInfo(pDnode); //monSendReport(); monSendPromReport(); diff --git a/source/libs/monitorfw/inc/taos_metric_sample_t.h b/source/libs/monitorfw/inc/taos_metric_sample_t.h index 59a398b938..92885acf4a 100644 --- a/source/libs/monitorfw/inc/taos_metric_sample_t.h +++ b/source/libs/monitorfw/inc/taos_metric_sample_t.h @@ -22,7 +22,7 @@ struct taos_metric_sample { taos_metric_type_t type; /**< type is the metric type for the sample */ char *l_value; /**< l_value is the full metric name and label set represeted as a string */ - _Atomic double r_value; /**< r_value is the value of the metric sample */ + /*_Atomic*/ int64_t r_value; /**< r_value is the value of the metric sample */ }; #endif // TAOS_METRIC_SAMPLE_T_H diff --git a/source/libs/monitorfw/src/taos_collector.c b/source/libs/monitorfw/src/taos_collector.c index 9d96f61b4b..d3228facb6 100644 --- a/source/libs/monitorfw/src/taos_collector.c +++ b/source/libs/monitorfw/src/taos_collector.c @@ -14,7 +14,6 @@ */ #include -#include // Public #include "taos_alloc.h" diff --git a/source/libs/monitorfw/src/taos_metric_formatter.c b/source/libs/monitorfw/src/taos_metric_formatter.c index 3529d9dda6..77361b8b03 100644 --- a/source/libs/monitorfw/src/taos_metric_formatter.c +++ b/source/libs/monitorfw/src/taos_metric_formatter.c @@ -171,7 +171,7 @@ int taos_metric_formatter_load_sample(taos_metric_formatter_t *self, taos_metric if (r) return r; char buffer[50]; - sprintf(buffer, "%.17g", sample->r_value); + sprintf(buffer, "%ld", sample->r_value); r = taos_string_builder_add_str(self->string_builder, buffer); if (r) return r; diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index 747eebc87c..10f626ba16 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -24,6 +24,7 @@ #include "taos_log.h" #include "taos_metric_sample_i.h" #include "taos_metric_sample_t.h" +#include "osAtomic.h" taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value) { taos_metric_sample_t *self = (taos_metric_sample_t *)taos_malloc(sizeof(taos_metric_sample_t)); @@ -62,13 +63,18 @@ int taos_metric_sample_add(taos_metric_sample_t *self, double r_value) { if (r_value < 0) { return 1; } + /* _Atomic double old = atomic_load(&self->r_value); + for (;;) { _Atomic double new = ATOMIC_VAR_INIT(old + r_value); if (atomic_compare_exchange_weak(&self->r_value, &old, new)) { return 0; } } + */ + atomic_fetch_add_64(&self->r_value, r_value); + return 0; } int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { @@ -77,6 +83,7 @@ int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); return 1; } + /* _Atomic double old = atomic_load(&self->r_value); for (;;) { _Atomic double new = ATOMIC_VAR_INIT(old - r_value); @@ -84,6 +91,9 @@ int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { return 0; } } + */ + atomic_fetch_sub_64(&self->r_value, r_value); + return 0; } int taos_metric_sample_set(taos_metric_sample_t *self, double r_value) { @@ -91,6 +101,9 @@ int taos_metric_sample_set(taos_metric_sample_t *self, double r_value) { TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); return 1; } + /* atomic_store(&self->r_value, r_value); + */ + atomic_store_64(&self->r_value, r_value); return 0; } From 8e007f199b522f89201e9fbe9995259a94d826e4 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 24 Oct 2023 11:31:50 +0000 Subject: [PATCH 009/181] format --- include/libs/monitorfw/taos_collector_registry.h | 2 +- source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 14 +++++++------- source/libs/monitor/src/monMain.c | 2 +- .../libs/monitorfw/inc/taos_metric_formatter_i.h | 8 ++++---- .../libs/monitorfw/src/taos_collector_registry.c | 4 ++-- source/libs/monitorfw/src/taos_metric_formatter.c | 13 +++++++------ 6 files changed, 22 insertions(+), 21 deletions(-) diff --git a/include/libs/monitorfw/taos_collector_registry.h b/include/libs/monitorfw/taos_collector_registry.h index f3c89ad543..915b132b11 100644 --- a/include/libs/monitorfw/taos_collector_registry.h +++ b/include/libs/monitorfw/taos_collector_registry.h @@ -103,7 +103,7 @@ int taos_collector_registry_register_collector(taos_collector_registry_t *self, * @param self The target taos_collector_registry_t* * @return The string int he default metric exposition format. */ -const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts); +const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts, char *format); int taos_collector_registry_clear_out(taos_collector_registry_t *self); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index 02d8bb43ea..c237ce73c0 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -99,13 +99,13 @@ void dmSendMonitorReport() { if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); - //SDnode *pDnode = dmInstance(); - //dmGetDmMonitorInfo(pDnode); - //dmGetMmMonitorInfo(pDnode); - //dmGetVmMonitorInfo(pDnode); - //dmGetQmMonitorInfo(pDnode); - //dmGetSmMonitorInfo(pDnode); - //monSendReport(); + SDnode *pDnode = dmInstance(); + dmGetDmMonitorInfo(pDnode); + dmGetMmMonitorInfo(pDnode); + dmGetVmMonitorInfo(pDnode); + dmGetQmMonitorInfo(pDnode); + dmGetSmMonitorInfo(pDnode); + monSendReport(); monSendPromReport(); } diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 4670334565..9c5b56f48e 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -554,7 +554,7 @@ void monSendReport() { void monSendPromReport() { char ts[50]; sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); - char *pCont = (char *)taos_collector_registry_bridge(TAOS_COLLECTOR_REGISTRY_DEFAULT, ts); + char *pCont = (char *)taos_collector_registry_bridge(TAOS_COLLECTOR_REGISTRY_DEFAULT, ts, "%" PRId64); //uInfoL("report cont:\n%s\n", pCont); if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_i.h b/source/libs/monitorfw/inc/taos_metric_formatter_i.h index a5dd23410f..fba30d9eeb 100644 --- a/source/libs/monitorfw/inc/taos_metric_formatter_i.h +++ b/source/libs/monitorfw/inc/taos_metric_formatter_i.h @@ -56,18 +56,18 @@ int taos_metric_formatter_load_l_value(taos_metric_formatter_t *metric_formatter /** * @brief API PRIVATE Loads the formatter with a metric sample */ -int taos_metric_formatter_load_sample( - taos_metric_formatter_t *metric_formatter, taos_metric_sample_t *sample, char *ts); +int taos_metric_formatter_load_sample(taos_metric_formatter_t *metric_formatter, taos_metric_sample_t *sample, + char *ts, char *format); /** * @brief API PRIVATE Loads a metric in the string exposition format */ -int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts); +int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format); /** * @brief API PRIVATE Loads the given metrics */ -int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts); +int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts, char *format); /** * @brief API PRIVATE Clear the underlying string_builder diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c index f94c8fdcd1..25e4b5b2ca 100644 --- a/source/libs/monitorfw/src/taos_collector_registry.c +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -186,9 +186,9 @@ int taos_collector_registry_validate_metric_name(taos_collector_registry_t *self return 0; } -const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts) { +const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts, char *format) { taos_metric_formatter_clear(self->metric_formatter); - taos_metric_formatter_load_metrics(self->metric_formatter, self->collectors, ts); + taos_metric_formatter_load_metrics(self->metric_formatter, self->collectors, ts, format); char *out = taos_metric_formatter_dump(self->metric_formatter); int r = 0; diff --git a/source/libs/monitorfw/src/taos_metric_formatter.c b/source/libs/monitorfw/src/taos_metric_formatter.c index 77361b8b03..8d1b44c149 100644 --- a/source/libs/monitorfw/src/taos_metric_formatter.c +++ b/source/libs/monitorfw/src/taos_metric_formatter.c @@ -158,7 +158,8 @@ int taos_metric_formatter_load_l_value(taos_metric_formatter_t *self, const char return 0; } -int taos_metric_formatter_load_sample(taos_metric_formatter_t *self, taos_metric_sample_t *sample, char *ts) { +int taos_metric_formatter_load_sample(taos_metric_formatter_t *self, taos_metric_sample_t *sample, + char *ts, char *format) { TAOS_ASSERT(self != NULL); if (self == NULL) return 1; @@ -171,7 +172,7 @@ int taos_metric_formatter_load_sample(taos_metric_formatter_t *self, taos_metric if (r) return r; char buffer[50]; - sprintf(buffer, "%ld", sample->r_value); + sprintf(buffer, format, sample->r_value); r = taos_string_builder_add_str(self->string_builder, buffer); if (r) return r; @@ -205,7 +206,7 @@ char *taos_metric_formatter_dump(taos_metric_formatter_t *self) { return data; } -int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts) { +int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format) { TAOS_ASSERT(self != NULL); if (self == NULL) return 1; @@ -225,14 +226,14 @@ int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric } else { taos_metric_sample_t *sample = (taos_metric_sample_t *)taos_map_get(metric->samples, key); if (sample == NULL) return 1; - r = taos_metric_formatter_load_sample(self, sample, ts); + r = taos_metric_formatter_load_sample(self, sample, ts, format); if (r) return r; } } return taos_string_builder_add_char(self->string_builder, '\n'); } -int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts) { +int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts, char *format) { TAOS_ASSERT(self != NULL); int r = 0; for (taos_linked_list_node_t *current_node = collectors->keys->head; current_node != NULL; @@ -249,7 +250,7 @@ int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t const char *metric_name = (const char *)current_node->item; taos_metric_t *metric = (taos_metric_t *)taos_map_get(metrics, metric_name); if (metric == NULL) return 1; - r = taos_metric_formatter_load_metric(self, metric, ts); + r = taos_metric_formatter_load_metric(self, metric, ts, format); if (r) return r; } } From 91092cb9092f03913666611af1187db5c18a5c5c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 25 Oct 2023 10:29:37 +0800 Subject: [PATCH 010/181] http handle empty packet --- source/libs/transport/src/thttp.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index c483d82027..b0a384cfcc 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -293,6 +293,11 @@ int32_t httpSendQuit() { static int32_t taosSendHttpReportImpl(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { + if (contLen == 0) { + tError("http-report failed to report empty packet"); + return -1; + } + SHttpModule* load = taosAcquireRef(httpRefMgt, httpRef); if (load == NULL) { tError("http-report already released"); From b403810f45d5d22b549cd20ecfa731378a9af50f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 25 Oct 2023 10:31:18 +0800 Subject: [PATCH 011/181] http handle empty packet --- source/libs/transport/src/thttp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index b0a384cfcc..63f07d7493 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -293,7 +293,7 @@ int32_t httpSendQuit() { static int32_t taosSendHttpReportImpl(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { - if (contLen == 0) { + if (pCont == NULL || contLen == 0) { tError("http-report failed to report empty packet"); return -1; } From 0b9058162a7e0d98c5dbafcbea83b2eea8a54dc2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 25 Oct 2023 10:38:57 +0800 Subject: [PATCH 012/181] http handle empty packet --- source/libs/transport/src/thttp.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 63f07d7493..f1bdd54948 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -293,6 +293,11 @@ int32_t httpSendQuit() { static int32_t taosSendHttpReportImpl(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { + if (server == NULL || uri == NULL) { + tError("http-report failed to report to invalid addr"); + return -1; + } + if (pCont == NULL || contLen == 0) { tError("http-report failed to report empty packet"); return -1; From 9af7b7b1789f44ca2ffaa4389355fe18824834c8 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 26 Oct 2023 11:50:34 +0000 Subject: [PATCH 013/181] init error --- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/monitorfw/src/taos_collector_registry.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index c4aa5c6428..a7179ae033 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1620,7 +1620,7 @@ _exit: int32_t label_count =1; const char *sample_labels[] = {"vgid"}; insert_counter = taos_counter_new("insert_counter", "counter for insert sql", label_count, sample_labels); - insert_counter = taos_collector_registry_must_register_metric(insert_counter); + taos_collector_registry_must_register_metric(insert_counter); } char vgId[50]; diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c index 25e4b5b2ca..711e66ed84 100644 --- a/source/libs/monitorfw/src/taos_collector_registry.c +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -122,7 +122,8 @@ int taos_collector_registry_register_metric(taos_metric_t *metric) { taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metric) { int err = taos_collector_registry_register_metric(metric); if (err != 0) { - exit(err); + //exit(err); + return NULL; } return metric; } From 21dd6f70a9687fd61562faecdae6178728fc0c56 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 27 Oct 2023 01:16:53 +0000 Subject: [PATCH 014/181] atomic init --- source/dnode/vnode/src/vnd/vnodeSvr.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index a7179ae033..7d8aeb7e60 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1619,8 +1619,13 @@ _exit: if(insert_counter == NULL){ int32_t label_count =1; const char *sample_labels[] = {"vgid"}; - insert_counter = taos_counter_new("insert_counter", "counter for insert sql", label_count, sample_labels); - taos_collector_registry_must_register_metric(insert_counter); + taos_counter_t *counter = taos_counter_new("insert_counter", "counter for insert sql", label_count, sample_labels); + if(taos_collector_registry_register_metric(counter) == 1){ + taos_counter_destroy(counter); + } + else{ + atomic_store_ptr(insert_counter, counter); + } } char vgId[50]; From ca5e252c0caee8092fee11d31184212133f58509 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 27 Oct 2023 01:20:33 +0000 Subject: [PATCH 015/181] atomic init --- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 7d8aeb7e60..758839f872 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1624,7 +1624,7 @@ _exit: taos_counter_destroy(counter); } else{ - atomic_store_ptr(insert_counter, counter); + atomic_store_ptr(&insert_counter, counter); } } From e861416e48902400ffb8169a8aed4153d4fcabe5 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 30 Oct 2023 11:50:51 +0000 Subject: [PATCH 016/181] uri --- source/libs/monitor/src/monMain.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 9c5b56f48e..d417a97db7 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -23,6 +23,7 @@ static SMonitor tsMonitor = {0}; static char* tsMonUri = "/report"; +static char* tsMonFwUri = "/td_metric"; void monRecordLog(int64_t ts, ELogLevel level, const char *content) { taosThreadMutexLock(&tsMonitor.lock); @@ -558,7 +559,7 @@ void monSendPromReport() { //uInfoL("report cont:\n%s\n", pCont); if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; - if (taosSendHttpReport(tsMonitor.cfg.server, tsMonUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { uError("failed to send monitor msg"); }else{ taos_collector_registry_clear_out(TAOS_COLLECTOR_REGISTRY_DEFAULT); @@ -571,7 +572,7 @@ void monSendContent(char *pCont) { //uInfoL("report cont:\n%s\n", pCont); if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; - if (taosSendHttpReport(tsMonitor.cfg.server, tsMonUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { uError("failed to send monitor msg"); } } From ffb1e1e8c3a25904a3fb0b6609258c197aeb9fc4 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 30 Oct 2023 12:05:20 +0000 Subject: [PATCH 017/181] end point --- source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 758839f872..a89a1f8dee 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1617,8 +1617,8 @@ _exit: } if(insert_counter == NULL){ - int32_t label_count =1; - const char *sample_labels[] = {"vgid"}; + int32_t label_count =2; + const char *sample_labels[] = {"vgid", "endpoint"}; taos_counter_t *counter = taos_counter_new("insert_counter", "counter for insert sql", label_count, sample_labels); if(taos_collector_registry_register_metric(counter) == 1){ taos_counter_destroy(counter); @@ -1630,7 +1630,7 @@ _exit: char vgId[50]; sprintf(vgId, "%"PRId32, TD_VID(pVnode)); - const char *sample_labels[] = {vgId}; + const char *sample_labels[] = {vgId, tsLocalEp}; taos_counter_inc(insert_counter, sample_labels); From 8318aa1418b9d39c67ad2d5c0622ff7bdd41df0c Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 3 Nov 2023 10:14:27 +0800 Subject: [PATCH 018/181] compile on windows --- include/os/osAtomic.h | 2 ++ source/libs/monitorfw/src/taos_metric_formatter.c | 1 - source/libs/monitorfw/src/taos_metric_sample.c | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index 9fd00cefb4..52be7200bf 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -20,6 +20,8 @@ extern "C" { #endif +#include "tdef.h" + // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC diff --git a/source/libs/monitorfw/src/taos_metric_formatter.c b/source/libs/monitorfw/src/taos_metric_formatter.c index 8d1b44c149..6c6f1974eb 100644 --- a/source/libs/monitorfw/src/taos_metric_formatter.c +++ b/source/libs/monitorfw/src/taos_metric_formatter.c @@ -28,7 +28,6 @@ #include "taos_metric_t.h" #include "taos_string_builder_i.h" -#include taos_metric_formatter_t *taos_metric_formatter_new() { taos_metric_formatter_t *self = (taos_metric_formatter_t *)taos_malloc(sizeof(taos_metric_formatter_t)); diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index 10f626ba16..e784712c3f 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include +//#include // Public #include "taos_alloc.h" @@ -30,7 +30,7 @@ taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char taos_metric_sample_t *self = (taos_metric_sample_t *)taos_malloc(sizeof(taos_metric_sample_t)); self->type = type; self->l_value = taos_strdup(l_value); - self->r_value = ATOMIC_VAR_INIT(r_value); + self->r_value = 0; return self; } From 84545d4e8da87da5571a0758bb1ce800b09043b1 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 3 Nov 2023 10:32:16 +0800 Subject: [PATCH 019/181] include os.h --- include/os/osAtomic.h | 2 -- source/libs/monitorfw/src/taos_metric_sample.c | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index 52be7200bf..9fd00cefb4 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -20,8 +20,6 @@ extern "C" { #endif -#include "tdef.h" - // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index e784712c3f..264fa220df 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -24,7 +24,7 @@ #include "taos_log.h" #include "taos_metric_sample_i.h" #include "taos_metric_sample_t.h" -#include "osAtomic.h" +#include "os.h" taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value) { taos_metric_sample_t *self = (taos_metric_sample_t *)taos_malloc(sizeof(taos_metric_sample_t)); From d0b6fc2ba6496a0ea0c29a45151e14e0957a267b Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 3 Nov 2023 09:17:21 +0000 Subject: [PATCH 020/181] atomic windows --- include/os/osAtomic.h | 2 ++ source/libs/monitorfw/src/taos_metric_sample.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index 9fd00cefb4..52be7200bf 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -20,6 +20,8 @@ extern "C" { #endif +#include "tdef.h" + // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index 264fa220df..e784712c3f 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -24,7 +24,7 @@ #include "taos_log.h" #include "taos_metric_sample_i.h" #include "taos_metric_sample_t.h" -#include "os.h" +#include "osAtomic.h" taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value) { taos_metric_sample_t *self = (taos_metric_sample_t *)taos_malloc(sizeof(taos_metric_sample_t)); From 68e2e4df74e23a123201a42c30513684e6d4ba9f Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Mon, 6 Nov 2023 11:26:42 +0800 Subject: [PATCH 021/181] atomic windows --- include/os/osAtomic.h | 2 -- source/libs/monitorfw/src/taos_metric_sample.c | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index 52be7200bf..9fd00cefb4 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -20,8 +20,6 @@ extern "C" { #endif -#include "tdef.h" - // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index e784712c3f..034657cb3d 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -24,6 +24,10 @@ #include "taos_log.h" #include "taos_metric_sample_i.h" #include "taos_metric_sample_t.h" + +#define ALLOW_FORBID_FUNC + +#include "tdef.h" #include "osAtomic.h" taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value) { From 0a16b758314cd7d0935ff88865519d5eb111c78c Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Thu, 2 Nov 2023 18:44:18 +0800 Subject: [PATCH 022/181] monitor client --- include/common/tglobal.h | 1 + include/libs/monitor/clientMonitor.h | 50 ++++++ source/client/CMakeLists.txt | 4 +- source/client/inc/clientInt.h | 9 ++ source/client/src/clientEnv.c | 1 + source/client/src/clientImpl.c | 13 ++ source/client/src/selectMonitor.c | 46 ++++++ source/client/src/slowQueryMonitor.c | 73 +++++++++ source/client/test/CMakeLists.txt | 17 ++ source/client/test/clientMonitorTests.cpp | 81 ++++++++++ source/common/src/tglobal.c | 3 + source/libs/monitor/CMakeLists.txt | 2 +- source/libs/monitor/inc/monInt.h | 1 + source/libs/monitor/src/clientMonitor.c | 189 ++++++++++++++++++++++ 14 files changed, 487 insertions(+), 3 deletions(-) create mode 100644 include/libs/monitor/clientMonitor.h create mode 100644 source/client/src/selectMonitor.c create mode 100644 source/client/src/slowQueryMonitor.c create mode 100644 source/client/test/clientMonitorTests.cpp create mode 100644 source/libs/monitor/src/clientMonitor.c diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 1d0da9e150..1765b0a234 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -147,6 +147,7 @@ extern int32_t tsMetaCacheMaxSize; extern int32_t tsSlowLogThreshold; extern int32_t tsSlowLogScope; extern int32_t tsTimeSeriesThreshold; +extern bool enableSlowQueryMonitor; // client extern int32_t tsMinSlidingTime; diff --git a/include/libs/monitor/clientMonitor.h b/include/libs/monitor/clientMonitor.h new file mode 100644 index 0000000000..0d238157c5 --- /dev/null +++ b/include/libs/monitor/clientMonitor.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_CLIENT_MONITOR_H +#define TDENGINE_CLIENT_MONITOR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "taos_monitor.h" +#include "thash.h" +#include "query.h" + +typedef struct { + char clusterKey[512]; + SEpSet epSet; + void* pTransporter; + taos_collector_registry_t* registry; + taos_collector_t* colector; + taos_counter_t* slow_query_counter; + taos_counter_t* select_counter; + SHashObj* counters; +} ClientMonitor; + +void clusterMonitorInit(const char* clusterKey, SEpSet epSet, void* pTransporter); +void clusterMonitorClose(const char* clusterKey); +taos_counter_t* createClusterCounter(const char* clusterKey, const char* name, const char* help, size_t label_key_count, + const char** label_keys); +int taosClusterCounterInc(const char* clusterKey, const char* counterName, const char** label_values); + +void cluster_monitor_stop(); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_CLIENT_MONITOR_H diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index f089247859..a17c27c297 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -20,7 +20,7 @@ target_include_directories( target_link_libraries( taos INTERFACE api - PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom geometry + PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry ) if(TD_DARWIN_ARM64) @@ -61,7 +61,7 @@ target_include_directories( target_link_libraries( taos_static INTERFACE api - PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom geometry + PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry ) if(${BUILD_TEST}) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 75003d76d8..ef19fb7c51 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -391,6 +391,8 @@ void hbRemoveAppHbMrg(SAppHbMgr** pAppHbMgr); void destroyAllRequests(SHashObj* pRequests); void stopAllRequests(SHashObj* pRequests); +SAppInstInfo* getAppInstInfo(const char* clusterKey); + // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); void hbDeregisterConn(STscObj* pTscObj, SClientHbKey connKey); @@ -426,6 +428,13 @@ void freeQueryParam(SSyncQueryParam* param); int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effeciveUser, SParseSqlRes* pRes); #endif +void clusterSlowQueryMonitorInit(const char* clusterKey); +void clusterSlowQueryLog(const char* clusterKey, int32_t cost); +void SlowQueryLog(int64_t connId, int32_t cost); + +void clusterSelectMonitorInit(const char* clusterKey); +void clusterSelectLog(const char* clusterKey); + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index b36ef20b53..26b076e5ee 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -114,6 +114,7 @@ static void deregisterRequest(SRequestObj *pRequest) { taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s", taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration, pRequest->sqlstr); + SlowQueryLog(pTscObj->connId, duration); } } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 395a396d89..e3aaa63c87 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -15,6 +15,7 @@ #include "cJSON.h" #include "clientInt.h" +#include "clientMonitor.h" #include "clientLog.h" #include "command.h" #include "scheduler.h" @@ -157,6 +158,8 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas tscDebug("new app inst mgr %p, user:%s, ip:%s, port:%d", p, user, epSet.epSet.eps[0].fqdn, epSet.epSet.eps[0].port); pInst = &p; + + clusterSlowQueryMonitorInit(p->instKey); } else { ASSERTS((*pInst) && (*pInst)->pAppHbMgr, "*pInst:%p, pAppHgMgr:%p", *pInst, (*pInst) ? (*pInst)->pAppHbMgr : NULL); // reset to 0 in case of conn with duplicated user key but its user has ever been dropped. @@ -166,9 +169,19 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas taosThreadMutexUnlock(&appInfo.mutex); taosMemoryFreeClear(key); + return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType); } +SAppInstInfo* getAppInstInfo(const char* clusterKey) { + SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey)); + if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) { + return *ppAppInstInfo; + } else { + return NULL; + } +} + void freeQueryParam(SSyncQueryParam* param) { if (param == NULL) return; tsem_destroy(¶m->sem); diff --git a/source/client/src/selectMonitor.c b/source/client/src/selectMonitor.c new file mode 100644 index 0000000000..0cb2073961 --- /dev/null +++ b/source/client/src/selectMonitor.c @@ -0,0 +1,46 @@ + /* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "clientInt.h" +#include "clientMonitor.h" +#include "clientLog.h" + +const char* selectMonitorName = "slow_query"; +const char* selectMonitorHelp = "slow query log when cost > 3s"; +const int selectMonitorLabelCount = 1; +const char* selectMonitorLabels[] = {"default"}; + +void clusterSelectMonitorInit(const char* clusterKey) { + SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey); + SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); + clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter); + createClusterCounter(clusterKey, selectMonitorName, selectMonitorHelp, selectMonitorLabelCount, selectMonitorLabels); +} + +void clusterSelectLog(const char* clusterKey) { + const char* selectMonitorLabelValues[] = {"default"}; + taosClusterCounterInc(clusterKey, selectMonitorName, selectMonitorLabelValues); +} + +void selectLog(int64_t connId) { + STscObj* pTscObj = acquireTscObj(connId); + if (pTscObj != NULL) { + if(pTscObj->pAppInfo == NULL) { + tscLog("selectLog, not found pAppInfo"); + } + return clusterSelectLog(pTscObj->pAppInfo->instKey); + } else { + tscLog("selectLog, not found connect ID"); + } +} diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c new file mode 100644 index 0000000000..b42f860d67 --- /dev/null +++ b/source/client/src/slowQueryMonitor.c @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "clientInt.h" +#include "clientMonitor.h" +#include "clientLog.h" +#include "tglobal.h" + +const char* slowQueryName = "slow_query"; +const char* slowQueryHelp = "slow query log when cost > 3s"; +const int slowQueryLabelCount = 1; +const char* slowQueryLabels[] = {"cost"}; + +const int64_t msInSeconds = 1000; +const int64_t msInMinutes = 60 * msInSeconds; + +static const char* getSlowQueryLableCostDesc(int64_t cost) { + if (cost >= 30 * msInMinutes) { + return " > 30 min"; + } else if (cost >= 10 * msInMinutes) { + return " > 10 min"; + } else if (cost >= 5 * msInMinutes) { + return " > 5 min"; + } else if (cost >= 1 * msInMinutes) { + return " > 1 min"; + } else if (cost >= 30 * msInSeconds) { + return " > 30 seconds"; + } else if (cost >= 10 * msInSeconds) { + return " > 10 seconds"; + } else if (cost >= 5 * msInSeconds) { + return " > 5 seconds"; + } else if (cost >= 3 * msInSeconds) { + return " > 3 seconds"; + } + return "< 3 s"; +} + +void clusterSlowQueryMonitorInit(const char* clusterKey) { + if (!enableSlowQueryMonitor) return; + SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey); + SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); + clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter); + createClusterCounter(clusterKey, slowQueryName, slowQueryHelp, slowQueryLabelCount, slowQueryLabels); +} + +void clusterSlowQueryLog(const char* clusterKey, int32_t cost) { + const char* slowQueryLabelValues[] = {getSlowQueryLableCostDesc(cost)}; + taosClusterCounterInc(clusterKey, slowQueryName, slowQueryLabelValues); +} + +void SlowQueryLog(int64_t connId, int32_t cost) { + if (!enableSlowQueryMonitor) return; + STscObj* pTscObj = acquireTscObj(connId); + if (pTscObj != NULL) { + if(pTscObj->pAppInfo == NULL) { + tscLog("SlowQueryLog, not found pAppInfo"); + } + return clusterSlowQueryLog(pTscObj->pAppInfo->instKey, cost); + } else { + tscLog("SlowQueryLog, not found connect ID"); + } +} diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 91f0d1eef8..7c3847e4a1 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -23,6 +23,12 @@ TARGET_LINK_LIBRARIES( PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom geometry ) +ADD_EXECUTABLE(clientMonitorTest clientMonitorTests.cpp) +TARGET_LINK_LIBRARIES( + clientMonitorTest + PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor +) + TARGET_INCLUDE_DIRECTORIES( clientTest PUBLIC "${TD_SOURCE_DIR}/include/client/" @@ -41,7 +47,18 @@ TARGET_INCLUDE_DIRECTORIES( PRIVATE "${TD_SOURCE_DIR}/source/client/inc" ) +TARGET_INCLUDE_DIRECTORIES( + clientMonitorTest + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" +) + add_test( NAME smlTest COMMAND smlTest ) + +# add_test( +# NAME clientMonitorTest +# COMMAND clientMonitorTest +# ) diff --git a/source/client/test/clientMonitorTests.cpp b/source/client/test/clientMonitorTests.cpp new file mode 100644 index 0000000000..8882e268e7 --- /dev/null +++ b/source/client/test/clientMonitorTests.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "clientInt.h" +#include "clientMonitor.h" +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "executor.h" +#include "taos.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(clientMonitorTest, monitorTest) { + const char* cluster1 = "cluster1"; + const char* cluster2 = "cluster2"; + SEpSet epSet; + clusterMonitorInit(cluster1, epSet, NULL); + const char* counterName1 = "slow_query"; + const char* counterName2 = "select_count"; + const char* help1 = "test for slowQuery"; + const char* help2 = "test for selectSQL"; + const char* lables[] = {"lable1"}; + taos_counter_t* c1 = createClusterCounter(cluster1, counterName1, help1, 1, lables); + ASSERT_TRUE(c1 != NULL); + taos_counter_t* c2 = createClusterCounter(cluster1, counterName2, help2, 1, lables); + ASSERT_TRUE(c2 != NULL); + ASSERT_TRUE(c1 != c2); + taos_counter_t* c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables); + ASSERT_TRUE(c21 == NULL); + clusterMonitorInit(cluster2, epSet, NULL); + c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables); + ASSERT_TRUE(c21 != NULL); + int i = 0; + while (i < 12) { + taosMsleep(10); + ++i; + } clusterMonitorClose(cluster1); + clusterMonitorClose(cluster2); +} + +TEST(clientMonitorTest, sendTest) { + TAOS* taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); + ASSERT_TRUE(taos != NULL); + printf("connect taosd sucessfully.\n"); + + int64_t connId = *(int64_t *)taos; + SlowQueryLog(connId, 1000); + int i = 0; + while (i < 20) { + SlowQueryLog(connId, i * 1000); + taosMsleep(10); + ++i; + } + + taos_close(taos); +} diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 8e824b4d67..d755ef0075 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -156,6 +156,7 @@ int32_t tsMetaCacheMaxSize = -1; // MB int32_t tsSlowLogThreshold = 3; // seconds int32_t tsSlowLogScope = SLOW_LOG_TYPE_ALL; int32_t tsTimeSeriesThreshold = 50; +bool enableSlowQueryMonitor = false; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -479,6 +480,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddString(pCfg, "slowLogScope", "", CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "enableSlowQueryMonitor", enableSlowQueryMonitor, CFG_SCOPE_CLIENT) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); @@ -991,6 +993,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) { return -1; } + enableSlowQueryMonitor = cfgGetItem(pCfg, "enableSlowQueryMonitor")->bval; tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; diff --git a/source/libs/monitor/CMakeLists.txt b/source/libs/monitor/CMakeLists.txt index 13523fd3cc..cc8f40fa4c 100644 --- a/source/libs/monitor/CMakeLists.txt +++ b/source/libs/monitor/CMakeLists.txt @@ -6,7 +6,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries(monitor os util common transport monitorfw) +target_link_libraries(monitor os util common qcom transport monitorfw) if(${BUILD_TEST}) add_subdirectory(test) diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h index ae1af4ba62..c5219c60b2 100644 --- a/source/libs/monitor/inc/monInt.h +++ b/source/libs/monitor/inc/monInt.h @@ -17,6 +17,7 @@ #define _TD_MONITOR_INT_H_ #include "monitor.h" +#include "query.h" #include "tjson.h" diff --git a/source/libs/monitor/src/clientMonitor.c b/source/libs/monitor/src/clientMonitor.c new file mode 100644 index 0000000000..941155f2fe --- /dev/null +++ b/source/libs/monitor/src/clientMonitor.c @@ -0,0 +1,189 @@ +#include "clientMonitor.h" +#include "os.h" +#include "tmisce.h" +#include "ttime.h" +#include "ttimer.h" + +SRWLatch monitorLock; +void* tmrClientMonitor; +tmr_h tmrStartHandle; +SHashObj* clusterMonitorInfoTable; + +static const int interval = 1000; // ms +static const int sendSize = 10; +static const int sleepTimeMS = 100; + +int32_t sendReport(ClientMonitor* pMonitor, char* pCont); +void generateClusterReport(ClientMonitor* pMonitor, bool send) { + char ts[50]; + sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + char* pCont = (char*)taos_collector_registry_bridge(pMonitor->registry, ts, "%" PRId64); + if (send && strlen(pCont) != TSDB_CODE_SUCCESS) { + if (sendReport(pMonitor, pCont) == 0) { + taos_collector_registry_clear_out(pMonitor->registry); + } + } +} + +void reportSendProcess(void* param, void* tmrId) { + taosRLockLatch(&monitorLock); + taosTmrReset(reportSendProcess, interval, NULL, tmrClientMonitor, &tmrStartHandle); + + static int index = 0; + index++; + ClientMonitor** ppMonitor = (ClientMonitor**)taosHashIterate(clusterMonitorInfoTable, NULL); + while (ppMonitor != NULL && *ppMonitor != NULL) { + ClientMonitor* pMonitor = *ppMonitor; + generateClusterReport(*ppMonitor, index == sendSize); + ppMonitor = taosHashIterate(clusterMonitorInfoTable, ppMonitor); + } + + if (index == sendSize) index = 0; + taosRUnLockLatch(&monitorLock); +} + +void monitorClientInitOnce() { + static int8_t init = 0; + if (atomic_exchange_8(&init, 1) == 0) { + uInfo("tscMonitorInit once."); + clusterMonitorInfoTable = + (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + + tmrClientMonitor = taosTmrInit(0, 0, 0, "MONITOR"); + tmrStartHandle = taosTmrStart(reportSendProcess, interval, NULL, tmrClientMonitor); + taosInitRWLatch(&monitorLock); + } +} + +void createMonitorClient(const char* clusterKey, SEpSet epSet, void* pTransporter) { + taosWLockLatch(&monitorLock); + if (clusterKey == NULL || strlen(clusterKey) == 0) { + uError("createMonitorClient failed, clusterKey is NULL"); + return; + } + + if (taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)) == NULL) { + uInfo("createMonitorClient for %s.", clusterKey); + ClientMonitor* pMonitor = taosMemoryCalloc(1, sizeof(ClientMonitor)); + snprintf(pMonitor->clusterKey, sizeof(pMonitor->clusterKey), "%s", clusterKey); + pMonitor->registry = taos_collector_registry_new(clusterKey); + pMonitor->colector = taos_collector_new(clusterKey); + pMonitor->epSet = epSet; + pMonitor->pTransporter = pTransporter; + + taos_collector_registry_register_collector(pMonitor->registry, pMonitor->colector); + pMonitor->counters = + (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + + taosHashPut(clusterMonitorInfoTable, clusterKey, strlen(clusterKey), &pMonitor, sizeof(ClientMonitor*)); + uInfo("createMonitorClient for %s finished %p.", clusterKey, pMonitor); + } + taosWUnLockLatch(&monitorLock); +} + +static int32_t monitorReportAsyncCB(void* param, SDataBuf* pMsg, int32_t code) { + static int32_t emptyRspNum = 0; + if (TSDB_CODE_SUCCESS != code) { + uError("found error in monitorReport send callback, code:%d, please check the network.", code); + } + return code; +} + +int32_t sendReport(ClientMonitor* pMonitor, char* pCont) { + SStatisReq sStatisReq; + sStatisReq.pCont = pCont; + sStatisReq.contLen = strlen(pCont); + + int tlen = tSerializeSStatisReq(NULL, 0, &sStatisReq); + if (tlen < 0) return 0; + void* buf = taosMemoryMalloc(tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + tSerializeSStatisReq(buf, tlen, &sStatisReq); + + SMsgSendInfo* pInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (pInfo == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pInfo->fp = monitorReportAsyncCB; + pInfo->msgInfo.pData = buf; + pInfo->msgInfo.len = tlen; + pInfo->msgType = TDMT_MND_STATIS; + // pInfo->param = taosMemoryMalloc(sizeof(int32_t)); + // *(int32_t*)pInfo->param = i; + pInfo->paramFreeFp = taosMemoryFree; + pInfo->requestId = tGenIdPI64(); + pInfo->requestObjRefId = 0; + + int64_t transporterId = 0; + return asyncSendMsgToServer(pMonitor->pTransporter, &pMonitor->epSet, &transporterId, pInfo); +} + +void clusterMonitorInit(const char* clusterKey, SEpSet epSet, void* pTransporter) { + monitorClientInitOnce(); + createMonitorClient(clusterKey, epSet, pTransporter); +} + +taos_counter_t* createClusterCounter(const char* clusterKey, const char* name, const char* help, size_t label_key_count, + const char** label_keys) { + ClientMonitor** ppMonitor = (ClientMonitor**)taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)); + + if (ppMonitor != NULL && *ppMonitor != NULL) { + ClientMonitor* pMonitor = *ppMonitor; + taos_counter_t** ppCounter = (taos_counter_t**)taosHashGet(pMonitor->counters, name, strlen(name)); + if (ppCounter != NULL && *ppCounter != NULL) { + taosHashRemove(pMonitor->counters, name, strlen(name)); + uInfo("createClusterCounter remove old counter: %s.", name); + } + + taos_counter_t* newCounter = taos_counter_new(name, help, label_key_count, label_keys); + if (newCounter != NULL) { + taos_collector_add_metric(pMonitor->colector, newCounter); + taosHashPut(pMonitor->counters, name, strlen(name), &newCounter, sizeof(taos_counter_t*)); + uInfo("createClusterCounter %s(%p):%s : %p.", pMonitor->clusterKey, pMonitor, name, newCounter); + return newCounter; + } else { + return NULL; + } + } else { + return NULL; + } + return NULL; +} + +int taosClusterCounterInc(const char* clusterKey, const char* counterName, const char** label_values) { + taosRLockLatch(&monitorLock); + ClientMonitor** ppMonitor = (ClientMonitor**)taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)); + + if (ppMonitor != NULL && *ppMonitor != NULL) { + ClientMonitor* pMonitor = *ppMonitor; + taos_counter_t** ppCounter = (taos_counter_t**)taosHashGet(pMonitor->counters, counterName, strlen(counterName)); + if (ppCounter != NULL && *ppCounter != NULL) { + int res = taos_counter_inc(*ppCounter, label_values); + } else { + uError("taosClusterCounterInc not found pCounter %s:%s.", clusterKey, counterName); + } + } else { + uError("taosClusterCounterInc not found pMonitor %s.", clusterKey); + } + taosRUnLockLatch(&monitorLock); + return 0; +} + +void clusterMonitorClose(const char* clusterKey) { + taosWLockLatch(&monitorLock); + ClientMonitor** ppMonitor = (ClientMonitor**)taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)); + + if (ppMonitor != NULL && *ppMonitor != NULL) { + ClientMonitor* pMonitor = *ppMonitor; + uInfo("clusterMonitorClose valule:%p clusterKey:%s.", pMonitor, pMonitor->clusterKey); + taosHashCleanup(pMonitor->counters); + taos_collector_registry_destroy(pMonitor->registry); + taosMemoryFree(pMonitor); + taosHashRemove(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)); + } + taosWUnLockLatch(&monitorLock); +} From 1494c6408278331c145e93384fe6ebcbfb9a9e76 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Wed, 8 Nov 2023 09:19:51 +0800 Subject: [PATCH 023/181] define const int direct --- source/client/src/slowQueryMonitor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c index b42f860d67..7d25cf87ca 100644 --- a/source/client/src/slowQueryMonitor.c +++ b/source/client/src/slowQueryMonitor.c @@ -23,7 +23,7 @@ const int slowQueryLabelCount = 1; const char* slowQueryLabels[] = {"cost"}; const int64_t msInSeconds = 1000; -const int64_t msInMinutes = 60 * msInSeconds; +const int64_t msInMinutes = 60 * 1000; static const char* getSlowQueryLableCostDesc(int64_t cost) { if (cost >= 30 * msInMinutes) { From ee006072096623f6e99810be0602d01e74c2ceca Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Wed, 8 Nov 2023 10:00:35 +0800 Subject: [PATCH 024/181] in line with api change on 3.0 --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 04521f4c43..b41a9454b8 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -474,7 +474,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddString(pCfg, "slowLogScope", "", CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "enableSlowQueryMonitor", enableSlowQueryMonitor, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "enableSlowQueryMonitor", enableSlowQueryMonitor, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); From c3d60989eff179e30279210b014d81ac2607e868 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Wed, 8 Nov 2023 14:11:55 +0800 Subject: [PATCH 025/181] lock optimization --- source/libs/monitor/src/clientMonitor.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/source/libs/monitor/src/clientMonitor.c b/source/libs/monitor/src/clientMonitor.c index 941155f2fe..15d047f3c0 100644 --- a/source/libs/monitor/src/clientMonitor.c +++ b/source/libs/monitor/src/clientMonitor.c @@ -10,8 +10,7 @@ tmr_h tmrStartHandle; SHashObj* clusterMonitorInfoTable; static const int interval = 1000; // ms -static const int sendSize = 10; -static const int sleepTimeMS = 100; +static const int sendBathchSize = 10; int32_t sendReport(ClientMonitor* pMonitor, char* pCont); void generateClusterReport(ClientMonitor* pMonitor, bool send) { @@ -26,19 +25,19 @@ void generateClusterReport(ClientMonitor* pMonitor, bool send) { } void reportSendProcess(void* param, void* tmrId) { - taosRLockLatch(&monitorLock); taosTmrReset(reportSendProcess, interval, NULL, tmrClientMonitor, &tmrStartHandle); + taosRLockLatch(&monitorLock); static int index = 0; index++; ClientMonitor** ppMonitor = (ClientMonitor**)taosHashIterate(clusterMonitorInfoTable, NULL); while (ppMonitor != NULL && *ppMonitor != NULL) { ClientMonitor* pMonitor = *ppMonitor; - generateClusterReport(*ppMonitor, index == sendSize); + generateClusterReport(*ppMonitor, index == sendBathchSize); ppMonitor = taosHashIterate(clusterMonitorInfoTable, ppMonitor); } - if (index == sendSize) index = 0; + if (index == sendBathchSize) index = 0; taosRUnLockLatch(&monitorLock); } @@ -56,12 +55,11 @@ void monitorClientInitOnce() { } void createMonitorClient(const char* clusterKey, SEpSet epSet, void* pTransporter) { - taosWLockLatch(&monitorLock); if (clusterKey == NULL || strlen(clusterKey) == 0) { uError("createMonitorClient failed, clusterKey is NULL"); return; } - + taosWLockLatch(&monitorLock); if (taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)) == NULL) { uInfo("createMonitorClient for %s.", clusterKey); ClientMonitor* pMonitor = taosMemoryCalloc(1, sizeof(ClientMonitor)); From ab12d110c0f59508c1dff7217c66b8b47645bf85 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Wed, 8 Nov 2023 14:28:37 +0800 Subject: [PATCH 026/181] delete unused variable --- include/libs/monitor/clientMonitor.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/libs/monitor/clientMonitor.h b/include/libs/monitor/clientMonitor.h index 0d238157c5..9d97c9004f 100644 --- a/include/libs/monitor/clientMonitor.h +++ b/include/libs/monitor/clientMonitor.h @@ -30,8 +30,6 @@ typedef struct { void* pTransporter; taos_collector_registry_t* registry; taos_collector_t* colector; - taos_counter_t* slow_query_counter; - taos_counter_t* select_counter; SHashObj* counters; } ClientMonitor; From 6b7a5b29103c8ed63def641e74b29523961a859d Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 14 Dec 2023 11:41:47 +0800 Subject: [PATCH 027/181] fix: get tscobj by rid --- source/client/inc/clientInt.h | 2 +- source/client/src/clientEnv.c | 2 +- source/client/src/selectMonitor.c | 6 +++--- source/client/src/slowQueryMonitor.c | 6 +++--- source/client/test/clientMonitorTests.cpp | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index ef19fb7c51..fe3e55092c 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -430,7 +430,7 @@ int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, boo void clusterSlowQueryMonitorInit(const char* clusterKey); void clusterSlowQueryLog(const char* clusterKey, int32_t cost); -void SlowQueryLog(int64_t connId, int32_t cost); +void SlowQueryLog(int64_t rid, int32_t cost); void clusterSelectMonitorInit(const char* clusterKey); void clusterSelectLog(const char* clusterKey); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 26b076e5ee..8c6292976f 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -114,7 +114,7 @@ static void deregisterRequest(SRequestObj *pRequest) { taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s", taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration, pRequest->sqlstr); - SlowQueryLog(pTscObj->connId, duration); + SlowQueryLog(pTscObj->id, duration); } } diff --git a/source/client/src/selectMonitor.c b/source/client/src/selectMonitor.c index 0cb2073961..c1cab23188 100644 --- a/source/client/src/selectMonitor.c +++ b/source/client/src/selectMonitor.c @@ -33,14 +33,14 @@ void clusterSelectLog(const char* clusterKey) { taosClusterCounterInc(clusterKey, selectMonitorName, selectMonitorLabelValues); } -void selectLog(int64_t connId) { - STscObj* pTscObj = acquireTscObj(connId); +void selectLog(int64_t rid) { + STscObj* pTscObj = acquireTscObj(rid); if (pTscObj != NULL) { if(pTscObj->pAppInfo == NULL) { tscLog("selectLog, not found pAppInfo"); } return clusterSelectLog(pTscObj->pAppInfo->instKey); } else { - tscLog("selectLog, not found connect ID"); + tscLog("selectLog, not found rid"); } } diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c index 7d25cf87ca..420b66a954 100644 --- a/source/client/src/slowQueryMonitor.c +++ b/source/client/src/slowQueryMonitor.c @@ -59,15 +59,15 @@ void clusterSlowQueryLog(const char* clusterKey, int32_t cost) { taosClusterCounterInc(clusterKey, slowQueryName, slowQueryLabelValues); } -void SlowQueryLog(int64_t connId, int32_t cost) { +void SlowQueryLog(int64_t rid, int32_t cost) { if (!enableSlowQueryMonitor) return; - STscObj* pTscObj = acquireTscObj(connId); + STscObj* pTscObj = acquireTscObj(rid); if (pTscObj != NULL) { if(pTscObj->pAppInfo == NULL) { tscLog("SlowQueryLog, not found pAppInfo"); } return clusterSlowQueryLog(pTscObj->pAppInfo->instKey, cost); } else { - tscLog("SlowQueryLog, not found connect ID"); + tscLog("SlowQueryLog, not found rid"); } } diff --git a/source/client/test/clientMonitorTests.cpp b/source/client/test/clientMonitorTests.cpp index 8882e268e7..5272a2ab3a 100644 --- a/source/client/test/clientMonitorTests.cpp +++ b/source/client/test/clientMonitorTests.cpp @@ -68,11 +68,11 @@ TEST(clientMonitorTest, sendTest) { ASSERT_TRUE(taos != NULL); printf("connect taosd sucessfully.\n"); - int64_t connId = *(int64_t *)taos; - SlowQueryLog(connId, 1000); + int64_t rid = *(int64_t *)taos; + SlowQueryLog(rid, 1000); int i = 0; while (i < 20) { - SlowQueryLog(connId, i * 1000); + SlowQueryLog(rid, i * 1000); taosMsleep(10); ++i; } From dce02868b7ff2e94bdd025972f9465317862f889 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 19 Jan 2024 19:27:36 +0800 Subject: [PATCH 028/181] fix: user sysinfo --- source/libs/parser/src/parTranslater.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 70e4744644..3f1ae127a3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -380,7 +380,10 @@ static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCac int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView) { int32_t code = TSDB_CODE_SUCCESS; - + if (!pParCxt->enableSysInfo && IS_SYS_DBNAME(pName->dbname)) { + code = TSDB_CODE_PAR_PERMISSION_DENIED; + return code; + } if (pParCxt->async) { code = getTableMetaFromCache(pMetaCache, pName, pMeta); #ifdef TD_ENTERPRISE From d9af07de6fe590e7960f1a8dddc5890b7ffd3b2b Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 24 Jan 2024 09:44:44 +0000 Subject: [PATCH 029/181] new table structure --- include/common/tglobal.h | 3 + include/libs/monitor/monitor.h | 7 +- include/libs/monitorfw/taos_collector.h | 2 + .../libs/monitorfw/taos_collector_registry.h | 6 +- include/libs/monitorfw/taos_gauge.h | 154 ++++ include/libs/monitorfw/taos_metric_sample.h | 2 + include/libs/monitorfw/taos_monitor.h | 2 + include/libs/monitorfw/taos_monitor_util.h | 31 + include/util/tdef.h | 6 + source/client/src/clientEnv.c | 5 + source/client/src/selectMonitor.c | 2 +- source/client/src/slowQueryMonitor.c | 2 +- source/common/src/tglobal.c | 14 +- source/dnode/mgmt/mgmt_dnode/inc/dmInt.h | 1 + source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 1 + source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 11 + source/dnode/mgmt/node_mgmt/inc/dmMgmt.h | 1 + source/dnode/mgmt/node_mgmt/src/dmEnv.c | 1 + source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 18 +- source/dnode/mgmt/node_util/inc/dmUtil.h | 1 + source/dnode/mnode/impl/src/mndDnode.c | 124 +++- source/dnode/mnode/impl/src/mndMain.c | 3 + source/dnode/vnode/src/inc/vnodeInt.h | 24 + source/dnode/vnode/src/vnd/vnodeOpen.c | 22 + source/dnode/vnode/src/vnd/vnodeSvr.c | 69 +- source/libs/monitor/inc/monInt.h | 15 + source/libs/monitor/src/clientMonitor.c | 7 +- source/libs/monitor/src/monFramework.c | 656 ++++++++++++++++++ source/libs/monitor/src/monMain.c | 118 ++-- source/libs/monitor/test/monTest.cpp | 4 +- .../monitorfw/inc/taos_collector_registry_t.h | 2 +- .../inc/taos_metric_formatter_custom_i.h | 31 + .../libs/monitorfw/inc/taos_metric_sample_t.h | 8 +- source/libs/monitorfw/inc/taos_metric_t.h | 2 +- .../libs/monitorfw/inc/taos_monitor_util_i.h | 26 + source/libs/monitorfw/src/taos_collector.c | 6 + .../monitorfw/src/taos_collector_registry.c | 89 ++- source/libs/monitorfw/src/taos_gauge.c | 100 +++ source/libs/monitorfw/src/taos_metric.c | 9 +- .../monitorfw/src/taos_metric_formatter.c | 2 +- .../src/taos_metric_formatter_custom.c | 234 +++++++ .../libs/monitorfw/src/taos_metric_sample.c | 53 +- source/libs/monitorfw/src/taos_monitor_util.c | 110 +++ 43 files changed, 1878 insertions(+), 106 deletions(-) create mode 100644 include/libs/monitorfw/taos_gauge.h create mode 100644 include/libs/monitorfw/taos_monitor_util.h create mode 100644 source/libs/monitor/src/monFramework.c create mode 100644 source/libs/monitorfw/inc/taos_metric_formatter_custom_i.h create mode 100644 source/libs/monitorfw/inc/taos_monitor_util_i.h create mode 100644 source/libs/monitorfw/src/taos_gauge.c create mode 100644 source/libs/monitorfw/src/taos_metric_formatter_custom.c create mode 100644 source/libs/monitorfw/src/taos_monitor_util.c diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 3a7ff40125..9b8d63b327 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -105,6 +105,9 @@ extern char tsMonitorFqdn[]; extern uint16_t tsMonitorPort; extern int32_t tsMonitorMaxLogs; extern bool tsMonitorComp; +extern bool tsMonitorLogProtocol; +extern int32_t tsMonitorIntervalForBasic; +extern bool tsMonitorForceV2; // audit extern bool tsEnableAudit; diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 1f3d21777d..4c6ffa0add 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -100,6 +100,7 @@ typedef struct { int32_t mnode_id; char mnode_ep[TSDB_EP_LEN]; char role[MON_ROLE_LEN]; + int32_t syncState; } SMonMnodeDesc; typedef struct { @@ -125,6 +126,7 @@ typedef struct { typedef struct { int32_t dnode_id; char vnode_role[MON_ROLE_LEN]; + int32_t syncState; } SMonVnodeDesc; typedef struct { @@ -221,9 +223,8 @@ void monSetVmInfo(SMonVmInfo *pInfo); void monSetQmInfo(SMonQmInfo *pInfo); void monSetSmInfo(SMonSmInfo *pInfo); void monSetBmInfo(SMonBmInfo *pInfo); -void monSendReport(); -void monSendPromReport(); -void monSendContent(char *pCont); +void monGenAndSendReport(); +void monGenAndSendReportBasic(); void tFreeSMonMmInfo(SMonMmInfo *pInfo); void tFreeSMonVmInfo(SMonVmInfo *pInfo); diff --git a/include/libs/monitorfw/taos_collector.h b/include/libs/monitorfw/taos_collector.h index 918395ae72..8fe304ed7d 100644 --- a/include/libs/monitorfw/taos_collector.h +++ b/include/libs/monitorfw/taos_collector.h @@ -76,6 +76,8 @@ int taos_collector_destroy_generic(void *gen); */ int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric); +taos_metric_t* taos_collector_get_metric(taos_collector_t *self, char *metric_name); + /** * @brief The collect function is responsible for doing any work involving a set of metrics and then returning them * for metric exposition. diff --git a/include/libs/monitorfw/taos_collector_registry.h b/include/libs/monitorfw/taos_collector_registry.h index 915b132b11..063e8afb50 100644 --- a/include/libs/monitorfw/taos_collector_registry.h +++ b/include/libs/monitorfw/taos_collector_registry.h @@ -86,6 +86,8 @@ taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metri */ int taos_collector_registry_register_metric(taos_metric_t *metric); +taos_metric_t *taos_collector_registry_get_metric(char* metric_name); + /** * @brief Register a collector with the given registry. Returns a non-zero integer value on failure. * @param self The target taos_collector_registry_t* @@ -105,7 +107,9 @@ int taos_collector_registry_register_collector(taos_collector_registry_t *self, */ const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts, char *format); -int taos_collector_registry_clear_out(taos_collector_registry_t *self); +int taos_collector_registry_clear_batch(taos_collector_registry_t *self); + +const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self, char *ts, char *format, char** prom_str); /** *@brief Validates that the given metric name complies with the specification: diff --git a/include/libs/monitorfw/taos_gauge.h b/include/libs/monitorfw/taos_gauge.h new file mode 100644 index 0000000000..a33661dfe4 --- /dev/null +++ b/include/libs/monitorfw/taos_gauge.h @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_gauge.h + * @brief https://prometheus.io/docs/concepts/metric_types/#gauge + */ + +#ifndef TAOS_GAUGE_H +#define TAOS_GAUGE_H + +#include + +#include "taos_metric.h" + +/** + * @brief A prometheus gauge. + * + * References + * * See https://prometheus.io/docs/concepts/metric_types/#gauge + */ +typedef taos_metric_t taos_gauge_t; + +/** + * @brief Constructs a taos_gauge_t* + * @param name The name of the metric + * @param help The metric description + * @param label_key_count The number of labels associated with the given metric. Pass 0 if the metric does not + * require labels. + * @param label_keys A collection of label keys. The number of keys MUST match the value passed as label_key_count. If + * no labels are required, pass NULL. Otherwise, it may be convenient to pass this value as a + * literal. + * @return The constructed taos_guage_t* + * + * // An example with labels + * taos_gauge_new("foo", "foo is a gauge with labels", 2, (const char**) { "one", "two" }); + * + * // An example without labels + * taos_gauge_new("foo", "foo is a gauge without labels", 0, NULL); + */ +taos_gauge_t *taos_gauge_new(const char *name, const char *help, size_t label_key_count, const char **label_keys); + +/** + * @brief Destroys a taos_gauge_t*. You must set self to NULL after destruction. A non-zero integer value will be + * returned on failure. + * @param self The target taos_gauge_t* + * @return A non-zero integer value upon failure + */ +int taos_gauge_destroy(taos_gauge_t *self); + +/** + * @brief Increment the taos_gauge_t* by 1. + * @param self The target taos_gauger_t* + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the gauge's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure + * *Example* + * + * // An example with labels + * taos_gauge_inc(foo_gauge, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_gauge_inc(foo_gauge, NULL); + */ +int taos_gauge_inc(taos_gauge_t *self, const char **label_values); + +/** + * @brief Decrement the taos_gauge_t* by 1. + * @param self The target taos_gauger_t* + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the gauge's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * *Example* + * + * // An example with labels + * taos_gauge_dec(foo_gauge, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_gauge_dec(foo_gauge, NULL); + */ +int taos_gauge_dec(taos_gauge_t *self, const char **label_values); + +/** + * @brief Add the value to the taos_gauge_t*. + * @param self The target taos_gauge_t* + * @param r_value The double to add to the taos_gauge_t passed as self. + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the gauge's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_gauge_add(foo_gauge 22, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_gauge_add(foo_gauge, 22, NULL); + */ +int taos_gauge_add(taos_gauge_t *self, double r_value, const char **label_values); + +/** + * @brief Subtract the value to the taos_gauge. A non-zero integer value will be returned on failure. + * @param self The target taos_gauge_t* + * @param r_value The double to add to the taos_gauge_t passed as self. + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the gauge's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_gauge_sub(foo_gauge 22, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_gauge_sub(foo_gauge, 22, NULL); + */ +int taos_gauge_sub(taos_gauge_t *self, double r_value, const char **label_values); + +/** + * @brief Set the value for the taos_gauge_t* + * @param self The target taos_gauge_t* + * @param r_value The double to which the taos_gauge_t* passed as self will be set + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the gauge's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_gauge_set(foo_gauge 22, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_gauge_set(foo_gauge, 22, NULL); + */ +int taos_gauge_set(taos_gauge_t *self, double r_value, const char **label_values); + +#endif // TAOS_GAUGE_H diff --git a/include/libs/monitorfw/taos_metric_sample.h b/include/libs/monitorfw/taos_metric_sample.h index 1c37f59e32..0283d56ad9 100644 --- a/include/libs/monitorfw/taos_metric_sample.h +++ b/include/libs/monitorfw/taos_metric_sample.h @@ -56,4 +56,6 @@ int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value); */ int taos_metric_sample_set(taos_metric_sample_t *self, double r_value); +int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, double* old_value); + #endif // TAOS_METRIC_SAMPLE_H diff --git a/include/libs/monitorfw/taos_monitor.h b/include/libs/monitorfw/taos_monitor.h index ec4f007c45..c8569dd0a0 100644 --- a/include/libs/monitorfw/taos_monitor.h +++ b/include/libs/monitorfw/taos_monitor.h @@ -126,5 +126,7 @@ limitations under the License. #include "taos_map.h" #include "taos_metric.h" #include "taos_metric_sample.h" +#include "taos_monitor_util.h" +#include "taos_gauge.h" #endif // TAOS_INCLUDED \ No newline at end of file diff --git a/include/libs/monitorfw/taos_monitor_util.h b/include/libs/monitorfw/taos_monitor_util.h new file mode 100644 index 0000000000..a42d1ddcef --- /dev/null +++ b/include/libs/monitorfw/taos_monitor_util.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_monitor_util.h + * @brief Functions for retrieving metric samples from metrics given an ordered set of labels + */ + +#ifndef TAOS_MONITOR_UTIL_H +#define TAOS_MONITOR_UTIL_H + +#include "taos_metric.h" +#include "tjson.h" + +void taos_monitor_split_str(char** arr, char* str, const char* del); +void taos_monitor_split_str_metric(char** arr, taos_metric_t* metric, const char* del, char** buf); +char* taos_monitor_get_metric_name(taos_metric_t* metric); + +#endif // TAOS_MONITOR_UTIL_H \ No newline at end of file diff --git a/include/util/tdef.h b/include/util/tdef.h index 875a6f5738..b6f64c368d 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -196,6 +196,8 @@ typedef enum ELogicConditionType { // ACCOUNT is a 32 bit positive integer // this is the length of its string representation, including the terminator zero #define TSDB_ACCT_ID_LEN 11 +#define TSDB_NODE_ID_LEN 11 +#define TSDB_VGROUP_ID_LEN 11 #define TSDB_MAX_COLUMNS 4096 #define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns @@ -543,6 +545,10 @@ enum { #define VNODE_TIMEOUT_SEC 60 #define MNODE_TIMEOUT_SEC 60 +#define MONITOR_TABLENAME_LEN 200 +#define MONITOR_TAG_NAME_LEN 100 +#define MONITOR_TAG_VALUE_LEN 300 +#define MONITOR_METRIC_NAME_LEN 100 #ifdef __cplusplus } #endif diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index b3d0023f9a..bb77c530c3 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -18,6 +18,7 @@ #include "clientLog.h" #include "functionMgt.h" #include "os.h" +#include "osSleep.h" #include "query.h" #include "qworker.h" #include "scheduler.h" @@ -82,6 +83,8 @@ static void deregisterRequest(SRequestObj *pRequest) { int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); int32_t reqType = SLOW_LOG_TYPE_OTHERS; + //taosSsleep(3); + int64_t duration = taosGetTimestampUs() - pRequest->metric.start; tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%.2f ms, " @@ -108,6 +111,8 @@ static void deregisterRequest(SRequestObj *pRequest) { } } + duration = 7000000; + if (duration >= (tsSlowLogThreshold * 1000000UL)) { atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); if (tsSlowLogScope & reqType) { diff --git a/source/client/src/selectMonitor.c b/source/client/src/selectMonitor.c index c1cab23188..58aba8de0b 100644 --- a/source/client/src/selectMonitor.c +++ b/source/client/src/selectMonitor.c @@ -16,7 +16,7 @@ #include "clientMonitor.h" #include "clientLog.h" -const char* selectMonitorName = "slow_query"; +const char* selectMonitorName = "slow_query:slow_query_metric"; const char* selectMonitorHelp = "slow query log when cost > 3s"; const int selectMonitorLabelCount = 1; const char* selectMonitorLabels[] = {"default"}; diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c index 420b66a954..4af00f5360 100644 --- a/source/client/src/slowQueryMonitor.c +++ b/source/client/src/slowQueryMonitor.c @@ -17,7 +17,7 @@ #include "clientLog.h" #include "tglobal.h" -const char* slowQueryName = "slow_query"; +const char* slowQueryName = "slow_query:slow_query_metric"; const char* slowQueryHelp = "slow query log when cost > 3s"; const int slowQueryLabelCount = 1; const char* slowQueryLabels[] = {"cost"}; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 30596d3e72..cb853c1a75 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -93,6 +93,9 @@ char tsMonitorFqdn[TSDB_FQDN_LEN] = {0}; uint16_t tsMonitorPort = 6043; int32_t tsMonitorMaxLogs = 100; bool tsMonitorComp = false; +bool tsMonitorLogProtocol = false; +int32_t tsMonitorIntervalForBasic = 30; +bool tsMonitorForceV2 = false; // audit bool tsEnableAudit = true; @@ -690,7 +693,11 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - + if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) @@ -1152,7 +1159,10 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32; tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval; tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32; - + tsMonitorLogProtocol = cfgGetItem(pCfg, "monitorLogProtocol")->bval; + tsMonitorIntervalForBasic = cfgGetItem(pCfg, "monitorIntervalForBasic")->i32; + tsMonitorForceV2 = cfgGetItem(pCfg, "monitorForceV2")->i32; + tsEnableAudit = cfgGetItem(pCfg, "audit")->bval; tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval; tsAuditInterval = cfgGetItem(pCfg, "auditInterval")->i32; diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 80502e2662..975246a10f 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -43,6 +43,7 @@ typedef struct SDnodeMgmt { GetMnodeLoadsFp getMnodeLoadsFp; GetQnodeLoadsFp getQnodeLoadsFp; int32_t statusSeq; + SendMonitorReportFp sendMonitorReportFpBasic; } SDnodeMgmt; // dmHandle.c diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index b9dd45f1c0..a651fbf060 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -65,6 +65,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->processDropNodeFp = pInput->processDropNodeFp; pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp; pMgmt->sendAuditRecordsFp = pInput->sendAuditRecordFp; + pMgmt->sendMonitorReportFpBasic = pInput->sendMonitorReportFpBasic; pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp; pMgmt->getVnodeLoadsLiteFp = pInput->getVnodeLoadsLiteFp; pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index af43804db4..1f2dc575d0 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -81,6 +81,7 @@ static void *dmNotifyThreadFp(void *param) { static void *dmMonitorThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); + int64_t lastTimeForBasic = taosGetTimestampMs(); setThreadName("dnode-monitor"); while (1) { @@ -88,12 +89,22 @@ static void *dmMonitorThreadFp(void *param) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; int64_t curTime = taosGetTimestampMs(); + if (curTime < lastTime) lastTime = curTime; float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsMonitorInterval) { (*pMgmt->sendMonitorReportFp)(); lastTime = curTime; } + + if(tsMonitorForceV2){ + if (curTime < lastTimeForBasic) lastTimeForBasic = curTime; + float intervalForBasic = (curTime - lastTimeForBasic) / 1000.0f; + if (intervalForBasic >= tsMonitorIntervalForBasic) { + (*pMgmt->sendMonitorReportFpBasic)(); + lastTimeForBasic = curTime; + } + } } return NULL; diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index c646bb4bdd..25e388672e 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -149,6 +149,7 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); // dmMonitor.c void dmSendMonitorReport(); void dmSendAuditRecords(); +void dmSendMonitorReportBasic(); void dmGetVnodeLoads(SMonVloadInfo *pInfo); void dmGetVnodeLoadsLite(SMonVloadInfo *pInfo); void dmGetMnodeLoads(SMonMloadInfo *pInfo); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index f9bba19fbb..b343be9426 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -398,6 +398,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) { .processDropNodeFp = dmProcessDropNodeReq, .sendMonitorReportFp = dmSendMonitorReport, .sendAuditRecordFp = auditSendRecordsInBatch, + .sendMonitorReportFpBasic = dmSendMonitorReportBasic, .getVnodeLoadsFp = dmGetVnodeLoads, .getVnodeLoadsLiteFp = dmGetVnodeLoadsLite, .getMnodeLoadsFp = dmGetMnodeLoads, diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index 69a7956c9c..590c25b936 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -19,7 +19,8 @@ #include "audit.h" static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { - pInfo->protocol = 1; + //pInfo->protocol = 1; + pInfo->protocol = 2; pInfo->dnode_id = pDnode->data.dnodeId; pInfo->cluster_id = pDnode->data.clusterId; tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); @@ -106,9 +107,20 @@ void dmSendMonitorReport() { dmGetVmMonitorInfo(pDnode); dmGetQmMonitorInfo(pDnode); dmGetSmMonitorInfo(pDnode); - monSendReport(); + monGenAndSendReport(); +} - monSendPromReport(); +void dmSendMonitorReportBasic() { + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; + dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); + + SDnode *pDnode = dmInstance(); + dmGetDmMonitorInfo(pDnode); + dmGetMmMonitorInfo(pDnode); + //dmGetVmMonitorInfo(pDnode); + //dmGetQmMonitorInfo(pDnode); + //dmGetSmMonitorInfo(pDnode); + monGenAndSendReportBasic(); } //Todo: put this in seperate file in the future diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 4769ef8538..16c464d17c 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -122,6 +122,7 @@ typedef struct { ProcessDropNodeFp processDropNodeFp; SendMonitorReportFp sendMonitorReportFp; SendAuditRecordsFp sendAuditRecordFp; + SendMonitorReportFp sendMonitorReportFpBasic; GetVnodeLoadsFp getVnodeLoadsFp; GetVnodeLoadsFp getVnodeLoadsLiteFp; GetMnodeLoadsFp getMnodeLoadsFp; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index b75753a87f..bd0c29f4c8 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#include "tjson.h" #include "mndDnode.h" #include "audit.h" #include "mndCluster.h" @@ -28,6 +29,7 @@ #include "mndVgroup.h" #include "tmisce.h" #include "tunit.h" +#include "taos_monitor.h" #define TSDB_DNODE_VER_NUMBER 2 #define TSDB_DNODE_RESERVE_SIZE 64 @@ -503,9 +505,129 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { goto _OVER; } - monSendContent(pReq->pCont); + if(tsMonitorLogProtocol){ + mInfo("process statis req,\n %s", statisReq.pCont); + } + + SJson* pJson = tjsonParse(statisReq.pCont); + + int32_t ts_size = tjsonGetArraySize(pJson); + + for(int32_t i = 0; i < ts_size; i++){ + SJson* item = tjsonGetArrayItem(pJson, i); + + SJson* tables = tjsonGetObjectItem(item, "tables"); + + int32_t tableSize = tjsonGetArraySize(tables); + for(int32_t i = 0; i < tableSize; i++){ + SJson* table = tjsonGetArrayItem(tables, i); + + char tableName[MONITOR_TABLENAME_LEN] = {0}; + tjsonGetStringValue(table, "name", tableName); + + SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups"); + + int32_t size = tjsonGetArraySize(metricGroups); + for(int32_t i = 0; i < size; i++){ + SJson* item = tjsonGetArrayItem(metricGroups, i); + + SJson* arrayTag = tjsonGetObjectItem(item, "tags"); + + int32_t tagSize = tjsonGetArraySize(arrayTag); + + char** labels = taosMemoryMalloc(sizeof(char*) * tagSize); + char** sample_labels = taosMemoryMalloc(sizeof(char*) * tagSize); + + for(int32_t j = 0; j < tagSize; j++){ + SJson* item = tjsonGetArrayItem(arrayTag, j); + + *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); + tjsonGetStringValue(item, "name", *(labels + j)); + + *(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN); + tjsonGetStringValue(item, "value", *(sample_labels + j)); + } + + SJson* metrics = tjsonGetObjectItem(item, "metrics"); + + int32_t metricLen = tjsonGetArraySize(metrics); + for(int32_t j = 0; j < metricLen; j++){ + SJson *item = tjsonGetArrayItem(metrics, j); + + char name[MONITOR_METRIC_NAME_LEN] = {0}; + tjsonGetStringValue(item, "name", name); + + double value = 0; + tjsonGetDoubleValue(item, "value", &value); + + double type = 0; + tjsonGetDoubleValue(item, "type", &type); + + int32_t metricNameLen = strlen(name) + strlen(tableName) + 2; + char* metricName = taosMemoryMalloc(metricNameLen); + memset(metricName, 0, metricNameLen); + sprintf(metricName, "%s:%s", tableName, name); + + taos_metric_t* metric = taos_collector_registry_get_metric(metricName); + if(metric == NULL){ + if(type == 0){ + metric = taos_counter_new(metricName, "", tagSize, (const char**)labels); + } + if(type == 1){ + metric = taos_gauge_new(metricName, "", tagSize, (const char**)labels); + } + mTrace("fail to get metric from registry, new one metric:%p", metric); + + if(taos_collector_registry_register_metric(metric) == 1){ + if(type == 0){ + taos_counter_destroy(metric); + } + if(type == 1){ + taos_gauge_destroy(metric); + } + + metric = taos_collector_registry_get_metric(metricName); + + mTrace("fail to register metric, get metric from registry:%p", metric); + } + else{ + mTrace("succeed to register metric:%p", metric); + } + } + else{ + mTrace("get metric from registry:%p", metric); + } + + if(type == 0){ + taos_counter_add(metric, value, (const char**)sample_labels); + } + if(type == 1){ + taos_gauge_set(metric, value, (const char**)sample_labels); + } + + taosMemoryFreeClear(metricName); + } + + for(int32_t j = 0; j < tagSize; j++){ + taosMemoryFreeClear(*(labels + j)); + taosMemoryFreeClear(*(sample_labels + j)); + } + + taosMemoryFreeClear(sample_labels); + taosMemoryFreeClear(labels); + } + } + + } + + code = 0; _OVER: + if(pJson != NULL){ + tjsonDelete(pJson); + pJson = NULL; + } + tFreeSStatisReq(&statisReq); return code; } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 38a92b43e7..51abcc4eae 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -859,8 +859,10 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f; // pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f); tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role)); + desc.syncState = TAOS_SYNC_STATE_LEADER; } else { tstrncpy(desc.role, syncStr(pObj->syncState), sizeof(desc.role)); + desc.syncState = pObj->syncState; } taosArrayPush(pClusterInfo->mnodes, &desc); sdbRelease(pSdb, pObj); @@ -891,6 +893,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr SMonVnodeDesc *pVnDesc = &desc.vnodes[i]; pVnDesc->dnode_id = pVgid->dnodeId; tstrncpy(pVnDesc->vnode_role, syncStr(pVgid->syncState), sizeof(pVnDesc->vnode_role)); + pVnDesc->syncState = pVgid->syncState; if (pVgid->syncState == TAOS_SYNC_STATE_LEADER) { tstrncpy(desc.status, "ready", sizeof(desc.status)); pClusterInfo->vgroups_alive++; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index c1a4754b62..eb9c3483d1 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -47,6 +47,8 @@ #include "vnode.h" +#include "taos_monitor.h" + #ifdef __cplusplus extern "C" { #endif @@ -104,6 +106,20 @@ typedef struct SQueryNode SQueryNode; #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" +#define VNODE_METRIC_SQL_COUNT "taoscd_sql_req:count" + +#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type" +#define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id" +#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id" +#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep" +#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id" +#define VNODE_METRIC_TAG_NAME_USERNAME "username" +#define VNODE_METRIC_TAG_NAME_RESULT "result" + +#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "insert_affected_rows" +//#define VNODE_METRIC_TAG_VALUE_INSERT "insert" +#define VNODE_METRIC_TAG_VALUE_DELETE "delete" + // vnd.h typedef int32_t (*_query_reseek_func_t)(void* pQHandle); struct SQueryNode { @@ -438,6 +454,13 @@ typedef struct SVCommitSched { int64_t maxWaitMs; } SVCommitSched; +typedef struct SVMonitorObj{ + char strClusterId[TSDB_CLUSTER_ID_LEN]; + char strDnodeId[TSDB_NODE_ID_LEN]; + char strVgId[TSDB_VGROUP_ID_LEN]; + taos_counter_t *insertCounter; +}SVMonitorObj; + struct SVnode { char* path; SVnodeCfg config; @@ -476,6 +499,7 @@ struct SVnode { int32_t blockSec; int64_t blockSeq; SQHandle* pQuery; + SVMonitorObj monitor; }; #define TD_VID(PVNODE) ((PVNODE)->config.vgId) diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 946ce9d278..e1f76b3a25 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -480,6 +480,28 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC vnodeRollback(pVnode); } + snprintf(pVnode->monitor.strClusterId, TSDB_CLUSTER_ID_LEN, "%"PRId64, pVnode->config.syncCfg.nodeInfo[0].clusterId); + snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%"PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId); + snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%"PRId32, pVnode->config.vgId); + + if(pVnode->monitor.insertCounter == NULL){ + int32_t label_count = 7; + const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID, + VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP, + VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME, + VNODE_METRIC_TAG_NAME_RESULT}; + taos_counter_t *counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql", + label_count, sample_labels); + vInfo("vgId:%d, new metric:%p",TD_VID(pVnode), counter); + if(taos_collector_registry_register_metric(counter) == 1){ + taos_counter_destroy(counter); + counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT); + vInfo("vgId:%d, get metric from registry:%p",TD_VID(pVnode), counter); + } + pVnode->monitor.insertCounter = counter; + vInfo("vgId:%d, succeed to set metric:%p",TD_VID(pVnode), counter); + } + return pVnode; _err: diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 1c45c93346..d79d239a50 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -21,8 +21,6 @@ #include "vnd.h" #include "vnode.h" #include "vnodeInt.h" -#include "taos_monitor.h" - static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -32,20 +30,21 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, SRpcMsg *pOriginRpc); -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg); static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg); static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessConfigChangeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); -taos_counter_t *insert_counter = NULL; extern int32_t vnodeProcessKillCompactReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); extern int32_t vnodeQueryCompactProgress(SVnode *pVnode, SRpcMsg *pMsg); @@ -535,10 +534,10 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg break; /* TSDB */ case TDMT_VND_SUBMIT: - if (vnodeProcessSubmitReq(pVnode, ver, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err; + if (vnodeProcessSubmitReq(pVnode, ver, pMsg->pCont, pMsg->contLen, pRsp, pMsg) < 0) goto _err; break; case TDMT_VND_DELETE: - if (vnodeProcessDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDeleteReq(pVnode, ver, pReq, len, pRsp, pMsg) < 0) goto _err; break; case TDMT_VND_BATCH_DEL: if (vnodeProcessBatchDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; @@ -1482,7 +1481,8 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) { return code; } -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg) { int32_t code = 0; terrno = 0; @@ -1700,28 +1700,33 @@ _exit: atomic_add_fetch_64(&pVnode->statis.nInsert, pSubmitRsp->affectedRows); atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows); atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); + + const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Success"}; + taos_counter_add(pVnode->monitor.insertCounter, pSubmitRsp->affectedRows, sample_labels); + if (code == 0) { atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); code = tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len); } + /* + if (code == 0) { + atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); + code = tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len); - if(insert_counter == NULL){ - int32_t label_count =2; - const char *sample_labels[] = {"vgid", "endpoint"}; - taos_counter_t *counter = taos_counter_new("insert_counter", "counter for insert sql", label_count, sample_labels); - if(taos_collector_registry_register_metric(counter) == 1){ - taos_counter_destroy(counter); - } - else{ - atomic_store_ptr(&insert_counter, counter); - } + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Success"}; + taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); } - - char vgId[50]; - sprintf(vgId, "%"PRId32, TD_VID(pVnode)); - const char *sample_labels[] = {vgId, tsLocalEp}; - - taos_counter_inc(insert_counter, sample_labels); + else{ + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Failed"}; + taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); + } + */ // clear taosArrayDestroy(newTbUids); @@ -1977,7 +1982,8 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pRe return 0; } -static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg) { int32_t code = 0; SDecoder *pCoder = &(SDecoder){0}; SDeleteRes *pRes = &(SDeleteRes){0}; @@ -2020,6 +2026,19 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, in return code; _err: + if(code == TSDB_CODE_SUCCESS){ + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_DELETE, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Success"}; + taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); + } + else{ + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_DELETE, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Failed"}; + taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); + } + return code; } static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h index c5219c60b2..7fc718393b 100644 --- a/source/libs/monitor/inc/monInt.h +++ b/source/libs/monitor/inc/monInt.h @@ -20,6 +20,7 @@ #include "query.h" #include "tjson.h" +#include "thash.h" typedef struct { int64_t curTime; @@ -45,8 +46,22 @@ typedef struct { SMonSmInfo smInfo; SMonQmInfo qmInfo; SMonBmInfo bmInfo; + SHashObj *metrics; } SMonitor; +void monGenClusterInfoTable(SMonInfo *pMonitor); +void monGenVgroupInfoTable(SMonInfo *pMonitor); +void monGenDnodeInfoTable(SMonInfo *pMonitor); +void monGenDnodeStatusInfoTable(SMonInfo *pMonitor); +void monGenDataDiskTable(SMonInfo *pMonitor); +void monGenLogDiskTable(SMonInfo *pMonitor); +void monGenMnodeRoleTable(SMonInfo *pMonitor); +void monGenVnodeRoleTable(SMonInfo *pMonitor); + +void monSendPromReport(); +void monInitMonitorFW(); +void monCleanupMonitorFW(); + #ifdef __cplusplus } #endif diff --git a/source/libs/monitor/src/clientMonitor.c b/source/libs/monitor/src/clientMonitor.c index 15d047f3c0..64df693e1e 100644 --- a/source/libs/monitor/src/clientMonitor.c +++ b/source/libs/monitor/src/clientMonitor.c @@ -10,16 +10,17 @@ tmr_h tmrStartHandle; SHashObj* clusterMonitorInfoTable; static const int interval = 1000; // ms -static const int sendBathchSize = 10; +static const int sendBathchSize = 1; int32_t sendReport(ClientMonitor* pMonitor, char* pCont); void generateClusterReport(ClientMonitor* pMonitor, bool send) { char ts[50]; sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); - char* pCont = (char*)taos_collector_registry_bridge(pMonitor->registry, ts, "%" PRId64); + char* pCont = (char*)taos_collector_registry_bridge_new(pMonitor->registry, ts, "%" PRId64, NULL); + uInfo("report cont:\n%s", pCont); if (send && strlen(pCont) != TSDB_CODE_SUCCESS) { if (sendReport(pMonitor, pCont) == 0) { - taos_collector_registry_clear_out(pMonitor->registry); + taos_collector_registry_clear_batch(pMonitor->registry); } } } diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c new file mode 100644 index 0000000000..3cc2ab8c90 --- /dev/null +++ b/source/libs/monitor/src/monFramework.c @@ -0,0 +1,656 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "monitor.h" +#include "monInt.h" + +#include "thash.h" +#include "taos_monitor.h" +#include "thttp.h" +#include "ttime.h" +#include "tglobal.h" + +extern SMonitor tsMonitor; +extern char* tsMonUri; +extern char* tsMonFwUri; + +#define LEVEL_LEN 11 + +#define CLUSTER_TABLE "taosd_cluster_info" + +#define MASTER_UPTIME CLUSTER_TABLE":master_uptime" +#define DBS_TOTAL CLUSTER_TABLE":dbs_total" +#define TBS_TOTAL CLUSTER_TABLE":tbs_total" +#define STBS_TOTAL CLUSTER_TABLE":stbs_total" +#define VGROUPS_TOTAL CLUSTER_TABLE":vgroups_total" +#define VGROUPS_ALIVE CLUSTER_TABLE":vgroups_alive" +#define VNODES_TOTAL CLUSTER_TABLE":vnodes_total" +#define VNODES_ALIVE CLUSTER_TABLE":vnodes_alive" +#define DNODES_TOTAL CLUSTER_TABLE":dnodes_total" +#define DNODES_ALIVE CLUSTER_TABLE":dnodes_alive" +#define CONNECTIONS_TOTAL CLUSTER_TABLE":connections_total" +#define TOPICS_TOTAL CLUSTER_TABLE":topics_total" +#define STREAMS_TOTAL CLUSTER_TABLE":streams_total" +#define EXPIRE_TIME CLUSTER_TABLE":expire_time" +#define TIMESERIES_USED CLUSTER_TABLE":timeseries_used" +#define TIMESERIES_TOTAL CLUSTER_TABLE":timeseries_total" + +#define VGROUP_TABLE "taosd_cluster_vgroups_info" + +#define TABLES_NUM VGROUP_TABLE":tables_num" +#define STATUS VGROUP_TABLE":status" + +#define DNODE_TABLE "taosd_dnodes_info" + +#define UPTIME DNODE_TABLE":uptime" +#define CPU_ENGINE DNODE_TABLE":cpu_engine" +#define CPU_SYSTEM DNODE_TABLE":cpu_system" +#define CPU_CORE DNODE_TABLE":cpu_cores" +#define MEM_ENGINE DNODE_TABLE":mem_engine" +#define MEM_SYSTEM DNODE_TABLE":mem_system" +#define MEM_TOTAL DNODE_TABLE":mem_total" +#define DISK_ENGINE DNODE_TABLE":disk_engine" +#define DISK_USED DNODE_TABLE":disk_used" +#define DISK_TOTAL DNODE_TABLE":disk_total" +#define NET_IN DNODE_TABLE":net_in" +#define NET_OUT DNODE_TABLE":net_out" +#define IO_READ DNODE_TABLE":io_read" +#define IO_WRITE DNODE_TABLE":io_write" +#define IO_READ_DISK DNODE_TABLE":io_read_disk" +#define IO_WRITE_DISK DNODE_TABLE":io_write_disk" +#define ERRORS DNODE_TABLE":errors" +#define VNODES_NUM DNODE_TABLE":vnodes_num" +#define MASTERS DNODE_TABLE":masters" +#define HAS_MNODE DNODE_TABLE":has_mnode" +#define HAS_QNODE DNODE_TABLE":has_qnode" +#define HAS_SNODE DNODE_TABLE":has_snode" +#define DNODE_LOG_ERROR DNODE_TABLE":ERROR" +#define DNODE_LOG_INFO DNODE_TABLE":INFO" +#define DNODE_LOG_DEBUG DNODE_TABLE":DEBUG" +#define DNODE_LOG_TRACE DNODE_TABLE":TRACE" + +#define DNODE_STATUS "taosd_dnodes_status:status" + +#define DATADIR_TABLE "taosd_dnodes_data_dirs" + +#define DNODE_DATA_AVAIL DATADIR_TABLE":avail" +#define DNODE_DATA_USED DATADIR_TABLE":used" +#define DNODE_DATA_TOTAL DATADIR_TABLE":total" + +#define LOGDIR_TABLE "taosd_dnodes_log_dirs" + +#define DNODE_LOG_AVAIL LOGDIR_TABLE":avail" +#define DNODE_LOG_USED LOGDIR_TABLE":used" +#define DNODE_LOG_TOTAL LOGDIR_TABLE":total" + +#define MNODE_ROLE "taosd_mnodes_info:role" +#define VNODE_ROLE "taosd_vnodes_info:role" + +void monInitMonitorFW(){ + taos_collector_registry_default_init(); + + tsMonitor.metrics = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + taos_gauge_t *gauge = NULL; + + int32_t label_count =1; + const char *sample_labels[] = {"cluster_id"}; + char *metric[] = {MASTER_UPTIME, DBS_TOTAL, TBS_TOTAL, STBS_TOTAL, VGROUPS_TOTAL, + VGROUPS_ALIVE, VNODES_TOTAL, VNODES_ALIVE, CONNECTIONS_TOTAL, TOPICS_TOTAL, STREAMS_TOTAL, + DNODES_TOTAL, DNODES_ALIVE, EXPIRE_TIME, TIMESERIES_USED, + TIMESERIES_TOTAL}; + for(int32_t i = 0; i < 16; i++){ + gauge= taos_gauge_new(metric[i], "", label_count, sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, metric[i], strlen(metric[i]), &gauge, sizeof(taos_gauge_t *)); + } + + int32_t vgroup_label_count = 3; + const char *vgroup_sample_labels[] = {"cluster_id", "vgroup_id", "database_name"}; + char *vgroup_metrics[] = {TABLES_NUM, STATUS}; + for(int32_t i = 0; i < 2; i++){ + gauge= taos_gauge_new(vgroup_metrics[i], "", vgroup_label_count, vgroup_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, vgroup_metrics[i], strlen(vgroup_metrics[i]), &gauge, sizeof(taos_gauge_t *)); + } + + int32_t dnodes_label_count = 3; + const char *dnodes_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep"}; + char *dnodes_gauges[] = {UPTIME, CPU_ENGINE, CPU_SYSTEM, CPU_CORE, MEM_ENGINE, MEM_SYSTEM, + MEM_TOTAL, DISK_ENGINE, DISK_USED, DISK_TOTAL, NET_IN, + NET_OUT, IO_READ, IO_WRITE, IO_READ_DISK, IO_WRITE_DISK, ERRORS, + VNODES_NUM, MASTERS, HAS_MNODE, HAS_QNODE, HAS_SNODE, DNODE_STATUS, + DNODE_LOG_ERROR, DNODE_LOG_INFO, DNODE_LOG_DEBUG, DNODE_LOG_TRACE}; + for(int32_t i = 0; i < 27; i++){ + gauge= taos_gauge_new(dnodes_gauges[i], "", dnodes_label_count, dnodes_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, dnodes_gauges[i], strlen(dnodes_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } + + int32_t dnodes_data_label_count = 5; + const char *dnodes_data_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep", "data_dir_name", "data_dir_level"}; + char *dnodes_data_gauges[] = {DNODE_DATA_AVAIL, DNODE_DATA_USED, DNODE_DATA_TOTAL}; + for(int32_t i = 0; i < 3; i++){ + gauge= taos_gauge_new(dnodes_data_gauges[i], "", dnodes_data_label_count, dnodes_data_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, dnodes_data_gauges[i], strlen(dnodes_data_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } + + int32_t dnodes_log_label_count = 4; + const char *dnodes_log_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep", "data_dir_name"}; + char *dnodes_log_gauges[] = {DNODE_LOG_AVAIL, DNODE_LOG_USED, DNODE_LOG_TOTAL}; + for(int32_t i = 0; i < 3; i++){ + gauge= taos_gauge_new(dnodes_log_gauges[i], "", dnodes_log_label_count, dnodes_log_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, dnodes_log_gauges[i], strlen(dnodes_log_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } + + int32_t mnodes_role_label_count = 3; + const char *mnodes_role_sample_labels[] = {"cluster_id", "mnode_id", "mnode_ep"}; + char *mnodes_role_gauges[] = {MNODE_ROLE}; + for(int32_t i = 0; i < 1; i++){ + gauge= taos_gauge_new(mnodes_role_gauges[i], "", mnodes_role_label_count, mnodes_role_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, mnodes_role_gauges[i], strlen(mnodes_role_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } + + int32_t vnodes_role_label_count = 4; + const char *vnodes_role_sample_labels[] = {"cluster_id", "vgroup_id", "database_name", "dnode_id"}; + char *vnodes_role_gauges[] = {VNODE_ROLE}; + for(int32_t i = 0; i < 1; i++){ + gauge= taos_gauge_new(vnodes_role_gauges[i], "", vnodes_role_label_count, vnodes_role_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, vnodes_role_gauges[i], strlen(vnodes_role_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } +} + +void monCleanupMonitorFW(){ + taosHashCleanup(tsMonitor.metrics); + taos_collector_registry_destroy(TAOS_COLLECTOR_REGISTRY_DEFAULT); + TAOS_COLLECTOR_REGISTRY_DEFAULT = NULL; +} + +void monGenClusterInfoTable(SMonInfo *pMonitor){ + SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; + SMonBasicInfo *pBasicInfo = &pMonitor->dmInfo.basic; + SMonGrantInfo *pGrantInfo = &pMonitor->mmInfo.grant; + + if(pBasicInfo->cluster_id == 0) { + uError("failed to generate dnode info table since cluster_id is 0"); + return; + } + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + //cluster info + char buf[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(buf, TSDB_CLUSTER_ID_LEN, "%"PRId64, pBasicInfo->cluster_id); + const char *sample_labels[] = {buf}; + + taos_gauge_t **metric = NULL; + + metric = taosHashGet(tsMonitor.metrics, MASTER_UPTIME, strlen(MASTER_UPTIME)); + taos_gauge_set(*metric, pInfo->master_uptime, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DBS_TOTAL, strlen(DBS_TOTAL)); + taos_gauge_set(*metric, pInfo->dbs_total, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, TBS_TOTAL, strlen(TBS_TOTAL)); + taos_gauge_set(*metric, pInfo->tbs_total, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, STBS_TOTAL, strlen(STBS_TOTAL)); + taos_gauge_set(*metric, pInfo->stbs_total, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, VGROUPS_TOTAL, strlen(VGROUPS_TOTAL)); + taos_gauge_set(*metric, pInfo->vgroups_total, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, VGROUPS_ALIVE, strlen(VGROUPS_ALIVE)); + taos_gauge_set(*metric, pInfo->vgroups_alive, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, VNODES_TOTAL, strlen(VNODES_TOTAL)); + taos_gauge_set(*metric, pInfo->vnodes_total, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, VNODES_ALIVE, strlen(VNODES_ALIVE)); + taos_gauge_set(*metric, pInfo->vnodes_alive, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, CONNECTIONS_TOTAL, strlen(CONNECTIONS_TOTAL)); + taos_gauge_set(*metric, pInfo->connections_total, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, TOPICS_TOTAL, strlen(TOPICS_TOTAL)); + taos_gauge_set(*metric, pInfo->topics_toal, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, STREAMS_TOTAL, strlen(STREAMS_TOTAL)); + taos_gauge_set(*metric, pInfo->streams_total, sample_labels); + + //dnodes number + int32_t dnode_total = taosArrayGetSize(pInfo->dnodes); + int32_t dnode_alive = 0; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) { + SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i); + + if(strcmp(pDnodeDesc->status, "ready") == 0){ + dnode_alive++; + } + } + + metric = taosHashGet(tsMonitor.metrics, DNODES_TOTAL, strlen(DNODES_TOTAL)); + taos_gauge_set(*metric, dnode_total, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODES_ALIVE, strlen(DNODES_ALIVE)); + taos_gauge_set(*metric, dnode_alive, sample_labels); + + //grant info + metric = taosHashGet(tsMonitor.metrics, EXPIRE_TIME, strlen(EXPIRE_TIME)); + taos_gauge_set(*metric, pGrantInfo->expire_time, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, TIMESERIES_USED, strlen(TIMESERIES_USED)); + taos_gauge_set(*metric, pGrantInfo->timeseries_used, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, TIMESERIES_TOTAL, strlen(TIMESERIES_TOTAL)); + taos_gauge_set(*metric, pGrantInfo->timeseries_total, sample_labels); +} + +void monGenVgroupInfoTable(SMonInfo *pMonitor){ + if(pMonitor->dmInfo.basic.cluster_id == 0) return; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + char cluster_id[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%"PRId64, pMonitor->dmInfo.basic.cluster_id); + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->vgroups); ++i) { + SMonVgroupDesc *pVgroupDesc = taosArrayGet(pInfo->vgroups, i); + + char vgroup_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(vgroup_id, TSDB_NODE_ID_LEN, "%"PRId32, pVgroupDesc->vgroup_id); + + const char *sample_labels[] = {cluster_id, vgroup_id, pVgroupDesc->database_name}; + + taos_gauge_t **metric = NULL; + + metric = taosHashGet(tsMonitor.metrics, TABLES_NUM, strlen(TABLES_NUM)); + taos_gauge_set(*metric, pVgroupDesc->tables_num, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, STATUS, strlen(STATUS)); + int32_t status = 0; + if(strcmp(pVgroupDesc->status, "ready") == 0){ + status = 1; + } + taos_gauge_set(*metric, status, sample_labels); + } +} + +void monGenDnodeInfoTable(SMonInfo *pMonitor) { + if(pMonitor->dmInfo.basic.cluster_id == 0) { + uError("failed to generate dnode info table since cluster_id is 0"); + return; + } + + char cluster_id[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%"PRId64, pMonitor->dmInfo.basic.cluster_id); + + char dnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMonitor->dmInfo.basic.dnode_id); + + const char *sample_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep}; + + taos_gauge_t **metric = NULL; + + //dnode info + SMonDnodeInfo *pInfo = &pMonitor->dmInfo.dnode; + SMonSysInfo *pSys = &pMonitor->dmInfo.sys; + SVnodesStat *pStat = &pMonitor->vmInfo.vstat; + SMonClusterInfo *pClusterInfo = &pMonitor->mmInfo.cluster; + + double interval = (pMonitor->curTime - pMonitor->lastTime) / 1000.0; + if (pMonitor->curTime - pMonitor->lastTime == 0) { + interval = 1; + } + + double cpu_engine = 0; + double mem_engine = 0; + double net_in = 0; + double net_out = 0; + double io_read = 0; + double io_write = 0; + double io_read_disk = 0; + double io_write_disk = 0; + + SMonSysInfo *sysArrays[6]; + sysArrays[0] = &pMonitor->dmInfo.sys; + sysArrays[1] = &pMonitor->mmInfo.sys; + sysArrays[2] = &pMonitor->vmInfo.sys; + sysArrays[3] = &pMonitor->qmInfo.sys; + sysArrays[4] = &pMonitor->smInfo.sys; + sysArrays[5] = &pMonitor->bmInfo.sys; + for (int32_t i = 0; i < 6; ++i) { + cpu_engine += sysArrays[i]->cpu_engine; + mem_engine += sysArrays[i]->mem_engine; + net_in += sysArrays[i]->net_in; + net_out += sysArrays[i]->net_out; + io_read += sysArrays[i]->io_read; + io_write += sysArrays[i]->io_write; + io_read_disk += sysArrays[i]->io_read_disk; + io_write_disk += sysArrays[i]->io_write_disk; + } + + double req_select_rate = pStat->numOfSelectReqs / interval; + double req_insert_rate = pStat->numOfInsertReqs / interval; + double req_insert_batch_rate = pStat->numOfBatchInsertReqs / interval; + double net_in_rate = net_in / interval; + double net_out_rate = net_out / interval; + double io_read_rate = io_read / interval; + double io_write_rate = io_write / interval; + double io_read_disk_rate = io_read_disk / interval; + double io_write_disk_rate = io_write_disk / interval; + + metric = taosHashGet(tsMonitor.metrics, UPTIME, strlen(UPTIME)); + taos_gauge_set(*metric, pInfo->uptime, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, CPU_ENGINE, strlen(CPU_ENGINE)); + taos_gauge_set(*metric, cpu_engine, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, CPU_SYSTEM, strlen(CPU_SYSTEM)); + taos_gauge_set(*metric, pSys->cpu_system, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, CPU_CORE, strlen(CPU_CORE)); + taos_gauge_set(*metric, pSys->cpu_cores, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, MEM_ENGINE, strlen(MEM_ENGINE)); + taos_gauge_set(*metric, mem_engine, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, MEM_SYSTEM, strlen(MEM_SYSTEM)); + taos_gauge_set(*metric, pSys->mem_system, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, MEM_TOTAL, strlen(MEM_TOTAL)); + taos_gauge_set(*metric, pSys->mem_total, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DISK_ENGINE, strlen(DISK_ENGINE)); + taos_gauge_set(*metric, pSys->disk_engine, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DISK_USED, strlen(DISK_USED)); + taos_gauge_set(*metric, pSys->disk_used, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DISK_TOTAL, strlen(DISK_TOTAL)); + taos_gauge_set(*metric, pSys->disk_total, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, NET_IN, strlen(NET_IN)); + taos_gauge_set(*metric, net_in_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, NET_OUT, strlen(NET_OUT)); + taos_gauge_set(*metric, net_out_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, IO_READ, strlen(IO_READ)); + taos_gauge_set(*metric, io_read_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, IO_WRITE, strlen(IO_WRITE)); + taos_gauge_set(*metric, io_write_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, IO_READ_DISK, strlen(IO_READ_DISK)); + taos_gauge_set(*metric, io_read_disk_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, IO_WRITE_DISK, strlen(IO_WRITE_DISK)); + taos_gauge_set(*metric, io_write_disk_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, ERRORS, strlen(ERRORS)); + taos_gauge_set(*metric, io_read_disk_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, VNODES_NUM, strlen(VNODES_NUM)); + taos_gauge_set(*metric, pStat->errors, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, MASTERS, strlen(MASTERS)); + taos_gauge_set(*metric, pStat->masterNum, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, HAS_MNODE, strlen(HAS_MNODE)); + taos_gauge_set(*metric, pInfo->has_mnode, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, HAS_QNODE, strlen(HAS_QNODE)); + taos_gauge_set(*metric, pInfo->has_qnode, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, HAS_SNODE, strlen(HAS_SNODE)); + taos_gauge_set(*metric, pInfo->has_snode, sample_labels); + + //log number + SMonLogs *logs[6]; + logs[0] = &pMonitor->log; + logs[1] = &pMonitor->mmInfo.log; + logs[2] = &pMonitor->vmInfo.log; + logs[3] = &pMonitor->smInfo.log; + logs[4] = &pMonitor->qmInfo.log; + logs[5] = &pMonitor->bmInfo.log; + + int32_t numOfErrorLogs = 0; + int32_t numOfInfoLogs = 0; + int32_t numOfDebugLogs = 0; + int32_t numOfTraceLogs = 0; + + for (int32_t j = 0; j < 6; j++) { + SMonLogs *pLog = logs[j]; + numOfErrorLogs += pLog->numOfErrorLogs; + numOfInfoLogs += pLog->numOfInfoLogs; + numOfDebugLogs += pLog->numOfDebugLogs; + numOfTraceLogs += pLog->numOfTraceLogs; + } + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_ERROR, strlen(DNODE_LOG_ERROR)); + taos_gauge_set(*metric, numOfErrorLogs, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_INFO, strlen(DNODE_LOG_INFO)); + taos_gauge_set(*metric, numOfInfoLogs, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_DEBUG, strlen(DNODE_LOG_DEBUG)); + taos_gauge_set(*metric, numOfDebugLogs, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_TRACE, strlen(DNODE_LOG_TRACE)); + taos_gauge_set(*metric, numOfTraceLogs, sample_labels); +} + +void monGenDnodeStatusInfoTable(SMonInfo *pMonitor){ + if(pMonitor->dmInfo.basic.cluster_id == 0) { + uError("failed to generate dnode info table since cluster_id is 0"); + return; + } + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + char cluster_id[TSDB_CLUSTER_ID_LEN]; + snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%"PRId64, pMonitor->dmInfo.basic.cluster_id); + + taos_gauge_t **metric = NULL; + //dnodes status + + SMonClusterInfo *pClusterInfo = &pMonitor->mmInfo.cluster; + + for (int32_t i = 0; i < taosArrayGetSize(pClusterInfo->dnodes); ++i) { + SMonDnodeDesc *pDnodeDesc = taosArrayGet(pClusterInfo->dnodes, i); + + char dnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pDnodeDesc->dnode_id); + + const char *sample_labels[] = {cluster_id, dnode_id, pDnodeDesc->dnode_ep}; + + metric = taosHashGet(tsMonitor.metrics, DNODE_STATUS, strlen(DNODE_STATUS)); + + int32_t status = 0; + if(strcmp(pDnodeDesc->status, "ready") == 0){ + status = 1; + } + taos_gauge_set(*metric, status, sample_labels); + } +} + +void monGenDataDiskTable(SMonInfo *pMonitor){ + if(pMonitor->dmInfo.basic.cluster_id == 0) return; + + SMonDiskInfo *pInfo = &pMonitor->vmInfo.tfs; + + char cluster_id[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%" PRId64, pMonitor->dmInfo.basic.cluster_id); + + char dnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMonitor->dmInfo.basic.dnode_id); + + taos_gauge_t **metric = NULL; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->datadirs); ++i) { + SMonDiskDesc *pDatadirDesc = taosArrayGet(pInfo->datadirs, i); + + char level[LEVEL_LEN] = {0}; + snprintf(level, LEVEL_LEN, "%"PRId32, pDatadirDesc->level); + + const char *sample_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep, pDatadirDesc->name, level}; + + metric = taosHashGet(tsMonitor.metrics, DNODE_DATA_AVAIL, strlen(DNODE_DATA_AVAIL)); + taos_gauge_set(*metric, pDatadirDesc->size.avail, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_DATA_USED, strlen(DNODE_DATA_USED)); + taos_gauge_set(*metric, pDatadirDesc->size.used, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_DATA_TOTAL, strlen(DNODE_DATA_TOTAL)); + taos_gauge_set(*metric, pDatadirDesc->size.total, sample_labels); + } +} + +void monGenLogDiskTable(SMonInfo *pMonitor){ + if(pMonitor->dmInfo.basic.cluster_id == 0) return; + + SMonDiskDesc *pLogDesc = &pMonitor->dmInfo.dnode.logdir; + SMonDiskDesc *pTempDesc = &pMonitor->dmInfo.dnode.tempdir; + + char cluster_id[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%" PRId64, pMonitor->dmInfo.basic.cluster_id); + + char dnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMonitor->dmInfo.basic.dnode_id); + + taos_gauge_t **metric = NULL; + + const char *sample_log_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep, pLogDesc->name}; + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_AVAIL, strlen(DNODE_LOG_AVAIL)); + taos_gauge_set(*metric, pLogDesc->size.avail, sample_log_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_USED, strlen(DNODE_LOG_USED)); + taos_gauge_set(*metric, pLogDesc->size.used, sample_log_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_TOTAL, strlen(DNODE_LOG_TOTAL)); + taos_gauge_set(*metric, pLogDesc->size.total, sample_log_labels); + + const char *sample_temp_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep, pTempDesc->name}; + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_AVAIL, strlen(DNODE_LOG_AVAIL)); + taos_gauge_set(*metric, pTempDesc->size.avail, sample_temp_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_USED, strlen(DNODE_LOG_USED)); + taos_gauge_set(*metric, pTempDesc->size.used, sample_temp_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_TOTAL, strlen(DNODE_LOG_TOTAL)); + taos_gauge_set(*metric, pTempDesc->size.total, sample_temp_labels); +} + +void monGenMnodeRoleTable(SMonInfo *pMonitor){ + SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + SMonBasicInfo *pBasicInfo = &pMonitor->dmInfo.basic; + if(pBasicInfo->cluster_id == 0) return; + + char buf[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(buf, TSDB_CLUSTER_ID_LEN, "%" PRId64, pBasicInfo->cluster_id); + + taos_gauge_t **metric = NULL; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->mnodes); ++i) { + + SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->mnodes, i); + + char mnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(mnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMnodeDesc->mnode_id); + + const char *sample_labels[] = {buf, mnode_id, pMnodeDesc->mnode_ep}; + + metric = taosHashGet(tsMonitor.metrics, MNODE_ROLE, strlen(MNODE_ROLE)); + taos_gauge_set(*metric, pMnodeDesc->syncState, sample_labels); + } +} + +void monGenVnodeRoleTable(SMonInfo *pMonitor){ + SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + SMonBasicInfo *pBasicInfo = &pMonitor->dmInfo.basic; + if(pBasicInfo->cluster_id == 0) return; + + char buf[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(buf, TSDB_CLUSTER_ID_LEN, "%" PRId64, pBasicInfo->cluster_id); + + taos_gauge_t **metric = NULL; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->vgroups); ++i) { + SMonVgroupDesc *pVgroupDesc = taosArrayGet(pInfo->vgroups, i); + + char vgroup_id[TSDB_VGROUP_ID_LEN] = {0}; + snprintf(vgroup_id, TSDB_VGROUP_ID_LEN, "%"PRId32, pVgroupDesc->vgroup_id); + + for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { + SMonVnodeDesc *pVnodeDesc = &pVgroupDesc->vnodes[j]; + if (pVnodeDesc->dnode_id <= 0) continue; + + char dnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pVnodeDesc->dnode_id); + + const char *sample_labels[] = {buf, vgroup_id, pVgroupDesc->database_name, dnode_id}; + + metric = taosHashGet(tsMonitor.metrics, VNODE_ROLE, strlen(VNODE_ROLE)); + taos_gauge_set(*metric, pVnodeDesc->syncState, sample_labels); + } + } +} + +void monSendPromReport() { + char ts[50] = {0}; + sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + + char* promStr = NULL; + char* pCont = (char *)taos_collector_registry_bridge_new(TAOS_COLLECTOR_REGISTRY_DEFAULT, ts, "%" PRId64, &promStr); + if(tsMonitorLogProtocol){ + uInfoL("report cont:\n%s\n", pCont); + uDebugL("report cont prom:\n%s\n", promStr); + } + if (pCont != NULL) { + EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + uError("failed to send monitor msg"); + }else{ + taos_collector_registry_clear_batch(TAOS_COLLECTOR_REGISTRY_DEFAULT); + } + taosMemoryFreeClear(pCont); + } + if(promStr != NULL){ + taosMemoryFreeClear(promStr); + } +} \ No newline at end of file diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index d417a97db7..83cde72a91 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -21,9 +21,9 @@ #include "taos_monitor.h" #include "tglobal.h" -static SMonitor tsMonitor = {0}; -static char* tsMonUri = "/report"; -static char* tsMonFwUri = "/td_metric"; +SMonitor tsMonitor = {0}; +char* tsMonUri = "/report"; +char* tsMonFwUri = "/td_metric"; void monRecordLog(int64_t ts, ELogLevel level, const char *content) { taosThreadMutexLock(&tsMonitor.lock); @@ -112,7 +112,7 @@ int32_t monInit(const SMonCfg *pCfg) { tsMonitor.lastTime = taosGetTimestampMs(); taosThreadMutexInit(&tsMonitor.lock, NULL); - taos_collector_registry_default_init(); + monInitMonitorFW(); return 0; } @@ -128,8 +128,7 @@ void monCleanup() { tFreeSMonBmInfo(&tsMonitor.bmInfo); taosThreadMutexDestroy(&tsMonitor.lock); - taos_collector_registry_destroy(TAOS_COLLECTOR_REGISTRY_DEFAULT); - TAOS_COLLECTOR_REGISTRY_DEFAULT = NULL; + monCleanupMonitorFW(); } static void monCleanupMonitorInfo(SMonInfo *pMonitor) { @@ -195,6 +194,22 @@ static void monGenBasicJson(SMonInfo *pMonitor) { tjsonAddDoubleToObject(pJson, "protocol", pInfo->protocol); } +static void monGenBasicJsonBasic(SMonInfo *pMonitor) { + SMonBasicInfo *pInfo = &pMonitor->dmInfo.basic; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + SJson *pJson = pMonitor->pJson; + char buf[40] = {0}; + + sprintf(buf, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + tjsonAddStringToObject(pJson, "ts", buf); + tjsonAddDoubleToObject(pJson, "dnode_id", pInfo->dnode_id); + tjsonAddStringToObject(pJson, "dnode_ep", pInfo->dnode_ep); + snprintf(buf, sizeof(buf), "%" PRId64, pInfo->cluster_id); + tjsonAddStringToObject(pJson, "cluster_id", buf); + tjsonAddDoubleToObject(pJson, "protocol", pInfo->protocol); +} + static void monGenClusterJson(SMonInfo *pMonitor) { SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; @@ -253,6 +268,16 @@ static void monGenClusterJson(SMonInfo *pMonitor) { } } +static void monGenClusterJsonBasic(SMonInfo *pMonitor) { + SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + tjsonAddStringToObject(pMonitor->pJson, "first_ep", pInfo->first_ep); + tjsonAddDoubleToObject(pMonitor->pJson, "first_ep_dnode_id", pInfo->first_ep_dnode_id); + tjsonAddStringToObject(pMonitor->pJson, "cluster_version", pInfo->version); + tjsonAddDoubleToObject(pMonitor->pJson, "monitor_interval", pInfo->monitor_interval); +} + static void monGenVgroupJson(SMonInfo *pMonitor) { SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; @@ -526,21 +551,11 @@ static void monGenLogJson(SMonInfo *pMonitor) { if (tjsonAddItemToArray(pSummaryJson, pLogTrace) != 0) tjsonDelete(pLogTrace); } -void monSendReport() { - SMonInfo *pMonitor = monCreateMonitorInfo(); - if (pMonitor == NULL) return; - - monGenBasicJson(pMonitor); - monGenClusterJson(pMonitor); - monGenVgroupJson(pMonitor); - monGenStbJson(pMonitor); - monGenGrantJson(pMonitor); - monGenDnodeJson(pMonitor); - monGenDiskJson(pMonitor); - monGenLogJson(pMonitor); - +void monSendReport(SMonInfo *pMonitor){ char *pCont = tjsonToString(pMonitor->pJson); - // uDebugL("report cont:%s\n", pCont); + if(tsMonitorLogProtocol){ + uInfoL("report cont basic:\n%s", pCont); + } if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; if (taosSendHttpReport(tsMonitor.cfg.server, tsMonUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { @@ -548,32 +563,49 @@ void monSendReport() { } taosMemoryFree(pCont); } +} + +void monGenAndSendReport() { + SMonInfo *pMonitor = monCreateMonitorInfo(); + if (pMonitor == NULL) return; + + if(!tsMonitorForceV2){ + monGenBasicJson(pMonitor); + monGenClusterJson(pMonitor); + monGenVgroupJson(pMonitor); + monGenStbJson(pMonitor); + monGenGrantJson(pMonitor); + monGenDnodeJson(pMonitor); + monGenDiskJson(pMonitor); + monGenLogJson(pMonitor); + + monSendReport(pMonitor); + } + else{ + monGenClusterInfoTable(pMonitor); + monGenVgroupInfoTable(pMonitor); + monGenDnodeInfoTable(pMonitor); + monGenDnodeStatusInfoTable(pMonitor); + monGenDataDiskTable(pMonitor); + monGenLogDiskTable(pMonitor); + monGenMnodeRoleTable(pMonitor); + monGenVnodeRoleTable(pMonitor); + + monSendPromReport(); + } monCleanupMonitorInfo(pMonitor); } -void monSendPromReport() { - char ts[50]; - sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); - char *pCont = (char *)taos_collector_registry_bridge(TAOS_COLLECTOR_REGISTRY_DEFAULT, ts, "%" PRId64); - //uInfoL("report cont:\n%s\n", pCont); - if (pCont != NULL) { - EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; - if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { - uError("failed to send monitor msg"); - }else{ - taos_collector_registry_clear_out(TAOS_COLLECTOR_REGISTRY_DEFAULT); - } - } -} +void monGenAndSendReportBasic() { + SMonInfo *pMonitor = monCreateMonitorInfo(); + if (pMonitor == NULL) return; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; -void monSendContent(char *pCont) { - if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; - //uInfoL("report cont:\n%s\n", pCont); - if (pCont != NULL) { - EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; - if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { - uError("failed to send monitor msg"); - } - } + monGenBasicJsonBasic(pMonitor); + monGenClusterJsonBasic(pMonitor); + + monSendReport(pMonitor); + + monCleanupMonitorInfo(pMonitor); } \ No newline at end of file diff --git a/source/libs/monitor/test/monTest.cpp b/source/libs/monitor/test/monTest.cpp index 3f7b1b51da..2660cff216 100644 --- a/source/libs/monitor/test/monTest.cpp +++ b/source/libs/monitor/test/monTest.cpp @@ -283,10 +283,10 @@ TEST_F(MonitorTest, 01_Full) { tFreeSMonSmInfo(&smInfo); tFreeSMonQmInfo(&qmInfo); tFreeSMonBmInfo(&bmInfo); - monSendReport(); + monGenAndSendReport(); } TEST_F(MonitorTest, 02_Log) { AddLogInfo2(); - monSendReport(); + monGenAndSendReport(); } diff --git a/source/libs/monitorfw/inc/taos_collector_registry_t.h b/source/libs/monitorfw/inc/taos_collector_registry_t.h index 2264d18081..8e8a881fca 100644 --- a/source/libs/monitorfw/inc/taos_collector_registry_t.h +++ b/source/libs/monitorfw/inc/taos_collector_registry_t.h @@ -34,7 +34,7 @@ struct taos_collector_registry { taos_string_builder_t *string_builder; /**< Enables string building */ taos_metric_formatter_t *metric_formatter; /**< metric formatter for metric exposition on bridge call */ pthread_rwlock_t *lock; /**< mutex for safety against concurrent registration */ - taos_string_builder_t *out; + taos_string_builder_t *string_builder_batch; }; #endif // TAOS_REGISTRY_T_H diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_custom_i.h b/source/libs/monitorfw/inc/taos_metric_formatter_custom_i.h new file mode 100644 index 0000000000..f3e4ebae75 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_formatter_custom_i.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_FORMATTER_CUSTOMV2_I_H +#define TAOS_METRIC_FORMATTER_CUSTOMV2_I_H + +#define ALLOW_FORBID_FUNC + +#include +#include "tjson.h" + +int taos_metric_formatter_load_sample_new(taos_metric_formatter_t *self, taos_metric_sample_t *sample, + char *ts, char *format, char *metricName, int32_t metric_type, + SJson *arrayMetricGroups); +int taos_metric_formatter_load_metric_new(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format, + SJson* tableArray); +int taos_metric_formatter_load_metrics_new(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts, + char *format, SJson* tableArray); +#endif // TAOS_METRIC_FORMATTER_CUSTOMV2_I_H \ No newline at end of file diff --git a/source/libs/monitorfw/inc/taos_metric_sample_t.h b/source/libs/monitorfw/inc/taos_metric_sample_t.h index 92885acf4a..3aa9b7bb99 100644 --- a/source/libs/monitorfw/inc/taos_metric_sample_t.h +++ b/source/libs/monitorfw/inc/taos_metric_sample_t.h @@ -19,10 +19,16 @@ #include "taos_metric_sample.h" #include "taos_metric_t.h" +#define DOUBLE_ATOMIC + struct taos_metric_sample { taos_metric_type_t type; /**< type is the metric type for the sample */ char *l_value; /**< l_value is the full metric name and label set represeted as a string */ - /*_Atomic*/ int64_t r_value; /**< r_value is the value of the metric sample */ +#ifdef DOUBLE_ATOMIC + _Atomic double r_value; /**< r_value is the value of the metric sample */ +#else + int64_t r_value; /**< r_value is the value of the metric sample */ +#endif }; #endif // TAOS_METRIC_SAMPLE_T_H diff --git a/source/libs/monitorfw/inc/taos_metric_t.h b/source/libs/monitorfw/inc/taos_metric_t.h index 806466528d..da237aa814 100644 --- a/source/libs/monitorfw/inc/taos_metric_t.h +++ b/source/libs/monitorfw/inc/taos_metric_t.h @@ -42,7 +42,7 @@ extern char *taos_metric_type_map[4]; */ struct taos_metric { taos_metric_type_t type; /**< metric_type The type of metric */ - const char *name; /**< name The name of the metric */ + char *name; /**< name The name of the metric */ const char *help; /**< help The help output for the metric */ taos_map_t *samples; /**< samples Map comprised of samples for the given metric */ size_t label_key_count; /**< label_keys_count The count of labe_keys*/ diff --git a/source/libs/monitorfw/inc/taos_monitor_util_i.h b/source/libs/monitorfw/inc/taos_monitor_util_i.h new file mode 100644 index 0000000000..fe072204a3 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_monitor_util_i.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_MONITOR_UTIL_I_H +#define TAOS_MONITOR_UTIL_I_H + +#include + +void taos_monitor_split_str(char** arr, char* str, const char* del); +int taos_monitor_count_occurrences(char *str, char *toSearch); +void taos_monitor_strip(char *s); +bool taos_monitor_is_match(const SJson* tags, char** pairs, int32_t count); + +#endif // TAOS_MONITOR_UTIL_I_H \ No newline at end of file diff --git a/source/libs/monitorfw/src/taos_collector.c b/source/libs/monitorfw/src/taos_collector.c index d3228facb6..997bf5587c 100644 --- a/source/libs/monitorfw/src/taos_collector.c +++ b/source/libs/monitorfw/src/taos_collector.c @@ -107,3 +107,9 @@ int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric) { } return taos_map_set(self->metrics, metric->name, metric); } + +taos_metric_t* taos_collector_get_metric(taos_collector_t *self, char *metric_name){ + TAOS_ASSERT(self != NULL); + if (self == NULL) return NULL; + return taos_map_get(self->metrics, metric_name); +} \ No newline at end of file diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c index 711e66ed84..c94675c95c 100644 --- a/source/libs/monitorfw/src/taos_collector_registry.c +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -33,6 +33,10 @@ #include "taos_metric_i.h" #include "taos_metric_t.h" #include "taos_string_builder_i.h" +#include "taos_metric_formatter_custom_i.h" + +#define ALLOW_FORBID_FUNC +#include "tjson.h" taos_collector_registry_t *TAOS_COLLECTOR_REGISTRY_DEFAULT; @@ -50,7 +54,7 @@ taos_collector_registry_t *taos_collector_registry_new(const char *name) { self->metric_formatter = taos_metric_formatter_new(); self->string_builder = taos_string_builder_new(); - self->out = taos_string_builder_new(); + self->string_builder_batch = taos_string_builder_new(); self->lock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t)); r = pthread_rwlock_init(self->lock, NULL); if (r) { @@ -88,8 +92,8 @@ int taos_collector_registry_destroy(taos_collector_registry_t *self) { self->string_builder = NULL; if (r) ret = r; - r = taos_string_builder_destroy(self->out); - self->out = NULL; + r = taos_string_builder_destroy(self->string_builder_batch); + self->string_builder_batch = NULL; if (r) ret = r; r = pthread_rwlock_destroy(self->lock); @@ -119,6 +123,19 @@ int taos_collector_registry_register_metric(taos_metric_t *metric) { return taos_collector_add_metric(default_collector, metric); } +taos_metric_t *taos_collector_registry_get_metric(char* metric_name){ + TAOS_ASSERT(metric != NULL); + + taos_collector_t *default_collector = + (taos_collector_t *)taos_map_get(TAOS_COLLECTOR_REGISTRY_DEFAULT->collectors, "default"); + + if (default_collector == NULL) { + return NULL; + } + + return taos_collector_get_metric(default_collector, metric_name); +} + taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metric) { int err = taos_collector_registry_register_metric(metric); if (err != 0) { @@ -193,13 +210,71 @@ const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char char *out = taos_metric_formatter_dump(self->metric_formatter); int r = 0; - r = taos_string_builder_add_str(self->out, out); + r = taos_string_builder_add_str(self->string_builder_batch, out); if (r) return NULL; taos_free(out); - return taos_string_builder_str(self->out); + return taos_string_builder_str(self->string_builder_batch); } -int taos_collector_registry_clear_out(taos_collector_registry_t *self){ - return taos_string_builder_clear(self->out); +int taos_collector_registry_clear_batch(taos_collector_registry_t *self){ + return taos_string_builder_clear(self->string_builder_batch); +} + +const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self, char *ts, char *format, char** prom_str) { + taos_metric_formatter_clear(self->metric_formatter); + + SJson* pJson = tjsonCreateArray(); + SJson* item = tjsonCreateObject(); + tjsonAddItemToArray(pJson, item); + tjsonAddStringToObject(item, "ts", ts); + tjsonAddDoubleToObject(item, "protocol", 2); + SJson* array = tjsonCreateArray(); + tjsonAddItemToObject(item, "tables", array); + + taos_metric_formatter_load_metrics_new(self->metric_formatter, self->collectors, ts, format, array); + + //caller free this + //generate prom protocol for debug + if(prom_str != NULL){ + *prom_str = taos_metric_formatter_dump(self->metric_formatter); + } + + //add this result to batch cache, format in batch cache is {},{} + int r = 0; + char* old_str = taos_string_builder_str(self->string_builder_batch); + if(old_str[0] != '\0'){ + r = taos_string_builder_add_str(self->string_builder_batch, ","); + if (r) return NULL; + } + char * item_str = tjsonToString(item); + r = taos_string_builder_add_str(self->string_builder_batch, item_str); + taos_free(item_str); + if (r) return NULL; + + tjsonDelete(pJson); + + //generate final array format result, ie, add [] to str in batch cache + taos_string_builder_t* tmp_builder = taos_string_builder_new(); + + r = taos_string_builder_add_str(tmp_builder, "["); + if (r) return NULL; + + r = taos_string_builder_add_str(tmp_builder, taos_string_builder_str(self->string_builder_batch)); + if (r) return NULL; + + r = taos_string_builder_add_str(tmp_builder, "]"); + if (r) return NULL; + + //caller free this + char *data = taos_string_builder_dump(tmp_builder); + if (data == NULL) return NULL; + r = taos_string_builder_clear(tmp_builder); + if (r) return NULL; + + r = taos_string_builder_destroy(tmp_builder); + tmp_builder = NULL; + if (r) return NULL; + + return data; } diff --git a/source/libs/monitorfw/src/taos_gauge.c b/source/libs/monitorfw/src/taos_gauge.c new file mode 100644 index 0000000000..74d2665194 --- /dev/null +++ b/source/libs/monitorfw/src/taos_gauge.c @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Public +#include "taos_gauge.h" + +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_metric_i.h" +#include "taos_metric_sample_i.h" +#include "taos_metric_sample_t.h" +#include "taos_metric_t.h" + +taos_gauge_t *taos_gauge_new(const char *name, const char *help, size_t label_key_count, const char **label_keys) { + return (taos_gauge_t *)taos_metric_new(TAOS_GAUGE, name, help, label_key_count, label_keys); +} + +int taos_gauge_destroy(taos_gauge_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + r = taos_metric_destroy(self); + self = NULL; + return r; +} + +int taos_gauge_inc(taos_gauge_t *self, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_add(sample, 1.0); +} + +int taos_gauge_dec(taos_gauge_t *self, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_sub(sample, 1.0); +} + +int taos_gauge_add(taos_gauge_t *self, double r_value, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_add(sample, r_value); +} + +int taos_gauge_sub(taos_gauge_t *self, double r_value, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_sub(sample, r_value); +} + +int taos_gauge_set(taos_gauge_t *self, double r_value, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_set(sample, r_value); +} diff --git a/source/libs/monitorfw/src/taos_metric.c b/source/libs/monitorfw/src/taos_metric.c index a9bb6d45a4..4e9af35f34 100644 --- a/source/libs/monitorfw/src/taos_metric.c +++ b/source/libs/monitorfw/src/taos_metric.c @@ -34,7 +34,11 @@ taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name, int r = 0; taos_metric_t *self = (taos_metric_t *)taos_malloc(sizeof(taos_metric_t)); self->type = metric_type; - self->name = name; + int len = strlen(name) + 1; + self->name = taos_malloc(len); + memset(self->name, 0, len); + strcpy(self->name, name); + //self->name = name; self->help = help; const char **k = (const char **)taos_malloc(sizeof(const char *) * label_key_count); @@ -111,6 +115,9 @@ int taos_metric_destroy(taos_metric_t *self) { taos_free(self->label_keys); self->label_keys = NULL; + taos_free(self->name); + self->name = NULL; + taos_free(self); self = NULL; diff --git a/source/libs/monitorfw/src/taos_metric_formatter.c b/source/libs/monitorfw/src/taos_metric_formatter.c index 6c6f1974eb..a9f35c3e8d 100644 --- a/source/libs/monitorfw/src/taos_metric_formatter.c +++ b/source/libs/monitorfw/src/taos_metric_formatter.c @@ -181,7 +181,7 @@ int taos_metric_formatter_load_sample(taos_metric_formatter_t *self, taos_metric r = taos_string_builder_add_str(self->string_builder, ts); if (r) return r; - taos_metric_sample_set(sample, 0); + //taos_metric_sample_set(sample, 0); return taos_string_builder_add_char(self->string_builder, '\n'); } diff --git a/source/libs/monitorfw/src/taos_metric_formatter_custom.c b/source/libs/monitorfw/src/taos_metric_formatter_custom.c new file mode 100644 index 0000000000..d7650098e0 --- /dev/null +++ b/source/libs/monitorfw/src/taos_metric_formatter_custom.c @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC + +#include +#include "taos_metric_formatter_i.h" +#include "taos_metric_sample_t.h" +#include "tjson.h" +#include "taos_monitor_util_i.h" +#include "taos_assert.h" +#include "tdef.h" +#include "taos_collector_t.h" + +int taos_metric_formatter_load_sample_new(taos_metric_formatter_t *self, taos_metric_sample_t *sample, + char *ts, char *format, char *metricName, int32_t metric_type, + SJson *arrayMetricGroups) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + char* start = strstr(sample->l_value, "{"); + char* end = strstr(sample->l_value, "}"); + + int32_t len = end -start; + + char* keyvalues = taosMemoryMalloc(len); + memset(keyvalues, 0, len); + memcpy(keyvalues, start + 1, len - 1); + + int32_t count = taos_monitor_count_occurrences(keyvalues, ","); + + char** keyvalue = taosMemoryMalloc(sizeof(char*) * (count + 1)); + memset(keyvalue, 0, sizeof(char*) * (count + 1)); + taos_monitor_split_str(keyvalue, keyvalues, ","); + + char** arr = taosMemoryMalloc(sizeof(char*) * (count + 1) * 2); + memset(arr, 0, sizeof(char*) * (count + 1) * 2); + + bool isfound = true; + for(int32_t i = 0; i < count + 1; i++){ + char* str = *(keyvalue + i); + + char** pair = arr + i * 2; + taos_monitor_split_str(pair, str, "="); + + taos_monitor_strip(pair[1]); + } + + int32_t table_size = tjsonGetArraySize(arrayMetricGroups); + + SJson* item = NULL; + for(int32_t i = 0; i < table_size; i++){ + SJson *cur = tjsonGetArrayItem(arrayMetricGroups, i); + + SJson* tag = tjsonGetObjectItem(cur, "tags"); + + if(taos_monitor_is_match(tag, arr, count + 1)) { + item = cur; + break; + } + } + + SJson* metrics = NULL; + if(item == NULL) { + item = tjsonCreateObject(); + + SJson* arrayTag = tjsonCreateArray(); + for(int32_t i = 0; i < count + 1; i++){ + char** pair = arr + i * 2; + + char* key = *pair; + char* value = *(pair + 1); + + SJson* tag = tjsonCreateObject(); + tjsonAddStringToObject(tag, "name", key); + tjsonAddStringToObject(tag, "value", value); + + tjsonAddItemToArray(arrayTag, tag); + } + tjsonAddItemToObject(item, "tags", arrayTag); + + metrics = tjsonCreateArray(); + tjsonAddItemToObject(item, "metrics", metrics); + + tjsonAddItemToArray(arrayMetricGroups, item); + } + else{ + metrics = tjsonGetObjectItem(item, "metrics"); + } + + taosMemoryFreeClear(arr); + taosMemoryFreeClear(keyvalue); + taosMemoryFreeClear(keyvalues); + + SJson* metric = tjsonCreateObject(); + tjsonAddStringToObject(metric, "name", metricName); + + double old_value = 0; +#define USE_EXCHANGE +#ifdef USE_EXCHANGE + taos_metric_sample_exchange(sample, 0, &old_value); +#else + old_value = sample->r_value; + taos_metric_sample_set(sample, 0); +#endif + + tjsonAddDoubleToObject(metric, "value", old_value); + tjsonAddDoubleToObject(metric, "type", metric_type); + tjsonAddItemToArray(metrics, metric); + + return 0; +} + +int taos_metric_formatter_load_metric_new(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format, + SJson* tableArray) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + int32_t size = strlen(metric->name); + char* name = taosMemoryMalloc(size + 1); + memset(name, 0, size + 1); + memcpy(name, metric->name, size); + char* arr[2] = {0}; //arr[0] is table name, arr[1] is metric name + taos_monitor_split_str((char**)&arr, name, ":"); + + bool isFound = false; + SJson* table = NULL; + SJson* arrayMetricGroups = NULL; + + int32_t table_count = tjsonGetArraySize(tableArray); + for(int32_t i = 0; i < table_count; i++){ + SJson* table = tjsonGetArrayItem(tableArray, i); + + char tableName[MONITOR_TABLENAME_LEN] = {0}; + tjsonGetStringValue(table, "name", tableName); + if(strcmp(tableName, arr[0]) == 0){ + isFound = true; + arrayMetricGroups = tjsonGetObjectItem(table, "metric_groups"); + break; + } + } + + if(!isFound){ + table = tjsonCreateObject(); + + tjsonAddStringToObject(table, "name", arr[0]); + + arrayMetricGroups = tjsonCreateArray(); + tjsonAddItemToObject(table, "metric_groups", arrayMetricGroups); + } + + int32_t sample_count = 0; + for (taos_linked_list_node_t *current_node = metric->samples->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *key = (const char *)current_node->item; + if (metric->type == TAOS_HISTOGRAM) { + + } else { + taos_metric_sample_t *sample = (taos_metric_sample_t *)taos_map_get(metric->samples, key); + if (sample == NULL) return 1; + r = taos_metric_formatter_load_sample_new(self, sample, ts, format, arr[1], metric->type, arrayMetricGroups); + if (r) return r; + } + sample_count++; + } + + if(!isFound && sample_count > 0){ + tjsonAddItemToArray(tableArray, table); + } + else{ + if(table != NULL) tjsonDelete(table); + } + + taosMemoryFreeClear(name); + return r; +} + +int taos_metric_formatter_load_metrics_new(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts, + char *format, SJson* tableArray) { + TAOS_ASSERT(self != NULL); + int r = 0; + + for (taos_linked_list_node_t *current_node = collectors->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *collector_name = (const char *)current_node->item; + taos_collector_t *collector = (taos_collector_t *)taos_map_get(collectors, collector_name); + if (collector == NULL) return 1; + + taos_map_t *metrics = collector->collect_fn(collector); + if (metrics == NULL) return 1; + + //if(strcmp(collector->name, "custom") != 0 ){ + + for (taos_linked_list_node_t *current_node = metrics->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *metric_name = (const char *)current_node->item; + taos_metric_t *metric = (taos_metric_t *)taos_map_get(metrics, metric_name); + if (metric == NULL) return 1; + r = taos_metric_formatter_load_metric_new(self, metric, ts, format, tableArray); + if (r) return r; + } + + //} + //else{ + + for (taos_linked_list_node_t *current_node = metrics->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *metric_name = (const char *)current_node->item; + taos_metric_t *metric = (taos_metric_t *)taos_map_get(metrics, metric_name); + if (metric == NULL) return 1; + r = taos_metric_formatter_load_metric(self, metric, ts, format); + if (r) return r; + } + + //} + } + return r; +} \ No newline at end of file diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index 034657cb3d..04a6045e76 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -//#include + // Public #include "taos_alloc.h" @@ -25,10 +25,14 @@ #include "taos_metric_sample_i.h" #include "taos_metric_sample_t.h" -#define ALLOW_FORBID_FUNC +#define DOUBLE_ATOMIC -#include "tdef.h" +#ifdef DOUBLE_ATOMIC +#include +#else +#define ALLOW_FORBID_FUNC #include "osAtomic.h" +#endif taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value) { taos_metric_sample_t *self = (taos_metric_sample_t *)taos_malloc(sizeof(taos_metric_sample_t)); @@ -67,7 +71,8 @@ int taos_metric_sample_add(taos_metric_sample_t *self, double r_value) { if (r_value < 0) { return 1; } - /* + +#ifdef DOUBLE_ATOMIC _Atomic double old = atomic_load(&self->r_value); for (;;) { @@ -76,8 +81,10 @@ int taos_metric_sample_add(taos_metric_sample_t *self, double r_value) { return 0; } } - */ +#else atomic_fetch_add_64(&self->r_value, r_value); +#endif + return 0; } @@ -87,7 +94,8 @@ int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); return 1; } - /* + +#ifdef DOUBLE_ATOMIC _Atomic double old = atomic_load(&self->r_value); for (;;) { _Atomic double new = ATOMIC_VAR_INIT(old - r_value); @@ -95,8 +103,10 @@ int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { return 0; } } - */ +#else atomic_fetch_sub_64(&self->r_value, r_value); +#endif + return 0; } @@ -105,9 +115,34 @@ int taos_metric_sample_set(taos_metric_sample_t *self, double r_value) { TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); return 1; } - /* + +#ifdef DOUBLE_ATOMIC atomic_store(&self->r_value, r_value); - */ +#else atomic_store_64(&self->r_value, r_value); +#endif + return 0; } + +int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, double* old_value) { + if (self->type != TAOS_GAUGE && self->type != TAOS_COUNTER) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + +#ifdef DOUBLE_ATOMIC + _Atomic double new = ATOMIC_VAR_INIT(r_value); + for (;;) { + _Atomic double old = atomic_load(&self->r_value); + *old_value = old; + if (atomic_compare_exchange_weak(&self->r_value, &old, new)) { + return 0; + } + } +#else + *old_value = atomic_exchange_64(&self->r_value, r_value); +#endif + + return 0; +} \ No newline at end of file diff --git a/source/libs/monitorfw/src/taos_monitor_util.c b/source/libs/monitorfw/src/taos_monitor_util.c new file mode 100644 index 0000000000..182402b3ff --- /dev/null +++ b/source/libs/monitorfw/src/taos_monitor_util.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +#define ALLOW_FORBID_FUNC +#include "tjson.h" +#include +#include +#include +#include "osMemory.h" +#include "tdef.h" + +#include "taos_metric_t.h" + +void taos_monitor_split_str(char** arr, char* str, const char* del) { + char *lasts; + char* s = strsep(&str, del); + while (s != NULL) { + *arr++ = s; + s = strsep(&str, del); + } +} + +void taos_monitor_split_str_metric(char** arr, taos_metric_t* metric, const char* del, char** buf) { + int32_t size = strlen(metric->name); + char* name = taosMemoryMalloc(size + 1); + memset(name, 0, size + 1); + memcpy(name, metric->name, size); + + char* s = strtok(name, del); + while (s != NULL) { + *arr++ = s; + s = strtok(NULL, del); + } + + *buf = name; +} + +const char* taos_monitor_get_metric_name(taos_metric_t* metric){ + return metric->name; +} + +int taos_monitor_count_occurrences(char *str, char *toSearch) { + int count = 0; + char *ptr = str; + while ((ptr = strstr(ptr, toSearch)) != NULL) { + count++; + ptr++; + } + return count; +} + +void taos_monitor_strip(char *s) +{ + size_t i; + size_t len = strlen(s); + size_t offset = 0; + for(i = 0; i < len; ++i){ + char c = s[i]; + if(c=='\"') ++offset; + else s[i-offset] = c; + } + s[len-offset] = '\0'; +} + +bool taos_monitor_is_match(const SJson* tags, char** pairs, int32_t count) { + int32_t size = tjsonGetArraySize(tags); + if(size != count) return false; + + for(int32_t i = 0; i < size; i++){ + SJson* item = tjsonGetArrayItem(tags, i); + + char item_name[MONITOR_TAG_NAME_LEN] = {0}; + tjsonGetStringValue(item, "name", item_name); + + char item_value[MONITOR_TAG_VALUE_LEN] = {0}; + tjsonGetStringValue(item, "value", item_value); + + bool isfound = false; + for(int32_t j = 0; j < count; j++){ + + char** pair = pairs + j * 2; + + char* key = *pair; + char* value = *(pair + 1); + + + if(strcmp(value, item_value) == 0 && strcmp(key, item_name) == 0){ + isfound = true; + break; + } + } + + if(!isfound) return false; + } + + return true; +} From f6ecd8d20477812493b8b4194fd62941900da493 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 25 Jan 2024 01:37:32 +0000 Subject: [PATCH 030/181] keep old protocal --- source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index 590c25b936..3d6be1f433 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -19,7 +19,13 @@ #include "audit.h" static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { - //pInfo->protocol = 1; + pInfo->protocol = 1; + pInfo->dnode_id = pDnode->data.dnodeId; + pInfo->cluster_id = pDnode->data.clusterId; + tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); +} + +static void dmGetMonitorBasicInfoBasic(SDnode *pDnode, SMonBasicInfo *pInfo) { pInfo->protocol = 2; pInfo->dnode_id = pDnode->data.dnodeId; pInfo->cluster_id = pDnode->data.clusterId; @@ -45,6 +51,12 @@ static void dmGetDmMonitorInfo(SDnode *pDnode) { monSetDmInfo(&dmInfo); } +static void dmGetDmMonitorInfoBasic(SDnode *pDnode) { + SMonDmInfo dmInfo = {0}; + dmGetMonitorBasicInfoBasic(pDnode, &dmInfo.basic); + monSetDmInfo(&dmInfo); +} + static void dmGetMmMonitorInfo(SDnode *pDnode) { SMgmtWrapper *pWrapper = &pDnode->wrappers[MNODE]; if (dmMarkWrapper(pWrapper) == 0) { @@ -115,11 +127,8 @@ void dmSendMonitorReportBasic() { dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); SDnode *pDnode = dmInstance(); - dmGetDmMonitorInfo(pDnode); + dmGetDmMonitorInfoBasic(pDnode); dmGetMmMonitorInfo(pDnode); - //dmGetVmMonitorInfo(pDnode); - //dmGetQmMonitorInfo(pDnode); - //dmGetSmMonitorInfo(pDnode); monGenAndSendReportBasic(); } From e7953ad541ade1c19eac0c6571bd0f316cd99344 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 25 Jan 2024 03:26:04 +0000 Subject: [PATCH 031/181] revert client change --- source/client/src/clientEnv.c | 4 ---- source/client/src/selectMonitor.c | 2 +- source/client/src/slowQueryMonitor.c | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index bb77c530c3..d5f444a0e3 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -83,8 +83,6 @@ static void deregisterRequest(SRequestObj *pRequest) { int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); int32_t reqType = SLOW_LOG_TYPE_OTHERS; - //taosSsleep(3); - int64_t duration = taosGetTimestampUs() - pRequest->metric.start; tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%.2f ms, " @@ -111,8 +109,6 @@ static void deregisterRequest(SRequestObj *pRequest) { } } - duration = 7000000; - if (duration >= (tsSlowLogThreshold * 1000000UL)) { atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); if (tsSlowLogScope & reqType) { diff --git a/source/client/src/selectMonitor.c b/source/client/src/selectMonitor.c index 58aba8de0b..c1cab23188 100644 --- a/source/client/src/selectMonitor.c +++ b/source/client/src/selectMonitor.c @@ -16,7 +16,7 @@ #include "clientMonitor.h" #include "clientLog.h" -const char* selectMonitorName = "slow_query:slow_query_metric"; +const char* selectMonitorName = "slow_query"; const char* selectMonitorHelp = "slow query log when cost > 3s"; const int selectMonitorLabelCount = 1; const char* selectMonitorLabels[] = {"default"}; diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c index 4af00f5360..420b66a954 100644 --- a/source/client/src/slowQueryMonitor.c +++ b/source/client/src/slowQueryMonitor.c @@ -17,7 +17,7 @@ #include "clientLog.h" #include "tglobal.h" -const char* slowQueryName = "slow_query:slow_query_metric"; +const char* slowQueryName = "slow_query"; const char* slowQueryHelp = "slow query log when cost > 3s"; const int slowQueryLabelCount = 1; const char* slowQueryLabels[] = {"cost"}; From 0f90422a39040ae8a82675c421de38dd411cc931 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 25 Jan 2024 03:34:45 +0000 Subject: [PATCH 032/181] revert client --- source/libs/monitor/src/clientMonitor.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/monitor/src/clientMonitor.c b/source/libs/monitor/src/clientMonitor.c index 64df693e1e..db5fde0d8d 100644 --- a/source/libs/monitor/src/clientMonitor.c +++ b/source/libs/monitor/src/clientMonitor.c @@ -10,14 +10,13 @@ tmr_h tmrStartHandle; SHashObj* clusterMonitorInfoTable; static const int interval = 1000; // ms -static const int sendBathchSize = 1; +static const int sendBathchSize = 10; int32_t sendReport(ClientMonitor* pMonitor, char* pCont); void generateClusterReport(ClientMonitor* pMonitor, bool send) { char ts[50]; sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); char* pCont = (char*)taos_collector_registry_bridge_new(pMonitor->registry, ts, "%" PRId64, NULL); - uInfo("report cont:\n%s", pCont); if (send && strlen(pCont) != TSDB_CODE_SUCCESS) { if (sendReport(pMonitor, pCont) == 0) { taos_collector_registry_clear_batch(pMonitor->registry); From b10d7e7c1f9ab27ed62746ce1529264ec73f3cfd Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 25 Jan 2024 07:30:54 +0000 Subject: [PATCH 033/181] compile atomic --- source/libs/monitorfw/inc/taos_metric_sample_t.h | 4 ++++ source/libs/monitorfw/src/taos_metric_sample.c | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/monitorfw/inc/taos_metric_sample_t.h b/source/libs/monitorfw/inc/taos_metric_sample_t.h index 3aa9b7bb99..fd796978b3 100644 --- a/source/libs/monitorfw/inc/taos_metric_sample_t.h +++ b/source/libs/monitorfw/inc/taos_metric_sample_t.h @@ -21,6 +21,10 @@ #define DOUBLE_ATOMIC +#ifdef DOUBLE_ATOMIC +#include +#endif + struct taos_metric_sample { taos_metric_type_t type; /**< type is the metric type for the sample */ char *l_value; /**< l_value is the full metric name and label set represeted as a string */ diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index 04a6045e76..1ee699fe2b 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -25,8 +25,6 @@ #include "taos_metric_sample_i.h" #include "taos_metric_sample_t.h" -#define DOUBLE_ATOMIC - #ifdef DOUBLE_ATOMIC #include #else From 2a8ef27f33934be8f1e7953239fe6a93fd07357e Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 25 Jan 2024 08:22:21 +0000 Subject: [PATCH 034/181] compile in macos --- source/libs/monitorfw/src/taos_metric_sample.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index 1ee699fe2b..c6c3844ab3 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -71,7 +71,7 @@ int taos_metric_sample_add(taos_metric_sample_t *self, double r_value) { } #ifdef DOUBLE_ATOMIC - _Atomic double old = atomic_load(&self->r_value); + /*_Atomic*/ double old = atomic_load(&self->r_value); for (;;) { _Atomic double new = ATOMIC_VAR_INIT(old + r_value); @@ -94,7 +94,7 @@ int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { } #ifdef DOUBLE_ATOMIC - _Atomic double old = atomic_load(&self->r_value); + /*_Atomic*/ double old = atomic_load(&self->r_value); for (;;) { _Atomic double new = ATOMIC_VAR_INIT(old - r_value); if (atomic_compare_exchange_weak(&self->r_value, &old, new)) { @@ -130,7 +130,7 @@ int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, doub } #ifdef DOUBLE_ATOMIC - _Atomic double new = ATOMIC_VAR_INIT(r_value); + /*_Atomic*/ double new = ATOMIC_VAR_INIT(r_value); for (;;) { _Atomic double old = atomic_load(&self->r_value); *old_value = old; From b693d990dc4fce295e75121156ce9611094e502c Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 25 Jan 2024 09:14:59 +0000 Subject: [PATCH 035/181] atomic macos --- source/libs/monitorfw/src/taos_metric_sample.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index c6c3844ab3..62b058d846 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -130,9 +130,9 @@ int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, doub } #ifdef DOUBLE_ATOMIC - /*_Atomic*/ double new = ATOMIC_VAR_INIT(r_value); + _Atomic double new = ATOMIC_VAR_INIT(r_value); for (;;) { - _Atomic double old = atomic_load(&self->r_value); + /*_Atomic*/ double old = atomic_load(&self->r_value); *old_value = old; if (atomic_compare_exchange_weak(&self->r_value, &old, new)) { return 0; From 580dd28565d5df248d1af6646847e693a08b83fa Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 26 Jan 2024 02:00:43 +0000 Subject: [PATCH 036/181] except windows --- source/libs/monitorfw/inc/taos_metric_sample_t.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/monitorfw/inc/taos_metric_sample_t.h b/source/libs/monitorfw/inc/taos_metric_sample_t.h index fd796978b3..1cc291b3cc 100644 --- a/source/libs/monitorfw/inc/taos_metric_sample_t.h +++ b/source/libs/monitorfw/inc/taos_metric_sample_t.h @@ -19,7 +19,9 @@ #include "taos_metric_sample.h" #include "taos_metric_t.h" +#if !defined(WINDOWS) #define DOUBLE_ATOMIC +#endif #ifdef DOUBLE_ATOMIC #include From c5b06d44fe45e3ba8d6af0ce3dff3b55f9e9c250 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 26 Jan 2024 03:03:34 +0000 Subject: [PATCH 037/181] windows osAtomic.h --- source/libs/monitorfw/src/taos_metric_sample.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index 62b058d846..9b16e6fe1a 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -29,6 +29,7 @@ #include #else #define ALLOW_FORBID_FUNC +#include "tdef.h" #include "osAtomic.h" #endif From 8d683983e0dddeaaf0f4aa61c2fb0c8c4bed06a7 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 29 Jan 2024 10:11:17 +0000 Subject: [PATCH 038/181] interface name --- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/libs/monitor/src/monMain.c | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index eb9c3483d1..bc54a6b3c8 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -106,7 +106,7 @@ typedef struct SQueryNode SQueryNode; #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" -#define VNODE_METRIC_SQL_COUNT "taoscd_sql_req:count" +#define VNODE_METRIC_SQL_COUNT "taos_sql_req:count" #define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type" #define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id" diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 83cde72a91..22b53340e1 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -23,7 +23,8 @@ SMonitor tsMonitor = {0}; char* tsMonUri = "/report"; -char* tsMonFwUri = "/td_metric"; +char* tsMonFwUri = "/general-metric"; +char* tsMonFwBasicUri = "/taosd-cluster-basic"; void monRecordLog(int64_t ts, ELogLevel level, const char *content) { taosThreadMutexLock(&tsMonitor.lock); @@ -554,7 +555,7 @@ static void monGenLogJson(SMonInfo *pMonitor) { void monSendReport(SMonInfo *pMonitor){ char *pCont = tjsonToString(pMonitor->pJson); if(tsMonitorLogProtocol){ - uInfoL("report cont basic:\n%s", pCont); + uInfoL("report cont:\n%s", pCont); } if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; @@ -597,6 +598,20 @@ void monGenAndSendReport() { monCleanupMonitorInfo(pMonitor); } +void monSendReportBasic(SMonInfo *pMonitor){ + char *pCont = tjsonToString(pMonitor->pJson); + if(tsMonitorLogProtocol){ + uInfoL("report cont basic:\n%s", pCont); + } + if (pCont != NULL) { + EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwBasicUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + uError("failed to send monitor msg"); + } + taosMemoryFree(pCont); + } +} + void monGenAndSendReportBasic() { SMonInfo *pMonitor = monCreateMonitorInfo(); if (pMonitor == NULL) return; @@ -605,7 +620,7 @@ void monGenAndSendReportBasic() { monGenBasicJsonBasic(pMonitor); monGenClusterJsonBasic(pMonitor); - monSendReport(pMonitor); + monSendReportBasic(pMonitor); monCleanupMonitorInfo(pMonitor); } \ No newline at end of file From 80d5f35bc5d742e1bb8782265981ee21e4c56f5b Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 31 Jan 2024 01:33:56 +0000 Subject: [PATCH 039/181] vgroups table name --- source/libs/monitor/src/monFramework.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c index 3cc2ab8c90..47912a188c 100644 --- a/source/libs/monitor/src/monFramework.c +++ b/source/libs/monitor/src/monFramework.c @@ -46,7 +46,7 @@ extern char* tsMonFwUri; #define TIMESERIES_USED CLUSTER_TABLE":timeseries_used" #define TIMESERIES_TOTAL CLUSTER_TABLE":timeseries_total" -#define VGROUP_TABLE "taosd_cluster_vgroups_info" +#define VGROUP_TABLE "taosd_vgroups_info" #define TABLES_NUM VGROUP_TABLE":tables_num" #define STATUS VGROUP_TABLE":status" From 907e06ad4deaa54d41d40b96be339a2f147465bb Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 1 Feb 2024 04:52:54 +0000 Subject: [PATCH 040/181] clusterid by taosd --- source/dnode/mnode/impl/src/mndDnode.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index bd0c29f4c8..50bc95f546 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#include #include "tjson.h" #include "mndDnode.h" #include "audit.h" @@ -500,6 +501,9 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { SStatisReq statisReq = {0}; int32_t code = -1; + char strClusterId[TSDB_CLUSTER_ID_LEN] = {0}; + sprintf(strClusterId, "%"PRId64, pMnode->clusterId); + if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; @@ -546,8 +550,21 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { *(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN); tjsonGetStringValue(item, "value", *(sample_labels + j)); + if(strncmp(*(labels + j), "cluster_id", MONITOR_TAG_NAME_LEN) == 0) { + strncpy(*(sample_labels + j), strClusterId, MONITOR_TAG_VALUE_LEN); + } } + /* + *(labels + tagSize) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); + strncpy(*(labels + tagSize), "cluster_id", MONITOR_TAG_NAME_LEN); + + *(sample_labels + tagSize) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN); + strncpy(*(sample_labels + tagSize), strClusterId, MONITOR_TAG_VALUE_LEN); + + tagSize++; + */ + SJson* metrics = tjsonGetObjectItem(item, "metrics"); int32_t metricLen = tjsonGetArraySize(metrics); From 4605f6b0111164e2e3bb6f4b4344c0a3ce02d1f9 Mon Sep 17 00:00:00 2001 From: dm chen Date: Thu, 1 Feb 2024 14:19:12 +0800 Subject: [PATCH 041/181] Windows double atomic --- .../libs/monitorfw/inc/taos_metric_sample_t.h | 2 +- .../libs/monitorfw/src/taos_metric_sample.c | 58 +++++++++++++++++-- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/source/libs/monitorfw/inc/taos_metric_sample_t.h b/source/libs/monitorfw/inc/taos_metric_sample_t.h index 1cc291b3cc..a955a1fa4d 100644 --- a/source/libs/monitorfw/inc/taos_metric_sample_t.h +++ b/source/libs/monitorfw/inc/taos_metric_sample_t.h @@ -33,7 +33,7 @@ struct taos_metric_sample { #ifdef DOUBLE_ATOMIC _Atomic double r_value; /**< r_value is the value of the metric sample */ #else - int64_t r_value; /**< r_value is the value of the metric sample */ + double r_value; /**< r_value is the value of the metric sample */ #endif }; diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index 9b16e6fe1a..d289b4e722 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -31,6 +31,10 @@ #define ALLOW_FORBID_FUNC #include "tdef.h" #include "osAtomic.h" +typedef union { + volatile int64_t i; + double d; +} double_number; #endif taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value) { @@ -81,7 +85,19 @@ int taos_metric_sample_add(taos_metric_sample_t *self, double r_value) { } } #else - atomic_fetch_add_64(&self->r_value, r_value); + for (;;) { + double_number old_num; + old_num.i = *(int64_t *)(&self->r_value); // current old value + + double_number new_num; + new_num.d = old_num.d + r_value; + + int64_t old_value = atomic_val_compare_exchange_64((volatile int64_t *)(&self->r_value), new_num.i, old_num.i); + + if (old_value == old_num.i) return 0; + } + + //atomic_fetch_add_64(&self->r_value, r_value); #endif return 0; @@ -103,7 +119,19 @@ int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { } } #else - atomic_fetch_sub_64(&self->r_value, r_value); + for (;;) { + double_number old_num; + old_num.i = *(int64_t *)(&self->r_value); // current old value + + double_number new_num; + new_num.d = old_num.d - r_value; + + int64_t old_value = atomic_val_compare_exchange_64((volatile int64_t *)(&self->r_value), new_num.i, old_num.i); + + if (old_value == old_num.i) return 0; + } + + //atomic_fetch_sub_64(&self->r_value, r_value); #endif return 0; @@ -118,7 +146,16 @@ int taos_metric_sample_set(taos_metric_sample_t *self, double r_value) { #ifdef DOUBLE_ATOMIC atomic_store(&self->r_value, r_value); #else - atomic_store_64(&self->r_value, r_value); + for (;;) { + int64_t iOld = *(int64_t *)(&self->r_value); // current old value + + int64_t iNew = *(int64_t *)(&r_value); + + int64_t old_value = atomic_val_compare_exchange_64((volatile int64_t *)(&self->r_value), iNew, iOld); + + if (old_value == iOld) return 0; + } + //atomic_store_64(&self->r_value, r_value); #endif return 0; @@ -140,7 +177,20 @@ int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, doub } } #else - *old_value = atomic_exchange_64(&self->r_value, r_value); + for (;;) { + int64_t iOld = *(int64_t *)(&self->r_value); // current old value + + int64_t iNew = *(int64_t *)(&r_value); + + int64_t iold_value = atomic_val_compare_exchange_64((volatile int64_t *)(&self->r_value), iNew, iOld); + + if (iold_value == iOld) { + *old_value = *(double *)(&iold_value); + return 0; + } + } + + //*old_value = atomic_exchange_64(&self->r_value, r_value); #endif return 0; From 7752a57580e8a02ce060067d554675bff52ad270 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 1 Feb 2024 07:26:05 +0000 Subject: [PATCH 042/181] not log null, memory leak, skip empty, default config --- source/common/src/tglobal.c | 2 +- source/libs/monitor/src/monFramework.c | 9 ++++-- source/libs/monitor/src/monMain.c | 13 ++++++--- .../monitorfw/src/taos_collector_registry.c | 29 ++++++++++++------- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index cb853c1a75..4a0a7a3a73 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -95,7 +95,7 @@ int32_t tsMonitorMaxLogs = 100; bool tsMonitorComp = false; bool tsMonitorLogProtocol = false; int32_t tsMonitorIntervalForBasic = 30; -bool tsMonitorForceV2 = false; +bool tsMonitorForceV2 = true; // audit bool tsEnableAudit = true; diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c index 47912a188c..b1e6265906 100644 --- a/source/libs/monitor/src/monFramework.c +++ b/source/libs/monitor/src/monFramework.c @@ -638,8 +638,13 @@ void monSendPromReport() { char* promStr = NULL; char* pCont = (char *)taos_collector_registry_bridge_new(TAOS_COLLECTOR_REGISTRY_DEFAULT, ts, "%" PRId64, &promStr); if(tsMonitorLogProtocol){ - uInfoL("report cont:\n%s\n", pCont); - uDebugL("report cont prom:\n%s\n", promStr); + if(pCont != NULL){ + uInfoL("report cont:\n%s", pCont); + uDebugL("report cont prom:\n%s", promStr); + } + else{ + uInfo("report cont is null"); + } } if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 22b53340e1..ba3db804c3 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -601,7 +601,12 @@ void monGenAndSendReport() { void monSendReportBasic(SMonInfo *pMonitor){ char *pCont = tjsonToString(pMonitor->pJson); if(tsMonitorLogProtocol){ - uInfoL("report cont basic:\n%s", pCont); + if(pCont != NULL){ + uInfoL("report cont basic:\n%s", pCont); + } + else{ + uInfo("report cont basic is null"); + } } if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; @@ -614,13 +619,13 @@ void monSendReportBasic(SMonInfo *pMonitor){ void monGenAndSendReportBasic() { SMonInfo *pMonitor = monCreateMonitorInfo(); - if (pMonitor == NULL) return; - if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; monGenBasicJsonBasic(pMonitor); monGenClusterJsonBasic(pMonitor); - monSendReportBasic(pMonitor); + if (pMonitor->mmInfo.cluster.first_ep_dnode_id != 0) { + monSendReportBasic(pMonitor); + } monCleanupMonitorInfo(pMonitor); } \ No newline at end of file diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c index c94675c95c..acf2cb4248 100644 --- a/source/libs/monitorfw/src/taos_collector_registry.c +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -234,6 +234,11 @@ const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self, taos_metric_formatter_load_metrics_new(self->metric_formatter, self->collectors, ts, format, array); + if(tjsonGetArraySize(array) == 0){ + tjsonDelete(pJson); + return NULL; + } + //caller free this //generate prom protocol for debug if(prom_str != NULL){ @@ -245,36 +250,40 @@ const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self, char* old_str = taos_string_builder_str(self->string_builder_batch); if(old_str[0] != '\0'){ r = taos_string_builder_add_str(self->string_builder_batch, ","); - if (r) return NULL; + if (r) goto _OVER; } char * item_str = tjsonToString(item); r = taos_string_builder_add_str(self->string_builder_batch, item_str); taos_free(item_str); - if (r) return NULL; - - tjsonDelete(pJson); + if (r) goto _OVER;; //generate final array format result, ie, add [] to str in batch cache taos_string_builder_t* tmp_builder = taos_string_builder_new(); r = taos_string_builder_add_str(tmp_builder, "["); - if (r) return NULL; + if (r) goto _OVER;; r = taos_string_builder_add_str(tmp_builder, taos_string_builder_str(self->string_builder_batch)); - if (r) return NULL; + if (r) goto _OVER;; r = taos_string_builder_add_str(tmp_builder, "]"); - if (r) return NULL; + if (r) goto _OVER;; //caller free this char *data = taos_string_builder_dump(tmp_builder); - if (data == NULL) return NULL; + if (data == NULL) goto _OVER;; r = taos_string_builder_clear(tmp_builder); - if (r) return NULL; + if (r) goto _OVER;; r = taos_string_builder_destroy(tmp_builder); tmp_builder = NULL; - if (r) return NULL; + if (r) goto _OVER;; + tjsonDelete(pJson); return data; + +_OVER: + tjsonDelete(pJson); + + return NULL; } From 97a120ee0968e09fff3afb79eb1a3e4e35e52860 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 1 Feb 2024 08:41:10 +0000 Subject: [PATCH 043/181] fix taosd monitor case --- tests/system-test/0-others/taosdMonitor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index 169c70e675..690de65dbb 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -267,6 +267,7 @@ class TDTestCase: updatecfgDict["monitorInterval"] = "5" updatecfgDict["monitorMaxLogs"] = "10" updatecfgDict["monitorComp"] = "1" + updatecfgDict["monitorForceV2"] = "0" updatecfgDict["audit"] = '0' From d3b668bd5fdd3ba81dc848612fe2773868aa74cb Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 2 Feb 2024 01:48:00 +0000 Subject: [PATCH 044/181] double atomic lib --- include/os/osAtomic.h | 4 ++ .../libs/monitorfw/src/taos_metric_sample.c | 58 ++---------------- source/os/src/osAtomic.c | 61 +++++++++++++++++++ 3 files changed, 69 insertions(+), 54 deletions(-) diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index 9fd00cefb4..48b7b8c56f 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -48,11 +48,13 @@ void atomic_store_8(int8_t volatile *ptr, int8_t val); void atomic_store_16(int16_t volatile *ptr, int16_t val); void atomic_store_32(int32_t volatile *ptr, int32_t val); void atomic_store_64(int64_t volatile *ptr, int64_t val); +double atomic_store_double(double volatile *ptr, double val); void atomic_store_ptr(void *ptr, void *val); int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val); int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val); int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val); int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val); +double atomic_exchange_double(double volatile *ptr, int64_t val); void *atomic_exchange_ptr(void *ptr, void *val); int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval); int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval); @@ -68,6 +70,7 @@ int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val); +double atomic_fetch_add_double(double volatile *ptr, double val); void *atomic_fetch_add_ptr(void *ptr, void *val); int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val); @@ -78,6 +81,7 @@ int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val); +double atomic_fetch_sub_double(double volatile *ptr, double val); void *atomic_fetch_sub_ptr(void *ptr, void *val); int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val); diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index d289b4e722..445568ed5b 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -31,10 +31,6 @@ #define ALLOW_FORBID_FUNC #include "tdef.h" #include "osAtomic.h" -typedef union { - volatile int64_t i; - double d; -} double_number; #endif taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value) { @@ -85,19 +81,7 @@ int taos_metric_sample_add(taos_metric_sample_t *self, double r_value) { } } #else - for (;;) { - double_number old_num; - old_num.i = *(int64_t *)(&self->r_value); // current old value - - double_number new_num; - new_num.d = old_num.d + r_value; - - int64_t old_value = atomic_val_compare_exchange_64((volatile int64_t *)(&self->r_value), new_num.i, old_num.i); - - if (old_value == old_num.i) return 0; - } - - //atomic_fetch_add_64(&self->r_value, r_value); + atomic_fetch_add_double(&self->r_value, r_value); #endif return 0; @@ -119,19 +103,7 @@ int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { } } #else - for (;;) { - double_number old_num; - old_num.i = *(int64_t *)(&self->r_value); // current old value - - double_number new_num; - new_num.d = old_num.d - r_value; - - int64_t old_value = atomic_val_compare_exchange_64((volatile int64_t *)(&self->r_value), new_num.i, old_num.i); - - if (old_value == old_num.i) return 0; - } - - //atomic_fetch_sub_64(&self->r_value, r_value); + atomic_fetch_sub_double(&self->r_value, r_value); #endif return 0; @@ -146,16 +118,7 @@ int taos_metric_sample_set(taos_metric_sample_t *self, double r_value) { #ifdef DOUBLE_ATOMIC atomic_store(&self->r_value, r_value); #else - for (;;) { - int64_t iOld = *(int64_t *)(&self->r_value); // current old value - - int64_t iNew = *(int64_t *)(&r_value); - - int64_t old_value = atomic_val_compare_exchange_64((volatile int64_t *)(&self->r_value), iNew, iOld); - - if (old_value == iOld) return 0; - } - //atomic_store_64(&self->r_value, r_value); + atomic_store_double(&self->r_value, r_value); #endif return 0; @@ -177,20 +140,7 @@ int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, doub } } #else - for (;;) { - int64_t iOld = *(int64_t *)(&self->r_value); // current old value - - int64_t iNew = *(int64_t *)(&r_value); - - int64_t iold_value = atomic_val_compare_exchange_64((volatile int64_t *)(&self->r_value), iNew, iOld); - - if (iold_value == iOld) { - *old_value = *(double *)(&iold_value); - return 0; - } - } - - //*old_value = atomic_exchange_64(&self->r_value, r_value); + *old_value = atomic_exchange_double(&self->r_value, r_value); #endif return 0; diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index a54c301de9..3736a71e08 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -16,6 +16,11 @@ #define ALLOW_FORBID_FUNC #include "os.h" +typedef union { + volatile int64_t i; + double d; +} double_number; + #ifdef WINDOWS // add @@ -339,6 +344,18 @@ void atomic_store_64(int64_t volatile* ptr, int64_t val) { #endif } +double atomic_store_double(double volatile *ptr, double val){ + for (;;) { + int64_t iOld = *(int64_t *)ptr; // current old value + + int64_t iNew = *(int64_t *)(&val); + + int64_t old_value = atomic_val_compare_exchange_64((volatile int64_t *)ptr, iOld, iNew); + + if (old_value == iOld) return old_value; + } +} + void atomic_store_ptr(void* ptr, void* val) { #ifdef WINDOWS ((*(void* volatile*)(ptr)) = (void*)(val)); @@ -393,6 +410,20 @@ int64_t atomic_exchange_64(int64_t volatile* ptr, int64_t val) { #endif } +double atomic_exchange_double(double volatile *ptr, int64_t val){ + for (;;) { + int64_t iOld = *(int64_t *)ptr; // current old value + + int64_t iNew = *(int64_t *)(&val); + + int64_t iold_value = atomic_val_compare_exchange_64((volatile int64_t *)ptr, iOld, iNew); + + if (iold_value == iOld) { + return *(double *)(&iold_value); + } + } +} + void* atomic_exchange_ptr(void* ptr, void* val) { #ifdef WINDOWS #ifdef _WIN64 @@ -551,6 +582,21 @@ int64_t atomic_fetch_add_64(int64_t volatile* ptr, int64_t val) { #endif } +double atomic_fetch_add_double(double volatile *ptr, double val){ + for (;;) { + double_number old_num = {0}; + old_num.i = *(int64_t *)ptr; // current old value + + double_number new_num = {0}; + new_num.d = old_num.d + val; + + double_number ret_num = {0}; + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + + if (ret_num.i == old_num.i) return ret_num.d; + } +} + void* atomic_fetch_add_ptr(void* ptr, void* val) { #ifdef WINDOWS return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)); @@ -657,6 +703,21 @@ int64_t atomic_fetch_sub_64(int64_t volatile* ptr, int64_t val) { #endif } +double atomic_fetch_sub_double(double volatile *ptr, double val){ + for (;;) { + double_number old_num = {0}; + old_num.i = *(int64_t *)ptr; // current old value + + double_number new_num = {0}; + new_num.d = old_num.d - val; + + double_number ret_num = {0}; + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + + if (ret_num.i == old_num.i) return ret_num.d; + } +} + void* atomic_fetch_sub_ptr(void* ptr, void* val) { #ifdef WINDOWS return interlocked_fetch_sub_ptr(ptr, val); From 7ae45a0a837d27d7f51e57f942356f820625095d Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 2 Feb 2024 02:21:06 +0000 Subject: [PATCH 045/181] atomic refactor --- .../libs/monitorfw/inc/taos_metric_sample_t.h | 6 ++-- .../libs/monitorfw/src/taos_metric_sample.c | 10 +++---- source/os/src/osAtomic.c | 28 +++++++++++-------- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/source/libs/monitorfw/inc/taos_metric_sample_t.h b/source/libs/monitorfw/inc/taos_metric_sample_t.h index a955a1fa4d..62d58c5b9f 100644 --- a/source/libs/monitorfw/inc/taos_metric_sample_t.h +++ b/source/libs/monitorfw/inc/taos_metric_sample_t.h @@ -20,17 +20,17 @@ #include "taos_metric_t.h" #if !defined(WINDOWS) -#define DOUBLE_ATOMIC +#define C11_ATOMIC #endif -#ifdef DOUBLE_ATOMIC +#ifdef C11_ATOMIC #include #endif struct taos_metric_sample { taos_metric_type_t type; /**< type is the metric type for the sample */ char *l_value; /**< l_value is the full metric name and label set represeted as a string */ -#ifdef DOUBLE_ATOMIC +#ifdef C11_ATOMIC _Atomic double r_value; /**< r_value is the value of the metric sample */ #else double r_value; /**< r_value is the value of the metric sample */ diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index 445568ed5b..c3f948434d 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -25,7 +25,7 @@ #include "taos_metric_sample_i.h" #include "taos_metric_sample_t.h" -#ifdef DOUBLE_ATOMIC +#ifdef C11_ATOMIC #include #else #define ALLOW_FORBID_FUNC @@ -71,7 +71,7 @@ int taos_metric_sample_add(taos_metric_sample_t *self, double r_value) { return 1; } -#ifdef DOUBLE_ATOMIC +#ifdef C11_ATOMIC /*_Atomic*/ double old = atomic_load(&self->r_value); for (;;) { @@ -94,7 +94,7 @@ int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { return 1; } -#ifdef DOUBLE_ATOMIC +#ifdef C11_ATOMIC /*_Atomic*/ double old = atomic_load(&self->r_value); for (;;) { _Atomic double new = ATOMIC_VAR_INIT(old - r_value); @@ -115,7 +115,7 @@ int taos_metric_sample_set(taos_metric_sample_t *self, double r_value) { return 1; } -#ifdef DOUBLE_ATOMIC +#ifdef C11_ATOMIC atomic_store(&self->r_value, r_value); #else atomic_store_double(&self->r_value, r_value); @@ -130,7 +130,7 @@ int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, doub return 1; } -#ifdef DOUBLE_ATOMIC +#ifdef C11_ATOMIC _Atomic double new = ATOMIC_VAR_INIT(r_value); for (;;) { /*_Atomic*/ double old = atomic_load(&self->r_value); diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index 3736a71e08..e56cf629bb 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -346,13 +346,16 @@ void atomic_store_64(int64_t volatile* ptr, int64_t val) { double atomic_store_double(double volatile *ptr, double val){ for (;;) { - int64_t iOld = *(int64_t *)ptr; // current old value + double_number old_num = {0}; + old_num.d = *ptr; // current old value - int64_t iNew = *(int64_t *)(&val); + double_number new_num = {0}; + new_num.d = val; - int64_t old_value = atomic_val_compare_exchange_64((volatile int64_t *)ptr, iOld, iNew); + double_number ret_num = {0}; + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); - if (old_value == iOld) return old_value; + if (ret_num.i == old_num.i) return ret_num.d; } } @@ -412,14 +415,17 @@ int64_t atomic_exchange_64(int64_t volatile* ptr, int64_t val) { double atomic_exchange_double(double volatile *ptr, int64_t val){ for (;;) { - int64_t iOld = *(int64_t *)ptr; // current old value + double_number old_num = {0}; + old_num.i = *ptr; // current old value - int64_t iNew = *(int64_t *)(&val); + double_number new_num = {0}; + int64_t iNew = val; - int64_t iold_value = atomic_val_compare_exchange_64((volatile int64_t *)ptr, iOld, iNew); + double_number ret_num = {0}; + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); - if (iold_value == iOld) { - return *(double *)(&iold_value); + if (ret_num.i == old_num.i) { + return ret_num.d; } } } @@ -585,7 +591,7 @@ int64_t atomic_fetch_add_64(int64_t volatile* ptr, int64_t val) { double atomic_fetch_add_double(double volatile *ptr, double val){ for (;;) { double_number old_num = {0}; - old_num.i = *(int64_t *)ptr; // current old value + old_num.d = *ptr; // current old value double_number new_num = {0}; new_num.d = old_num.d + val; @@ -706,7 +712,7 @@ int64_t atomic_fetch_sub_64(int64_t volatile* ptr, int64_t val) { double atomic_fetch_sub_double(double volatile *ptr, double val){ for (;;) { double_number old_num = {0}; - old_num.i = *(int64_t *)ptr; // current old value + old_num.d = *ptr; // current old value double_number new_num = {0}; new_num.d = old_num.d - val; From 010d3fc860ae7078609ca1ed1febb2864de8a94c Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 2 Feb 2024 13:39:31 +0800 Subject: [PATCH 046/181] add select count --- include/libs/monitor/clientMonitor.h | 11 ++++- source/client/inc/clientInt.h | 5 +-- source/client/src/clientEnv.c | 3 +- source/client/src/clientImpl.c | 1 + .../{selectMonitor.c => clientSqlMonitor.c} | 29 +++++++++---- source/client/src/slowQueryMonitor.c | 42 ++++++++++--------- source/libs/monitor/src/clientMonitor.c | 28 ++++++++++--- 7 files changed, 80 insertions(+), 39 deletions(-) rename source/client/src/{selectMonitor.c => clientSqlMonitor.c} (60%) diff --git a/include/libs/monitor/clientMonitor.h b/include/libs/monitor/clientMonitor.h index 9d97c9004f..9c0302a15f 100644 --- a/include/libs/monitor/clientMonitor.h +++ b/include/libs/monitor/clientMonitor.h @@ -24,6 +24,14 @@ extern "C" { #include "thash.h" #include "query.h" +typedef enum SQL_RESULT_CODE { + SQL_RESULT_SUCCESS = 0, + SQL_RESULT_FAILED = 1, + SQL_RESULT_CANCEL = 2, +} SQL_RESULT_CODE; + +const char* resultStr(SQL_RESULT_CODE code); + typedef struct { char clusterKey[512]; SEpSet epSet; @@ -38,8 +46,7 @@ void clusterMonitorClose(const char* clusterKey); taos_counter_t* createClusterCounter(const char* clusterKey, const char* name, const char* help, size_t label_key_count, const char** label_keys); int taosClusterCounterInc(const char* clusterKey, const char* counterName, const char** label_values); - -void cluster_monitor_stop(); +void cluster_monitor_stop(); #ifdef __cplusplus } diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index bae0627055..257ee35aaf 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -432,11 +432,10 @@ int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, boo #endif void clusterSlowQueryMonitorInit(const char* clusterKey); -void clusterSlowQueryLog(const char* clusterKey, int32_t cost); -void SlowQueryLog(int64_t rid, int32_t cost); +void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost); void clusterSelectMonitorInit(const char* clusterKey); -void clusterSelectLog(const char* clusterKey); +void selectLog(int64_t rid, bool killed, int32_t code); #ifdef __cplusplus } diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index d5f444a0e3..bd83f15e7e 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -105,6 +105,7 @@ static void deregisterRequest(SRequestObj *pRequest) { pRequest->metric.planCostUs, pRequest->metric.execCostUs); atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); + selectLog(pTscObj->id, pRequest->killed, pRequest->code); reqType = SLOW_LOG_TYPE_QUERY; } } @@ -115,7 +116,7 @@ static void deregisterRequest(SRequestObj *pRequest) { taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s", taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration, pRequest->sqlstr); - SlowQueryLog(pTscObj->id, duration); + SlowQueryLog(pTscObj->id, pRequest->killed, pRequest->code, duration); } } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 405df7377f..a165ef6e88 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -160,6 +160,7 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas pInst = &p; clusterSlowQueryMonitorInit(p->instKey); + clusterSelectMonitorInit(p->instKey); } else { ASSERTS((*pInst) && (*pInst)->pAppHbMgr, "*pInst:%p, pAppHgMgr:%p", *pInst, (*pInst) ? (*pInst)->pAppHbMgr : NULL); // reset to 0 in case of conn with duplicated user key but its user has ever been dropped. diff --git a/source/client/src/selectMonitor.c b/source/client/src/clientSqlMonitor.c similarity index 60% rename from source/client/src/selectMonitor.c rename to source/client/src/clientSqlMonitor.c index c1cab23188..79a862275f 100644 --- a/source/client/src/selectMonitor.c +++ b/source/client/src/clientSqlMonitor.c @@ -16,10 +16,12 @@ #include "clientMonitor.h" #include "clientLog.h" -const char* selectMonitorName = "slow_query"; -const char* selectMonitorHelp = "slow query log when cost > 3s"; -const int selectMonitorLabelCount = 1; -const char* selectMonitorLabels[] = {"default"}; +const char* selectMonitorName = "select sql"; +const char* selectMonitorHelp = "count for select sql"; +const int selectMonitorLabelCount = 4; +const char* selectMonitorLabels[] = {"cluster_id", "sql_type", "username", "result"}; + +static const char* defaultClusterID = ""; void clusterSelectMonitorInit(const char* clusterKey) { SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey); @@ -28,18 +30,27 @@ void clusterSelectMonitorInit(const char* clusterKey) { createClusterCounter(clusterKey, selectMonitorName, selectMonitorHelp, selectMonitorLabelCount, selectMonitorLabels); } -void clusterSelectLog(const char* clusterKey) { - const char* selectMonitorLabelValues[] = {"default"}; +void clusterSelectLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result) { + const char* selectMonitorLabelValues[] = {defaultClusterID, "select", user, resultStr(result)}; taosClusterCounterInc(clusterKey, selectMonitorName, selectMonitorLabelValues); } -void selectLog(int64_t rid) { +void selectLog(int64_t rid, bool killed, int32_t code) { + SQL_RESULT_CODE result = SQL_RESULT_SUCCESS; + if (TSDB_CODE_SUCCESS != code) { + result = SQL_RESULT_FAILED; + } + // to do Distinguish active Kill events + // else if (killed) { + // result = SQL_RESULT_CANCEL; + // } + STscObj* pTscObj = acquireTscObj(rid); if (pTscObj != NULL) { - if(pTscObj->pAppInfo == NULL) { + if (pTscObj->pAppInfo == NULL) { tscLog("selectLog, not found pAppInfo"); } - return clusterSelectLog(pTscObj->pAppInfo->instKey); + return clusterSelectLog(pTscObj->pAppInfo->instKey, pTscObj->user, result); } else { tscLog("selectLog, not found rid"); } diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c index 420b66a954..8f6253007f 100644 --- a/source/client/src/slowQueryMonitor.c +++ b/source/client/src/slowQueryMonitor.c @@ -18,28 +18,23 @@ #include "tglobal.h" const char* slowQueryName = "slow_query"; -const char* slowQueryHelp = "slow query log when cost > 3s"; -const int slowQueryLabelCount = 1; -const char* slowQueryLabels[] = {"cost"}; +const char* slowQueryHelp = "slow query log when cost over than config duration"; +const int slowQueryLabelCount = 4; +const char* slowQueryLabels[] = {"cluster_id", "username", "result", "duration"}; +static const char* defaultClusterID = ""; const int64_t msInSeconds = 1000; const int64_t msInMinutes = 60 * 1000; static const char* getSlowQueryLableCostDesc(int64_t cost) { - if (cost >= 30 * msInMinutes) { - return " > 30 min"; - } else if (cost >= 10 * msInMinutes) { - return " > 10 min"; - } else if (cost >= 5 * msInMinutes) { - return " > 5 min"; - } else if (cost >= 1 * msInMinutes) { - return " > 1 min"; - } else if (cost >= 30 * msInSeconds) { - return " > 30 seconds"; + if (cost >= 10000 * msInSeconds) { + return " > 10000 seconds"; + } else if (cost >= 1000 * msInSeconds) { + return " > 1000 seconds"; + } else if (cost >= 100 * msInSeconds) { + return " > 100 seconds"; } else if (cost >= 10 * msInSeconds) { return " > 10 seconds"; - } else if (cost >= 5 * msInSeconds) { - return " > 5 seconds"; } else if (cost >= 3 * msInSeconds) { return " > 3 seconds"; } @@ -54,19 +49,28 @@ void clusterSlowQueryMonitorInit(const char* clusterKey) { createClusterCounter(clusterKey, slowQueryName, slowQueryHelp, slowQueryLabelCount, slowQueryLabels); } -void clusterSlowQueryLog(const char* clusterKey, int32_t cost) { - const char* slowQueryLabelValues[] = {getSlowQueryLableCostDesc(cost)}; +void clusterSlowQueryLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int32_t cost) { + const char* slowQueryLabelValues[] = {defaultClusterID, user, resultStr(result), getSlowQueryLableCostDesc(cost)}; taosClusterCounterInc(clusterKey, slowQueryName, slowQueryLabelValues); } -void SlowQueryLog(int64_t rid, int32_t cost) { +void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost) { if (!enableSlowQueryMonitor) return; + SQL_RESULT_CODE result = SQL_RESULT_SUCCESS; + if (TSDB_CODE_SUCCESS != code) { + result = SQL_RESULT_FAILED; + } + // to do Distinguish active Kill events + // else if (killed) { + // result = SQL_RESULT_CANCEL; + // } + STscObj* pTscObj = acquireTscObj(rid); if (pTscObj != NULL) { if(pTscObj->pAppInfo == NULL) { tscLog("SlowQueryLog, not found pAppInfo"); } - return clusterSlowQueryLog(pTscObj->pAppInfo->instKey, cost); + return clusterSlowQueryLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, cost); } else { tscLog("SlowQueryLog, not found rid"); } diff --git a/source/libs/monitor/src/clientMonitor.c b/source/libs/monitor/src/clientMonitor.c index db5fde0d8d..e3b49da858 100644 --- a/source/libs/monitor/src/clientMonitor.c +++ b/source/libs/monitor/src/clientMonitor.c @@ -3,21 +3,26 @@ #include "tmisce.h" #include "ttime.h" #include "ttimer.h" +#include "tglobal.h" SRWLatch monitorLock; void* tmrClientMonitor; tmr_h tmrStartHandle; SHashObj* clusterMonitorInfoTable; -static const int interval = 1000; // ms -static const int sendBathchSize = 10; +static int interval = 30 * 1000; +static int sendBathchSize = 1; int32_t sendReport(ClientMonitor* pMonitor, char* pCont); void generateClusterReport(ClientMonitor* pMonitor, bool send) { char ts[50]; sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); char* pCont = (char*)taos_collector_registry_bridge_new(pMonitor->registry, ts, "%" PRId64, NULL); - if (send && strlen(pCont) != TSDB_CODE_SUCCESS) { + if(NULL == pCont) { + uError("generateClusterReport failed, get null content."); + return; + } + if (send && strlen(pCont) != 0) { if (sendReport(pMonitor, pCont) == 0) { taos_collector_registry_clear_batch(pMonitor->registry); } @@ -25,7 +30,7 @@ void generateClusterReport(ClientMonitor* pMonitor, bool send) { } void reportSendProcess(void* param, void* tmrId) { - taosTmrReset(reportSendProcess, interval, NULL, tmrClientMonitor, &tmrStartHandle); + taosTmrReset(reportSendProcess, tsMonitorInterval * 1000, NULL, tmrClientMonitor, &tmrStartHandle); taosRLockLatch(&monitorLock); static int index = 0; @@ -49,7 +54,15 @@ void monitorClientInitOnce() { (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); tmrClientMonitor = taosTmrInit(0, 0, 0, "MONITOR"); - tmrStartHandle = taosTmrStart(reportSendProcess, interval, NULL, tmrClientMonitor); + tmrStartHandle = taosTmrStart(reportSendProcess, tsMonitorInterval * 1000, NULL, tmrClientMonitor); + if(tsMonitorInterval < 1){ + interval = 30 * 1000; + } else { + interval = tsMonitorInterval * 1000; + } + if (tsMonitorInterval < 10) { + sendBathchSize = (10 / sendBathchSize) + 1; + } taosInitRWLatch(&monitorLock); } } @@ -185,3 +198,8 @@ void clusterMonitorClose(const char* clusterKey) { } taosWUnLockLatch(&monitorLock); } + +const char* resultStr(SQL_RESULT_CODE code) { + static const char* result_state[] = {"Success", "Failed", "Cancel"}; + return result_state[code]; +} From 881df9d9363c17e3e22b1fb327a5502af11849fb Mon Sep 17 00:00:00 2001 From: cadem Date: Fri, 2 Feb 2024 13:47:11 +0800 Subject: [PATCH 047/181] windows atomic --- source/libs/monitorfw/inc/taos_metric_sample_t.h | 6 ++++++ source/libs/monitorfw/src/taos_metric_sample.c | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/source/libs/monitorfw/inc/taos_metric_sample_t.h b/source/libs/monitorfw/inc/taos_metric_sample_t.h index 62d58c5b9f..6ebc000df5 100644 --- a/source/libs/monitorfw/inc/taos_metric_sample_t.h +++ b/source/libs/monitorfw/inc/taos_metric_sample_t.h @@ -21,6 +21,8 @@ #if !defined(WINDOWS) #define C11_ATOMIC +#else +//#define DOUBLE_ATOMIC #endif #ifdef C11_ATOMIC @@ -33,7 +35,11 @@ struct taos_metric_sample { #ifdef C11_ATOMIC _Atomic double r_value; /**< r_value is the value of the metric sample */ #else +#ifdef DOUBLE_ATOMIC double r_value; /**< r_value is the value of the metric sample */ +#else + int64_t r_value; +#endif #endif }; diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index c3f948434d..ca6ea30028 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -81,7 +81,11 @@ int taos_metric_sample_add(taos_metric_sample_t *self, double r_value) { } } #else +#ifdef DOUBLE_ATOMIC atomic_fetch_add_double(&self->r_value, r_value); +#else + atomic_fetch_add_64(&self->r_value, r_value); +#endif #endif return 0; @@ -103,7 +107,11 @@ int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { } } #else +#ifdef DOUBLE_ATOMIC atomic_fetch_sub_double(&self->r_value, r_value); +#else + atomic_fetch_sub_64(&self->r_value, r_value); +#endif #endif return 0; @@ -118,7 +126,11 @@ int taos_metric_sample_set(taos_metric_sample_t *self, double r_value) { #ifdef C11_ATOMIC atomic_store(&self->r_value, r_value); #else +#ifdef DOUBLE_ATOMIC atomic_store_double(&self->r_value, r_value); +#else + atomic_store_64(&self->r_value, r_value); +#endif #endif return 0; @@ -140,7 +152,11 @@ int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, doub } } #else +#ifdef DOUBLE_ATOMIC *old_value = atomic_exchange_double(&self->r_value, r_value); +#else + *old_value = atomic_exchange_64(&self->r_value, r_value); +#endif #endif return 0; From 75539fb9080eeb30f7b71ecf09e8079cdc000a2d Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Fri, 2 Feb 2024 14:16:28 +0800 Subject: [PATCH 048/181] fix: table name --- source/client/src/clientSqlMonitor.c | 2 +- source/client/src/slowQueryMonitor.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientSqlMonitor.c b/source/client/src/clientSqlMonitor.c index 79a862275f..2f8aa1db5d 100644 --- a/source/client/src/clientSqlMonitor.c +++ b/source/client/src/clientSqlMonitor.c @@ -16,7 +16,7 @@ #include "clientMonitor.h" #include "clientLog.h" -const char* selectMonitorName = "select sql"; +const char* selectMonitorName = "taos_sql_req:count"; const char* selectMonitorHelp = "count for select sql"; const int selectMonitorLabelCount = 4; const char* selectMonitorLabels[] = {"cluster_id", "sql_type", "username", "result"}; diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c index 8f6253007f..0818aaeae4 100644 --- a/source/client/src/slowQueryMonitor.c +++ b/source/client/src/slowQueryMonitor.c @@ -17,7 +17,7 @@ #include "clientLog.h" #include "tglobal.h" -const char* slowQueryName = "slow_query"; +const char* slowQueryName = "taos_slow_sql:count"; const char* slowQueryHelp = "slow query log when cost over than config duration"; const int slowQueryLabelCount = 4; const char* slowQueryLabels[] = {"cluster_id", "username", "result", "duration"}; From b552a16743405d61b76e6a7f3c20295e9177ff5a Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Fri, 2 Feb 2024 14:52:12 +0800 Subject: [PATCH 049/181] fix: monitor switch --- source/client/src/clientSqlMonitor.c | 2 ++ source/client/src/slowQueryMonitor.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientSqlMonitor.c b/source/client/src/clientSqlMonitor.c index 2f8aa1db5d..911d589096 100644 --- a/source/client/src/clientSqlMonitor.c +++ b/source/client/src/clientSqlMonitor.c @@ -24,6 +24,7 @@ const char* selectMonitorLabels[] = {"cluster_id", "sql_type", "username", "resu static const char* defaultClusterID = ""; void clusterSelectMonitorInit(const char* clusterKey) { + if (!tsEnableMonitor) return; SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey); SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter); @@ -36,6 +37,7 @@ void clusterSelectLog(const char* clusterKey, const char* user, SQL_RESULT_CODE } void selectLog(int64_t rid, bool killed, int32_t code) { + if (!tsEnableMonitor) return; SQL_RESULT_CODE result = SQL_RESULT_SUCCESS; if (TSDB_CODE_SUCCESS != code) { result = SQL_RESULT_FAILED; diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c index 0818aaeae4..98785205b9 100644 --- a/source/client/src/slowQueryMonitor.c +++ b/source/client/src/slowQueryMonitor.c @@ -42,7 +42,7 @@ static const char* getSlowQueryLableCostDesc(int64_t cost) { } void clusterSlowQueryMonitorInit(const char* clusterKey) { - if (!enableSlowQueryMonitor) return; + if (!tsEnableMonitor || !enableSlowQueryMonitor) return; SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey); SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter); @@ -55,7 +55,7 @@ void clusterSlowQueryLog(const char* clusterKey, const char* user, SQL_RESULT_CO } void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost) { - if (!enableSlowQueryMonitor) return; + if (!tsEnableMonitor || !enableSlowQueryMonitor) return; SQL_RESULT_CODE result = SQL_RESULT_SUCCESS; if (TSDB_CODE_SUCCESS != code) { result = SQL_RESULT_FAILED; From a72593e66dbecb755a9678b73df9ceb59777abc8 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Fri, 2 Feb 2024 14:54:31 +0800 Subject: [PATCH 050/181] fix: include head file --- source/client/src/clientSqlMonitor.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/client/src/clientSqlMonitor.c b/source/client/src/clientSqlMonitor.c index 911d589096..85458b6960 100644 --- a/source/client/src/clientSqlMonitor.c +++ b/source/client/src/clientSqlMonitor.c @@ -15,6 +15,7 @@ #include "clientInt.h" #include "clientMonitor.h" #include "clientLog.h" +#include "tglobal.h" const char* selectMonitorName = "taos_sql_req:count"; const char* selectMonitorHelp = "count for select sql"; From e0a1b886dfba4985c0b6a7c670915cae0ce4d0ba Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Fri, 2 Feb 2024 15:25:35 +0800 Subject: [PATCH 051/181] function rename --- source/client/inc/clientInt.h | 6 +++--- source/client/src/clientEnv.c | 2 +- source/client/src/clientImpl.c | 4 ++-- source/client/src/clientSqlMonitor.c | 12 ++++++------ source/client/src/slowQueryMonitor.c | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 257ee35aaf..8fe717a7fa 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -431,11 +431,11 @@ void freeQueryParam(SSyncQueryParam* param); int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effeciveUser, SParseSqlRes* pRes); #endif -void clusterSlowQueryMonitorInit(const char* clusterKey); +void clientSlowQueryMonitorInit(const char* clusterKey); void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost); -void clusterSelectMonitorInit(const char* clusterKey); -void selectLog(int64_t rid, bool killed, int32_t code); +void clientSQLReqMonitorInit(const char* clusterKey); +void sqlReqLog(int64_t rid, bool killed, int32_t code); #ifdef __cplusplus } diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index bd83f15e7e..d5523d6d7e 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -105,7 +105,7 @@ static void deregisterRequest(SRequestObj *pRequest) { pRequest->metric.planCostUs, pRequest->metric.execCostUs); atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); - selectLog(pTscObj->id, pRequest->killed, pRequest->code); + sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code); reqType = SLOW_LOG_TYPE_QUERY; } } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index a165ef6e88..200a940924 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -159,8 +159,8 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas pInst = &p; - clusterSlowQueryMonitorInit(p->instKey); - clusterSelectMonitorInit(p->instKey); + clientSlowQueryMonitorInit(p->instKey); + clientSQLReqMonitorInit(p->instKey); } else { ASSERTS((*pInst) && (*pInst)->pAppHbMgr, "*pInst:%p, pAppHgMgr:%p", *pInst, (*pInst) ? (*pInst)->pAppHbMgr : NULL); // reset to 0 in case of conn with duplicated user key but its user has ever been dropped. diff --git a/source/client/src/clientSqlMonitor.c b/source/client/src/clientSqlMonitor.c index 85458b6960..80375bafae 100644 --- a/source/client/src/clientSqlMonitor.c +++ b/source/client/src/clientSqlMonitor.c @@ -24,7 +24,7 @@ const char* selectMonitorLabels[] = {"cluster_id", "sql_type", "username", "resu static const char* defaultClusterID = ""; -void clusterSelectMonitorInit(const char* clusterKey) { +void clientSQLReqMonitorInit(const char* clusterKey) { if (!tsEnableMonitor) return; SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey); SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); @@ -32,12 +32,12 @@ void clusterSelectMonitorInit(const char* clusterKey) { createClusterCounter(clusterKey, selectMonitorName, selectMonitorHelp, selectMonitorLabelCount, selectMonitorLabels); } -void clusterSelectLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result) { +void clientSQLReqLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result) { const char* selectMonitorLabelValues[] = {defaultClusterID, "select", user, resultStr(result)}; taosClusterCounterInc(clusterKey, selectMonitorName, selectMonitorLabelValues); } -void selectLog(int64_t rid, bool killed, int32_t code) { +void sqlReqLog(int64_t rid, bool killed, int32_t code) { if (!tsEnableMonitor) return; SQL_RESULT_CODE result = SQL_RESULT_SUCCESS; if (TSDB_CODE_SUCCESS != code) { @@ -51,10 +51,10 @@ void selectLog(int64_t rid, bool killed, int32_t code) { STscObj* pTscObj = acquireTscObj(rid); if (pTscObj != NULL) { if (pTscObj->pAppInfo == NULL) { - tscLog("selectLog, not found pAppInfo"); + tscLog("sqlReqLog, not found pAppInfo"); } - return clusterSelectLog(pTscObj->pAppInfo->instKey, pTscObj->user, result); + return clientSQLReqLog(pTscObj->pAppInfo->instKey, pTscObj->user, result); } else { - tscLog("selectLog, not found rid"); + tscLog("sqlReqLog, not found rid"); } } diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c index 98785205b9..c4c623c7be 100644 --- a/source/client/src/slowQueryMonitor.c +++ b/source/client/src/slowQueryMonitor.c @@ -41,7 +41,7 @@ static const char* getSlowQueryLableCostDesc(int64_t cost) { return "< 3 s"; } -void clusterSlowQueryMonitorInit(const char* clusterKey) { +void clientSlowQueryMonitorInit(const char* clusterKey) { if (!tsEnableMonitor || !enableSlowQueryMonitor) return; SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey); SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); @@ -49,7 +49,7 @@ void clusterSlowQueryMonitorInit(const char* clusterKey) { createClusterCounter(clusterKey, slowQueryName, slowQueryHelp, slowQueryLabelCount, slowQueryLabels); } -void clusterSlowQueryLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int32_t cost) { +void clientSlowQueryLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int32_t cost) { const char* slowQueryLabelValues[] = {defaultClusterID, user, resultStr(result), getSlowQueryLableCostDesc(cost)}; taosClusterCounterInc(clusterKey, slowQueryName, slowQueryLabelValues); } @@ -70,7 +70,7 @@ void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost) { if(pTscObj->pAppInfo == NULL) { tscLog("SlowQueryLog, not found pAppInfo"); } - return clusterSlowQueryLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, cost); + return clientSlowQueryLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, cost); } else { tscLog("SlowQueryLog, not found rid"); } From 0159a046be049a113a1b651fd82bb01209747c19 Mon Sep 17 00:00:00 2001 From: dmchen Date: Sun, 4 Feb 2024 01:26:27 +0000 Subject: [PATCH 052/181] winows atomic --- source/os/src/osAtomic.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index e56cf629bb..c891ee4579 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -18,7 +18,8 @@ typedef union { volatile int64_t i; - double d; + volatile double d; + //double d; } double_number; #ifdef WINDOWS @@ -416,7 +417,7 @@ int64_t atomic_exchange_64(int64_t volatile* ptr, int64_t val) { double atomic_exchange_double(double volatile *ptr, int64_t val){ for (;;) { double_number old_num = {0}; - old_num.i = *ptr; // current old value + old_num.d = *ptr; // current old value double_number new_num = {0}; int64_t iNew = val; From c9fd8d2c7c2631a226b7c7ba32dfafc94dcb6224 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Sun, 4 Feb 2024 09:57:57 +0800 Subject: [PATCH 053/181] fix: test build failed --- source/client/test/clientMonitorTests.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/test/clientMonitorTests.cpp b/source/client/test/clientMonitorTests.cpp index 5272a2ab3a..f38ee74284 100644 --- a/source/client/test/clientMonitorTests.cpp +++ b/source/client/test/clientMonitorTests.cpp @@ -69,10 +69,10 @@ TEST(clientMonitorTest, sendTest) { printf("connect taosd sucessfully.\n"); int64_t rid = *(int64_t *)taos; - SlowQueryLog(rid, 1000); + SlowQueryLog(rid, false, -1, 1000); int i = 0; while (i < 20) { - SlowQueryLog(rid, i * 1000); + SlowQueryLog(rid, false, 0, i * 1000); taosMsleep(10); ++i; } From 2108f69c848e51d46e3c495322f07d3d37803045 Mon Sep 17 00:00:00 2001 From: cadem Date: Mon, 5 Feb 2024 07:32:15 +0800 Subject: [PATCH 054/181] client memory leak --- source/libs/monitor/src/clientMonitor.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/monitor/src/clientMonitor.c b/source/libs/monitor/src/clientMonitor.c index e3b49da858..4064f600b4 100644 --- a/source/libs/monitor/src/clientMonitor.c +++ b/source/libs/monitor/src/clientMonitor.c @@ -27,6 +27,7 @@ void generateClusterReport(ClientMonitor* pMonitor, bool send) { taos_collector_registry_clear_batch(pMonitor->registry); } } + taosMemoryFreeClear(pCont); } void reportSendProcess(void* param, void* tmrId) { From 71d9065d479e2163cf397dcf1ee598f3fe6b8e5a Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 5 Feb 2024 00:12:35 +0000 Subject: [PATCH 055/181] tmp log --- source/dnode/mnode/impl/src/mndDnode.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 50bc95f546..13a0d6cfcc 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -509,9 +509,9 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { goto _OVER; } - if(tsMonitorLogProtocol){ + //if(tsMonitorLogProtocol){ mInfo("process statis req,\n %s", statisReq.pCont); - } + //} SJson* pJson = tjsonParse(statisReq.pCont); From 16aef662603ad563084394a97803a19951eb0735 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 5 Feb 2024 01:34:20 +0000 Subject: [PATCH 056/181] ci case fail --- source/dnode/vnode/src/inc/vnodeInt.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index bc54a6b3c8..1aaab9e81d 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -106,7 +106,7 @@ typedef struct SQueryNode SQueryNode; #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" -#define VNODE_METRIC_SQL_COUNT "taos_sql_req:count" +#define VNODE_METRIC_SQL_COUNT "taos_sql_req:inserted_rows" #define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type" #define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id" @@ -116,7 +116,7 @@ typedef struct SQueryNode SQueryNode; #define VNODE_METRIC_TAG_NAME_USERNAME "username" #define VNODE_METRIC_TAG_NAME_RESULT "result" -#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "insert_affected_rows" +#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows" //#define VNODE_METRIC_TAG_VALUE_INSERT "insert" #define VNODE_METRIC_TAG_VALUE_DELETE "delete" From 5c7f839da8a8a2408d6968e661aed67042ce8c14 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 5 Feb 2024 03:28:33 +0000 Subject: [PATCH 057/181] remove tmp log --- source/dnode/mnode/impl/src/mndDnode.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 13a0d6cfcc..50bc95f546 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -509,9 +509,9 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { goto _OVER; } - //if(tsMonitorLogProtocol){ + if(tsMonitorLogProtocol){ mInfo("process statis req,\n %s", statisReq.pCont); - //} + } SJson* pJson = tjsonParse(statisReq.pCont); From 6c9bfb8f62b108174cebc418a568b2dbac0ee372 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 5 Feb 2024 15:18:09 +0800 Subject: [PATCH 058/181] fix: release obj reference --- source/client/src/clientSqlMonitor.c | 4 +++- source/client/src/slowQueryMonitor.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientSqlMonitor.c b/source/client/src/clientSqlMonitor.c index 80375bafae..7c54eb7b01 100644 --- a/source/client/src/clientSqlMonitor.c +++ b/source/client/src/clientSqlMonitor.c @@ -52,8 +52,10 @@ void sqlReqLog(int64_t rid, bool killed, int32_t code) { if (pTscObj != NULL) { if (pTscObj->pAppInfo == NULL) { tscLog("sqlReqLog, not found pAppInfo"); + } else { + clientSQLReqLog(pTscObj->pAppInfo->instKey, pTscObj->user, result); } - return clientSQLReqLog(pTscObj->pAppInfo->instKey, pTscObj->user, result); + releaseTscObj(rid); } else { tscLog("sqlReqLog, not found rid"); } diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c index c4c623c7be..61add665dc 100644 --- a/source/client/src/slowQueryMonitor.c +++ b/source/client/src/slowQueryMonitor.c @@ -69,8 +69,10 @@ void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost) { if (pTscObj != NULL) { if(pTscObj->pAppInfo == NULL) { tscLog("SlowQueryLog, not found pAppInfo"); + } else { + clientSlowQueryLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, cost); } - return clientSlowQueryLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, cost); + releaseTscObj(rid); } else { tscLog("SlowQueryLog, not found rid"); } From 930158088625580c6759393efae6f074b5ddbcf6 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 6 Feb 2024 13:06:56 +0800 Subject: [PATCH 059/181] fix: free mem in callback --- source/client/inc/clientInt.h | 2 + source/client/src/clientEnv.c | 1 + source/client/src/clientMain.c | 1 + source/client/src/clientSqlMonitor.c | 5 + source/client/test/clientMonitorTests.cpp | 3 +- source/libs/monitor/src/clientMonitor.c | 8 +- tests/system-test/2-query/stable_1.csv | 364 ++++++++++++++++++ .../2-query/stable_null_childtable.csv | 64 +++ .../system-test/2-query/stable_null_data.csv | 64 +++ 9 files changed, 509 insertions(+), 3 deletions(-) create mode 100644 tests/system-test/2-query/stable_1.csv create mode 100644 tests/system-test/2-query/stable_null_childtable.csv create mode 100644 tests/system-test/2-query/stable_null_data.csv diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 8fe717a7fa..246d86ca54 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -437,6 +437,8 @@ void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost); void clientSQLReqMonitorInit(const char* clusterKey); void sqlReqLog(int64_t rid, bool killed, int32_t code); +void clientMonitorClose(const char* clusterKey); + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index d5523d6d7e..93ee530b41 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -227,6 +227,7 @@ void destroyAppInst(SAppInstInfo *pAppInfo) { taosThreadMutexLock(&appInfo.mutex); + clientMonitorClose(pAppInfo->instKey); hbRemoveAppHbMrg(&pAppInfo->pAppHbMgr); taosHashRemove(appInfo.pInstMap, pAppInfo->instKey, strlen(pAppInfo->instKey)); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index b99654172c..4a80aba971 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -279,6 +279,7 @@ void taos_close_internal(void *taos) { STscObj *pTscObj = (STscObj *)taos; tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs); + // clientMonitorClose(pTscObj->pAppInfo->instKey); taosRemoveRef(clientConnRefPool, pTscObj->id); } diff --git a/source/client/src/clientSqlMonitor.c b/source/client/src/clientSqlMonitor.c index 7c54eb7b01..89b91646ff 100644 --- a/source/client/src/clientSqlMonitor.c +++ b/source/client/src/clientSqlMonitor.c @@ -60,3 +60,8 @@ void sqlReqLog(int64_t rid, bool killed, int32_t code) { tscLog("sqlReqLog, not found rid"); } } + +void clientMonitorClose(const char* clusterKey) { + tscLog("clientMonitorClose, key:%s", clusterKey); + clusterMonitorClose(clusterKey); +} diff --git a/source/client/test/clientMonitorTests.cpp b/source/client/test/clientMonitorTests.cpp index f38ee74284..f7ddc1a5cd 100644 --- a/source/client/test/clientMonitorTests.cpp +++ b/source/client/test/clientMonitorTests.cpp @@ -59,7 +59,8 @@ TEST(clientMonitorTest, monitorTest) { while (i < 12) { taosMsleep(10); ++i; - } clusterMonitorClose(cluster1); + } + clusterMonitorClose(cluster1); clusterMonitorClose(cluster2); } diff --git a/source/libs/monitor/src/clientMonitor.c b/source/libs/monitor/src/clientMonitor.c index 4064f600b4..0891932583 100644 --- a/source/libs/monitor/src/clientMonitor.c +++ b/source/libs/monitor/src/clientMonitor.c @@ -80,7 +80,7 @@ void createMonitorClient(const char* clusterKey, SEpSet epSet, void* pTransporte snprintf(pMonitor->clusterKey, sizeof(pMonitor->clusterKey), "%s", clusterKey); pMonitor->registry = taos_collector_registry_new(clusterKey); pMonitor->colector = taos_collector_new(clusterKey); - pMonitor->epSet = epSet; + epsetAssign(&pMonitor->epSet, &epSet); pMonitor->pTransporter = pTransporter; taos_collector_registry_register_collector(pMonitor->registry, pMonitor->colector); @@ -98,6 +98,10 @@ static int32_t monitorReportAsyncCB(void* param, SDataBuf* pMsg, int32_t code) { if (TSDB_CODE_SUCCESS != code) { uError("found error in monitorReport send callback, code:%d, please check the network.", code); } + if (pMsg) { + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); + } return code; } @@ -174,7 +178,7 @@ int taosClusterCounterInc(const char* clusterKey, const char* counterName, const ClientMonitor* pMonitor = *ppMonitor; taos_counter_t** ppCounter = (taos_counter_t**)taosHashGet(pMonitor->counters, counterName, strlen(counterName)); if (ppCounter != NULL && *ppCounter != NULL) { - int res = taos_counter_inc(*ppCounter, label_values); + taos_counter_inc(*ppCounter, label_values); } else { uError("taosClusterCounterInc not found pCounter %s:%s.", clusterKey, counterName); } diff --git a/tests/system-test/2-query/stable_1.csv b/tests/system-test/2-query/stable_1.csv new file mode 100644 index 0000000000..7b0587f527 --- /dev/null +++ b/tests/system-test/2-query/stable_1.csv @@ -0,0 +1,364 @@ +tbname,ts,q_int,q_binary +"stable_1_1","2021-08-27 01:46:40.000",-143708439,"binary.wEvcyPTYhxxKhVFrShTZ" +"stable_1_1","2021-08-27 05:56:40.000",-2044434228,"binary.iVmwRqGqAbGImLpoUiKX" +"stable_1_1","2021-08-27 10:06:40.000",1109067209,"binary.mJPDrXToNeEldaahHrjD" +"stable_1_1","2021-08-27 14:16:40.000",1148326681,"binary.NHZMOcXRwhSPvmIIxdxp" +"stable_1_1","2021-08-27 18:26:40.000",723117353,"binary.TublFbdwZUkjDXBnzoCS" +"stable_1_1","2021-08-27 22:36:40.000",-1735114764,"binary.TCDlFpIyMpXfcghfWlUe" +"stable_1_1","2021-08-28 02:46:40.000",1973623075,"binary.xkRVpVawmLMQlKCdvZtZ" +"stable_1_1","2021-08-28 06:56:40.000",1693347900,"binary.CRmkBadMfWkcImouLOYC" +"stable_1_1","2021-08-28 11:06:40.000",1041114692,"binary.FplIhrrnZDEqcPpMXcCz" +"stable_1_1","2021-08-28 15:16:40.000",-1384036824,"binary.NpfAGysaedqwHzyjOEhb" +"stable_1_1","2021-08-28 19:26:40.000",1841425030,"binary.wpRXidgNPOERhEaZlJAM" +"stable_1_1","2021-08-28 23:36:40.000",-270068972,"binary.FNcyuyrPVVVfxbzDuGlf" +"stable_1_1","2021-08-29 03:46:40.000",418330248,"binary.KevLpYjpMOfugEsBCBlm" +"stable_1_1","2021-08-29 07:56:40.000",-1261626278,"binary.DmaGJbQkLmJQaURuaowv" +"stable_1_1","2021-08-29 12:06:40.000",665794589,"binary.IKfpbOnsEUqZvDUDXwxE" +"stable_1_1","2021-08-29 16:16:40.000",1556546304,"binary.GKOAUmTHRsUYNGpJuJXS" +"stable_1_1","2021-08-29 20:26:40.000",-635394652,"binary.UPKoftKIqXYBGkgijulk" +"stable_1_1","2021-08-30 00:36:40.000",1654794184,"binary.QHlEvczXvDhrjeXKechh" +"stable_1_1","2021-08-30 04:46:40.000",41007230,"binary.wuOctyBeTsCZynkMWYzu" +"stable_1_1","2021-08-30 08:56:40.000",1387304128,"binary.QwtMztxuEtuQdccEPSgc" +"stable_1_1","2021-08-30 13:06:40.000",-1036308321,"binary.CHRMHcuiDKIrbquzMWnc" +"stable_1_1","2021-08-30 17:16:40.000",-1768408453,"binary.OaHLnnqkcUEKxOOHCYCJ" +"stable_1_1","2021-08-30 21:26:40.000",125729034,"binary.YAKBKrKRqlgNweLacnpB" +"stable_1_1","2021-08-31 01:36:40.000",-125236222,"binary.UJkpaKsbERScbXdQuhvo" +"stable_1_1","2021-08-31 05:46:40.000",-677274672,"binary.jwdoZKFVRIbdNbxvzglZ" +"stable_1_2","2021-08-27 01:46:39.999",1048090051,"binary.QFXcjYxRIgvVcaabbAuo" +"stable_1_2","2021-08-27 01:46:40.001",-115380468,"binary.ilLTZcPTGRSwIjACLGlM" +"stable_1_2","2021-08-27 05:56:39.999",1095266727,"binary.NMQGxjbGhMVVbcQEBaEK" +"stable_1_2","2021-08-27 05:56:40.001",-1664825220,"binary.LZlUlCryyKuhhwGGlSjc" +"stable_1_2","2021-08-27 10:06:39.999",1270783858,"binary.GqXavHpdBwhbCDrQnpSF" +"stable_1_2","2021-08-27 10:06:40.001",-1718628594,"binary.wPOmeglMuNsatgqVOYvh" +"stable_1_2","2021-08-27 14:16:39.999",781814072,"binary.LBBuZbsGbCPwrtpgpJUU" +"stable_1_2","2021-08-27 14:16:40.001",-222510400,"binary.BgnCkJAICiNgxqiaowKU" +"stable_1_2","2021-08-27 18:26:39.999",260526220,"binary.uSifMBsrTaQhLuYRzSNB" +"stable_1_2","2021-08-27 18:26:40.001",-1734497021,"binary.UnQSdMFNesxovxaQqywB" +"stable_1_2","2021-08-27 22:36:39.999",856956546,"binary.ctAKBcRCfQQDgayszuAG" +"stable_1_2","2021-08-27 22:36:40.001",-2035071163,"binary.evhulVVFzQzmcxSxrhKQ" +"stable_1_2","2021-08-28 02:46:39.999",2024548336,"binary.HAdhzWKQINYLaYkXNWMA" +"stable_1_2","2021-08-28 02:46:40.001",-1388002413,"binary.uYfWFNKpQafMupBPxsaQ" +"stable_1_2","2021-08-28 06:56:39.999",896614799,"binary.eLMnpehchKlvWYooNwzT" +"stable_1_2","2021-08-28 06:56:40.001",-449300050,"binary.qKdRIooLiEPovATVrIjv" +"stable_1_2","2021-08-28 11:06:39.999",640308875,"binary.OzGnekdKxbZxbKgGFJlg" +"stable_1_2","2021-08-28 11:06:40.001",-1961834598,"binary.FbQxRmecVULCPbyUFBtd" +"stable_1_2","2021-08-28 15:16:39.999",152017229,"binary.CoERNXcHahySItezBpen" +"stable_1_2","2021-08-28 15:16:40.001",-873242705,"binary.LLhLPAXTmmXelsILHdQZ" +"stable_1_2","2021-08-28 19:26:39.999",7371013,"binary.OtbSjvYZOPWbIQOdIrKG" +"stable_1_2","2021-08-28 19:26:40.001",-259884599,"binary.ILwvbvFGWLKDelTSCcDm" +"stable_1_2","2021-08-28 23:36:39.999",1060176497,"binary.cZSBdkTCfHzKdaUbRObM" +"stable_1_2","2021-08-28 23:36:40.001",-1895192712,"binary.CYsQdbzmYnNmadYZeaGH" +"stable_1_2","2021-08-29 03:46:39.999",778869650,"binary.HYiRCyeEqGzFGwoHupCH" +"stable_1_2","2021-08-29 03:46:40.001",-1070149378,"binary.VZKQmmtwMfYTGzlJOyzv" +"stable_1_2","2021-08-29 07:56:39.999",2139522991,"binary.WgIELLWDqNqWvdsblKCB" +"stable_1_2","2021-08-29 07:56:40.001",-1450415363,"binary.THXDnYCLdvuTXRSUsMnH" +"stable_1_2","2021-08-29 12:06:39.999",1202083307,"binary.jzdrBvoIOBydYtlFDBfZ" +"stable_1_2","2021-08-29 12:06:40.001",-808169724,"binary.sUZnozuiMlgMQjgaKLWT" +"stable_1_2","2021-08-29 16:16:39.999",202190259,"binary.QFvJnTjavOlzsXAEZVwH" +"stable_1_2","2021-08-29 16:16:40.001",-409805763,"binary.sbSiqKmjPKvnXbSeKwXG" +"stable_1_2","2021-08-29 20:26:39.999",238657263,"binary.OgrrbsocdxzSAsMKmGFW" +"stable_1_2","2021-08-29 20:26:40.001",-1447960279,"binary.gMIOOqmPvHzbiBiPEVsv" +"stable_1_2","2021-08-30 00:36:39.999",954795210,"binary.VOdZRucmuTuXzTGKuWfA" +"stable_1_2","2021-08-30 00:36:40.001",-185956520,"binary.LsRcBtiQqicggCLXZoWe" +"stable_1_2","2021-08-30 04:46:39.999",1365142146,"binary.EnWxuBUlakKcHrXrPBOj" +"stable_1_2","2021-08-30 04:46:40.001",-210822579,"binary.ZKvMAWaDuBEInoXJgQSL" +"stable_1_2","2021-08-30 08:56:39.999",952122349,"binary.lcXghoPWtfUTfSAAJDPS" +"stable_1_2","2021-08-30 08:56:40.001",-1152099753,"binary.vtvGwZIEcxVUBWDFVniL" +"stable_1_2","2021-08-30 13:06:39.999",1198995002,"binary.CobYIOXzviHBQmtndNfY" +"stable_1_2","2021-08-30 13:06:40.001",-1498342366,"binary.qfSLXCSfQexiQHdraTYY" +"stable_1_2","2021-08-30 17:16:39.999",388708472,"binary.VWdcyMxbxjRZCcDesjWH" +"stable_1_2","2021-08-30 17:16:40.001",-1688761021,"binary.fsNWygGviERgbHDCTNoU" +"stable_1_2","2021-08-30 21:26:39.999",1046617386,"binary.OmMMfoOWmDphnVXsqVny" +"stable_1_2","2021-08-30 21:26:40.001",-509016842,"binary.miOrdOLIjNjowuqZLMJY" +"stable_1_2","2021-08-31 01:36:39.999",1470657859,"binary.QLjYhzfGDKcPmHFQjBub" +"stable_1_2","2021-08-31 01:36:40.001",-720454315,"binary.GaEJJOwaHtRTijYNSKrt" +"stable_1_2","2021-08-31 05:46:39.999",1874112564,"binary.tNjvCgdWlmcOHVbLJRMN" +"stable_1_2","2021-08-31 05:46:40.001",-805620343,"binary.rMWNLTrygVpKqbvfAPEQ" +"stable_1_2","2021-08-31 09:56:39.999",1624260248,"binary.OMehRclQHcJBTynDoScY" +"stable_1_2","2021-08-31 09:56:40.001",-321755377,"binary.pYwaFBbMgRmreHVLLqHM" +"stable_1_2","2021-08-31 14:06:39.999",1338486910,"binary.PewhoryOLSZsZOExXrsO" +"stable_1_2","2021-08-31 14:06:40.001",-1972396465,"binary.HCciTUVnuKaUQHOTLmoy" +"stable_1_2","2021-08-31 18:16:39.999",586219277,"binary.ejuJVeKrOYTBIOGtnvwN" +"stable_1_2","2021-08-31 18:16:40.001",-23773554,"binary.HMzLCwXDnCBrSjyvRiij" +"stable_1_2","2021-08-31 22:26:39.999",403592186,"binary.KbgXOhPmkagxFsFbQakf" +"stable_1_2","2021-08-31 22:26:40.001",-1017841441,"binary.BONmREpjVojUyNnkKVDr" +"stable_1_2","2021-09-01 02:36:39.999",2123582595,"binary.JqrmmSVJyyJSphzDfdfF" +"stable_1_2","2021-09-01 02:36:40.001",-541998051,"binary.DAHIBWzujRgGmXHITFdP" +"stable_1_2","2021-09-01 06:46:39.999",863783720,"binary.iZrgQLIwNNvEhiErAwkt" +"stable_1_2","2021-09-01 06:46:40.001",-1507574370,"binary.nVGhmvtMlkPpeaBpAFTM" +"stable_1_2","2021-09-01 10:56:39.999",1652311395,"binary.GFGXaNXjaQWBuiuIuflv" +"stable_1_2","2021-09-01 10:56:40.001",-881848122,"binary.WKsWUZoBjTlKVGgAJxPx" +"stable_1_2","2021-09-01 15:06:39.999",133942525,"binary.jFkkgXUPSCrZnMUmiDca" +"stable_1_2","2021-09-01 15:06:40.001",-1161482672,"binary.TMzTATElhmlhyqFvwFKJ" +"stable_1_2","2021-09-01 19:16:39.999",1748308690,"binary.bsqrWOhUgZliFzqtMCyZ" +"stable_1_2","2021-09-01 19:16:40.001",-45395742,"binary.SEnorgonMIgtUKkNTOxP" +"stable_1_2","2021-09-01 23:26:39.999",59717021,"binary.XBJIPnZODnweOOXmzRkO" +"stable_1_2","2021-09-01 23:26:40.001",-369410792,"binary.HBmqhBQjhNYwAaLJWbpE" +"stable_1_2","2021-09-02 03:36:39.999",1042899672,"binary.SyRZdWnAthyOUqpOtOos" +"stable_1_2","2021-09-02 03:36:40.001",-1784712756,"binary.FsBnuFMAwIdTjEjYkjBR" +"stable_1_2","2021-09-02 07:46:39.999",656087591,"binary.vsIjUPiZNVxiFWHzGqkg" +"stable_1_2","2021-09-02 07:46:40.001",-70790307,"binary.GLAJYWukLZueieYRsidQ" +"stable_1_2","2021-09-02 11:56:39.999",396390446,"binary.pgBGZOKYSliCxHFmGsyp" +"stable_1_2","2021-09-02 11:56:40.001",-1965530504,"binary.ECLVeNvkmqcTHEoSSxYT" +"stable_1_2","2021-09-02 16:06:39.999",554982384,"binary.zpGkiHSJABZuoZlpNXMp" +"stable_1_2","2021-09-02 16:06:40.001",-2032946157,"binary.lyLVozekCioDlqntmATw" +"stable_1_2","2021-09-02 20:16:39.999",677947760,"binary.vfXYvrHXFnDSAHwCndcp" +"stable_1_2","2021-09-02 20:16:40.001",-494862894,"binary.zffLjgnXcDlwdKAxADMr" +"stable_1_2","2021-09-03 00:26:39.999",1558178384,"binary.bKLXWDPaBjWRXgOeesiD" +"stable_1_2","2021-09-03 00:26:40.001",-475557451,"binary.wnZulRregqgognSRRYWI" +"stable_1_2","2021-09-03 04:36:39.999",632223220,"binary.JqnabMfPCrzsTaYczTLh" +"stable_1_2","2021-09-03 04:36:40.001",-951785691,"binary.sKKfLrkShxlGFGromDFZ" +"stable_1_2","2021-09-03 08:46:39.999",826686189,"binary.shGShhPNnWdZQDAdbiHo" +"stable_1_2","2021-09-03 08:46:40.001",-667795153,"binary.OaUjyowfAGaGXKoVLWZR" +"stable_1_2","2021-09-03 12:56:39.999",1420828021,"binary.AYcgokdEsFMUOIcUpxnM" +"stable_1_2","2021-09-03 12:56:40.001",-2009472018,"binary.JpOZwpoVKCDjiqsFhbaS" +"stable_1_2","2021-09-03 17:06:39.999",395324903,"binary.vWVYYMXIWQAEHBTKKWjP" +"stable_1_2","2021-09-03 17:06:40.001",-9754404,"binary.QNAVDZqImSFgnHpVdWtS" +"stable_1_2","2021-09-03 21:16:39.999",2068284680,"binary.wNSWJmuVBjpRVVCpksQC" +"stable_1_2","2021-09-03 21:16:40.001",-853667209,"binary.OgZwwWPZuNLXAAdBcanX" +"stable_1_2","2021-09-04 01:26:39.999",155983724,"binary.MRJjXYWHBZvzaAjTzrli" +"stable_1_2","2021-09-04 01:26:40.001",-1760453704,"binary.pARRcJciJQzdzcCZRqZA" +"stable_1_2","2021-09-04 05:36:39.999",570191599,"binary.VVxoQGPpWrOYSXzSxWlM" +"stable_1_2","2021-09-04 05:36:40.001",-1289549437,"binary.mtOMrfEeDMegNqMauPgK" +"stable_1_2","2021-09-04 09:46:39.999",1998845359,"binary.IROrsLaBEXcULJZvtXDS" +"stable_1_2","2021-09-04 09:46:40.001",-1746905381,"binary.WatcMYFnezOmZtsAGUDK" +"stable_1_2","2021-09-04 13:56:39.999",2123054685,"binary.cRdRGuitRXDuGAWVDxAE" +"stable_1_2","2021-09-04 13:56:40.001",-45388705,"binary.HMSrCuTIHQiHmuAoGqJK" +"stable_1_2","2021-09-04 18:06:39.999",422908755,"binary.xeqmTUMylYPDFkFJGLQY" +"stable_1_2","2021-09-04 18:06:40.001",-1902203841,"binary.dEfjmLzekcjvysMUToXa" +"stable_1_2","2021-09-04 22:16:39.999",1335747951,"binary.ugUzQSUcPjEXMIDrFobB" +"stable_1_2","2021-09-04 22:16:40.001",-1552553924,"binary.gZMscjyRXoGyNsFLHpoR" +"stable_1_2","2021-09-05 02:26:39.999",1092904458,"binary.IAiAMcGMqeACFMkfgGju" +"stable_1_2","2021-09-05 02:26:40.001",-437886009,"binary.jWguWEtBAUgcMKtgXieL" +"stable_1_2","2021-09-05 06:36:39.999",433850999,"binary.QBIulMvNBsGMfStAKgCQ" +"stable_1_2","2021-09-05 06:36:40.001",-1219816305,"binary.aSnbvFgLheGIahlWoBEf" +"stable_1_2","2021-09-05 10:46:39.999",1771552222,"binary.YdwgZmlYfeivmxtaWueZ" +"stable_1_2","2021-09-05 10:46:40.001",-429657495,"binary.yGIWxOkDtQCQgbPJhHFi" +"stable_1_2","2021-09-05 14:56:39.999",1560886565,"binary.OlTzMGFNSfSrcixpefQx" +"stable_1_2","2021-09-05 14:56:40.001",-2082927804,"binary.FTNnvEphxFByhzkfmAhb" +"stable_1_2","2021-09-05 19:06:39.999",312047911,"binary.yaVlkhtTxxgrtZTsFdxa" +"stable_1_2","2021-09-05 19:06:40.001",-1417705247,"binary.tHnemGaRugDffcmoyWli" +"stable_1_2","2021-09-05 23:16:39.999",1667164436,"binary.NdSzaJxdoQPGYefWPJyi" +"stable_1_2","2021-09-05 23:16:40.001",-2145233720,"binary.jghuIXEGcUtKikJpzLOh" +"stable_1_2","2021-09-06 03:26:39.999",488072919,"binary.RLQFgpAcQCfhuZExHLkx" +"stable_1_2","2021-09-06 03:26:40.001",-1943550230,"binary.ozdeiPLKjSgywqNpCBYN" +"stable_1_2","2021-09-06 07:36:39.999",724025582,"binary.XZmZYlSRZZddcxEOjfEd" +"stable_1_2","2021-09-06 07:36:40.001",-2130070769,"binary.qLvSSOSazLTUKvqpCCBL" +"stable_1_2","2021-09-06 11:46:39.999",941556630,"binary.WrcojJGdsiZyAaRxYrdw" +"stable_1_2","2021-09-06 11:46:40.001",-343938435,"binary.HoqLkegpacFGqzFWeVXL" +"stable_1_2","2021-09-06 15:56:39.999",462886464,"binary.BRImBBEftaaKZeOGRSvW" +"stable_1_2","2021-09-06 15:56:40.001",-674623669,"binary.XevOOqIPqCmaJUcXaaKz" +"stable_1_2","2021-09-06 20:06:39.999",1589240480,"binary.wJyZOXguJRYIbgIuIRye" +"stable_1_2","2021-09-06 20:06:40.001",-253589870,"binary.CzIxfDKtViliFCPuYkSq" +"stable_1_2","2021-09-07 00:16:39.999",701182235,"binary.wFVimJeupABVGEvwMiTv" +"stable_1_2","2021-09-07 00:16:40.001",-1563988450,"binary.OjWhbhEvRlHpqFEHXmfR" +"stable_1_2","2021-09-07 04:26:39.999",1631111972,"binary.QZSHkvsYmRHnGFnyMije" +"stable_1_2","2021-09-07 04:26:40.001",-1081644320,"binary.TDrOTkriDBpZtXBadhIx" +"stable_1_2","2021-09-07 08:36:39.999",1021775776,"binary.AstGPfaVxEefogcWzDzq" +"stable_1_2","2021-09-07 08:36:40.001",-1715903601,"binary.lSSCcLFElDWxKZTdHRLN" +"stable_1_2","2021-09-07 12:46:39.999",1492685945,"binary.esYXdqdUGVFhLCHHciWA" +"stable_1_2","2021-09-07 12:46:40.001",-457991373,"binary.OCGshioisqgLTUFLRMSc" +"stable_1_2","2021-09-07 16:56:39.999",306491871,"binary.WPbbRgxYIRecrOFSlRYM" +"stable_1_2","2021-09-07 16:56:40.001",-709883988,"binary.cADQdfbZkXfcNPPOPQwp" +"stable_1_2","2021-09-07 21:06:39.999",1639981656,"binary.mVTonpWtYFauqLNntbcK" +"stable_1_2","2021-09-07 21:06:40.001",-40134567,"binary.hSZKVixlyoxpHpdvwYxY" +"stable_1_2","2021-09-08 01:16:39.999",1412945598,"binary.EPkNzxzaeCAWQetzzzlv" +"stable_1_2","2021-09-08 01:16:40.001",-517552216,"binary.AmPftzlwURAILqlHhYtv" +"stable_1_2","2021-09-08 05:26:39.999",1210716085,"binary.xPKETDcCdbWtoYoUCgre" +"stable_1_2","2021-09-08 05:26:40.001",-1186882087,"binary.HVKExGKBzdrjjQbKdgim" +"stable_1_2","2021-09-08 09:36:39.999",1209625354,"binary.xDqLioOjrUudmVTRaKmP" +"stable_1_2","2021-09-08 09:36:40.001",-1049869765,"binary.ZcJrqAQpBfTRFJupVZDH" +"stable_1_2","2021-09-08 13:46:39.999",482971664,"binary.ZumfNrRUgHCXCqqjBwgX" +"stable_1_2","2021-09-08 13:46:40.001",-1695698428,"binary.VzjkizbqoeqlOEtsqzTd" +"stable_1_2","2021-09-08 17:56:39.999",369111551,"binary.hBVqWjfnRCktdjBpGTYb" +"stable_1_2","2021-09-08 17:56:40.001",-1324385506,"binary.aBZiPEKTvPqdXPsDXgvy" +"stable_1_2","2021-09-08 22:06:39.999",1370148713,"binary.ApKvhLsWkpRndmodhPEN" +"stable_1_2","2021-09-08 22:06:40.001",-1440742963,"binary.nsuhquhIJtXKINhuhbZm" +"stable_1_2","2021-09-09 02:16:39.999",1951461498,"binary.hZDumWlAzVpYfXkEaPlW" +"stable_1_2","2021-09-09 02:16:40.001",-1663211983,"binary.wBVWJVLFoOdXTzTtRNuT" +"stable_1_2","2021-09-09 06:26:39.999",2092906691,"binary.hLAAdJsauemmUBKhTtqW" +"stable_1_2","2021-09-09 06:26:40.001",-2137737633,"binary.wRfxAxVfEcdfKteGLuaX" +"stable_1_2","2021-09-09 10:36:39.999",870002160,"binary.JCROwpmJQkPqZkIzrRkU" +"stable_1_2","2021-09-09 10:36:40.001",-837033747,"binary.YKouegacQvxoSSgTaPrv" +"stable_1_2","2021-09-09 14:46:39.999",1397781824,"binary.hwfhcmJDDTvQaCRmlKCd" +"stable_1_2","2021-09-09 14:46:40.001",-1573224659,"binary.QpcyRVyxqqnHSSLxWCdz" +"stable_1_2","2021-09-09 18:56:39.999",1354974803,"binary.mgwWzOiTRoTHUbvzEMSm" +"stable_1_2","2021-09-09 18:56:40.001",-1306330996,"binary.xazZjdXNjaykPYfDWhos" +"stable_1_2","2021-09-09 23:06:39.999",1089140558,"binary.tqyfrLsfYAZeSvFCwYXQ" +"stable_1_2","2021-09-09 23:06:40.001",-2070609011,"binary.HtBovhAaClqmUKDVoPoA" +"stable_1_2","2021-09-10 03:16:39.999",243460896,"binary.GwEkcVzBVjqKKvBBYCHl" +"stable_1_2","2021-09-10 03:16:40.001",-442417658,"binary.aMVpynzemnksvKTWGcDC" +"stable_1_2","2021-09-10 07:26:39.999",993462171,"binary.akkJQvtKVOUMeEJdtLWQ" +"stable_1_2","2021-09-10 07:26:40.001",-1147877996,"binary.UsylaHHTjnwUCmWOFRop" +"stable_1_1","2021-08-31 09:56:40.000",1859005003,"binary.uDiGDbJBCeWVuQCYGwQU" +"stable_1_1","2021-08-31 14:06:40.000",620402706,"binary.kcxyJbYKMUthZCemDZwN" +"stable_1_1","2021-08-31 18:16:40.000",-697189031,"binary.mjvvuLgBZRyYZvSnmrpW" +"stable_1_1","2021-08-31 22:26:40.000",-1223538957,"binary.YhGtRmitXcCooBPlNxlh" +"stable_1_1","2021-09-01 02:36:40.000",-838875167,"binary.AXEKjEPNWZzTiqwFpkrg" +"stable_1_1","2021-09-01 06:46:40.000",-765134201,"binary.uytkzHZmorAgiDiISVoG" +"stable_1_1","2021-09-01 10:56:40.000",-314548498,"binary.zPUYZbaXGMPJOywgECDZ" +"stable_1_1","2021-09-01 15:06:40.000",780205345,"binary.dyNGyigdTXCBLJSGKHrJ" +"stable_1_1","2021-09-01 19:16:40.000",-179608812,"binary.VMzHtbnrmjokPDZCMXsk" +"stable_1_1","2021-09-01 23:26:40.000",-138813503,"binary.cvjJVfNMFXXyaKDjkvRu" +"stable_1_1","2021-09-02 03:36:40.000",-319429462,"binary.INFGvqMFdDTmdeFePZnQ" +"stable_1_1","2021-09-02 07:46:40.000",-298407008,"binary.ligoRoVSAHzFhUvIDwlE" +"stable_1_1","2021-09-02 11:56:40.000",632563376,"binary.RsQnCVXQtzoeXuamUiBl" +"stable_1_1","2021-09-02 16:06:40.000",446510409,"binary.FEewTpfZgngFmPSTSvxJ" +"stable_1_1","2021-09-02 20:16:40.000",640199308,"binary.bRHphFiZkeWtyDjXqNaO" +"stable_1_1","2021-09-03 00:26:40.000",790711145,"binary.uCOCqMGewUkNeUhAbNNf" +"stable_1_1","2021-09-03 04:36:40.000",1664157316,"binary.AyZCLhYiMxZgfYXlVciw" +"stable_1_1","2021-09-03 08:46:40.000",-1983928412,"binary.CaPFXqhDLSOaNmNnOsLh" +"stable_1_1","2021-09-03 12:56:40.000",-412013965,"binary.qEwLmiFwCrLZumFHOYjb" +"stable_1_1","2021-09-03 17:06:40.000",-1579595934,"binary.xUUoAUpZeJJsquiJOKsO" +"stable_1_1","2021-09-03 21:16:40.000",-98786523,"binary.XQfGXHbnwGQPulAcrAPX" +"stable_1_1","2021-09-04 01:26:40.000",-287136285,"binary.OdsJNUSWeVvTxQdKykmk" +"stable_1_1","2021-09-04 05:36:40.000",1051015641,"binary.HfnUyPAwYxEHQotoCfFT" +"stable_1_1","2021-09-04 09:46:40.000",-1161611813,"binary.jEENQpaYOWcVHUYKPaPn" +"stable_1_1","2021-09-04 13:56:40.000",-1162065386,"binary.ltmEKwzkvDqQIZWAxJtQ" +"stable_1_1","2021-09-04 18:06:40.000",1289374504,"binary.SHhbtXRCqxYOqQcWUDtU" +"stable_1_1","2021-09-04 22:16:40.000",-115300759,"binary.CowDUEaCtAEdafPCLLwM" +"stable_1_1","2021-09-05 02:26:40.000",-97528818,"binary.pqfTTMuvVmSCfxCmFTwS" +"stable_1_1","2021-09-05 06:36:40.000",-738727537,"binary.zWAWphEtCuSMACUmkfvN" +"stable_1_1","2021-09-05 10:46:40.000",1063250813,"binary.XwbrXvujPArWLXYhQRQK" +"stable_1_1","2021-09-05 14:56:40.000",1579885922,"binary.FEXGjykuWeGEfkKImWDw" +"stable_1_1","2021-09-05 19:06:40.000",-714663691,"binary.tdLqigCoasQnRffCEaDD" +"stable_1_1","2021-09-05 23:16:40.000",1437018826,"binary.XZcaYfmvLZtxhrmzHWEJ" +"stable_1_1","2021-09-06 03:26:40.000",1811347552,"binary.riDaPlhuskGRZsNxzooI" +"stable_1_1","2021-09-06 07:36:40.000",-2123493348,"binary.ZKZunxmtjBCRRCZBHTsL" +"stable_1_1","2021-09-06 11:46:40.000",2108150002,"binary.eEaxRyDoGrrYZQfYhDwC" +"stable_1_1","2021-09-06 15:56:40.000",-1893993866,"binary.bMuXKucyDJjBggYIIAvd" +"stable_1_1","2021-09-06 20:06:40.000",-1494759877,"binary.GGbtLZVCzfmnUdMHExqv" +"stable_1_1","2021-09-07 00:16:40.000",2135168110,"binary.ICfwWUnCEvXrJWGQNYjv" +"stable_1_1","2021-09-07 04:26:40.000",15376875,"binary.lwtmrfdurUFDSkHMZuOe" +"stable_1_1","2021-09-07 08:36:40.000",1998856189,"binary.JCxsLYijtQSKzNLYAICB" +"stable_1_1","2021-09-07 12:46:40.000",-1190594011,"binary.lTlvbMfWCVccDOCnIDBK" +"stable_1_1","2021-09-07 16:56:40.000",-925781701,"binary.iLESXKenJaWTIeKHFleM" +"stable_1_1","2021-09-07 21:06:40.000",-487874477,"binary.AfIofVnHdvWMCGyeajlC" +"stable_1_1","2021-09-08 01:16:40.000",212157003,"binary.jDxeCyLAKzKgEHGHAlif" +"stable_1_1","2021-09-08 05:26:40.000",-1001974902,"binary.ELgeHxhmjoFyFgkFXZui" +"stable_1_1","2021-09-08 09:36:40.000",-607426580,"binary.NuToHMvRscGxwMENZqOS" +"stable_1_1","2021-09-08 13:46:40.000",630458653,"binary.SrRBINnQSvdBTBKNGHFz" +"stable_1_1","2021-09-08 17:56:40.000",1299240809,"binary.eqzlYcordfJQQJMBljjF" +"stable_1_1","2021-09-08 22:06:40.000",1560015019,"binary.fWtsEHhFIrXfqBGYlTNJ" +"stable_1_1","2021-09-09 02:16:40.000",-1618319757,"binary.xQXfGqKJJntmbtHJzqHc" +"stable_1_1","2021-09-09 06:26:40.000",989052773,"binary.SifrNRSsBKcJKDqsnfXN" +"stable_1_1","2021-09-09 10:36:40.000",-1117272908,"binary.HlxBvTuGkChJDLzDaxyu" +"stable_1_1","2021-09-09 14:46:40.000",2139488417,"binary.yymGCDKbqZkixkIjIIGT" +"stable_1_1","2021-09-09 18:56:40.000",-1462514908,"binary.gMqqCIzQCNGNgbSwolZX" +"stable_1_1","2021-09-09 23:06:40.000",-1984142506,"binary.pfhSySLjKZbOfIPTmqAB" +"stable_1_1","2021-09-10 03:16:40.000",-2106211575,"binary.lENnnzItGfhHBhuBSKkg" +"stable_1_1","2021-09-10 07:26:40.000",-456444500,"binary.zrOBefSbOEUgRLtVobMw" +"stable_1_1","2021-09-10 11:36:40.000",-294434895,"binary.olALLZcgYbqmLLsQAfvv" +"stable_1_1","2021-09-10 15:46:40.000",1229299947,"binary.TIeullzMmPeJOEDzERVs" +"stable_1_1","2021-09-10 19:56:40.000",1290141025,"binary.loHjtwzgyvJksRhJNiQS" +"stable_1_1","2021-09-11 00:06:40.000",-770991939,"binary.QKhLWpoMPsfwyQYNoPZA" +"stable_1_1","2021-09-11 04:16:40.000",262080824,"binary.hIMCRvrXPAOVpFRmYnUy" +"stable_1_1","2021-09-11 08:26:40.000",-493256377,"binary.PqdASxtDLyeIFzBiaFGV" +"stable_1_1","2021-09-11 12:36:40.000",408968903,"binary.GLyIXkgZbApVCOoVSqaO" +"stable_1_1","2021-09-11 16:46:40.000",29364546,"binary.SOQDYnfDtUghMpNrrrvd" +"stable_1_1","2021-09-11 20:56:40.000",568429328,"binary.IGDZBmjwYmOzJaUbkiKN" +"stable_1_1","2021-09-12 01:06:40.000",1015255006,"binary.dcFQefENHBmSbWfsxkNZ" +"stable_1_1","2021-09-12 05:16:40.000",110327094,"binary.jBWuQXhwybyeWaUeuiCo" +"stable_1_1","2021-09-12 09:26:40.000",1886325467,"binary.QSJUCkzpJcOiAmsyKmUs" +"stable_1_1","2021-09-12 13:36:40.000",2071766507,"binary.sROneWSFNuXBalpsjyEo" +"stable_1_1","2021-09-12 17:46:40.000",-1084304030,"binary.TyxrsRIKPjtzXgKyuAEH" +"stable_1_1","2021-09-12 21:56:40.000",113309185,"binary.DJuxhSLmckBxGAZrTrdg" +"stable_1_1","2021-09-13 02:06:40.000",222279926,"binary.VsnXIRHAihqVgXYwhHKL" +"stable_1_1","2021-09-13 06:16:40.000",-1420187501,"binary.SlMenUlCCEwyXsFKefLW" +"stable_1_2","2021-09-10 11:36:39.999",224419688,"binary.FsYcmhXFIVEwuCmapLFd" +"stable_1_2","2021-09-10 11:36:40.001",-111362062,"binary.wugzMLvmLdkxesgmixOH" +"stable_1_2","2021-09-10 15:46:39.999",2133083813,"binary.buDeWRmTqpbBhNKoIWjt" +"stable_1_2","2021-09-10 15:46:40.001",-88622269,"binary.fgqZcQmlErAwLKYvDwkA" +"stable_1_2","2021-09-10 19:56:39.999",521859506,"binary.NgxLslBkOvvWWVcDCUlC" +"stable_1_2","2021-09-10 19:56:40.001",-1577694201,"binary.zgfPMXusJGeFxLLImITl" +"stable_1_2","2021-09-11 00:06:39.999",896685945,"binary.NEkCzZWDoXmvngPlArSM" +"stable_1_2","2021-09-11 00:06:40.001",-102246541,"binary.qiNYdNPnHVKQBpzXATmm" +"stable_1_2","2021-09-11 04:16:39.999",2143525047,"binary.eIPLVCyRogBRyzGbrFic" +"stable_1_2","2021-09-11 04:16:40.001",-489200958,"binary.bCLmDBrnkgDcXvcwrBqA" +"stable_1_2","2021-09-11 08:26:39.999",1129994268,"binary.rqJyBwQmuzFVARUJRgKb" +"stable_1_2","2021-09-11 08:26:40.001",-1449143866,"binary.fPxCvZJXReoaPBmALmPQ" +"stable_1_2","2021-09-11 12:36:39.999",2026186883,"binary.fOSZlDCKdxiQaysyscfx" +"stable_1_2","2021-09-11 12:36:40.001",-2001883206,"binary.IUNzHoyqCMDBQckAVEQz" +"stable_1_2","2021-09-11 16:46:39.999",341004545,"binary.PlQgjTLDuHwIfIkGmfIq" +"stable_1_2","2021-09-11 16:46:40.001",-2139193188,"binary.LSzPYUKvvRlNrhoKoYkL" +"stable_1_2","2021-09-11 20:56:39.999",2050068224,"binary.fzVzLhgOcjBWiooIbfZy" +"stable_1_2","2021-09-11 20:56:40.001",-1606114187,"binary.DQLAjLojVsSPrXVqoaFU" +"stable_1_2","2021-09-12 01:06:39.999",869226799,"binary.thrXsOuUUQEwGkXiZqVD" +"stable_1_2","2021-09-12 01:06:40.001",-2127572203,"binary.MtiGMwxbCoGxBrSCfmnb" +"stable_1_2","2021-09-12 05:16:39.999",415752279,"binary.pNjKEKLwWZUgNZshphnH" +"stable_1_2","2021-09-12 05:16:40.001",-1943166726,"binary.hSqhMEcJIhZRmIFrQDKl" +"stable_1_2","2021-09-12 09:26:39.999",1895012884,"binary.KBNSdgumJOCWoMmWtLJU" +"stable_1_2","2021-09-12 09:26:40.001",-1384685953,"binary.orjVZeKJsgJsTEqHljkf" +"stable_1_2","2021-09-12 13:36:39.999",2108909701,"binary.vYUalkfScnQyDPeEqCTq" +"stable_1_2","2021-09-12 13:36:40.001",-52260486,"binary.erVQJAvbzEahBSRwIflK" +"stable_1_2","2021-09-12 17:46:39.999",20091569,"binary.kZJLmYOxtxuJZnMBDufv" +"stable_1_2","2021-09-12 17:46:40.001",-958070866,"binary.RZliiclFrDaNImulqJTo" +"stable_1_2","2021-09-12 21:56:39.999",420485592,"binary.PUybFnyERTpaOdCfdUeZ" +"stable_1_2","2021-09-12 21:56:40.001",-136337675,"binary.AvuVCfvKjWXieILiINcT" +"stable_1_2","2021-09-13 02:06:39.999",447814637,"binary.fyooIrohcePtHVwJGMuD" +"stable_1_2","2021-09-13 02:06:40.001",-994364364,"binary.qpXjZYHYtFFCkEkBnwAb" +"stable_1_2","2021-09-13 06:16:39.999",1572668480,"binary.qWPhdNHKBtjYxVZgdXOh" +"stable_1_2","2021-09-13 06:16:40.001",-375162227,"binary.yMDiebwbXIHZiaQKPRut" +"stable_1_1","2024-01-30 20:18:24.652",1,NULL +"stable_1_1","2024-01-30 20:18:24.653",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.654",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.655",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.656",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.657",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.658",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.659",NULL,"1" +"stable_1_1","2024-01-30 20:18:24.660",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.661",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.672",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.673",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.674",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.675",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.676",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.677",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.678",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.679",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.680",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.681",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.692",1,NULL +"stable_1_1","2024-01-30 20:18:24.693",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.694",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.695",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.696",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.697",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.698",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.699",NULL,"1" +"stable_1_1","2024-01-30 20:18:24.700",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.701",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.834",1,NULL +"stable_1_1","2024-01-30 20:18:24.835",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.836",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.837",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.838",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.839",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.840",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.841",NULL,"1" +"stable_1_1","2024-01-30 20:18:24.842",NULL,NULL +"stable_1_1","2024-01-30 20:18:24.843",NULL,NULL +"stable_1_1","2024-01-30 20:18:34.387",NULL,NULL +"stable_1_1","2024-01-30 20:18:34.400",1,NULL +"stable_1_1","2024-01-30 20:18:34.432",NULL,NULL +"stable_1_1","2024-01-30 20:18:34.637",NULL,NULL +"stable_1_1","2024-01-30 20:18:35.402",NULL,NULL +"stable_1_1","2024-01-30 20:18:35.437",NULL,NULL +"stable_1_1","2024-01-30 20:18:36.404",NULL,NULL +"stable_1_1","2024-01-30 20:18:36.439",NULL,NULL +"stable_1_1","2024-01-30 20:18:37.406",NULL,NULL +"stable_1_1","2024-01-30 20:18:37.441",NULL,NULL +"stable_1_1","2024-01-30 20:18:38.408",NULL,NULL +"stable_1_1","2024-01-30 20:18:38.443",NULL,NULL +"stable_1_1","2024-01-30 20:18:39.410",NULL,NULL +"stable_1_1","2024-01-30 20:18:39.445",NULL,NULL +"stable_1_1","2024-01-30 20:18:40.413",NULL,NULL +"stable_1_1","2024-01-30 20:18:40.447",NULL,NULL +"stable_1_1","2024-01-30 20:18:41.414",NULL,"1" +"stable_1_1","2024-01-30 20:18:41.449",NULL,NULL +"stable_1_1","2024-01-30 20:18:42.417",NULL,NULL +"stable_1_1","2024-01-30 20:18:42.451",NULL,NULL +"stable_1_1","2024-01-30 20:18:43.419",NULL,NULL +"stable_1_1","2024-01-30 20:18:43.453",NULL,NULL +"!@!@$$^$","2024-01-30 20:18:24.844",1,NULL diff --git a/tests/system-test/2-query/stable_null_childtable.csv b/tests/system-test/2-query/stable_null_childtable.csv new file mode 100644 index 0000000000..ec2a2da4fd --- /dev/null +++ b/tests/system-test/2-query/stable_null_childtable.csv @@ -0,0 +1,64 @@ +tbname,ts,q_int,q_binary +"$^%$%^&","2024-01-30 20:18:24.844",1,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.773",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.774",1,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.775",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.776",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.777",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.778",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.779",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.780",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.781",NULL,"1" +"stable_null_childtable_1","2024-01-30 20:18:24.782",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.783",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.794",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.795",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.796",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.797",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.798",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.799",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.800",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.801",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.802",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.803",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.814",1,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.815",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.816",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.817",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.818",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.819",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.820",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.821",NULL,"1" +"stable_null_childtable_1","2024-01-30 20:18:24.822",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.823",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.834",1,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.835",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.836",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.837",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.838",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.839",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.840",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.841",NULL,"1" +"stable_null_childtable_1","2024-01-30 20:18:24.842",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:24.843",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:34.539",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:34.552",1,NULL +"stable_null_childtable_1","2024-01-30 20:18:34.582",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:35.554",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:35.587",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:36.557",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:36.589",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:37.559",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:37.592",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:38.561",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:38.594",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:39.563",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:39.596",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:40.565",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:40.598",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:41.567",NULL,"1" +"stable_null_childtable_1","2024-01-30 20:18:41.600",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:42.569",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:42.602",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:43.571",NULL,NULL +"stable_null_childtable_1","2024-01-30 20:18:43.604",NULL,NULL diff --git a/tests/system-test/2-query/stable_null_data.csv b/tests/system-test/2-query/stable_null_data.csv new file mode 100644 index 0000000000..06534ac559 --- /dev/null +++ b/tests/system-test/2-query/stable_null_data.csv @@ -0,0 +1,64 @@ +tbname,ts,q_int,q_binary +"stable_null_data_1","2024-01-30 20:18:24.712",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.713",1,NULL +"stable_null_data_1","2024-01-30 20:18:24.714",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.715",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.716",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.717",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.718",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.719",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.720",NULL,"1" +"stable_null_data_1","2024-01-30 20:18:24.721",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.722",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.733",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.734",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.735",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.736",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.737",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.738",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.739",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.740",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.741",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.742",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.753",1,NULL +"stable_null_data_1","2024-01-30 20:18:24.754",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.755",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.756",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.757",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.758",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.759",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.760",NULL,"1" +"stable_null_data_1","2024-01-30 20:18:24.761",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.762",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.834",1,NULL +"stable_null_data_1","2024-01-30 20:18:24.835",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.836",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.837",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.838",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.839",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.840",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.841",NULL,"1" +"stable_null_data_1","2024-01-30 20:18:24.842",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:24.843",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:34.466",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:34.477",1,NULL +"stable_null_data_1","2024-01-30 20:18:34.506",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:35.479",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:35.512",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:36.481",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:36.514",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:37.483",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:37.516",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:38.485",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:38.518",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:39.487",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:39.520",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:40.489",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:40.522",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:41.491",NULL,"1" +"stable_null_data_1","2024-01-30 20:18:41.524",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:42.493",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:42.526",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:43.495",NULL,NULL +"stable_null_data_1","2024-01-30 20:18:43.528",NULL,NULL +"%^$^&^&","2024-01-30 20:18:24.844",1,NULL From cd80560c94998af9bbc3afe2f5f0c8ca5f01fae3 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 6 Feb 2024 06:40:38 +0000 Subject: [PATCH 060/181] windows double --- source/libs/monitorfw/inc/taos_metric_sample_t.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/monitorfw/inc/taos_metric_sample_t.h b/source/libs/monitorfw/inc/taos_metric_sample_t.h index 6ebc000df5..387512af9d 100644 --- a/source/libs/monitorfw/inc/taos_metric_sample_t.h +++ b/source/libs/monitorfw/inc/taos_metric_sample_t.h @@ -22,7 +22,7 @@ #if !defined(WINDOWS) #define C11_ATOMIC #else -//#define DOUBLE_ATOMIC +#define DOUBLE_ATOMIC #endif #ifdef C11_ATOMIC From e5dcef0794a018d99122f6b035b7a4dc88880a1b Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 6 Feb 2024 08:29:09 +0000 Subject: [PATCH 061/181] forward client directly --- include/libs/monitor/monitor.h | 1 + include/util/tjson.h | 1 + source/dnode/mnode/impl/src/mndDnode.c | 70 ++++++++++++++++++++++---- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/libs/monitor/src/monMain.c | 15 ++++++ source/util/src/tjson.c | 6 ++- 6 files changed, 82 insertions(+), 13 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 4c6ffa0add..55b8117f3d 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -225,6 +225,7 @@ void monSetSmInfo(SMonSmInfo *pInfo); void monSetBmInfo(SMonBmInfo *pInfo); void monGenAndSendReport(); void monGenAndSendReportBasic(); +void monSendContent(char *pCont); void tFreeSMonMmInfo(SMonMmInfo *pInfo); void tFreeSMonVmInfo(SMonVmInfo *pInfo); diff --git a/include/util/tjson.h b/include/util/tjson.h index 6922930c13..b9ea72b4bb 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -103,6 +103,7 @@ char* tjsonToUnformattedString(const SJson* pJson); SJson* tjsonParse(const char* pStr); bool tjsonValidateJson(const char* pJson); const char* tjsonGetError(); +void tjsonDeleteItemFromObject(const SJson* pJson, const char* pName); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 50bc95f546..422821301b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -506,7 +506,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; + return code; } if(tsMonitorLogProtocol){ @@ -517,6 +517,63 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { int32_t ts_size = tjsonGetArraySize(pJson); + for(int32_t i = 0; i < ts_size; i++){ + SJson* item = tjsonGetArrayItem(pJson, i); + + SJson* tables = tjsonGetObjectItem(item, "tables"); + + int32_t tableSize = tjsonGetArraySize(tables); + for(int32_t i = 0; i < tableSize; i++){ + SJson* table = tjsonGetArrayItem(tables, i); + + char tableName[MONITOR_TABLENAME_LEN] = {0}; + tjsonGetStringValue(table, "name", tableName); + + SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups"); + + int32_t size = tjsonGetArraySize(metricGroups); + for(int32_t i = 0; i < size; i++){ + SJson* item = tjsonGetArrayItem(metricGroups, i); + + SJson* arrayTag = tjsonGetObjectItem(item, "tags"); + + int32_t tagSize = tjsonGetArraySize(arrayTag); + for(int32_t j = 0; j < tagSize; j++){ + SJson* item = tjsonGetArrayItem(arrayTag, j); + + char tagName[MONITOR_TAG_NAME_LEN] = {0}; + tjsonGetStringValue(item, "name", tagName); + + if(strncmp(tagName, "cluster_id", MONITOR_TAG_NAME_LEN) == 0) { + tjsonDeleteItemFromObject(item, "value"); + tjsonAddStringToObject(item, "value", strClusterId); + } + } + } + } + } + + char *pCont = tjsonToString(pJson); + monSendContent(pCont); + + if(pJson != NULL){ + tjsonDelete(pJson); + pJson = NULL; + } + + if(pCont != NULL){ + taosMemoryFree(pCont); + pCont = NULL; + } + + tFreeSStatisReq(&statisReq); + return 0; + +/* + SJson* pJson = tjsonParse(statisReq.pCont); + + int32_t ts_size = tjsonGetArraySize(pJson); + for(int32_t i = 0; i < ts_size; i++){ SJson* item = tjsonGetArrayItem(pJson, i); @@ -555,16 +612,6 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { } } - /* - *(labels + tagSize) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); - strncpy(*(labels + tagSize), "cluster_id", MONITOR_TAG_NAME_LEN); - - *(sample_labels + tagSize) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN); - strncpy(*(sample_labels + tagSize), strClusterId, MONITOR_TAG_VALUE_LEN); - - tagSize++; - */ - SJson* metrics = tjsonGetObjectItem(item, "metrics"); int32_t metricLen = tjsonGetArraySize(metrics); @@ -647,6 +694,7 @@ _OVER: tFreeSStatisReq(&statisReq); return code; + */ } static int32_t mndProcessStatusReq(SRpcMsg *pReq) { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 1aaab9e81d..bfd19f2346 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -106,7 +106,7 @@ typedef struct SQueryNode SQueryNode; #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" -#define VNODE_METRIC_SQL_COUNT "taos_sql_req:inserted_rows" +#define VNODE_METRIC_SQL_COUNT "taos_sql_req:count" #define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type" #define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id" diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index ba3db804c3..a849dc8f06 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -628,4 +628,19 @@ void monGenAndSendReportBasic() { } monCleanupMonitorInfo(pMonitor); +} + +void monSendContent(char *pCont) { + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; + if(tsMonitorLogProtocol){ + if (pCont != NULL){ + uInfoL("report client cont:\n%s\n", pCont); + } + } + if (pCont != NULL) { + EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + uError("failed to send monitor msg"); + } + } } \ No newline at end of file diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 0b7ad330a7..82993e8449 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -375,4 +375,8 @@ bool tjsonValidateJson(const char* jIn) { return true; } -const char* tjsonGetError() { return cJSON_GetErrorPtr(); } \ No newline at end of file +const char* tjsonGetError() { return cJSON_GetErrorPtr(); } + +void tjsonDeleteItemFromObject(const SJson* pJson, const char* pName) { + cJSON_DeleteItemFromObject((cJSON*)pJson, pName); +} \ No newline at end of file From 52c5295ce8e1ad7a6484bfe6d0be8cc557fdcd46 Mon Sep 17 00:00:00 2001 From: dmchen Date: Sat, 17 Feb 2024 09:08:03 +0000 Subject: [PATCH 062/181] mnode number --- source/libs/monitor/src/monFramework.c | 34 +++++++++++++++++++++----- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c index b1e6265906..f3ba050d68 100644 --- a/source/libs/monitor/src/monFramework.c +++ b/source/libs/monitor/src/monFramework.c @@ -29,7 +29,7 @@ extern char* tsMonFwUri; #define CLUSTER_TABLE "taosd_cluster_info" -#define MASTER_UPTIME CLUSTER_TABLE":master_uptime" +#define MASTER_UPTIME CLUSTER_TABLE":cluster_uptime" #define DBS_TOTAL CLUSTER_TABLE":dbs_total" #define TBS_TOTAL CLUSTER_TABLE":tbs_total" #define STBS_TOTAL CLUSTER_TABLE":stbs_total" @@ -39,12 +39,14 @@ extern char* tsMonFwUri; #define VNODES_ALIVE CLUSTER_TABLE":vnodes_alive" #define DNODES_TOTAL CLUSTER_TABLE":dnodes_total" #define DNODES_ALIVE CLUSTER_TABLE":dnodes_alive" +#define MNODES_TOTAL CLUSTER_TABLE":mnodes_total" +#define MNODES_ALIVE CLUSTER_TABLE":mnodes_alive" #define CONNECTIONS_TOTAL CLUSTER_TABLE":connections_total" #define TOPICS_TOTAL CLUSTER_TABLE":topics_total" #define STREAMS_TOTAL CLUSTER_TABLE":streams_total" -#define EXPIRE_TIME CLUSTER_TABLE":expire_time" -#define TIMESERIES_USED CLUSTER_TABLE":timeseries_used" -#define TIMESERIES_TOTAL CLUSTER_TABLE":timeseries_total" +#define EXPIRE_TIME CLUSTER_TABLE":grants_expire_time" +#define TIMESERIES_USED CLUSTER_TABLE":grants_timeseries_used" +#define TIMESERIES_TOTAL CLUSTER_TABLE":grants_timeseries_total" #define VGROUP_TABLE "taosd_vgroups_info" @@ -106,10 +108,11 @@ void monInitMonitorFW(){ int32_t label_count =1; const char *sample_labels[] = {"cluster_id"}; char *metric[] = {MASTER_UPTIME, DBS_TOTAL, TBS_TOTAL, STBS_TOTAL, VGROUPS_TOTAL, - VGROUPS_ALIVE, VNODES_TOTAL, VNODES_ALIVE, CONNECTIONS_TOTAL, TOPICS_TOTAL, STREAMS_TOTAL, + VGROUPS_ALIVE, VNODES_TOTAL, VNODES_ALIVE, MNODES_TOTAL, MNODES_ALIVE, + CONNECTIONS_TOTAL, TOPICS_TOTAL, STREAMS_TOTAL, DNODES_TOTAL, DNODES_ALIVE, EXPIRE_TIME, TIMESERIES_USED, TIMESERIES_TOTAL}; - for(int32_t i = 0; i < 16; i++){ + for(int32_t i = 0; i < 18; i++){ gauge= taos_gauge_new(metric[i], "", label_count, sample_labels); if(taos_collector_registry_register_metric(gauge) == 1){ taos_counter_destroy(gauge); @@ -263,6 +266,25 @@ void monGenClusterInfoTable(SMonInfo *pMonitor){ metric = taosHashGet(tsMonitor.metrics, DNODES_ALIVE, strlen(DNODES_ALIVE)); taos_gauge_set(*metric, dnode_alive, sample_labels); + //mnodes number + int32_t mnode_total = taosArrayGetSize(pInfo->mnodes); + int32_t mnode_alive = 0; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->mnodes); ++i) { + + SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->mnodes, i); + + if(pMnodeDesc->syncState != 0){ + mnode_alive++; + } + } + + metric = taosHashGet(tsMonitor.metrics, MNODES_TOTAL, strlen(MNODES_TOTAL)); + taos_gauge_set(*metric, mnode_total, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, MNODES_ALIVE, strlen(MNODES_ALIVE)); + taos_gauge_set(*metric, mnode_alive, sample_labels); + //grant info metric = taosHashGet(tsMonitor.metrics, EXPIRE_TIME, strlen(EXPIRE_TIME)); taos_gauge_set(*metric, pGrantInfo->expire_time, sample_labels); From b8bc14f9dfe8c86445242f0e6d978ce2ceeb5d43 Mon Sep 17 00:00:00 2001 From: dmchen Date: Sun, 18 Feb 2024 09:17:59 +0000 Subject: [PATCH 063/181] log count --- source/libs/monitor/src/monFramework.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c index f3ba050d68..bf651f1dee 100644 --- a/source/libs/monitor/src/monFramework.c +++ b/source/libs/monitor/src/monFramework.c @@ -77,10 +77,10 @@ extern char* tsMonFwUri; #define HAS_MNODE DNODE_TABLE":has_mnode" #define HAS_QNODE DNODE_TABLE":has_qnode" #define HAS_SNODE DNODE_TABLE":has_snode" -#define DNODE_LOG_ERROR DNODE_TABLE":ERROR" -#define DNODE_LOG_INFO DNODE_TABLE":INFO" -#define DNODE_LOG_DEBUG DNODE_TABLE":DEBUG" -#define DNODE_LOG_TRACE DNODE_TABLE":TRACE" +#define DNODE_LOG_ERROR DNODE_TABLE":error_log_count" +#define DNODE_LOG_INFO DNODE_TABLE":info_log_count" +#define DNODE_LOG_DEBUG DNODE_TABLE":debug_log_count" +#define DNODE_LOG_TRACE DNODE_TABLE":trace_log_count" #define DNODE_STATUS "taosd_dnodes_status:status" From 53a3ab3edcdafb2153741e9755b2797e42e91d64 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Mon, 19 Feb 2024 11:41:58 +0800 Subject: [PATCH 064/181] fix: delete slowSQLLog switch --- include/common/tglobal.h | 1 - source/client/src/slowQueryMonitor.c | 4 +- source/common/src/tglobal.c | 3 - tests/system-test/2-query/stable_1.csv | 364 ------------------ .../2-query/stable_null_childtable.csv | 64 --- .../system-test/2-query/stable_null_data.csv | 64 --- 6 files changed, 2 insertions(+), 498 deletions(-) delete mode 100644 tests/system-test/2-query/stable_1.csv delete mode 100644 tests/system-test/2-query/stable_null_childtable.csv delete mode 100644 tests/system-test/2-query/stable_null_data.csv diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 697b41543c..4f87ebcbd9 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -153,7 +153,6 @@ extern int32_t tsMetaCacheMaxSize; extern int32_t tsSlowLogThreshold; extern int32_t tsSlowLogScope; extern int32_t tsTimeSeriesThreshold; -extern bool enableSlowQueryMonitor; // client extern int32_t tsMinSlidingTime; diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c index 61add665dc..933ef5de16 100644 --- a/source/client/src/slowQueryMonitor.c +++ b/source/client/src/slowQueryMonitor.c @@ -42,7 +42,7 @@ static const char* getSlowQueryLableCostDesc(int64_t cost) { } void clientSlowQueryMonitorInit(const char* clusterKey) { - if (!tsEnableMonitor || !enableSlowQueryMonitor) return; + if (!tsEnableMonitor) return; SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey); SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter); @@ -55,7 +55,7 @@ void clientSlowQueryLog(const char* clusterKey, const char* user, SQL_RESULT_COD } void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost) { - if (!tsEnableMonitor || !enableSlowQueryMonitor) return; + if (!tsEnableMonitor) return; SQL_RESULT_CODE result = SQL_RESULT_SUCCESS; if (TSDB_CODE_SUCCESS != code) { result = SQL_RESULT_FAILED; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 39b7a23e3a..63f69d94a8 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -165,7 +165,6 @@ int32_t tsMetaCacheMaxSize = -1; // MB int32_t tsSlowLogThreshold = 3; // seconds int32_t tsSlowLogScope = SLOW_LOG_TYPE_ALL; int32_t tsTimeSeriesThreshold = 50; -bool enableSlowQueryMonitor = false; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -511,7 +510,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddString(pCfg, "slowLogScope", "", CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "enableSlowQueryMonitor", enableSlowQueryMonitor, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); @@ -1090,7 +1088,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) { return -1; } - enableSlowQueryMonitor = cfgGetItem(pCfg, "enableSlowQueryMonitor")->bval; tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; diff --git a/tests/system-test/2-query/stable_1.csv b/tests/system-test/2-query/stable_1.csv deleted file mode 100644 index 7b0587f527..0000000000 --- a/tests/system-test/2-query/stable_1.csv +++ /dev/null @@ -1,364 +0,0 @@ -tbname,ts,q_int,q_binary -"stable_1_1","2021-08-27 01:46:40.000",-143708439,"binary.wEvcyPTYhxxKhVFrShTZ" -"stable_1_1","2021-08-27 05:56:40.000",-2044434228,"binary.iVmwRqGqAbGImLpoUiKX" -"stable_1_1","2021-08-27 10:06:40.000",1109067209,"binary.mJPDrXToNeEldaahHrjD" -"stable_1_1","2021-08-27 14:16:40.000",1148326681,"binary.NHZMOcXRwhSPvmIIxdxp" -"stable_1_1","2021-08-27 18:26:40.000",723117353,"binary.TublFbdwZUkjDXBnzoCS" -"stable_1_1","2021-08-27 22:36:40.000",-1735114764,"binary.TCDlFpIyMpXfcghfWlUe" -"stable_1_1","2021-08-28 02:46:40.000",1973623075,"binary.xkRVpVawmLMQlKCdvZtZ" -"stable_1_1","2021-08-28 06:56:40.000",1693347900,"binary.CRmkBadMfWkcImouLOYC" -"stable_1_1","2021-08-28 11:06:40.000",1041114692,"binary.FplIhrrnZDEqcPpMXcCz" -"stable_1_1","2021-08-28 15:16:40.000",-1384036824,"binary.NpfAGysaedqwHzyjOEhb" -"stable_1_1","2021-08-28 19:26:40.000",1841425030,"binary.wpRXidgNPOERhEaZlJAM" -"stable_1_1","2021-08-28 23:36:40.000",-270068972,"binary.FNcyuyrPVVVfxbzDuGlf" -"stable_1_1","2021-08-29 03:46:40.000",418330248,"binary.KevLpYjpMOfugEsBCBlm" -"stable_1_1","2021-08-29 07:56:40.000",-1261626278,"binary.DmaGJbQkLmJQaURuaowv" -"stable_1_1","2021-08-29 12:06:40.000",665794589,"binary.IKfpbOnsEUqZvDUDXwxE" -"stable_1_1","2021-08-29 16:16:40.000",1556546304,"binary.GKOAUmTHRsUYNGpJuJXS" -"stable_1_1","2021-08-29 20:26:40.000",-635394652,"binary.UPKoftKIqXYBGkgijulk" -"stable_1_1","2021-08-30 00:36:40.000",1654794184,"binary.QHlEvczXvDhrjeXKechh" -"stable_1_1","2021-08-30 04:46:40.000",41007230,"binary.wuOctyBeTsCZynkMWYzu" -"stable_1_1","2021-08-30 08:56:40.000",1387304128,"binary.QwtMztxuEtuQdccEPSgc" -"stable_1_1","2021-08-30 13:06:40.000",-1036308321,"binary.CHRMHcuiDKIrbquzMWnc" -"stable_1_1","2021-08-30 17:16:40.000",-1768408453,"binary.OaHLnnqkcUEKxOOHCYCJ" -"stable_1_1","2021-08-30 21:26:40.000",125729034,"binary.YAKBKrKRqlgNweLacnpB" -"stable_1_1","2021-08-31 01:36:40.000",-125236222,"binary.UJkpaKsbERScbXdQuhvo" -"stable_1_1","2021-08-31 05:46:40.000",-677274672,"binary.jwdoZKFVRIbdNbxvzglZ" -"stable_1_2","2021-08-27 01:46:39.999",1048090051,"binary.QFXcjYxRIgvVcaabbAuo" -"stable_1_2","2021-08-27 01:46:40.001",-115380468,"binary.ilLTZcPTGRSwIjACLGlM" -"stable_1_2","2021-08-27 05:56:39.999",1095266727,"binary.NMQGxjbGhMVVbcQEBaEK" -"stable_1_2","2021-08-27 05:56:40.001",-1664825220,"binary.LZlUlCryyKuhhwGGlSjc" -"stable_1_2","2021-08-27 10:06:39.999",1270783858,"binary.GqXavHpdBwhbCDrQnpSF" -"stable_1_2","2021-08-27 10:06:40.001",-1718628594,"binary.wPOmeglMuNsatgqVOYvh" -"stable_1_2","2021-08-27 14:16:39.999",781814072,"binary.LBBuZbsGbCPwrtpgpJUU" -"stable_1_2","2021-08-27 14:16:40.001",-222510400,"binary.BgnCkJAICiNgxqiaowKU" -"stable_1_2","2021-08-27 18:26:39.999",260526220,"binary.uSifMBsrTaQhLuYRzSNB" -"stable_1_2","2021-08-27 18:26:40.001",-1734497021,"binary.UnQSdMFNesxovxaQqywB" -"stable_1_2","2021-08-27 22:36:39.999",856956546,"binary.ctAKBcRCfQQDgayszuAG" -"stable_1_2","2021-08-27 22:36:40.001",-2035071163,"binary.evhulVVFzQzmcxSxrhKQ" -"stable_1_2","2021-08-28 02:46:39.999",2024548336,"binary.HAdhzWKQINYLaYkXNWMA" -"stable_1_2","2021-08-28 02:46:40.001",-1388002413,"binary.uYfWFNKpQafMupBPxsaQ" -"stable_1_2","2021-08-28 06:56:39.999",896614799,"binary.eLMnpehchKlvWYooNwzT" -"stable_1_2","2021-08-28 06:56:40.001",-449300050,"binary.qKdRIooLiEPovATVrIjv" -"stable_1_2","2021-08-28 11:06:39.999",640308875,"binary.OzGnekdKxbZxbKgGFJlg" -"stable_1_2","2021-08-28 11:06:40.001",-1961834598,"binary.FbQxRmecVULCPbyUFBtd" -"stable_1_2","2021-08-28 15:16:39.999",152017229,"binary.CoERNXcHahySItezBpen" -"stable_1_2","2021-08-28 15:16:40.001",-873242705,"binary.LLhLPAXTmmXelsILHdQZ" -"stable_1_2","2021-08-28 19:26:39.999",7371013,"binary.OtbSjvYZOPWbIQOdIrKG" -"stable_1_2","2021-08-28 19:26:40.001",-259884599,"binary.ILwvbvFGWLKDelTSCcDm" -"stable_1_2","2021-08-28 23:36:39.999",1060176497,"binary.cZSBdkTCfHzKdaUbRObM" -"stable_1_2","2021-08-28 23:36:40.001",-1895192712,"binary.CYsQdbzmYnNmadYZeaGH" -"stable_1_2","2021-08-29 03:46:39.999",778869650,"binary.HYiRCyeEqGzFGwoHupCH" -"stable_1_2","2021-08-29 03:46:40.001",-1070149378,"binary.VZKQmmtwMfYTGzlJOyzv" -"stable_1_2","2021-08-29 07:56:39.999",2139522991,"binary.WgIELLWDqNqWvdsblKCB" -"stable_1_2","2021-08-29 07:56:40.001",-1450415363,"binary.THXDnYCLdvuTXRSUsMnH" -"stable_1_2","2021-08-29 12:06:39.999",1202083307,"binary.jzdrBvoIOBydYtlFDBfZ" -"stable_1_2","2021-08-29 12:06:40.001",-808169724,"binary.sUZnozuiMlgMQjgaKLWT" -"stable_1_2","2021-08-29 16:16:39.999",202190259,"binary.QFvJnTjavOlzsXAEZVwH" -"stable_1_2","2021-08-29 16:16:40.001",-409805763,"binary.sbSiqKmjPKvnXbSeKwXG" -"stable_1_2","2021-08-29 20:26:39.999",238657263,"binary.OgrrbsocdxzSAsMKmGFW" -"stable_1_2","2021-08-29 20:26:40.001",-1447960279,"binary.gMIOOqmPvHzbiBiPEVsv" -"stable_1_2","2021-08-30 00:36:39.999",954795210,"binary.VOdZRucmuTuXzTGKuWfA" -"stable_1_2","2021-08-30 00:36:40.001",-185956520,"binary.LsRcBtiQqicggCLXZoWe" -"stable_1_2","2021-08-30 04:46:39.999",1365142146,"binary.EnWxuBUlakKcHrXrPBOj" -"stable_1_2","2021-08-30 04:46:40.001",-210822579,"binary.ZKvMAWaDuBEInoXJgQSL" -"stable_1_2","2021-08-30 08:56:39.999",952122349,"binary.lcXghoPWtfUTfSAAJDPS" -"stable_1_2","2021-08-30 08:56:40.001",-1152099753,"binary.vtvGwZIEcxVUBWDFVniL" -"stable_1_2","2021-08-30 13:06:39.999",1198995002,"binary.CobYIOXzviHBQmtndNfY" -"stable_1_2","2021-08-30 13:06:40.001",-1498342366,"binary.qfSLXCSfQexiQHdraTYY" -"stable_1_2","2021-08-30 17:16:39.999",388708472,"binary.VWdcyMxbxjRZCcDesjWH" -"stable_1_2","2021-08-30 17:16:40.001",-1688761021,"binary.fsNWygGviERgbHDCTNoU" -"stable_1_2","2021-08-30 21:26:39.999",1046617386,"binary.OmMMfoOWmDphnVXsqVny" -"stable_1_2","2021-08-30 21:26:40.001",-509016842,"binary.miOrdOLIjNjowuqZLMJY" -"stable_1_2","2021-08-31 01:36:39.999",1470657859,"binary.QLjYhzfGDKcPmHFQjBub" -"stable_1_2","2021-08-31 01:36:40.001",-720454315,"binary.GaEJJOwaHtRTijYNSKrt" -"stable_1_2","2021-08-31 05:46:39.999",1874112564,"binary.tNjvCgdWlmcOHVbLJRMN" -"stable_1_2","2021-08-31 05:46:40.001",-805620343,"binary.rMWNLTrygVpKqbvfAPEQ" -"stable_1_2","2021-08-31 09:56:39.999",1624260248,"binary.OMehRclQHcJBTynDoScY" -"stable_1_2","2021-08-31 09:56:40.001",-321755377,"binary.pYwaFBbMgRmreHVLLqHM" -"stable_1_2","2021-08-31 14:06:39.999",1338486910,"binary.PewhoryOLSZsZOExXrsO" -"stable_1_2","2021-08-31 14:06:40.001",-1972396465,"binary.HCciTUVnuKaUQHOTLmoy" -"stable_1_2","2021-08-31 18:16:39.999",586219277,"binary.ejuJVeKrOYTBIOGtnvwN" -"stable_1_2","2021-08-31 18:16:40.001",-23773554,"binary.HMzLCwXDnCBrSjyvRiij" -"stable_1_2","2021-08-31 22:26:39.999",403592186,"binary.KbgXOhPmkagxFsFbQakf" -"stable_1_2","2021-08-31 22:26:40.001",-1017841441,"binary.BONmREpjVojUyNnkKVDr" -"stable_1_2","2021-09-01 02:36:39.999",2123582595,"binary.JqrmmSVJyyJSphzDfdfF" -"stable_1_2","2021-09-01 02:36:40.001",-541998051,"binary.DAHIBWzujRgGmXHITFdP" -"stable_1_2","2021-09-01 06:46:39.999",863783720,"binary.iZrgQLIwNNvEhiErAwkt" -"stable_1_2","2021-09-01 06:46:40.001",-1507574370,"binary.nVGhmvtMlkPpeaBpAFTM" -"stable_1_2","2021-09-01 10:56:39.999",1652311395,"binary.GFGXaNXjaQWBuiuIuflv" -"stable_1_2","2021-09-01 10:56:40.001",-881848122,"binary.WKsWUZoBjTlKVGgAJxPx" -"stable_1_2","2021-09-01 15:06:39.999",133942525,"binary.jFkkgXUPSCrZnMUmiDca" -"stable_1_2","2021-09-01 15:06:40.001",-1161482672,"binary.TMzTATElhmlhyqFvwFKJ" -"stable_1_2","2021-09-01 19:16:39.999",1748308690,"binary.bsqrWOhUgZliFzqtMCyZ" -"stable_1_2","2021-09-01 19:16:40.001",-45395742,"binary.SEnorgonMIgtUKkNTOxP" -"stable_1_2","2021-09-01 23:26:39.999",59717021,"binary.XBJIPnZODnweOOXmzRkO" -"stable_1_2","2021-09-01 23:26:40.001",-369410792,"binary.HBmqhBQjhNYwAaLJWbpE" -"stable_1_2","2021-09-02 03:36:39.999",1042899672,"binary.SyRZdWnAthyOUqpOtOos" -"stable_1_2","2021-09-02 03:36:40.001",-1784712756,"binary.FsBnuFMAwIdTjEjYkjBR" -"stable_1_2","2021-09-02 07:46:39.999",656087591,"binary.vsIjUPiZNVxiFWHzGqkg" -"stable_1_2","2021-09-02 07:46:40.001",-70790307,"binary.GLAJYWukLZueieYRsidQ" -"stable_1_2","2021-09-02 11:56:39.999",396390446,"binary.pgBGZOKYSliCxHFmGsyp" -"stable_1_2","2021-09-02 11:56:40.001",-1965530504,"binary.ECLVeNvkmqcTHEoSSxYT" -"stable_1_2","2021-09-02 16:06:39.999",554982384,"binary.zpGkiHSJABZuoZlpNXMp" -"stable_1_2","2021-09-02 16:06:40.001",-2032946157,"binary.lyLVozekCioDlqntmATw" -"stable_1_2","2021-09-02 20:16:39.999",677947760,"binary.vfXYvrHXFnDSAHwCndcp" -"stable_1_2","2021-09-02 20:16:40.001",-494862894,"binary.zffLjgnXcDlwdKAxADMr" -"stable_1_2","2021-09-03 00:26:39.999",1558178384,"binary.bKLXWDPaBjWRXgOeesiD" -"stable_1_2","2021-09-03 00:26:40.001",-475557451,"binary.wnZulRregqgognSRRYWI" -"stable_1_2","2021-09-03 04:36:39.999",632223220,"binary.JqnabMfPCrzsTaYczTLh" -"stable_1_2","2021-09-03 04:36:40.001",-951785691,"binary.sKKfLrkShxlGFGromDFZ" -"stable_1_2","2021-09-03 08:46:39.999",826686189,"binary.shGShhPNnWdZQDAdbiHo" -"stable_1_2","2021-09-03 08:46:40.001",-667795153,"binary.OaUjyowfAGaGXKoVLWZR" -"stable_1_2","2021-09-03 12:56:39.999",1420828021,"binary.AYcgokdEsFMUOIcUpxnM" -"stable_1_2","2021-09-03 12:56:40.001",-2009472018,"binary.JpOZwpoVKCDjiqsFhbaS" -"stable_1_2","2021-09-03 17:06:39.999",395324903,"binary.vWVYYMXIWQAEHBTKKWjP" -"stable_1_2","2021-09-03 17:06:40.001",-9754404,"binary.QNAVDZqImSFgnHpVdWtS" -"stable_1_2","2021-09-03 21:16:39.999",2068284680,"binary.wNSWJmuVBjpRVVCpksQC" -"stable_1_2","2021-09-03 21:16:40.001",-853667209,"binary.OgZwwWPZuNLXAAdBcanX" -"stable_1_2","2021-09-04 01:26:39.999",155983724,"binary.MRJjXYWHBZvzaAjTzrli" -"stable_1_2","2021-09-04 01:26:40.001",-1760453704,"binary.pARRcJciJQzdzcCZRqZA" -"stable_1_2","2021-09-04 05:36:39.999",570191599,"binary.VVxoQGPpWrOYSXzSxWlM" -"stable_1_2","2021-09-04 05:36:40.001",-1289549437,"binary.mtOMrfEeDMegNqMauPgK" -"stable_1_2","2021-09-04 09:46:39.999",1998845359,"binary.IROrsLaBEXcULJZvtXDS" -"stable_1_2","2021-09-04 09:46:40.001",-1746905381,"binary.WatcMYFnezOmZtsAGUDK" -"stable_1_2","2021-09-04 13:56:39.999",2123054685,"binary.cRdRGuitRXDuGAWVDxAE" -"stable_1_2","2021-09-04 13:56:40.001",-45388705,"binary.HMSrCuTIHQiHmuAoGqJK" -"stable_1_2","2021-09-04 18:06:39.999",422908755,"binary.xeqmTUMylYPDFkFJGLQY" -"stable_1_2","2021-09-04 18:06:40.001",-1902203841,"binary.dEfjmLzekcjvysMUToXa" -"stable_1_2","2021-09-04 22:16:39.999",1335747951,"binary.ugUzQSUcPjEXMIDrFobB" -"stable_1_2","2021-09-04 22:16:40.001",-1552553924,"binary.gZMscjyRXoGyNsFLHpoR" -"stable_1_2","2021-09-05 02:26:39.999",1092904458,"binary.IAiAMcGMqeACFMkfgGju" -"stable_1_2","2021-09-05 02:26:40.001",-437886009,"binary.jWguWEtBAUgcMKtgXieL" -"stable_1_2","2021-09-05 06:36:39.999",433850999,"binary.QBIulMvNBsGMfStAKgCQ" -"stable_1_2","2021-09-05 06:36:40.001",-1219816305,"binary.aSnbvFgLheGIahlWoBEf" -"stable_1_2","2021-09-05 10:46:39.999",1771552222,"binary.YdwgZmlYfeivmxtaWueZ" -"stable_1_2","2021-09-05 10:46:40.001",-429657495,"binary.yGIWxOkDtQCQgbPJhHFi" -"stable_1_2","2021-09-05 14:56:39.999",1560886565,"binary.OlTzMGFNSfSrcixpefQx" -"stable_1_2","2021-09-05 14:56:40.001",-2082927804,"binary.FTNnvEphxFByhzkfmAhb" -"stable_1_2","2021-09-05 19:06:39.999",312047911,"binary.yaVlkhtTxxgrtZTsFdxa" -"stable_1_2","2021-09-05 19:06:40.001",-1417705247,"binary.tHnemGaRugDffcmoyWli" -"stable_1_2","2021-09-05 23:16:39.999",1667164436,"binary.NdSzaJxdoQPGYefWPJyi" -"stable_1_2","2021-09-05 23:16:40.001",-2145233720,"binary.jghuIXEGcUtKikJpzLOh" -"stable_1_2","2021-09-06 03:26:39.999",488072919,"binary.RLQFgpAcQCfhuZExHLkx" -"stable_1_2","2021-09-06 03:26:40.001",-1943550230,"binary.ozdeiPLKjSgywqNpCBYN" -"stable_1_2","2021-09-06 07:36:39.999",724025582,"binary.XZmZYlSRZZddcxEOjfEd" -"stable_1_2","2021-09-06 07:36:40.001",-2130070769,"binary.qLvSSOSazLTUKvqpCCBL" -"stable_1_2","2021-09-06 11:46:39.999",941556630,"binary.WrcojJGdsiZyAaRxYrdw" -"stable_1_2","2021-09-06 11:46:40.001",-343938435,"binary.HoqLkegpacFGqzFWeVXL" -"stable_1_2","2021-09-06 15:56:39.999",462886464,"binary.BRImBBEftaaKZeOGRSvW" -"stable_1_2","2021-09-06 15:56:40.001",-674623669,"binary.XevOOqIPqCmaJUcXaaKz" -"stable_1_2","2021-09-06 20:06:39.999",1589240480,"binary.wJyZOXguJRYIbgIuIRye" -"stable_1_2","2021-09-06 20:06:40.001",-253589870,"binary.CzIxfDKtViliFCPuYkSq" -"stable_1_2","2021-09-07 00:16:39.999",701182235,"binary.wFVimJeupABVGEvwMiTv" -"stable_1_2","2021-09-07 00:16:40.001",-1563988450,"binary.OjWhbhEvRlHpqFEHXmfR" -"stable_1_2","2021-09-07 04:26:39.999",1631111972,"binary.QZSHkvsYmRHnGFnyMije" -"stable_1_2","2021-09-07 04:26:40.001",-1081644320,"binary.TDrOTkriDBpZtXBadhIx" -"stable_1_2","2021-09-07 08:36:39.999",1021775776,"binary.AstGPfaVxEefogcWzDzq" -"stable_1_2","2021-09-07 08:36:40.001",-1715903601,"binary.lSSCcLFElDWxKZTdHRLN" -"stable_1_2","2021-09-07 12:46:39.999",1492685945,"binary.esYXdqdUGVFhLCHHciWA" -"stable_1_2","2021-09-07 12:46:40.001",-457991373,"binary.OCGshioisqgLTUFLRMSc" -"stable_1_2","2021-09-07 16:56:39.999",306491871,"binary.WPbbRgxYIRecrOFSlRYM" -"stable_1_2","2021-09-07 16:56:40.001",-709883988,"binary.cADQdfbZkXfcNPPOPQwp" -"stable_1_2","2021-09-07 21:06:39.999",1639981656,"binary.mVTonpWtYFauqLNntbcK" -"stable_1_2","2021-09-07 21:06:40.001",-40134567,"binary.hSZKVixlyoxpHpdvwYxY" -"stable_1_2","2021-09-08 01:16:39.999",1412945598,"binary.EPkNzxzaeCAWQetzzzlv" -"stable_1_2","2021-09-08 01:16:40.001",-517552216,"binary.AmPftzlwURAILqlHhYtv" -"stable_1_2","2021-09-08 05:26:39.999",1210716085,"binary.xPKETDcCdbWtoYoUCgre" -"stable_1_2","2021-09-08 05:26:40.001",-1186882087,"binary.HVKExGKBzdrjjQbKdgim" -"stable_1_2","2021-09-08 09:36:39.999",1209625354,"binary.xDqLioOjrUudmVTRaKmP" -"stable_1_2","2021-09-08 09:36:40.001",-1049869765,"binary.ZcJrqAQpBfTRFJupVZDH" -"stable_1_2","2021-09-08 13:46:39.999",482971664,"binary.ZumfNrRUgHCXCqqjBwgX" -"stable_1_2","2021-09-08 13:46:40.001",-1695698428,"binary.VzjkizbqoeqlOEtsqzTd" -"stable_1_2","2021-09-08 17:56:39.999",369111551,"binary.hBVqWjfnRCktdjBpGTYb" -"stable_1_2","2021-09-08 17:56:40.001",-1324385506,"binary.aBZiPEKTvPqdXPsDXgvy" -"stable_1_2","2021-09-08 22:06:39.999",1370148713,"binary.ApKvhLsWkpRndmodhPEN" -"stable_1_2","2021-09-08 22:06:40.001",-1440742963,"binary.nsuhquhIJtXKINhuhbZm" -"stable_1_2","2021-09-09 02:16:39.999",1951461498,"binary.hZDumWlAzVpYfXkEaPlW" -"stable_1_2","2021-09-09 02:16:40.001",-1663211983,"binary.wBVWJVLFoOdXTzTtRNuT" -"stable_1_2","2021-09-09 06:26:39.999",2092906691,"binary.hLAAdJsauemmUBKhTtqW" -"stable_1_2","2021-09-09 06:26:40.001",-2137737633,"binary.wRfxAxVfEcdfKteGLuaX" -"stable_1_2","2021-09-09 10:36:39.999",870002160,"binary.JCROwpmJQkPqZkIzrRkU" -"stable_1_2","2021-09-09 10:36:40.001",-837033747,"binary.YKouegacQvxoSSgTaPrv" -"stable_1_2","2021-09-09 14:46:39.999",1397781824,"binary.hwfhcmJDDTvQaCRmlKCd" -"stable_1_2","2021-09-09 14:46:40.001",-1573224659,"binary.QpcyRVyxqqnHSSLxWCdz" -"stable_1_2","2021-09-09 18:56:39.999",1354974803,"binary.mgwWzOiTRoTHUbvzEMSm" -"stable_1_2","2021-09-09 18:56:40.001",-1306330996,"binary.xazZjdXNjaykPYfDWhos" -"stable_1_2","2021-09-09 23:06:39.999",1089140558,"binary.tqyfrLsfYAZeSvFCwYXQ" -"stable_1_2","2021-09-09 23:06:40.001",-2070609011,"binary.HtBovhAaClqmUKDVoPoA" -"stable_1_2","2021-09-10 03:16:39.999",243460896,"binary.GwEkcVzBVjqKKvBBYCHl" -"stable_1_2","2021-09-10 03:16:40.001",-442417658,"binary.aMVpynzemnksvKTWGcDC" -"stable_1_2","2021-09-10 07:26:39.999",993462171,"binary.akkJQvtKVOUMeEJdtLWQ" -"stable_1_2","2021-09-10 07:26:40.001",-1147877996,"binary.UsylaHHTjnwUCmWOFRop" -"stable_1_1","2021-08-31 09:56:40.000",1859005003,"binary.uDiGDbJBCeWVuQCYGwQU" -"stable_1_1","2021-08-31 14:06:40.000",620402706,"binary.kcxyJbYKMUthZCemDZwN" -"stable_1_1","2021-08-31 18:16:40.000",-697189031,"binary.mjvvuLgBZRyYZvSnmrpW" -"stable_1_1","2021-08-31 22:26:40.000",-1223538957,"binary.YhGtRmitXcCooBPlNxlh" -"stable_1_1","2021-09-01 02:36:40.000",-838875167,"binary.AXEKjEPNWZzTiqwFpkrg" -"stable_1_1","2021-09-01 06:46:40.000",-765134201,"binary.uytkzHZmorAgiDiISVoG" -"stable_1_1","2021-09-01 10:56:40.000",-314548498,"binary.zPUYZbaXGMPJOywgECDZ" -"stable_1_1","2021-09-01 15:06:40.000",780205345,"binary.dyNGyigdTXCBLJSGKHrJ" -"stable_1_1","2021-09-01 19:16:40.000",-179608812,"binary.VMzHtbnrmjokPDZCMXsk" -"stable_1_1","2021-09-01 23:26:40.000",-138813503,"binary.cvjJVfNMFXXyaKDjkvRu" -"stable_1_1","2021-09-02 03:36:40.000",-319429462,"binary.INFGvqMFdDTmdeFePZnQ" -"stable_1_1","2021-09-02 07:46:40.000",-298407008,"binary.ligoRoVSAHzFhUvIDwlE" -"stable_1_1","2021-09-02 11:56:40.000",632563376,"binary.RsQnCVXQtzoeXuamUiBl" -"stable_1_1","2021-09-02 16:06:40.000",446510409,"binary.FEewTpfZgngFmPSTSvxJ" -"stable_1_1","2021-09-02 20:16:40.000",640199308,"binary.bRHphFiZkeWtyDjXqNaO" -"stable_1_1","2021-09-03 00:26:40.000",790711145,"binary.uCOCqMGewUkNeUhAbNNf" -"stable_1_1","2021-09-03 04:36:40.000",1664157316,"binary.AyZCLhYiMxZgfYXlVciw" -"stable_1_1","2021-09-03 08:46:40.000",-1983928412,"binary.CaPFXqhDLSOaNmNnOsLh" -"stable_1_1","2021-09-03 12:56:40.000",-412013965,"binary.qEwLmiFwCrLZumFHOYjb" -"stable_1_1","2021-09-03 17:06:40.000",-1579595934,"binary.xUUoAUpZeJJsquiJOKsO" -"stable_1_1","2021-09-03 21:16:40.000",-98786523,"binary.XQfGXHbnwGQPulAcrAPX" -"stable_1_1","2021-09-04 01:26:40.000",-287136285,"binary.OdsJNUSWeVvTxQdKykmk" -"stable_1_1","2021-09-04 05:36:40.000",1051015641,"binary.HfnUyPAwYxEHQotoCfFT" -"stable_1_1","2021-09-04 09:46:40.000",-1161611813,"binary.jEENQpaYOWcVHUYKPaPn" -"stable_1_1","2021-09-04 13:56:40.000",-1162065386,"binary.ltmEKwzkvDqQIZWAxJtQ" -"stable_1_1","2021-09-04 18:06:40.000",1289374504,"binary.SHhbtXRCqxYOqQcWUDtU" -"stable_1_1","2021-09-04 22:16:40.000",-115300759,"binary.CowDUEaCtAEdafPCLLwM" -"stable_1_1","2021-09-05 02:26:40.000",-97528818,"binary.pqfTTMuvVmSCfxCmFTwS" -"stable_1_1","2021-09-05 06:36:40.000",-738727537,"binary.zWAWphEtCuSMACUmkfvN" -"stable_1_1","2021-09-05 10:46:40.000",1063250813,"binary.XwbrXvujPArWLXYhQRQK" -"stable_1_1","2021-09-05 14:56:40.000",1579885922,"binary.FEXGjykuWeGEfkKImWDw" -"stable_1_1","2021-09-05 19:06:40.000",-714663691,"binary.tdLqigCoasQnRffCEaDD" -"stable_1_1","2021-09-05 23:16:40.000",1437018826,"binary.XZcaYfmvLZtxhrmzHWEJ" -"stable_1_1","2021-09-06 03:26:40.000",1811347552,"binary.riDaPlhuskGRZsNxzooI" -"stable_1_1","2021-09-06 07:36:40.000",-2123493348,"binary.ZKZunxmtjBCRRCZBHTsL" -"stable_1_1","2021-09-06 11:46:40.000",2108150002,"binary.eEaxRyDoGrrYZQfYhDwC" -"stable_1_1","2021-09-06 15:56:40.000",-1893993866,"binary.bMuXKucyDJjBggYIIAvd" -"stable_1_1","2021-09-06 20:06:40.000",-1494759877,"binary.GGbtLZVCzfmnUdMHExqv" -"stable_1_1","2021-09-07 00:16:40.000",2135168110,"binary.ICfwWUnCEvXrJWGQNYjv" -"stable_1_1","2021-09-07 04:26:40.000",15376875,"binary.lwtmrfdurUFDSkHMZuOe" -"stable_1_1","2021-09-07 08:36:40.000",1998856189,"binary.JCxsLYijtQSKzNLYAICB" -"stable_1_1","2021-09-07 12:46:40.000",-1190594011,"binary.lTlvbMfWCVccDOCnIDBK" -"stable_1_1","2021-09-07 16:56:40.000",-925781701,"binary.iLESXKenJaWTIeKHFleM" -"stable_1_1","2021-09-07 21:06:40.000",-487874477,"binary.AfIofVnHdvWMCGyeajlC" -"stable_1_1","2021-09-08 01:16:40.000",212157003,"binary.jDxeCyLAKzKgEHGHAlif" -"stable_1_1","2021-09-08 05:26:40.000",-1001974902,"binary.ELgeHxhmjoFyFgkFXZui" -"stable_1_1","2021-09-08 09:36:40.000",-607426580,"binary.NuToHMvRscGxwMENZqOS" -"stable_1_1","2021-09-08 13:46:40.000",630458653,"binary.SrRBINnQSvdBTBKNGHFz" -"stable_1_1","2021-09-08 17:56:40.000",1299240809,"binary.eqzlYcordfJQQJMBljjF" -"stable_1_1","2021-09-08 22:06:40.000",1560015019,"binary.fWtsEHhFIrXfqBGYlTNJ" -"stable_1_1","2021-09-09 02:16:40.000",-1618319757,"binary.xQXfGqKJJntmbtHJzqHc" -"stable_1_1","2021-09-09 06:26:40.000",989052773,"binary.SifrNRSsBKcJKDqsnfXN" -"stable_1_1","2021-09-09 10:36:40.000",-1117272908,"binary.HlxBvTuGkChJDLzDaxyu" -"stable_1_1","2021-09-09 14:46:40.000",2139488417,"binary.yymGCDKbqZkixkIjIIGT" -"stable_1_1","2021-09-09 18:56:40.000",-1462514908,"binary.gMqqCIzQCNGNgbSwolZX" -"stable_1_1","2021-09-09 23:06:40.000",-1984142506,"binary.pfhSySLjKZbOfIPTmqAB" -"stable_1_1","2021-09-10 03:16:40.000",-2106211575,"binary.lENnnzItGfhHBhuBSKkg" -"stable_1_1","2021-09-10 07:26:40.000",-456444500,"binary.zrOBefSbOEUgRLtVobMw" -"stable_1_1","2021-09-10 11:36:40.000",-294434895,"binary.olALLZcgYbqmLLsQAfvv" -"stable_1_1","2021-09-10 15:46:40.000",1229299947,"binary.TIeullzMmPeJOEDzERVs" -"stable_1_1","2021-09-10 19:56:40.000",1290141025,"binary.loHjtwzgyvJksRhJNiQS" -"stable_1_1","2021-09-11 00:06:40.000",-770991939,"binary.QKhLWpoMPsfwyQYNoPZA" -"stable_1_1","2021-09-11 04:16:40.000",262080824,"binary.hIMCRvrXPAOVpFRmYnUy" -"stable_1_1","2021-09-11 08:26:40.000",-493256377,"binary.PqdASxtDLyeIFzBiaFGV" -"stable_1_1","2021-09-11 12:36:40.000",408968903,"binary.GLyIXkgZbApVCOoVSqaO" -"stable_1_1","2021-09-11 16:46:40.000",29364546,"binary.SOQDYnfDtUghMpNrrrvd" -"stable_1_1","2021-09-11 20:56:40.000",568429328,"binary.IGDZBmjwYmOzJaUbkiKN" -"stable_1_1","2021-09-12 01:06:40.000",1015255006,"binary.dcFQefENHBmSbWfsxkNZ" -"stable_1_1","2021-09-12 05:16:40.000",110327094,"binary.jBWuQXhwybyeWaUeuiCo" -"stable_1_1","2021-09-12 09:26:40.000",1886325467,"binary.QSJUCkzpJcOiAmsyKmUs" -"stable_1_1","2021-09-12 13:36:40.000",2071766507,"binary.sROneWSFNuXBalpsjyEo" -"stable_1_1","2021-09-12 17:46:40.000",-1084304030,"binary.TyxrsRIKPjtzXgKyuAEH" -"stable_1_1","2021-09-12 21:56:40.000",113309185,"binary.DJuxhSLmckBxGAZrTrdg" -"stable_1_1","2021-09-13 02:06:40.000",222279926,"binary.VsnXIRHAihqVgXYwhHKL" -"stable_1_1","2021-09-13 06:16:40.000",-1420187501,"binary.SlMenUlCCEwyXsFKefLW" -"stable_1_2","2021-09-10 11:36:39.999",224419688,"binary.FsYcmhXFIVEwuCmapLFd" -"stable_1_2","2021-09-10 11:36:40.001",-111362062,"binary.wugzMLvmLdkxesgmixOH" -"stable_1_2","2021-09-10 15:46:39.999",2133083813,"binary.buDeWRmTqpbBhNKoIWjt" -"stable_1_2","2021-09-10 15:46:40.001",-88622269,"binary.fgqZcQmlErAwLKYvDwkA" -"stable_1_2","2021-09-10 19:56:39.999",521859506,"binary.NgxLslBkOvvWWVcDCUlC" -"stable_1_2","2021-09-10 19:56:40.001",-1577694201,"binary.zgfPMXusJGeFxLLImITl" -"stable_1_2","2021-09-11 00:06:39.999",896685945,"binary.NEkCzZWDoXmvngPlArSM" -"stable_1_2","2021-09-11 00:06:40.001",-102246541,"binary.qiNYdNPnHVKQBpzXATmm" -"stable_1_2","2021-09-11 04:16:39.999",2143525047,"binary.eIPLVCyRogBRyzGbrFic" -"stable_1_2","2021-09-11 04:16:40.001",-489200958,"binary.bCLmDBrnkgDcXvcwrBqA" -"stable_1_2","2021-09-11 08:26:39.999",1129994268,"binary.rqJyBwQmuzFVARUJRgKb" -"stable_1_2","2021-09-11 08:26:40.001",-1449143866,"binary.fPxCvZJXReoaPBmALmPQ" -"stable_1_2","2021-09-11 12:36:39.999",2026186883,"binary.fOSZlDCKdxiQaysyscfx" -"stable_1_2","2021-09-11 12:36:40.001",-2001883206,"binary.IUNzHoyqCMDBQckAVEQz" -"stable_1_2","2021-09-11 16:46:39.999",341004545,"binary.PlQgjTLDuHwIfIkGmfIq" -"stable_1_2","2021-09-11 16:46:40.001",-2139193188,"binary.LSzPYUKvvRlNrhoKoYkL" -"stable_1_2","2021-09-11 20:56:39.999",2050068224,"binary.fzVzLhgOcjBWiooIbfZy" -"stable_1_2","2021-09-11 20:56:40.001",-1606114187,"binary.DQLAjLojVsSPrXVqoaFU" -"stable_1_2","2021-09-12 01:06:39.999",869226799,"binary.thrXsOuUUQEwGkXiZqVD" -"stable_1_2","2021-09-12 01:06:40.001",-2127572203,"binary.MtiGMwxbCoGxBrSCfmnb" -"stable_1_2","2021-09-12 05:16:39.999",415752279,"binary.pNjKEKLwWZUgNZshphnH" -"stable_1_2","2021-09-12 05:16:40.001",-1943166726,"binary.hSqhMEcJIhZRmIFrQDKl" -"stable_1_2","2021-09-12 09:26:39.999",1895012884,"binary.KBNSdgumJOCWoMmWtLJU" -"stable_1_2","2021-09-12 09:26:40.001",-1384685953,"binary.orjVZeKJsgJsTEqHljkf" -"stable_1_2","2021-09-12 13:36:39.999",2108909701,"binary.vYUalkfScnQyDPeEqCTq" -"stable_1_2","2021-09-12 13:36:40.001",-52260486,"binary.erVQJAvbzEahBSRwIflK" -"stable_1_2","2021-09-12 17:46:39.999",20091569,"binary.kZJLmYOxtxuJZnMBDufv" -"stable_1_2","2021-09-12 17:46:40.001",-958070866,"binary.RZliiclFrDaNImulqJTo" -"stable_1_2","2021-09-12 21:56:39.999",420485592,"binary.PUybFnyERTpaOdCfdUeZ" -"stable_1_2","2021-09-12 21:56:40.001",-136337675,"binary.AvuVCfvKjWXieILiINcT" -"stable_1_2","2021-09-13 02:06:39.999",447814637,"binary.fyooIrohcePtHVwJGMuD" -"stable_1_2","2021-09-13 02:06:40.001",-994364364,"binary.qpXjZYHYtFFCkEkBnwAb" -"stable_1_2","2021-09-13 06:16:39.999",1572668480,"binary.qWPhdNHKBtjYxVZgdXOh" -"stable_1_2","2021-09-13 06:16:40.001",-375162227,"binary.yMDiebwbXIHZiaQKPRut" -"stable_1_1","2024-01-30 20:18:24.652",1,NULL -"stable_1_1","2024-01-30 20:18:24.653",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.654",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.655",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.656",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.657",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.658",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.659",NULL,"1" -"stable_1_1","2024-01-30 20:18:24.660",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.661",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.672",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.673",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.674",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.675",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.676",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.677",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.678",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.679",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.680",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.681",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.692",1,NULL -"stable_1_1","2024-01-30 20:18:24.693",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.694",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.695",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.696",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.697",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.698",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.699",NULL,"1" -"stable_1_1","2024-01-30 20:18:24.700",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.701",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.834",1,NULL -"stable_1_1","2024-01-30 20:18:24.835",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.836",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.837",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.838",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.839",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.840",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.841",NULL,"1" -"stable_1_1","2024-01-30 20:18:24.842",NULL,NULL -"stable_1_1","2024-01-30 20:18:24.843",NULL,NULL -"stable_1_1","2024-01-30 20:18:34.387",NULL,NULL -"stable_1_1","2024-01-30 20:18:34.400",1,NULL -"stable_1_1","2024-01-30 20:18:34.432",NULL,NULL -"stable_1_1","2024-01-30 20:18:34.637",NULL,NULL -"stable_1_1","2024-01-30 20:18:35.402",NULL,NULL -"stable_1_1","2024-01-30 20:18:35.437",NULL,NULL -"stable_1_1","2024-01-30 20:18:36.404",NULL,NULL -"stable_1_1","2024-01-30 20:18:36.439",NULL,NULL -"stable_1_1","2024-01-30 20:18:37.406",NULL,NULL -"stable_1_1","2024-01-30 20:18:37.441",NULL,NULL -"stable_1_1","2024-01-30 20:18:38.408",NULL,NULL -"stable_1_1","2024-01-30 20:18:38.443",NULL,NULL -"stable_1_1","2024-01-30 20:18:39.410",NULL,NULL -"stable_1_1","2024-01-30 20:18:39.445",NULL,NULL -"stable_1_1","2024-01-30 20:18:40.413",NULL,NULL -"stable_1_1","2024-01-30 20:18:40.447",NULL,NULL -"stable_1_1","2024-01-30 20:18:41.414",NULL,"1" -"stable_1_1","2024-01-30 20:18:41.449",NULL,NULL -"stable_1_1","2024-01-30 20:18:42.417",NULL,NULL -"stable_1_1","2024-01-30 20:18:42.451",NULL,NULL -"stable_1_1","2024-01-30 20:18:43.419",NULL,NULL -"stable_1_1","2024-01-30 20:18:43.453",NULL,NULL -"!@!@$$^$","2024-01-30 20:18:24.844",1,NULL diff --git a/tests/system-test/2-query/stable_null_childtable.csv b/tests/system-test/2-query/stable_null_childtable.csv deleted file mode 100644 index ec2a2da4fd..0000000000 --- a/tests/system-test/2-query/stable_null_childtable.csv +++ /dev/null @@ -1,64 +0,0 @@ -tbname,ts,q_int,q_binary -"$^%$%^&","2024-01-30 20:18:24.844",1,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.773",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.774",1,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.775",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.776",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.777",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.778",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.779",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.780",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.781",NULL,"1" -"stable_null_childtable_1","2024-01-30 20:18:24.782",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.783",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.794",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.795",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.796",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.797",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.798",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.799",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.800",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.801",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.802",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.803",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.814",1,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.815",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.816",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.817",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.818",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.819",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.820",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.821",NULL,"1" -"stable_null_childtable_1","2024-01-30 20:18:24.822",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.823",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.834",1,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.835",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.836",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.837",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.838",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.839",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.840",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.841",NULL,"1" -"stable_null_childtable_1","2024-01-30 20:18:24.842",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:24.843",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:34.539",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:34.552",1,NULL -"stable_null_childtable_1","2024-01-30 20:18:34.582",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:35.554",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:35.587",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:36.557",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:36.589",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:37.559",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:37.592",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:38.561",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:38.594",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:39.563",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:39.596",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:40.565",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:40.598",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:41.567",NULL,"1" -"stable_null_childtable_1","2024-01-30 20:18:41.600",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:42.569",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:42.602",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:43.571",NULL,NULL -"stable_null_childtable_1","2024-01-30 20:18:43.604",NULL,NULL diff --git a/tests/system-test/2-query/stable_null_data.csv b/tests/system-test/2-query/stable_null_data.csv deleted file mode 100644 index 06534ac559..0000000000 --- a/tests/system-test/2-query/stable_null_data.csv +++ /dev/null @@ -1,64 +0,0 @@ -tbname,ts,q_int,q_binary -"stable_null_data_1","2024-01-30 20:18:24.712",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.713",1,NULL -"stable_null_data_1","2024-01-30 20:18:24.714",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.715",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.716",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.717",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.718",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.719",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.720",NULL,"1" -"stable_null_data_1","2024-01-30 20:18:24.721",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.722",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.733",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.734",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.735",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.736",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.737",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.738",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.739",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.740",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.741",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.742",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.753",1,NULL -"stable_null_data_1","2024-01-30 20:18:24.754",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.755",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.756",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.757",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.758",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.759",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.760",NULL,"1" -"stable_null_data_1","2024-01-30 20:18:24.761",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.762",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.834",1,NULL -"stable_null_data_1","2024-01-30 20:18:24.835",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.836",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.837",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.838",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.839",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.840",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.841",NULL,"1" -"stable_null_data_1","2024-01-30 20:18:24.842",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:24.843",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:34.466",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:34.477",1,NULL -"stable_null_data_1","2024-01-30 20:18:34.506",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:35.479",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:35.512",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:36.481",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:36.514",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:37.483",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:37.516",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:38.485",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:38.518",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:39.487",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:39.520",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:40.489",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:40.522",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:41.491",NULL,"1" -"stable_null_data_1","2024-01-30 20:18:41.524",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:42.493",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:42.526",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:43.495",NULL,NULL -"stable_null_data_1","2024-01-30 20:18:43.528",NULL,NULL -"%^$^&^&","2024-01-30 20:18:24.844",1,NULL From a59fd502cf3830175e95e6e7addcb07f954c36b9 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 19 Feb 2024 04:21:53 +0000 Subject: [PATCH 065/181] vnodes_num --- source/libs/monitor/src/monFramework.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c index bf651f1dee..0ce6923326 100644 --- a/source/libs/monitor/src/monFramework.c +++ b/source/libs/monitor/src/monFramework.c @@ -441,10 +441,10 @@ void monGenDnodeInfoTable(SMonInfo *pMonitor) { taos_gauge_set(*metric, io_write_disk_rate, sample_labels); metric = taosHashGet(tsMonitor.metrics, ERRORS, strlen(ERRORS)); - taos_gauge_set(*metric, io_read_disk_rate, sample_labels); + taos_gauge_set(*metric, pStat->errors, sample_labels); metric = taosHashGet(tsMonitor.metrics, VNODES_NUM, strlen(VNODES_NUM)); - taos_gauge_set(*metric, pStat->errors, sample_labels); + taos_gauge_set(*metric, pStat->totalVnodes, sample_labels); metric = taosHashGet(tsMonitor.metrics, MASTERS, strlen(MASTERS)); taos_gauge_set(*metric, pStat->masterNum, sample_labels); From b554887d4d384949d7c0f74765c9bdd0668b1178 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 19 Feb 2024 14:27:54 +0800 Subject: [PATCH 066/181] enh: insert/delete sql count --- source/client/inc/clientInt.h | 9 ++++++++- source/client/src/clientEnv.c | 5 ++++- source/client/src/clientSqlMonitor.c | 20 ++++++++++++++++---- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 7e52344ba5..989c6614a6 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -438,7 +438,14 @@ void clientSlowQueryMonitorInit(const char* clusterKey); void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost); void clientSQLReqMonitorInit(const char* clusterKey); -void sqlReqLog(int64_t rid, bool killed, int32_t code); + +enum { + MONITORSQLTYPESELECT = 0, + MONITORSQLTYPEINSERT = 1, + MONITORSQLTYPEDELETE = 2 +}; + +void sqlReqLog(int64_t rid, bool killed, int32_t code, int8_t type); void clientMonitorClose(const char* clusterKey); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 902c407eb5..77ca10c905 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -98,6 +98,7 @@ static void deregisterRequest(SRequestObj *pRequest) { pRequest->metric.planCostUs, pRequest->metric.execCostUs); atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); reqType = SLOW_LOG_TYPE_INSERT; + sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT); } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { tscDebug("query duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64 "us, planCost:%" PRId64 "us, exec:%" PRId64 "us", @@ -105,8 +106,10 @@ static void deregisterRequest(SRequestObj *pRequest) { pRequest->metric.planCostUs, pRequest->metric.execCostUs); atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); - sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code); reqType = SLOW_LOG_TYPE_QUERY; + sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPESELECT); + } else if (QUERY_NODE_DELETE_STMT == pRequest->stmtType) { + sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEDELETE); } } diff --git a/source/client/src/clientSqlMonitor.c b/source/client/src/clientSqlMonitor.c index 89b91646ff..572af7ff55 100644 --- a/source/client/src/clientSqlMonitor.c +++ b/source/client/src/clientSqlMonitor.c @@ -32,12 +32,24 @@ void clientSQLReqMonitorInit(const char* clusterKey) { createClusterCounter(clusterKey, selectMonitorName, selectMonitorHelp, selectMonitorLabelCount, selectMonitorLabels); } -void clientSQLReqLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result) { - const char* selectMonitorLabelValues[] = {defaultClusterID, "select", user, resultStr(result)}; +void clientSQLReqLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int8_t type) { + const char* typeStr; + switch (type) { + case MONITORSQLTYPEDELETE: + typeStr = "delete"; + break; + case MONITORSQLTYPEINSERT: + typeStr = "insert"; + break; + default: + typeStr = "select"; + break; + } + const char* selectMonitorLabelValues[] = {defaultClusterID, typeStr, user, resultStr(result)}; taosClusterCounterInc(clusterKey, selectMonitorName, selectMonitorLabelValues); } -void sqlReqLog(int64_t rid, bool killed, int32_t code) { +void sqlReqLog(int64_t rid, bool killed, int32_t code, int8_t type) { if (!tsEnableMonitor) return; SQL_RESULT_CODE result = SQL_RESULT_SUCCESS; if (TSDB_CODE_SUCCESS != code) { @@ -53,7 +65,7 @@ void sqlReqLog(int64_t rid, bool killed, int32_t code) { if (pTscObj->pAppInfo == NULL) { tscLog("sqlReqLog, not found pAppInfo"); } else { - clientSQLReqLog(pTscObj->pAppInfo->instKey, pTscObj->user, result); + clientSQLReqLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, type); } releaseTscObj(rid); } else { From b2ad162a8b770032594020bd18f600f57b761610 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 20 Feb 2024 17:13:58 +0800 Subject: [PATCH 067/181] enh: remove obsolete codes --- include/common/tmsg.h | 1 - source/dnode/mnode/impl/src/mndDnode.c | 3 --- 2 files changed, 4 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1b438ba026..87d8ed76a8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1613,7 +1613,6 @@ typedef struct { SEp ep; char active[TSDB_ACTIVE_KEY_LEN]; char connActive[TSDB_CONN_ACTIVE_KEY_LEN]; - char machineId[TSDB_MACHINE_ID_LEN + 1]; } SDnodeInfo; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index cb8a24e675..6974c6b0e8 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -410,9 +410,6 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) { dInfo.ep.port = pDnode->port; dInfo.offlineReason = pDnode->offlineReason; tstrncpy(dInfo.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - tstrncpy(dInfo.active, pDnode->active, TSDB_ACTIVE_KEY_LEN); - tstrncpy(dInfo.connActive, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN); - tstrncpy(dInfo.machineId, pDnode->machineId, TSDB_MACHINE_ID_LEN + 1); sdbRelease(pSdb, pDnode); if (mndIsMnode(pMnode, pDnode->id)) { dInfo.isMnode = 1; From 87ad111b2c7177eab7751ffc32098fe41227a884 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 20 Feb 2024 18:40:25 +0800 Subject: [PATCH 068/181] coverage: snapshot add stop taosd and start --- tests/army/community/cluster/snapshot.py | 18 +++++++++++++++++- tests/army/frame/srvCtl.py | 4 ++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index b21cbb1ad8..eef650cc77 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -25,6 +25,7 @@ from frame.cases import * from frame.sql import * from frame.caseBase import * from frame import * +from frame.srvCtl import * class TDTestCase(TBase): @@ -65,6 +66,21 @@ class TDTestCase(TBase): sql = f"select avg(dc) from {self.db}.{self.stb}" tdSql.checkFirstValue(sql, 200) + def alterReplica3(self): + sql = f"alter database {self.db} replica 3" + tdSql.execute(sql, show=True) + time.sleep(2) + sc.dnodeStop(2) + sc.dnodeStop(3) + time.sleep(5) + sc.dnodeStart(2) + sc.dnodeStart(3) + + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + def doAction(self): tdLog.info(f"do action.") self.flushDb() @@ -81,7 +97,7 @@ class TDTestCase(TBase): self.alterReplica(1) self.checkAggCorrect() self.compactDb() - self.alterReplica(3) + self.alterReplica3() vgids = self.getVGroup(self.db) selid = random.choice(vgids) diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index 091856056b..3a9b0cdf4b 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -33,14 +33,14 @@ class srvCtl: # control server # - # start + # start idx base is 1 def dnodeStart(self, idx): if clusterDnodes.getModel() == 'cluster': return clusterDnodes.starttaosd(idx) return tdDnodes.starttaosd(idx) - # stop + # stop idx base is 1 def dnodeStop(self, idx): if clusterDnodes.getModel() == 'cluster': return clusterDnodes.stoptaosd(idx) From 73ec02ceb4cfb8a8195ab5cd04a2c024201ce7bf Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 20 Feb 2024 18:48:06 +0800 Subject: [PATCH 069/181] coverage: s3 and stop and start taosd --- tests/army/enterprise/s3/s3_basic.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index e7bc188ca5..58dbd12d7b 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -81,11 +81,15 @@ class TDTestCase(TBase): cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data" tdLog.info(cmd) loop = 0 - while len(eos.runRetList(cmd)) > 0 and loop < 40: + while len(eos.runRetList(cmd)) > 0 and loop < 100: time.sleep(5) self.trimDb(True) loop += 1 tdLog.info(f"loop={loop} wait 5s...") + if loop == 4: + sc.dnodeStop(1) + time.sleep(2) + sc.dnodeStart(1) def checkStreamCorrect(self): sql = f"select count(*) from {self.db}.stm1" From c7932675ce271bb98506cea5a1be7600c0583304 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 21 Feb 2024 09:07:38 +0800 Subject: [PATCH 070/181] enh: get machine id --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 7 +------ source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 11 +++++++++++ source/dnode/mgmt/node_util/inc/dmUtil.h | 1 + source/dnode/mnode/impl/src/mndDnode.c | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index a3a29f6f77..fa59f56496 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "dmInt.h" #include "systable.h" -#include "tgrant.h" extern SConfig *tsCfg; @@ -118,11 +117,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { req.memTotal = tsTotalMemoryKB * 1024; req.memAvail = req.memTotal - tsRpcQueueMemoryAllowed - 16 * 1024 * 1024; tstrncpy(req.dnodeEp, tsLocalEp, TSDB_EP_LEN); - char *machine = tGetMachineId(); - if (machine) { - tstrncpy(req.machineId, machine, TSDB_MACHINE_ID_LEN + 1); - taosMemoryFreeClear(machine); - } + tstrncpy(req.machineId, pMgmt->pData->machineId, TSDB_MACHINE_ID_LEN + 1); req.clusterCfg.statusInterval = tsStatusInterval; req.clusterCfg.checkTime = 0; diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 6cbf31b15f..049b1bdf84 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -22,6 +22,7 @@ #ifdef TD_TSZ #include "tcompression.h" #include "tglobal.h" +#include "tgrant.h" #endif static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { @@ -137,6 +138,16 @@ int32_t dmInitVars(SDnode *pDnode) { pData->rebootTime = taosGetTimestampMs(); pData->dropped = 0; pData->stopped = 0; + char *machineId = tGetMachineId(); + if (machineId) { + tstrncpy(pData->machineId, machineId, TSDB_MACHINE_ID_LEN + 1); + taosMemoryFreeClear(machineId); + } else { +#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG) + terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE; + return -1; +#endif + } pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); if (pData->dnodeHash == NULL) { diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 4769ef8538..504c405506 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -109,6 +109,7 @@ typedef struct { SMsgCb msgCb; bool validMnodeEps; int64_t ipWhiteVer; + char machineId[TSDB_MACHINE_ID_LEN + 1]; } SDnodeData; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 6974c6b0e8..c65c228224 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -141,7 +141,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { memcpy(dnodeObj.machineId, machineId, TSDB_MACHINE_ID_LEN); taosMemoryFreeClear(machineId); } else { -#ifdef TD_ENTERPRISE +#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG) terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE; goto _OVER; #endif From c55e9b18775cc1027c2d0cb0a291f44526ea9bcf Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 21 Feb 2024 04:12:45 +0000 Subject: [PATCH 071/181] insert user empty --- source/dnode/vnode/src/vnd/vnodeSvr.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 7498431227..108ee706c0 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1702,10 +1702,12 @@ _exit: atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows); atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); - const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS, pVnode->monitor.strClusterId, - pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, - pOriginalMsg->info.conn.user, "Success"}; - taos_counter_add(pVnode->monitor.insertCounter, pSubmitRsp->affectedRows, sample_labels); + if(pSubmitRsp->affectedRows > 0 && strlen(pOriginalMsg->info.conn.user) > 0){ + const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Success"}; + taos_counter_add(pVnode->monitor.insertCounter, pSubmitRsp->affectedRows, sample_labels); + } if (code == 0) { atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); From 95a6af54a90c42c10923bd7ed18f948fb6a0cf5c Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 21 Feb 2024 06:33:12 +0000 Subject: [PATCH 072/181] remove monitor_interval --- source/libs/monitor/src/monMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index a849dc8f06..6c2344826a 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -276,7 +276,7 @@ static void monGenClusterJsonBasic(SMonInfo *pMonitor) { tjsonAddStringToObject(pMonitor->pJson, "first_ep", pInfo->first_ep); tjsonAddDoubleToObject(pMonitor->pJson, "first_ep_dnode_id", pInfo->first_ep_dnode_id); tjsonAddStringToObject(pMonitor->pJson, "cluster_version", pInfo->version); - tjsonAddDoubleToObject(pMonitor->pJson, "monitor_interval", pInfo->monitor_interval); + //tjsonAddDoubleToObject(pMonitor->pJson, "monitor_interval", pInfo->monitor_interval); } static void monGenVgroupJson(SMonInfo *pMonitor) { From 606aa61c4f5838848c71770e4295c94d6ef80abd Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 21 Feb 2024 08:06:45 +0000 Subject: [PATCH 073/181] remove errors and delete --- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 ++ source/libs/monitor/src/monFramework.c | 10 +++++----- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9d882af1bd..ac28723fc3 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -118,7 +118,7 @@ typedef struct SQueryNode SQueryNode; #define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows" //#define VNODE_METRIC_TAG_VALUE_INSERT "insert" -#define VNODE_METRIC_TAG_VALUE_DELETE "delete" +//#define VNODE_METRIC_TAG_VALUE_DELETE "delete" // vnd.h typedef int32_t (*_query_reseek_func_t)(void* pQHandle); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 108ee706c0..e928326c67 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -2029,6 +2029,7 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, in return code; _err: + /* if(code == TSDB_CODE_SUCCESS){ const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_DELETE, pVnode->monitor.strClusterId, pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, @@ -2041,6 +2042,7 @@ _err: pOriginalMsg->info.conn.user, "Failed"}; taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); } + */ return code; } diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c index 0ce6923326..15793d07fb 100644 --- a/source/libs/monitor/src/monFramework.c +++ b/source/libs/monitor/src/monFramework.c @@ -71,7 +71,7 @@ extern char* tsMonFwUri; #define IO_WRITE DNODE_TABLE":io_write" #define IO_READ_DISK DNODE_TABLE":io_read_disk" #define IO_WRITE_DISK DNODE_TABLE":io_write_disk" -#define ERRORS DNODE_TABLE":errors" +//#define ERRORS DNODE_TABLE":errors" #define VNODES_NUM DNODE_TABLE":vnodes_num" #define MASTERS DNODE_TABLE":masters" #define HAS_MNODE DNODE_TABLE":has_mnode" @@ -135,10 +135,10 @@ void monInitMonitorFW(){ const char *dnodes_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep"}; char *dnodes_gauges[] = {UPTIME, CPU_ENGINE, CPU_SYSTEM, CPU_CORE, MEM_ENGINE, MEM_SYSTEM, MEM_TOTAL, DISK_ENGINE, DISK_USED, DISK_TOTAL, NET_IN, - NET_OUT, IO_READ, IO_WRITE, IO_READ_DISK, IO_WRITE_DISK, ERRORS, + NET_OUT, IO_READ, IO_WRITE, IO_READ_DISK, IO_WRITE_DISK, /*ERRORS,*/ VNODES_NUM, MASTERS, HAS_MNODE, HAS_QNODE, HAS_SNODE, DNODE_STATUS, DNODE_LOG_ERROR, DNODE_LOG_INFO, DNODE_LOG_DEBUG, DNODE_LOG_TRACE}; - for(int32_t i = 0; i < 27; i++){ + for(int32_t i = 0; i < 26; i++){ gauge= taos_gauge_new(dnodes_gauges[i], "", dnodes_label_count, dnodes_sample_labels); if(taos_collector_registry_register_metric(gauge) == 1){ taos_counter_destroy(gauge); @@ -440,8 +440,8 @@ void monGenDnodeInfoTable(SMonInfo *pMonitor) { metric = taosHashGet(tsMonitor.metrics, IO_WRITE_DISK, strlen(IO_WRITE_DISK)); taos_gauge_set(*metric, io_write_disk_rate, sample_labels); - metric = taosHashGet(tsMonitor.metrics, ERRORS, strlen(ERRORS)); - taos_gauge_set(*metric, pStat->errors, sample_labels); + //metric = taosHashGet(tsMonitor.metrics, ERRORS, strlen(ERRORS)); + //taos_gauge_set(*metric, pStat->errors, sample_labels); metric = taosHashGet(tsMonitor.metrics, VNODES_NUM, strlen(VNODES_NUM)); taos_gauge_set(*metric, pStat->totalVnodes, sample_labels); From fa9f48866c3b6b9086e27412c95df356fed2958b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 21 Feb 2024 20:02:22 +0800 Subject: [PATCH 074/181] update retry --- source/libs/transport/inc/transComm.h | 2 ++ source/libs/transport/src/transCli.c | 4 ++-- source/libs/transport/src/transComm.c | 13 ++++++++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 7483588593..5b18d56d70 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -427,6 +427,8 @@ SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, void transDQCancel(SDelayQueue* queue, SDelayTask* task); bool transEpSetIsEqual(SEpSet* a, SEpSet* b); + +bool transEpSetIsEqual2(SEpSet* a, SEpSet* b); /* * init global func */ diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index b6942655a9..63f684fd50 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2229,7 +2229,7 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { EPSET_FORWARD_INUSE(&pCtx->epSet); } } else { - if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { + if (!transEpSetIsEqual2(&pCtx->epSet, &epSet)) { tDebug("epset not equal, retry new epset1"); transPrintEpSet(&pCtx->epSet); transPrintEpSet(&epSet); @@ -2256,7 +2256,7 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { EPSET_FORWARD_INUSE(&pCtx->epSet); } } else { - if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { + if (!transEpSetIsEqual2(&pCtx->epSet, &epSet)) { tDebug("epset not equal, retry new epset2"); transPrintEpSet(&pCtx->epSet); transPrintEpSet(&epSet); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index b1fb9a2450..e9ba0128b3 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -70,7 +70,7 @@ int32_t transDecompressMsg(char** msg, int32_t len) { char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead)); STransMsgHead* pNewHead = (STransMsgHead*)buf; int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), (char*)pNewHead->content, - len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen); + len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen); memcpy((char*)pNewHead, (char*)pHead, sizeof(STransMsgHead)); pNewHead->msgLen = htonl(oriLen + sizeof(STransMsgHead)); @@ -602,6 +602,17 @@ bool transEpSetIsEqual(SEpSet* a, SEpSet* b) { } return true; } +bool transEpSetIsEqual2(SEpSet* a, SEpSet* b) { + if (a->numOfEps != b->numOfEps) { + return false; + } + for (int i = 0; i < a->numOfEps; i++) { + if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { + return false; + } + } + return true; +} static void transInitEnv() { refMgt = transOpenRefMgt(50000, transDestroyExHandle); From 46ee4cd00f00e022d5aa680d51770426bd0d5940 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 22 Feb 2024 00:49:51 +0000 Subject: [PATCH 075/181] uptime --- include/libs/monitor/monitor.h | 3 ++- source/dnode/mnode/impl/src/mndMain.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 55b8117f3d..6c28c77cc7 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -107,7 +107,8 @@ typedef struct { char first_ep[TSDB_EP_LEN]; int32_t first_ep_dnode_id; char version[MON_VER_LEN]; - float master_uptime; // day + //float master_uptime; // day + int64_t master_uptime; //second int32_t monitor_interval; // sec int32_t dbs_total; int32_t stbs_total; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 8b4d8eda8a..249ddb74c6 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -911,7 +911,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr if (pObj->id == pMnode->selfDnodeId) { pClusterInfo->first_ep_dnode_id = pObj->id; tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep)); - pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f; + //pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f; + pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode); // pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f); tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role)); desc.syncState = TAOS_SYNC_STATE_LEADER; From ee84e2d75e6b3f4ce4c151be405239b1cbaf5942 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 22 Feb 2024 02:00:33 +0000 Subject: [PATCH 076/181] uptime unit --- include/libs/monitor/monitor.h | 3 ++- source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 6c28c77cc7..ecd59a51a8 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -76,7 +76,8 @@ typedef struct { } SMonBasicInfo; typedef struct { - float uptime; // day + //float uptime; // day + int64_t uptime; // second int8_t has_mnode; int8_t has_qnode; int8_t has_snode; diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index 3d6be1f433..413135a08c 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -33,7 +33,8 @@ static void dmGetMonitorBasicInfoBasic(SDnode *pDnode, SMonBasicInfo *pInfo) { } static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { - pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f); + //pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f); + pInfo->uptime = taosGetTimestampMs() - pDnode->data.rebootTime; pInfo->has_mnode = pDnode->wrappers[MNODE].required; pInfo->has_qnode = pDnode->wrappers[QNODE].required; pInfo->has_snode = pDnode->wrappers[SNODE].required; From 6e13e4aa638dacfc34e6652355fa9ecf2deb5ed1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 22 Feb 2024 10:58:32 +0800 Subject: [PATCH 077/181] fix:add excluded msg for meta in tmq --- include/common/tmsg.h | 16 ++++----- source/client/src/clientRawBlockWrite.c | 41 ++++++++++++++++++----- source/client/src/clientTmq.c | 2 +- source/common/src/tmsg.c | 22 ++++++++---- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/src/mndStb.c | 10 ++++-- source/dnode/vnode/src/sma/smaTimeRange.c | 2 +- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/dnode/vnode/src/tq/tqScan.c | 4 +-- source/dnode/vnode/src/tq/tqSink.c | 4 +-- source/dnode/vnode/src/tq/tqUtil.c | 29 ++++++++++++++++ source/libs/parser/src/parInsertUtil.c | 3 +- 12 files changed, 100 insertions(+), 36 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1b438ba026..587e2f9f3e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -207,9 +207,6 @@ typedef enum _mgmt_table { #define TD_CHILD_TABLE TSDB_CHILD_TABLE #define TD_NORMAL_TABLE TSDB_NORMAL_TABLE -#define TD_REQ_FROM_APP 0 -#define TD_REQ_FROM_TAOX 1 - typedef enum ENodeType { // Syntax nodes are used in parser and planner module, and some are also used in executor module, such as COLUMN, // VALUE, OPERATOR, FUNCTION and so on. @@ -759,7 +756,7 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaW typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; - int8_t source; // 1-taosX or 0-taosClient + int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient int8_t reserved[6]; tb_uid_t suid; int64_t delay1; @@ -802,7 +799,7 @@ void tFreeSMCreateStbRsp(SMCreateStbRsp* pRsp); typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igNotExists; - int8_t source; // 1-taosX or 0-taosClient + int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient int8_t reserved[6]; tb_uid_t suid; int32_t sqlLen; @@ -2661,6 +2658,7 @@ typedef struct SVCreateStbReq { SRSmaParam rsmaParam; int32_t alterOriDataLen; void* alterOriData; + int8_t source; } SVCreateStbReq; int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq); @@ -2730,6 +2728,7 @@ typedef struct { SVCreateTbReq* pReqs; SArray* pArray; }; + int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient } SVCreateTbBatchReq; int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq); @@ -2822,6 +2821,7 @@ typedef struct { int32_t newCommentLen; char* newComment; int64_t ctimeMs; // fill by vnode + int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient } SVAlterTbReq; int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq); @@ -3919,12 +3919,13 @@ int32_t tDeatroySMqHbRsp(SMqHbRsp* pRsp); int32_t tSerializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq); int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq); +#define TD_REQ_FROM_APP 0x0 #define SUBMIT_REQ_AUTO_CREATE_TABLE 0x1 #define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2 #define SUBMIT_REQ_FROM_FILE 0x4 +#define TD_REQ_FROM_TAOX 0x8 -#define SOURCE_NULL 0 -#define SOURCE_TAOSX 1 +#define TD_REQ_FROM_TAOX_OLD 0x1 // for compatibility typedef struct { int32_t flags; @@ -3937,7 +3938,6 @@ typedef struct { SArray* aCol; }; int64_t ctimeMs; - int8_t source; } SSubmitTbData; typedef struct { diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 448243cc3d..f143624bab 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1001,6 +1001,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq)); taosArrayPush(tBatch.req.pArray, pCreateReq); + tBatch.req.source = TD_REQ_FROM_TAOX; taosHashPut(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId), &tBatch, sizeof(tBatch)); } else { // add to the correct vgroup @@ -1276,7 +1277,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { return terrno; } SVAlterTbReq req = {0}; - SDecoder coder = {0}; + SDecoder dcoder = {0}; int32_t code = TSDB_CODE_SUCCESS; SRequestObj* pRequest = NULL; SQuery* pQuery = NULL; @@ -1297,8 +1298,8 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { // decode and process req void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); - tDecoderInit(&coder, data, len); - if (tDecodeSVAlterTbReq(&coder, &req) < 0) { + tDecoderInit(&dcoder, data, len); + if (tDecodeSVAlterTbReq(&dcoder, &req) < 0) { code = TSDB_CODE_INVALID_PARA; goto end; } @@ -1340,14 +1341,36 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } pVgData->vg = pInfo; - pVgData->pData = taosMemoryMalloc(metaLen); - if (NULL == pVgData->pData) { + + int tlen = 0; + req.source = TD_REQ_FROM_TAOX; + tEncodeSize(tEncodeSVAlterTbReq, &req, tlen, code); + if(code != 0){ code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - memcpy(pVgData->pData, meta, metaLen); - ((SMsgHead*)pVgData->pData)->vgId = htonl(pInfo.vgId); - pVgData->size = metaLen; + tlen += sizeof(SMsgHead); + void* pMsg = taosMemoryMalloc(tlen); + if (NULL == pMsg) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + ((SMsgHead*)pMsg)->vgId = htonl(pInfo.vgId); + ((SMsgHead*)pMsg)->contLen = htonl(tlen); + void* pBuf = POINTER_SHIFT(pMsg, sizeof(SMsgHead)); + SEncoder coder = {0}; + tEncoderInit(&coder, pBuf, tlen - sizeof(SMsgHead)); + code = tEncodeSVAlterTbReq(&coder, &req); + if(code != 0){ + tEncoderClear(&coder); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + tEncoderClear(&coder); + + pVgData->pData = pMsg; + pVgData->size = tlen; + pVgData->numOfTables = 1; taosArrayPush(pArray, &pVgData); @@ -1387,7 +1410,7 @@ end: if (pVgData) taosMemoryFreeClear(pVgData->pData); taosMemoryFreeClear(pVgData); destroyRequest(pRequest); - tDecoderClear(&coder); + tDecoderClear(&dcoder); qDestroyQuery(pQuery); terrno = code; return code; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 1904874a0b..b141921a77 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -389,7 +389,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } if (strcasecmp(key, "msg.consume.excluded") == 0) { - conf->sourceExcluded = taosStr2int64(value); + conf->sourceExcluded = (taosStr2int64(value) != 0) ? TD_REQ_FROM_TAOX : 0; return TMQ_CONF_OK; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 2ffa12f2c1..85f5d462c7 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -7513,6 +7513,7 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) { if (pReq->alterOriDataLen > 0) { if (tEncodeBinary(pCoder, pReq->alterOriData, pReq->alterOriDataLen) < 0) return -1; } + if (tEncodeI8(pCoder, pReq->source) < 0) return -1; tEndEncode(pCoder); return 0; @@ -7535,6 +7536,10 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) { if (tDecodeBinary(pCoder, (uint8_t **)&pReq->alterOriData, NULL) < 0) return -1; } + if (!tDecodeIsEnd(pCoder)) { + if (tDecodeI8(pCoder, &pReq->source) < 0) return -1; + } + tEndDecode(pCoder); return 0; } @@ -7663,6 +7668,8 @@ int tEncodeSVCreateTbBatchReq(SEncoder *pCoder, const SVCreateTbBatchReq *pReq) if (tEncodeSVCreateTbReq(pCoder, (SVCreateTbReq *)taosArrayGet(pReq->pArray, iReq)) < 0) return -1; } + if (tEncodeI8(pCoder, pReq->source) < 0) return -1; + tEndEncode(pCoder); return 0; } @@ -7677,6 +7684,10 @@ int tDecodeSVCreateTbBatchReq(SDecoder *pCoder, SVCreateTbBatchReq *pReq) { if (tDecodeSVCreateTbReq(pCoder, pReq->pReqs + iReq) < 0) return -1; } + if (!tDecodeIsEnd(pCoder)) { + if (tDecodeI8(pCoder, &pReq->source) < 0) return -1; + } + tEndDecode(pCoder); return 0; } @@ -8034,6 +8045,7 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) { break; } if (tEncodeI64(pEncoder, pReq->ctimeMs) < 0) return -1; + if (tEncodeI8(pEncoder, pReq->source) < 0) return -1; tEndEncode(pEncoder); return 0; @@ -8094,6 +8106,9 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) { if (!tDecodeIsEnd(pDecoder)) { if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1; } + if (!tDecodeIsEnd(pDecoder)) { + if (tDecodeI8(pDecoder, &pReq->source) < 0) return -1; + } tEndDecode(pDecoder); return 0; @@ -8670,7 +8685,6 @@ static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubm } } if (tEncodeI64(pCoder, pSubmitTbData->ctimeMs) < 0) return -1; - if (tEncodeI8(pCoder, pSubmitTbData->source) < 0) return -1; tEndEncode(pCoder); return 0; @@ -8758,12 +8772,6 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa goto _exit; } } - if (!tDecodeIsEnd(pCoder)) { - if (tDecodeI8(pCoder, &pSubmitTbData->source) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto _exit; - } - } tEndDecode(pCoder); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index fc58fabad5..4a092057ce 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -464,6 +464,7 @@ typedef struct { char* pAst1; char* pAst2; SRWLatch lock; + int8_t source; } SStbObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 1ddd2f34e6..aad1dc881b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -458,6 +458,7 @@ void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int3 req.rollup = pStb->ast1Len > 0 ? 1 : 0; req.alterOriData = alterOriData; req.alterOriDataLen = alterOriDataLen; + req.source = pStb->source; // todo req.schemaRow.nCols = pStb->numOfColumns; req.schemaRow.version = pStb->colVer; @@ -774,7 +775,8 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat pDst->createdTime = taosGetTimestampMs(); pDst->updateTime = pDst->createdTime; pDst->uid = - (pCreate->source == TD_REQ_FROM_TAOX) ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + (pCreate->source == TD_REQ_FROM_TAOX_OLD || pCreate->source == TD_REQ_FROM_TAOX) + ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); pDst->dbUid = pDb->uid; pDst->tagVer = 1; pDst->colVer = 1; @@ -790,6 +792,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat pDst->numOfFuncs = pCreate->numOfFuncs; pDst->commentLen = pCreate->commentLen; pDst->pFuncs = pCreate->pFuncs; + pDst->source = pCreate->source; pCreate->pFuncs = NULL; if (pDst->commentLen > 0) { @@ -1033,6 +1036,7 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq memcpy(pDst, pStb, sizeof(SStbObj)); taosRUnLockLatch(&pStb->lock); + pDst->source = createReq->source; pDst->updateTime = taosGetTimestampMs(); pDst->numOfColumns = createReq->numOfColumns; pDst->numOfTags = createReq->numOfTags; @@ -1141,7 +1145,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { } } else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) { goto _OVER; - } else if (createReq.source == TD_REQ_FROM_TAOX && (createReq.tagVer != 1 || createReq.colVer != 1)) { + } else if ((createReq.source == TD_REQ_FROM_TAOX_OLD || createReq.source == TD_REQ_FROM_TAOX) && (createReq.tagVer != 1 || createReq.colVer != 1)) { mInfo("stb:%s, alter table does not need to be done, because table is deleted", createReq.name); code = 0; goto _OVER; @@ -2572,7 +2576,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { } } - if (dropReq.source == TD_REQ_FROM_TAOX && pStb->uid != dropReq.suid) { + if ((dropReq.source == TD_REQ_FROM_TAOX_OLD || dropReq.source == TD_REQ_FROM_TAOX) && pStb->uid != dropReq.suid) { code = 0; goto _OVER; } diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 54d859992c..fb898c02f8 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -193,7 +193,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * continue; } - SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE, .source = SOURCE_NULL}; + SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE}; int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1; tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 727157a2f8..7a737fbb5d 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -392,7 +392,7 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { pReader->msg.ver); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); - if ((pSubmitTbData->source & sourceExcluded) != 0) { + if ((pSubmitTbData->flags & sourceExcluded) != 0) { pReader->nextBlk += 1; continue; } diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index f108cedcf5..103007eb57 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -267,7 +267,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_table; } - if ((pSubmitTbDataRet->source & sourceExcluded) != 0) { + if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) { goto loop_table; } if (pRsp->withTbName) { @@ -335,7 +335,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_db; } - if ((pSubmitTbDataRet->source & sourceExcluded) != 0) { + if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) { goto loop_db; } if (pRsp->withTbName) { diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index b56bf3e0fe..42bd1bc59c 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -815,7 +815,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { return; } - SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .source = SOURCE_NULL}; + SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP}; code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); if (code != TSDB_CODE_SUCCESS) { continue; @@ -860,7 +860,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { pTask->execInfo.sink.numOfBlocks += 1; uint64_t groupId = pDataBlock->info.id.groupId; - SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .source = SOURCE_NULL}; + SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP}; int32_t* index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId)); if (index == NULL) { // no data yet, append it diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 31267dbf52..dad1211326 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -171,6 +171,22 @@ end : { } } +#define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC) \ + SDecoder decoder = {0};\ + TYPE req = {0}; \ + void* data = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)); \ + int32_t len = pHead->bodyLen - sizeof(SMsgHead); \ + tDecoderInit(&decoder, data, len); \ + if (DECODE_FUNC(&decoder, &req) == 0 && (req.source & TD_REQ_FROM_TAOX) != 0) { \ + tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, jump meta for, vgId:%d offset %" PRId64 " msgType %d", \ + pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType); \ + fetchVer++; \ + tDecoderClear(&decoder); \ + continue; \ + } \ + tDecoderClear(&decoder); + + static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* offset) { int code = 0; @@ -239,6 +255,19 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, goto end; } + if ((pRequest->sourceExcluded & TD_REQ_FROM_TAOX) != 0) { + if (pHead->msgType == TDMT_VND_CREATE_TABLE) { + PROCESS_EXCLUDED_MSG(SVCreateTbBatchReq, tDecodeSVCreateTbBatchReq) + } else if (pHead->msgType == TDMT_VND_ALTER_TABLE) { + PROCESS_EXCLUDED_MSG(SVAlterTbReq, tDecodeSVAlterTbReq) + } else if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB) { + PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq) + } else if (pHead->msgType == TDMT_VND_DELETE) { + fetchVer++; + continue; + } + } + tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType)); tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer + 1); metaRsp.resMsgType = pHead->msgType; diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 8900ef8e7f..cd38b67fe9 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -285,7 +285,6 @@ static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) { pTmp->suid = pSrc->suid; pTmp->uid = pSrc->uid; pTmp->sver = pSrc->sver; - pTmp->source = pSrc->source; pTmp->pCreateTbReq = NULL; if (pTmp->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { if (pSrc->pCreateTbReq) { @@ -653,7 +652,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate goto end; } - pTableCxt->pData->source = SOURCE_TAOSX; + pTableCxt->pData->flags |= TD_REQ_FROM_TAOX; if(tmp == NULL){ ret = initTableColSubmitData(pTableCxt); if (ret != TSDB_CODE_SUCCESS) { From c22360a529ef92e3773d3399684ff3b5a9fe2831 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 22 Feb 2024 11:32:45 +0800 Subject: [PATCH 078/181] fix:add excluded msg for meta in tmq --- tests/system-test/7-tmq/tmq_taosx.py | 4 ++-- utils/test/c/tmq_taosx_ci.c | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index ac8a6f377c..c547385d70 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -323,7 +323,7 @@ class TDTestCase: tdSql.query("select * from st") tdSql.checkRows(8) - tdSql.execute(f'create topic topic_excluded with meta as database d1') + tdSql.execute(f'create topic topic_all with meta as database d1') consumer_dict = { "group.id": "g1", "td.connect.user": "root", @@ -333,7 +333,7 @@ class TDTestCase: consumer = Consumer(consumer_dict) try: - consumer.subscribe(["topic_excluded"]) + consumer.subscribe(["topic_all"]) except TmqError: tdLog.exit(f"subscribe error") diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 056b7dc6cf..2257089f06 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -20,6 +20,7 @@ #include #include "taos.h" #include "types.h" +#include "tmsg.h" static int running = 1; TdFilePtr g_fp = NULL; @@ -966,7 +967,14 @@ void testConsumeExcluded(int topic_type){ tmq_raw_data raw = {0}; tmq_get_raw(msg, &raw); if(topic_type == 1){ - assert(raw.raw_type != 2 && raw.raw_type != 4); + assert(raw.raw_type != 2 && raw.raw_type != 4 && + raw.raw_type != TDMT_VND_CREATE_STB && + raw.raw_type != TDMT_VND_ALTER_STB && + raw.raw_type != TDMT_VND_CREATE_TABLE && + raw.raw_type != TDMT_VND_ALTER_TABLE && + raw.raw_type != TDMT_VND_DELETE); + assert(raw.raw_type == TDMT_VND_DROP_STB || + raw.raw_type == TDMT_VND_DROP_TABLE); }else if(topic_type == 2){ assert(0); } From ab3fd3164fbc36a88faaabe1ba00ae500e55010c Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 22 Feb 2024 11:42:34 +0800 Subject: [PATCH 079/181] fix: tag desc --- source/client/src/slowQueryMonitor.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c index 933ef5de16..caa8fe1995 100644 --- a/source/client/src/slowQueryMonitor.c +++ b/source/client/src/slowQueryMonitor.c @@ -27,18 +27,16 @@ const int64_t msInSeconds = 1000; const int64_t msInMinutes = 60 * 1000; static const char* getSlowQueryLableCostDesc(int64_t cost) { - if (cost >= 10000 * msInSeconds) { - return " > 10000 seconds"; - } else if (cost >= 1000 * msInSeconds) { - return " > 1000 seconds"; + if (cost >= 1000 * msInSeconds) { + return "1000s-"; } else if (cost >= 100 * msInSeconds) { - return " > 100 seconds"; + return "100-1000s"; } else if (cost >= 10 * msInSeconds) { - return " > 10 seconds"; + return "10-100s"; } else if (cost >= 3 * msInSeconds) { - return " > 3 seconds"; + return "3-10s"; } - return "< 3 s"; + return "0-3s"; } void clientSlowQueryMonitorInit(const char* clusterKey) { From 2f6fc4a9ce29516234c71202985d573615750e7f Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 22 Feb 2024 04:55:54 +0000 Subject: [PATCH 080/181] grant default --- include/libs/monitor/monitor.h | 2 +- source/dnode/mnode/impl/src/mndMain.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index ecd59a51a8..256be26999 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -153,7 +153,7 @@ typedef struct { } SMonStbInfo; typedef struct { - uint32_t expire_time; + int64_t expire_time; int64_t timeseries_used; int64_t timeseries_total; } SMonGrantInfo; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 249ddb74c6..b188d314d9 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -989,8 +989,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 1000; pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed; if (pMnode->grant.expireTimeMS == 0) { - pGrantInfo->expire_time = INT32_MAX; - pGrantInfo->timeseries_total = INT32_MAX; + pGrantInfo->expire_time = 0; + pGrantInfo->timeseries_total = 0; } mndReleaseRpc(pMnode); From 607971b51455500695844d2125afda23acceda0a Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 22 Feb 2024 06:54:06 +0000 Subject: [PATCH 081/181] uptime unit --- source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index 413135a08c..21e25f5535 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -34,7 +34,7 @@ static void dmGetMonitorBasicInfoBasic(SDnode *pDnode, SMonBasicInfo *pInfo) { static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { //pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f); - pInfo->uptime = taosGetTimestampMs() - pDnode->data.rebootTime; + pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) /1000.0f; pInfo->has_mnode = pDnode->wrappers[MNODE].required; pInfo->has_qnode = pDnode->wrappers[QNODE].required; pInfo->has_snode = pDnode->wrappers[SNODE].required; From c859213ce54a9faba038ca89ddbb0a6e90d7b89d Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 22 Feb 2024 15:34:12 +0800 Subject: [PATCH 082/181] fix: free datablocks that is skipped decided by firt row ts --- source/libs/executor/src/tsort.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index fde5f22aaa..9ff903cdb9 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -1125,8 +1125,11 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { int64_t firstRowTs = *(int64_t*)tsCol->pData; if ((pOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs) || (pOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) { - continue; - } + if (bExtractedBlock) { + blockDataDestroy(pBlk); + } + continue; + } } if (pBlk != NULL) { @@ -1149,10 +1152,11 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { tSimpleHashClear(mUidBlk); code = sortBlocksToExtSource(pHandle, aBlkSort, pOrder, aExtSrc); if (code != TSDB_CODE_SUCCESS) { - tSimpleHashCleanup(mUidBlk); - taosArrayDestroy(aBlkSort); - taosArrayDestroy(aExtSrc); - return code; + for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) { + blockDataDestroy(taosArrayGetP(aBlkSort, i)); + } + taosArrayClear(aBlkSort); + break; } int64_t el = taosGetTimestampUs() - p; @@ -1165,6 +1169,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { szSort = 0; qDebug("source %zu created", taosArrayGetSize(aExtSrc)); } + if (pBlk == NULL) { break; } @@ -1180,6 +1185,9 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { } tSimpleHashCleanup(mUidBlk); + for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) { + blockDataDestroy(taosArrayGetP(aBlkSort, i)); + } taosArrayDestroy(aBlkSort); tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL); if (!tsortIsClosed(pHandle)) { @@ -1188,7 +1196,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { taosArrayDestroy(aExtSrc); tSimpleHashCleanup(mTableNumRows); pHandle->type = SORT_SINGLESOURCE_SORT; - return TSDB_CODE_SUCCESS; + return code; } static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { From 4037a7a1a5070666b5bb312e995a10ce90b3b6bb Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 22 Feb 2024 15:44:50 +0800 Subject: [PATCH 083/181] fix:change datablock to old version for compatibility --- include/common/tdatablock.h | 5 ++++- source/client/src/clientImpl.c | 10 +++++----- source/client/src/clientMsgHandler.c | 4 ++-- source/common/src/tdatablock.c | 10 ++++++---- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mnode/impl/src/mndShow.c | 2 +- source/dnode/mnode/impl/src/mndStb.c | 2 +- source/dnode/vnode/src/tq/tqScan.c | 2 +- source/libs/command/src/command.c | 2 +- source/libs/command/src/explain.c | 2 +- source/libs/executor/src/dataDispatcher.c | 2 +- source/libs/parser/src/parInsertUtil.c | 4 ++-- source/libs/stream/src/streamDispatch.c | 4 ++-- 13 files changed, 28 insertions(+), 23 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index bcb40f4175..e305f4c2ec 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -32,6 +32,9 @@ typedef struct SBlockOrderInfo { SColumnInfoData* pColData; } SBlockOrderInfo; +#define BLOCK_VERSION_1 1 +#define BLOCK_VERSION_2 2 + #define NBIT (3u) #define BitPos(_n) ((_n) & ((1 << NBIT) - 1)) #define BMCharPos(bm_, r_) ((bm_)[(r_) >> NBIT]) @@ -253,7 +256,7 @@ SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); int32_t blockGetEncodeSize(const SSDataBlock* pBlock); -int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols); +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols, int32_t bVersion); const char* blockDecode(SSDataBlock* pBlock, const char* pData); // for debug diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 9800d233e9..a8ebef763a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1811,7 +1811,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i char* pStart = p + len; for (int32_t i = 0; i < numOfCols; ++i) { - int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i]; + int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i]; if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) { int32_t* offset = (int32_t*)pStart; @@ -1910,8 +1910,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int char* pStart = p; char* pStart1 = p1; for (int32_t i = 0; i < numOfCols; ++i) { - int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i]; - int32_t colLen1 = (blockVersion == 1) ? htonl(colLength1[i]) : colLength1[i]; + int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i]; + int32_t colLen1 = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength1[i]) : colLength1[i]; if (ASSERT(colLen < dataLen)) { tscError("doConvertJson error: colLen:%d >= dataLen:%d", colLen, dataLen); return TSDB_CODE_TSC_INTERNAL_ERROR; @@ -1970,7 +1970,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int } colLen1 = len; totalLen += colLen1; - colLength1[i] = (blockVersion == 1) ? htonl(len) : len; + colLength1[i] = (blockVersion == BLOCK_VERSION_1) ? htonl(len) : len; } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { len = numOfRows * sizeof(int32_t); memcpy(pStart1, pStart, len); @@ -2059,7 +2059,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 char* pStart = p; for (int32_t i = 0; i < numOfCols; ++i) { - if(blockVersion == 1){ + if(blockVersion == BLOCK_VERSION_1){ colLength[i] = htonl(colLength[i]); } if (colLength[i] >= dataLen) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index e0cedb9924..3aa46d135e 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -499,7 +499,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); - int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); + int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS, BLOCK_VERSION_1); blockDataDestroy(pBlock); if (len != rspSize - sizeof(SRetrieveTableRsp)) { @@ -611,7 +611,7 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS); - int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS); + int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS, BLOCK_VERSION_1); blockDataDestroy(pBlock); if (len != rspSize - sizeof(SRetrieveTableRsp)) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5382259899..c6f004f83b 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2191,12 +2191,12 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha return TSDB_CODE_SUCCESS; } -int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols, int32_t bVersion) { int32_t dataLen = 0; // todo extract method int32_t* version = (int32_t*)data; - *version = 2; + *version = bVersion; data += sizeof(int32_t); int32_t* actualLen = (int32_t*)data; @@ -2277,7 +2277,9 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { data += colSizes[col]; } -// colSizes[col] = htonl(colSizes[col]); + if(bVersion == BLOCK_VERSION_1){ + colSizes[col] = htonl(colSizes[col]); + } // uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type, // htonl(colSizes[col]), colSizes[col]); } @@ -2338,7 +2340,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { pStart += sizeof(int32_t) * numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { - if(version == 1){ + if(version == BLOCK_VERSION_1){ colLen[i] = htonl(colLen[i]); } ASSERT(colLen[i] >= 0); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 14853009e0..b36dc71322 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -401,7 +401,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { pStart += sizeof(SSysTableSchema); } - int32_t len = blockEncode(pBlock, pStart, numOfCols); + int32_t len = blockEncode(pBlock, pStart, numOfCols, BLOCK_VERSION_1); pRsp->numOfRows = htonl(pBlock->info.rows); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 8e7e72aa0e..e46c057129 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -328,7 +328,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { pStart += sizeof(SSysTableSchema); } - int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); + int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns, BLOCK_VERSION_1); } pRsp->numOfRows = htonl(rowsRead); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 1ddd2f34e6..ed1192c490 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -3150,7 +3150,7 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t // pStart += sizeof(SSysTableSchema); // } // -// int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); +// int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns, BLOCK_VERSION_1); // } // // pRsp->numOfRows = htonl(rowsRead); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 5432637482..4e4e1481f9 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -28,7 +28,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t pRetrieve->compressed = 0; pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_2); actualLen += sizeof(SRetrieveTableRspForTmq); taosArrayPush(pRsp->blockDataLen, &actualLen); taosArrayPush(pRsp->blockData, &buf); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 6bae0e1022..e74c735497 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -40,7 +40,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); - int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); + int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols, BLOCK_VERSION_1); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 66b50bcb47..d2d14bb9f3 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1817,7 +1817,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htobe64((int64_t)rowNum); - int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); + int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock), BLOCK_VERSION_1); rsp->compLen = htonl(len); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index abe566473f..fb57d6b531 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -76,7 +76,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pEntry->dataLen = 0; pBuf->useSize = sizeof(SDataCacheEntry); - pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); + pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols, BLOCK_VERSION_1); // ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); // ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 6b655bfae6..514f2ca64e 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -718,7 +718,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate goto end; } fields += sizeof(int8_t) + sizeof(int32_t); - if (needChangeLength && version == 1) { + if (needChangeLength && version == BLOCK_VERSION_1) { pStart += htonl(colLength[j]); } else { pStart += colLength[j]; @@ -749,7 +749,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate goto end; } fields += sizeof(int8_t) + sizeof(int32_t); - if (needChangeLength && version == 1) { + if (needChangeLength && version == BLOCK_VERSION_1) { pStart += htonl(colLength[i]); } else { pStart += colLength[i]; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index b51845d152..080c2c9556 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -188,7 +188,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_1); SStreamRetrieveReq req = { .streamId = pTask->id.streamId, @@ -772,7 +772,7 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); pRetrieve->numOfCols = htonl(numOfCols); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_1); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pReq->dataLen, &actualLen); From a6f70ad8c203233b625bef6d6d52c9614998daad Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Thu, 22 Feb 2024 16:13:50 +0800 Subject: [PATCH 084/181] fix: slow query desc --- source/client/src/slowQueryMonitor.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c index caa8fe1995..b41343443d 100644 --- a/source/client/src/slowQueryMonitor.c +++ b/source/client/src/slowQueryMonitor.c @@ -23,17 +23,17 @@ const int slowQueryLabelCount = 4; const char* slowQueryLabels[] = {"cluster_id", "username", "result", "duration"}; static const char* defaultClusterID = ""; -const int64_t msInSeconds = 1000; +const int64_t usInSeconds = 1000 * 1000; const int64_t msInMinutes = 60 * 1000; static const char* getSlowQueryLableCostDesc(int64_t cost) { - if (cost >= 1000 * msInSeconds) { + if (cost >= 1000 * usInSeconds) { return "1000s-"; - } else if (cost >= 100 * msInSeconds) { + } else if (cost >= 100 * usInSeconds) { return "100-1000s"; - } else if (cost >= 10 * msInSeconds) { + } else if (cost >= 10 * usInSeconds) { return "10-100s"; - } else if (cost >= 3 * msInSeconds) { + } else if (cost >= 3 * usInSeconds) { return "3-10s"; } return "0-3s"; From ac36dd46d5e5f4b27b5639b6bc0bffa29afcaa78 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 22 Feb 2024 16:32:09 +0800 Subject: [PATCH 085/181] fix: add test case --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/tms_memleak.py | 51 ++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 tests/system-test/2-query/tms_memleak.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index cc15df33a1..12f1f62f63 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -49,6 +49,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/compact-col.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tms_memleak.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 3 diff --git a/tests/system-test/2-query/tms_memleak.py b/tests/system-test/2-query/tms_memleak.py new file mode 100644 index 0000000000..0d5cdd8272 --- /dev/null +++ b/tests/system-test/2-query/tms_memleak.py @@ -0,0 +1,51 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TD-] + ''' + return + + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.conn = conn + + def restartTaosd(self, index=1, dbname="db"): + tdDnodes.stop(index) + tdDnodes.startWithoutSleep(index) + tdSql.execute(f"use tms_memleak") + + def run(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists tms_memleak") + tdSql.execute("create database if not exists tms_memleak") + tdSql.execute('use tms_memleak') + + tdSql.execute('create table st(ts timestamp, f int) tags (t int);') + + tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 1)('2021-04-19 00:00:02', 2)('2021-04-19 00:00:03', 3)('2021-04-19 00:00:04', 4)") + + tdSql.execute("insert into ct2 using st tags(2) values('2021-04-20 00:00:01', 5)('2021-04-20 00:00:02', 6)('2021-04-20 00:00:03', 7)('2021-04-20 00:00:04', 8)") + + tdSql.execute("insert into ct3 using st tags(3) values('2021-04-21 00:00:01', 5)('2021-04-21 00:00:02', 6)('2021-04-21 00:00:03', 7)('2021-04-21 00:00:04', 8)") + + tdSql.execute("insert into ct4 using st tags(4) values('2021-04-22 00:00:01', 5)('2021-04-22 00:00:02', 6)('2021-04-22 00:00:03', 7)('2021-04-22 00:00:04', 8)") + + tdSql.query("select * from st order by ts limit 1 "); + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1); + + tdSql.execute('drop database tms_memleak') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From d8361e979c50911b281ae1b6237b75ab65e79f84 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 22 Feb 2024 16:47:27 +0800 Subject: [PATCH 086/181] fix: add check s3 upload over logic --- tests/army/enterprise/s3/s3_basic.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 58dbd12d7b..5d9a52ebc4 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -81,15 +81,23 @@ class TDTestCase(TBase): cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data" tdLog.info(cmd) loop = 0 - while len(eos.runRetList(cmd)) > 0 and loop < 100: - time.sleep(5) + rets = [] + while loop < 500: + rets = eos.runRetList(cmd) + cnt = len(rets) + if cnt == 0: + tdLog.info("All data file upload to server over.") + break + time.sleep(3) self.trimDb(True) loop += 1 - tdLog.info(f"loop={loop} wait 5s...") + tdLog.info(f"loop={loop} data files have {cnt} wait 5s...") if loop == 4: sc.dnodeStop(1) time.sleep(2) sc.dnodeStart(1) + if len(rets) > 0: + tdLog.exit(f"s3 can not upload all data to server. data files={rets}") def checkStreamCorrect(self): sql = f"select count(*) from {self.db}.stm1" From 37fd11adf264c3002ef5f32c199a2322d01fae5c Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 22 Feb 2024 08:53:11 +0000 Subject: [PATCH 087/181] mnode role offline --- source/libs/monitor/src/monFramework.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c index 15793d07fb..cc0a846e3b 100644 --- a/source/libs/monitor/src/monFramework.c +++ b/source/libs/monitor/src/monFramework.c @@ -615,8 +615,27 @@ void monGenMnodeRoleTable(SMonInfo *pMonitor){ const char *sample_labels[] = {buf, mnode_id, pMnodeDesc->mnode_ep}; + bool dnodeIsOnline = false; + for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) { + SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i); + + if(pDnodeDesc->dnode_id == pMnodeDesc->mnode_id){ + if(strcmp(pDnodeDesc->status, "ready") == 0){ + dnodeIsOnline = true; + } + } + } + metric = taosHashGet(tsMonitor.metrics, MNODE_ROLE, strlen(MNODE_ROLE)); - taos_gauge_set(*metric, pMnodeDesc->syncState, sample_labels); + + if(dnodeIsOnline){ + taos_gauge_set(*metric, pMnodeDesc->syncState, sample_labels); + } + else{ + taos_gauge_set(*metric, 0, sample_labels); + } + //metric = taosHashGet(tsMonitor.metrics, MNODE_ROLE, strlen(MNODE_ROLE)); + //taos_gauge_set(*metric, pMnodeDesc->syncState, sample_labels); } } From 6ac6e8f2bd382c2e3b41dd604e43f197135a770c Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 22 Feb 2024 17:46:36 +0800 Subject: [PATCH 088/181] enh: unit test and optimization for base58 --- include/util/tbase58.h | 3 ++ source/util/src/tbase58.c | 34 +++++++++--- source/util/test/CMakeLists.txt | 8 +++ source/util/test/tbaseCodecTest.cpp | 80 +++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+), 8 deletions(-) create mode 100644 source/util/test/tbaseCodecTest.cpp diff --git a/include/util/tbase58.h b/include/util/tbase58.h index e1b03f8a8f..ab62e1926e 100644 --- a/include/util/tbase58.h +++ b/include/util/tbase58.h @@ -22,6 +22,9 @@ extern "C" { #endif +#define TBASE_MAX_ILEN 4096 +#define TBASE_MAX_OLEN 5653 + uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen); char *base58_encode(const uint8_t *value, int32_t vlen); diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c index fa3ecadd14..e1cee72b47 100644 --- a/source/util/src/tbase58.c +++ b/source/util/src/tbase58.c @@ -18,24 +18,29 @@ #include #include -#define BASE_BUF_SIZE 256 +#define TBASE_BUF_SIZE 256 static const char *basis_58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; char *base58_encode(const uint8_t *value, int32_t vlen) { const uint8_t *pb = value; const uint8_t *pe = pb + vlen; - uint8_t buf[BASE_BUF_SIZE] = {0}; + uint8_t buf[TBASE_BUF_SIZE] = {0}; uint8_t *pbuf = &buf[0]; bool bfree = false; int32_t nz = 0, size = 0, len = 0; + if (vlen > TBASE_MAX_ILEN) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + while (pb != pe && *pb == 0) { ++pb; ++nz; } size = (pe - pb) * 69 / 50 + 1; - if (size > BASE_BUF_SIZE) { + if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -47,7 +52,7 @@ char *base58_encode(const uint8_t *value, int32_t vlen) { int32_t num = *pb; int32_t i = 0; for (int32_t j = (int32_t)size - 1; (num != 0 || i < len) && j >= 0; --j, ++i) { - num += ((int32_t)buf[j]) << 8; + num += ((int32_t)pbuf[j]) << 8; pbuf[j] = num % 58; num /= 58; } @@ -57,7 +62,7 @@ char *base58_encode(const uint8_t *value, int32_t vlen) { const uint8_t *pi = pbuf + (size - len); while (pi != pbuf + size && *pi == 0) ++pi; - uint8_t *result = taosMemoryCalloc(1, size + 1); + uint8_t *result = taosMemoryCalloc(1, nz + (pbuf + size - pi) + 1); if (!result) { terrno = TSDB_CODE_OUT_OF_MEMORY; if (bfree) taosMemoryFree(pbuf); @@ -83,11 +88,23 @@ static const signed char index_58[256] = { uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { const char *pe = value + inlen; - uint8_t buf[BASE_BUF_SIZE] = {0}; + uint8_t buf[TBASE_BUF_SIZE] = {0}; uint8_t *pbuf = &buf[0]; bool bfree = false; int32_t nz = 0, size = 0, len = 0; + if (inlen > TBASE_MAX_OLEN) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + + for (int32_t i = 0; i < inlen; ++i) { + if (value[i] == 0) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + } + while (*value && isspace(*value)) ++value; while (*value == '1') { ++nz; @@ -95,7 +112,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { } size = (int32_t)(pe - value) * 733 / 1000 + 1; - if (size > BASE_BUF_SIZE) { + if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -106,6 +123,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { while (*value && !isspace(*value)) { int32_t num = index_58[(uint8_t)*value]; if (num == -1) { + terrno = TSDB_CODE_INVALID_PARA; if (bfree) taosMemoryFree(pbuf); return NULL; } @@ -127,7 +145,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { const uint8_t *it = pbuf + (size - len); while (it != pbuf + size && *it == 0) ++it; - uint8_t *result = taosMemoryCalloc(1, size + 1); + uint8_t *result = taosMemoryCalloc(1, inlen + 1); if (!result) { if (bfree) taosMemoryFree(pbuf); terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index f4f3880388..5e32bd82b7 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -100,3 +100,11 @@ add_test( NAME talgoTest COMMAND talgoTest ) + +# tbaseCodecTest +add_executable(tbaseCodecTest "tbaseCodecTest.cpp") +target_link_libraries(tbaseCodecTest os util common gtest_main) +add_test( + NAME tbaseCodecTest + COMMAND tbaseCodecTest + diff --git a/source/util/test/tbaseCodecTest.cpp b/source/util/test/tbaseCodecTest.cpp new file mode 100644 index 0000000000..70639e7ec9 --- /dev/null +++ b/source/util/test/tbaseCodecTest.cpp @@ -0,0 +1,80 @@ +#include +#include + +#include +#include "os.h" +#include "taos.h" +#include "taoserror.h" +#include "tbase58.h" +#include "tbase64.h" +#include "tglobal.h" + +using namespace std; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +static void checkBase58Codec(uint8_t *pRaw, int32_t rawLen, int32_t index) { + char *pEnc = base58_encode((const uint8_t *)pRaw, rawLen); + ASSERT_NE(nullptr, pEnc); + + int32_t encLen = strlen(pEnc); + std::cout << "index:" << index << ", encLen is " << encLen << std::endl; + int32_t decLen = 0; + char *pDec = (char *)base58_decode((const char *)pEnc, encLen, &decLen); + std::cout << "index:" << index << ", decLen is " << decLen << std::endl; + ASSERT_NE(nullptr, pDec); + ASSERT_EQ(rawLen, decLen); + ASSERT_LE(rawLen, encLen); + ASSERT_EQ(0, strncmp((char *)pRaw, pDec, rawLen)); + taosMemoryFreeClear(pDec); + taosMemoryFreeClear(pEnc); +} + +TEST(TD_BASE_CODEC_TEST, tbase58_test) { + const int32_t TEST_LEN_MAX = TBASE_MAX_ILEN; + const int32_t TEST_LEN_STEP = 10; + int32_t rawLen = 0; + uint8_t *pRaw = NULL; + + pRaw = (uint8_t *)taosMemoryCalloc(1, TEST_LEN_MAX); + ASSERT_NE(nullptr, pRaw); + + // 1. normal case + // string blend with char and '\0' + rawLen = TEST_LEN_MAX; + for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) { + checkBase58Codec(pRaw, rawLen, i); + pRaw[i] = i & 127; + } + + // string without '\0' + for (int32_t i = 0; i < TEST_LEN_MAX; ++i) { + pRaw[i] = i & 127; + } + checkBase58Codec(pRaw, TEST_LEN_MAX, 0); + for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) { + rawLen = i; + checkBase58Codec(pRaw, rawLen, i); + } + taosMemoryFreeClear(pRaw); + ASSERT_EQ(nullptr, pRaw); + + // 2. overflow case + char tmp[1]; + char *pEnc = base58_encode((const uint8_t *)tmp, TBASE_MAX_ILEN + 1); + ASSERT_EQ(nullptr, pEnc); + char *pDec = (char *)base58_decode((const char *)tmp, TBASE_MAX_OLEN + 1, NULL); + ASSERT_EQ(nullptr, pDec); + + taosMemoryFreeClear(pRaw); + ASSERT_EQ(nullptr, pRaw); +} \ No newline at end of file From 604ae220f306d5a794d4082560aa3e2367f1aaf8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 22 Feb 2024 17:48:33 +0800 Subject: [PATCH 089/181] fix:change datablock to old version for compatibility --- include/common/tdatablock.h | 2 +- source/client/src/clientMsgHandler.c | 4 +- source/client/src/clientTmq.c | 67 ++++++++++++++------- source/common/src/tdatablock.c | 12 ++-- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mnode/impl/src/mndShow.c | 2 +- source/dnode/mnode/impl/src/mndStb.c | 2 +- source/dnode/vnode/src/tq/tqScan.c | 2 +- source/libs/command/src/command.c | 2 +- source/libs/command/src/explain.c | 2 +- source/libs/executor/src/dataDispatcher.c | 2 +- source/libs/stream/src/streamDispatch.c | 4 +- 12 files changed, 62 insertions(+), 41 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index e305f4c2ec..292e9a3181 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -256,7 +256,7 @@ SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); int32_t blockGetEncodeSize(const SSDataBlock* pBlock); -int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols, int32_t bVersion); +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols); const char* blockDecode(SSDataBlock* pBlock, const char* pData); // for debug diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 3aa46d135e..e0cedb9924 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -499,7 +499,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); - int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS, BLOCK_VERSION_1); + int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); blockDataDestroy(pBlock); if (len != rspSize - sizeof(SRetrieveTableRsp)) { @@ -611,7 +611,7 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS); - int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS, BLOCK_VERSION_1); + int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS); blockDataDestroy(pBlock); if (len != rspSize - sizeof(SRetrieveTableRsp)) { diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 69681b9ae0..ebd4e1c7b6 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1562,17 +1562,39 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { return pRspObj; } -SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { - SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); - pRspObj->resType = RES_TYPE__TMQ; +void changeByteEndian(char* pData){ + char* p = pData; + + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | + // version: + int32_t blockVersion = *(int32_t*)p; + ASSERT(blockVersion == BLOCK_VERSION_1); + *(int32_t*)p = BLOCK_VERSION_2; + + p += sizeof(int32_t); + p += sizeof(int32_t); + p += sizeof(int32_t); + int32_t cols = *(int32_t*)p; + p += sizeof(int32_t); + p += sizeof(int32_t); + p += sizeof(uint64_t); + // check fields + p += cols * (sizeof(int8_t) + sizeof(int32_t)); + + int32_t* colLength = (int32_t*)p; + for (int32_t i = 0; i < cols; ++i) { + colLength[i] = htonl(colLength[i]); + } +} + +static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj* pRspObj) { (*numOfRows) = 0; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); pRspObj->vgId = pWrapper->vgHandle->vgId; pRspObj->resIter = -1; - memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; @@ -1584,11 +1606,21 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, } // extract the rows in this data packet for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { - SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i); - int64_t rows = htobe64(pRetrieve->numOfRows); + void* pRetrieve = taosArrayGetP(pRspObj->rsp.blockData, i); + void* rawData = NULL; + int64_t rows = 0; + // deal with compatibility + if(*(int64_t*)pRetrieve == 0){ + rawData = ((SRetrieveTableRsp*)pRetrieve)->data; + rows = htobe64(((SRetrieveTableRsp*)pRetrieve)->numOfRows); + }else if(*(int64_t*)pRetrieve == 1){ + rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data; + rows = htobe64(((SRetrieveTableRspForTmq*)pRetrieve)->numOfRows); + } + pVg->numOfRows += rows; (*numOfRows) += rows; - + changeByteEndian(rawData); if (needTransformSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); if(schema){ @@ -1596,29 +1628,22 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, } } } +} +SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { + SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); + pRspObj->resType = RES_TYPE__TMQ; + memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); + tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, pRspObj); return pRspObj; } SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); pRspObj->resType = RES_TYPE__TMQ_METADATA; - tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); - tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); - pRspObj->vgId = pWrapper->vgHandle->vgId; - pRspObj->resIter = -1; memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp)); - pRspObj->resInfo.totalRows = 0; - pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; - - // extract the rows in this data packet - for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { - SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i); - int64_t rows = htobe64(pRetrieve->numOfRows); - pVg->numOfRows += rows; - (*numOfRows) += rows; - } + tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, (SMqRspObj*)pRspObj); return pRspObj; } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c6f004f83b..49f1f1eb0e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2191,12 +2191,12 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha return TSDB_CODE_SUCCESS; } -int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols, int32_t bVersion) { +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { int32_t dataLen = 0; // todo extract method int32_t* version = (int32_t*)data; - *version = bVersion; + *version = BLOCK_VERSION_1; data += sizeof(int32_t); int32_t* actualLen = (int32_t*)data; @@ -2277,9 +2277,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols, in data += colSizes[col]; } - if(bVersion == BLOCK_VERSION_1){ - colSizes[col] = htonl(colSizes[col]); - } + colSizes[col] = htonl(colSizes[col]); // uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type, // htonl(colSizes[col]), colSizes[col]); } @@ -2340,9 +2338,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { pStart += sizeof(int32_t) * numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { - if(version == BLOCK_VERSION_1){ - colLen[i] = htonl(colLen[i]); - } + colLen[i] = htonl(colLen[i]); ASSERT(colLen[i] >= 0); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index b36dc71322..14853009e0 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -401,7 +401,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { pStart += sizeof(SSysTableSchema); } - int32_t len = blockEncode(pBlock, pStart, numOfCols, BLOCK_VERSION_1); + int32_t len = blockEncode(pBlock, pStart, numOfCols); pRsp->numOfRows = htonl(pBlock->info.rows); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index e46c057129..8e7e72aa0e 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -328,7 +328,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { pStart += sizeof(SSysTableSchema); } - int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns, BLOCK_VERSION_1); + int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); } pRsp->numOfRows = htonl(rowsRead); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index ed1192c490..1ddd2f34e6 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -3150,7 +3150,7 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t // pStart += sizeof(SSysTableSchema); // } // -// int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns, BLOCK_VERSION_1); +// int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); // } // // pRsp->numOfRows = htonl(rowsRead); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 4e4e1481f9..5432637482 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -28,7 +28,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t pRetrieve->compressed = 0; pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_2); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRspForTmq); taosArrayPush(pRsp->blockDataLen, &actualLen); taosArrayPush(pRsp->blockData, &buf); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index e74c735497..6bae0e1022 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -40,7 +40,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); - int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols, BLOCK_VERSION_1); + int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index d2d14bb9f3..66b50bcb47 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1817,7 +1817,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htobe64((int64_t)rowNum); - int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock), BLOCK_VERSION_1); + int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index fb57d6b531..abe566473f 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -76,7 +76,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pEntry->dataLen = 0; pBuf->useSize = sizeof(SDataCacheEntry); - pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols, BLOCK_VERSION_1); + pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); // ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); // ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 080c2c9556..b51845d152 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -188,7 +188,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_1); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); SStreamRetrieveReq req = { .streamId = pTask->id.streamId, @@ -772,7 +772,7 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); pRetrieve->numOfCols = htonl(numOfCols); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_1); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pReq->dataLen, &actualLen); From b9b14c4633b168262fed787bb1022e50e424e46b Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 22 Feb 2024 17:50:41 +0800 Subject: [PATCH 090/181] fix: monitor client config --- source/common/src/tglobal.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f89cc1f4fa..72038957e0 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -538,6 +538,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { return -1; if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; return 0; } @@ -1088,6 +1090,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsQueryMaxConcurrentTables = cfgGetItem(pCfg, "queryMaxConcurrentTables")->i64; tsMetaCacheMaxSize = cfgGetItem(pCfg, "metaCacheMaxSize")->i32; tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32; + tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; + tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) { return -1; } From 5b0fc1544d08e74487deca8f5300bb4e99ba23f7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 22 Feb 2024 18:00:12 +0800 Subject: [PATCH 091/181] enh: unit test and optimization for base58 --- source/util/test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 5e32bd82b7..3514c578e9 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -107,4 +107,4 @@ target_link_libraries(tbaseCodecTest os util common gtest_main) add_test( NAME tbaseCodecTest COMMAND tbaseCodecTest - +) From 75caea4fbf6b47a17b87528960f69ce5d79e40d9 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 22 Feb 2024 19:04:21 +0800 Subject: [PATCH 092/181] enh: code optimization for tbase58 --- source/util/src/tbase58.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c index e1cee72b47..2566d54431 100644 --- a/source/util/src/tbase58.c +++ b/source/util/src/tbase58.c @@ -145,7 +145,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { const uint8_t *it = pbuf + (size - len); while (it != pbuf + size && *it == 0) ++it; - uint8_t *result = taosMemoryCalloc(1, inlen + 1); + uint8_t *result = taosMemoryCalloc(1, nz + (pbuf + size - it) + 1); if (!result) { if (bfree) taosMemoryFree(pbuf); terrno = TSDB_CODE_OUT_OF_MEMORY; From af1f61cd3902761b6655456ed60147cb128047dc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Feb 2024 19:05:30 +0800 Subject: [PATCH 093/181] fix(stream): fix the compatible issue when the fill-history exists. --- source/libs/stream/src/streamTask.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index f3494377d6..c0cd297286 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -494,6 +494,8 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i pRange->range.maxVer = ver; pRange->range.minVer = ver; } else { + // the initial value of processedVer/nextProcessVer/checkpointVer for stream task with related fill-history task + // is set at the mnode. if (pTask->info.fillHistory == 1) { pChkInfo->checkpointVer = pRange->range.maxVer; pChkInfo->processedVer = pRange->range.maxVer; @@ -502,6 +504,14 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i pChkInfo->checkpointVer = pRange->range.minVer - 1; pChkInfo->processedVer = pRange->range.minVer - 1; pChkInfo->nextProcessVer = pRange->range.minVer; + + { // for compatible purpose, remove it later + if (pRange->range.minVer == 0) { + pChkInfo->checkpointVer = 0; + pChkInfo->processedVer = 0; + stDebug("s-task:%s update the processedVer to 0 from -1 due to compatible purpose", pTask->id.idStr); + } + } } } } From eb670e12ee3e4562407328db29520f0bb5b5d480 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 22 Feb 2024 11:39:57 +0000 Subject: [PATCH 094/181] first_ep --- source/libs/monitor/src/monMain.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 6c2344826a..4ecc6f1261 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -273,7 +273,8 @@ static void monGenClusterJsonBasic(SMonInfo *pMonitor) { SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; - tjsonAddStringToObject(pMonitor->pJson, "first_ep", pInfo->first_ep); + //tjsonAddStringToObject(pMonitor->pJson, "first_ep", pInfo->first_ep); + tjsonAddStringToObject(pMonitor->pJson, "first_ep", tsFirst); tjsonAddDoubleToObject(pMonitor->pJson, "first_ep_dnode_id", pInfo->first_ep_dnode_id); tjsonAddStringToObject(pMonitor->pJson, "cluster_version", pInfo->version); //tjsonAddDoubleToObject(pMonitor->pJson, "monitor_interval", pInfo->monitor_interval); From c77fee12e03e10b9623b780139ec586e4655fe37 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 22 Feb 2024 12:07:28 +0000 Subject: [PATCH 095/181] mnode alive --- source/libs/monitor/src/monFramework.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c index cc0a846e3b..06b4a3ee3b 100644 --- a/source/libs/monitor/src/monFramework.c +++ b/source/libs/monitor/src/monFramework.c @@ -273,9 +273,22 @@ void monGenClusterInfoTable(SMonInfo *pMonitor){ for (int32_t i = 0; i < taosArrayGetSize(pInfo->mnodes); ++i) { SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->mnodes, i); + + bool dnodeIsOnline = false; + for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) { + SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i); - if(pMnodeDesc->syncState != 0){ - mnode_alive++; + if(pDnodeDesc->dnode_id == pMnodeDesc->mnode_id){ + if(strcmp(pDnodeDesc->status, "ready") == 0){ + dnodeIsOnline = true; + } + } + } + + if(dnodeIsOnline){ + if(pMnodeDesc->syncState != 0){ + mnode_alive++; + } } } From c7a50acd632af1789f2a195ce4652c268d359a32 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 23 Feb 2024 09:05:30 +0800 Subject: [PATCH 096/181] rebuild sliding window --- source/libs/executor/src/streamcountwindowoperator.c | 3 +++ source/libs/stream/src/streamSessionState.c | 1 + 2 files changed, 4 insertions(+) diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 689ba54ca6..294c2730df 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -99,6 +99,9 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, (void**)&pCurWin->winInfo.pStatePos, &size); } } + if (ts < pCurWin->winInfo.sessionWin.win.ekey) { + pBuffInfo->rebuildWindow = true; + } } else { code = pAggSup->stateStore.streamStateCountWinAddIfNotExist( pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size); diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index e2aae130e5..3d0241df75 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -488,6 +488,7 @@ SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SS void* pFileStore = getStateFileStore(pFileState); SStreamStateCur* pCur = streamStateSessionSeekKeyPrev_rocksdb(pFileStore, pWinKey); if (pCur) { + pCur->pStreamFileState = pFileState; SSessionKey key = {0}; void* pVal = NULL; int len = 0; From d8ffa65b62c215f883fda66cb45456067198bfbd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Feb 2024 09:12:27 +0800 Subject: [PATCH 097/181] fix(stream): fix the compatible issue when the fill-history exists. --- source/libs/stream/src/streamTask.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index c0cd297286..b63dc50836 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -509,6 +509,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i if (pRange->range.minVer == 0) { pChkInfo->checkpointVer = 0; pChkInfo->processedVer = 0; + pChkInfo->nextProcessVer = 1; stDebug("s-task:%s update the processedVer to 0 from -1 due to compatible purpose", pTask->id.idStr); } } From 4248da14e7ed9ce03fe99611cf939cfed61fefed Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Feb 2024 10:02:54 +0800 Subject: [PATCH 098/181] fix(stream): inc the default stream threads for each vnode. --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index ee85a909e7..7a5d554b97 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -58,7 +58,7 @@ int32_t tsNumOfMnodeQueryThreads = 4; int32_t tsNumOfMnodeFetchThreads = 1; int32_t tsNumOfMnodeReadThreads = 1; int32_t tsNumOfVnodeQueryThreads = 4; -float tsRatioOfVnodeStreamThreads = 0.5F; +float tsRatioOfVnodeStreamThreads = 1.5F; int32_t tsNumOfVnodeFetchThreads = 4; int32_t tsNumOfVnodeRsmaThreads = 2; int32_t tsNumOfQnodeQueryThreads = 4; From e80ac5a4a69e42fa6ccc48cbee67a6513c5945cb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 23 Feb 2024 10:14:59 +0800 Subject: [PATCH 099/181] fix(vnd/query): not release reader if not initialized --- source/dnode/vnode/src/vnd/vnodeQuery.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 4fc7a88494..d1c811858a 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "vnd.h" #include "tsdb.h" +#include "vnd.h" #define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType, tags) \ do { \ @@ -49,7 +49,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { // decode req if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) { code = TSDB_CODE_INVALID_MSG; - goto _exit; + goto _exit4; } metaRsp.dbId = pVnode->config.dbId; @@ -59,7 +59,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName); code = vnodeValidateTableHash(pVnode, tableFName); if (code) { - goto _exit; + goto _exit4; } // query meta @@ -67,7 +67,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { if (metaGetTableEntryByName(&mer1, infoReq.tbName) < 0) { code = terrno; - goto _exit; + goto _exit3; } metaRsp.tableType = mer1.me.type; @@ -81,7 +81,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaRsp.suid = mer1.me.uid; } else if (mer1.me.type == TSDB_CHILD_TABLE) { metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); - if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; + if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit2; strcpy(metaRsp.stbName, mer2.me.name); metaRsp.suid = mer2.me.uid; @@ -125,6 +125,12 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); _exit: + taosMemoryFree(metaRsp.pSchemas); +_exit2: + metaReaderClear(&mer2); +_exit3: + metaReaderClear(&mer1); +_exit4: rpcMsg.info = pMsg->info; rpcMsg.pCont = pRsp; rpcMsg.contLen = rspLen; @@ -141,9 +147,6 @@ _exit: *pMsg = rpcMsg; } - taosMemoryFree(metaRsp.pSchemas); - metaReaderClear(&mer2); - metaReaderClear(&mer1); return TSDB_CODE_SUCCESS; } @@ -706,5 +709,5 @@ void *vnodeGetIvtIdx(void *pVnode) { } int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid) { - return tsdbGetTableSchema(((SVnode*)pVnode)->pMeta, uid, pSchema, suid); + return tsdbGetTableSchema(((SVnode *)pVnode)->pMeta, uid, pSchema, suid); } From 31cf877f50c6b2221daa48b3e252323b81588938 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 23 Feb 2024 10:16:27 +0800 Subject: [PATCH 100/181] enh: code optimization for tbase58 --- source/util/src/tbase58.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c index 2566d54431..5eb72879c3 100644 --- a/source/util/src/tbase58.c +++ b/source/util/src/tbase58.c @@ -87,6 +87,7 @@ static const signed char index_58[256] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { + const char *pb = value; const char *pe = value + inlen; uint8_t buf[TBASE_BUF_SIZE] = {0}; uint8_t *pbuf = &buf[0]; @@ -98,20 +99,22 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { return NULL; } - for (int32_t i = 0; i < inlen; ++i) { - if (value[i] == 0) { + while (pb != pe) { + if (*pb == 0) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } + ++pb; } - while (*value && isspace(*value)) ++value; - while (*value == '1') { + pb = value; + while (pb != pe && *pb && isspace(*pb)) ++pb; + while (pb != pe && *pb == '1') { ++nz; - ++value; + ++pb; } - size = (int32_t)(pe - value) * 733 / 1000 + 1; + size = (int32_t)(pe - pb) * 733 / 1000 + 1; if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -120,8 +123,8 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { bfree = true; } - while (*value && !isspace(*value)) { - int32_t num = index_58[(uint8_t)*value]; + while (pb != pe && *pb && !isspace(*pb)) { + int32_t num = index_58[(uint8_t)*pb]; if (num == -1) { terrno = TSDB_CODE_INVALID_PARA; if (bfree) taosMemoryFree(pbuf); @@ -134,11 +137,11 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { num >>= 8; } len = i; - ++value; + ++pb; } - while (isspace(*value)) ++value; - if (*value != 0) { + while (pb != pe && isspace(*pb)) ++pb; + if (*pb != 0) { if (bfree) taosMemoryFree(pbuf); return NULL; } From 03d68a037693120001a69d6653d38c8da5e1464f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 21 Feb 2024 13:37:15 +0800 Subject: [PATCH 101/181] enh: change redo logs of trans create-stb as prepare logs --- source/dnode/mnode/impl/src/mndStb.c | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index aad1dc881b..a1aed7b735 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -617,10 +617,10 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { return 0; } -static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { +static int32_t mndSetCreateStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { sdbFreeRaw(pRedoRaw); return -1; } @@ -629,18 +629,6 @@ static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -static int32_t mndSetCreateStbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { - SSdbRaw *pUndoRaw = mndStbActionEncode(pStb); - if (pUndoRaw == NULL) return -1; - if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - sdbFreeRaw(pUndoRaw); - return -1; - } - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; - - return 0; -} - static int32_t mndSetCreateStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); if (pCommitRaw == NULL) return -1; @@ -913,8 +901,7 @@ _OVER: int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { mndTransSetDbName(pTrans, pDb->name, pStb->name); if (mndTransCheckConflict(pMnode, pTrans) != 0) return -1; - if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; - if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; + if (mndSetCreateStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) return -1; if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb) != 0) return -1; From 3a81c9777ca3297a6e7ca682a84707154add14b3 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 21 Feb 2024 20:00:04 +0800 Subject: [PATCH 102/181] enh: change redo logs of trans alter-stb and drop-stb as prepare logs --- source/dnode/mnode/impl/src/mndStb.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index a1aed7b735..b6e49b4d8b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1733,10 +1733,10 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO return 0; } -static int32_t mndSetAlterStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { +static int32_t mndSetAlterStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { sdbFreeRaw(pRedoRaw); return -1; } @@ -2142,7 +2142,7 @@ static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbOb mndTransSetRpcRsp(pTrans, pCont, contLen); } - if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -2179,11 +2179,11 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO if (mndGetIdxsByTagName(pMnode, pStb, pField0->name, &idxObj) == 0) { exist = true; } - if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (exist == true) { - if (mndSetDropIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; + if (mndSetDropIdxPrepareLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; if (mndSetDropIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; } @@ -2202,13 +2202,13 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO exist = true; } - if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (exist == true) { memcpy(idxObj.colName, nTagName, strlen(nTagName)); idxObj.colName[strlen(nTagName)] = 0; - if (mndSetAlterIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; + if (mndSetAlterIdxPrepareLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; if (mndSetAlterIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; } @@ -2341,10 +2341,10 @@ _OVER: return code; } -static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { +static int32_t mndSetDropStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { sdbFreeRaw(pRedoRaw); return -1; } @@ -2414,7 +2414,7 @@ static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *p mndTransSetDbName(pTrans, pDb->name, pStb->name); if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; - if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER; + if (mndSetDropStbPrepareLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndDropIdxsByStb(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; @@ -3534,7 +3534,7 @@ static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) { mndTransSetDbName(pTrans, pDb->name, pStb->name); if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; - if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndSetAlterStbRedoActions2(pMnode, pTrans, pDb, pStb, sql, len) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; From 8f3b6a717df0eb8a4ae4b464ff2c00ac3f9ebbfe Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 21 Feb 2024 20:03:21 +0800 Subject: [PATCH 103/181] enh: change redo logs of trans alter-db and drop-db as prepare logs --- source/dnode/mnode/impl/src/mndDb.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 077c0a9c2a..2ea1cdfe2a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -884,10 +884,10 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { return terrno; } -static int32_t mndSetAlterDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { +static int32_t mndSetAlterDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { SSdbRaw *pRedoRaw = mndDbActionEncode(pOld); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { sdbFreeRaw(pRedoRaw); return -1; } @@ -943,7 +943,7 @@ static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *p mndTransSetDbName(pTrans, pOld->name, NULL); if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; - if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndSetAlterDbPrepareLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -1120,10 +1120,10 @@ _OVER: return code; } -static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { +static int32_t mndSetDropDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { SSdbRaw *pRedoRaw = mndDbActionEncode(pDb); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1; if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; return 0; @@ -1257,7 +1257,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { goto _OVER; } - if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto _OVER; + if (mndSetDropDbPrepareLogs(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER; /*if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ /*if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ From 94f7e0c811af2a7ce4ab4e8ed9ebc853230b7341 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 21 Feb 2024 20:07:45 +0800 Subject: [PATCH 104/181] enh: change redo logs of trans create-stb-index and drop-index as prepare logs --- source/dnode/mnode/impl/inc/mndIndex.h | 8 ++++---- source/dnode/mnode/impl/src/mndIndex.c | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndIndex.h b/source/dnode/mnode/impl/inc/mndIndex.h index 2d5479bc9b..3b9b68a1b5 100644 --- a/source/dnode/mnode/impl/inc/mndIndex.h +++ b/source/dnode/mnode/impl/inc/mndIndex.h @@ -19,16 +19,16 @@ int32_t mndGetTableIdx(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq); -int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); +int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); -int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); +int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); -int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); +int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); #ifdef __cplusplus } #endif -#endif /*_TD_MND_IDX_H_*/ \ No newline at end of file +#endif /*_TD_MND_IDX_H_*/ diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index 041cc664e5..622ed0080f 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -331,10 +331,10 @@ SDbObj *mndAcquireDbByIdx(SMnode *pMnode, const char *idxName) { return mndAcquireDb(pMnode, db); } -int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { +int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1; if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; return 0; @@ -349,10 +349,10 @@ int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) return 0; } -int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { +int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { sdbFreeRaw(pRedoRaw); return -1; } @@ -482,10 +482,10 @@ _OVER: return code; } -int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { +int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1; if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; return 0; @@ -652,7 +652,7 @@ int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pSt mndTransSetSerial(pTrans); - if (mndSetCreateIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; + if (mndSetCreateIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; if (mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1) != 0) goto _OVER; @@ -771,7 +771,7 @@ static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *p if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; mndTransSetSerial(pTrans); - if (mndSetDropIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; + if (mndSetDropIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0) != 0) goto _OVER; From fbd2a0035c2a6122dc30f8332905b2ec0f9d8b3b Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 21 Feb 2024 20:09:01 +0800 Subject: [PATCH 105/181] enh: change type of trans create-snode and drop-snode as serial --- source/dnode/mnode/impl/src/mndSnode.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 1275ba7962..4243ccb77c 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -257,6 +257,7 @@ static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode"); if (pTrans == NULL) goto _OVER; + mndTransSetSerial(pTrans); mInfo("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); @@ -383,6 +384,7 @@ static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode"); if (pTrans == NULL) goto _OVER; + mndTransSetSerial(pTrans); mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER; From a371afa58e054d02fe62975ea7ff3984a33ee66f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 21 Feb 2024 20:12:29 +0800 Subject: [PATCH 106/181] enh: change type of trans create-qnode and drop-qnode as serial --- source/dnode/mnode/impl/src/mndQnode.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index a7d8c81d3f..2f4ad69e2a 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -257,6 +257,7 @@ static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-qnode"); if (pTrans == NULL) goto _OVER; + mndTransSetSerial(pTrans); mInfo("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId); if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) goto _OVER; @@ -380,6 +381,7 @@ static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-qnode"); if (pTrans == NULL) goto _OVER; + mndTransSetSerial(pTrans); mInfo("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id); if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER; From ef7b38d4f69257df739a6e602deca68482e4bd81 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 23 Feb 2024 10:30:47 +0800 Subject: [PATCH 107/181] enh: code optimization for tbase58 --- source/util/test/tbaseCodecTest.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/source/util/test/tbaseCodecTest.cpp b/source/util/test/tbaseCodecTest.cpp index 70639e7ec9..4c56979885 100644 --- a/source/util/test/tbaseCodecTest.cpp +++ b/source/util/test/tbaseCodecTest.cpp @@ -3,10 +3,10 @@ #include #include "os.h" +#include "osTime.h" #include "taos.h" #include "taoserror.h" #include "tbase58.h" -#include "tbase64.h" #include "tglobal.h" using namespace std; @@ -23,14 +23,17 @@ int main(int argc, char **argv) { } static void checkBase58Codec(uint8_t *pRaw, int32_t rawLen, int32_t index) { - char *pEnc = base58_encode((const uint8_t *)pRaw, rawLen); + int64_t start = taosGetTimestampUs(); + char *pEnc = base58_encode((const uint8_t *)pRaw, rawLen); ASSERT_NE(nullptr, pEnc); int32_t encLen = strlen(pEnc); - std::cout << "index:" << index << ", encLen is " << encLen << std::endl; + int64_t endOfEnc = taosGetTimestampUs(); + std::cout << "index:" << index << ", encLen is " << encLen << ", cost:" << endOfEnc - start << " us" << std::endl; int32_t decLen = 0; char *pDec = (char *)base58_decode((const char *)pEnc, encLen, &decLen); - std::cout << "index:" << index << ", decLen is " << decLen << std::endl; + std::cout << "index:" << index << ", decLen is " << decLen << ", cost:" << taosGetTimestampUs() - endOfEnc << " us" + << std::endl; ASSERT_NE(nullptr, pDec); ASSERT_EQ(rawLen, decLen); ASSERT_LE(rawLen, encLen); @@ -51,7 +54,7 @@ TEST(TD_BASE_CODEC_TEST, tbase58_test) { // 1. normal case // string blend with char and '\0' rawLen = TEST_LEN_MAX; - for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) { + for (int32_t i = 0; i < TEST_LEN_MAX; i += 500) { checkBase58Codec(pRaw, rawLen, i); pRaw[i] = i & 127; } @@ -61,7 +64,7 @@ TEST(TD_BASE_CODEC_TEST, tbase58_test) { pRaw[i] = i & 127; } checkBase58Codec(pRaw, TEST_LEN_MAX, 0); - for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) { + for (int32_t i = 0; i < TEST_LEN_MAX; i += 500) { rawLen = i; checkBase58Codec(pRaw, rawLen, i); } From dfef26866a4ac267457d5207b9dc16ffc6198430 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 22 Feb 2024 10:34:00 +0800 Subject: [PATCH 108/181] enh: decode sdb raw object in SdbValidateFp instead --- source/dnode/mnode/impl/src/mndDb.c | 20 +++++++++++++----- source/dnode/mnode/impl/src/mndSync.c | 17 ++-------------- source/dnode/mnode/impl/src/mndVgroup.c | 27 +++++++++++++++++-------- source/dnode/mnode/sdb/inc/sdb.h | 2 +- 4 files changed, 37 insertions(+), 29 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 2ea1cdfe2a..9f644fb290 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -41,7 +41,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew); -static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj); +static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw); static int32_t mndProcessCreateDbReq(SRpcMsg *pReq); static int32_t mndProcessAlterDbReq(SRpcMsg *pReq); @@ -256,17 +256,27 @@ _OVER: return pRow; } -static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj) { - SDbObj *pNewDb = pObj; +static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) { + SSdb *pSdb = pMnode->pSdb; + SSdbRow *pRow = NULL; + int code = -1; + + pRow = mndDbActionDecode(pRaw); + if (pRow == NULL) goto _OVER; + SDbObj *pNewDb = sdbGetRowObj(pRow); + if (pNewDb == NULL) goto _OVER; SDbObj *pOldDb = sdbAcquire(pMnode->pSdb, SDB_DB, pNewDb->name); if (pOldDb != NULL) { mError("trans:%d, db name already in use. name: %s", pTrans->id, pNewDb->name); sdbRelease(pMnode->pSdb, pOldDb); - return -1; + goto _OVER; } - return 0; + code = 0; +_OVER: + taosMemoryFree(pRow); + return code; } static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb) { diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 0fc8dad420..a1490a3813 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -77,29 +77,16 @@ static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { static int32_t mndTransValidatePrepareAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { SSdbRaw *pRaw = pAction->pRaw; SSdb *pSdb = pMnode->pSdb; - SSdbRow *pRow = NULL; - void *pObj = NULL; - int code = -1; + int code = 0; if (pRaw->status != SDB_STATUS_CREATING) goto _OUT; - pRow = (pSdb->decodeFps[pRaw->type])(pRaw); - if (pRow == NULL) goto _OUT; - pObj = sdbGetRowObj(pRow); - if (pObj == NULL) goto _OUT; - SdbValidateFp validateFp = pSdb->validateFps[pRaw->type]; - code = 0; if (validateFp) { - code = validateFp(pMnode, pTrans, pObj); + code = validateFp(pMnode, pTrans, pRaw); } _OUT: - if (pRow) { - SdbDeleteFp deleteFp = pSdb->deleteFps[pRaw->type]; - if (deleteFp) (*deleteFp)(pSdb, pRow->pObj, false); - taosMemoryFreeClear(pRow); - } return code; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index a5df9ad820..6bd29f7910 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -31,10 +31,10 @@ #define VGROUP_VER_NUMBER 1 #define VGROUP_RESERVE_SIZE 64 -static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); -static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); -static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); -static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj); +static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); +static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); +static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); +static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw); static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); @@ -181,15 +181,26 @@ _OVER: return pRow; } -static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj) { - SVgObj *pVgroup = pObj; +static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) { + SSdb *pSdb = pMnode->pSdb; + SSdbRow *pRow = NULL; + int code = -1; + + pRow = mndVgroupActionDecode(pRaw); + if (pRow == NULL) goto _OVER; + SVgObj *pVgroup = sdbGetRowObj(pRow); + if (pVgroup == NULL) goto _OVER; int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); if (maxVgId > pVgroup->vgId) { mError("trans:%d, vgroup id %d already in use. maxVgId:%d", pTrans->id, pVgroup->vgId, maxVgId); - return -1; + goto _OVER; } - return 0; + + code = 0; +_OVER: + taosMemoryFree(pRow); + return code; } static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup) { diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index 9d1125dad2..e2d4a1f6fc 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -106,7 +106,7 @@ typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj); typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj); typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc); typedef int32_t (*SdbDeployFp)(SMnode *pMnode); -typedef int32_t (*SdbValidateFp)(SMnode *pMnode, void *pTrans, void *pObj); +typedef int32_t (*SdbValidateFp)(SMnode *pMnode, void *pTrans, SSdbRaw *pRaw); typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw); typedef SSdbRaw *(*SdbEncodeFp)(void *pObj); typedef bool (*sdbTraverseFp)(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3); From 1b2bccd1d9866c78ec9c80002e530ef515bfed76 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 22 Feb 2024 15:49:30 +0800 Subject: [PATCH 109/181] enh: not use prepare log in trans create-stb anymore --- source/dnode/mnode/impl/src/mndStb.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index b6e49b4d8b..7ee1b36916 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -901,7 +901,6 @@ _OVER: int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { mndTransSetDbName(pTrans, pDb->name, pStb->name); if (mndTransCheckConflict(pMnode, pTrans) != 0) return -1; - if (mndSetCreateStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) return -1; if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb) != 0) return -1; From d3dd15e621d67ff2cb2fcd4589f3719c36ada6a5 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 23 Feb 2024 11:18:29 +0800 Subject: [PATCH 110/181] count window support having --- .../libs/executor/src/countwindowoperator.c | 12 +++-- tests/script/tsim/query/query_count0.sim | 46 +++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/countwindowoperator.c b/source/libs/executor/src/countwindowoperator.c index 3980e5ae4d..1d2a55fac8 100644 --- a/source/libs/executor/src/countwindowoperator.c +++ b/source/libs/executor/src/countwindowoperator.c @@ -129,7 +129,7 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) { return code; } -static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock) { +static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SExecTaskInfo* pTaskInfo, SFilterInfo* pFilterInfo, SSDataBlock* pBlock) { SResultRow* pResultRow = NULL; for (int32_t i = 0; i < taosArrayGetSize(pCountSup->pWinStates); i++) { SCountWindowResult* pBuff = setCountWindowOutputBuff(pExprSup, pCountSup, &pResultRow); @@ -143,6 +143,7 @@ static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, S clearWinStateBuff(pBuff); clearResultRowInitFlag(pExprSup->pCtx, pExprSup->numOfExprs); } + doFilter(pBlock, pFilterInfo, NULL); } static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) { @@ -177,7 +178,7 @@ static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) { if (pInfo->groupId == 0) { pInfo->groupId = pBlock->info.id.groupId; } else if (pInfo->groupId != pBlock->info.id.groupId) { - buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pRes); + buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pOperator->exprSupp.pFilterInfo, pRes); pInfo->groupId = pBlock->info.id.groupId; } @@ -187,7 +188,7 @@ static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) { } } - buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pRes); + buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pOperator->exprSupp.pFilterInfo, pRes); return pRes->info.rows == 0 ? NULL : pRes; } @@ -246,6 +247,11 @@ SOperatorInfo* createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo pInfo->countSup.stateIndex = 0; + code = filterInitFromNode((SNode*)pCountWindowNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); setOperatorInfo(pOperator, "CountWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT, true, OP_NOT_OPENED, pInfo, diff --git a/tests/script/tsim/query/query_count0.sim b/tests/script/tsim/query/query_count0.sim index c3a75d635b..5b95d4fad7 100644 --- a/tests/script/tsim/query/query_count0.sim +++ b/tests/script/tsim/query/query_count0.sim @@ -175,5 +175,51 @@ if $data33 != 3 then goto loop3 endi +print step3 +print =============== create database +sql create database test3 vgroups 1; +sql use test3; + +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql insert into t1 values(1648791213000,0,1,3,1.0); +sql insert into t1 values(1648791213001,2,2,3,1.1); +sql insert into t1 values(1648791223002,0,3,3,2.1); +sql insert into t1 values(1648791223003,1,4,3,3.1); +sql insert into t1 values(1648791223004,1,5,3,4.1); +sql insert into t1 values(1648791223005,2,6,3,4.1); + +$loop_count = 0 +loop4: + +sleep 300 +print 1 sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 3; +sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 3; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $rows != 1 then + print ======rows=$rows + goto loop4 +endi + +print 1 sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 6; +sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 6; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 + +# row 0 +if $rows != 0 then + print ======rows=$rows + return -1 +endi + print query_count0 end system sh/exec.sh -n dnode1 -s stop -x SIGINT From ec093cb47732f3fc63e439014d44d68c43960e60 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 23 Feb 2024 11:25:42 +0800 Subject: [PATCH 111/181] enh: ensure decoded obj cleared in new obj action validate --- source/dnode/mnode/impl/src/mndDb.c | 6 ++++-- source/dnode/mnode/impl/src/mndVgroup.c | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9f644fb290..37c2d19bd4 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -259,11 +259,12 @@ _OVER: static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) { SSdb *pSdb = pMnode->pSdb; SSdbRow *pRow = NULL; + SDbObj *pNewDb = NULL; int code = -1; pRow = mndDbActionDecode(pRaw); if (pRow == NULL) goto _OVER; - SDbObj *pNewDb = sdbGetRowObj(pRow); + pNewDb = sdbGetRowObj(pRow); if (pNewDb == NULL) goto _OVER; SDbObj *pOldDb = sdbAcquire(pMnode->pSdb, SDB_DB, pNewDb->name); @@ -275,7 +276,8 @@ static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *p code = 0; _OVER: - taosMemoryFree(pRow); + if (pNewDb) mndDbActionDelete(pSdb, pNewDb); + taosMemoryFreeClear(pRow); return code; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 6bd29f7910..b0290191bc 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -184,11 +184,12 @@ _OVER: static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) { SSdb *pSdb = pMnode->pSdb; SSdbRow *pRow = NULL; + SVgObj *pVgroup = NULL; int code = -1; pRow = mndVgroupActionDecode(pRaw); if (pRow == NULL) goto _OVER; - SVgObj *pVgroup = sdbGetRowObj(pRow); + pVgroup = sdbGetRowObj(pRow); if (pVgroup == NULL) goto _OVER; int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); @@ -199,7 +200,8 @@ static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *p code = 0; _OVER: - taosMemoryFree(pRow); + if (pVgroup) mndVgroupActionDelete(pSdb, pVgroup); + taosMemoryFreeClear(pRow); return code; } From acdc21ff6c3aaadf0d68cbbe94b2e692126dae09 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 23 Feb 2024 11:38:04 +0800 Subject: [PATCH 112/181] fix: modify retry to upload to 8 minutes --- tests/army/enterprise/s3/s3_basic.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 5d9a52ebc4..45519d925f 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -28,12 +28,12 @@ from frame import * from frame.eos import * # -# 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa +# 192.168.1.52 MINIO S3 # ''' s3EndPoint http://192.168.1.52:9000 -s3AccessKey MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa +s3AccessKey 'zOgllR6bSnw2Ah3mCNel:cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX' s3BucketName ci-bucket s3UploadDelaySec 60 ''' @@ -42,7 +42,7 @@ s3UploadDelaySec 60 class TDTestCase(TBase): updatecfgDict = { 's3EndPoint': 'http://192.168.1.52:9000', - 's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa', + 's3AccessKey': 'zOgllR6bSnw2Ah3mCNel:cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX', 's3BucketName': 'ci-bucket', 's3BlockSize': '10240', 's3BlockCacheSize': '320', @@ -78,26 +78,27 @@ class TDTestCase(TBase): self.trimDb(True) rootPath = sc.clusterRootPath() - cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data" + cmd = f"ls {rootPath}/dnode1/data2*/vnode/vnode*/tsdb/*.data" tdLog.info(cmd) loop = 0 rets = [] - while loop < 500: + while loop < 180: + time.sleep(3) rets = eos.runRetList(cmd) cnt = len(rets) if cnt == 0: tdLog.info("All data file upload to server over.") - break - time.sleep(3) + break self.trimDb(True) - loop += 1 - tdLog.info(f"loop={loop} data files have {cnt} wait 5s...") - if loop == 4: + tdLog.info(f"loop={loop} no upload {cnt} data files wait 3s retry ...") + if loop == 0: sc.dnodeStop(1) time.sleep(2) sc.dnodeStart(1) + loop += 1 + if len(rets) > 0: - tdLog.exit(f"s3 can not upload all data to server. data files={rets}") + tdLog.exit(f"s3 can not upload all data to server. data files cnt={len(rets)} list={rets}") def checkStreamCorrect(self): sql = f"select count(*) from {self.db}.stm1" From f2c1b7da05eaca83d30b3ed5f5a8d490be78d097 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 23 Feb 2024 13:16:59 +0800 Subject: [PATCH 113/181] enh: keep in redo stage on context switch for rollback trans --- source/dnode/mnode/impl/src/mndTrans.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 1adc4ed4bf..5c76baff08 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1419,7 +1419,7 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool pTrans->stage = TRN_STAGE_COMMIT; mInfo("trans:%d, stage from redoAction to commit", pTrans->id); continueExec = true; - } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) { + } else if (code == TSDB_CODE_ACTION_IN_PROGRESS || code == TSDB_CODE_MND_TRANS_CTX_SWITCH) { mInfo("trans:%d, stage keep on redoAction since %s", pTrans->id, tstrerror(code)); continueExec = false; } else { From b6d28b9b9d0aebe6c516688b0352ec6e98901135 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 23 Feb 2024 14:00:54 +0800 Subject: [PATCH 114/181] fix:[TD-28698] error message --- include/util/taoserror.h | 1 + source/libs/parser/src/parTranslater.c | 4 ++-- source/libs/parser/src/parUtil.c | 2 ++ source/util/src/terror.c | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index c0c20a0fde..556479f547 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -757,6 +757,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_COL_QUERY_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x266D) #define TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE TAOS_DEF_ERROR_CODE(0, 0x266E) #define TSDB_CODE_PAR_ORDERBY_AMBIGUOUS TAOS_DEF_ERROR_CODE(0, 0x266F) +#define TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT TAOS_DEF_ERROR_CODE(0, 0x2670) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index abe50a27da..bb22e04cdf 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1672,11 +1672,11 @@ static int32_t dataTypeComp(const SDataType* l, const SDataType* r) { static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { if (isMultiResFunc(pOp->pLeft)) { - generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); + generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT, ((SExprNode*)(pOp->pLeft))->userAlias); return DEAL_RES_ERROR; } if (isMultiResFunc(pOp->pRight)) { - generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT, ((SExprNode*)(pOp->pRight))->userAlias); return DEAL_RES_ERROR; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index dfe33ce55e..76d8022578 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -192,6 +192,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Out of memory"; case TSDB_CODE_PAR_ORDERBY_AMBIGUOUS: return "ORDER BY \"%s\" is ambiguous"; + case TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT: + return "Operator not supported multi result: %s"; default: return "Unknown error"; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 8c73604727..836fd980d0 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -619,6 +619,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Invalid stream quer TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VIEW_QUERY, "Invalid view query type") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_COL_QUERY_MISMATCH, "Columns number mismatch with query result") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE, "View name is conflict with table") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT, "Operator not supported multi result") //planner TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "Planner internal error") From e4fa6871004c80a928ac89b968bd0190686bb9a0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 23 Feb 2024 14:36:38 +0800 Subject: [PATCH 115/181] enh: adjust code format in logging msg of mndTransSync --- source/dnode/mnode/impl/src/mndTrans.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 1adc4ed4bf..7c9ea69a02 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -751,8 +751,8 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { pTrans->createdTime); int32_t code = mndSyncPropose(pMnode, pRaw, pTrans->id); if (code != 0) { - mError("trans:%d, failed to sync, errno:%s code:%s createTime:%" PRId64 " saved trans:%d", pTrans->id, terrstr(), - tstrerror(code), pTrans->createdTime, pMnode->syncMgmt.transId); + mError("trans:%d, failed to sync, errno:%s code:0x%x createTime:%" PRId64 " saved trans:%d", pTrans->id, terrstr(), + code, pTrans->createdTime, pMnode->syncMgmt.transId); sdbFreeRaw(pRaw); return -1; } From 7835cb9c4202b98ebc46ea96e582ef07808e754d Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Fri, 23 Feb 2024 16:09:16 +0800 Subject: [PATCH 116/181] fix: delete sql count --- source/client/src/clientEnv.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 448adce15a..6054bb6982 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -108,9 +108,10 @@ static void deregisterRequest(SRequestObj *pRequest) { atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); reqType = SLOW_LOG_TYPE_QUERY; sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPESELECT); - } else if (QUERY_NODE_DELETE_STMT == pRequest->stmtType) { - sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEDELETE); - } + } + } + if (QUERY_NODE_DELETE_STMT == pRequest->stmtType) { + sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEDELETE); } if (duration >= (tsSlowLogThreshold * 1000000UL)) { From 9fd26a5949637e8af4bb9eb178f29fd0277953e9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 23 Feb 2024 16:13:11 +0800 Subject: [PATCH 117/181] docs:fix syntax error --- docs/zh/07-develop/07-tmq.mdx | 1135 +++++++++++++++--------------- docs/zh/07-develop/_sub_java.mdx | 1 + 2 files changed, 570 insertions(+), 566 deletions(-) diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index dc08f4e24d..881838f8f0 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -15,11 +15,11 @@ import Node from "./_sub_node.mdx"; import CSharp from "./_sub_cs.mdx"; import CDemo from "./_sub_c.mdx"; - 为了帮助应用实时获取写入 TDengine 的数据，或者以事件到达顺序处理数据，TDengine 提供了类似 kafka 的数据订阅功能。这样在很多场景下，采用 TDengine 的时序数据处理系统不再需要集成消息队列产品，比如 kafka, 从而简化系统设计的复杂度，降低运营维护成本。 -# 介绍 -## 主题 +## 数据订阅介绍 + +### 主题 与 kafka 一样，你需要定义 topic, TDengine 的 topic 有三种，可以是数据库，超级表，或者一个 `SELECT` 语句，具体的语法参见 [CREATE TOPIC](../../taos-sql/tmq)。与其他消息队列软件相比，这是 TDengine 数据订阅功能的最大的优势，它提供了更大的灵活性，数据的颗粒度可以由应用随时调整，而且数据的过滤与预处理交给 TDengine，而不是应用完成，有效的减少传输的数据量与应用的复杂度。 如下图，每个 topic 涉及到的数据表可能分布在多个 vnode（相当于 kafka 里的 partition） 上，每个 vnode 上的数据保存在 WAL(Write-Ahead-Log) 文件中，WAL 文件里的数据是顺序写入的（由于 WAL 文件中存储的不只有数据，还有元数据，写入消息等，所以数据的版本号不是连续的）。 @@ -30,11 +30,12 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提 对于 `SELECT` 语句形式的 topic，在消费时，TDengine 根据当前消费进度从 WAL 直接读取数据，并使用统一的查询引擎实现过滤、变换等操作，将数据推送给消费者。 -## 生产者 +### 生产者 写入 topic 相关联的数据表中数据的都是生产者，生产者实际生产的数据写入到了子表或普通表中，即表所在 vnode 的 WAL 里。 -## 消费者 -### 消费者组 +### 消费者 + +#### 消费者组 消费者订阅 topic 后，可以消费 topic 里的所有数据（这些数据所在的表可能分布在多个 vnode 上，即 db 所在的所有 vnode）。订阅 topic 时，需要指定一个消费者组 (consumer group)，如果这个消费者组里只有一个消费者，那么这个消费者会顺序的消费这些 vnode 上的数据。 为了提高消费速度，便于多线程、分布式地消费数据，可以在一个消费组里添加多个消费者，这些消费者将均分数据所在的 vnode 进行消费（比如数据分布在 4 个 vnode 上，有 2 个消费者的话，那么每个消费者消费 2 个 vnode；有 3 个消费者的话，2 个消费者各消费 1 个 vnode，1 个消费者消费 2 个 vnode；有 5 个消费者的话，4 个各分配 1 个 vnode 消费，另外 1 个不消费），如下图： @@ -44,7 +45,8 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提 在一个消费组里添加一个消费者后，在 Mnode 上通过 rebalance 的机制实现消费者的重新分配，该操作对用户是透明的。 一个消费者可以订阅多个 topic。TDengine 的数据订阅在宕机、重启等复杂环境下确保 at least once 消费。 -### 消费进度 + +#### 消费进度 在 topic 的一个消费组的一个 vnode 上有消费进度。消费者消费的同时，可以提交消费进度，消费进度即 vnode 上 WAL 的版本号（对于 kafka 里的 offset），消费进度可以手动提交，也可以通过参数（auto.commit.interval.ms）设置为周期性自动提交。 首次消费数据时通过订阅参数（auto.offset.reset）来确定消费位置为最新数据（latest）还是最旧数据（earliest）。 @@ -59,16 +61,16 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提 作为一个数据库产品， WAL 文件中存储的不全是数据，也包括其他写入消息，元数据等，所以消费进度不是连续的。 -##说明 +### 说明 从3.2.0.0版本开始，数据订阅支持vnode迁移和分裂。 由于数据订阅依赖wal文件，而在vnode迁移和分裂的过程中，wal并不会同步过去，所以迁移或分裂后，之前没消费完的wal数据后消费不到。所以请保证迁移和分裂之前把数据全部消费完后，再进行vnode迁移或分裂，否则，消费会丢失数据。 -# 语法说明 +## 数据订阅语法说明 具体的语法参见 [数据订阅](../../taos-sql/tmq) -# 消费参数 +## 数据订阅相关参数 消费参数主要用于消费者创建时指定，基础配置项如下表所示： @@ -86,229 +88,232 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提 | `msg.with.table.name` | boolean | 是否允许从消息中解析表名, 不适用于列订阅（列订阅时可将 tbname 作为列写入 subquery 语句）（从3.2.0.0版本该参数废弃，恒为true） |默认关闭 | | `enable.replay` | boolean | 是否开启数据回放功能 |默认关闭 | -# 主要数据结构和 API 接口 +## 数据订阅主要 API 接口 -不同语言下， TMQ 订阅相关的 API 及数据结构如下（详细的接口说明可以参考连接器章节，注意consumer结构不是线程安全的，在一个线程使用consumer时，不要在另一个线程close这个consumer）： +不同语言下， TMQ 订阅相关的 API 及数据结构如下（详细的接口说明可以参考连接器章节数据订阅部分，注意consumer结构不是线程安全的，在一个线程使用consumer时，不要在另一个线程close这个consumer）： - - + - ```c - typedef struct tmq_t tmq_t; - typedef struct tmq_conf_t tmq_conf_t; - typedef struct tmq_list_t tmq_list_t; +```c + typedef struct tmq_t tmq_t; + typedef struct tmq_conf_t tmq_conf_t; + typedef struct tmq_list_t tmq_list_t; - typedef void(tmq_commit_cb(tmq_t *tmq, int32_t code, void *param)); + typedef void(tmq_commit_cb(tmq_t *tmq, int32_t code, void *param)); - typedef enum tmq_conf_res_t { - TMQ_CONF_UNKNOWN = -2, - TMQ_CONF_INVALID = -1, - TMQ_CONF_OK = 0, - } tmq_conf_res_t; + typedef enum tmq_conf_res_t { + TMQ_CONF_UNKNOWN = -2, + TMQ_CONF_INVALID = -1, + TMQ_CONF_OK = 0, + } tmq_conf_res_t; - typedef struct tmq_topic_assignment { - int32_t vgId; - int64_t currentOffset; - int64_t begin; - int64_t end; - } tmq_topic_assignment; + typedef struct tmq_topic_assignment { + int32_t vgId; + int64_t currentOffset; + int64_t begin; + int64_t end; + } tmq_topic_assignment; - DLL_EXPORT tmq_conf_t *tmq_conf_new(); - DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value); - DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); - DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param); + DLL_EXPORT tmq_conf_t *tmq_conf_new(); + DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value); + DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); + DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param); - DLL_EXPORT tmq_list_t *tmq_list_new(); - DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); - DLL_EXPORT void tmq_list_destroy(tmq_list_t *); - DLL_EXPORT int32_t tmq_list_get_size(const tmq_list_t *); - DLL_EXPORT char **tmq_list_to_c_array(const tmq_list_t *); + DLL_EXPORT tmq_list_t *tmq_list_new(); + DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); + DLL_EXPORT void tmq_list_destroy(tmq_list_t *); + DLL_EXPORT int32_t tmq_list_get_size(const tmq_list_t *); + DLL_EXPORT char **tmq_list_to_c_array(const tmq_list_t *); - DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen); - DLL_EXPORT int32_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list); - DLL_EXPORT int32_t tmq_unsubscribe(tmq_t *tmq); - DLL_EXPORT int32_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); - DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout); - DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq); - DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg); - DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param); - DLL_EXPORT int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset); - DLL_EXPORT void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param); - DLL_EXPORT int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment,int32_t *numOfAssignment); - DLL_EXPORT void tmq_free_assignment(tmq_topic_assignment* pAssignment); - DLL_EXPORT int32_t tmq_offset_seek(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset); - DLL_EXPORT int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId); - DLL_EXPORT int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId); + DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen); + DLL_EXPORT int32_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list); + DLL_EXPORT int32_t tmq_unsubscribe(tmq_t *tmq); + DLL_EXPORT int32_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); + DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout); + DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq); + DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg); + DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param); + DLL_EXPORT int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset); + DLL_EXPORT void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param); + DLL_EXPORT int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment,int32_t *numOfAssignment); + DLL_EXPORT void tmq_free_assignment(tmq_topic_assignment* pAssignment); + DLL_EXPORT int32_t tmq_offset_seek(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset); + DLL_EXPORT int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId); + DLL_EXPORT int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId); - DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); - DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); - DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); - DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res); - DLL_EXPORT const char *tmq_err2str(int32_t code); - ``` - - + DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); + DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); + DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); + DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res); + DLL_EXPORT const char *tmq_err2str(int32_t code); +``` - ```java - void subscribe(Collection topics) throws SQLException; +下面介绍一下它们的具体用法（超级表和子表结构请参考“数据建模”一节），完整的示例代码请见下面 C 语言的示例代码。 - void unsubscribe() throws SQLException; + + - Set subscription() throws SQLException; +```java +void subscribe(Collection topics) throws SQLException; - ConsumerRecords poll(Duration timeout) throws SQLException; +void unsubscribe() throws SQLException; - Set assignment() throws SQLException; - long position(TopicPartition partition) throws SQLException; - Map position(String topic) throws SQLException; - Map beginningOffsets(String topic) throws SQLException; - Map endOffsets(String topic) throws SQLException; - Map committed(Set partitions) throws SQLException; +Set subscription() throws SQLException; - void seek(TopicPartition partition, long offset) throws SQLException; - void seekToBeginning(Collection partitions) throws SQLException; - void seekToEnd(Collection partitions) throws SQLException; +ConsumerRecords poll(Duration timeout) throws SQLException; - void commitSync() throws SQLException; - void commitSync(Map offsets) throws SQLException; +Set assignment() throws SQLException; +long position(TopicPartition partition) throws SQLException; +Map position(String topic) throws SQLException; +Map beginningOffsets(String topic) throws SQLException; +Map endOffsets(String topic) throws SQLException; +Map committed(Set partitions) throws SQLException; - void close() throws SQLException; - ``` +void seek(TopicPartition partition, long offset) throws SQLException; +void seekToBeginning(Collection partitions) throws SQLException; +void seekToEnd(Collection partitions) throws SQLException; - +void commitSync() throws SQLException; +void commitSync(Map offsets) throws SQLException; - +void close() throws SQLException; +``` - ```python - class Consumer: - def subscribe(self, topics): + + + + +```python +class Consumer: + def subscribe(self, topics): pass - def unsubscribe(self): + def unsubscribe(self): pass - def poll(self, timeout: float = 1.0): + def poll(self, timeout: float = 1.0): pass - def assignment(self): + def assignment(self): pass - def seek(self, partition): + def seek(self, partition): pass - def close(self): + def close(self): pass - def commit(self, message): + def commit(self, message): pass - ``` +``` - + - + - ```go - func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error) +```go +func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error) - // 出于兼容目的保留 rebalanceCb 参数，当前未使用 - func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error +// 出于兼容目的保留 rebalanceCb 参数，当前未使用 +func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error - // 出于兼容目的保留 rebalanceCb 参数，当前未使用 - func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error +// 出于兼容目的保留 rebalanceCb 参数，当前未使用 +func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error - func (c *Consumer) Poll(timeoutMs int) tmq.Event +func (c *Consumer) Poll(timeoutMs int) tmq.Event - // 出于兼容目的保留 tmq.TopicPartition 参数，当前未使用 - func (c *Consumer) Commit() ([]tmq.TopicPartition, error) +// 出于兼容目的保留 tmq.TopicPartition 参数，当前未使用 +func (c *Consumer) Commit() ([]tmq.TopicPartition, error) - func (c *Consumer) Unsubscribe() error +func (c *Consumer) Unsubscribe() error - func (c *Consumer) Close() error - ``` +func (c *Consumer) Close() error +``` - + - + - ```rust - impl TBuilder for TmqBuilder - fn from_dsn(dsn: D) -> Result - fn build(&self) -> Result +```rust +impl TBuilder for TmqBuilder + fn from_dsn(dsn: D) -> Result + fn build(&self) -> Result - impl AsAsyncConsumer for Consumer - async fn subscribe, I: IntoIterator + Send>( +impl AsAsyncConsumer for Consumer + async fn subscribe, I: IntoIterator + Send>( &mut self, topics: I, - ) -> Result<(), Self::Error>; - fn stream( + ) -> Result<(), Self::Error>; + fn stream( &self, - ) -> Pin< - Box< + ) -> Pin< + Box< dyn '_ + Send + futures::Stream< Item = Result<(Self::Offset, MessageSet), Self::Error>, >, - >, - >; - async fn commit(&self, offset: Self::Offset) -> Result<(), Self::Error>; + >, + >; + async fn commit(&self, offset: Self::Offset) -> Result<(), Self::Error>; - async fn unsubscribe(self); - ``` + async fn unsubscribe(self); +``` - 可在 上查看详细 API 说明。 +可在 上查看详细 API 说明。 - + - + - ```js - function TMQConsumer(config) - - function subscribe(topic) - - function consume(timeout) - - function subscription() - - function unsubscribe() - - function commit(msg) - - function close() - ``` +```js +function TMQConsumer(config) - +function subscribe(topic) - +function consume(timeout) - ```csharp - class ConsumerBuilder - - ConsumerBuilder(IEnumerable> config) - - public IConsumer Build() - - void Subscribe(IEnumerable topics) - - void Subscribe(string topic) - - ConsumeResult Consume(int millisecondsTimeout) - - List Subscription() - - void Unsubscribe() - - List Commit() - - void Close() - ``` - +function subscription() + +function unsubscribe() + +function commit(msg) + +function close() +``` + + + + + +```csharp +class ConsumerBuilder + +ConsumerBuilder(IEnumerable> config) + +public IConsumer Build() + +void Subscribe(IEnumerable topics) + +void Subscribe(string topic) + +ConsumeResult Consume(int millisecondsTimeout) + +List Subscription() + +void Unsubscribe() + +List Commit() + +void Close() +``` + + -# 数据订阅示例 -## 写入数据 +## 数据订阅示例 +### 写入数据 首先完成建库、建一张超级表和多张子表操作，然后就可以写入数据了，比如： @@ -321,7 +326,7 @@ CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1"); INSERT INTO tmqdb.ctb0 VALUES(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00'); INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11'); ``` -## 创建 topic +### 创建 topic 使用 SQL 创建一个 topic： @@ -329,516 +334,514 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11'); CREATE TOPIC topic_name AS SELECT ts, c1, c2, c3 FROM tmqdb.stb WHERE c1 > 1; ``` -## 创建消费者 *consumer* +### 创建消费者 consumer 对于不同编程语言，其设置方式如下： + - - ```c - /* 根据需要，设置消费组 (group.id)、自动提交 (enable.auto.commit)、 - 自动提交时间间隔 (auto.commit.interval.ms)、用户名 (td.connect.user)、密码 (td.connect.pass) 等参数 */ - tmq_conf_t* conf = tmq_conf_new(); - tmq_conf_set(conf, "enable.auto.commit", "true"); - tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - tmq_conf_set(conf, "group.id", "cgrpName"); - tmq_conf_set(conf, "td.connect.user", "root"); - tmq_conf_set(conf, "td.connect.pass", "taosdata"); - tmq_conf_set(conf, "auto.offset.reset", "latest"); - tmq_conf_set(conf, "msg.with.table.name", "true"); - tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); +```c +/* 根据需要，设置消费组 (group.id)、自动提交 (enable.auto.commit)、 + 自动提交时间间隔 (auto.commit.interval.ms)、用户名 (td.connect.user)、密码 (td.connect.pass) 等参数 */ +tmq_conf_t* conf = tmq_conf_new(); +tmq_conf_set(conf, "enable.auto.commit", "true"); +tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); +tmq_conf_set(conf, "group.id", "cgrpName"); +tmq_conf_set(conf, "td.connect.user", "root"); +tmq_conf_set(conf, "td.connect.pass", "taosdata"); +tmq_conf_set(conf, "auto.offset.reset", "latest"); +tmq_conf_set(conf, "msg.with.table.name", "true"); +tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); - tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); - tmq_conf_destroy(conf); - ``` - +tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); +tmq_conf_destroy(conf); +``` - + + - 对于 Java 程序，还可以使用如下配置项： - - | 参数名称 | 类型 | 参数说明 | - | ----------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------- | - | `td.connect.type` | string | 连接类型，"jni" 指原生连接，"ws" 指 websocket 连接，默认值为 "jni" | - | `bootstrap.servers` | string | 连接地址，如 `localhost:6030` | - | `value.deserializer` | string | 值解析方法，使用此方法应实现 `com.taosdata.jdbc.tmq.Deserializer` 接口或继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer` 类 | - | `value.deserializer.encoding` | string | 指定字符串解析的字符集 | | - - 需要注意：此处使用 `bootstrap.servers` 替代 `td.connect.ip` 和 `td.connect.port`，以提供与 Kafka 一致的接口。 - - ```java - Properties properties = new Properties(); - properties.setProperty("enable.auto.commit", "true"); - properties.setProperty("auto.commit.interval.ms", "1000"); - properties.setProperty("group.id", "cgrpName"); - properties.setProperty("bootstrap.servers", "127.0.0.1:6030"); - properties.setProperty("td.connect.user", "root"); - properties.setProperty("td.connect.pass", "taosdata"); - properties.setProperty("auto.offset.reset", "latest"); - properties.setProperty("msg.with.table.name", "true"); - properties.setProperty("value.deserializer", "com.taos.example.MetersDeserializer"); - - TaosConsumer consumer = new TaosConsumer<>(properties); - - /* value deserializer definition. */ - import com.taosdata.jdbc.tmq.ReferenceDeserializer; - - public class MetersDeserializer extends ReferenceDeserializer { - } - ``` - +对于 Java 程序，还可以使用如下配置项： - - - ```go - conf := &tmq.ConfigMap{ - "group.id": "test", - "auto.offset.reset": "latest", - "td.connect.ip": "127.0.0.1", - "td.connect.user": "root", - "td.connect.pass": "taosdata", - "td.connect.port": "6030", - "client.id": "test_tmq_c", - "enable.auto.commit": "false", - "msg.with.table.name": "true", - } - consumer, err := NewConsumer(conf) - ``` - - +| 参数名称 | 类型 | 参数说明 | +| ----------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------- | +| `td.connect.type` | string | 连接类型，"jni" 指原生连接，"ws" 指 websocket 连接，默认值为 "jni" | +| `bootstrap.servers` | string | 连接地址，如 `localhost:6030` | +| `value.deserializer` | string | 值解析方法，使用此方法应实现 `com.taosdata.jdbc.tmq.Deserializer` 接口或继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer` 类 | +| `value.deserializer.encoding` | string | 指定字符串解析的字符集 | | - - - ```rust - let mut dsn: Dsn = "taos://".parse()?; - dsn.set("group.id", "group1"); - dsn.set("client.id", "test"); - dsn.set("auto.offset.reset", "latest"); - - let tmq = TmqBuilder::from_dsn(dsn)?; - - let mut consumer = tmq.build()?; - ``` - - +需要注意：此处使用 `bootstrap.servers` 替代 `td.connect.ip` 和 `td.connect.port`，以提供与 Kafka 一致的接口。 - - - Python 语言下引入 `taos` 库的 `Consumer` 类，创建一个 Consumer 示例： - - ```python - from taos.tmq import Consumer - - # Syntax: `consumer = Consumer(configs)` - # - # Example: - consumer = Consumer( - { - "group.id": "local", - "client.id": "1", - "enable.auto.commit": "true", - "auto.commit.interval.ms": "1000", - "td.connect.ip": "127.0.0.1", - "td.connect.user": "root", - "td.connect.pass": "taosdata", - "auto.offset.reset": "latest", - "msg.with.table.name": "true", - } - ) - ``` - - +```java +Properties properties = new Properties(); +properties.setProperty("enable.auto.commit", "true"); +properties.setProperty("auto.commit.interval.ms", "1000"); +properties.setProperty("group.id", "cgrpName"); +properties.setProperty("bootstrap.servers", "127.0.0.1:6030"); +properties.setProperty("td.connect.user", "root"); +properties.setProperty("td.connect.pass", "taosdata"); +properties.setProperty("auto.offset.reset", "latest"); +properties.setProperty("msg.with.table.name", "true"); +properties.setProperty("value.deserializer", "com.taos.example.MetersDeserializer"); - - - ```js - // 根据需要，设置消费组 (group.id)、自动提交 (enable.auto.commit)、 - // 自动提交时间间隔 (auto.commit.interval.ms)、用户名 (td.connect.user)、密码 (td.connect.pass) 等参数 - - let consumer = taos.consumer({ - 'enable.auto.commit': 'true', - 'auto.commit.interval.ms','1000', - 'group.id': 'tg2', - 'td.connect.user': 'root', - 'td.connect.pass': 'taosdata', - 'auto.offset.reset','latest', - 'msg.with.table.name': 'true', - 'td.connect.ip','127.0.0.1', - 'td.connect.port','6030' - }); - ``` - - +TaosConsumer consumer = new TaosConsumer<>(properties); + +/* value deserializer definition. */ +import com.taosdata.jdbc.tmq.ReferenceDeserializer; + +public class MetersDeserializer extends ReferenceDeserializer { +} +``` + + + + + +```go +conf := &tmq.ConfigMap{ + "group.id": "test", + "auto.offset.reset": "latest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_c", + "enable.auto.commit": "false", + "msg.with.table.name": "true", +} +consumer, err := NewConsumer(conf) +``` + + + + + +```rust +let mut dsn: Dsn = "taos://".parse()?; +dsn.set("group.id", "group1"); +dsn.set("client.id", "test"); +dsn.set("auto.offset.reset", "latest"); + +let tmq = TmqBuilder::from_dsn(dsn)?; + +let mut consumer = tmq.build()?; +``` + + + + + +Python 语言下引入 `taos` 库的 `Consumer` 类，创建一个 Consumer 示例： + +```python +from taos.tmq import Consumer + +# Syntax: `consumer = Consumer(configs)` +# +# Example: +consumer = Consumer( + { + "group.id": "local", + "client.id": "1", + "enable.auto.commit": "true", + "auto.commit.interval.ms": "1000", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "latest", + "msg.with.table.name": "true", + } +) +``` + + + + + +```js +// 根据需要，设置消费组 (group.id)、自动提交 (enable.auto.commit)、 +// 自动提交时间间隔 (auto.commit.interval.ms)、用户名 (td.connect.user)、密码 (td.connect.pass) 等参数 + +let consumer = taos.consumer({ + 'enable.auto.commit': 'true', + 'auto.commit.interval.ms','1000', + 'group.id': 'tg2', + 'td.connect.user': 'root', + 'td.connect.pass': 'taosdata', + 'auto.offset.reset','latest', + 'msg.with.table.name': 'true', + 'td.connect.ip','127.0.0.1', + 'td.connect.port','6030' + }); +``` + + + + + +```csharp +var cfg = new Dictionary() +{ + { "group.id", "group1" }, + { "auto.offset.reset", "latest" }, + { "td.connect.ip", "127.0.0.1" }, + { "td.connect.user", "root" }, + { "td.connect.pass", "taosdata" }, + { "td.connect.port", "6030" }, + { "client.id", "tmq_example" }, + { "enable.auto.commit", "true" }, + { "msg.with.table.name", "false" }, +}; +var consumer = new ConsumerBuilder>(cfg).Build(); +``` + + - - - ```csharp - var cfg = new Dictionary() - { - { "group.id", "group1" }, - { "auto.offset.reset", "latest" }, - { "td.connect.ip", "127.0.0.1" }, - { "td.connect.user", "root" }, - { "td.connect.pass", "taosdata" }, - { "td.connect.port", "6030" }, - { "client.id", "tmq_example" }, - { "enable.auto.commit", "true" }, - { "msg.with.table.name", "false" }, - }; - var consumer = new ConsumerBuilder>(cfg).Build(); - ``` - - - 上述配置中包括 consumer group ID，如果多个 consumer 指定的 consumer group ID 一样，则自动形成一个 consumer group，共享消费进度。 -## 订阅 *topics* +### 订阅 topics 一个 consumer 支持同时订阅多个 topic。 + - - - ```c - // 创建订阅 topics 列表 - tmq_list_t* topicList = tmq_list_new(); - tmq_list_append(topicList, "topicName"); - // 启动订阅 - tmq_subscribe(tmq, topicList); - tmq_list_destroy(topicList); - - ``` - - - - - ```java - List topics = new ArrayList<>(); - topics.add("tmq_topic"); - consumer.subscribe(topics); - ``` - - - +```c +// 创建订阅 topics 列表 +tmq_list_t* topicList = tmq_list_new(); +tmq_list_append(topicList, "topicName"); +// 启动订阅 +tmq_subscribe(tmq, topicList); +tmq_list_destroy(topicList); + +``` - ```go - err = consumer.Subscribe("example_tmq_topic", nil) - if err != nil { - panic(err) - } - ``` + + - - +```java +List topics = new ArrayList<>(); +topics.add("tmq_topic"); +consumer.subscribe(topics); +``` - ```rust - consumer.subscribe(["tmq_meters"]).await?; - ``` + + - +```go +err = consumer.Subscribe("example_tmq_topic", nil) +if err != nil { + panic(err) +} +``` - + + - ```python - consumer.subscribe(['topic1', 'topic2']) - ``` +```rust +consumer.subscribe(["tmq_meters"]).await?; +``` - + - + - ```js - // 创建订阅 topics 列表 - let topics = ['topic_test'] +```python +consumer.subscribe(['topic1', 'topic2']) +``` - // 启动订阅 - consumer.subscribe(topics); - ``` + - + - +```js +// 创建订阅 topics 列表 +let topics = ['topic_test'] - ```csharp - // 创建订阅 topics 列表 - List topics = new List(); - topics.add("tmq_topic"); - // 启动订阅 - consumer.Subscribe(topics); - ``` +// 启动订阅 +consumer.subscribe(topics); +``` - + + + + +```csharp +// 创建订阅 topics 列表 +List topics = new List(); +topics.add("tmq_topic"); +// 启动订阅 +consumer.Subscribe(topics); +``` + + -## 消费 +### 消费 以下代码展示了不同语言下如何对 TMQ 消息进行消费。 + - +```c +// 消费数据 +while (running) { + TAOS_RES* msg = tmq_consumer_poll(tmq, timeOut); + msg_process(msg); +} +``` - ```c - // 消费数据 - while (running) { - TAOS_RES* msg = tmq_consumer_poll(tmq, timeOut); - msg_process(msg); - } - ``` +这里是一个 **while** 循环，每调用一次 tmq_consumer_poll()，获取一个消息，该消息与普通查询返回的结果集完全相同，可以使用相同的解析 API 完成消息内容的解析。 - 这里是一个 **while** 循环，每调用一次 tmq_consumer_poll()，获取一个消息，该消息与普通查询返回的结果集完全相同，可以使用相同的解析 API 完成消息内容的解析。 + + - - - - ```java - while(running){ - ConsumerRecords meters = consumer.poll(Duration.ofMillis(100)); - for (Meters meter : meters) { - processMsg(meter); - } - } - ``` - - +```java +while(running){ + ConsumerRecords meters = consumer.poll(Duration.ofMillis(100)); + for (Meters meter : meters) { + processMsg(meter); + } +} +``` - + - ```go - for { - ev := consumer.Poll(0) - if ev != nil { - switch e := ev.(type) { - case *tmqcommon.DataMessage: - fmt.Println(e.Value()) - case tmqcommon.Error: - fmt.Fprintf(os.Stderr, "%% Error: %v: %v\n", e.Code(), e) - panic(e) - } - consumer.Commit() - } - } - ``` + - +```go +for { + ev := consumer.Poll(0) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Println(e.Value()) + case tmqcommon.Error: + fmt.Fprintf(os.Stderr, "%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() + } +} +``` - + - ```rust - { - let mut stream = consumer.stream(); + - while let Some((offset, message)) = stream.try_next().await? { - // get information from offset +```rust +{ + let mut stream = consumer.stream(); - // the topic - let topic = offset.topic(); - // the vgroup id, like partition id in kafka. - let vgroup_id = offset.vgroup_id(); - println!("* in vgroup id {vgroup_id} of topic {topic}\n"); + while let Some((offset, message)) = stream.try_next().await? { + // get information from offset - if let Some(data) = message.into_data() { + // the topic + let topic = offset.topic(); + // the vgroup id, like partition id in kafka. + let vgroup_id = offset.vgroup_id(); + println!("* in vgroup id {vgroup_id} of topic {topic}\n"); + + if let Some(data) = message.into_data() { while let Some(block) = data.fetch_raw_block().await? { - // one block for one table, get table name if needed - let name = block.table_name(); - let records: Vec = block.deserialize().try_collect()?; - println!( - "** table: {}, got {} records: {:#?}\n", - name.unwrap(), - records.len(), - records - ); + // one block for one table, get table name if needed + let name = block.table_name(); + let records: Vec = block.deserialize().try_collect()?; + println!( + "** table: {}, got {} records: {:#?}\n", + name.unwrap(), + records.len(), + records + ); + } } - } - consumer.commit(offset).await?; - } - } - ``` + consumer.commit(offset).await?; + } +} +``` - - + + - ```python - while True: - res = consumer.poll(100) - if not res: +```python +while True: + res = consumer.poll(100) + if not res: continue - err = res.error() - if err is not None: + err = res.error() + if err is not None: raise err - val = res.value() + val = res.value() - for block in val: + for block in val: print(block.fetchall()) - ``` +``` - + - + - ```js - while(true){ - msg = consumer.consume(200); - // process message(consumeResult) - console.log(msg.topicPartition); - console.log(msg.block); - console.log(msg.fields) - } - ``` +```js +while(true){ + msg = consumer.consume(200); + // process message(consumeResult) + console.log(msg.topicPartition); + console.log(msg.block); + console.log(msg.fields) +} +``` - + - + - ```csharp - // 消费数据 - while (true) - { - using (var result = consumer.Consume(500)) - { - if (result == null) continue; - ProcessMsg(result); - consumer.Commit(); - } - } - ``` +```csharp +// 消费数据 +while (true) +{ + using (var result = consumer.Consume(500)) + { + if (result == null) continue; + ProcessMsg(result); + consumer.Commit(); + } +} +``` - + -## 结束消费 +### 结束消费 消费结束后，应当取消订阅。 - - + - ```c - /* 取消订阅 */ - tmq_unsubscribe(tmq); +```c +/* 取消订阅 */ +tmq_unsubscribe(tmq); - /* 关闭消费者对象 */ - tmq_consumer_close(tmq); - ``` +/* 关闭消费者对象 */ +tmq_consumer_close(tmq); +``` - - + + - ```java - /* 取消订阅 */ - consumer.unsubscribe(); +```java +/* 取消订阅 */ +consumer.unsubscribe(); - /* 关闭消费 */ - consumer.close(); - ``` +/* 关闭消费 */ +consumer.close(); +``` - - - + - ```go - /* Unsubscribe */ - _ = consumer.Unsubscribe() + - /* Close consumer */ - _ = consumer.Close() - ``` +```go +/* Unsubscribe */ +_ = consumer.Unsubscribe() - - - +/* Close consumer */ +_ = consumer.Close() +``` - ```rust - consumer.unsubscribe().await; - ``` + - + - +```rust +consumer.unsubscribe().await; +``` - ```py - # 取消订阅 - consumer.unsubscribe() - # 关闭消费 - consumer.close() - ``` + - - + - ```js - consumer.unsubscribe(); - consumer.close(); - ``` +```py +# 取消订阅 +consumer.unsubscribe() +# 关闭消费 +consumer.close() +``` - + + - +```js +consumer.unsubscribe(); +consumer.close(); +``` - ```csharp - // 取消订阅 - consumer.Unsubscribe(); + - // 关闭消费 - consumer.Close(); - ``` + - +```csharp +// 取消订阅 +consumer.Unsubscribe(); + +// 关闭消费 +consumer.Close(); +``` + + -## 完整示例代码 +### 完整示例代码 以下是各语言的完整示例代码。 - - - + + + - - - - - - - - - - + + + + + + + + + + - - - + + + - - - + + + - - - + + + - - - + + + + + + + - - - - -#订阅高级功能 -##数据回放 +## 数据订阅高级功能 +### 数据回放 - 订阅支持 replay 功能，按照数据写入的时间回放。 比如，如下时间写入三条数据 ```sql @@ -854,4 +857,4 @@ CREATE TOPIC topic_name AS SELECT ts, c1, c2, c3 FROM tmqdb.stb WHERE c1 > 1; - 超级表和库订阅不支持回放 - enable.replay 参数，true表示开启订阅回放功能，false表示不开启订阅回放功能，默认不开启。 - 回放不支持进度保存，所以回放参数 enable.replay = true 时，auto commit 自动关闭 -- 因为数据回放本身需要处理时间，所以回放的精度存在几十ms的误差 \ No newline at end of file +- 因为数据回放本身需要处理时间，所以回放的精度存在几十ms的误差 diff --git a/docs/zh/07-develop/_sub_java.mdx b/docs/zh/07-develop/_sub_java.mdx index 60810ec275..c0e9e6c937 100644 --- a/docs/zh/07-develop/_sub_java.mdx +++ b/docs/zh/07-develop/_sub_java.mdx @@ -6,3 +6,4 @@ ``` ```java {{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}} +``` From 22bd628c14049efefeb7a81f38e01d27af40ee2c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 23 Feb 2024 16:36:53 +0800 Subject: [PATCH 118/181] fix:[TD-28698] unit test error --- source/libs/parser/test/parSelectTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 390faab537..d7b31d2ac8 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -381,7 +381,7 @@ TEST_F(ParserSelectTest, semanticCheck) { // TSDB_CODE_PAR_WRONG_VALUE_TYPE run("SELECT timestamp '2010a' FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE); - run("SELECT LAST(*) + SUM(c1) FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE); + run("SELECT LAST(*) + SUM(c1) FROM t1", TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT); run("SELECT CEIL(LAST(ts, c1)) FROM t1", TSDB_CODE_FUNC_FUNTION_PARA_NUM); From 3233f2dee03b0307f27e25efda664b8b67afa338 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Fri, 23 Feb 2024 17:02:41 +0800 Subject: [PATCH 119/181] fix: insert sql count --- source/client/src/clientEnv.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 6054bb6982..1df50a51da 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -98,7 +98,6 @@ static void deregisterRequest(SRequestObj *pRequest) { pRequest->metric.planCostUs, pRequest->metric.execCostUs); atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); reqType = SLOW_LOG_TYPE_INSERT; - sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT); } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { tscDebug("query duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64 "us, planCost:%" PRId64 "us, exec:%" PRId64 "us", @@ -107,10 +106,14 @@ static void deregisterRequest(SRequestObj *pRequest) { atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); reqType = SLOW_LOG_TYPE_QUERY; - sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPESELECT); } } - if (QUERY_NODE_DELETE_STMT == pRequest->stmtType) { + + if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) { + sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT); + } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { + sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPESELECT); + } else if (QUERY_NODE_DELETE_STMT == pRequest->stmtType) { sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEDELETE); } From a164e36b6ddd250fb5eb96f9700d178b9f8aa0ae Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 23 Feb 2024 10:24:00 +0000 Subject: [PATCH 120/181] add remove metric --- include/libs/monitorfw/taos_collector.h | 2 + .../libs/monitorfw/taos_collector_registry.h | 2 + source/libs/monitor/src/monFramework.c | 38 +++++++++------- source/libs/monitorfw/inc/taos_log.h | 4 +- source/libs/monitorfw/inc/taos_map_i.h | 2 + source/libs/monitorfw/src/taos_collector.c | 6 +++ .../monitorfw/src/taos_collector_registry.c | 19 +++++++- source/libs/monitorfw/src/taos_linked_list.c | 33 ++++++++++++++ source/libs/monitorfw/src/taos_map.c | 12 ++++- .../src/taos_metric_formatter_custom.c | 45 ++++++++++++++++--- 10 files changed, 138 insertions(+), 25 deletions(-) diff --git a/include/libs/monitorfw/taos_collector.h b/include/libs/monitorfw/taos_collector.h index 8fe304ed7d..c37cb612d2 100644 --- a/include/libs/monitorfw/taos_collector.h +++ b/include/libs/monitorfw/taos_collector.h @@ -76,6 +76,8 @@ int taos_collector_destroy_generic(void *gen); */ int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric); +int taos_collector_remove_metric(taos_collector_t *self, const char* key); + taos_metric_t* taos_collector_get_metric(taos_collector_t *self, char *metric_name); /** diff --git a/include/libs/monitorfw/taos_collector_registry.h b/include/libs/monitorfw/taos_collector_registry.h index 063e8afb50..81d1f8050c 100644 --- a/include/libs/monitorfw/taos_collector_registry.h +++ b/include/libs/monitorfw/taos_collector_registry.h @@ -86,6 +86,8 @@ taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metri */ int taos_collector_registry_register_metric(taos_metric_t *metric); +int taos_collector_registry_deregister_metric(const char *key); + taos_metric_t *taos_collector_registry_get_metric(char* metric_name); /** diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c index 06b4a3ee3b..9a2857d442 100644 --- a/source/libs/monitor/src/monFramework.c +++ b/source/libs/monitor/src/monFramework.c @@ -120,17 +120,6 @@ void monInitMonitorFW(){ taosHashPut(tsMonitor.metrics, metric[i], strlen(metric[i]), &gauge, sizeof(taos_gauge_t *)); } - int32_t vgroup_label_count = 3; - const char *vgroup_sample_labels[] = {"cluster_id", "vgroup_id", "database_name"}; - char *vgroup_metrics[] = {TABLES_NUM, STATUS}; - for(int32_t i = 0; i < 2; i++){ - gauge= taos_gauge_new(vgroup_metrics[i], "", vgroup_label_count, vgroup_sample_labels); - if(taos_collector_registry_register_metric(gauge) == 1){ - taos_counter_destroy(gauge); - } - taosHashPut(tsMonitor.metrics, vgroup_metrics[i], strlen(vgroup_metrics[i]), &gauge, sizeof(taos_gauge_t *)); - } - int32_t dnodes_label_count = 3; const char *dnodes_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep"}; char *dnodes_gauges[] = {UPTIME, CPU_ENGINE, CPU_SYSTEM, CPU_CORE, MEM_ENGINE, MEM_SYSTEM, @@ -316,6 +305,27 @@ void monGenVgroupInfoTable(SMonInfo *pMonitor){ SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + int32_t vgroup_label_count = 3; + const char *vgroup_sample_labels[] = {"cluster_id", "vgroup_id", "database_name"}; + + if(taos_collector_registry_deregister_metric(TABLES_NUM) != 0){ + uError("failed to delete metric "TABLES_NUM); + } + + taos_gauge_t *tableNumGauge = taos_gauge_new(TABLES_NUM, "", vgroup_label_count, vgroup_sample_labels); + if(taos_collector_registry_register_metric(tableNumGauge) == 1){ + taos_counter_destroy(tableNumGauge); + } + + if(taos_collector_registry_deregister_metric(STATUS) != 0){ + uError("failed to delete metric "STATUS); + } + + taos_gauge_t *statusGauge = taos_gauge_new(STATUS, "", vgroup_label_count, vgroup_sample_labels); + if(taos_collector_registry_register_metric(statusGauge) == 1){ + taos_counter_destroy(statusGauge); + } + char cluster_id[TSDB_CLUSTER_ID_LEN] = {0}; snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%"PRId64, pMonitor->dmInfo.basic.cluster_id); @@ -329,15 +339,13 @@ void monGenVgroupInfoTable(SMonInfo *pMonitor){ taos_gauge_t **metric = NULL; - metric = taosHashGet(tsMonitor.metrics, TABLES_NUM, strlen(TABLES_NUM)); - taos_gauge_set(*metric, pVgroupDesc->tables_num, sample_labels); + taos_gauge_set(tableNumGauge, pVgroupDesc->tables_num, sample_labels); - metric = taosHashGet(tsMonitor.metrics, STATUS, strlen(STATUS)); int32_t status = 0; if(strcmp(pVgroupDesc->status, "ready") == 0){ status = 1; } - taos_gauge_set(*metric, status, sample_labels); + taos_gauge_set(statusGauge, status, sample_labels); } } diff --git a/source/libs/monitorfw/inc/taos_log.h b/source/libs/monitorfw/inc/taos_log.h index 6d6c9e1fb3..fecbda59e5 100644 --- a/source/libs/monitorfw/inc/taos_log.h +++ b/source/libs/monitorfw/inc/taos_log.h @@ -18,8 +18,10 @@ #ifndef TAOS_LOG_H #define TAOS_LOG_H +//#define TAOS_LOG_ENABLE + #ifdef TAOS_LOG_ENABLE -#define TAOS_LOG(msg) printf("%s %s %s %s %d %s\n", __DATE__, __TIME__, __FILE__, __FUNCTION__, __LINE__, msg); +#define TAOS_LOG(msg) printf("monitor_log %s %s %s %s %d %s\n", __DATE__, __TIME__, __FILE__, __FUNCTION__, __LINE__, msg); #else #define TAOS_LOG(msg) #endif // TAOS_LOG_ENABLE diff --git a/source/libs/monitorfw/inc/taos_map_i.h b/source/libs/monitorfw/inc/taos_map_i.h index 808548e96a..55f248b96a 100644 --- a/source/libs/monitorfw/inc/taos_map_i.h +++ b/source/libs/monitorfw/inc/taos_map_i.h @@ -24,6 +24,8 @@ int taos_map_set_free_value_fn(taos_map_t *self, taos_map_node_free_value_fn fre void *taos_map_get(taos_map_t *self, const char *key); +void *taos_map_get_withoutlock(taos_map_t *self, const char *key); + int taos_map_set(taos_map_t *self, const char *key, void *value); int taos_map_delete(taos_map_t *self, const char *key); diff --git a/source/libs/monitorfw/src/taos_collector.c b/source/libs/monitorfw/src/taos_collector.c index 997bf5587c..17d324462c 100644 --- a/source/libs/monitorfw/src/taos_collector.c +++ b/source/libs/monitorfw/src/taos_collector.c @@ -108,6 +108,12 @@ int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric) { return taos_map_set(self->metrics, metric->name, metric); } +int taos_collector_remove_metric(taos_collector_t *self, const char* key){ + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + return taos_map_delete(self->metrics, key); +} + taos_metric_t* taos_collector_get_metric(taos_collector_t *self, char *metric_name){ TAOS_ASSERT(self != NULL); if (self == NULL) return NULL; diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c index acf2cb4248..55b08775b4 100644 --- a/source/libs/monitorfw/src/taos_collector_registry.c +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -123,6 +123,19 @@ int taos_collector_registry_register_metric(taos_metric_t *metric) { return taos_collector_add_metric(default_collector, metric); } +int taos_collector_registry_deregister_metric(const char *key) { + TAOS_ASSERT(metric != NULL); + + taos_collector_t *default_collector = + (taos_collector_t *)taos_map_get(TAOS_COLLECTOR_REGISTRY_DEFAULT->collectors, "default"); + + if (default_collector == NULL) { + return 1; + } + + return taos_collector_remove_metric(default_collector, key); +} + taos_metric_t *taos_collector_registry_get_metric(char* metric_name){ TAOS_ASSERT(metric != NULL); @@ -232,7 +245,11 @@ const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self, SJson* array = tjsonCreateArray(); tjsonAddItemToObject(item, "tables", array); - taos_metric_formatter_load_metrics_new(self->metric_formatter, self->collectors, ts, format, array); + if(taos_metric_formatter_load_metrics_new(self->metric_formatter, self->collectors, ts, format, array) != 0){ + TAOS_LOG("failed to load metrics"); + tjsonDelete(pJson); + return NULL; + } if(tjsonGetArraySize(array) == 0){ tjsonDelete(pJson); diff --git a/source/libs/monitorfw/src/taos_linked_list.c b/source/libs/monitorfw/src/taos_linked_list.c index ab4e23ff29..675400a6fa 100644 --- a/source/libs/monitorfw/src/taos_linked_list.c +++ b/source/libs/monitorfw/src/taos_linked_list.c @@ -147,9 +147,26 @@ int taos_linked_list_remove(taos_linked_list_t *self, void *item) { if (self == NULL) return 1; taos_linked_list_node_t *node; taos_linked_list_node_t *prev_node = NULL; +#ifdef TAOS_LOG_ENABLE + int32_t count = 0; + char tmp[200] = {0}; + + count = 0; + for (node = self->head; node != NULL; node = node->next) { + count++; + } + sprintf(tmp, "list count:%d", count); + TAOS_LOG(tmp); +#endif // Locate the node +#ifdef TAOS_LOG_ENABLE + count = 0; +#endif for (node = self->head; node != NULL; node = node->next) { +#ifdef TAOS_LOG_ENABLE + count++; +#endif if (self->compare_fn) { if ((*self->compare_fn)(node->item, item) == TAOS_EQUAL) { break; @@ -162,6 +179,11 @@ int taos_linked_list_remove(taos_linked_list_t *self, void *item) { prev_node = node; } +#ifdef TAOS_LOG_ENABLE + sprintf(tmp, "remove item:%d", count); + TAOS_LOG(tmp); +#endif + if (node == NULL) return 0; if (prev_node) { @@ -185,6 +207,17 @@ int taos_linked_list_remove(taos_linked_list_t *self, void *item) { taos_free(node); node = NULL; self->size--; + +#ifdef TAOS_LOG_ENABLE + count = 0; + for (node = self->head; node != NULL; node = node->next) { + count++; + } + + sprintf(tmp, "list count:%d", count); + TAOS_LOG(tmp); +#endif + return 0; } diff --git a/source/libs/monitorfw/src/taos_map.c b/source/libs/monitorfw/src/taos_map.c index fce308f11d..8f0b83884e 100644 --- a/source/libs/monitorfw/src/taos_map.c +++ b/source/libs/monitorfw/src/taos_map.c @@ -215,6 +215,14 @@ void *taos_map_get(taos_map_t *self, const char *key) { return payload; } +void *taos_map_get_withoutlock(taos_map_t *self, const char *key) { + TAOS_ASSERT(self != NULL); + int r = 0; + void *payload = + taos_map_get_internal(key, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn); + return payload; +} + static int taos_map_set_internal(const char *key, void *value, size_t *size, size_t *max_size, taos_linked_list_t *keys, taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn, bool destroy_current_value) { @@ -368,10 +376,10 @@ static int taos_map_delete_internal(const char *key, size_t *size, size_t *max_s taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item; taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, temp_map_node); if (result == TAOS_EQUAL) { - r = taos_linked_list_remove(list, current_node); + r = taos_linked_list_remove(keys, (char*)current_map_node->key); if (r) return r; - r = taos_linked_list_remove(keys, (char *)current_map_node->key); + r = taos_linked_list_remove(list, current_node->item); if (r) return r; (*size)--; diff --git a/source/libs/monitorfw/src/taos_metric_formatter_custom.c b/source/libs/monitorfw/src/taos_metric_formatter_custom.c index d7650098e0..3b1318dfc0 100644 --- a/source/libs/monitorfw/src/taos_metric_formatter_custom.c +++ b/source/libs/monitorfw/src/taos_metric_formatter_custom.c @@ -23,6 +23,7 @@ #include "taos_assert.h" #include "tdef.h" #include "taos_collector_t.h" +#include "taos_log.h" int taos_metric_formatter_load_sample_new(taos_metric_formatter_t *self, taos_metric_sample_t *sample, char *ts, char *format, char *metricName, int32_t metric_type, @@ -207,18 +208,50 @@ int taos_metric_formatter_load_metrics_new(taos_metric_formatter_t *self, taos_m //if(strcmp(collector->name, "custom") != 0 ){ + r = pthread_rwlock_wrlock(metrics->rwlock); + if (r) { + TAOS_LOG("failed to lock"); + return r; + } + +#ifdef TAOS_LOG_ENABLE + int32_t count = 0; +#endif for (taos_linked_list_node_t *current_node = metrics->keys->head; current_node != NULL; current_node = current_node->next) { +#ifdef TAOS_LOG_ENABLE + count++; +#endif const char *metric_name = (const char *)current_node->item; - taos_metric_t *metric = (taos_metric_t *)taos_map_get(metrics, metric_name); - if (metric == NULL) return 1; + taos_metric_t *metric = (taos_metric_t *)taos_map_get_withoutlock(metrics, metric_name); + if (metric == NULL) { +#ifdef TAOS_LOG_ENABLE + char tmp[200] = {0}; + sprintf(tmp, "fail to get metric(%d):%s", count, metric_name); + TAOS_LOG(tmp); +#endif + continue;; + } r = taos_metric_formatter_load_metric_new(self, metric, ts, format, tableArray); - if (r) return r; + if (r) { + TAOS_LOG("failed to load metric"); + continue; + } + } + +#ifdef TAOS_LOG_ENABLE + char tmp[20] = {0}; + sprintf(tmp, "list count:%d", count); + TAOS_LOG(tmp); +#endif + r = pthread_rwlock_unlock(metrics->rwlock); + if (r) { + TAOS_LOG("failed to unlock"); + return r; } - //} //else{ - + /* for (taos_linked_list_node_t *current_node = metrics->keys->head; current_node != NULL; current_node = current_node->next) { const char *metric_name = (const char *)current_node->item; @@ -227,7 +260,7 @@ int taos_metric_formatter_load_metrics_new(taos_metric_formatter_t *self, taos_m r = taos_metric_formatter_load_metric(self, metric, ts, format); if (r) return r; } - + */ //} } return r; From ddc0aba1d377ce1bc9a4f6773f7f53dcd98164f9 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 23 Feb 2024 11:13:25 +0000 Subject: [PATCH 121/181] remove cluster metric --- source/libs/monitor/src/monFramework.c | 163 +++++++++++++++---------- 1 file changed, 96 insertions(+), 67 deletions(-) diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c index 9a2857d442..3a5dca3e92 100644 --- a/source/libs/monitor/src/monFramework.c +++ b/source/libs/monitor/src/monFramework.c @@ -105,21 +105,6 @@ void monInitMonitorFW(){ tsMonitor.metrics = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); taos_gauge_t *gauge = NULL; - int32_t label_count =1; - const char *sample_labels[] = {"cluster_id"}; - char *metric[] = {MASTER_UPTIME, DBS_TOTAL, TBS_TOTAL, STBS_TOTAL, VGROUPS_TOTAL, - VGROUPS_ALIVE, VNODES_TOTAL, VNODES_ALIVE, MNODES_TOTAL, MNODES_ALIVE, - CONNECTIONS_TOTAL, TOPICS_TOTAL, STREAMS_TOTAL, - DNODES_TOTAL, DNODES_ALIVE, EXPIRE_TIME, TIMESERIES_USED, - TIMESERIES_TOTAL}; - for(int32_t i = 0; i < 18; i++){ - gauge= taos_gauge_new(metric[i], "", label_count, sample_labels); - if(taos_collector_registry_register_metric(gauge) == 1){ - taos_counter_destroy(gauge); - } - taosHashPut(tsMonitor.metrics, metric[i], strlen(metric[i]), &gauge, sizeof(taos_gauge_t *)); - } - int32_t dnodes_label_count = 3; const char *dnodes_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep"}; char *dnodes_gauges[] = {UPTIME, CPU_ENGINE, CPU_SYSTEM, CPU_CORE, MEM_ENGINE, MEM_SYSTEM, @@ -156,28 +141,6 @@ void monInitMonitorFW(){ } taosHashPut(tsMonitor.metrics, dnodes_log_gauges[i], strlen(dnodes_log_gauges[i]), &gauge, sizeof(taos_gauge_t *)); } - - int32_t mnodes_role_label_count = 3; - const char *mnodes_role_sample_labels[] = {"cluster_id", "mnode_id", "mnode_ep"}; - char *mnodes_role_gauges[] = {MNODE_ROLE}; - for(int32_t i = 0; i < 1; i++){ - gauge= taos_gauge_new(mnodes_role_gauges[i], "", mnodes_role_label_count, mnodes_role_sample_labels); - if(taos_collector_registry_register_metric(gauge) == 1){ - taos_counter_destroy(gauge); - } - taosHashPut(tsMonitor.metrics, mnodes_role_gauges[i], strlen(mnodes_role_gauges[i]), &gauge, sizeof(taos_gauge_t *)); - } - - int32_t vnodes_role_label_count = 4; - const char *vnodes_role_sample_labels[] = {"cluster_id", "vgroup_id", "database_name", "dnode_id"}; - char *vnodes_role_gauges[] = {VNODE_ROLE}; - for(int32_t i = 0; i < 1; i++){ - gauge= taos_gauge_new(vnodes_role_gauges[i], "", vnodes_role_label_count, vnodes_role_sample_labels); - if(taos_collector_registry_register_metric(gauge) == 1){ - taos_counter_destroy(gauge); - } - taosHashPut(tsMonitor.metrics, vnodes_role_gauges[i], strlen(vnodes_role_gauges[i]), &gauge, sizeof(taos_gauge_t *)); - } } void monCleanupMonitorFW(){ @@ -191,51 +154,77 @@ void monGenClusterInfoTable(SMonInfo *pMonitor){ SMonBasicInfo *pBasicInfo = &pMonitor->dmInfo.basic; SMonGrantInfo *pGrantInfo = &pMonitor->mmInfo.grant; + char *metric_names[] = {MASTER_UPTIME, DBS_TOTAL, TBS_TOTAL, STBS_TOTAL, VGROUPS_TOTAL, + VGROUPS_ALIVE, VNODES_TOTAL, VNODES_ALIVE, MNODES_TOTAL, MNODES_ALIVE, + CONNECTIONS_TOTAL, TOPICS_TOTAL, STREAMS_TOTAL, + DNODES_TOTAL, DNODES_ALIVE, EXPIRE_TIME, TIMESERIES_USED, + TIMESERIES_TOTAL}; + + for(int32_t i = 0; i < 18; i++){ + if(taos_collector_registry_deregister_metric(metric_names[i]) != 0){ + uError("failed to delete metric %s", metric_names[i]); + } + + taosHashRemove(tsMonitor.metrics, metric_names[i], strlen(metric_names[i])); + } + if(pBasicInfo->cluster_id == 0) { uError("failed to generate dnode info table since cluster_id is 0"); return; } if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; - //cluster info + //cluster info + taos_gauge_t *gauge = NULL; + int32_t label_count =1; + const char *sample_labels1[] = {"cluster_id"}; + + for(int32_t i = 0; i < 18; i++){ + gauge= taos_gauge_new(metric_names[i], "", label_count, sample_labels1); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, metric_names[i], strlen(metric_names[i]), &gauge, sizeof(taos_gauge_t *)); + } + char buf[TSDB_CLUSTER_ID_LEN] = {0}; snprintf(buf, TSDB_CLUSTER_ID_LEN, "%"PRId64, pBasicInfo->cluster_id); - const char *sample_labels[] = {buf}; + const char *sample_label_values[] = {buf}; taos_gauge_t **metric = NULL; metric = taosHashGet(tsMonitor.metrics, MASTER_UPTIME, strlen(MASTER_UPTIME)); - taos_gauge_set(*metric, pInfo->master_uptime, sample_labels); + taos_gauge_set(*metric, pInfo->master_uptime, sample_label_values); metric = taosHashGet(tsMonitor.metrics, DBS_TOTAL, strlen(DBS_TOTAL)); - taos_gauge_set(*metric, pInfo->dbs_total, sample_labels); + taos_gauge_set(*metric, pInfo->dbs_total, sample_label_values); metric = taosHashGet(tsMonitor.metrics, TBS_TOTAL, strlen(TBS_TOTAL)); - taos_gauge_set(*metric, pInfo->tbs_total, sample_labels); + taos_gauge_set(*metric, pInfo->tbs_total, sample_label_values); metric = taosHashGet(tsMonitor.metrics, STBS_TOTAL, strlen(STBS_TOTAL)); - taos_gauge_set(*metric, pInfo->stbs_total, sample_labels); + taos_gauge_set(*metric, pInfo->stbs_total, sample_label_values); metric = taosHashGet(tsMonitor.metrics, VGROUPS_TOTAL, strlen(VGROUPS_TOTAL)); - taos_gauge_set(*metric, pInfo->vgroups_total, sample_labels); + taos_gauge_set(*metric, pInfo->vgroups_total, sample_label_values); metric = taosHashGet(tsMonitor.metrics, VGROUPS_ALIVE, strlen(VGROUPS_ALIVE)); - taos_gauge_set(*metric, pInfo->vgroups_alive, sample_labels); + taos_gauge_set(*metric, pInfo->vgroups_alive, sample_label_values); metric = taosHashGet(tsMonitor.metrics, VNODES_TOTAL, strlen(VNODES_TOTAL)); - taos_gauge_set(*metric, pInfo->vnodes_total, sample_labels); + taos_gauge_set(*metric, pInfo->vnodes_total, sample_label_values); metric = taosHashGet(tsMonitor.metrics, VNODES_ALIVE, strlen(VNODES_ALIVE)); - taos_gauge_set(*metric, pInfo->vnodes_alive, sample_labels); + taos_gauge_set(*metric, pInfo->vnodes_alive, sample_label_values); metric = taosHashGet(tsMonitor.metrics, CONNECTIONS_TOTAL, strlen(CONNECTIONS_TOTAL)); - taos_gauge_set(*metric, pInfo->connections_total, sample_labels); + taos_gauge_set(*metric, pInfo->connections_total, sample_label_values); metric = taosHashGet(tsMonitor.metrics, TOPICS_TOTAL, strlen(TOPICS_TOTAL)); - taos_gauge_set(*metric, pInfo->topics_toal, sample_labels); + taos_gauge_set(*metric, pInfo->topics_toal, sample_label_values); metric = taosHashGet(tsMonitor.metrics, STREAMS_TOTAL, strlen(STREAMS_TOTAL)); - taos_gauge_set(*metric, pInfo->streams_total, sample_labels); + taos_gauge_set(*metric, pInfo->streams_total, sample_label_values); //dnodes number int32_t dnode_total = taosArrayGetSize(pInfo->dnodes); @@ -250,10 +239,10 @@ void monGenClusterInfoTable(SMonInfo *pMonitor){ } metric = taosHashGet(tsMonitor.metrics, DNODES_TOTAL, strlen(DNODES_TOTAL)); - taos_gauge_set(*metric, dnode_total, sample_labels); + taos_gauge_set(*metric, dnode_total, sample_label_values); metric = taosHashGet(tsMonitor.metrics, DNODES_ALIVE, strlen(DNODES_ALIVE)); - taos_gauge_set(*metric, dnode_alive, sample_labels); + taos_gauge_set(*metric, dnode_alive, sample_label_values); //mnodes number int32_t mnode_total = taosArrayGetSize(pInfo->mnodes); @@ -282,23 +271,31 @@ void monGenClusterInfoTable(SMonInfo *pMonitor){ } metric = taosHashGet(tsMonitor.metrics, MNODES_TOTAL, strlen(MNODES_TOTAL)); - taos_gauge_set(*metric, mnode_total, sample_labels); + taos_gauge_set(*metric, mnode_total, sample_label_values); metric = taosHashGet(tsMonitor.metrics, MNODES_ALIVE, strlen(MNODES_ALIVE)); - taos_gauge_set(*metric, mnode_alive, sample_labels); + taos_gauge_set(*metric, mnode_alive, sample_label_values); //grant info metric = taosHashGet(tsMonitor.metrics, EXPIRE_TIME, strlen(EXPIRE_TIME)); - taos_gauge_set(*metric, pGrantInfo->expire_time, sample_labels); + taos_gauge_set(*metric, pGrantInfo->expire_time, sample_label_values); metric = taosHashGet(tsMonitor.metrics, TIMESERIES_USED, strlen(TIMESERIES_USED)); - taos_gauge_set(*metric, pGrantInfo->timeseries_used, sample_labels); + taos_gauge_set(*metric, pGrantInfo->timeseries_used, sample_label_values); metric = taosHashGet(tsMonitor.metrics, TIMESERIES_TOTAL, strlen(TIMESERIES_TOTAL)); - taos_gauge_set(*metric, pGrantInfo->timeseries_total, sample_labels); + taos_gauge_set(*metric, pGrantInfo->timeseries_total, sample_label_values); } void monGenVgroupInfoTable(SMonInfo *pMonitor){ + if(taos_collector_registry_deregister_metric(TABLES_NUM) != 0){ + uError("failed to delete metric "TABLES_NUM); + } + + if(taos_collector_registry_deregister_metric(STATUS) != 0){ + uError("failed to delete metric "STATUS); + } + if(pMonitor->dmInfo.basic.cluster_id == 0) return; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; @@ -307,20 +304,10 @@ void monGenVgroupInfoTable(SMonInfo *pMonitor){ int32_t vgroup_label_count = 3; const char *vgroup_sample_labels[] = {"cluster_id", "vgroup_id", "database_name"}; - - if(taos_collector_registry_deregister_metric(TABLES_NUM) != 0){ - uError("failed to delete metric "TABLES_NUM); - } - taos_gauge_t *tableNumGauge = taos_gauge_new(TABLES_NUM, "", vgroup_label_count, vgroup_sample_labels); if(taos_collector_registry_register_metric(tableNumGauge) == 1){ taos_counter_destroy(tableNumGauge); } - - if(taos_collector_registry_deregister_metric(STATUS) != 0){ - uError("failed to delete metric "STATUS); - } - taos_gauge_t *statusGauge = taos_gauge_new(STATUS, "", vgroup_label_count, vgroup_sample_labels); if(taos_collector_registry_register_metric(statusGauge) == 1){ taos_counter_destroy(statusGauge); @@ -617,11 +604,32 @@ void monGenLogDiskTable(SMonInfo *pMonitor){ } void monGenMnodeRoleTable(SMonInfo *pMonitor){ + char *mnodes_role_gauges[] = {MNODE_ROLE}; + + for(int32_t i = 0; i < 1; i++){ + if(taos_collector_registry_deregister_metric(mnodes_role_gauges[i]) != 0){ + uError("failed to delete metric %s", mnodes_role_gauges[i]); + } + + taosHashRemove(tsMonitor.metrics, mnodes_role_gauges[i], strlen(mnodes_role_gauges[i])); + } + SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; SMonBasicInfo *pBasicInfo = &pMonitor->dmInfo.basic; if(pBasicInfo->cluster_id == 0) return; + taos_gauge_t *gauge = NULL; + int32_t mnodes_role_label_count = 3; + const char *mnodes_role_sample_labels[] = {"cluster_id", "mnode_id", "mnode_ep"}; + for(int32_t i = 0; i < 1; i++){ + gauge= taos_gauge_new(mnodes_role_gauges[i], "", mnodes_role_label_count, mnodes_role_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, mnodes_role_gauges[i], strlen(mnodes_role_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } + char buf[TSDB_CLUSTER_ID_LEN] = {0}; snprintf(buf, TSDB_CLUSTER_ID_LEN, "%" PRId64, pBasicInfo->cluster_id); @@ -661,12 +669,33 @@ void monGenMnodeRoleTable(SMonInfo *pMonitor){ } void monGenVnodeRoleTable(SMonInfo *pMonitor){ + char *vnodes_role_gauges[] = {VNODE_ROLE}; + taos_gauge_t *gauge = NULL; + + for(int32_t i = 0; i < 1; i++){ + if(taos_collector_registry_deregister_metric(vnodes_role_gauges[i]) != 0){ + uError("failed to delete metric %s", vnodes_role_gauges[i]); + } + + taosHashRemove(tsMonitor.metrics, vnodes_role_gauges[i], strlen(vnodes_role_gauges[i])); + } + SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; SMonBasicInfo *pBasicInfo = &pMonitor->dmInfo.basic; if(pBasicInfo->cluster_id == 0) return; + int32_t vnodes_role_label_count = 4; + const char *vnodes_role_sample_labels[] = {"cluster_id", "vgroup_id", "database_name", "dnode_id"}; + for(int32_t i = 0; i < 1; i++){ + gauge= taos_gauge_new(vnodes_role_gauges[i], "", vnodes_role_label_count, vnodes_role_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, vnodes_role_gauges[i], strlen(vnodes_role_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } + char buf[TSDB_CLUSTER_ID_LEN] = {0}; snprintf(buf, TSDB_CLUSTER_ID_LEN, "%" PRId64, pBasicInfo->cluster_id); From 1ca24fae6e37122ba8b03de8d6adc8a3be4dbc9f Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 23 Feb 2024 11:44:55 +0000 Subject: [PATCH 122/181] fix/TD-28776 --- source/dnode/mnode/impl/src/mndTrans.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 1adc4ed4bf..1fe18ec320 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1100,7 +1100,10 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->rawWritten) return 0; - if (topHalf) return TSDB_CODE_MND_TRANS_CTX_SWITCH; + if (topHalf) { + terrno = TSDB_CODE_MND_TRANS_CTX_SWITCH; + return TSDB_CODE_MND_TRANS_CTX_SWITCH; + } int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pAction->pRaw); if (code == 0 || terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { From df2488431e40e6edd887b075a8e4bb51d8a496a4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Feb 2024 00:40:26 +0800 Subject: [PATCH 123/181] fix(stream): add logs. --- source/common/src/tdatablock.c | 16 ++++++++-------- source/dnode/vnode/src/tq/tqSink.c | 13 +++++++------ source/libs/stream/src/streamDispatch.c | 2 +- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 9439c172c4..35c886dd90 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2118,28 +2118,28 @@ _end: return TSDB_CODE_SUCCESS; } -void buildCtbNameAddGruopId(char* ctbName, uint64_t groupId){ +void buildCtbNameAddGroupId(char* ctbName, uint64_t groupId){ char tmp[TSDB_TABLE_NAME_LEN] = {0}; snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%"PRIu64, groupId); ctbName[TSDB_TABLE_NAME_LEN - strlen(tmp) - 1] = 0; // put groupId to the end strcat(ctbName, tmp); } -bool isAutoTableName(char* ctbName){ - return (strlen(ctbName) == 34 && ctbName[0] == 't' && ctbName[1] == '_'); -} +// auto stream subtable name starts with 't_', followed by the first segment of MD5 digest for group vals. +// the total length is fixed to be 34 bytes. +bool isAutoTableName(char* ctbName) { return (strlen(ctbName) == 34 && ctbName[0] == 't' && ctbName[1] == '_'); } -bool alreadyAddGroupId(char* ctbName){ +bool alreadyAddGroupId(char* ctbName) { size_t len = strlen(ctbName); size_t _location = len - 1; - while(_location > 0){ - if(ctbName[_location] < '0' || ctbName[_location] > '9'){ + while (_location > 0) { + if (ctbName[_location] < '0' || ctbName[_location] > '9') { break; } _location--; } - return ctbName[_location] == '_' && len - 1 - _location > 15; //15 means the min length of groupid + return ctbName[_location] == '_' && len - 1 - _location > 15; // 15 means the min length of groupid } char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index b56bf3e0fe..70519a5cd2 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -72,7 +72,7 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); memcpy(name, varDataVal(varTbName), varDataLen(varTbName)); if (newSubTableRule && !isAutoTableName(name) && !alreadyAddGroupId(name) && groupId != 0) { - buildCtbNameAddGruopId(name, groupId); + buildCtbNameAddGroupId(name, groupId); } } else if (stbFullName) { name = buildCtbNameByGroupId(stbFullName, groupId); @@ -185,7 +185,7 @@ void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDa !alreadyAddGroupId(pDataBlock->info.parTbName) && gid != 0) { pCreateTableReq->name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); strcpy(pCreateTableReq->name, pDataBlock->info.parTbName); - buildCtbNameAddGruopId(pCreateTableReq->name, gid); + buildCtbNameAddGroupId(pCreateTableReq->name, gid); } else { pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName); } @@ -196,12 +196,12 @@ void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDa static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, int64_t suid) { - tqDebug("s-task:%s build create table msg", pTask->id.idStr); - STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; int32_t rows = pDataBlock->info.rows; SArray* tagArray = taosArrayInit(4, sizeof(STagVal)); - ; + + tqDebug("s-task:%s build create %d table(s) msg", pTask->id.idStr, rows); + int32_t code = 0; SVCreateTbBatchReq reqs = {0}; @@ -670,7 +670,8 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat } else { if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) && !alreadyAddGroupId(dstTableName) && groupId != 0) { - buildCtbNameAddGruopId(dstTableName, groupId); + tqDebug("s-task:%s append groupId:%" PRId64 " for generated dstTable:%s", id, groupId, dstTableName); + buildCtbNameAddGroupId(dstTableName, groupId); } } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 9383383dc0..60c545b9e5 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -575,7 +575,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S !isAutoTableName(pDataBlock->info.parTbName) && !alreadyAddGroupId(pDataBlock->info.parTbName) && groupId != 0){ - buildCtbNameAddGruopId(pDataBlock->info.parTbName, groupId); + buildCtbNameAddGroupId(pDataBlock->info.parTbName, groupId); } } else { buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName); From c4562b5d2620ec886f55188cec7c1f30bc8e6a7a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Feb 2024 00:54:57 +0800 Subject: [PATCH 124/181] fix(stream): fix syntax error. --- include/common/tdatablock.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index bcb40f4175..bac3031a0b 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -264,7 +264,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pData bool alreadyAddGroupId(char* ctbName); bool isAutoTableName(char* ctbName); -void buildCtbNameAddGruopId(char* ctbName, uint64_t groupId); +void buildCtbNameAddGroupId(char* ctbName, uint64_t groupId); char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* pBuf); From bb31b0c12175ee3f5cb1e1f8adebd7cfe47f2f4f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Feb 2024 02:08:49 +0800 Subject: [PATCH 125/181] fix(stream): add logs. --- source/dnode/vnode/src/tq/tqSink.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 800b84df61..c9df5cfce0 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -186,11 +186,14 @@ void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDa pCreateTableReq->name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); strcpy(pCreateTableReq->name, pDataBlock->info.parTbName); buildCtbNameAddGroupId(pCreateTableReq->name, gid); + tqDebug("gen name from:%s", pDataBlock->info.parTbName); } else { pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName); + tqDebug("copy name:%s", pDataBlock->info.parTbName); } } else { pCreateTableReq->name = buildCtbNameByGroupId(stbFullName, gid); + tqDebug("gen name from stbFullName:%s gid:%"PRId64, stbFullName, gid); } } From 3cd158f681c9952091832159e5d7b1c7f586a000 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Feb 2024 02:19:40 +0800 Subject: [PATCH 126/181] fix(stream): fix the invalid check for subtable name. --- source/common/src/tdatablock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 033791a572..9f55b67ea3 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2139,7 +2139,7 @@ bool alreadyAddGroupId(char* ctbName) { _location--; } - return ctbName[_location] == '_' && len - 1 - _location > 15; // 15 means the min length of groupid + return ctbName[_location] == '_' && len - 1 - _location >= 15; // 15 means the min length of groupid } char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { From ace5983fae48e4e64b8ce6c09c67ff5ddb261fe3 Mon Sep 17 00:00:00 2001 From: dmchen Date: Sat, 24 Feb 2024 07:04:32 +0000 Subject: [PATCH 127/181] remove dnodes status --- source/libs/monitor/src/monFramework.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c index 3a5dca3e92..d58cee7b9a 100644 --- a/source/libs/monitor/src/monFramework.c +++ b/source/libs/monitor/src/monFramework.c @@ -110,9 +110,9 @@ void monInitMonitorFW(){ char *dnodes_gauges[] = {UPTIME, CPU_ENGINE, CPU_SYSTEM, CPU_CORE, MEM_ENGINE, MEM_SYSTEM, MEM_TOTAL, DISK_ENGINE, DISK_USED, DISK_TOTAL, NET_IN, NET_OUT, IO_READ, IO_WRITE, IO_READ_DISK, IO_WRITE_DISK, /*ERRORS,*/ - VNODES_NUM, MASTERS, HAS_MNODE, HAS_QNODE, HAS_SNODE, DNODE_STATUS, + VNODES_NUM, MASTERS, HAS_MNODE, HAS_QNODE, HAS_SNODE, DNODE_LOG_ERROR, DNODE_LOG_INFO, DNODE_LOG_DEBUG, DNODE_LOG_TRACE}; - for(int32_t i = 0; i < 26; i++){ + for(int32_t i = 0; i < 25; i++){ gauge= taos_gauge_new(dnodes_gauges[i], "", dnodes_label_count, dnodes_sample_labels); if(taos_collector_registry_register_metric(gauge) == 1){ taos_counter_destroy(gauge); @@ -502,16 +502,29 @@ void monGenDnodeInfoTable(SMonInfo *pMonitor) { } void monGenDnodeStatusInfoTable(SMonInfo *pMonitor){ + if(taos_collector_registry_deregister_metric(DNODE_STATUS) != 0){ + uError("failed to delete metric "DNODE_STATUS); + } + if(pMonitor->dmInfo.basic.cluster_id == 0) { uError("failed to generate dnode info table since cluster_id is 0"); return; } if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + taos_gauge_t *gauge = NULL; + + int32_t dnodes_label_count = 3; + const char *dnodes_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep"}; + + gauge= taos_gauge_new(DNODE_STATUS, "", dnodes_label_count, dnodes_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + char cluster_id[TSDB_CLUSTER_ID_LEN]; snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%"PRId64, pMonitor->dmInfo.basic.cluster_id); - - taos_gauge_t **metric = NULL; + //dnodes status SMonClusterInfo *pClusterInfo = &pMonitor->mmInfo.cluster; @@ -524,13 +537,11 @@ void monGenDnodeStatusInfoTable(SMonInfo *pMonitor){ const char *sample_labels[] = {cluster_id, dnode_id, pDnodeDesc->dnode_ep}; - metric = taosHashGet(tsMonitor.metrics, DNODE_STATUS, strlen(DNODE_STATUS)); - int32_t status = 0; if(strcmp(pDnodeDesc->status, "ready") == 0){ status = 1; } - taos_gauge_set(*metric, status, sample_labels); + taos_gauge_set(gauge, status, sample_labels); } } From 2df6ae111617df634819e20a542136aa4037d5da Mon Sep 17 00:00:00 2001 From: dmchen Date: Sat, 24 Feb 2024 09:48:28 +0000 Subject: [PATCH 128/181] first net_in net_out --- source/os/src/osSysinfo.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index fea7a4f63d..fad5da3f79 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -905,8 +905,15 @@ void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) { int64_t cur_receive_bytes = 0; int64_t cur_transmit_bytes = 0; if (taosGetCardInfo(&cur_receive_bytes, &cur_transmit_bytes) == 0) { - *receive_bytes = cur_receive_bytes - last_receive_bytes; - *transmit_bytes = cur_transmit_bytes - last_transmit_bytes; + if(last_receive_bytes > 0 && last_receive_bytes > 0){ + *receive_bytes = cur_receive_bytes - last_receive_bytes; + *transmit_bytes = cur_transmit_bytes - last_transmit_bytes; + } + else{ + *receive_bytes = 0; + *transmit_bytes = 0; + } + last_receive_bytes = cur_receive_bytes; last_transmit_bytes = cur_transmit_bytes; } else { From 917edde91fca2220a2a73ae50e5dd82d57be7233 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 24 Feb 2024 22:25:30 +0800 Subject: [PATCH 129/181] user sysinfo --- source/common/src/systable.c | 50 +++++++++++++------------- source/libs/parser/src/parTranslater.c | 4 --- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 47eac317ec..ff1eeba0a2 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -223,12 +223,12 @@ static const SSysDbTableSchema userTblDistSchema[] = { }; static const SSysDbTableSchema userUsersSchema[] = { - {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false}, - {.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false}, - {.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, - {.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, + {.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, + {.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, + {.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; GRANTS_SCHEMA; @@ -304,7 +304,7 @@ static const SSysDbTableSchema subscriptionSchema[] = { static const SSysDbTableSchema vnodesSchema[] = { {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = true}, {.name = "status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, @@ -312,12 +312,12 @@ static const SSysDbTableSchema vnodesSchema[] = { }; static const SSysDbTableSchema userUserPrivilegesSchema[] = { - {.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "notes", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "notes", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema userViewsSchema[] = { @@ -349,21 +349,21 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = { }; static const SSysDbTableSchema useGrantsFullSchema[] = { - {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema useGrantsLogsSchema[] = { - {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema useMachinesSchema[] = { - {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysTableMeta infosMeta[] = { @@ -395,9 +395,9 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false}, {TSDB_INS_TABLE_COMPACTS, userCompactsSchema, tListLen(userCompactsSchema), false}, {TSDB_INS_TABLE_COMPACT_DETAILS, userCompactsDetailSchema, tListLen(userCompactsDetailSchema), false}, - {TSDB_INS_TABLE_GRANTS_FULL, useGrantsFullSchema, tListLen(useGrantsFullSchema), false}, - {TSDB_INS_TABLE_GRANTS_LOGS, useGrantsLogsSchema, tListLen(useGrantsLogsSchema), false}, - {TSDB_INS_TABLE_MACHINES, useMachinesSchema, tListLen(useMachinesSchema), false}, + {TSDB_INS_TABLE_GRANTS_FULL, useGrantsFullSchema, tListLen(useGrantsFullSchema), true}, + {TSDB_INS_TABLE_GRANTS_LOGS, useGrantsLogsSchema, tListLen(useGrantsLogsSchema), true}, + {TSDB_INS_TABLE_MACHINES, useMachinesSchema, tListLen(useMachinesSchema), true}, }; static const SSysDbTableSchema connectionsSchema[] = { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index cea82ef3fb..c94cf3f174 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -399,10 +399,6 @@ static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCac int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView) { int32_t code = TSDB_CODE_SUCCESS; - if (!pParCxt->enableSysInfo && IS_SYS_DBNAME(pName->dbname)) { - code = TSDB_CODE_PAR_PERMISSION_DENIED; - return code; - } if (pParCxt->async) { code = getTableMetaFromCache(pMetaCache, pName, pMeta); #ifdef TD_ENTERPRISE From d4ef374108a27d1bd3da28ef13310767f9f8a54e Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 24 Feb 2024 22:48:28 +0800 Subject: [PATCH 130/181] user sysinfo --- include/common/tgrant.h | 40 ++++++++++---------- tests/script/tsim/user/privilege_sysinfo.sim | 6 +++ 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 6d11608d21..ca9e030117 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -67,28 +67,28 @@ int32_t grantCheckLE(EGrantType grant); // #ifndef GRANTS_CFG #ifdef TD_ENTERPRISE -#define GRANTS_SCHEMA \ - static const SSysDbTableSchema grantsSchema[] = { \ - {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ +#define GRANTS_SCHEMA \ + static const SSysDbTableSchema grantsSchema[] = { \ + {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ } #else -#define GRANTS_SCHEMA \ - static const SSysDbTableSchema grantsSchema[] = { \ - {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ +#define GRANTS_SCHEMA \ + static const SSysDbTableSchema grantsSchema[] = { \ + {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ } #endif // #define GRANT_CFG_ADD diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index 567d963c86..40098504ae 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -129,6 +129,9 @@ sql show topics sql show subscriptions sql show functions sql_error show grants +sql_error show grants full; +sql_error show grants logs; +sql_error show cluster machines; sql show queries sql show connections sql show apps @@ -166,6 +169,9 @@ sql select * from information_schema.ins_topics sql select * from information_schema.ins_subscriptions sql select * from information_schema.ins_streams sql_error select * from information_schema.ins_grants +sql_error select * from information_schema.ins_grants_full +sql_error select * from information_schema.ins_grants_logs +sql_error select * from information_schema.ins_machines sql_error select * from information_schema.ins_vgroups sql select * from information_schema.ins_configs sql_error select * from information_schema.ins_dnode_variables From 59ad27fcae94668887dc7448bd47eb5897acce5b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Feb 2024 22:50:17 +0800 Subject: [PATCH 131/181] other(stream): comment some logs. --- source/dnode/vnode/src/tq/tqSink.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index c9df5cfce0..9ff4e3ba62 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -186,14 +186,14 @@ void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDa pCreateTableReq->name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); strcpy(pCreateTableReq->name, pDataBlock->info.parTbName); buildCtbNameAddGroupId(pCreateTableReq->name, gid); - tqDebug("gen name from:%s", pDataBlock->info.parTbName); +// tqDebug("gen name from:%s", pDataBlock->info.parTbName); } else { pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName); - tqDebug("copy name:%s", pDataBlock->info.parTbName); +// tqDebug("copy name:%s", pDataBlock->info.parTbName); } } else { pCreateTableReq->name = buildCtbNameByGroupId(stbFullName, gid); - tqDebug("gen name from stbFullName:%s gid:%"PRId64, stbFullName, gid); +// tqDebug("gen name from stbFullName:%s gid:%"PRId64, stbFullName, gid); } } From a38a69557a52ea8c5056d19650cea4b59cd29c5f Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 24 Feb 2024 23:05:39 +0800 Subject: [PATCH 132/181] user sysinfo --- source/libs/parser/src/parAuthenticator.c | 2 ++ source/libs/parser/src/parTranslater.c | 1 + tests/script/tsim/user/privilege_sysinfo.sim | 3 ++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index c2cd4786db..3703c27c66 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -349,6 +349,8 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: case QUERY_NODE_SHOW_VNODES_STMT: case QUERY_NODE_SHOW_SCORES_STMT: + case QUERY_NODE_SHOW_USERS_STMT: + case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: case QUERY_NODE_SHOW_GRANTS_FULL_STMT: case QUERY_NODE_SHOW_GRANTS_LOGS_STMT: case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT: diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c94cf3f174..c30cc88f67 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -399,6 +399,7 @@ static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCac int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView) { int32_t code = TSDB_CODE_SUCCESS; + if (pParCxt->async) { code = getTableMetaFromCache(pMetaCache, pName, pMeta); #ifdef TD_ENTERPRISE diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index 40098504ae..135e05c3b3 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -101,6 +101,7 @@ sql select * from d2.ntb2 print =============== check show sql_error show users +sql_error show user privileges sql_error show cluster sql_error select * from information_schema.ins_dnodes sql_error select * from information_schema.ins_mnodes @@ -158,7 +159,7 @@ sql_error select * from information_schema.ins_modules sql_error select * from information_schema.ins_qnodes sql_error select * from information_schema.ins_cluster sql_error select * from information_schema.ins_users -sql_error select * from information_schema.ins_user_privilege +sql_error select * from information_schema.ins_user_privileges sql select * from information_schema.ins_databases sql select * from information_schema.ins_functions sql select * from information_schema.ins_indexes From 8ad1ed885192efa4b91f9826fc49c96dba9f907c Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 24 Feb 2024 23:08:01 +0800 Subject: [PATCH 133/181] user sysinfo --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c30cc88f67..abe50a27da 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -399,7 +399,7 @@ static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCac int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView) { int32_t code = TSDB_CODE_SUCCESS; - + if (pParCxt->async) { code = getTableMetaFromCache(pMetaCache, pName, pMeta); #ifdef TD_ENTERPRISE From 41a79866b123cf022cedda34c262a12957ce8cb1 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 25 Feb 2024 10:23:05 +0800 Subject: [PATCH 134/181] test: check user sysinfo during alter repeatedly --- source/common/src/systable.c | 2 +- source/libs/parser/src/parAuthenticator.c | 1 + tests/script/tsim/user/privilege_sysinfo.sim | 123 ++++++++++++++++++- 3 files changed, 121 insertions(+), 5 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index ff1eeba0a2..25cc5d7c79 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -276,7 +276,7 @@ static const SSysDbTableSchema configSchema[] = { }; static const SSysDbTableSchema variablesSchema[] = { - {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "value", .bytes = TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "scope", .bytes = TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index 3703c27c66..4f523bf4f2 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -347,6 +347,7 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: + case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: case QUERY_NODE_SHOW_VNODES_STMT: case QUERY_NODE_SHOW_SCORES_STMT: case QUERY_NODE_SHOW_USERS_STMT: diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index 135e05c3b3..ebdd42e1a8 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -99,7 +99,11 @@ sql select * from d2.stb2 sql select * from d2.ctb2 sql select * from d2.ntb2 -print =============== check show +$loop_cnt = 0 +$loop_flag = 0 + +loop_check_sysinfo_0: +print =============== check show of sysinfo 0 sql_error show users sql_error show user privileges sql_error show cluster @@ -146,7 +150,7 @@ sql_error show dnode 1 variables; sql show variables; -print =============== check information_schema +print =============== check information_schema of sysinfo 0 sql show databases if $rows != 3 then return -1 @@ -177,7 +181,7 @@ sql_error select * from information_schema.ins_vgroups sql select * from information_schema.ins_configs sql_error select * from information_schema.ins_dnode_variables -print =============== check performance_schema +print =============== check performance_schema of sysinfo 0 sql use performance_schema; sql select * from performance_schema.perf_connections sql select * from performance_schema.perf_queries @@ -185,4 +189,115 @@ sql select * from performance_schema.perf_consumers sql select * from performance_schema.perf_trans sql select * from performance_schema.perf_apps -#system sh/exec.sh -n dnode1 -s stop -x SIGINT +goto loop_check_switch + +loop_check_sysinfo_1: +print =============== check show of sysinfo 1 +sql show users +sql show user privileges +sql show cluster +sql select * from information_schema.ins_dnodes +sql select * from information_schema.ins_mnodes +sql show snodes +sql select * from information_schema.ins_qnodes +sql show dnodes +sql show snodes +sql show qnodes +sql show mnodes +sql show db.vgroups +sql_error show db.stables +sql_error show db.tables +sql show indexes from stb from db +sql show databases +sql show d2.vgroups +sql show d2.stables +sql show d2.tables +sql show indexes from stb2 from d2 +#sql_error show create database db +sql_error show create table db.stb; +sql_error show create table db.ctb; +sql_error show create table db.ntb; +sql show streams +sql show consumers +sql show topics +sql show subscriptions +sql show functions +sql show grants +sql show grants full; +sql show grants logs; +sql show cluster machines; +sql show queries +sql show connections +sql show apps +sql show transactions +sql show create database d2 +sql show create table d2.stb2; +sql show create table d2.ctb2; +sql show create table d2.ntb2; +sql show local variables; +sql_error show dnode 1 variables; +sql show variables; + + +print =============== check information_schema of sysinfo 1 +sql show databases +if $rows != 3 then + return -1 +endi + +sql use information_schema; +sql select * from information_schema.ins_dnodes +sql select * from information_schema.ins_mnodes +sql_error select * from information_schema.ins_modules +sql select * from information_schema.ins_qnodes +sql select * from information_schema.ins_cluster +sql select * from information_schema.ins_users +sql select * from information_schema.ins_user_privileges +sql select * from information_schema.ins_databases +sql select * from information_schema.ins_functions +sql select * from information_schema.ins_indexes +sql select * from information_schema.ins_stables +sql select * from information_schema.ins_tables +sql select * from information_schema.ins_tags +sql select * from information_schema.ins_topics +sql select * from information_schema.ins_subscriptions +sql select * from information_schema.ins_streams +sql select * from information_schema.ins_grants +sql select * from information_schema.ins_grants_full +sql select * from information_schema.ins_grants_logs +sql select * from information_schema.ins_machines +sql select * from information_schema.ins_vgroups +sql select * from information_schema.ins_configs +sql_error select * from information_schema.ins_dnode_variables + +print =============== check performance_schema of sysinfo 1 +sql use performance_schema; +sql select * from performance_schema.perf_connections +sql select * from performance_schema.perf_queries +sql select * from performance_schema.perf_consumers +sql select * from performance_schema.perf_trans +sql select * from performance_schema.perf_apps + +loop_check_switch: +if $loop_cnt > 5 then + goto loop_check_end +endi +$loop_cnt = $loop_cnt + 1 + +if $loop_flag == 0 then + system taos -P7100 -s 'alter user sysinfo0 sysinfo 1' + sleep 2000 + $loop_flag = 1 + goto loop_check_sysinfo_1 +else + system taos -P7100 -s 'alter user sysinfo0 sysinfo 0' + sleep 2000 + $loop_flag = 0 + goto loop_check_sysinfo_0 +endi +goto loop_check_sysinfo_0 + +loop_check_end: +print =============== check end + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From c4b42dbbff701f320640742642ad00b8bbad5796 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 25 Feb 2024 10:34:49 +0800 Subject: [PATCH 135/181] test: check user sysinfo during alter repeatedly --- tests/script/tsim/user/privilege_sysinfo.sim | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index ebdd42e1a8..625b1ec04c 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -99,7 +99,8 @@ sql select * from d2.stb2 sql select * from d2.ctb2 sql select * from d2.ntb2 -$loop_cnt = 0 +$loop_cnt = 5 +$loop_idx = 0 $loop_flag = 0 loop_check_sysinfo_0: @@ -279,19 +280,19 @@ sql select * from performance_schema.perf_trans sql select * from performance_schema.perf_apps loop_check_switch: -if $loop_cnt > 5 then +if $loop_idx > $loop_cnt then goto loop_check_end endi -$loop_cnt = $loop_cnt + 1 +$loop_idx = $loop_idx + 1 if $loop_flag == 0 then system taos -P7100 -s 'alter user sysinfo0 sysinfo 1' - sleep 2000 + sleep 3000 $loop_flag = 1 goto loop_check_sysinfo_1 else system taos -P7100 -s 'alter user sysinfo0 sysinfo 0' - sleep 2000 + sleep 3000 $loop_flag = 0 goto loop_check_sysinfo_0 endi From e27138e4135ec52758e9c83d098249e466f2f743 Mon Sep 17 00:00:00 2001 From: dmchen Date: Sun, 25 Feb 2024 03:45:30 +0000 Subject: [PATCH 136/181] first statis --- source/os/src/osSysinfo.c | 54 ++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index fad5da3f79..173bb982db 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -563,9 +563,9 @@ int32_t taosGetCpuCores(float *numOfCores, bool physical) { } void taosGetCpuUsage(double *cpu_system, double *cpu_engine) { - static int64_t lastSysUsed = 0; - static int64_t lastSysTotal = 0; - static int64_t lastProcTotal = 0; + static int64_t lastSysUsed = -1; + static int64_t lastSysTotal = -1; + static int64_t lastProcTotal = -1; static int64_t curSysUsed = 0; static int64_t curSysTotal = 0; static int64_t curProcTotal = 0; @@ -580,14 +580,20 @@ void taosGetCpuUsage(double *cpu_system, double *cpu_engine) { curSysTotal = curSysUsed + sysCpu.idle; curProcTotal = procCpu.utime + procCpu.stime + procCpu.cutime + procCpu.cstime; - if (curSysTotal - lastSysTotal > 0 && curSysUsed >= lastSysUsed && curProcTotal >= lastProcTotal) { - if (cpu_system != NULL) { - *cpu_system = (curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100; - } - if (cpu_engine != NULL) { - *cpu_engine = (curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100; + if(lastSysUsed >= 0 && lastSysTotal >=0 && lastProcTotal >=0){ + if (curSysTotal - lastSysTotal > 0 && curSysUsed >= lastSysUsed && curProcTotal >= lastProcTotal) { + if (cpu_system != NULL) { + *cpu_system = (curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100; + } + if (cpu_engine != NULL) { + *cpu_engine = (curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100; + } } } + else{ + *cpu_system = 0; + *cpu_engine = 0; + } lastSysUsed = curSysUsed; lastSysTotal = curSysTotal; @@ -821,19 +827,27 @@ int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int } void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { - static int64_t last_rchars = 0; - static int64_t last_wchars = 0; - static int64_t last_read_bytes = 0; - static int64_t last_write_bytes = 0; + static int64_t last_rchars = -1; + static int64_t last_wchars = -1; + static int64_t last_read_bytes = -1; + static int64_t last_write_bytes = -1; static int64_t cur_rchars = 0; static int64_t cur_wchars = 0; static int64_t cur_read_bytes = 0; static int64_t cur_write_bytes = 0; if (taosGetProcIO(&cur_rchars, &cur_wchars, &cur_read_bytes, &cur_write_bytes) == 0) { - *rchars = cur_rchars - last_rchars; - *wchars = cur_wchars - last_wchars; - *read_bytes = cur_read_bytes - last_read_bytes; - *write_bytes = cur_write_bytes - last_write_bytes; + if(last_rchars >=0 && last_wchars >=0 && last_read_bytes >=0 && last_write_bytes >= 0){ + *rchars = cur_rchars - last_rchars; + *wchars = cur_wchars - last_wchars; + *read_bytes = cur_read_bytes - last_read_bytes; + *write_bytes = cur_write_bytes - last_write_bytes; + } + else{ + *rchars = 0; + *wchars = 0; + *read_bytes = 0; + *write_bytes = 0; + } last_rchars = cur_rchars; last_wchars = cur_wchars; last_read_bytes = cur_read_bytes; @@ -900,12 +914,12 @@ int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { } void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) { - static int64_t last_receive_bytes = 0; - static int64_t last_transmit_bytes = 0; + static int64_t last_receive_bytes = -1; + static int64_t last_transmit_bytes = -1; int64_t cur_receive_bytes = 0; int64_t cur_transmit_bytes = 0; if (taosGetCardInfo(&cur_receive_bytes, &cur_transmit_bytes) == 0) { - if(last_receive_bytes > 0 && last_receive_bytes > 0){ + if(last_receive_bytes >= 0 && last_transmit_bytes >= 0){ *receive_bytes = cur_receive_bytes - last_receive_bytes; *transmit_bytes = cur_transmit_bytes - last_transmit_bytes; } From 0d5b997d943cefd0160590b5d6ee7fb617596496 Mon Sep 17 00:00:00 2001 From: dmchen Date: Sun, 25 Feb 2024 03:56:12 +0000 Subject: [PATCH 137/181] first set --- source/os/src/osSysinfo.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 173bb982db..187461826a 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -590,10 +590,6 @@ void taosGetCpuUsage(double *cpu_system, double *cpu_engine) { } } } - else{ - *cpu_system = 0; - *cpu_engine = 0; - } lastSysUsed = curSysUsed; lastSysTotal = curSysTotal; From 6ddd34c4d1def076671ca46cecd5897ab309d1e6 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 25 Feb 2024 15:21:37 +0800 Subject: [PATCH 138/181] other: trigger CI From e1add42485a33dd7238e68cacce62a9a033ad9a9 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 25 Feb 2024 17:58:20 +0800 Subject: [PATCH 139/181] Update privilege_sysinfo.sim --- tests/script/tsim/user/privilege_sysinfo.sim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index 625b1ec04c..cf87f63b8c 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -296,9 +296,8 @@ else $loop_flag = 0 goto loop_check_sysinfo_0 endi -goto loop_check_sysinfo_0 loop_check_end: print =============== check end -#system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +#system sh/exec.sh -n dnode1 -s stop -x SIGINT From 24ec06bfe4b85ba6ec53757072042903bb5ec114 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Sun, 25 Feb 2024 19:28:32 +0800 Subject: [PATCH 140/181] reset close window state --- source/libs/executor/src/streamcountwindowoperator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 294c2730df..3daefcdc7e 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -287,6 +287,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + curWin.winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &key); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); @@ -330,6 +331,7 @@ static SSDataBlock* buildCountResult(SOperatorInfo* pOperator) { int32_t doStreamCountEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator, bool isParent) { SStreamCountAggOperatorInfo* pInfo = pOperator->info; + return 0; if (!pInfo) { return 0; } From 27219f70afb35ab44259ddbcdce37f5d55067d4c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Sun, 25 Feb 2024 19:30:56 +0800 Subject: [PATCH 141/181] reset close window state --- source/libs/executor/src/streamcountwindowoperator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 3daefcdc7e..2d523a49fd 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -331,7 +331,6 @@ static SSDataBlock* buildCountResult(SOperatorInfo* pOperator) { int32_t doStreamCountEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator, bool isParent) { SStreamCountAggOperatorInfo* pInfo = pOperator->info; - return 0; if (!pInfo) { return 0; } From 9d560c494d655e723f41501a1b5320947e9f4332 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Sun, 25 Feb 2024 19:56:38 +0800 Subject: [PATCH 142/181] reset close window state --- source/libs/executor/src/streamcountwindowoperator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 2d523a49fd..70aee38264 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -290,6 +290,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl curWin.winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &key); + qDebug("===stream=== window close %" PRId64, key.win.skey); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); } @@ -345,7 +346,7 @@ int32_t doStreamCountEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper size_t keyLen = 0; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pInfo->streamAggSup.pResultRows, pIte, &iter)) != NULL) { - void* key = taosHashGetKey(pIte, &keyLen); + void* key = tSimpleHashGetKey(pIte, &keyLen); tlen += encodeSSessionKey(buf, key); tlen += encodeSResultWindowInfo(buf, pIte, pInfo->streamAggSup.resultRowSize); } From 176afbb74a8db188cd5f2e9d50b9b65eda259029 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Sun, 25 Feb 2024 20:07:04 +0800 Subject: [PATCH 143/181] reset close window state --- source/libs/executor/src/streamcountwindowoperator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 70aee38264..93bff984d3 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -290,7 +290,6 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl curWin.winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &key); - qDebug("===stream=== window close %" PRId64, key.win.skey); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); } From 97f56686c0447728084a8c71c199ab13bdf92e06 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 25 Feb 2024 22:40:51 +0800 Subject: [PATCH 144/181] feat: autoshell add news keyword --- source/libs/parser/src/parTranslater.c | 8 ++ tests/army/community/query/show.py | 124 +++++++++++++++++++++++++ tests/army/frame/autogen.py | 21 +++-- tools/shell/src/shellAuto.c | 47 +++++++++- 4 files changed, 188 insertions(+), 12 deletions(-) create mode 100644 tests/army/community/query/show.py diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index abe50a27da..3f00b4b8c8 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -373,6 +373,7 @@ static int32_t collectUseTable(const SName* pName, SHashObj* pTable) { return taosHashPut(pTable, fullName, strlen(fullName), pName, sizeof(SName)); } +#ifdef BUILD_NO_CALL static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) { #ifndef TD_ENTERPRISE return TSDB_CODE_PAR_TABLE_NOT_EXIST; @@ -396,6 +397,7 @@ static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCac } return code; } +#endif int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView) { int32_t code = TSDB_CODE_SUCCESS; @@ -774,9 +776,11 @@ static bool isAggFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)); } +#ifdef BUILD_NO_CALL static bool isSelectFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsSelectFunc(((SFunctionNode*)pNode)->funcId)); } +#endif static bool isWindowPseudoColumnFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsWindowPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); @@ -790,9 +794,11 @@ static bool isInterpPseudoColumnFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsInterpPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); } +#ifdef BUILD_NO_CALL static bool isTimelineFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsTimelineFunc(((SFunctionNode*)pNode)->funcId)); } +#endif static bool isImplicitTsFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsImplicitTsFunc(((SFunctionNode*)pNode)->funcId)); @@ -7738,9 +7744,11 @@ static int32_t addSubtableInfoToCreateStreamQuery(STranslateContext* pCxt, STabl return code; } +#ifdef BUILD_NO_CALL static bool isEventWindowQuery(SSelectStmt* pSelect) { return NULL != pSelect->pWindow && QUERY_NODE_EVENT_WINDOW == nodeType(pSelect->pWindow); } +#endif static bool hasJsonTypeProjection(SSelectStmt* pSelect) { SNode* pProj = NULL; diff --git a/tests/army/community/query/show.py b/tests/army/community/query/show.py new file mode 100644 index 0000000000..05f410b639 --- /dev/null +++ b/tests/army/community/query/show.py @@ -0,0 +1,124 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import random + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.autogen import * + + +class TDTestCase(TBase): + updatecfgDict = { + } + + def insertData(self): + tdLog.info(f"create table and insert data.") + self.stb = "stb" + self.db = "db" + self.childtable_count = 10 + self.insert_rows = 10000 + + self.autoGen = AutoGen(startTs = 1600000000000*1000*1000, batch=500, fillOne=True) + self.autoGen.create_db(self.db, 2, 3, "precision 'ns'") + self.autoGen.create_stable(stbname = self.stb, tag_cnt = 5, column_cnt = 20, binary_len = 10, nchar_len = 5) + self.autoGen.create_child(self.stb, "child", self.childtable_count) + self.autoGen.insert_data(self.insert_rows, True) + + tdLog.info("create view.") + tdSql.execute(f"use {self.db}") + sqls = [ + "create view viewc0c1 as select c0,c1 from stb ", + "create view viewc0c1c2 as select c0,c1,c2 from stb ", + "create view viewc0c3 as select c0,c3 from stb where c3=1", + "create view viewc0c4c5 as select c4,c5 from stb ", + "create view viewc0c6 as select c0,c1,c6 from stb ", + "create view viewc0c7 as select c0,c1 from stb ", + "create view viewc0c7c8 as select c0,c7,c8 from stb where c8>0", + "create view viewc0c3c1 as select c0,c3,c1 from stb ", + "create view viewc2c4 as select c2,c4 from stb ", + "create view viewc2c5 as select c2,c5 from stb ", + ] + tdSql.executes(sqls) + + def checkView(self): + tdLog.info(f"check view like.") + + # like + sql = f"show views like 'view%'" + tdSql.query(sql) + tdSql.checkRows(10) + + sql = f"show views like 'vie_c0c1c2'" + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0,0,"viewc0c1c2") + + sql = f"show views like '%c2c_'" + tdSql.query(sql) + tdSql.checkRows(2) + tdSql.checkData(0,0, "viewc2c4") + tdSql.checkData(1,0, "viewc2c5") + + sql = f"show views like '%' " + tdSql.query(sql) + tdSql.checkRows(10) + + # zero + sql = "show views like '_' " + tdSql.query(sql) + tdSql.checkRows(0) + sql = "show views like 'a%' " + tdSql.query(sql) + tdSql.checkRows(0) + + + def doQuery(self): + tdLog.info(f"do query.") + + # __group_key + sql = f"select count(*) from {self.stb} " + tdSql.query(sql) + # column index 1 value same with 2 + allRows = self.insert_rows * self.childtable_count + tdSql.checkFirstValue(sql, allRows) + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check view + self.checkView() + + # do action + self.doQuery() + + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py index d1f02e7865..cf21977c75 100644 --- a/tests/army/frame/autogen.py +++ b/tests/army/frame/autogen.py @@ -14,15 +14,18 @@ import time # Auto Gen class # class AutoGen: - def __init__(self, fillOne=False): - self.ts = 1600000000000 - self.batch_size = 100 + def __init__(self, startTs = 1600000000000, step = 1000, batch = 100, fillOne=False): + self.startTs = startTs + self.ts = startTs + self.step = step + self.batch_size = batch + self.fillOne = fillOne seed = time.time() % 10000 random.seed(seed) - self.fillOne = fillOne # set start ts def set_start_ts(self, ts): + self.startTs = ts self.ts = ts # set batch size @@ -111,9 +114,9 @@ class AutoGen: return ''.join(random.choice(letters) for i in range(count)) # create db - def create_db(self, dbname, vgroups = 2, replica = 1): + def create_db(self, dbname, vgroups = 2, replica = 1, others=""): self.dbname = dbname - tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica}') + tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica} {others}') # create table or stable def create_stable(self, stbname, tag_cnt, column_cnt, binary_len, nchar_len): @@ -167,12 +170,12 @@ class AutoGen: def insert_data(self, cnt, bContinue=False): if not bContinue: - self.ts = 1600000000000 + self.ts = self.startTs - currTs = 1600000000000 + currTs = self.startTs for i in range(self.child_cnt): name = f"{self.child_name}{i}" - currTs = self.insert_data_child(name, cnt, self.batch_size, 1) + currTs = self.insert_data_child(name, cnt, self.batch_size, self.step) self.ts = currTs tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}") diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index e9b9b9e944..0a2ce4f316 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -105,7 +105,8 @@ SWords shellCommands[] = { {"create or replace aggregate function as outputtype bufsize language ", 0, 0, NULL}, {"create user pass sysinfo 0;", 0, 0, NULL}, {"create user pass sysinfo 1;", 0, 0, NULL}, -#ifdef TD_ENTERPRISE +#ifdef TD_ENTERPRISE + {"create view as select", 0, 0, NULL}, {"compact database ", 0, 0, NULL}, #endif {"describe ", 0, 0, NULL}, @@ -162,13 +163,19 @@ SWords shellCommands[] = { {"show create database \\G;", 0, 0, NULL}, {"show create stable \\G;", 0, 0, NULL}, {"show create table \\G;", 0, 0, NULL}, +#ifdef TD_ENTERPRISE + {"show create view \\G;", 0, 0, NULL}, +#endif {"show connections;", 0, 0, NULL}, + {"show compact", 0, 0, NULL}, + {"show compacts;", 0, 0, NULL}, {"show cluster;", 0, 0, NULL}, {"show cluster alive;", 0, 0, NULL}, {"show databases;", 0, 0, NULL}, {"show dnodes;", 0, 0, NULL}, {"show dnode variables;", 0, 0, NULL}, {"show functions;", 0, 0, NULL}, + {"show licences;", 0, 0, NULL}, {"show mnodes;", 0, 0, NULL}, {"show queries;", 0, 0, NULL}, // 80 @@ -185,6 +192,7 @@ SWords shellCommands[] = { {"show table distributed ", 0, 0, NULL}, {"show tags from ", 0, 0, NULL}, {"show tags from ", 0, 0, NULL}, + {"show table tags from ", 0, 0, NULL}, {"show topics;", 0, 0, NULL}, {"show transactions;", 0, 0, NULL}, {"show users;", 0, 0, NULL}, @@ -194,6 +202,8 @@ SWords shellCommands[] = { {"show vgroups;", 0, 0, NULL}, {"show consumers;", 0, 0, NULL}, {"show grants;", 0, 0, NULL}, + {"show grants full;", 0, 0, NULL}, + {"show grants logs;", 0, 0, NULL}, #ifdef TD_ENTERPRISE {"split vgroup ", 0, 0, NULL}, #endif @@ -302,6 +312,20 @@ char* key_systable[] = { char* udf_language[] = {"\'Python\'", "\'C\'"}; +// global keys can tips on anywhere +char* global_keys[] = { + "tbname", + "now", + "_wstart", + "_wend", + "_wduration", + "_qstart", + "_qend", + "_qduration", + "_qtag", + "_isfilled" + }; + // // ------- global variant define --------- // @@ -341,8 +365,9 @@ bool waitAutoFill = false; #define WT_VAR_KEYSELECT 20 #define WT_VAR_SYSTABLE 21 #define WT_VAR_LANGUAGE 22 +#define WT_VAR_GLOBALKEYS 23 -#define WT_VAR_CNT 23 +#define WT_VAR_CNT 24 #define WT_TEXT 0xFF @@ -498,6 +523,7 @@ void showHelp() { show dnodes;\n\ show dnode variables;\n\ show functions;\n\ + show licences;\n\ show mnodes;\n\ show queries;\n\ show query ;\n\ @@ -513,6 +539,7 @@ void showHelp() { show table distributed ;\n\ show tags from \n\ show tags from \n\ + show table tags from \n\ show topics;\n\ show transactions;\n\ show users;\n\ @@ -522,6 +549,8 @@ void showHelp() { show vgroups;\n\ show consumers;\n\ show grants;\n\ + show grants full;\n\ + show grants logs;\n\ ----- T ----- \n\ trim database ;\n\ ----- U ----- \n\ @@ -534,8 +563,12 @@ void showHelp() { balance vgroup ;\n\ balance vgroup leader on \n\ compact database ; \n\ + crate view as select ...\n\ redistribute vgroup dnode ;\n\ - split vgroup ;"); + split vgroup ;\n\ + show compacts;\n\ + show compact \n\ + show create view ;"); #endif printf("\n\n"); @@ -699,6 +732,7 @@ bool shellAutoInit() { GenerateVarType(WT_VAR_KEYSELECT, key_select, sizeof(key_select) / sizeof(char*)); GenerateVarType(WT_VAR_SYSTABLE, key_systable, sizeof(key_systable) / sizeof(char*)); GenerateVarType(WT_VAR_LANGUAGE, udf_language, sizeof(udf_language) / sizeof(char*)); + GenerateVarType(WT_VAR_GLOBALKEYS, global_keys, sizeof(global_keys) / sizeof(char*)); return true; } @@ -1800,6 +1834,13 @@ bool matchEnd(TAOS* con, SShellCmd* cmd) { goto _return; } + // global keys + if (fillWithType(con, cmd, last, WT_VAR_GLOBALKEYS)) { + ret = true; + goto _return; + } + + _return: taosMemoryFree(ps); return ret; From 4ceb0856131e88851caaa4e9dc614436f15a1ad9 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 26 Feb 2024 01:03:08 +0800 Subject: [PATCH 145/181] enh: coverity of trow --- include/common/trow.h | 4 +- source/common/src/trow.c | 33 ++-- source/common/test/dataformatTest.cpp | 248 ++++++++++++++++---------- 3 files changed, 174 insertions(+), 111 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 8332c10ed2..92713e8e63 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -289,7 +289,7 @@ int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValT */ static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) { - pBuilder->rowType = TD_ROW_TP; // default STpRow + pBuilder->rowType = pBuilder->rowType; pBuilder->sver = sver; } int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen); @@ -331,7 +331,7 @@ void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow); bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal); -int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow); +int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t rowType); bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag); diff --git a/source/common/src/trow.c b/source/common/src/trow.c index b91562be7a..cf6d4752d9 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -231,7 +231,7 @@ bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCe return true; } -#ifdef BUILD_NO_CALL +// #ifdef BUILD_NO_CALL const uint8_t tdVTypeByte[2][3] = {{ // 2 bits TD_VTYPE_NORM_BYTE_II, @@ -439,13 +439,14 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { return true; } -int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { +int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t rowType) { STColumn *pTColumn; SColVal *pColVal; int32_t nColVal = taosArrayGetSize(pArray); int32_t varDataLen = 0; int32_t maxVarDataLen = 0; int32_t iColVal = 0; + int32_t nBound = 0; void *varBuf = NULL; bool isAlloc = false; @@ -458,6 +459,9 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { } else { pColVal = NULL; } + if (pColVal && !COL_VAL_IS_NONE(pColVal)) { + ++nBound; + } if (iColumn == 0) { ASSERT(pColVal && pColVal->cid == pTColumn->colId); @@ -512,10 +516,11 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { } } - SRowBuilder rb = {0}; + SRowBuilder rb = {.rowType = rowType}; tdSRowInit(&rb, pTSchema->version); - tdSRowSetInfo(&rb, pTSchema->numOfCols, pTSchema->numOfCols, pTSchema->flen); + tdSRowSetInfo(&rb, pTSchema->numOfCols, nBound, pTSchema->flen); tdSRowResetBuf(&rb, *ppRow); + int32_t iBound = 0; iColVal = 0; for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; ++iColumn) { @@ -529,21 +534,28 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { valType = TD_VTYPE_NONE; } else if (COL_VAL_IS_NULL(pColVal)) { valType = TD_VTYPE_NULL; + ++iBound; } else if (IS_VAR_DATA_TYPE(pTColumn->type)) { varDataSetLen(varBuf, pColVal->value.nData); if (pColVal->value.nData != 0) { memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData); } val = varBuf; + ++iBound; } else { val = (const void *)&pColVal->value.val; + ++iBound; } } else { // pColVal = NULL; valType = TD_VTYPE_NONE; } - tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal); + if (TD_IS_TP_ROW(rb.pBuf)) { + tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal); + } else { + tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, rb.offset, iBound); + } ++iColVal; } @@ -761,11 +773,11 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) { - if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) { + if (colIdx < 1) { terrno = TSDB_CODE_INVALID_PARA; + ASSERTS(0, "colIdx is %" PRIi64, colIdx); return terrno; } - offset -= sizeof(SKvRowIdx); --colIdx; #ifdef TD_SUPPORT_BITMAP @@ -779,6 +791,7 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); pColIdx->colId = colId; pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN + pBuilder->offset += sizeof(SKvRowIdx); if (valType == TD_VTYPE_NORM) { char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); if (IS_VAR_DATA_TYPE(colType)) { @@ -904,6 +917,7 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { TD_ROW_SET_LEN(pBuilder->pBuf, len); TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); + pBuilder->offset = 0; break; default: terrno = TSDB_CODE_INVALID_PARA; @@ -1056,7 +1070,7 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColVal) { STColumn *pTColumn = &pTSchema->columns[iCol]; SCellVal cv = {0}; - SValue value = {0}; + // SValue value = {0}; ASSERT((pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) || (iCol > 0)); @@ -1084,5 +1098,4 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV memcpy(&pColVal->value.val, cv.val, tDataTypes[pTColumn->type].bytes); } } -} -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index b1083e4590..81407c7ef0 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#if 0 #include #include @@ -59,7 +58,6 @@ STSchema *genSTSchema(int16_t nCols) { case 1: { pSchema[i].type = TSDB_DATA_TYPE_INT; pSchema[i].bytes = TYPE_BYTES[pSchema[i].type]; - ; } break; case 2: { pSchema[i].type = TSDB_DATA_TYPE_BIGINT; @@ -123,7 +121,8 @@ STSchema *genSTSchema(int16_t nCols) { return pResult; } -// ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, c9 bool +// ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, +// c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) { if (!(*pArray)) { *pArray = taosArrayInit(nCols, sizeof(SColVal)); @@ -142,59 +141,76 @@ static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) { taosArrayPush(*pArray, &colVal); continue; } - switch (i) { case 0: - sscanf(data[i], "%" PRIi64, &colVal.value.ts); + colVal.type = TSDB_DATA_TYPE_TIMESTAMP; + sscanf(data[i], "%" PRIi64, &colVal.value.val); break; case 1: - sscanf(data[i], "%" PRIi32, &colVal.value.i32); + colVal.type = TSDB_DATA_TYPE_INT; + sscanf(data[i], "%" PRIi32, (int32_t *)&colVal.value.val); break; case 2: - sscanf(data[i], "%" PRIi64, &colVal.value.i64); + colVal.type = TSDB_DATA_TYPE_BIGINT; + sscanf(data[i], "%" PRIi64, &colVal.value.val); break; case 3: - sscanf(data[i], "%f", &colVal.value.f); + colVal.type = TSDB_DATA_TYPE_FLOAT; + sscanf(data[i], "%f", (float *)&colVal.value.val); break; case 4: - sscanf(data[i], "%lf", &colVal.value.d); + colVal.type = TSDB_DATA_TYPE_DOUBLE; + sscanf(data[i], "%lf", (double *)&colVal.value.val); break; case 5: { + colVal.type = TSDB_DATA_TYPE_BINARY; int16_t dataLen = strlen(data[i]) + 1; colVal.value.nData = dataLen < 10 ? dataLen : 10; colVal.value.pData = (uint8_t *)data[i]; } break; case 6: { + colVal.type = TSDB_DATA_TYPE_NCHAR; int16_t dataLen = strlen(data[i]) + 1; colVal.value.nData = dataLen < 40 ? dataLen : 40; colVal.value.pData = (uint8_t *)data[i]; // just for test, not real nchar } break; - case 7: - case 9: { + case 7: { + colVal.type = TSDB_DATA_TYPE_TINYINT; int32_t d8; sscanf(data[i], "%" PRId32, &d8); - colVal.value.i8 = (int8_t)d8; - } break; + colVal.value.val = (int8_t)d8; + } case 8: { + colVal.type = TSDB_DATA_TYPE_SMALLINT; int32_t d16; sscanf(data[i], "%" PRId32, &d16); - colVal.value.i16 = (int16_t)d16; + colVal.value.val = (int16_t)d16; + } break; + case 9: { + colVal.type = TSDB_DATA_TYPE_BOOL; + int32_t d8; + sscanf(data[i], "%" PRId32, &d8); + colVal.value.val = (int8_t)d8; } break; case 10: { + colVal.type = TSDB_DATA_TYPE_UTINYINT; uint32_t u8; sscanf(data[i], "%" PRId32, &u8); - colVal.value.u8 = (uint8_t)u8; + colVal.value.val = (uint8_t)u8; } break; case 11: { + colVal.type = TSDB_DATA_TYPE_USMALLINT; uint32_t u16; sscanf(data[i], "%" PRId32, &u16); - colVal.value.u16 = (uint16_t)u16; + colVal.value.val = (uint16_t)u16; } break; case 12: { - sscanf(data[i], "%" PRIu32, &colVal.value.u32); + colVal.type = TSDB_DATA_TYPE_UINT; + sscanf(data[i], "%" PRIu32, (uint32_t *)&colVal.value.val); } break; case 13: { - sscanf(data[i], "%" PRIu64, &colVal.value.u64); + colVal.type = TSDB_DATA_TYPE_UBIGINT; + sscanf(data[i], "%" PRIu64, (uint64_t *)&colVal.value.val); } break; default: ASSERT(0); @@ -215,25 +231,25 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) { } switch (type) { case TSDB_DATA_TYPE_BOOL: - printf("%s ", cv->value.i8 == 0 ? "false" : "true"); + printf("%s ", cv->value.val == 0 ? "false" : "true"); break; case TSDB_DATA_TYPE_TINYINT: - printf("%" PRIi8 " ", cv->value.i8); + printf("%" PRIi8 " ", *(int8_t *)&cv->value.val); break; case TSDB_DATA_TYPE_SMALLINT: - printf("%" PRIi16 " ", cv->value.i16); + printf("%" PRIi16 " ", *(int16_t *)&cv->value.val); break; case TSDB_DATA_TYPE_INT: - printf("%" PRIi32 " ", cv->value.i32); + printf("%" PRIi32 " ", *(int32_t *)&cv->value.val); break; case TSDB_DATA_TYPE_BIGINT: - printf("%" PRIi64 " ", cv->value.i64); + printf("%" PRIi64 " ", cv->value.val); break; case TSDB_DATA_TYPE_FLOAT: - printf("%f ", cv->value.f); + printf("%f ", *(float *)&cv->value.val); break; case TSDB_DATA_TYPE_DOUBLE: - printf("%lf ", cv->value.d); + printf("%lf ", *(double *)&cv->value.val); break; case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_GEOMETRY: { @@ -242,7 +258,7 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) { printf("%s ", tv); } break; case TSDB_DATA_TYPE_TIMESTAMP: - printf("%" PRIi64 " ", cv->value.i64); + printf("%" PRIi64 " ", cv->value.val); break; case TSDB_DATA_TYPE_NCHAR: { char tv[15] = {0}; @@ -250,16 +266,16 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) { printf("%s ", tv); } break; case TSDB_DATA_TYPE_UTINYINT: - printf("%" PRIu8 " ", cv->value.u8); + printf("%" PRIu8 " ", *(uint8_t *)&cv->value.val); break; case TSDB_DATA_TYPE_USMALLINT: - printf("%" PRIu16 " ", cv->value.u16); + printf("%" PRIu16 " ", *(uint16_t *)&cv->value.val); break; case TSDB_DATA_TYPE_UINT: - printf("%" PRIu32 " ", cv->value.u32); + printf("%" PRIu32 " ", *(uint32_t *)&cv->value.val); break; case TSDB_DATA_TYPE_UBIGINT: - printf("%" PRIu64 " ", cv->value.u64); + printf("%" PRIu64 " ", *(uint64_t *)&cv->value.val); break; case TSDB_DATA_TYPE_JSON: printf("JSON "); @@ -286,11 +302,11 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) { return 0; } -void debugPrintTSRow(STSRow2 *row, STSchema *pTSchema, const char *tags, int32_t ln) { - // printf("%s:%d %s:v%d:%d ", tags, ln, (row->flags & 0xf0) ? "KV" : "TP", row->sver, row->nData); +void debugPrintTSRow(STSRow *row, STSchema *pTSchema, const char *tags, int32_t ln) { + printf("%s:%d %s:v%d:len-%u ", tags, ln, (row->type) ? "KV" : "TP", row->sver, row->len); for (int16_t i = 0; i < pTSchema->numOfCols; ++i) { SColVal cv = {0}; - tTSRowGet(row, pTSchema, i, &cv); + tTSRowGetVal(row, pTSchema, i, &cv); debugPrintSColVal(&cv, pTSchema->columns[i].type); } printf("\n"); @@ -315,36 +331,36 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) { case TSDB_DATA_TYPE_TINYINT: { int32_t d8; sscanf(rawVal, "%" PRId32, &d8); - EXPECT_EQ(cv->value.i8, (int8_t)d8); + EXPECT_EQ((int8_t)cv->value.val, (int8_t)d8); } break; case TSDB_DATA_TYPE_SMALLINT: { int32_t d16; sscanf(rawVal, "%" PRId32, &d16); - EXPECT_EQ(cv->value.i16, (int16_t)d16); + EXPECT_EQ((int16_t)cv->value.val, (int16_t)d16); } break; case TSDB_DATA_TYPE_INT: { - sscanf(rawVal, "%" PRId32, &rawSVal.i32); - EXPECT_EQ(cv->value.i32, rawSVal.i32); + sscanf(rawVal, "%" PRId32, (int32_t *)&rawSVal.val); + EXPECT_EQ((int32_t)cv->value.val, (int32_t)rawSVal.val); } break; case TSDB_DATA_TYPE_BIGINT: { - sscanf(rawVal, "%" PRIi64, &rawSVal.i64); - EXPECT_EQ(cv->value.i64, rawSVal.i64); + sscanf(rawVal, "%" PRIi64, &rawSVal.val); + EXPECT_EQ(cv->value.val, rawSVal.val); } break; case TSDB_DATA_TYPE_FLOAT: { - sscanf(rawVal, "%f", &rawSVal.f); - EXPECT_FLOAT_EQ(cv->value.f, rawSVal.f); + sscanf(rawVal, "%f", (float *)&rawSVal.val); + EXPECT_FLOAT_EQ((float)cv->value.val, (float)rawSVal.val); } break; case TSDB_DATA_TYPE_DOUBLE: { - sscanf(rawVal, "%lf", &rawSVal.d); - EXPECT_DOUBLE_EQ(cv->value.d, rawSVal.d); + sscanf(rawVal, "%lf", (double *)&rawSVal.val); + EXPECT_DOUBLE_EQ((double)cv->value.val, (double)rawSVal.val); } break; case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_GEOMETRY: { EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData); } break; case TSDB_DATA_TYPE_TIMESTAMP: { - sscanf(rawVal, "%" PRIi64, &rawSVal.ts); - EXPECT_DOUBLE_EQ(cv->value.ts, rawSVal.ts); + sscanf(rawVal, "%" PRIi64, &rawSVal.val); + EXPECT_DOUBLE_EQ(cv->value.val, rawSVal.val); } break; case TSDB_DATA_TYPE_NCHAR: { EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData); // informal nchar comparsion @@ -352,20 +368,20 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) { case TSDB_DATA_TYPE_UTINYINT: { uint32_t u8; sscanf(rawVal, "%" PRIu32, &u8); - EXPECT_EQ(cv->value.u8, (uint8_t)u8); + EXPECT_EQ((uint8_t)cv->value.val, (uint8_t)u8); } break; case TSDB_DATA_TYPE_USMALLINT: { uint32_t u16; sscanf(rawVal, "%" PRIu32, &u16); - EXPECT_EQ(cv->value.u16, (uint16_t)u16); + EXPECT_EQ((uint16_t)cv->value.val, (uint16_t)u16); } break; case TSDB_DATA_TYPE_UINT: { - sscanf(rawVal, "%" PRIu32, &rawSVal.u32); - EXPECT_EQ(cv->value.u32, rawSVal.u32); + sscanf(rawVal, "%" PRIu32, (uint32_t *)&rawSVal.val); + EXPECT_EQ((uint32_t)cv->value.val, (uint32_t)rawSVal.val); } break; case TSDB_DATA_TYPE_UBIGINT: { - sscanf(rawVal, "%" PRIu64, &rawSVal.u64); - EXPECT_EQ(cv->value.u64, rawSVal.u64); + sscanf(rawVal, "%" PRIu64, (uint64_t *)&rawSVal.val); + EXPECT_EQ((uint64_t)cv->value.val, (uint64_t)rawSVal.val); } break; case TSDB_DATA_TYPE_JSON: printf("JSON "); @@ -395,36 +411,63 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) { return 0; } -static void checkTSRow(const char **data, STSRow2 *row, STSchema *pTSchema) { +static void checkTSRow(const char **data, STSRow *row, STSchema *pTSchema) { for (int16_t i = 0; i < pTSchema->numOfCols; ++i) { SColVal cv = {0}; - tTSRowGet(row, pTSchema, i, &cv); + tTSRowGetVal(row, pTSchema, i, &cv); checkSColVal(data[i], &cv, pTSchema->columns[i].type); } + + STSRowIter rowIter = {.pSchema = pTSchema}; + tdSTSRowIterReset(&rowIter, row); + for (int32_t i = 0; i < pTSchema->numOfCols; ++i) { + STColumn *pCol = pTSchema->columns + i; + SColVal colVal = {0}; + SCellVal cv = {0}; + if (!tdSTSRowIterFetch(&rowIter, pCol->colId, pCol->type, &cv)) { + break; + } + if (tdValTypeIsNone(cv.valType)) { + colVal = COL_VAL_NONE(pCol->colId, pCol->type); + } else if (tdValTypeIsNull(cv.valType)) { + colVal = COL_VAL_NULL(pCol->colId, pCol->type); + } else { + colVal.cid = pCol->colId; + colVal.type = pCol->type; + colVal.flag = CV_FLAG_VALUE; + + if (IS_VAR_DATA_TYPE(pCol->type)) { + colVal.value.nData = varDataLen(cv.val); + colVal.value.pData = (uint8_t *)varDataVal(cv.val); + } else { + memcpy(&colVal.value.val, cv.val, tDataTypes[pCol->type].bytes); + } + } + checkSColVal(data[i], &colVal, pCol->type); + } } TEST(testCase, AllNormTest) { - int16_t nCols = 14; - STSRowBuilder rb = {0}; - STSRow2 *row = nullptr; - SArray *pArray = taosArrayInit(nCols, sizeof(SColVal)); + int16_t nCols = 14; + STSRow *row = nullptr; + SArray *pArray = taosArrayInit(nCols, sizeof(SColVal)); EXPECT_NE(pArray, nullptr); STSchema *pTSchema = genSTSchema(nCols); EXPECT_NE(pTSchema, nullptr); - // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, - // c9 bool - char *data[14] = {"1653694220000", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"}; + // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, + // c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned + char *data[14] = { + "1653694220000", "10", "20", "10.1", "10.1", "binary10", "nchar10", "10", "10", "10", "10", "20", "30", "40"}; - genTestData((const char **)&data, nCols, &pArray); + // genTestData((const char **)&data, nCols, &pArray); - tTSRowNew(&rb, pArray, pTSchema, &row); + // tdSTSRowNew(pArray, pTSchema, &row, TD_ROW_TP); + // debugPrintTSRow(row, pTSchema, __func__, __LINE__); + // tdSRowPrint(row, pTSchema, __func__); + // checkTSRow((const char **)&data, row, pTSchema); - debugPrintTSRow(row, pTSchema, __func__, __LINE__); - checkTSRow((const char **)&data, row, pTSchema); - - tsRowBuilderClear(&rb); taosArrayDestroy(pArray); taosMemoryFree(pTSchema); } @@ -433,7 +476,7 @@ TEST(testCase, AllNormTest) { TEST(testCase, NoneTest) { const static int nCols = 14; const static int nRows = 20; - STSRow2 *row = nullptr; + STSRow *row = nullptr; SArray *pArray = taosArrayInit(nCols, sizeof(SColVal)); EXPECT_NE(pArray, nullptr); @@ -442,42 +485,49 @@ TEST(testCase, NoneTest) { // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, // c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned - const char *data[nRows][nCols] = { - {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"}, - {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, - {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, - {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, - {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"}, - {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, - {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, - {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"}, - {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"}, - {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"}, - {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"}, - {"1653694220011", "-2147483648", "nu", "nu", "nu", "biy10", "nu", "nu", "32767", "no", "nu", "nu", "nu", "100"}, - {"1653694220012", "2147483647", "nu", "nu", "nu", "ary10", "nu", "nu", "-32768", "no", "nu", "nu", "nu", "100"}, - {"1653694220013", "no", "-9223372036854775818", "nu", "nu", "b1", "nu", "nu", "10", "no", "nu", "nu", "nu", "nu"}, - {"1653694220014", "no", "nu", "nu", "nu", "b0", "nu", "nu", "10", "no", "nu", "nu", "nu", "9223372036854775808"}, - {"1653694220015", "no", "nu", "nu", "nu", "binary30", "char4", "nu", "10", "no", "nu", "nu", "nu", - "18446744073709551615"}, - {"1653694220016", "2147483647", "nu", "nu", "nu", "bin50", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"}, - {"1653694220017", "2147483646", "0", "0", "0", "binary10", "0", "0", "0", "0", "255", "0", "0", "0"}, - {"1653694220018", "no", "-9223372036854775808", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", - "4294967295", "100"}, - {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu", - "no"}}; + // const int8_t rowType[nRows] = {TD_ROW_TP, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, + // TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP, TD_ROW_KV, TD_ROW_KV, + // TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP}; + const int8_t rowType[nRows] = {TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, + TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP, TD_ROW_KV, TD_ROW_KV, + TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP}; +const char *data[nRows][nCols] = { + // {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"}, + // {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, + // {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, + // {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, + {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"}, + {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, + {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, + {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"}, + {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"}, + {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"}, + {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"}, + {"1653694220011", "-2147483648", "nu", "nu", "nu", "biy10", "nu", "nu", "32767", "no", "nu", "nu", "nu", "100"}, + {"1653694220012", "2147483647", "nu", "nu", "nu", "ary10", "nu", "nu", "-32768", "no", "nu", "nu", "nu", "100"}, + {"1653694220013", "no", "-9223372036854775818", "nu", "nu", "b1", "nu", "nu", "10", "no", "nu", "nu", "nu", "nu"}, + {"1653694220014", "no", "nu", "nu", "nu", "b0", "nu", "nu", "10", "no", "nu", "nu", "nu", "9223372036854775808"}, + {"1653694220015", "no", "nu", "nu", "nu", "binary30", "char4", "nu", "10", "no", "nu", "nu", "nu", + "18446744073709551615"}, + {"1653694220016", "2147483647", "nu", "nu", "nu", "bin50", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"}, + {"1653694220017", "2147483646", "0", "0", "0", "binary10", "0", "0", "0", "0", "255", "0", "0", "0"}, + {"1653694220018", "no", "-9223372036854775808", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", + "4294967295", "100"}, + {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu", + "no"}}; - for (int r = 0; r < nRows; ++r) { - genTestData((const char **)&data[r], nCols, &pArray); - tTSRowNew(NULL, pArray, pTSchema, &row); - debugPrintTSRow(row, pTSchema, __func__, __LINE__); // debug print - checkTSRow((const char **)&data[r], row, pTSchema); // check - tTSRowFree(row); - taosArrayClear(pArray); +for (int r = 0; r < nRows; ++r) { + printf("%s:%d index:%d\n", __func__, __LINE__, r); + genTestData((const char **)&data[r], nCols, &pArray); + tdSTSRowNew(pArray, pTSchema, &row, rowType[r]); + debugPrintTSRow(row, pTSchema, __func__, __LINE__); // debug print + tdSRowPrint(row, pTSchema, __func__); + checkTSRow((const char **)&data[r], row, pTSchema); // check + taosMemoryFreeClear(row); + taosArrayClear(pArray); } taosArrayDestroy(pArray); taosMemoryFree(pTSchema); } -#endif #endif \ No newline at end of file From c54dfc36c20992d72ad4647f67de90bdb3c57066 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 26 Feb 2024 01:35:18 +0800 Subject: [PATCH 146/181] enh: coverity for dataformatTest --- source/common/src/trow.c | 21 +++++++++++++++----- source/common/test/dataformatTest.cpp | 28 +++++++++++++-------------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index cf6d4752d9..e2a8d5d5c3 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -444,6 +444,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r SColVal *pColVal; int32_t nColVal = taosArrayGetSize(pArray); int32_t varDataLen = 0; + int32_t nonVarDataLen = 0; int32_t maxVarDataLen = 0; int32_t iColVal = 0; int32_t nBound = 0; @@ -488,16 +489,26 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r } } } + } else { + if(pColVal && COL_VAL_IS_VALUE(pColVal)) { + nonVarDataLen += TYPE_BYTES[pTColumn->type]; + } } } ++iColVal; } - if (!(*ppRow)) { - *ppRow = (STSRow *)taosMemoryCalloc( - 1, sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1)); - isAlloc = true; + int32_t rowTotalLen = 0; + if (rowType == TD_ROW_TP) { + rowTotalLen = sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1); + } else { + rowTotalLen = sizeof(STSRow) + sizeof(col_id_t) + varDataLen + nonVarDataLen + (nBound - 1) * sizeof(SKvRowIdx) + + TD_BITMAP_BYTES(nBound - 1); + } + if (!(*ppRow)) { + *ppRow = (STSRow *)taosMemoryCalloc(1, rowTotalLen); + isAlloc = true; } if (!(*ppRow)) { @@ -554,7 +565,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r if (TD_IS_TP_ROW(rb.pBuf)) { tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal); } else { - tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, rb.offset, iBound); + tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, rb.offset, iBound - 1); } ++iColVal; diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index 81407c7ef0..80631fa1b3 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -461,13 +461,14 @@ TEST(testCase, AllNormTest) { char *data[14] = { "1653694220000", "10", "20", "10.1", "10.1", "binary10", "nchar10", "10", "10", "10", "10", "20", "30", "40"}; - // genTestData((const char **)&data, nCols, &pArray); + genTestData((const char **)&data, nCols, &pArray); - // tdSTSRowNew(pArray, pTSchema, &row, TD_ROW_TP); - // debugPrintTSRow(row, pTSchema, __func__, __LINE__); - // tdSRowPrint(row, pTSchema, __func__); - // checkTSRow((const char **)&data, row, pTSchema); + tdSTSRowNew(pArray, pTSchema, &row, TD_ROW_TP); + debugPrintTSRow(row, pTSchema, __func__, __LINE__); + tdSRowPrint(row, pTSchema, __func__); + checkTSRow((const char **)&data, row, pTSchema); + taosMemoryFreeClear(row); taosArrayDestroy(pArray); taosMemoryFree(pTSchema); } @@ -485,17 +486,15 @@ TEST(testCase, NoneTest) { // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, // c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned - // const int8_t rowType[nRows] = {TD_ROW_TP, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, - // TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP, TD_ROW_KV, TD_ROW_KV, - // TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP}; - const int8_t rowType[nRows] = {TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, - TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP, TD_ROW_KV, TD_ROW_KV, + const int8_t rowType[nRows] = {TD_ROW_TP, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, + TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP}; + const char *data[nRows][nCols] = { - // {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"}, - // {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, - // {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, - // {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, + {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"}, + {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, + {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, + {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"}, {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, @@ -517,7 +516,6 @@ const char *data[nRows][nCols] = { "no"}}; for (int r = 0; r < nRows; ++r) { - printf("%s:%d index:%d\n", __func__, __LINE__, r); genTestData((const char **)&data[r], nCols, &pArray); tdSTSRowNew(pArray, pTSchema, &row, rowType[r]); debugPrintTSRow(row, pTSchema, __func__, __LINE__); // debug print From 785d1672b849a12e0bc220a23810d7144ab63e0d Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 26 Feb 2024 02:05:00 +0800 Subject: [PATCH 147/181] test: add test for dataformat --- source/common/test/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/common/test/CMakeLists.txt b/source/common/test/CMakeLists.txt index eb79e79afa..51c733e796 100644 --- a/source/common/test/CMakeLists.txt +++ b/source/common/test/CMakeLists.txt @@ -35,6 +35,10 @@ target_include_directories( PUBLIC "${TD_SOURCE_DIR}/include/common" PUBLIC "${TD_SOURCE_DIR}/include/util" ) +add_test( + NAME dataformatTest + COMMAND dataformatTest +) # tmsg test # add_executable(tmsgTest "") From 9cd6da76d90f81a039aa6d56586eaff3680ecead Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 26 Feb 2024 02:10:30 +0800 Subject: [PATCH 148/181] other: trigger CI From f2840cd7ba43c89ced2bed1b66d9421160cec1d6 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 26 Feb 2024 08:48:47 +0800 Subject: [PATCH 149/181] fix: task reschedule issue --- source/libs/scheduler/inc/schInt.h | 2 +- source/libs/scheduler/src/scheduler.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 1c90e61ea3..1aa21e73a3 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -57,7 +57,7 @@ typedef enum { #define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ #define SCHEDULE_DEFAULT_MAX_NODE_NUM 20 -#define SCH_DEFAULT_TASK_TIMEOUT_USEC 5000000 +#define SCH_DEFAULT_TASK_TIMEOUT_USEC 30000000 #define SCH_MAX_TASK_TIMEOUT_USEC 300000000 #define SCH_DEFAULT_MAX_RETRY_NUM 6 #define SCH_MIN_AYSNC_EXEC_NUM 3 diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 841066a4c9..795c21c234 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -33,7 +33,7 @@ int32_t schedulerInit() { schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_MAX_JOB_NUM; schMgmt.cfg.maxNodeTableNum = tsQueryMaxConcurrentTables; schMgmt.cfg.schPolicy = SCHEDULE_DEFAULT_POLICY; - schMgmt.cfg.enableReSchedule = true; + schMgmt.cfg.enableReSchedule = false; qDebug("schedule init, policy: %d, maxNodeTableNum: %" PRId64", reSchedule:%d", schMgmt.cfg.schPolicy, schMgmt.cfg.maxNodeTableNum, schMgmt.cfg.enableReSchedule); From bc456e61832dfeff7d40248a2ef0d759ff241383 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 26 Feb 2024 08:51:06 +0800 Subject: [PATCH 150/181] fix: grammar of init on windows --- source/common/test/dataformatTest.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index 80631fa1b3..ab7496a474 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -418,7 +418,8 @@ static void checkTSRow(const char **data, STSRow *row, STSchema *pTSchema) { checkSColVal(data[i], &cv, pTSchema->columns[i].type); } - STSRowIter rowIter = {.pSchema = pTSchema}; + STSRowIter rowIter = {0}; + rowIter.pSchema = pTSchema; tdSTSRowIterReset(&rowIter, row); for (int32_t i = 0; i < pTSchema->numOfCols; ++i) { STColumn *pCol = pTSchema->columns + i; From 17ae867ec6215ac629bb7026484154fb6aef2a7c Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 26 Feb 2024 09:07:11 +0800 Subject: [PATCH 151/181] fix: grammar of init on windows --- source/common/src/trow.c | 12 ++++++++++-- source/common/test/dataformatTest.cpp | 9 +++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index e2a8d5d5c3..6a6758f834 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -137,6 +137,7 @@ int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pVa return TSDB_CODE_SUCCESS; } +#if 0 int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { terrno = TSDB_CODE_INVALID_PARA; @@ -177,16 +178,19 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal } return TSDB_CODE_SUCCESS; } +#endif int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) { switch (bitmapMode) { case 0: tdGetBitmapValTypeII(pBitmap, colIdx, pValType); break; +#if 0 case -1: case 1: tdGetBitmapValTypeI(pBitmap, colIdx, pValType); break; +#endif default: terrno = TSDB_CODE_INVALID_PARA; return TSDB_CODE_FAILED; @@ -628,7 +632,7 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell return true; } - +#if 0 int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { if (!pBitmap || colIdx < 0) { terrno = TSDB_CODE_INVALID_PARA; @@ -678,6 +682,7 @@ int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { } return TSDB_CODE_SUCCESS; } +#endif int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) { #ifdef TD_SUPPORT_BITMAP @@ -854,7 +859,7 @@ int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const vo return 0; } - +#if 0 int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, int32_t allNullLen, int32_t boundNullLen) { if ((boundNullLen > 0) && (allNullLen > 0) && (nBoundCols > 0)) { @@ -890,6 +895,7 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou #endif return TSDB_CODE_SUCCESS; } +#endif int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; @@ -1065,10 +1071,12 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int case 0: tdSetBitmapValTypeII(pBitmap, colIdx, valType); break; +#if 0 case -1: case 1: tdSetBitmapValTypeI(pBitmap, colIdx, valType); break; +#endif default: terrno = TSDB_CODE_INVALID_PARA; return TSDB_CODE_FAILED; diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index ab7496a474..8bc7d47ebf 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -428,13 +428,14 @@ static void checkTSRow(const char **data, STSRow *row, STSchema *pTSchema) { if (!tdSTSRowIterFetch(&rowIter, pCol->colId, pCol->type, &cv)) { break; } + + colVal.cid = pCol->colId; + colVal.type = pCol->type; if (tdValTypeIsNone(cv.valType)) { - colVal = COL_VAL_NONE(pCol->colId, pCol->type); + colVal.flag = CV_FLAG_NONE; } else if (tdValTypeIsNull(cv.valType)) { - colVal = COL_VAL_NULL(pCol->colId, pCol->type); + colVal.flag = CV_FLAG_NULL; } else { - colVal.cid = pCol->colId; - colVal.type = pCol->type; colVal.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pCol->type)) { From 4afd45992a1b7a68834db76aac23bbc7f7ff055e Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 26 Feb 2024 09:30:33 +0800 Subject: [PATCH 152/181] fix: sysinfo of dnode variables --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 4 ++-- tests/script/tsim/user/privilege_sysinfo.sim | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index fa59f56496..aa5e87e8b1 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -360,12 +360,12 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { terrno = TSDB_CODE_INVALID_MSG; return -1; } - +#if 0 if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) { terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } - +#endif if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) { terrno = TSDB_CODE_INVALID_MSG; return -1; diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index cf87f63b8c..f193e375d9 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -236,7 +236,7 @@ sql show create table d2.stb2; sql show create table d2.ctb2; sql show create table d2.ntb2; sql show local variables; -sql_error show dnode 1 variables; +sql show dnode 1 variables; sql show variables; @@ -269,7 +269,7 @@ sql select * from information_schema.ins_grants_logs sql select * from information_schema.ins_machines sql select * from information_schema.ins_vgroups sql select * from information_schema.ins_configs -sql_error select * from information_schema.ins_dnode_variables +sql select * from information_schema.ins_dnode_variables print =============== check performance_schema of sysinfo 1 sql use performance_schema; From 12ef8897bf28c15f2b1ccb860b32dfa20e4fbe32 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 26 Feb 2024 09:39:45 +0800 Subject: [PATCH 153/181] enh: remove obsolote code --- source/common/src/trow.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 6a6758f834..6e9278c630 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -1089,7 +1089,6 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColVal) { STColumn *pTColumn = &pTSchema->columns[iCol]; SCellVal cv = {0}; - // SValue value = {0}; ASSERT((pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) || (iCol > 0)); From 6e891b01450c7d3902eb7005f6c1e8d6ad332a23 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Feb 2024 10:22:34 +0800 Subject: [PATCH 154/181] fix(stream): add into buffer before start the create stream trans. --- source/dnode/mnode/impl/src/mndStream.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 56adfa9072..8da56d2d46 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -721,6 +721,11 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } + taosThreadMutexLock(&execInfo.lock); + mDebug("stream stream:%s tasks register into node list", createReq.name); + saveStreamTasksInfo(&streamObj, &execInfo); + taosThreadMutexUnlock(&execInfo.lock); + // execute creation if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -730,12 +735,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { mndTransDrop(pTrans); - taosThreadMutexLock(&execInfo.lock); - - mDebug("stream tasks register into node list"); - saveStreamTasksInfo(&streamObj, &execInfo); - taosThreadMutexUnlock(&execInfo.lock); - SName dbname = {0}; tNameFromString(&dbname, createReq.sourceDB, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); From ca6ed89f6f566fe16fcb0915d7f5eeddd9d23851 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 26 Feb 2024 10:22:48 +0800 Subject: [PATCH 155/181] fix: return error code instead of just out of memory --- source/libs/executor/src/projectoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index cc83ecd84e..691210e5ba 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -461,7 +461,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy _error: destroyIndefinitOperatorInfo(pInfo); taosMemoryFree(pOperator); - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + pTaskInfo->code = code; return NULL; } From 5c4f0ae053af2c84d2c0f51aedbd06ee0280ab97 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 26 Feb 2024 09:54:39 +0800 Subject: [PATCH 156/181] set max delay info --- source/libs/executor/src/streamcountwindowoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 93bff984d3..f9c7b51316 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -438,7 +438,7 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, &key); } while (1) { - SSessionKey tmpKey = {0}; + SSessionKey tmpKey = {.groupId = gpDatas[i], .win.skey = INT64_MIN, .win.ekey = INT64_MIN}; int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void **)&pPos, &size); if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) { pAggSup->stateStore.streamStateFreeCur(pCur); From 760dc57dcf49db908117980abe9f45ce72c268b1 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 26 Feb 2024 10:30:13 +0800 Subject: [PATCH 157/181] converage --- include/common/tmsg.h | 5 ++--- source/common/src/tmsg.c | 48 +++++++++++++++------------------------- 2 files changed, 20 insertions(+), 33 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index fe53217eac..d2e256f844 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1324,14 +1324,13 @@ typedef struct { } SDnodeListReq; int32_t tSerializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq); -int32_t tDeserializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq); typedef struct { int32_t useless; // useless } SServerVerReq; int32_t tSerializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq); -int32_t tDeserializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq); +// int32_t tDeserializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq); typedef struct { char ver[TSDB_VERSION_LEN]; @@ -1907,7 +1906,7 @@ typedef struct { } SShowVariablesReq; int32_t tSerializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq); -int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq); +// int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq); typedef struct { char name[TSDB_CONFIG_OPTION_LEN + 1]; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 85f5d462c7..6addd36958 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3280,18 +3280,6 @@ int32_t tSerializeSDnodeListReq(void *buf, int32_t bufLen, SDnodeListReq *pReq) return tlen; } -int32_t tDeserializeSDnodeListReq(void *buf, int32_t bufLen, SDnodeListReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->rowNum) < 0) return -1; - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; -} - int32_t tSerializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3305,17 +3293,17 @@ int32_t tSerializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) return tlen; } -int32_t tDeserializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSServerVerRsp(void *buf, int32_t bufLen, SServerVerRsp *pRsp) { SEncoder encoder = {0}; @@ -4233,17 +4221,17 @@ int32_t tSerializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq return tlen; } -int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) { if (tEncodeCStr(pEncoder, pInfo->name) < 0) return -1; From 323938df369be7c61c982aaaaf74e8710f339a15 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 26 Feb 2024 12:29:33 +0800 Subject: [PATCH 158/181] fix(meta/table): remove cache table dropping --- source/dnode/vnode/src/meta/metaTable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index a2c7a04ab3..4a978c8f41 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -1008,11 +1008,11 @@ void metaDropTables(SMeta *pMeta, SArray *tbUids) { } tSimpleHashPut(suidHash, &suid, sizeof(tb_uid_t), &nCtbDropped, sizeof(int64_t)); } - + /* if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { tsdbCacheDropTable(pMeta->pVnode->pTsdb, uid, suid, NULL); } - + */ metaDebug("batch drop table:%" PRId64, uid); } metaULock(pMeta); From baafcba7ca0115fa1df5430b8773ea52fe9972e4 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 26 Feb 2024 16:27:32 +0800 Subject: [PATCH 159/181] fix: incShapshot.py change argument and s3 increase data file size --- tests/army/enterprise/s3/s3_basic.json | 8 ++++---- tests/army/enterprise/s3/s3_basic.py | 2 +- tests/parallel_test/cases.task | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json index d7544a897c..d6847826db 100644 --- a/tests/army/enterprise/s3/s3_basic.json +++ b/tests/army/enterprise/s3/s3_basic.json @@ -18,19 +18,19 @@ "drop": "yes", "vgroups": 2, "replica": 1, - "duration":"1d", - "keep": "3d,6d,30d" + "duration":"10d", + "keep": "30d,60d,100d" }, "super_tables": [ { "name": "stb", "child_table_exists": "no", - "childtable_count": 4, + "childtable_count": 6, "insert_rows": 1000000, "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"now-13d", + "start_timestamp":"now-90d", "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 45519d925f..7071c678a4 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -58,7 +58,7 @@ class TDTestCase(TBase): tdSql.execute(f"use {self.db}") # come from s3_basic.json - self.childtable_count = 4 + self.childtable_count = 6 self.insert_rows = 1000000 self.timestamp_step = 1000 diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 12f1f62f63..abaeb44047 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -15,7 +15,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f community/query/function/test_func_elapsed.py ,,y,army,./pytest.sh python3 ./test.py -f community/query/fill/fill_desc.py -N 3 -L 3 -D 2 -,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py From 1043f46a75d2a6781c7848b07ad9f42d640cdd6a Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 26 Feb 2024 16:28:10 +0800 Subject: [PATCH 160/181] enh: converage --- include/common/tmsg.h | 16 +- source/common/src/tmsg.c | 318 +++++++++++++++++++-------------------- 2 files changed, 167 insertions(+), 167 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d2e256f844..1476750737 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -583,7 +583,7 @@ typedef struct { int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); -void tFreeSSubmitBlkRsp(void* param); +// void tFreeSSubmitBlkRsp(void* param); void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_SMA_ON ((int8_t)0x1) @@ -2221,8 +2221,8 @@ typedef struct { char ckey[TSDB_PASSWORD_LEN]; } SAuthReq, SAuthRsp; -int32_t tSerializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); -int32_t tDeserializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); +// int32_t tSerializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); +// int32_t tDeserializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); typedef struct { int32_t statusCode; @@ -2755,8 +2755,8 @@ typedef struct { int tEncodeSVCreateTbBatchRsp(SEncoder* pCoder, const SVCreateTbBatchRsp* pRsp); int tDecodeSVCreateTbBatchRsp(SDecoder* pCoder, SVCreateTbBatchRsp* pRsp); -int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); -int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); +// int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); +// int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); // TDMT_VND_DROP_TABLE ================= typedef struct { @@ -3211,8 +3211,8 @@ typedef struct { int32_t hasCheckPoint; } SMVStreamGatherInfoReq; -int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq); -int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq); +// int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq); +// int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq); typedef struct { int64_t leftForVer; @@ -3489,7 +3489,7 @@ void* tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq); int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq); int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq); int32_t tEncodeSVDropTSmaReq(SEncoder* pCoder, const SVDropTSmaReq* pReq); -int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq); +// int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq); typedef struct { int32_t number; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 6addd36958..574ad917fe 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -5877,38 +5877,38 @@ int32_t tDeserializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq return 0; } -int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; - if (tEncodeI8(&encoder, pReq->spi) < 0) return -1; - if (tEncodeI8(&encoder, pReq->encrypt) < 0) return -1; - if (tEncodeBinary(&encoder, pReq->secret, TSDB_PASSWORD_LEN) < 0) return -1; - if (tEncodeBinary(&encoder, pReq->ckey, TSDB_PASSWORD_LEN) < 0) return -1; - tEndEncode(&encoder); +// if (tStartEncode(&encoder) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->spi) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->encrypt) < 0) return -1; +// if (tEncodeBinary(&encoder, pReq->secret, TSDB_PASSWORD_LEN) < 0) return -1; +// if (tEncodeBinary(&encoder, pReq->ckey, TSDB_PASSWORD_LEN) < 0) return -1; +// tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } -int32_t tDeserializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->spi) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->encrypt) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->secret) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->ckey) < 0) return -1; - tEndDecode(&decoder); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; +// if (tDecodeI8(&decoder, &pReq->spi) < 0) return -1; +// if (tDecodeI8(&decoder, &pReq->encrypt) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->secret) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->ckey) < 0) return -1; +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSServerStatusRsp(void *buf, int32_t bufLen, SServerStatusRsp *pRsp) { SEncoder encoder = {0}; @@ -6946,30 +6946,30 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp * void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp) { taosArrayDestroy(pRsp->taskStatus); } -int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { - // SEncoder encoder = {0}; - // tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { +// // SEncoder encoder = {0}; +// // tEncoderInit(&encoder, buf, bufLen); - // if (tStartEncode(&encoder) < 0) return -1; - // if (pRsp->rspList) { - // int32_t num = taosArrayGetSize(pRsp->rspList); - // if (tEncodeI32(&encoder, num) < 0) return -1; - // for (int32_t i = 0; i < num; ++i) { - // SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i); - // if (tEncodeI32(&encoder, rsp->code) < 0) return -1; - // } - // } else { - // if (tEncodeI32(&encoder, 0) < 0) return -1; - // } - // tEndEncode(&encoder); +// // if (tStartEncode(&encoder) < 0) return -1; +// // if (pRsp->rspList) { +// // int32_t num = taosArrayGetSize(pRsp->rspList); +// // if (tEncodeI32(&encoder, num) < 0) return -1; +// // for (int32_t i = 0; i < num; ++i) { +// // SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i); +// // if (tEncodeI32(&encoder, rsp->code) < 0) return -1; +// // } +// // } else { +// // if (tEncodeI32(&encoder, 0) < 0) return -1; +// // } +// // tEndEncode(&encoder); - // int32_t tlen = encoder.pos; - // tEncoderClear(&encoder); - // reture tlen; - return 0; -} +// // int32_t tlen = encoder.pos; +// // tEncoderClear(&encoder); +// // reture tlen; +// return 0; +// } -int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { +//int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { // SDecoder decoder = {0}; // int32_t num = 0; // tDecoderInit(&decoder, buf, bufLen); @@ -6990,8 +6990,8 @@ int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatc // tEndDecode(&decoder); // tDecoderClear(&decoder); - return 0; -} +// return 0; +//} int tEncodeSVCreateTbBatchRsp(SEncoder *pCoder, const SVCreateTbBatchRsp *pRsp) { int32_t nRsps = taosArrayGetSize(pRsp->pArray); @@ -7126,15 +7126,15 @@ int32_t tEncodeSVDropTSmaReq(SEncoder *pCoder, const SVDropTSmaReq *pReq) { return 0; } -int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { - if (tStartDecode(pCoder) < 0) return -1; +// int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { +// if (tStartDecode(pCoder) < 0) return -1; - if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; - if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1; +// if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; +// if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1; - tEndDecode(pCoder); - return 0; -} +// tEndDecode(pCoder); +// return 0; +// } int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { int32_t headLen = sizeof(SMsgHead); @@ -7415,34 +7415,34 @@ int32_t tDeserializeSMDropStreamReq(void *buf, int32_t bufLen, SMDropStreamReq * void tFreeMDropStreamReq(SMDropStreamReq *pReq) { FREESQL(); } -int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; - if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; +// if (tStartEncode(&encoder) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; - tEndEncode(&encoder); +// tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } -int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStreamReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStreamReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; +// if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; - tEndDecode(&decoder); +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { if (NULL == pReq) { @@ -7842,37 +7842,37 @@ int32_t tDecodeSVDropStbReq(SDecoder *pCoder, SVDropStbReq *pReq) { return 0; } -static int32_t tEncodeSVSubmitBlk(SEncoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) { - if (tStartEncode(pCoder) < 0) return -1; +// static int32_t tEncodeSVSubmitBlk(SEncoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) { +// if (tStartEncode(pCoder) < 0) return -1; - if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1; - if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1; - if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1; - if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1; +// if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1; +// if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1; +// if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1; +// if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1; - if (flags & TD_AUTO_CREATE_TABLE) { - if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; - } +// if (flags & TD_AUTO_CREATE_TABLE) { +// if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; +// } - tEndEncode(pCoder); - return 0; -} +// tEndEncode(pCoder); +// return 0; +// } -static int32_t tDecodeSVSubmitBlk(SDecoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) { - if (tStartDecode(pCoder) < 0) return -1; +// static int32_t tDecodeSVSubmitBlk(SDecoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) { +// if (tStartDecode(pCoder) < 0) return -1; - if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1; - if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1; - if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1; - if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1; +// if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1; +// if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1; +// if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1; +// if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1; - if (flags & TD_AUTO_CREATE_TABLE) { - if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; - } +// if (flags & TD_AUTO_CREATE_TABLE) { +// if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; +// } - tEndDecode(pCoder); - return 0; -} +// tEndDecode(pCoder); +// return 0; +// } static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBlock) { if (tStartEncode(pEncoder) < 0) return -1; @@ -7955,19 +7955,19 @@ int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) { return 0; } -void tFreeSSubmitBlkRsp(void *param) { - if (NULL == param) { - return; - } +// void tFreeSSubmitBlkRsp(void *param) { +// if (NULL == param) { +// return; +// } - SSubmitBlkRsp *pRsp = (SSubmitBlkRsp *)param; +// SSubmitBlkRsp *pRsp = (SSubmitBlkRsp *)param; - taosMemoryFree(pRsp->tblFName); - if (pRsp->pMeta) { - taosMemoryFree(pRsp->pMeta->pSchemas); - taosMemoryFree(pRsp->pMeta); - } -} +// taosMemoryFree(pRsp->tblFName); +// if (pRsp->pMeta) { +// taosMemoryFree(pRsp->pMeta->pSchemas); +// taosMemoryFree(pRsp->pMeta); +// } +// } void tFreeSSubmitRsp(SSubmitRsp *pRsp) { if (NULL == pRsp) return; @@ -8141,23 +8141,23 @@ int32_t tDecodeSVAlterTbRsp(SDecoder *pDecoder, SVAlterTbRsp *pRsp) { return 0; } -int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) { - int32_t meta = 0; - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) { +// int32_t meta = 0; +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1; - if (tDecodeI32(&decoder, &meta) < 0) return -1; - if (meta) { - pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - if (NULL == pRsp->pMeta) return -1; - if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; - } - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1; +// if (tDecodeI32(&decoder, &meta) < 0) return -1; +// if (meta) { +// pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); +// if (NULL == pRsp->pMeta) return -1; +// if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; +// } +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } int32_t tEncodeSMAlterStbRsp(SEncoder *pEncoder, const SMAlterStbRsp *pRsp) { if (tStartEncode(pEncoder) < 0) return -1; @@ -8182,22 +8182,22 @@ int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp) { return 0; } -int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) { - int32_t meta = 0; - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) { +// int32_t meta = 0; +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &meta) < 0) return -1; - if (meta) { - pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - if (NULL == pRsp->pMeta) return -1; - if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; - } - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &meta) < 0) return -1; +// if (meta) { +// pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); +// if (NULL == pRsp->pMeta) return -1; +// if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; +// } +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) { if (NULL == pRsp) { @@ -8233,22 +8233,22 @@ int32_t tDecodeSMCreateStbRsp(SDecoder *pDecoder, SMCreateStbRsp *pRsp) { return 0; } -int32_t tDeserializeSMCreateStbRsp(void *buf, int32_t bufLen, SMCreateStbRsp *pRsp) { - int32_t meta = 0; - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSMCreateStbRsp(void *buf, int32_t bufLen, SMCreateStbRsp *pRsp) { +// int32_t meta = 0; +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &meta) < 0) return -1; - if (meta) { - pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - if (NULL == pRsp->pMeta) return -1; - if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; - } - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &meta) < 0) return -1; +// if (meta) { +// pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); +// if (NULL == pRsp->pMeta) return -1; +// if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; +// } +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } void tFreeSMCreateStbRsp(SMCreateStbRsp *pRsp) { if (NULL == pRsp) { From 831f3e0c3d3c503afa5bb126664d513d018f322b Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 26 Feb 2024 16:56:29 +0800 Subject: [PATCH 161/181] enh: remove useless code --- include/common/tmsg.h | 14 ++--- source/common/src/tmsg.c | 130 +++++++++++++++++++-------------------- 2 files changed, 72 insertions(+), 72 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1476750737..c24709bdea 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1628,14 +1628,14 @@ typedef struct { } SMTimerReq; int32_t tSerializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq); -int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq); +// int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq); typedef struct { int64_t tick; } SMStreamTickReq; int32_t tSerializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq); -int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq); +// int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq); typedef struct { int32_t id; @@ -1936,7 +1936,7 @@ typedef struct { } SShowReq; int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); -int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); +// int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); void tFreeSShowReq(SShowReq* pReq); typedef struct { @@ -1944,9 +1944,9 @@ typedef struct { STableMetaRsp tableMeta; } SShowRsp, SVShowTablesRsp; -int32_t tSerializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); -int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); -void tFreeSShowRsp(SShowRsp* pRsp); +// int32_t tSerializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); +// int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); +// void tFreeSShowRsp(SShowRsp* pRsp); typedef struct { char db[TSDB_DB_FNAME_LEN]; @@ -2204,7 +2204,7 @@ typedef struct { } SForceBecomeFollowerReq; int32_t tSerializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq); -int32_t tDeserializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq); +// int32_t tDeserializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq); typedef struct { int32_t vgId; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 574ad917fe..f92a078088 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4313,24 +4313,24 @@ int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { return tlen; } -int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->type) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1; - if (pReq->payloadLen > 0) { - pReq->payload = taosMemoryMalloc(pReq->payloadLen); - if (pReq->payload == NULL) return -1; - if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1; - } +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->type) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1; +// if (pReq->payloadLen > 0) { +// pReq->payload = taosMemoryMalloc(pReq->payloadLen); +// if (pReq->payload == NULL) return -1; +// if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1; +// } - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } void tFreeSShowReq(SShowReq *pReq) { taosMemoryFreeClear(pReq->payload); } @@ -4583,34 +4583,34 @@ void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) { taosArrayDestroy(pRsp->pIndexRsp); } -int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI64(&encoder, pRsp->showId) < 0) return -1; - if (tEncodeSTableMetaRsp(&encoder, &pRsp->tableMeta) < 0) return -1; - tEndEncode(&encoder); +// if (tStartEncode(&encoder) < 0) return -1; +// if (tEncodeI64(&encoder, pRsp->showId) < 0) return -1; +// if (tEncodeSTableMetaRsp(&encoder, &pRsp->tableMeta) < 0) return -1; +// tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } -int32_t tDeserializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI64(&decoder, &pRsp->showId) < 0) return -1; - if (tDecodeSTableMetaRsp(&decoder, &pRsp->tableMeta) < 0) return -1; +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI64(&decoder, &pRsp->showId) < 0) return -1; +// if (tDecodeSTableMetaRsp(&decoder, &pRsp->tableMeta) < 0) return -1; - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } -void tFreeSShowRsp(SShowRsp *pRsp) { tFreeSTableMetaRsp(&pRsp->tableMeta); } +// void tFreeSShowRsp(SShowRsp *pRsp) { tFreeSTableMetaRsp(&pRsp->tableMeta); } int32_t tSerializeSTableInfoReq(void *buf, int32_t bufLen, STableInfoReq *pReq) { int32_t headLen = sizeof(SMsgHead); @@ -4918,17 +4918,17 @@ int32_t tSerializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) { return tlen; } -int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1; - tEndDecode(&decoder); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1; +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) { SEncoder encoder = {0}; @@ -4943,17 +4943,17 @@ int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pR return tlen; } -int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1; - tEndDecode(&decoder); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1; +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } int32_t tEncodeSReplica(SEncoder *pEncoder, SReplica *pReplica) { if (tEncodeI32(pEncoder, pReplica->id) < 0) return -1; @@ -5818,17 +5818,17 @@ int32_t tSerializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecom return tlen; } -int32_t tDeserializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecomeFollowerReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecomeFollowerReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; - tEndDecode(&decoder); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq *pReq) { SEncoder encoder = {0}; From a00daa0287bcf681f36f3c67282abe10523c2c9d Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 26 Feb 2024 11:37:10 +0000 Subject: [PATCH 162/181] column name --- source/libs/monitor/src/monFramework.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c index d58cee7b9a..70b9dd9f0e 100644 --- a/source/libs/monitor/src/monFramework.c +++ b/source/libs/monitor/src/monFramework.c @@ -65,8 +65,8 @@ extern char* tsMonFwUri; #define DISK_ENGINE DNODE_TABLE":disk_engine" #define DISK_USED DNODE_TABLE":disk_used" #define DISK_TOTAL DNODE_TABLE":disk_total" -#define NET_IN DNODE_TABLE":net_in" -#define NET_OUT DNODE_TABLE":net_out" +#define NET_IN DNODE_TABLE":system_net_in" +#define NET_OUT DNODE_TABLE":system_net_out" #define IO_READ DNODE_TABLE":io_read" #define IO_WRITE DNODE_TABLE":io_write" #define IO_READ_DISK DNODE_TABLE":io_read_disk" From 49fe44ab7439aee571c416a172dd835e2927f903 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 26 Feb 2024 11:46:39 +0000 Subject: [PATCH 163/181] column name --- source/libs/monitor/src/monFramework.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c index 70b9dd9f0e..488b4d1d58 100644 --- a/source/libs/monitor/src/monFramework.c +++ b/source/libs/monitor/src/monFramework.c @@ -60,7 +60,7 @@ extern char* tsMonFwUri; #define CPU_SYSTEM DNODE_TABLE":cpu_system" #define CPU_CORE DNODE_TABLE":cpu_cores" #define MEM_ENGINE DNODE_TABLE":mem_engine" -#define MEM_SYSTEM DNODE_TABLE":mem_system" +#define MEM_SYSTEM DNODE_TABLE":mem_free" #define MEM_TOTAL DNODE_TABLE":mem_total" #define DISK_ENGINE DNODE_TABLE":disk_engine" #define DISK_USED DNODE_TABLE":disk_used" From d814eec5e5487e983970ab7a356174e4afb97997 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 26 Feb 2024 19:47:45 +0800 Subject: [PATCH 164/181] fix: diff step is 1000 --- tests/army/community/cluster/incSnapshot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py index 6bcf547136..85f030eb03 100644 --- a/tests/army/community/cluster/incSnapshot.py +++ b/tests/army/community/cluster/incSnapshot.py @@ -67,7 +67,6 @@ class TDTestCase(TBase): dirs = glob.glob(dnodesRootDir) for dir in dirs: if os.path.isdir(dir): - tdLog.debug("delete dir: %s " % (dnodesRootDir)) self.remove_directory(os.path.join(dir, "wal")) sc.dnodeStart(1) @@ -88,7 +87,7 @@ class TDTestCase(TBase): if bFinish: break - self.timestamp_step = 1 + self.timestamp_step = 1000 self.insert_rows = 6000 self.checkInsertCorrect() self.checkAggCorrect() From bbe6fe02d24b4ff391ebceeba5832e2a2fc8711d Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 26 Feb 2024 20:15:27 +0800 Subject: [PATCH 165/181] enh: use optimistic locking to protect snapshot reader open against vnode commit --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 ++-- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 3 ++- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 17 +++++++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 6aff1c2930..81a136a0e3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -448,8 +448,8 @@ _exit: taosMemoryFree(reader[0]); reader[0] = NULL; } else { - tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), - sver, ever, type); + tsdbInfo("vgId:%d, tsdb snapshot incremental reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d", + TD_VID(tsdb->pVnode), sver, ever, type); } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index b7c22aa0e9..ae840eb6d2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -69,7 +69,8 @@ _exit: taosMemoryFree(reader[0]); reader[0] = NULL; } else { - tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), ever, type); + tsdbInfo("vgId:%d, tsdb snapshot raw reader opened. sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), ever, + type); } return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index ed1dcc64c9..b4c8a26f5c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -172,6 +172,23 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader goto _err; } + // open tsdb snapshot raw reader + if (!pReader->tsdbRAWDone) { + ASSERT(pReader->sver == 0); + code = tsdbSnapRAWReaderOpen(pVnode->pTsdb, ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader); + if (code) goto _err; + } + + // check snapshot ever + SSnapshot snapshot = {0}; + vnodeGetSnapshot(pVnode, &snapshot); + if (ever != snapshot.lastApplyIndex) { + vError("vgId:%d, abort reader open due to vnode snapshot changed. ever:%" PRId64 ", commit ver:%" PRId64, + TD_VID(pVnode), ever, snapshot.lastApplyIndex); + code = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _err; + } + vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever); *ppReader = pReader; return code; From 8cf6fdce52b8e7c5ab507048aa59560f7eceaa7d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 26 Feb 2024 20:26:47 +0800 Subject: [PATCH 166/181] fix: ic set to 1 --- tests/army/community/query/show.py | 25 +++++++++++++++++++++++++ tests/army/enterprise/s3/s3_basic.json | 4 ++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/army/community/query/show.py b/tests/army/community/query/show.py index 05f410b639..d1bc1d9934 100644 --- a/tests/army/community/query/show.py +++ b/tests/army/community/query/show.py @@ -102,6 +102,31 @@ class TDTestCase(TBase): allRows = self.insert_rows * self.childtable_count tdSql.checkFirstValue(sql, allRows) + def checkShow(self): + # not support + sql = "show accounts;" + tdSql.error(sql) + + # check result + sql = "SHOW CLUSTER;" + tdSql.query(sql) + tdSql.checkRows(1) + sql = "SHOW COMPACTS;" + tdSql.query(sql) + tdSql.checkRows(0) + sql = "SHOW COMPACT 1;" + tdSql.query(sql) + tdSql.checkRows(0) + + # run to check crash + sqls = [ + "show scores;", + "SHOW CLUSTER VARIABLES", + "SHOW BNODES;", + ] + tdSql.executes(sqls) + + # run def run(self): tdLog.debug(f"start to excute {__file__}") diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json index d6847826db..f5bd7564f5 100644 --- a/tests/army/enterprise/s3/s3_basic.json +++ b/tests/army/enterprise/s3/s3_basic.json @@ -35,9 +35,9 @@ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, { "type": "double", "name": "dc"}, - { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "tinyint", "name": "ti"}, { "type": "smallint", "name": "si" }, - { "type": "int", "name": "ic" }, + { "type": "int", "name": "ic" , "max" : "1", "min" : "1" }, { "type": "bigint", "name": "bi" }, { "type": "utinyint", "name": "uti"}, { "type": "usmallint", "name": "usi"}, From a279fcac636883c61127ebe2b6b36aa242a54c66 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 26 Feb 2024 20:36:58 +0800 Subject: [PATCH 167/181] fix: ic set to 1 --- tests/army/enterprise/s3/s3_basic.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json index f5bd7564f5..54b8808f2f 100644 --- a/tests/army/enterprise/s3/s3_basic.json +++ b/tests/army/enterprise/s3/s3_basic.json @@ -37,7 +37,7 @@ { "type": "double", "name": "dc"}, { "type": "tinyint", "name": "ti"}, { "type": "smallint", "name": "si" }, - { "type": "int", "name": "ic" , "max" : "1", "min" : "1" }, + { "type": "int", "name": "ic" ,"max": 1,"min": 1}, { "type": "bigint", "name": "bi" }, { "type": "utinyint", "name": "uti"}, { "type": "usmallint", "name": "usi"}, From 9a0c402c116121ecbaea094c490e82f43a48769d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 26 Feb 2024 20:43:01 +0800 Subject: [PATCH 168/181] fix: add show.py --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index abaeb44047..bdccf33c32 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -19,6 +19,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py +,,n,army,python3 ./test.py -f community/query/show.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1 # From 81a1eac87a9e17a240631f34720518f982bb6f32 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Feb 2024 22:49:38 +0800 Subject: [PATCH 169/181] fix(stream): add some logs. --- source/dnode/mnode/impl/src/mndScheduler.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index ef9a7205e1..7831048f2d 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -300,14 +300,15 @@ static int32_t doAddShuffleSinkTask(SMnode* pMnode, SStreamObj* pStream, SEpSet* } static int64_t getVgroupLastVer(const SArray* pList, int32_t vgId) { - for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) { + int32_t size = (int32_t) taosArrayGetSize(pList); + for (int32_t i = 0; i < size; ++i) { SVgroupVer* pVer = taosArrayGet(pList, i); if (pVer->vgId == vgId) { return pVer->ver; } } - mError("failed to find the vgId:%d for extract last version", vgId); + mError("failed to find the vgId:%d for extract last version, total existed vgs:%d", vgId, size); return -1; } @@ -472,6 +473,9 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream int code = doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, false, useTriggerParam); if (code != 0) { + mError("create stream task, code:%s", tstrerror(code)); + + // todo drop the added source tasks. sdbRelease(pSdb, pVgroup); return code; } From fbc5699215b0d690ac2cecd85f97c92548efc01b Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 27 Feb 2024 09:13:56 +0800 Subject: [PATCH 170/181] fix: add show cluster machines --- tests/army/community/query/show.py | 3 +++ tests/army/enterprise/s3/s3_basic.py | 2 ++ tools/shell/src/shellAuto.c | 2 ++ 3 files changed, 7 insertions(+) diff --git a/tests/army/community/query/show.py b/tests/army/community/query/show.py index d1bc1d9934..8b6844820e 100644 --- a/tests/army/community/query/show.py +++ b/tests/army/community/query/show.py @@ -117,6 +117,9 @@ class TDTestCase(TBase): sql = "SHOW COMPACT 1;" tdSql.query(sql) tdSql.checkRows(0) + sql = "SHOW CLUSTER MACHINES;" + tdSql.query(sql) + tdSql.checkRows(1) # run to check crash sqls = [ diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 7071c678a4..c255ca3bb1 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -91,10 +91,12 @@ class TDTestCase(TBase): break self.trimDb(True) tdLog.info(f"loop={loop} no upload {cnt} data files wait 3s retry ...") + ''' if loop == 0: sc.dnodeStop(1) time.sleep(2) sc.dnodeStart(1) + ''' loop += 1 if len(rets) > 0: diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 0a2ce4f316..b6917e5a76 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -171,6 +171,7 @@ SWords shellCommands[] = { {"show compacts;", 0, 0, NULL}, {"show cluster;", 0, 0, NULL}, {"show cluster alive;", 0, 0, NULL}, + {"show cluster machines;", 0, 0, NULL}, {"show databases;", 0, 0, NULL}, {"show dnodes;", 0, 0, NULL}, {"show dnode variables;", 0, 0, NULL}, @@ -519,6 +520,7 @@ void showHelp() { show connections;\n\ show cluster;\n\ show cluster alive;\n\ + show cluster machines;\n\ show databases;\n\ show dnodes;\n\ show dnode variables;\n\ From abd000ed036f72a9b160fbf224b1282d58065f83 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 27 Feb 2024 10:29:15 +0800 Subject: [PATCH 171/181] disable query count --- include/common/tglobal.h | 1 + source/common/src/tglobal.c | 8 +++++++- source/libs/planner/src/planLogicCreater.c | 5 +++++ tests/script/tsim/query/query_count0.sim | 2 ++ tests/script/tsim/query/query_count1.sim | 2 ++ tests/script/tsim/query/query_count_sliding0.sim | 2 ++ 6 files changed, 19 insertions(+), 1 deletion(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index e65b0f9a1a..469bc6227e 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -219,6 +219,7 @@ extern bool tsFilterScalarMode; extern int32_t tsMaxStreamBackendCache; extern int32_t tsPQSortMemThreshold; extern int32_t tsResolveFQDNRetryTime; +extern bool tsDisableCount; extern bool tsExperimental; // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 4583f3ba2c..3d8c7c7f8b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -269,6 +269,7 @@ int64_t tsStreamBufferSize = 128 * 1024 * 1024; bool tsFilterScalarMode = false; int tsResolveFQDNRetryTime = 100; // seconds int tsStreamAggCnt = 1000; +bool tsDisableCount = true; char tsS3Endpoint[TSDB_FQDN_LEN] = ""; char tsS3AccessKey[TSDB_FQDN_LEN] = ""; @@ -540,6 +541,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + + if (cfgAddBool(pCfg, "disableCount", tsDisableCount, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; return 0; } @@ -1106,6 +1109,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32; tsExperimental = cfgGetItem(pCfg, "experimental")->bval; + + tsDisableCount = cfgGetItem(pCfg, "disableCount")->bval; return 0; } @@ -1732,7 +1737,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) { {"shellActivityTimer", &tsShellActivityTimer}, {"slowLogThreshold", &tsSlowLogThreshold}, {"useAdapter", &tsUseAdapter}, - {"experimental", &tsExperimental}}; + {"experimental", &tsExperimental}, + {"disableCount", &tsDisableCount}}; if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { taosCfgSetOption(options, tListLen(options), pItem, false); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index c5e84898ed..066ea13791 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -16,6 +16,7 @@ #include "planInt.h" #include "filter.h" #include "functionMgt.h" +#include "tglobal.h" typedef struct SLogicPlanContext { SPlanContext* pPlanCxt; @@ -1015,6 +1016,10 @@ static int32_t createWindowLogicNodeByCount(SLogicPlanContext* pCxt, SCountWindo return TSDB_CODE_OUT_OF_MEMORY; } + if (tsDisableCount) { + return TSDB_CODE_FAILED; + } + pWindow->winType = WINDOW_TYPE_COUNT; pWindow->node.groupAction = getGroupAction(pCxt, pSelect); pWindow->node.requireDataOrder = diff --git a/tests/script/tsim/query/query_count0.sim b/tests/script/tsim/query/query_count0.sim index 5b95d4fad7..b7c629e538 100644 --- a/tests/script/tsim/query/query_count0.sim +++ b/tests/script/tsim/query/query_count0.sim @@ -9,6 +9,8 @@ print =============== create database sql create database test vgroups 1; sql use test; +sql alter local 'disableCount' '0' ; + sql create table t1(ts timestamp, a int, b int , c int, d double); sql insert into t1 values(1648791213000,0,1,1,1.0); diff --git a/tests/script/tsim/query/query_count1.sim b/tests/script/tsim/query/query_count1.sim index 0694ab062a..0c40303e57 100644 --- a/tests/script/tsim/query/query_count1.sim +++ b/tests/script/tsim/query/query_count1.sim @@ -9,6 +9,8 @@ print =============== create database sql create database test vgroups 4; sql use test; +sql alter local 'disableCount' '0' ; + sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); diff --git a/tests/script/tsim/query/query_count_sliding0.sim b/tests/script/tsim/query/query_count_sliding0.sim index 464aec6b97..13a6c94451 100644 --- a/tests/script/tsim/query/query_count_sliding0.sim +++ b/tests/script/tsim/query/query_count_sliding0.sim @@ -9,6 +9,8 @@ print =============== create database sql create database test vgroups 1; sql use test; +sql alter local 'disableCount' '0' ; + sql create table t1(ts timestamp, a int, b int , c int, d double); sql insert into t1 values(1648791213000,0,1,1,1.0); From dafac591f7d380c651bb61ea4b7c999b77150c69 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Feb 2024 10:53:25 +0800 Subject: [PATCH 172/181] enh(stream): add rsp for req-checkpoint --- include/dnode/vnode/tqCommon.h | 1 + source/dnode/mgmt/mgmt_snode/src/smHandle.c | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/mnode/impl/inc/mndStream.h | 4 ++++ source/dnode/mnode/impl/src/mndScheduler.c | 4 ++-- source/dnode/mnode/impl/src/mndStream.c | 13 ++++++++++++- source/dnode/mnode/impl/src/mndStreamHb.c | 4 ---- source/dnode/snode/src/snode.c | 2 ++ source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tq/tq.c | 4 ++++ source/dnode/vnode/src/tqCommon/tqCommon.c | 6 +++++- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 ++ source/libs/stream/src/streamTask.c | 3 +-- 13 files changed, 36 insertions(+), 10 deletions(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index 84d0dd4982..ec00855666 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -27,6 +27,7 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored); int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen); diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 8c726f9f03..99777bbbb0 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -89,6 +89,7 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; code = 0; _OVER: diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 4213541351..9336bb7141 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -838,6 +838,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index aed49809dd..57fd187da3 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -86,6 +86,10 @@ typedef struct SOrphanTask { int32_t nodeId; } SOrphanTask; +typedef struct { + SMsgHead head; +} SMStreamHbRspMsg, SMStreamReqCheckpointRspMsg; + int32_t mndInitStream(SMnode *pMnode); void mndCleanupStream(SMnode *pMnode); SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 7831048f2d..9aba428ff6 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -308,8 +308,8 @@ static int64_t getVgroupLastVer(const SArray* pList, int32_t vgId) { } } - mError("failed to find the vgId:%d for extract last version, total existed vgs:%d", vgId, size); - return -1; + mDebug("no data in vgId:%d for extract last version, set to be 0, total existed vgs:%d", vgId, size); + return 1; } static void streamTaskSetDataRange(SStreamTask* pTask, int64_t skey, SArray* pVerList, int32_t vgId) { diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 8da56d2d46..3ef2f64df7 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -877,7 +877,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre int64_t ts = taosGetTimestampMs(); if (mndTrigger == 1 && (ts - pStream->checkpointFreq < tsStreamCheckpointInterval * 1000)) { // mWarn("checkpoint interval less than the threshold, ignore it"); - return -1; + return TSDB_CODE_SUCCESS; } bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHECKPOINT_NAME, lock); @@ -2179,5 +2179,16 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { mndReleaseStream(pMnode, pStream); taosThreadMutexUnlock(&execInfo.lock); + { + SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRspMsg)}; + rsp.pCont = rpcMallocCont(rsp.contLen); + SMsgHead* pHead = rsp.pCont; + pHead->vgId = htonl(req.nodeId); + + tmsgSendRsp(&rsp); + + pReq->info.handle = NULL; // disable auto rsp + } + return 0; } diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 1d296a1c6e..14f3c533e3 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -16,10 +16,6 @@ #include "mndStream.h" #include "mndTrans.h" -typedef struct { - SMsgHead head; -} SMStreamHbRspMsg; - typedef struct SFailedCheckpointInfo { int64_t streamUid; int64_t checkpointId; diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 34623c021b..e4b3cab256 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -180,6 +180,8 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { return tqStreamTaskProcessCheckpointReadyMsg(pSnode->pMeta, pMsg); case TDMT_MND_STREAM_HEARTBEAT_RSP: return tqStreamProcessStreamHbRsp(pSnode->pMeta, pMsg); + case TDMT_MND_STREAM_REQ_CHKPT_RSP: + return tqStreamProcessReqCheckpointRsp(pSnode->pMeta, pMsg); default: sndError("invalid snode msg:%d", pMsg->msgType); ASSERT(0); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index b83af84086..3d45a0765e 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -243,6 +243,7 @@ int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); int32_t tqScanWal(STQ* pTq); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a3fd19c7f8..7459551329 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1225,3 +1225,7 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg) { return tqStreamProcessStreamHbRsp(pTq->pStreamMeta, pMsg); } + +int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg) { + return tqStreamProcessReqCheckpointRsp(pTq->pStreamMeta, pMsg); +} diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 4e49091da4..b66a114ef8 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -938,9 +938,13 @@ int32_t tqStreamTasksGetTotalNum(SStreamMeta* pMeta) { return taosArrayGetSize(pMeta->pTaskList); } -int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { +static int32_t doProcessDummyRspMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) { rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; return TSDB_CODE_SUCCESS; } + +int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } + +int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 86355cdbb0..7763729c2c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -798,6 +798,8 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) return tqProcessTaskCheckpointReadyMsg(pVnode->pTq, pMsg); case TDMT_MND_STREAM_HEARTBEAT_RSP: return tqProcessStreamHbRsp(pVnode->pTq, pMsg); + case TDMT_MND_STREAM_REQ_CHKPT_RSP: + return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg); default: vError("unknown msg type:%d in stream queue", pMsg->msgType); return TSDB_CODE_APP_ERROR; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index b63dc50836..9639921c77 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -934,9 +934,8 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) { } tEncoderClear(&encoder); - SRpcMsg msg = {.info.noResp = 1}; + SRpcMsg msg = {0}; initRpcMsg(&msg, TDMT_MND_STREAM_REQ_CHKPT, buf, tlen); - stDebug("s-task:%s vgId:%d build and send task checkpoint req", id, vgId); tmsgSendReq(&pTask->info.mnodeEpset, &msg); From 6a916738805a5beb6176f707552a479a6f9c0484 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 27 Feb 2024 10:57:43 +0800 Subject: [PATCH 173/181] fix: add flush_each_batch to json --- tests/army/enterprise/s3/s3_basic.json | 3 ++- tests/army/enterprise/s3/s3_basic.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json index 54b8808f2f..24b46b668d 100644 --- a/tests/army/enterprise/s3/s3_basic.json +++ b/tests/army/enterprise/s3/s3_basic.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "connection_pool_size": 8, - "num_of_records_per_req": 2000, + "num_of_records_per_req": 4000, "prepared_rand": 1000, "thread_count": 2, "create_table_thread_count": 1, @@ -19,6 +19,7 @@ "vgroups": 2, "replica": 1, "duration":"10d", + "flush_each_batch":"yes", "keep": "30d,60d,100d" }, "super_tables": [ diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index c255ca3bb1..7071c678a4 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -91,12 +91,10 @@ class TDTestCase(TBase): break self.trimDb(True) tdLog.info(f"loop={loop} no upload {cnt} data files wait 3s retry ...") - ''' if loop == 0: sc.dnodeStop(1) time.sleep(2) sc.dnodeStart(1) - ''' loop += 1 if len(rets) > 0: From c4f08b82757ddcab76e441d7acfd35c988acf296 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 27 Feb 2024 03:05:45 +0000 Subject: [PATCH 174/181] fix/TD-28776 --- source/dnode/mnode/impl/src/mndTrans.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 4dcbfe169d..04d74112d4 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1217,7 +1217,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA if (numOfActions == 0) return 0; if ((code = mndTransExecSingleActions(pMnode, pTrans, pArray, topHalf)) != 0) { - return -1; + return code; } int32_t numOfExecuted = 0; From aee0460daf81110ec9175e84a0e5b3a7204236e9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Feb 2024 11:15:09 +0800 Subject: [PATCH 175/181] enh(stream):rsp the checkpoint ready msg. --- include/dnode/vnode/tqCommon.h | 1 + source/dnode/mgmt/mgmt_snode/src/smHandle.c | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/snode/src/snode.c | 2 ++ source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tq/tq.c | 4 ++++ source/dnode/vnode/src/tqCommon/tqCommon.c | 4 ++++ source/dnode/vnode/src/vnd/vnodeSvr.c | 2 ++ source/libs/stream/src/streamDispatch.c | 1 - 9 files changed, 16 insertions(+), 1 deletion(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index ec00855666..22a176f0bb 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -28,6 +28,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored); int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen); diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 99777bbbb0..1b1dcc9b54 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -87,6 +87,7 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 9336bb7141..bfac0bab9d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -835,6 +835,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index e4b3cab256..3bef5b595b 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -182,6 +182,8 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { return tqStreamProcessStreamHbRsp(pSnode->pMeta, pMsg); case TDMT_MND_STREAM_REQ_CHKPT_RSP: return tqStreamProcessReqCheckpointRsp(pSnode->pMeta, pMsg); + case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP: + return tqStreamProcessCheckpointReadyRsp(pSnode->pMeta, pMsg); default: sndError("invalid snode msg:%d", pMsg->msgType); ASSERT(0); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 3d45a0765e..06051ee5c8 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -244,6 +244,7 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); int32_t tqScanWal(STQ* pTq); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7459551329..e442af4a3d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1229,3 +1229,7 @@ int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg) { return tqStreamProcessReqCheckpointRsp(pTq->pStreamMeta, pMsg); } + +int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg) { + return tqStreamProcessCheckpointReadyRsp(pTq->pStreamMeta, pMsg); +} diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index b66a114ef8..c1c10b2fc2 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -948,3 +948,7 @@ static int32_t doProcessDummyRspMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } + +int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { + return doProcessDummyRspMsg(pMeta, pMsg); +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 7763729c2c..84cbbfd4b2 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -800,6 +800,8 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) return tqProcessStreamHbRsp(pVnode->pTq, pMsg); case TDMT_MND_STREAM_REQ_CHKPT_RSP: return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg); + case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP: + return tqProcessTaskCheckpointReadyRsp(pVnode->pTq, pMsg); default: vError("unknown msg type:%d in stream queue", pMsg->msgType); return TSDB_CODE_APP_ERROR; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 60c545b9e5..78b914c3db 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -907,7 +907,6 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, SStreamChkptReadyInfo info = {.upStreamTaskId = pInfo->taskId, .upstreamNodeEpset = pInfo->epSet}; initRpcMsg(&info.msg, TDMT_STREAM_TASK_CHECKPOINT_READY, buf, tlen + sizeof(SMsgHead)); - info.msg.info.noResp = 1; // refactor later. stDebug("s-task:%s (level:%d) prepare checkpoint ready msg to upstream s-task:0x%" PRIx64 ":0x%x (vgId:%d) idx:%d, vgId:%d", From 80d027da196ffde1a7d7b2b1c58befaf34dcd25c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Feb 2024 11:22:58 +0800 Subject: [PATCH 176/181] fix(stream): restore from the crash. --- source/dnode/vnode/src/tq/tq.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e442af4a3d..e279b4bc1d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1175,7 +1175,11 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) return TSDB_CODE_SUCCESS; } } else { - ASSERT(status == TASK_STATUS__HALT); +// ASSERT(status == TASK_STATUS__HALT); + if (status != TASK_STATUS__HALT) { + tqError("s-task:%s should in halt status, let's halt it directly", pTask->id.idStr); + streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT); + } } // check if the checkpoint msg already sent or not. From 402a43bf67143a71521dfdafbdc0c4f9a084be50 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Feb 2024 11:25:32 +0800 Subject: [PATCH 177/181] fix(stream): disable halt. --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e279b4bc1d..011e62cb89 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1178,7 +1178,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) // ASSERT(status == TASK_STATUS__HALT); if (status != TASK_STATUS__HALT) { tqError("s-task:%s should in halt status, let's halt it directly", pTask->id.idStr); - streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT); +// streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT); } } From 654aa681371b10f7beb852be07c992cdccc6febb Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 27 Feb 2024 11:41:44 +0800 Subject: [PATCH 178/181] disable query count --- source/libs/planner/src/planLogicCreater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 066ea13791..0f5ddf926d 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1016,7 +1016,7 @@ static int32_t createWindowLogicNodeByCount(SLogicPlanContext* pCxt, SCountWindo return TSDB_CODE_OUT_OF_MEMORY; } - if (tsDisableCount) { + if (!pCxt->pPlanCxt->streamQuery && tsDisableCount) { return TSDB_CODE_FAILED; } From d964caf31ef238fb6e2766e067a7f68624d0278c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 27 Feb 2024 12:43:45 +0800 Subject: [PATCH 179/181] fix: adjust 2 tables to write --- tests/army/enterprise/s3/s3_basic.json | 8 ++++---- tests/army/enterprise/s3/s3_basic.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json index 24b46b668d..747ac7c8ec 100644 --- a/tests/army/enterprise/s3/s3_basic.json +++ b/tests/army/enterprise/s3/s3_basic.json @@ -18,16 +18,16 @@ "drop": "yes", "vgroups": 2, "replica": 1, - "duration":"10d", + "duration":"15d", "flush_each_batch":"yes", - "keep": "30d,60d,100d" + "keep": "60d,100d,200d" }, "super_tables": [ { "name": "stb", "child_table_exists": "no", - "childtable_count": 6, - "insert_rows": 1000000, + "childtable_count": 2, + "insert_rows": 2000000, "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 7071c678a4..e9173dda00 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -58,8 +58,8 @@ class TDTestCase(TBase): tdSql.execute(f"use {self.db}") # come from s3_basic.json - self.childtable_count = 6 - self.insert_rows = 1000000 + self.childtable_count = 2 + self.insert_rows = 2000000 self.timestamp_step = 1000 def createStream(self, sname): From de1c67f77381e28af14d76621dfe0a511cd3f6bc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Feb 2024 12:57:43 +0800 Subject: [PATCH 180/181] fix(stream): rsp the checkpoint ready msg. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index c1c10b2fc2..e3dbaa8803 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -485,6 +485,10 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe return code; } +typedef struct SMStreamCheckpointReadyRspMsg { + int8_t placeholder; +}SMStreamCheckpointReadyRspMsg; + int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t vgId = pMeta->vgId; char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); @@ -513,6 +517,18 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) streamProcessCheckpointReadyMsg(pTask); streamMetaReleaseTask(pMeta, pTask); + + { // send checkpoint ready rsp + SRpcMsg rsp = {.code = 0, .info = pMsg->info, .contLen = sizeof(SMStreamCheckpointReadyRspMsg)}; + rsp.pCont = rpcMallocCont(rsp.contLen); + SMsgHead* pHead = rsp.pCont; + pHead->vgId = htonl(req.downstreamNodeId); + + tmsgSendRsp(&rsp); + + pMsg->info.handle = NULL; // disable auto rsp + } + return code; } From d05c55f3fe40e5bc16432f1da3a726a85f298b97 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Feb 2024 12:59:08 +0800 Subject: [PATCH 181/181] fix(stream): update the msg struct. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index e3dbaa8803..a2d45062b9 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -486,7 +486,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe } typedef struct SMStreamCheckpointReadyRspMsg { - int8_t placeholder; + SMsgHead head; }SMStreamCheckpointReadyRspMsg; int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) {