From dec257ff599d78ec12e58c0527531e8498a15a0a Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 23 Oct 2023 08:59:54 +0000 Subject: [PATCH] TD-26529 --- include/common/tmsg.h | 9 + include/common/tmsgdef.h | 1 + include/libs/monitor/monitor.h | 2 + include/libs/monitorfw/taos_alloc.h | 47 ++ include/libs/monitorfw/taos_collector.h | 89 ++++ .../libs/monitorfw/taos_collector_registry.h | 123 ++++++ include/libs/monitorfw/taos_counter.h | 102 +++++ include/libs/monitorfw/taos_linked_list.h | 27 ++ include/libs/monitorfw/taos_map.h | 25 ++ include/libs/monitorfw/taos_metric.h | 49 +++ include/libs/monitorfw/taos_metric_sample.h | 59 +++ include/libs/monitorfw/taos_monitor.h | 130 ++++++ include/util/tlog.h | 1 + source/common/src/tmsg.c | 38 ++ source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 3 +- source/dnode/mgmt/node_mgmt/CMakeLists.txt | 2 +- source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 2 + source/dnode/mgmt/node_util/CMakeLists.txt | 2 +- source/dnode/mgmt/node_util/inc/dmUtil.h | 1 + source/dnode/mnode/impl/CMakeLists.txt | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 19 + source/dnode/mnode/impl/src/mndMnode.c | 1 + source/dnode/vnode/src/vnd/vnodeSvr.c | 16 + source/libs/CMakeLists.txt | 1 + source/libs/monitor/CMakeLists.txt | 2 +- source/libs/monitor/src/monMain.c | 28 ++ source/libs/monitorfw/CMakeLists.txt | 9 + source/libs/monitorfw/inc/taos_assert.h | 27 ++ .../monitorfw/inc/taos_collector_registry_i.h | 25 ++ .../monitorfw/inc/taos_collector_registry_t.h | 40 ++ source/libs/monitorfw/inc/taos_collector_t.h | 32 ++ source/libs/monitorfw/inc/taos_errors.h | 25 ++ .../libs/monitorfw/inc/taos_linked_list_i.h | 93 ++++ .../libs/monitorfw/inc/taos_linked_list_t.h | 53 +++ source/libs/monitorfw/inc/taos_log.h | 27 ++ source/libs/monitorfw/inc/taos_map_i.h | 37 ++ source/libs/monitorfw/inc/taos_map_t.h | 44 ++ .../monitorfw/inc/taos_metric_formatter_i.h | 82 ++++ .../monitorfw/inc/taos_metric_formatter_t.h | 26 ++ source/libs/monitorfw/inc/taos_metric_i.h | 43 ++ .../libs/monitorfw/inc/taos_metric_sample_i.h | 48 ++ .../libs/monitorfw/inc/taos_metric_sample_t.h | 28 ++ source/libs/monitorfw/inc/taos_metric_t.h | 54 +++ .../monitorfw/inc/taos_string_builder_i.h | 77 ++++ .../monitorfw/inc/taos_string_builder_t.h | 25 ++ source/libs/monitorfw/src/taos_collector.c | 110 +++++ .../monitorfw/src/taos_collector_registry.c | 200 +++++++++ source/libs/monitorfw/src/taos_counter.c | 65 +++ source/libs/monitorfw/src/taos_linked_list.c | 220 ++++++++++ source/libs/monitorfw/src/taos_map.c | 414 ++++++++++++++++++ source/libs/monitorfw/src/taos_metric.c | 173 ++++++++ .../monitorfw/src/taos_metric_formatter.c | 258 +++++++++++ .../libs/monitorfw/src/taos_metric_sample.c | 96 ++++ .../libs/monitorfw/src/taos_string_builder.c | 152 +++++++ 54 files changed, 3259 insertions(+), 5 deletions(-) create mode 100644 include/libs/monitorfw/taos_alloc.h create mode 100644 include/libs/monitorfw/taos_collector.h create mode 100644 include/libs/monitorfw/taos_collector_registry.h create mode 100644 include/libs/monitorfw/taos_counter.h create mode 100644 include/libs/monitorfw/taos_linked_list.h create mode 100644 include/libs/monitorfw/taos_map.h create mode 100644 include/libs/monitorfw/taos_metric.h create mode 100644 include/libs/monitorfw/taos_metric_sample.h create mode 100644 include/libs/monitorfw/taos_monitor.h create mode 100644 source/libs/monitorfw/CMakeLists.txt create mode 100644 source/libs/monitorfw/inc/taos_assert.h create mode 100644 source/libs/monitorfw/inc/taos_collector_registry_i.h create mode 100644 source/libs/monitorfw/inc/taos_collector_registry_t.h create mode 100644 source/libs/monitorfw/inc/taos_collector_t.h create mode 100644 source/libs/monitorfw/inc/taos_errors.h create mode 100644 source/libs/monitorfw/inc/taos_linked_list_i.h create mode 100644 source/libs/monitorfw/inc/taos_linked_list_t.h create mode 100644 source/libs/monitorfw/inc/taos_log.h create mode 100644 source/libs/monitorfw/inc/taos_map_i.h create mode 100644 source/libs/monitorfw/inc/taos_map_t.h create mode 100644 source/libs/monitorfw/inc/taos_metric_formatter_i.h create mode 100644 source/libs/monitorfw/inc/taos_metric_formatter_t.h create mode 100644 source/libs/monitorfw/inc/taos_metric_i.h create mode 100644 source/libs/monitorfw/inc/taos_metric_sample_i.h create mode 100644 source/libs/monitorfw/inc/taos_metric_sample_t.h create mode 100644 source/libs/monitorfw/inc/taos_metric_t.h create mode 100644 source/libs/monitorfw/inc/taos_string_builder_i.h create mode 100644 source/libs/monitorfw/inc/taos_string_builder_t.h create mode 100644 source/libs/monitorfw/src/taos_collector.c create mode 100644 source/libs/monitorfw/src/taos_collector_registry.c create mode 100644 source/libs/monitorfw/src/taos_counter.c create mode 100644 source/libs/monitorfw/src/taos_linked_list.c create mode 100644 source/libs/monitorfw/src/taos_map.c create mode 100644 source/libs/monitorfw/src/taos_metric.c create mode 100644 source/libs/monitorfw/src/taos_metric_formatter.c create mode 100644 source/libs/monitorfw/src/taos_metric_sample.c create mode 100644 source/libs/monitorfw/src/taos_string_builder.c diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 18b10a1749..6c6bacb6ad 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1544,6 +1544,15 @@ int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); void tFreeSStatusReq(SStatusReq* pReq); +typedef struct { + int32_t contLen; + char* pCont; +} SStatisReq; + +int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); +int32_t tDeserializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); +void tFreeSStatisReq(SStatisReq *pReq); + typedef struct { int32_t dnodeId; int64_t clusterId; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index b92bba831c..2fe7657b7f 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -189,6 +189,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_MND_STREAM_NODECHANGE_CHECK, "stream-nodechange-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB_TIMER, "trim-db-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_STATIS, "statis", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_NEW_MSG_SEG(TDMT_VND_MSG) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 91b3a54ea1..1f3d21777d 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -222,6 +222,8 @@ void monSetQmInfo(SMonQmInfo *pInfo); void monSetSmInfo(SMonSmInfo *pInfo); void monSetBmInfo(SMonBmInfo *pInfo); void monSendReport(); +void monSendPromReport(); +void monSendContent(char *pCont); void tFreeSMonMmInfo(SMonMmInfo *pInfo); void tFreeSMonVmInfo(SMonVmInfo *pInfo); diff --git a/include/libs/monitorfw/taos_alloc.h b/include/libs/monitorfw/taos_alloc.h new file mode 100644 index 0000000000..893ffc7e9b --- /dev/null +++ b/include/libs/monitorfw/taos_alloc.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_alloc.h + * @brief memory management + */ + +#ifndef TAOS_ALLOC_H +#define TAOS_ALLOC_H + +#include +#include + +/** + * @brief Redefine this macro if you wish to override it. The default value is malloc. + */ +#define taos_malloc malloc + +/** + * @brief Redefine this macro if you wish to override it. The default value is realloc. + */ +#define taos_realloc realloc + +/** + * @brief Redefine this macro if you wish to override it. The default value is strdup. + */ +#define taos_strdup strdup + +/** + * @brief Redefine this macro if you wish to override it. The default value is free. + */ +#define taos_free free + +#endif // TAOS_ALLOC_H diff --git a/include/libs/monitorfw/taos_collector.h b/include/libs/monitorfw/taos_collector.h new file mode 100644 index 0000000000..918395ae72 --- /dev/null +++ b/include/libs/monitorfw/taos_collector.h @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_COLLECTOR_H +#define TAOS_COLLECTOR_H + +#include "taos_map.h" +#include "taos_metric.h" + +/** + * @file taos_collector.h + * @brief A Prometheus collector returns a collection of metrics + */ + +/** + * @brief A prometheus collector calls collect to prepare metrics and return them to the registry to which it is + * registered. + */ +typedef struct taos_collector taos_collector_t; + +/** + * @brief The function responsible for preparing metric data and returning metrics for a given collector. + * + * If you use the default collector registry, this should not concern you. If you are using a custom collector, you may + * set this function on your collector to do additional work before returning the contained metrics. + * + * @param self The target taos_collector_t* + * @return The taos_map_t* containing the collected metrics + */ +typedef taos_map_t *taos_collect_fn(taos_collector_t *self); + +/** + * @brief Create a collector + * @param name The name of the collector. The name MUST NOT be default or process. + * @return The constructed taos_collector_t* + */ +taos_collector_t *taos_collector_new(const char *name); + +/** + * @brief Destroy a collector. You MUST set self to NULL after destruction. + * @param self The target taos_collector_t* + * @return A non-zero integer value upon failure. + */ +int taos_collector_destroy(taos_collector_t *self); + +/** + * @brief Frees a collector passed as a void pointer. You MUST set self to NULL after destruction. + * @param gen The target taos_collector_t* represented as a void* + */ +void taos_collector_free_generic(void *gen); + +/** + * @brief Destroys a collector passed as a void pointer. You MUST set self to NULL after destruction. + * @param gen The target taos_collector_t* represented as a void* + * @return A non-zero integer value upon failure. + */ +int taos_collector_destroy_generic(void *gen); + +/** + * @brief Add a metric to a collector + * @param self The target taos_collector_t* + * @param metric the taos_metric_t* to add to the taos_collector_t* passed as self. + * @return A non-zero integer value upon failure. + */ +int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric); + +/** + * @brief The collect function is responsible for doing any work involving a set of metrics and then returning them + * for metric exposition. + * @param self The target taos_collector_t* + * @param fn The taos_collect_fn* which will be responsible for handling any metric collection operations before + * returning the collected metrics for exposition. + * @return A non-zero integer value upon failure. + */ +int taos_collector_set_collect_fn(taos_collector_t *self, taos_collect_fn *fn); + +#endif // TAOS_COLLECTOR_H diff --git a/include/libs/monitorfw/taos_collector_registry.h b/include/libs/monitorfw/taos_collector_registry.h new file mode 100644 index 0000000000..f67deb074b --- /dev/null +++ b/include/libs/monitorfw/taos_collector_registry.h @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_collector_registry.h + * @brief The collector registry registers collectors for metric exposition. + */ + +#ifndef TAOS_REGISTRY_H +#define TAOS_REGISTRY_H + +#include "taos_collector.h" +#include "taos_metric.h" + +/** + * @brief A taos_registry_t is responsible for registering metrics and briding them to the string exposition format + */ +typedef struct taos_collector_registry taos_collector_registry_t; + +/** + * @brief Initialize the default registry by calling taos_collector_registry_init within your program. You MUST NOT + * modify this value. + */ +extern taos_collector_registry_t *TAOS_COLLECTOR_REGISTRY_DEFAULT; + +/** + * @brief Initializes the default collector registry and enables metric collection on the executing process + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_default_init(void); + +/** + * @brief Constructs a taos_collector_registry_t* + * @param name The name of the collector registry. It MUST NOT be default. + * @return The constructed taos_collector_registry_t* + */ +taos_collector_registry_t *taos_collector_registry_new(const char *name); + +/** + * @brief Destroy a collector registry. You MUST set self to NULL after destruction. + * @param self The target taos_collector_registry_t* + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_destroy(taos_collector_registry_t *self); + +/** + * @brief Enable process metrics on the given collector registry + * @param self The target taos_collector_registry_t* + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_enable_process_metrics(taos_collector_registry_t *self); + +/** + * @brief Registers a metric with the default collector on TAOS_DEFAULT_COLLECTOR_REGISTRY + * + * The metric to be registered MUST NOT already be registered with the given . If so, the program will + * halt. It returns a taos_metric_t* to simplify metric creation and registration. Furthermore, + * TAOS_DEFAULT_COLLECTOR_REGISTRY must be registered via taos_collector_registry_default_init() prior to calling this + * function. The metric will be added to the default registry's default collector. + * + * @param metric The metric to register on TAOS_DEFAULT_COLLECTOR_REGISTRY* + * @return The registered taos_metric_t* + */ +taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metric); + +/** + * @brief Registers a metric with the default collector on TAOS_DEFAULT_COLLECTOR_REGISTRY. Returns an non-zero integer + * value on failure. + * + * See taos_collector_registry_must_register_metric. + * + * @param metric The metric to register on TAOS_DEFAULT_COLLECTOR_REGISTRY* + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_register_metric(taos_metric_t *metric); + +/** + * @brief Register a collector with the given registry. Returns a non-zero integer value on failure. + * @param self The target taos_collector_registry_t* + * @param collector The taos_collector_t* to register onto the taos_collector_registry_t* as self + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_register_collector(taos_collector_registry_t *self, taos_collector_t *collector); + +/** + * @brief Returns a string in the default metric exposition format. The string MUST be freed to avoid unnecessary heap + * memory growth. + * + * Reference: https://prometheus.io/docs/instrumenting/exposition_formats/ + * + * @param self The target taos_collector_registry_t* + * @return The string int he default metric exposition format. + */ +const char *taos_collector_registry_bridge(taos_collector_registry_t *self, int64_t ts); + +int taos_collector_registry_clear_out(taos_collector_registry_t *self); + +/** + *@brief Validates that the given metric name complies with the specification: + * + * Reference: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels + * + * Returns a non-zero integer value on failure. + * + * @param self The target taos_collector_registry_t* + * @param metric_name The metric name to validate + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_validate_metric_name(taos_collector_registry_t *self, const char *metric_name); + +#endif // TAOS_H diff --git a/include/libs/monitorfw/taos_counter.h b/include/libs/monitorfw/taos_counter.h new file mode 100644 index 0000000000..28a9eed41c --- /dev/null +++ b/include/libs/monitorfw/taos_counter.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_COUNTER_H +#define TAOS_COUNTER_H + +#include + +#include "taos_metric.h" + +/** + * @file taos_counter.h + * @brief https://prometheus.io/docs/concepts/metric_types/#counter + */ + +/** + * @brief A prometheus counter. + * + * References + * * See https://prometheus.io/docs/concepts/metric_types/#counter + */ +typedef taos_metric_t taos_counter_t; + +/** + * @brief Construct a taos_counter_t* + * @param name The name of the metric + * @param help The metric description + * @param label_key_count The number of labels associated with the given metric. Pass 0 if the metric does not + * require labels. + * @param label_keys A collection of label keys. The number of keys MUST match the value passed as label_key_count. If + * no labels are required, pass NULL. Otherwise, it may be convenient to pass this value as a + * literal. + * @return The constructed taos_counter_t* + * + * *Example* + * + * // An example with labels + * taos_counter_new("foo", "foo is a counter with labels", 2, (const char**) { "one", "two" }); + * + * // An example without labels + * taos_counter_new("foo", "foo is a counter without labels", 0, NULL); + */ +taos_counter_t *taos_counter_new(const char *name, const char *help, size_t label_key_count, const char **label_keys); + +/** + * @brief Destroys a taos_counter_t*. You must set self to NULL after destruction. A non-zero integer value will be + * returned on failure. + * @param self A taos_counter_t* + * @return A non-zero integer value upon failure. + */ +int taos_counter_destroy(taos_counter_t *self); + +/** + * @brief Increment the taos_counter_t by 1. A non-zero integer value will be returned on failure. + * @param self The target taos_counter_t* + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the counter's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_counter_inc(foo_counter, (const char**) { "bar", "bang" }); + ** + * // An example without labels + * taos_counter_inc(foo_counter, NULL); + */ +int taos_counter_inc(taos_counter_t *self, const char **label_values); + +/** + * @brief Add the value to the taos_counter_t*. A non-zero integer value will be returned on failure. + * @param self The target taos_counter_t* + * @param r_value The double to add to the taos_counter_t passed as self. The value MUST be greater than or equal to 0. + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the counter's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_counter_add(foo_counter, 22, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_counter_add(foo_counter, 22, NULL); + */ +int taos_counter_add(taos_counter_t *self, double r_value, const char **label_values); + +#endif // TAOS_COUNTER_H diff --git a/include/libs/monitorfw/taos_linked_list.h b/include/libs/monitorfw/taos_linked_list.h new file mode 100644 index 0000000000..1fff5e65a6 --- /dev/null +++ b/include/libs/monitorfw/taos_linked_list.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_LIST_H +#define TAOS_LIST_H + +#include + +struct taos_linked_list; +/** + * @brief Provides a generic linked list + */ +typedef struct taos_linked_list taos_linked_list_t; + +#endif // TAOS_LIST_H diff --git a/include/libs/monitorfw/taos_map.h b/include/libs/monitorfw/taos_map.h new file mode 100644 index 0000000000..fa554d569c --- /dev/null +++ b/include/libs/monitorfw/taos_map.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_MAP_H +#define TAOS_MAP_H + +struct taos_map; +typedef struct taos_map taos_map_t; + +struct taos_map_node; +typedef struct taos_map_node taos_map_node_t; + +#endif // TAOS_MAP_H diff --git a/include/libs/monitorfw/taos_metric.h b/include/libs/monitorfw/taos_metric.h new file mode 100644 index 0000000000..a377311fa0 --- /dev/null +++ b/include/libs/monitorfw/taos_metric.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_metric.h + * @brief Functions for retrieving metric samples from metrics given an ordered set of labels + */ + +#ifndef TAOS_METRIC_H +#define TAOS_METRIC_H + +#include "taos_metric_sample.h" + +struct taos_metric; +/** + * @brief A prometheus metric. + * + * Reference: https://prometheus.io/docs/concepts/data_model + */ +typedef struct taos_metric taos_metric_t; + +/** + * @brief Returns a taos_metric_sample_t*. The order of label_values is significant. + * + * You may use this function to cache metric samples to avoid sample lookup. Metric samples are stored in a hash map + * with O(1) lookups in average case; nonethless, caching metric samples and updating them directly might be + * preferrable in performance-sensitive situations. + * + * @param self The target taos_metric_t* + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the counter's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A taos_metric_sample_t* + */ +taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const char **label_values); + +#endif // TAOS_METRIC_H diff --git a/include/libs/monitorfw/taos_metric_sample.h b/include/libs/monitorfw/taos_metric_sample.h new file mode 100644 index 0000000000..1c37f59e32 --- /dev/null +++ b/include/libs/monitorfw/taos_metric_sample.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_metric_sample.h + * @brief Functions for interfacting with metric samples directly + */ + +#ifndef TAOS_METRIC_SAMPLE_H +#define TAOS_METRIC_SAMPLE_H + +struct taos_metric_sample; +/** + * @brief Contains the specific metric and value given the name and label set + * Reference: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels + */ +typedef struct taos_metric_sample taos_metric_sample_t; + +/** + * @brief Add the r_value to the sample. The value must be greater than or equal to zero. + * @param self The target taos_metric_sample_t* + * @param r_value The double to add to taos_metric_sample_t* provided by self + * @return Non-zero integer value upon failure + */ +int taos_metric_sample_add(taos_metric_sample_t *self, double r_value); + +/** + * @brief Subtract the r_value from the sample. + * + * This operation MUST be called a sample derived from a gauge metric. + * @param self The target taos_metric_sample_t* + * @param r_value The double to subtract from the taos_metric_sample_t* provided by self + * @return Non-zero integer value upon failure + */ +int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value); + +/** + * @brief Set the r_value of the sample. + * + * This operation MUST be called on a sample derived from a gauge metric. + * @param self The target taos_metric_sample_t* + * @param r_value The double which will be set to the taos_metric_sample_t* provided by self + * @return Non-zero integer value upon failure + */ +int taos_metric_sample_set(taos_metric_sample_t *self, double r_value); + +#endif // TAOS_METRIC_SAMPLE_H diff --git a/include/libs/monitorfw/taos_monitor.h b/include/libs/monitorfw/taos_monitor.h new file mode 100644 index 0000000000..ec4f007c45 --- /dev/null +++ b/include/libs/monitorfw/taos_monitor.h @@ -0,0 +1,130 @@ +/* +Copyright 2019 DigitalOcean Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/** + * @file taos_monitor.h + * @brief Include taos_monitor.h to include the entire public API + * @mainpage Welcome to the documentation site for prometheus-client-c! + * @tableofcontents + * @section Introduction + * + * prometheus-client-c is a small suite of Prometheus client libraries targeted for the C programming language. + * In this brief tutorial you will learn how to create and register metrics, update metric samples, and expose metrics + * over HTTP. + * + * @section Creating-and-Registering-Metrics Creating and Registering Metrics + * + * prometheus-client-c supports the following metric types: + * + * * [Counter](https://prometheus.io/docs/concepts/metric_types/#counter) + * * [Gauge](https://prometheus.io/docs/concepts/metric_types/#gauge) + * * [Histogram](https://prometheus.io/docs/concepts/metric_types/#histogram) + * + * To get started using one of the metric types, declare the metric at file scope. For example: + * + * @code{.c} + * + * #incldue "taos_monitor.h" + * + * taos_counter_t *my_counter; + * + * @endcode + * + * Next, create a metric initialization function. You can create the metric and register it with the default metric + * collector registry in one chain of functions. A metric collector is responsible for collecting metrics and returning + * them. A metric collector registry is declared in global scope and contains metric collectors. More on this later... + * + * To create a metric and register it with the default metric collector registry in one shot, you may chain the metric + * constructor into the taos_collector_registry_must_register_metric function. For example: + * + * @code{.c} + * + * void foo_metric_init(void) { + * my_counter = taos_collector_registry_must_register_metric(taos_counter_new("my_counter", "counts things", 0, NULL)); + * } + * + * @endcode + * + * The first argument to taos_counter_new is the counter name. The second argument is the counter description. The third + * argument is the number of metric labels. In this case, we will only have one metric sample for this metric so we pass + * 0 to specify that no labels will be used. The 4th argument is an array of strings storing the metric labels. Since we + * have none, we pass NULL. A call to foo_metric_init within the program's main function will initialize the metrics + * for the file we just created to the default prometheus metric collector registery called + * TAOS_COLLECTOR_REGISTRY_DEFAULT + * + * @section Updating-Metric-Sample-Values Updating Metric Sample Values + * + * Now that we have a metric configured for creation and registration, we can update our metric within any of the + * functions of the file in which it was declared. For example: + * + * @code{.c} + * + * void my_lib_do_something(void) { + * printf("I did a really important thing!\n"); + * taos_counter_inc(my_counter, NULL); + * } + * @endcode + * + * This function will increment the default metric sample for my_counter. Since we are not using metric labels, we pass + * NULL as the second argument. + * + * @section Program-Initialization Program Initialization + * + * At the start of the program's main function you need to do two things: + * + * * Initialize the default metric collector registry: + * + * @code{.c} + * + * taos_collector_registry_default_init(); + * + * @endcode + * + * * For each file containing prometheus metrics, call its corresponding metric initialization function + * + * @code{.c} + * + * foo_metric_init() + * + * @endcode + * + * After initialization is complete, you may proceed to do work and update your metrics. + * + * @section Metric-Exposition-Over-HTTP Metric Exposition Over HTTP + * + * @todo Describe how to use libpromhttp to expose metrics over HTTP + * + * @section Where-To-Go-From-Here Where to Go From Here? + * + * Take a look at the [Files](https://github.internal.digitalocean.com/pages/timeseries/prometheus-client-c/files.html) + * tab in this documentation site for more information about the public API available to you. Also, you can take a look + * at the examples directory at the + * [Github repository](https://github.internal.digitalocean.com/timeseries/prometheus-client-c) for inspiration. + */ + +#ifndef TAOS_INCLUDED +#define TAOS_INCLUDED + +#include "taos_alloc.h" +#include "taos_collector.h" +#include "taos_collector_registry.h" +#include "taos_counter.h" +#include "taos_linked_list.h" +#include "taos_map.h" +#include "taos_metric.h" +#include "taos_metric_sample.h" + +#endif // TAOS_INCLUDED \ No newline at end of file diff --git a/include/util/tlog.h b/include/util/tlog.h index a6d146a79e..1d795045ef 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -118,6 +118,7 @@ void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile); #define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }} #define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", DEBUG_TRACE, uDebugFlag, __VA_ARGS__); }} #define uDebugL(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLongString("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }} +#define uInfoL(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLongString("UTL ", DEBUG_INFO, uDebugFlag, __VA_ARGS__); }} #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); } #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index bd6eb46bad..d22d4c4cf5 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1423,6 +1423,44 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) { void tFreeSStatusRsp(SStatusRsp *pRsp) { taosArrayDestroy(pRsp->pDnodeEps); } +int32_t tSerializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + + if (tEncodeI32(&encoder, pReq->contLen) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->pCont) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeI32(&decoder, &pReq->contLen) < 0) return -1; + if (pReq->contLen > 0) { + pReq->pCont = taosMemoryMalloc(pReq->contLen + 1); + if (pReq->pCont == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->pCont) < 0) return -1; + } + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} + +void tFreeSStatisReq(SStatisReq *pReq) { + taosMemoryFreeClear(pReq->pCont); +} + int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index d5488da770..8f60a3dc85 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -189,7 +189,8 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_INDEX, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_INDEX, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RESTORE_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - + if (dmSetMgmtHandle(pArray, TDMT_MND_STATIS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_CONTINUE, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/node_mgmt/CMakeLists.txt b/source/dnode/mgmt/node_mgmt/CMakeLists.txt index f1be20289a..3761020560 100644 --- a/source/dnode/mgmt/node_mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/node_mgmt/CMakeLists.txt @@ -1,7 +1,7 @@ aux_source_directory(src IMPLEMENT_SRC) add_library(dnode STATIC ${IMPLEMENT_SRC}) target_link_libraries( - dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode + dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw ) target_include_directories( dnode diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index b3db7c3058..c237ce73c0 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -106,6 +106,8 @@ void dmSendMonitorReport() { dmGetQmMonitorInfo(pDnode); dmGetSmMonitorInfo(pDnode); monSendReport(); + + monSendPromReport(); } void dmGetVnodeLoads(SMonVloadInfo *pInfo) { diff --git a/source/dnode/mgmt/node_util/CMakeLists.txt b/source/dnode/mgmt/node_util/CMakeLists.txt index 5c670cbdd3..d882d784de 100644 --- a/source/dnode/mgmt/node_util/CMakeLists.txt +++ b/source/dnode/mgmt/node_util/CMakeLists.txt @@ -6,5 +6,5 @@ target_include_directories( PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor + node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor monitorfw ) \ No newline at end of file diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 0a52c578a5..4c00a48796 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -39,6 +39,7 @@ #include "sync.h" #include "tfs.h" #include "wal.h" +#include "taos_monitor.h" #ifdef __cplusplus extern "C" { diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 48dc71a12b..8f9564b072 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -16,7 +16,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser audit + mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser audit monitorfw ) IF (TD_GRANT) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index b53dee7bff..01332472a1 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -73,6 +73,7 @@ static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessStatusReq(SRpcMsg *pReq); static int32_t mndProcessNotifyReq(SRpcMsg *pReq); static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq); +static int32_t mndProcessStatisReq(SRpcMsg *pReq); static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); @@ -108,6 +109,7 @@ int32_t mndInitDnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq); mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq); mndSetMsgHandle(pMnode, TDMT_MND_RESTORE_DNODE, mndProcessRestoreDnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_STATIS, mndProcessStatisReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndRetrieveConfigs); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndCancelGetNextConfig); @@ -489,6 +491,23 @@ static bool mndUpdateMnodeState(SMnodeObj *pObj, SMnodeLoad *pMload) { return stateChanged; } +static int32_t mndProcessStatisReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SStatisReq statisReq = {0}; + int32_t code = -1; + + if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + monSendContent(pReq->pCont); + +_OVER: + tFreeSStatisReq(&statisReq); + return code; +} + static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStatusReq statusReq = {0}; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 22b2fec857..a93973fd13 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -23,6 +23,7 @@ #include "mndTrans.h" #include "tmisce.h" #include "audit.h" +#include "taos_monitor.h" #define MNODE_VER_NUMBER 2 #define MNODE_RESERVE_SIZE 64 diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index efa722d41a..c4aa5c6428 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -20,6 +20,7 @@ #include "vnode.h" #include "vnodeInt.h" #include "audit.h" +#include "taos_monitor.h" static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -40,6 +41,8 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessConfigChangeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +taos_counter_t *insert_counter = NULL; + static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t btime, int64_t *pUid) { int32_t code = 0; int32_t lino = 0; @@ -1613,6 +1616,19 @@ _exit: tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT); } + if(insert_counter == NULL){ + int32_t label_count =1; + const char *sample_labels[] = {"vgid"}; + insert_counter = taos_counter_new("insert_counter", "counter for insert sql", label_count, sample_labels); + insert_counter = taos_collector_registry_must_register_metric(insert_counter); + } + + char vgId[50]; + sprintf(vgId, "%"PRId32, TD_VID(pVnode)); + const char *sample_labels[] = {vgId}; + + taos_counter_inc(insert_counter, sample_labels); + // clear taosArrayDestroy(newTbUids); tDestroySubmitReq(pSubmitReq, 0 == pMsg->version ? TSDB_MSG_FLG_CMPT : TSDB_MSG_FLG_DECODE); diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 9f812517c1..6196d30a58 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -8,6 +8,7 @@ add_subdirectory(qcom) add_subdirectory(nodes) add_subdirectory(catalog) add_subdirectory(audit) +add_subdirectory(monitorfw) add_subdirectory(scalar) add_subdirectory(function) diff --git a/source/libs/monitor/CMakeLists.txt b/source/libs/monitor/CMakeLists.txt index 30dce7aaef..13523fd3cc 100644 --- a/source/libs/monitor/CMakeLists.txt +++ b/source/libs/monitor/CMakeLists.txt @@ -6,7 +6,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries(monitor os util common transport) +target_link_libraries(monitor os util common transport monitorfw) if(${BUILD_TEST}) add_subdirectory(test) diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 56cf0a2b51..93abda1281 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -18,6 +18,7 @@ #include "taoserror.h" #include "thttp.h" #include "ttime.h" +#include "taos_monitor.h" static SMonitor tsMonitor = {0}; static char* tsMonUri = "/report"; @@ -108,6 +109,9 @@ int32_t monInit(const SMonCfg *pCfg) { tsLogFp = monRecordLog; tsMonitor.lastTime = taosGetTimestampMs(); taosThreadMutexInit(&tsMonitor.lock, NULL); + + taos_collector_registry_default_init(); + return 0; } @@ -542,3 +546,27 @@ void monSendReport() { monCleanupMonitorInfo(pMonitor); } + +void monSendPromReport() { + char *pCont = (char *)taos_collector_registry_bridge( + TAOS_COLLECTOR_REGISTRY_DEFAULT, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + uInfoL("report cont:\n%s\n", pCont); + if (pCont != NULL) { + EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + uError("failed to send monitor msg"); + }else{ + taos_collector_registry_clear_out(TAOS_COLLECTOR_REGISTRY_DEFAULT); + } + } +} + +void monSendContent(char *pCont) { + uInfoL("report cont:\n%s\n", pCont); + if (pCont != NULL) { + EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + uError("failed to send monitor msg"); + } + } +} \ No newline at end of file diff --git a/source/libs/monitorfw/CMakeLists.txt b/source/libs/monitorfw/CMakeLists.txt new file mode 100644 index 0000000000..610cd63985 --- /dev/null +++ b/source/libs/monitorfw/CMakeLists.txt @@ -0,0 +1,9 @@ +aux_source_directory(src MONITOR2_SRC) +add_library(monitorfw STATIC ${MONITOR2_SRC}) +target_include_directories( + monitorfw + PUBLIC "${TD_SOURCE_DIR}/include/libs/monitorfw" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries(monitorfw os util common transport) diff --git a/source/libs/monitorfw/inc/taos_assert.h b/source/libs/monitorfw/inc/taos_assert.h new file mode 100644 index 0000000000..d3226eed26 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_assert.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#ifndef TAOS_ASSERT_H +#define TAOS_ASSERT_H + +#ifdef TAOS_ASSERT_ENABLE +#define TAOS_ASSERT(i) assert(i); +#else +#define TAOS_ASSERT(i) +#endif // TAOS_TEST + +#endif // TAOS_ASSERT_H diff --git a/source/libs/monitorfw/inc/taos_collector_registry_i.h b/source/libs/monitorfw/inc/taos_collector_registry_i.h new file mode 100644 index 0000000000..ed5bb1fe19 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_collector_registry_i.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "taos_collector_registry_t.h" + +#ifndef TAOS_COLLECTOR_REGISTRY_I_INCLUDED +#define TAOS_COLLECTOR_REGISTRY_I_INCLUDED + +int taos_collector_registry_enable_custom_process_metrics(taos_collector_registry_t *self, + const char *process_limits_path, + const char *process_stats_path); + +#endif // TAOS_COLLECTOR_REGISTRY_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_collector_registry_t.h b/source/libs/monitorfw/inc/taos_collector_registry_t.h new file mode 100644 index 0000000000..2264d18081 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_collector_registry_t.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_REGISTRY_T_H +#define TAOS_REGISTRY_T_H + +#include +#include + +// Public +#include "taos_collector_registry.h" + +// Private +#include "taos_map_t.h" +#include "taos_metric_formatter_t.h" +#include "taos_string_builder_t.h" + +struct taos_collector_registry { + const char *name; + bool disable_process_metrics; /**< Disables the collection of process metrics */ + taos_map_t *collectors; /**< Map of collectors keyed by name */ + taos_string_builder_t *string_builder; /**< Enables string building */ + taos_metric_formatter_t *metric_formatter; /**< metric formatter for metric exposition on bridge call */ + pthread_rwlock_t *lock; /**< mutex for safety against concurrent registration */ + taos_string_builder_t *out; +}; + +#endif // TAOS_REGISTRY_T_H diff --git a/source/libs/monitorfw/inc/taos_collector_t.h b/source/libs/monitorfw/inc/taos_collector_t.h new file mode 100644 index 0000000000..264e8e9ad9 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_collector_t.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_COLLECTOR_T_H +#define TAOS_COLLECTOR_T_H + +#include "taos_collector.h" +#include "taos_map_t.h" +#include "taos_string_builder_t.h" + +struct taos_collector { + const char *name; + taos_map_t *metrics; + taos_collect_fn *collect_fn; + taos_string_builder_t *string_builder; + const char *proc_limits_file_path; + const char *proc_stat_file_path; +}; + +#endif // TAOS_COLLECTOR_T_H diff --git a/source/libs/monitorfw/inc/taos_errors.h b/source/libs/monitorfw/inc/taos_errors.h new file mode 100644 index 0000000000..ee2a894df3 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_errors.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define TAOS_STDIO_CLOSE_DIR_ERROR "failed to close dir" +#define TAOS_STDIO_OPEN_DIR_ERROR "failed to open dir" +#define TAOS_METRIC_INCORRECT_TYPE "incorrect metric type" +#define TAOS_METRIC_INVALID_LABEL_NAME "invalid label name" +#define TAOS_PTHREAD_RWLOCK_DESTROY_ERROR "failed to destroy the pthread_rwlock_t*" +#define TAOS_PTHREAD_RWLOCK_INIT_ERROR "failed to initialize the pthread_rwlock_t*" +#define TAOS_PTHREAD_RWLOCK_LOCK_ERROR "failed to lock the pthread_rwlock_t*" +#define TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR "failed to unlock the pthread_rwlock_t*" +#define TAOS_REGEX_REGCOMP_ERROR "failed to compile the regular expression" +#define TAOS_REGEX_REGEXEC_ERROR "failed to execute the regular expression" diff --git a/source/libs/monitorfw/inc/taos_linked_list_i.h b/source/libs/monitorfw/inc/taos_linked_list_i.h new file mode 100644 index 0000000000..ed11a76427 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_linked_list_i.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_LIST_I_INCLUDED +#define TAOS_LIST_I_INCLUDED + +// Private +#include "taos_linked_list_t.h" + +/** + * @brief API PRIVATE Returns a pointer to a taos_linked_list + */ +taos_linked_list_t *taos_linked_list_new(void); + +/** + * @brief API PRIVATE removes all nodes from the given taos_linked_list * + */ +int taos_linked_list_purge(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Destroys a taos_linked_list + */ +int taos_linked_list_destroy(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Append an item to the back of the list + */ +int taos_linked_list_append(taos_linked_list_t *self, void *item); + +/** + * @brief API PRIVATE Push an item onto the front of the list + */ +int taos_linked_list_push(taos_linked_list_t *self, void *item); + +/** + * @brief API PRIVATE Pop the first item off of the list + */ +void *taos_linked_list_pop(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Returns the item at the head of the list or NULL if not present + */ +void *taos_linked_list_first(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Returns the item at the tail of the list or NULL if not present + */ +void *taos_linked_list_last(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Removes an item from the linked list + */ +int taos_linked_list_remove(taos_linked_list_t *self, void *item); + +/** + * @brief API PRIVATE Compares two items within a linked list + */ +taos_linked_list_compare_t taos_linked_list_compare(taos_linked_list_t *self, void *item_a, void *node_b); + +/** + * @brief API PRIVATE Get the size + */ +size_t taos_linked_list_size(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Set the free_fn member on taos_linked_list + */ +int taos_linked_list_set_free_fn(taos_linked_list_t *self, taos_linked_list_free_item_fn free_fn); + +/** + * @brief API PRIVATE Set the compare_fn member on the taos_linked_list + */ +int taos_linked_list_set_compare_fn(taos_linked_list_t *self, taos_linked_list_compare_item_fn compare_fn); + +/** + * API PRIVATE + * @brief does nothing + */ +void taos_linked_list_no_op_free(void *item); + +#endif // TAOS_LIST_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_linked_list_t.h b/source/libs/monitorfw/inc/taos_linked_list_t.h new file mode 100644 index 0000000000..ccd82ffb61 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_linked_list_t.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_LIST_T_H +#define TAOS_LIST_T_H + +#include "taos_linked_list.h" + +typedef enum { TAOS_LESS = -1, TAOS_EQUAL = 0, TAOS_GREATER = 1 } taos_linked_list_compare_t; + +/** + * @brief API PRIVATE Frees an item in a taos_linked_list_node + */ +typedef void (*taos_linked_list_free_item_fn)(void *); + +/** + * @brief API PRIVATE Compares two items within a taos_linked_list + */ +typedef taos_linked_list_compare_t (*taos_linked_list_compare_item_fn)(void *item_a, void *item_b); + +/** + * @brief API PRIVATE A struct containing a generic item, represented as a void pointer, and next, a pointer to the + * next taos_linked_list_node* + */ +typedef struct taos_linked_list_node { + struct taos_linked_list_node *next; + void *item; +} taos_linked_list_node_t; + +/** + * @brief API PRIVATE A linked list comprised of taos_linked_list_node* instances + */ +struct taos_linked_list { + taos_linked_list_node_t *head; + taos_linked_list_node_t *tail; + size_t size; + taos_linked_list_free_item_fn free_fn; + taos_linked_list_compare_item_fn compare_fn; +}; + +#endif // TAOS_LIST_T_H diff --git a/source/libs/monitorfw/inc/taos_log.h b/source/libs/monitorfw/inc/taos_log.h new file mode 100644 index 0000000000..6d6c9e1fb3 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_log.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#ifndef TAOS_LOG_H +#define TAOS_LOG_H + +#ifdef TAOS_LOG_ENABLE +#define TAOS_LOG(msg) printf("%s %s %s %s %d %s\n", __DATE__, __TIME__, __FILE__, __FUNCTION__, __LINE__, msg); +#else +#define TAOS_LOG(msg) +#endif // TAOS_LOG_ENABLE + +#endif // TAOS_LOG_H diff --git a/source/libs/monitorfw/inc/taos_map_i.h b/source/libs/monitorfw/inc/taos_map_i.h new file mode 100644 index 0000000000..808548e96a --- /dev/null +++ b/source/libs/monitorfw/inc/taos_map_i.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_MAP_I_INCLUDED +#define TAOS_MAP_I_INCLUDED + +#include "taos_map_t.h" + +taos_map_t *taos_map_new(void); + +int taos_map_set_free_value_fn(taos_map_t *self, taos_map_node_free_value_fn free_value_fn); + +void *taos_map_get(taos_map_t *self, const char *key); + +int taos_map_set(taos_map_t *self, const char *key, void *value); + +int taos_map_delete(taos_map_t *self, const char *key); + +int taos_map_destroy(taos_map_t *self); + +size_t taos_map_size(taos_map_t *self); + +taos_map_node_t *taos_map_node_new(const char *key, void *value, taos_map_node_free_value_fn free_value_fn); + +#endif // TAOS_MAP_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_map_t.h b/source/libs/monitorfw/inc/taos_map_t.h new file mode 100644 index 0000000000..6fcbda1366 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_map_t.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_MAP_T_H +#define TAOS_MAP_T_H + +#include + +// Public +#include "taos_map.h" + +// Private +#include "taos_linked_list_t.h" + +typedef void (*taos_map_node_free_value_fn)(void *); + +struct taos_map_node { + const char *key; + void *value; + taos_map_node_free_value_fn free_value_fn; +}; + +struct taos_map { + size_t size; /**< contains the size of the map */ + size_t max_size; /**< stores the current max_size */ + taos_linked_list_t *keys; /**< linked list containing containing all keys present */ + taos_linked_list_t **addrs; /**< Sequence of linked lists. Each list contains nodes with the same index */ + pthread_rwlock_t *rwlock; + taos_map_node_free_value_fn free_value_fn; +}; + +#endif // TAOS_MAP_T_H diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_i.h b/source/libs/monitorfw/inc/taos_metric_formatter_i.h new file mode 100644 index 0000000000..9a63850ea4 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_formatter_i.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_FORMATTER_I_H +#define TAOS_METRIC_FORMATTER_I_H + +// Private +#include "taos_metric_formatter_t.h" +#include "taos_metric_t.h" + +/** + * @brief API PRIVATE taos_metric_formatter constructor + */ +taos_metric_formatter_t *taos_metric_formatter_new(); + +/** + * @brief API PRIVATE taos_metric_formatter destructor + */ +int taos_metric_formatter_destroy(taos_metric_formatter_t *self); + +/** + * @brief API PRIVATE Loads the help text + */ +int taos_metric_formatter_load_help(taos_metric_formatter_t *self, const char *name, const char *help); + +/** + * @brief API PRIVATE Loads the type text + */ +int taos_metric_formatter_load_type(taos_metric_formatter_t *self, const char *name, taos_metric_type_t metric_type); + +/** + * @brief API PRIVATE Loads the formatter with a metric sample L-value + * @param name The metric name + * @param suffix The metric suffix. This is applicable to Summary and Histogram metric types. + * @param label_count The number of labels for the given metric. + * @param label_keys An array of constant strings. + * @param label_values An array of constant strings. + * + * The number of const char **and taos_label_value must be the same. + */ +int taos_metric_formatter_load_l_value(taos_metric_formatter_t *metric_formatter, const char *name, const char *suffix, + size_t label_count, const char **label_keys, const char **label_values); + +/** + * @brief API PRIVATE Loads the formatter with a metric sample + */ +int taos_metric_formatter_load_sample( + taos_metric_formatter_t *metric_formatter, taos_metric_sample_t *sample, int64_t ts); + +/** + * @brief API PRIVATE Loads a metric in the string exposition format + */ +int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, int64_t ts); + +/** + * @brief API PRIVATE Loads the given metrics + */ +int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, int64_t ts); + +/** + * @brief API PRIVATE Clear the underlying string_builder + */ +int taos_metric_formatter_clear(taos_metric_formatter_t *self); + +/** + * @brief API PRIVATE Returns the string built by taos_metric_formatter + */ +char *taos_metric_formatter_dump(taos_metric_formatter_t *metric_formatter); + +#endif // TAOS_METRIC_FORMATTER_I_H diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_t.h b/source/libs/monitorfw/inc/taos_metric_formatter_t.h new file mode 100644 index 0000000000..0d7425aa59 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_formatter_t.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_FORMATTER_T_H +#define TAOS_METRIC_FORMATTER_T_H + +#include "taos_string_builder_t.h" + +typedef struct taos_metric_formatter { + taos_string_builder_t *string_builder; + taos_string_builder_t *err_builder; +} taos_metric_formatter_t; + +#endif // TAOS_METRIC_FORMATTER_T_H diff --git a/source/libs/monitorfw/inc/taos_metric_i.h b/source/libs/monitorfw/inc/taos_metric_i.h new file mode 100644 index 0000000000..e8ae799547 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_i.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Private +#include "taos_metric_t.h" + +#ifndef TAOS_METRIC_I_INCLUDED +#define TAOS_METRIC_I_INCLUDED + +/** + * @brief API PRIVATE Returns a *taos_metric + */ +taos_metric_t *taos_metric_new(taos_metric_type_t type, const char *name, const char *help, size_t label_key_count, + const char **label_keys); + +/** + * @brief API PRIVATE Destroys a *taos_metric + */ +int taos_metric_destroy(taos_metric_t *self); + +/** + * @brief API PRIVATE takes a generic item, casts to a *taos_metric_t and destroys it + */ +int taos_metric_destroy_generic(void *item); + +/** + * @brief API Private takes a generic item, casts to a *taos_metric_t and destroys it. Discards any errors. + */ +void taos_metric_free_generic(void *item); + +#endif // TAOS_METRIC_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_metric_sample_i.h b/source/libs/monitorfw/inc/taos_metric_sample_i.h new file mode 100644 index 0000000000..b5e90f1933 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_sample_i.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "taos_metric_sample_t.h" +#include "taos_metric_t.h" + +#ifndef TAOS_METRIC_SAMPLE_I_H +#define TAOS_METRIC_SAMPLE_I_H + +/** + * @brief API PRIVATE Return a taos_metric_sample_t* + * + * @param type The type of metric sample + * @param l_value The entire left value of the metric e.g metric_name{foo="bar"} + * @param r_value A double representing the value of the sample + */ +taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value); + +/** + * @brief API PRIVATE Destroy the taos_metric_sample** + */ +int taos_metric_sample_destroy(taos_metric_sample_t *self); + +/** + * @brief API PRIVATE A taos_linked_list_free_item_fn to enable item destruction within a linked list's destructor + */ +int taos_metric_sample_destroy_generic(void *); + +/** + * @brief API PRIVATE A taos_linked_list_free_item_fn to enable item destruction within a linked list's destructor. + * + * This function ignores any errors. + */ +void taos_metric_sample_free_generic(void *gen); + +#endif // TAOS_METRIC_SAMPLE_I_H diff --git a/source/libs/monitorfw/inc/taos_metric_sample_t.h b/source/libs/monitorfw/inc/taos_metric_sample_t.h new file mode 100644 index 0000000000..59a398b938 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_sample_t.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_SAMPLE_T_H +#define TAOS_METRIC_SAMPLE_T_H + +#include "taos_metric_sample.h" +#include "taos_metric_t.h" + +struct taos_metric_sample { + taos_metric_type_t type; /**< type is the metric type for the sample */ + char *l_value; /**< l_value is the full metric name and label set represeted as a string */ + _Atomic double r_value; /**< r_value is the value of the metric sample */ +}; + +#endif // TAOS_METRIC_SAMPLE_T_H diff --git a/source/libs/monitorfw/inc/taos_metric_t.h b/source/libs/monitorfw/inc/taos_metric_t.h new file mode 100644 index 0000000000..806466528d --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_t.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_T_H +#define TAOS_METRIC_T_H + +#include + +// Public +#include "taos_metric.h" + +// Private +#include "taos_map_i.h" +#include "taos_map_t.h" +#include "taos_metric_formatter_t.h" + +/** + * @brief API PRIVATE Contains metric type constants + */ +typedef enum taos_metric_type { TAOS_COUNTER, TAOS_GAUGE, TAOS_HISTOGRAM, TAOS_SUMMARY } taos_metric_type_t; + +/** + * @brief API PRIVATE Maps metric type constants to human readable string values + */ +extern char *taos_metric_type_map[4]; + +/** + * @brief API PRIVATE An opaque struct to users containing metric metadata; one or more metric samples; and a metric + * formatter for locating metric samples and exporting metric data + */ +struct taos_metric { + taos_metric_type_t type; /**< metric_type The type of metric */ + const char *name; /**< name The name of the metric */ + const char *help; /**< help The help output for the metric */ + taos_map_t *samples; /**< samples Map comprised of samples for the given metric */ + size_t label_key_count; /**< label_keys_count The count of labe_keys*/ + taos_metric_formatter_t *formatter; /**< formatter The metric formatter */ + pthread_rwlock_t *rwlock; /**< rwlock Required for locking on certain non-atomic operations */ + const char **label_keys; /**< labels Array comprised of const char **/ +}; + +#endif // TAOS_METRIC_T_H diff --git a/source/libs/monitorfw/inc/taos_string_builder_i.h b/source/libs/monitorfw/inc/taos_string_builder_i.h new file mode 100644 index 0000000000..142ca020ba --- /dev/null +++ b/source/libs/monitorfw/inc/taos_string_builder_i.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_STRING_BUILDER_I_H +#define TAOS_STRING_BUILDER_I_H + +#include + +#include "taos_string_builder_t.h" + +/** + * API PRIVATE + * @brief Constructor for taos_string_builder + */ +taos_string_builder_t *taos_string_builder_new(void); + +/** + * API PRIVATE + * @brief Destroys a taos_string_builder* + */ +int taos_string_builder_destroy(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Adds a string + */ +int taos_string_builder_add_str(taos_string_builder_t *self, const char *str); + +/** + * API PRIVATE + * @brief Adds a char + */ +int taos_string_builder_add_char(taos_string_builder_t *self, char c); + +/** + * API PRIVATE + * @brief Clear the string + */ +int taos_string_builder_clear(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Remove data from the end + */ +int taos_string_buillder_truncate(taos_string_builder_t *self, size_t len); + +/** + * API PRIVATE + * @brief Returns the length of the string + */ +size_t taos_string_builder_len(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Returns a copy of the string. The returned string must be deallocated when no longer needed. + */ +char *taos_string_builder_dump(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Getter for str member + */ +char *taos_string_builder_str(taos_string_builder_t *self); + +#endif // TAOS_STRING_BUILDER_I_H diff --git a/source/libs/monitorfw/inc/taos_string_builder_t.h b/source/libs/monitorfw/inc/taos_string_builder_t.h new file mode 100644 index 0000000000..edd3d574fa --- /dev/null +++ b/source/libs/monitorfw/inc/taos_string_builder_t.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_STRING_BUILDER_T_H +#define TAOS_STRING_BUILDER_T_H + +struct taos_string_builder; +/** + * @brief API PRIVATE A structure with functions responsible for building a string + */ +typedef struct taos_string_builder taos_string_builder_t; + +#endif // TAOS_STRING_BUILDER_T_H diff --git a/source/libs/monitorfw/src/taos_collector.c b/source/libs/monitorfw/src/taos_collector.c new file mode 100644 index 0000000000..9d96f61b4b --- /dev/null +++ b/source/libs/monitorfw/src/taos_collector.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +// Public +#include "taos_alloc.h" +#include "taos_collector.h" +#include "taos_collector_registry.h" + +// Private +#include "taos_assert.h" +#include "taos_collector_t.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_metric_i.h" +#include "taos_string_builder_i.h" + +taos_map_t *taos_collector_default_collect(taos_collector_t *self) { return self->metrics; } + +taos_collector_t *taos_collector_new(const char *name) { + int r = 0; + taos_collector_t *self = (taos_collector_t *)taos_malloc(sizeof(taos_collector_t)); + self->name = taos_strdup(name); + self->metrics = taos_map_new(); + if (self->metrics == NULL) { + taos_collector_destroy(self); + return NULL; + } + r = taos_map_set_free_value_fn(self->metrics, &taos_metric_free_generic); + if (r) { + taos_collector_destroy(self); + return NULL; + } + self->collect_fn = &taos_collector_default_collect; + self->string_builder = taos_string_builder_new(); + if (self->string_builder == NULL) { + taos_collector_destroy(self); + return NULL; + } + self->proc_limits_file_path = NULL; + self->proc_stat_file_path = NULL; + return self; +} + +int taos_collector_destroy(taos_collector_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_map_destroy(self->metrics); + if (r) ret = r; + self->metrics = NULL; + + r = taos_string_builder_destroy(self->string_builder); + if (r) ret = r; + self->string_builder = NULL; + + taos_free((char *)self->name); + self->name = NULL; + taos_free(self); + self = NULL; + + return ret; +} + +int taos_collector_destroy_generic(void *gen) { + int r = 0; + taos_collector_t *self = (taos_collector_t *)gen; + r = taos_collector_destroy(self); + self = NULL; + return r; +} + +void taos_collector_free_generic(void *gen) { + taos_collector_t *self = (taos_collector_t *)gen; + taos_collector_destroy(self); +} + +int taos_collector_set_collect_fn(taos_collector_t *self, taos_collect_fn *fn) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->collect_fn = fn; + return 0; +} + +int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (taos_map_get(self->metrics, metric->name) != NULL) { + TAOS_LOG("metric already found in collector"); + return 1; + } + return taos_map_set(self->metrics, metric->name, metric); +} diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c new file mode 100644 index 0000000000..7efe82ad23 --- /dev/null +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +// Public +#include "taos_alloc.h" +#include "taos_collector.h" +#include "taos_collector_registry.h" + +// Private +#include "taos_assert.h" +#include "taos_collector_registry_t.h" +#include "taos_collector_t.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_metric_formatter_i.h" +#include "taos_metric_i.h" +#include "taos_metric_t.h" +#include "taos_string_builder_i.h" + +taos_collector_registry_t *TAOS_COLLECTOR_REGISTRY_DEFAULT; + +taos_collector_registry_t *taos_collector_registry_new(const char *name) { + int r = 0; + + taos_collector_registry_t *self = (taos_collector_registry_t *)taos_malloc(sizeof(taos_collector_registry_t)); + + self->disable_process_metrics = false; + + self->name = taos_strdup(name); + self->collectors = taos_map_new(); + taos_map_set_free_value_fn(self->collectors, &taos_collector_free_generic); + taos_map_set(self->collectors, "default", taos_collector_new("default")); + + self->metric_formatter = taos_metric_formatter_new(); + self->string_builder = taos_string_builder_new(); + self->out = taos_string_builder_new(); + self->lock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t)); + r = pthread_rwlock_init(self->lock, NULL); + if (r) { + TAOS_LOG("failed to initialize rwlock"); + return NULL; + } + return self; +} + +int taos_collector_registry_default_init(void) { + if (TAOS_COLLECTOR_REGISTRY_DEFAULT != NULL) return 0; + + TAOS_COLLECTOR_REGISTRY_DEFAULT = taos_collector_registry_new("default"); + //if (TAOS_COLLECTOR_REGISTRY_DEFAULT) { + // return taos_collector_registry_enable_process_metrics(TAOS_COLLECTOR_REGISTRY_DEFAULT); + //} + return 1; +} + +int taos_collector_registry_destroy(taos_collector_registry_t *self) { + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_map_destroy(self->collectors); + self->collectors = NULL; + if (r) ret = r; + + r = taos_metric_formatter_destroy(self->metric_formatter); + self->metric_formatter = NULL; + if (r) ret = r; + + r = taos_string_builder_destroy(self->string_builder); + self->string_builder = NULL; + if (r) ret = r; + + r = pthread_rwlock_destroy(self->lock); + taos_free(self->lock); + self->lock = NULL; + if (r) ret = r; + + taos_free((char *)self->name); + self->name = NULL; + + taos_free(self); + self = NULL; + + return ret; +} + +int taos_collector_registry_register_metric(taos_metric_t *metric) { + TAOS_ASSERT(metric != NULL); + + taos_collector_t *default_collector = + (taos_collector_t *)taos_map_get(TAOS_COLLECTOR_REGISTRY_DEFAULT->collectors, "default"); + + if (default_collector == NULL) { + return 1; + } + + return taos_collector_add_metric(default_collector, metric); +} + +taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metric) { + int err = taos_collector_registry_register_metric(metric); + if (err != 0) { + exit(err); + } + return metric; +} + +int taos_collector_registry_register_collector(taos_collector_registry_t *self, taos_collector_t *collector) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = pthread_rwlock_wrlock(self->lock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + return 1; + } + if (taos_map_get(self->collectors, collector->name) != NULL) { + TAOS_LOG("the given taos_collector_t* is already registered"); + int rr = pthread_rwlock_unlock(self->lock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return 1; + } + } + r = taos_map_set(self->collectors, collector->name, collector); + if (r) { + int rr = pthread_rwlock_unlock(self->lock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return r; + } + } + r = pthread_rwlock_unlock(self->lock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return 1; + } + return 0; +} + +int taos_collector_registry_validate_metric_name(taos_collector_registry_t *self, const char *metric_name) { + regex_t r; + int ret = 0; + ret = regcomp(&r, "^[a-zA-Z_:][a-zA-Z0-9_:]*$", REG_EXTENDED); + if (ret) { + TAOS_LOG(TAOS_REGEX_REGCOMP_ERROR); + regfree(&r); + return ret; + } + + ret = regexec(&r, metric_name, 0, NULL, 0); + if (ret) { + TAOS_LOG(TAOS_REGEX_REGEXEC_ERROR); + regfree(&r); + return ret; + } + regfree(&r); + return 0; +} + +const char *taos_collector_registry_bridge(taos_collector_registry_t *self, int64_t ts) { + taos_metric_formatter_clear(self->metric_formatter); + taos_metric_formatter_load_metrics(self->metric_formatter, self->collectors, ts); + char *out = taos_metric_formatter_dump(self->metric_formatter); + + int r = 0; + r = taos_string_builder_add_str(self->out, out); + if (r) return NULL; + taos_free(out); + + return taos_string_builder_str(self->out); +} + +int taos_collector_registry_clear_out(taos_collector_registry_t *self){ + return taos_string_builder_clear(self->out); +} diff --git a/source/libs/monitorfw/src/taos_counter.c b/source/libs/monitorfw/src/taos_counter.c new file mode 100644 index 0000000000..d522411b2b --- /dev/null +++ b/source/libs/monitorfw/src/taos_counter.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Public +#include "taos_counter.h" + +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_metric_i.h" +#include "taos_metric_sample_i.h" +#include "taos_metric_sample_t.h" +#include "taos_metric_t.h" + +taos_counter_t *taos_counter_new(const char *name, const char *help, size_t label_key_count, const char **label_keys) { + return (taos_counter_t *)taos_metric_new(TAOS_COUNTER, name, help, label_key_count, label_keys); +} + +int taos_counter_destroy(taos_counter_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + int r = 0; + r = taos_metric_destroy(self); + self = NULL; + return r; +} + +int taos_counter_inc(taos_counter_t *self, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_COUNTER) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_add(sample, 1.0); +} + +int taos_counter_add(taos_counter_t *self, double r_value, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_COUNTER) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_add(sample, r_value); +} diff --git a/source/libs/monitorfw/src/taos_linked_list.c b/source/libs/monitorfw/src/taos_linked_list.c new file mode 100644 index 0000000000..ab4e23ff29 --- /dev/null +++ b/source/libs/monitorfw/src/taos_linked_list.c @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_linked_list_i.h" +#include "taos_linked_list_t.h" +#include "taos_log.h" + +taos_linked_list_t *taos_linked_list_new(void) { + taos_linked_list_t *self = (taos_linked_list_t *)taos_malloc(sizeof(taos_linked_list_t)); + self->head = NULL; + self->tail = NULL; + self->free_fn = NULL; + self->compare_fn = NULL; + self->size = 0; + return self; +} + +int taos_linked_list_purge(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node = self->head; + while (node != NULL) { + taos_linked_list_node_t *next = node->next; + if (node->item != NULL) { + if (self->free_fn) { + (*self->free_fn)(node->item); + } else { + taos_free(node->item); + } + } + taos_free(node); + node = NULL; + node = next; + } + self->head = NULL; + self->tail = NULL; + self->size = 0; + return 0; +} + +int taos_linked_list_destroy(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + int ret = 0; + + r = taos_linked_list_purge(self); + if (r) ret = r; + taos_free(self); + self = NULL; + return ret; +} + +void *taos_linked_list_first(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self->head) { + return self->head->item; + } else { + return NULL; + } +} + +void *taos_linked_list_last(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self->tail) { + return self->tail->item; + } else { + return NULL; + } +} + +int taos_linked_list_append(taos_linked_list_t *self, void *item) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node = (taos_linked_list_node_t *)taos_malloc(sizeof(taos_linked_list_node_t)); + + node->item = item; + if (self->tail) { + self->tail->next = node; + } else { + self->head = node; + } + self->tail = node; + node->next = NULL; + self->size++; + return 0; +} + +int taos_linked_list_push(taos_linked_list_t *self, void *item) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node = (taos_linked_list_node_t *)taos_malloc(sizeof(taos_linked_list_node_t)); + + node->item = item; + node->next = self->head; + self->head = node; + if (self->tail == NULL) { + self->tail = node; + } + self->size++; + return 0; +} + +void *taos_linked_list_pop(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return NULL; + taos_linked_list_node_t *node = self->head; + void *item = NULL; + if (node != NULL) { + item = node->item; + self->head = node->next; + if (self->tail == node) { + self->tail = NULL; + } + if (node->item != NULL) { + if (self->free_fn) { + (*self->free_fn)(node->item); + } else { + taos_free(node->item); + } + } + node->item = NULL; + node = NULL; + self->size--; + } + return item; +} + +int taos_linked_list_remove(taos_linked_list_t *self, void *item) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node; + taos_linked_list_node_t *prev_node = NULL; + + // Locate the node + for (node = self->head; node != NULL; node = node->next) { + if (self->compare_fn) { + if ((*self->compare_fn)(node->item, item) == TAOS_EQUAL) { + break; + } + } else { + if (node->item == item) { + break; + } + } + prev_node = node; + } + + if (node == NULL) return 0; + + if (prev_node) { + prev_node->next = node->next; + } else { + self->head = node->next; + } + if (node->next == NULL) { + self->tail = prev_node; + } + + if (node->item != NULL) { + if (self->free_fn) { + (*self->free_fn)(node->item); + } else { + taos_free(node->item); + } + } + + node->item = NULL; + taos_free(node); + node = NULL; + self->size--; + return 0; +} + +taos_linked_list_compare_t taos_linked_list_compare(taos_linked_list_t *self, void *item_a, void *item_b) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->compare_fn) { + return (*self->compare_fn)(item_a, item_b); + } else { + return strcmp(item_a, item_b); + } +} + +size_t taos_linked_list_size(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + return self->size; +} + +int taos_linked_list_set_free_fn(taos_linked_list_t *self, taos_linked_list_free_item_fn free_fn) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->free_fn = free_fn; + return 0; +} + +int taos_linked_list_set_compare_fn(taos_linked_list_t *self, taos_linked_list_compare_item_fn compare_fn) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->compare_fn = compare_fn; + return 0; +} + +void taos_linked_list_no_op_free(void *item) {} diff --git a/source/libs/monitorfw/src/taos_map.c b/source/libs/monitorfw/src/taos_map.c new file mode 100644 index 0000000000..fce308f11d --- /dev/null +++ b/source/libs/monitorfw/src/taos_map.c @@ -0,0 +1,414 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_linked_list_i.h" +#include "taos_linked_list_t.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_map_t.h" + +#define TAOS_MAP_INITIAL_SIZE 32 + +static void destroy_map_node_value_no_op(void *value) {} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// taos_map_node +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +taos_map_node_t *taos_map_node_new(const char *key, void *value, taos_map_node_free_value_fn free_value_fn) { + taos_map_node_t *self = taos_malloc(sizeof(taos_map_node_t)); + self->key = taos_strdup(key); + self->value = value; + self->free_value_fn = free_value_fn; + return self; +} + +int taos_map_node_destroy(taos_map_node_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + taos_free((void *)self->key); + self->key = NULL; + if (self->value != NULL) (*self->free_value_fn)(self->value); + self->value = NULL; + taos_free(self); + self = NULL; + return 0; +} + +void taos_map_node_free(void *item) { + taos_map_node_t *map_node = (taos_map_node_t *)item; + taos_map_node_destroy(map_node); +} + +taos_linked_list_compare_t taos_map_node_compare(void *item_a, void *item_b) { + taos_map_node_t *map_node_a = (taos_map_node_t *)item_a; + taos_map_node_t *map_node_b = (taos_map_node_t *)item_b; + + return strcmp(map_node_a->key, map_node_b->key); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// taos_map +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +taos_map_t *taos_map_new() { + int r = 0; + + taos_map_t *self = (taos_map_t *)taos_malloc(sizeof(taos_map_t)); + self->size = 0; + self->max_size = TAOS_MAP_INITIAL_SIZE; + + self->keys = taos_linked_list_new(); + if (self->keys == NULL) return NULL; + + // These each key will be allocated once by taos_map_node_new and used here as well to save memory. With that said + // we will only have to deallocate each key once. That will happen on taos_map_node_destroy. + r = taos_linked_list_set_free_fn(self->keys, taos_linked_list_no_op_free); + if (r) { + taos_map_destroy(self); + return NULL; + } + + self->addrs = taos_malloc(sizeof(taos_linked_list_t) * self->max_size); + self->free_value_fn = destroy_map_node_value_no_op; + + for (int i = 0; i < self->max_size; i++) { + self->addrs[i] = taos_linked_list_new(); + r = taos_linked_list_set_free_fn(self->addrs[i], taos_map_node_free); + if (r) { + taos_map_destroy(self); + return NULL; + } + r = taos_linked_list_set_compare_fn(self->addrs[i], taos_map_node_compare); + if (r) { + taos_map_destroy(self); + return NULL; + } + } + + self->rwlock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t)); + r = pthread_rwlock_init(self->rwlock, NULL); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_INIT_ERROR); + taos_map_destroy(self); + return NULL; + } + + return self; +} + +int taos_map_destroy(taos_map_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + int ret = 0; + + r = taos_linked_list_destroy(self->keys); + if (r) ret = r; + self->keys = NULL; + + for (size_t i = 0; i < self->max_size; i++) { + r = taos_linked_list_destroy(self->addrs[i]); + if (r) ret = r; + self->addrs[i] = NULL; + } + taos_free(self->addrs); + self->addrs = NULL; + + r = pthread_rwlock_destroy(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_DESTROY_ERROR) + ret = r; + } + + taos_free(self->rwlock); + self->rwlock = NULL; + taos_free(self); + self = NULL; + + return ret; +} + +static size_t taos_map_get_index_internal(const char *key, size_t *size, size_t *max_size) { + size_t index; + size_t a = 31415, b = 27183; + for (index = 0; *key != '\0'; key++, a = a * b % (*max_size - 1)) { + index = (a * index + *key) % *max_size; + } + return index; +} + +/** + * @brief API PRIVATE hash function that returns an array index from the given key and taos_map. + * + * The algorithm is based off of Horner's method. In a simpler version, you set the return value to 0. Next, for each + * character in the string, you add the integer value of the current character to the product of the prime number and + * the current return value, set the result to the return value, then finally return the return value. + * + * In this version of the algorithm, we attempt to achieve a probabily of key to index conversion collisions to + * 1/M (with M being the max_size of the map). This optimizes dispersion and consequently, evens out the performance + * for gets and sets for each item. Instead of using a fixed prime number, we generate a coefficient for each iteration + * through the loop. + * + * Reference: + * * Algorithms in C: Third Edition by Robert Sedgewick, p579 + */ +size_t taos_map_get_index(taos_map_t *self, const char *key) { + return taos_map_get_index_internal(key, &self->size, &self->max_size); +} + +static void *taos_map_get_internal(const char *key, size_t *size, size_t *max_size, taos_linked_list_t *keys, + taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn) { + size_t index = taos_map_get_index_internal(key, size, max_size); + taos_linked_list_t *list = addrs[index]; + taos_map_node_t *temp_map_node = taos_map_node_new(key, NULL, free_value_fn); + + for (taos_linked_list_node_t *current_node = list->head; current_node != NULL; current_node = current_node->next) { + taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item; + taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, temp_map_node); + if (result == TAOS_EQUAL) { + taos_map_node_destroy(temp_map_node); + temp_map_node = NULL; + return current_map_node->value; + } + } + taos_map_node_destroy(temp_map_node); + temp_map_node = NULL; + return NULL; +} + +void *taos_map_get(taos_map_t *self, const char *key) { + TAOS_ASSERT(self != NULL); + int r = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + NULL; + } + void *payload = + taos_map_get_internal(key, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn); + r = pthread_rwlock_unlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return NULL; + } + return payload; +} + +static int taos_map_set_internal(const char *key, void *value, size_t *size, size_t *max_size, taos_linked_list_t *keys, + taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn, + bool destroy_current_value) { + taos_map_node_t *map_node = taos_map_node_new(key, value, free_value_fn); + if (map_node == NULL) return 1; + + size_t index = taos_map_get_index_internal(key, size, max_size); + taos_linked_list_t *list = addrs[index]; + for (taos_linked_list_node_t *current_node = list->head; current_node != NULL; current_node = current_node->next) { + taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item; + taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, map_node); + if (result == TAOS_EQUAL) { + if (destroy_current_value) { + free_value_fn(current_map_node->value); + current_map_node->value = NULL; + } + taos_free((char *)current_map_node->key); + current_map_node->key = NULL; + taos_free(current_map_node); + current_map_node = NULL; + current_node->item = map_node; + return 0; + } + } + taos_linked_list_append(list, map_node); + taos_linked_list_append(keys, (char *)map_node->key); + (*size)++; + return 0; +} + +int taos_map_ensure_space(taos_map_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + + if (self->size <= self->max_size / 2) { + return 0; + } + + // Increase the max size + size_t new_max = self->max_size * 2; + size_t new_size = 0; + + // Create a new list of keys + taos_linked_list_t *new_keys = taos_linked_list_new(); + if (new_keys == NULL) return 1; + + r = taos_linked_list_set_free_fn(new_keys, taos_linked_list_no_op_free); + if (r) return r; + + // Create a new array of addrs + taos_linked_list_t **new_addrs = taos_malloc(sizeof(taos_linked_list_t) * new_max); + + // Initialize the new array + for (int i = 0; i < new_max; i++) { + new_addrs[i] = taos_linked_list_new(); + r = taos_linked_list_set_free_fn(new_addrs[i], taos_map_node_free); + if (r) return r; + r = taos_linked_list_set_compare_fn(new_addrs[i], taos_map_node_compare); + if (r) return r; + } + + // Iterate through each linked-list at each memory region in the map's backbone + for (int i = 0; i < self->max_size; i++) { + // Create a new map node for each node in the linked list and insert it into the new map. Afterwards, deallocate + // the old map node + taos_linked_list_t *list = self->addrs[i]; + taos_linked_list_node_t *current_node = list->head; + while (current_node != NULL) { + taos_map_node_t *map_node = (taos_map_node_t *)current_node->item; + r = taos_map_set_internal(map_node->key, map_node->value, &new_size, &new_max, new_keys, new_addrs, + self->free_value_fn, false); + if (r) return r; + + taos_linked_list_node_t *next = current_node->next; + taos_free(current_node); + current_node = NULL; + taos_free((void *)map_node->key); + map_node->key = NULL; + taos_free(map_node); + map_node = NULL; + current_node = next; + } + // We're done deallocating each map node in the linked list, so deallocate the linked-list object + taos_free(self->addrs[i]); + self->addrs[i] = NULL; + } + // Destroy the collection of keys in the map + taos_linked_list_destroy(self->keys); + self->keys = NULL; + + // Deallocate the backbone of the map + taos_free(self->addrs); + self->addrs = NULL; + + // Update the members of the current map + self->size = new_size; + self->max_size = new_max; + self->keys = new_keys; + self->addrs = new_addrs; + + return 0; +} + +int taos_map_set(taos_map_t *self, const char *key, void *value) { + TAOS_ASSERT(self != NULL); + int r = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + return r; + } + + r = taos_map_ensure_space(self); + if (r) { + int rr = 0; + rr = pthread_rwlock_unlock(self->rwlock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return r; + } + } + r = taos_map_set_internal(key, value, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn, + true); + if (r) { + int rr = 0; + rr = pthread_rwlock_unlock(self->rwlock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return r; + } + } + r = pthread_rwlock_unlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + } + return r; +} + +static int taos_map_delete_internal(const char *key, size_t *size, size_t *max_size, taos_linked_list_t *keys, + taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn) { + int r = 0; + size_t index = taos_map_get_index_internal(key, size, max_size); + taos_linked_list_t *list = addrs[index]; + taos_map_node_t *temp_map_node = taos_map_node_new(key, NULL, free_value_fn); + + for (taos_linked_list_node_t *current_node = list->head; current_node != NULL; current_node = current_node->next) { + taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item; + taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, temp_map_node); + if (result == TAOS_EQUAL) { + r = taos_linked_list_remove(list, current_node); + if (r) return r; + + r = taos_linked_list_remove(keys, (char *)current_map_node->key); + if (r) return r; + + (*size)--; + break; + } + } + r = taos_map_node_destroy(temp_map_node); + temp_map_node = NULL; + return r; +} + +int taos_map_delete(taos_map_t *self, const char *key) { + TAOS_ASSERT(self != NULL); + int r = 0; + int ret = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + ret = r; + } + r = taos_map_delete_internal(key, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn); + if (r) ret = r; + r = pthread_rwlock_unlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + ret = r; + } + return ret; +} + +int taos_map_set_free_value_fn(taos_map_t *self, taos_map_node_free_value_fn free_value_fn) { + TAOS_ASSERT(self != NULL); + self->free_value_fn = free_value_fn; + return 0; +} + +size_t taos_map_size(taos_map_t *self) { + TAOS_ASSERT(self != NULL); + return self->size; +} diff --git a/source/libs/monitorfw/src/taos_metric.c b/source/libs/monitorfw/src/taos_metric.c new file mode 100644 index 0000000000..a9bb6d45a4 --- /dev/null +++ b/source/libs/monitorfw/src/taos_metric.c @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_metric_formatter_i.h" +#include "taos_metric_i.h" +#include "taos_metric_sample_i.h" + +char *taos_metric_type_map[4] = {"counter", "gauge", "histogram", "summary"}; + +taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name, const char *help, + size_t label_key_count, const char **label_keys) { + int r = 0; + taos_metric_t *self = (taos_metric_t *)taos_malloc(sizeof(taos_metric_t)); + self->type = metric_type; + self->name = name; + self->help = help; + + const char **k = (const char **)taos_malloc(sizeof(const char *) * label_key_count); + + for (int i = 0; i < label_key_count; i++) { + if (strcmp(label_keys[i], "le") == 0) { + TAOS_LOG(TAOS_METRIC_INVALID_LABEL_NAME); + taos_metric_destroy(self); + return NULL; + } + if (strcmp(label_keys[i], "quantile") == 0) { + TAOS_LOG(TAOS_METRIC_INVALID_LABEL_NAME); + taos_metric_destroy(self); + return NULL; + } + k[i] = taos_strdup(label_keys[i]); + } + self->label_keys = k; + self->label_key_count = label_key_count; + self->samples = taos_map_new(); + + if (metric_type == TAOS_HISTOGRAM) { + + } else { + r = taos_map_set_free_value_fn(self->samples, &taos_metric_sample_free_generic); + if (r) { + taos_metric_destroy(self); + return NULL; + } + } + + self->formatter = taos_metric_formatter_new(); + if (self->formatter == NULL) { + taos_metric_destroy(self); + return NULL; + } + self->rwlock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t)); + r = pthread_rwlock_init(self->rwlock, NULL); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_INIT_ERROR); + return NULL; + } + return self; +} + +int taos_metric_destroy(taos_metric_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_map_destroy(self->samples); + self->samples = NULL; + if (r) ret = r; + + r = taos_metric_formatter_destroy(self->formatter); + self->formatter = NULL; + if (r) ret = r; + + r = pthread_rwlock_destroy(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_DESTROY_ERROR); + ret = r; + } + + taos_free(self->rwlock); + self->rwlock = NULL; + + for (int i = 0; i < self->label_key_count; i++) { + taos_free((void *)self->label_keys[i]); + self->label_keys[i] = NULL; + } + taos_free(self->label_keys); + self->label_keys = NULL; + + taos_free(self); + self = NULL; + + return ret; +} + +int taos_metric_destroy_generic(void *item) { + int r = 0; + taos_metric_t *self = (taos_metric_t *)item; + r = taos_metric_destroy(self); + self = NULL; + return r; +} + +void taos_metric_free_generic(void *item) { + taos_metric_t *self = (taos_metric_t *)item; + taos_metric_destroy(self); +} + +taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const char **label_values) { + TAOS_ASSERT(self != NULL); + int r = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + return NULL; + } + +#define TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK() \ + r = pthread_rwlock_unlock(self->rwlock); \ + if (r) TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); \ + return NULL; + + // Get l_value + r = taos_metric_formatter_load_l_value(self->formatter, self->name, NULL, self->label_key_count, self->label_keys, + label_values); + if (r) { + TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); + } + + // This must be freed before returning + const char *l_value = taos_metric_formatter_dump(self->formatter); + if (l_value == NULL) { + TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); + } + + // Get sample + taos_metric_sample_t *sample = (taos_metric_sample_t *)taos_map_get(self->samples, l_value); + if (sample == NULL) { + sample = taos_metric_sample_new(self->type, l_value, 0.0); + r = taos_map_set(self->samples, l_value, sample); + if (r) { + TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); + } + } + pthread_rwlock_unlock(self->rwlock); + taos_free((void *)l_value); + return sample; +} + diff --git a/source/libs/monitorfw/src/taos_metric_formatter.c b/source/libs/monitorfw/src/taos_metric_formatter.c new file mode 100644 index 0000000000..235ecb2bce --- /dev/null +++ b/source/libs/monitorfw/src/taos_metric_formatter.c @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_collector_t.h" +#include "taos_linked_list_t.h" +#include "taos_map_i.h" +#include "taos_metric_formatter_i.h" +#include "taos_metric_sample_t.h" +#include "taos_metric_t.h" +#include "taos_string_builder_i.h" + +#include + +taos_metric_formatter_t *taos_metric_formatter_new() { + taos_metric_formatter_t *self = (taos_metric_formatter_t *)taos_malloc(sizeof(taos_metric_formatter_t)); + self->string_builder = taos_string_builder_new(); + if (self->string_builder == NULL) { + taos_metric_formatter_destroy(self); + return NULL; + } + self->err_builder = taos_string_builder_new(); + if (self->err_builder == NULL) { + taos_metric_formatter_destroy(self); + return NULL; + } + return self; +} + +int taos_metric_formatter_destroy(taos_metric_formatter_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_string_builder_destroy(self->string_builder); + self->string_builder = NULL; + if (r) ret = r; + + r = taos_string_builder_destroy(self->err_builder); + self->err_builder = NULL; + if (r) ret = r; + + taos_free(self); + self = NULL; + return ret; +} + +int taos_metric_formatter_load_help(taos_metric_formatter_t *self, const char *name, const char *help) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, "# HELP "); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, name); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, help); + if (r) return r; + + return taos_string_builder_add_char(self->string_builder, '\n'); +} + +int taos_metric_formatter_load_type(taos_metric_formatter_t *self, const char *name, taos_metric_type_t metric_type) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, "# TYPE "); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, name); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, taos_metric_type_map[metric_type]); + if (r) return r; + + return taos_string_builder_add_char(self->string_builder, '\n'); +} + +int taos_metric_formatter_load_l_value(taos_metric_formatter_t *self, const char *name, const char *suffix, + size_t label_count, const char **label_keys, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, name); + if (r) return r; + + if (suffix != NULL) { + r = taos_string_builder_add_char(self->string_builder, '_'); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, suffix); + if (r) return r; + } + + if (label_count == 0) return 0; + + for (int i = 0; i < label_count; i++) { + if (i == 0) { + r = taos_string_builder_add_char(self->string_builder, '{'); + if (r) return r; + } + r = taos_string_builder_add_str(self->string_builder, (const char *)label_keys[i]); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, '='); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, '"'); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, (const char *)label_values[i]); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, '"'); + if (r) return r; + + if (i == label_count - 1) { + r = taos_string_builder_add_char(self->string_builder, '}'); + if (r) return r; + } else { + r = taos_string_builder_add_char(self->string_builder, ','); + if (r) return r; + } + } + return 0; +} + +int taos_metric_formatter_load_sample(taos_metric_formatter_t *self, taos_metric_sample_t *sample, int64_t ts) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, sample->l_value); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + char buffer[50]; + sprintf(buffer, "%.17g", sample->r_value); + r = taos_string_builder_add_str(self->string_builder, buffer); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + sprintf(buffer, "%ld", ts); + r = taos_string_builder_add_str(self->string_builder, buffer); + if (r) return r; + + taos_metric_sample_set(sample, 0); + + return taos_string_builder_add_char(self->string_builder, '\n'); +} + +int taos_metric_formatter_clear(taos_metric_formatter_t *self) { + TAOS_ASSERT(self != NULL); + return taos_string_builder_clear(self->string_builder); +} + +char *taos_metric_formatter_dump(taos_metric_formatter_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + if (self == NULL) return NULL; + char *data = taos_string_builder_dump(self->string_builder); + if (data == NULL) return NULL; + r = taos_string_builder_clear(self->string_builder); + if (r) { + taos_free(data); + return NULL; + } + return data; +} + +int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, int64_t ts) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_metric_formatter_load_help(self, metric->name, metric->help); + if (r) return r; + + r = taos_metric_formatter_load_type(self, metric->name, metric->type); + if (r) return r; + + for (taos_linked_list_node_t *current_node = metric->samples->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *key = (const char *)current_node->item; + if (metric->type == TAOS_HISTOGRAM) { + + } else { + taos_metric_sample_t *sample = (taos_metric_sample_t *)taos_map_get(metric->samples, key); + if (sample == NULL) return 1; + r = taos_metric_formatter_load_sample(self, sample, ts); + if (r) return r; + } + } + return taos_string_builder_add_char(self->string_builder, '\n'); +} + +int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, int64_t ts) { + TAOS_ASSERT(self != NULL); + int r = 0; + for (taos_linked_list_node_t *current_node = collectors->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *collector_name = (const char *)current_node->item; + taos_collector_t *collector = (taos_collector_t *)taos_map_get(collectors, collector_name); + if (collector == NULL) return 1; + + taos_map_t *metrics = collector->collect_fn(collector); + if (metrics == NULL) return 1; + + for (taos_linked_list_node_t *current_node = metrics->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *metric_name = (const char *)current_node->item; + taos_metric_t *metric = (taos_metric_t *)taos_map_get(metrics, metric_name); + if (metric == NULL) return 1; + r = taos_metric_formatter_load_metric(self, metric, ts); + if (r) return r; + } + } + return r; +} diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c new file mode 100644 index 0000000000..747eebc87c --- /dev/null +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_metric_sample_i.h" +#include "taos_metric_sample_t.h" + +taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value) { + taos_metric_sample_t *self = (taos_metric_sample_t *)taos_malloc(sizeof(taos_metric_sample_t)); + self->type = type; + self->l_value = taos_strdup(l_value); + self->r_value = ATOMIC_VAR_INIT(r_value); + return self; +} + +int taos_metric_sample_destroy(taos_metric_sample_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + taos_free((void *)self->l_value); + self->l_value = NULL; + taos_free((void *)self); + self = NULL; + return 0; +} + +int taos_metric_sample_destroy_generic(void *gen) { + int r = 0; + + taos_metric_sample_t *self = (taos_metric_sample_t *)gen; + r = taos_metric_sample_destroy(self); + self = NULL; + return r; +} + +void taos_metric_sample_free_generic(void *gen) { + taos_metric_sample_t *self = (taos_metric_sample_t *)gen; + taos_metric_sample_destroy(self); +} + +int taos_metric_sample_add(taos_metric_sample_t *self, double r_value) { + TAOS_ASSERT(self != NULL); + if (r_value < 0) { + return 1; + } + _Atomic double old = atomic_load(&self->r_value); + for (;;) { + _Atomic double new = ATOMIC_VAR_INIT(old + r_value); + if (atomic_compare_exchange_weak(&self->r_value, &old, new)) { + return 0; + } + } +} + +int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { + TAOS_ASSERT(self != NULL); + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + _Atomic double old = atomic_load(&self->r_value); + for (;;) { + _Atomic double new = ATOMIC_VAR_INIT(old - r_value); + if (atomic_compare_exchange_weak(&self->r_value, &old, new)) { + return 0; + } + } +} + +int taos_metric_sample_set(taos_metric_sample_t *self, double r_value) { + if (self->type != TAOS_GAUGE && self->type != TAOS_COUNTER) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + atomic_store(&self->r_value, r_value); + return 0; +} diff --git a/source/libs/monitorfw/src/taos_string_builder.c b/source/libs/monitorfw/src/taos_string_builder.c new file mode 100644 index 0000000000..0f3940cdab --- /dev/null +++ b/source/libs/monitorfw/src/taos_string_builder.c @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_string_builder_i.h" +#include "taos_string_builder_t.h" + +// The initial size of a string created via taos_string_builder +#define TAOS_STRING_BUILDER_INIT_SIZE 32 + +// taos_string_builder_init prototype declaration +int taos_string_builder_init(taos_string_builder_t *self); + +struct taos_string_builder { + char *str; /**< the target string */ + size_t allocated; /**< the size allocated to the string in bytes */ + size_t len; /**< the length of str */ + size_t init_size; /**< the initialize size of space to allocate */ +}; + +taos_string_builder_t *taos_string_builder_new(void) { + int r = 0; + + taos_string_builder_t *self = (taos_string_builder_t *)taos_malloc(sizeof(taos_string_builder_t)); + self->init_size = TAOS_STRING_BUILDER_INIT_SIZE; + r = taos_string_builder_init(self); + if (r) { + taos_string_builder_destroy(self); + return NULL; + } + + return self; +} + +int taos_string_builder_init(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->str = (char *)taos_malloc(self->init_size); + *self->str = '\0'; + self->allocated = self->init_size; + self->len = 0; + return 0; +} + +int taos_string_builder_destroy(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + taos_free(self->str); + self->str = NULL; + taos_free(self); + self = NULL; + return 0; +} + +/** + * @brief API PRIVATE Grows the size of the string given the value we want to add + * + * The method continuously shifts left until the new size is large enough to accommodate add_len. This private method + * is called in methods that need to add one or more characters to the underlying string. + */ +static int taos_string_builder_ensure_space(taos_string_builder_t *self, size_t add_len) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (add_len == 0 || self->allocated >= self->len + add_len + 1) return 0; + while (self->allocated < self->len + add_len + 1) self->allocated <<= 1; + self->str = (char *)taos_realloc(self->str, self->allocated); + return 0; +} + +int taos_string_builder_add_str(taos_string_builder_t *self, const char *str) { + TAOS_ASSERT(self != NULL); + int r = 0; + + if (self == NULL) return 1; + if (str == NULL || *str == '\0') return 0; + + size_t len = strlen(str); + r = taos_string_builder_ensure_space(self, len); + if (r) return r; + + memcpy(self->str + self->len, str, len); + self->len += len; + self->str[self->len] = '\0'; + return 0; +} + +int taos_string_builder_add_char(taos_string_builder_t *self, char c) { + TAOS_ASSERT(self != NULL); + int r = 0; + + if (self == NULL) return 1; + r = taos_string_builder_ensure_space(self, 1); + if (r) return r; + + self->str[self->len] = c; + self->len++; + self->str[self->len] = '\0'; + return 0; +} + +int taos_string_builder_truncate(taos_string_builder_t *self, size_t len) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (len >= self->len) return 0; + + self->len = len; + self->str[self->len] = '\0'; + return 0; +} + +int taos_string_builder_clear(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + taos_free(self->str); + self->str = NULL; + return taos_string_builder_init(self); +} + +size_t taos_string_builder_len(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + return self->len; +} + +char *taos_string_builder_dump(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + // +1 to accommodate \0 + char *out = (char *)taos_malloc((self->len + 1) * sizeof(char)); + memcpy(out, self->str, self->len + 1); + return out; +} + +char *taos_string_builder_str(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + return self->str; +}