diff --git a/cmake/cmake.version b/cmake/cmake.version index 0e35fa316f..4abc854e71 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.2.3.0.alpha") + SET(TD_VER_NUMBER "3.2.4.0.alpha") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/deps/arm/dm_static/libdmodule.a b/deps/arm/dm_static/libdmodule.a index dbad0112cf..37077ef63b 100644 Binary files a/deps/arm/dm_static/libdmodule.a and b/deps/arm/dm_static/libdmodule.a differ diff --git a/deps/darwin/arm/dm_static/libdmodule.a b/deps/darwin/arm/dm_static/libdmodule.a index 2aab587b18..246b2247af 100644 Binary files a/deps/darwin/arm/dm_static/libdmodule.a and b/deps/darwin/arm/dm_static/libdmodule.a differ diff --git a/deps/darwin/x64/dm_static/libdmodule.a b/deps/darwin/x64/dm_static/libdmodule.a index 1fb6794f65..8745f57636 100644 Binary files a/deps/darwin/x64/dm_static/libdmodule.a and b/deps/darwin/x64/dm_static/libdmodule.a differ diff --git a/deps/mips/dm_static/libdmodule.a b/deps/mips/dm_static/libdmodule.a index d4b0582498..855a6a41d9 100644 Binary files a/deps/mips/dm_static/libdmodule.a and b/deps/mips/dm_static/libdmodule.a differ diff --git a/deps/x86/dm_static/libdmodule.a b/deps/x86/dm_static/libdmodule.a index 9d37818a79..6a3c0d45c2 100644 Binary files a/deps/x86/dm_static/libdmodule.a and b/deps/x86/dm_static/libdmodule.a differ diff --git a/docs/en/08-client-libraries/07-python.mdx b/docs/en/08-client-libraries/07-python.mdx index aacfd0fe53..3110afcf10 100644 --- a/docs/en/08-client-libraries/07-python.mdx +++ b/docs/en/08-client-libraries/07-python.mdx @@ -842,12 +842,12 @@ consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) In addition to native connections, the client library also supports subscriptions via websockets. -The syntax for creating a consumer is "consumer = consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer). +The syntax for creating a consumer is "consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer). ```python import taosws -consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) +consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) ``` @@ -887,13 +887,13 @@ The `poll` function is used to consume data in tmq. The parameter of the `poll` ```python while True: - res = consumer.poll(1) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() + err = message.error() if err is not None: raise err - val = res.value() + val = message.value() for block in val: print(block.fetchall()) @@ -902,16 +902,14 @@ while True: -The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. You have to handle error messages in response data. +The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. ```python while True: - res = consumer.poll(timeout=1.0) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() - if err is not None: - raise err + for block in message: for row in block: print(row) diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 1fc6fb7e67..a2e6bca46c 100755 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] [TAGS] select_list hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */ hint: - BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP + BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT select_list: select_expr [, select_expr] ... @@ -87,12 +87,13 @@ Hints are a means of user control over query optimization for individual stateme The list of currently supported Hints is as follows: -| **Hint** | **Params** | **Comment** | **Scopt** | +| **Hint** | **Params** | **Comment** | **Scope** | | :-----------: | -------------- | -------------------------- | -----------------------------------| | BATCH_SCAN | None | Batch table scan | JOIN statment for stable | | NO_BATCH_SCAN | None | Sequential table scan | JOIN statment for stable | | SORT_FOR_GROUP| None | Use sort for partition, conflict with PARTITION_FIRST | With normal column in partition by list | | PARTITION_FIRST| None | Use Partition before aggregate, conflict with SORT_FOR_GROUP | With normal column in partition by list | +| PARA_TABLES_SORT| None | When sorting the supertable rows by timestamp, No temporary disk space is used. When there are numerous tables, each with long rows, the corresponding algorithm associated with this prompt may consume a substantial amount of memory, potentially leading to an Out Of Memory (OOM) situation. | Sorting the supertable rows by timestamp | For example: @@ -100,6 +101,7 @@ For example: SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0 and a.ts = b.ts; SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1; SELECT /*+ PARTITION_FIRST() */ count(*), c1 FROM stable1 PARTITION BY c1; +SELECT /*+ PARA_TABLES_SORT() */ * from stable1 order by ts; ``` ## Lists diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index 6f2343d347..e1bf18c854 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -41,16 +41,28 @@ window_clause: { SESSION(ts_col, tol_val) | STATE_WINDOW(col) | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] + | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition + | COUNT_WINDOW(count_val[, sliding_val]) } ``` `SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically. +`EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` can be any conditional expressions supported by TDengine and can include multiple columns. + +`COUNT_WINDOW` is a counting window that is divided by a fixed number of data rows.`count_val`: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. `count_val` represents the maximum number of data rows contained in each `COUNT_WINDOW`. When the total number of data rows cannot be divided by `count_val`, the number of rows in the last window will be less than `count_val`. `sliding_val`: is a constant that represents the number of window slides, similar to `SLIDING` in `INTERVAL`. + For example, the following SQL statement creates a stream and automatically creates a supertable named `avg_vol`. The stream has a 1 minute time window that slides forward in 30 second intervals to calculate the average voltage of the meters supertable. ```sql CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); + +CREATE STREAM streams0 INTO streamt0 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9; + +CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10); ``` ## Partitions of Stream diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index f5a4789976..902e62de73 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t import Release from "/components/ReleaseV3"; +## 3.2.3.0 + + + ## 3.2.2.0 diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index e7160ab094..71dc82316e 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -856,7 +856,7 @@ taosws `Consumer` API 提供了基于 Websocket 订阅 TMQ 数据的 API。创 ```python import taosws -consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) +consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) ``` @@ -896,13 +896,13 @@ Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 flo ```python while True: - res = consumer.poll(1) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() + err = message.error() if err is not None: raise err - val = res.value() + val = message.value() for block in val: print(block.fetchall()) @@ -911,16 +911,14 @@ while True: -Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。消费者必须通过 Message 的 `error()` 方法校验返回数据的 error 信息。 +Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。 ```python while True: - res = consumer.poll(timeout=1.0) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() - if err is not None: - raise err + for block in message: for row in block: print(row) diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 2a7dff6f7d..eec947ea23 100755 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] [TAGS] select_list hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */ hint: - BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP + BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT select_list: select_expr [, select_expr] ... @@ -93,13 +93,14 @@ Hints 是用户控制单个语句查询优化的一种手段，当 Hint 不适 | NO_BATCH_SCAN | 无 | 采用顺序读表的方式 | 超级表 JOIN 语句 | | SORT_FOR_GROUP| 无 | 采用sort方式进行分组, 与PARTITION_FIRST冲突 | partition by 列表有普通列时 | | PARTITION_FIRST| 无 | 在聚合之前使用PARTITION计算分组, 与SORT_FOR_GROUP冲突 | partition by 列表有普通列时 | - +| PARA_TABLES_SORT| 无 | 超级表的数据按时间戳排序时, 不使用临时磁盘空间, 只使用内存。当子表数量多, 行长比较大时候, 会使用大量内存, 可能发生OOM | 超级表的数据按时间戳排序时 | 举例： ```sql SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0 and a.ts = b.ts; SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1; SELECT /*+ PARTITION_FIRST() */ count(*), c1 FROM stable1 PARTITION BY c1; +SELECT /*+ PARA_TABLES_SORT() */ * from stable1 order by ts; ``` ## 列表 diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 979fc436b9..8868b728f8 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -49,10 +49,14 @@ window_clause: { SESSION(ts_col, tol_val) | STATE_WINDOW(col) | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] + | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition + | COUNT_WINDOW(count_val[, sliding_val]) } ``` 其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。 +EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。 +COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished) @@ -61,6 +65,12 @@ window_clause: { ```sql CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); + +CREATE STREAM streams0 INTO streamt0 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9; + +CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10); ``` ## 流式计算的 partition diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index b0a81e01a1..1c51f934fe 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.2.3.0 + + + ## 3.2.2.0 diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index bcb40f4175..fa6e4c3ae8 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -32,6 +32,9 @@ typedef struct SBlockOrderInfo { SColumnInfoData* pColData; } SBlockOrderInfo; +#define BLOCK_VERSION_1 1 +#define BLOCK_VERSION_2 2 + #define NBIT (3u) #define BitPos(_n) ((_n) & ((1 << NBIT) - 1)) #define BMCharPos(bm_, r_) ((bm_)[(r_) >> NBIT]) @@ -264,7 +267,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pData bool alreadyAddGroupId(char* ctbName); bool isAutoTableName(char* ctbName); -void buildCtbNameAddGruopId(char* ctbName, uint64_t groupId); +void buildCtbNameAddGroupId(char* ctbName, uint64_t groupId); char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* pBuf); diff --git a/include/common/tglobal.h b/include/common/tglobal.h index ef3e61e1a2..93f17fa887 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -106,6 +106,9 @@ extern char tsMonitorFqdn[]; extern uint16_t tsMonitorPort; extern int32_t tsMonitorMaxLogs; extern bool tsMonitorComp; +extern bool tsMonitorLogProtocol; +extern int32_t tsMonitorIntervalForBasic; +extern bool tsMonitorForceV2; // audit extern bool tsEnableAudit; @@ -230,10 +233,10 @@ int32_t taosCfgDynamicOptions(SConfig *pCfg, char *name, bool forServer); struct SConfig *taosGetCfg(); -void taosSetAllDebugFlag(int32_t flag); -void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal); -void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); -int8_t taosGranted(int8_t type); +void taosSetGlobalDebugFlag(int32_t flag); +void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal); +void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); +int8_t taosGranted(int8_t type); #ifdef __cplusplus } diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 6d11608d21..ca9e030117 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -67,28 +67,28 @@ int32_t grantCheckLE(EGrantType grant); // #ifndef GRANTS_CFG #ifdef TD_ENTERPRISE -#define GRANTS_SCHEMA \ - static const SSysDbTableSchema grantsSchema[] = { \ - {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ +#define GRANTS_SCHEMA \ + static const SSysDbTableSchema grantsSchema[] = { \ + {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ } #else -#define GRANTS_SCHEMA \ - static const SSysDbTableSchema grantsSchema[] = { \ - {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ +#define GRANTS_SCHEMA \ + static const SSysDbTableSchema grantsSchema[] = { \ + {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ } #endif // #define GRANT_CFG_ADD diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 4fa244a07c..db2c51fb01 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -581,9 +581,9 @@ typedef struct { }; } SSubmitRsp; -int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); -int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); -void tFreeSSubmitBlkRsp(void* param); +// int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); +// int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); +// void tFreeSSubmitBlkRsp(void* param); void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_SMA_ON ((int8_t)0x1) @@ -885,8 +885,8 @@ typedef struct { int64_t maxStorage; } SCreateAcctReq, SAlterAcctReq; -int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); -int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); +// int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); +// int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); typedef struct { char user[TSDB_USER_LEN]; @@ -1324,14 +1324,13 @@ typedef struct { } SDnodeListReq; int32_t tSerializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq); -int32_t tDeserializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq); typedef struct { int32_t useless; // useless } SServerVerReq; int32_t tSerializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq); -int32_t tDeserializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq); +// int32_t tDeserializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq); typedef struct { char ver[TSDB_VERSION_LEN]; @@ -1582,6 +1581,15 @@ int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); void tFreeSStatusReq(SStatusReq* pReq); +typedef struct { + int32_t contLen; + char* pCont; +} SStatisReq; + +int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); +int32_t tDeserializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); +void tFreeSStatisReq(SStatisReq *pReq); + typedef struct { int32_t dnodeId; int64_t clusterId; @@ -1610,7 +1618,6 @@ typedef struct { SEp ep; char active[TSDB_ACTIVE_KEY_LEN]; char connActive[TSDB_CONN_ACTIVE_KEY_LEN]; - char machineId[TSDB_MACHINE_ID_LEN + 1]; } SDnodeInfo; typedef struct { @@ -1630,14 +1637,14 @@ typedef struct { } SMTimerReq; int32_t tSerializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq); -int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq); +// int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq); typedef struct { int64_t tick; } SMStreamTickReq; int32_t tSerializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq); -int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq); +// int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq); typedef struct { int32_t id; @@ -1908,7 +1915,7 @@ typedef struct { } SShowVariablesReq; int32_t tSerializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq); -int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq); +// int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq); typedef struct { char name[TSDB_CONFIG_OPTION_LEN + 1]; @@ -1938,7 +1945,7 @@ typedef struct { } SShowReq; int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); -int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); +// int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); void tFreeSShowReq(SShowReq* pReq); typedef struct { @@ -1946,9 +1953,9 @@ typedef struct { STableMetaRsp tableMeta; } SShowRsp, SVShowTablesRsp; -int32_t tSerializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); -int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); -void tFreeSShowRsp(SShowRsp* pRsp); +// int32_t tSerializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); +// int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); +// void tFreeSShowRsp(SShowRsp* pRsp); typedef struct { char db[TSDB_DB_FNAME_LEN]; @@ -2206,7 +2213,7 @@ typedef struct { } SForceBecomeFollowerReq; int32_t tSerializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq); -int32_t tDeserializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq); +// int32_t tDeserializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq); typedef struct { int32_t vgId; @@ -2223,8 +2230,8 @@ typedef struct { char ckey[TSDB_PASSWORD_LEN]; } SAuthReq, SAuthRsp; -int32_t tSerializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); -int32_t tDeserializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); +// int32_t tSerializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); +// int32_t tDeserializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); typedef struct { int32_t statusCode; @@ -2757,8 +2764,8 @@ typedef struct { int tEncodeSVCreateTbBatchRsp(SEncoder* pCoder, const SVCreateTbBatchRsp* pRsp); int tDecodeSVCreateTbBatchRsp(SDecoder* pCoder, SVCreateTbBatchRsp* pRsp); -int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); -int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); +// int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); +// int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); // TDMT_VND_DROP_TABLE ================= typedef struct { @@ -3213,8 +3220,8 @@ typedef struct { int32_t hasCheckPoint; } SMVStreamGatherInfoReq; -int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq); -int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq); +// int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq); +// int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq); typedef struct { int64_t leftForVer; @@ -3439,7 +3446,7 @@ int32_t tDeserializeSCreateTagIdxReq(void* buf, int32_t bufLen, SCreateTagIndexR typedef SMDropSmaReq SDropTagIndexReq; -int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); +// int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); int32_t tDeserializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); typedef struct { @@ -3491,7 +3498,7 @@ void* tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq); int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq); int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq); int32_t tEncodeSVDropTSmaReq(SEncoder* pCoder, const SVDropTSmaReq* pReq); -int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq); +// int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq); typedef struct { int32_t number; @@ -3560,8 +3567,8 @@ typedef struct { int8_t igNotExists; } SMDropFullTextReq; -int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); -int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); +// int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); +// int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); typedef struct { char indexFName[TSDB_INDEX_FNAME_LEN]; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index dc5cb24564..62392a9caa 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -215,6 +215,7 @@ TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VIEW, "create-view", SCMCreateViewReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_STATIS, "statis", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_COMPACT, "kill-compact", SKillCompactReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL) diff --git a/include/common/trow.h b/include/common/trow.h index 8332c10ed2..92713e8e63 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -289,7 +289,7 @@ int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValT */ static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) { - pBuilder->rowType = TD_ROW_TP; // default STpRow + pBuilder->rowType = pBuilder->rowType; pBuilder->sver = sver; } int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen); @@ -331,7 +331,7 @@ void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow); bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal); -int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow); +int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t rowType); bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag); diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index fc9b88340f..22a176f0bb 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -26,6 +26,9 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored); int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen); diff --git a/include/libs/monitor/clientMonitor.h b/include/libs/monitor/clientMonitor.h new file mode 100644 index 0000000000..9c0302a15f --- /dev/null +++ b/include/libs/monitor/clientMonitor.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_CLIENT_MONITOR_H +#define TDENGINE_CLIENT_MONITOR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "taos_monitor.h" +#include "thash.h" +#include "query.h" + +typedef enum SQL_RESULT_CODE { + SQL_RESULT_SUCCESS = 0, + SQL_RESULT_FAILED = 1, + SQL_RESULT_CANCEL = 2, +} SQL_RESULT_CODE; + +const char* resultStr(SQL_RESULT_CODE code); + +typedef struct { + char clusterKey[512]; + SEpSet epSet; + void* pTransporter; + taos_collector_registry_t* registry; + taos_collector_t* colector; + SHashObj* counters; +} ClientMonitor; + +void clusterMonitorInit(const char* clusterKey, SEpSet epSet, void* pTransporter); +void clusterMonitorClose(const char* clusterKey); +taos_counter_t* createClusterCounter(const char* clusterKey, const char* name, const char* help, size_t label_key_count, + const char** label_keys); +int taosClusterCounterInc(const char* clusterKey, const char* counterName, const char** label_values); +void cluster_monitor_stop(); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_CLIENT_MONITOR_H diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 91b3a54ea1..256be26999 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -76,7 +76,8 @@ typedef struct { } SMonBasicInfo; typedef struct { - float uptime; // day + //float uptime; // day + int64_t uptime; // second int8_t has_mnode; int8_t has_qnode; int8_t has_snode; @@ -100,13 +101,15 @@ typedef struct { int32_t mnode_id; char mnode_ep[TSDB_EP_LEN]; char role[MON_ROLE_LEN]; + int32_t syncState; } SMonMnodeDesc; typedef struct { char first_ep[TSDB_EP_LEN]; int32_t first_ep_dnode_id; char version[MON_VER_LEN]; - float master_uptime; // day + //float master_uptime; // day + int64_t master_uptime; //second int32_t monitor_interval; // sec int32_t dbs_total; int32_t stbs_total; @@ -125,6 +128,7 @@ typedef struct { typedef struct { int32_t dnode_id; char vnode_role[MON_ROLE_LEN]; + int32_t syncState; } SMonVnodeDesc; typedef struct { @@ -149,7 +153,7 @@ typedef struct { } SMonStbInfo; typedef struct { - uint32_t expire_time; + int64_t expire_time; int64_t timeseries_used; int64_t timeseries_total; } SMonGrantInfo; @@ -221,7 +225,9 @@ void monSetVmInfo(SMonVmInfo *pInfo); void monSetQmInfo(SMonQmInfo *pInfo); void monSetSmInfo(SMonSmInfo *pInfo); void monSetBmInfo(SMonBmInfo *pInfo); -void monSendReport(); +void monGenAndSendReport(); +void monGenAndSendReportBasic(); +void monSendContent(char *pCont); void tFreeSMonMmInfo(SMonMmInfo *pInfo); void tFreeSMonVmInfo(SMonVmInfo *pInfo); diff --git a/include/libs/monitorfw/taos_alloc.h b/include/libs/monitorfw/taos_alloc.h new file mode 100644 index 0000000000..893ffc7e9b --- /dev/null +++ b/include/libs/monitorfw/taos_alloc.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_alloc.h + * @brief memory management + */ + +#ifndef TAOS_ALLOC_H +#define TAOS_ALLOC_H + +#include +#include + +/** + * @brief Redefine this macro if you wish to override it. The default value is malloc. + */ +#define taos_malloc malloc + +/** + * @brief Redefine this macro if you wish to override it. The default value is realloc. + */ +#define taos_realloc realloc + +/** + * @brief Redefine this macro if you wish to override it. The default value is strdup. + */ +#define taos_strdup strdup + +/** + * @brief Redefine this macro if you wish to override it. The default value is free. + */ +#define taos_free free + +#endif // TAOS_ALLOC_H diff --git a/include/libs/monitorfw/taos_collector.h b/include/libs/monitorfw/taos_collector.h new file mode 100644 index 0000000000..c37cb612d2 --- /dev/null +++ b/include/libs/monitorfw/taos_collector.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_COLLECTOR_H +#define TAOS_COLLECTOR_H + +#include "taos_map.h" +#include "taos_metric.h" + +/** + * @file taos_collector.h + * @brief A Prometheus collector returns a collection of metrics + */ + +/** + * @brief A prometheus collector calls collect to prepare metrics and return them to the registry to which it is + * registered. + */ +typedef struct taos_collector taos_collector_t; + +/** + * @brief The function responsible for preparing metric data and returning metrics for a given collector. + * + * If you use the default collector registry, this should not concern you. If you are using a custom collector, you may + * set this function on your collector to do additional work before returning the contained metrics. + * + * @param self The target taos_collector_t* + * @return The taos_map_t* containing the collected metrics + */ +typedef taos_map_t *taos_collect_fn(taos_collector_t *self); + +/** + * @brief Create a collector + * @param name The name of the collector. The name MUST NOT be default or process. + * @return The constructed taos_collector_t* + */ +taos_collector_t *taos_collector_new(const char *name); + +/** + * @brief Destroy a collector. You MUST set self to NULL after destruction. + * @param self The target taos_collector_t* + * @return A non-zero integer value upon failure. + */ +int taos_collector_destroy(taos_collector_t *self); + +/** + * @brief Frees a collector passed as a void pointer. You MUST set self to NULL after destruction. + * @param gen The target taos_collector_t* represented as a void* + */ +void taos_collector_free_generic(void *gen); + +/** + * @brief Destroys a collector passed as a void pointer. You MUST set self to NULL after destruction. + * @param gen The target taos_collector_t* represented as a void* + * @return A non-zero integer value upon failure. + */ +int taos_collector_destroy_generic(void *gen); + +/** + * @brief Add a metric to a collector + * @param self The target taos_collector_t* + * @param metric the taos_metric_t* to add to the taos_collector_t* passed as self. + * @return A non-zero integer value upon failure. + */ +int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric); + +int taos_collector_remove_metric(taos_collector_t *self, const char* key); + +taos_metric_t* taos_collector_get_metric(taos_collector_t *self, char *metric_name); + +/** + * @brief The collect function is responsible for doing any work involving a set of metrics and then returning them + * for metric exposition. + * @param self The target taos_collector_t* + * @param fn The taos_collect_fn* which will be responsible for handling any metric collection operations before + * returning the collected metrics for exposition. + * @return A non-zero integer value upon failure. + */ +int taos_collector_set_collect_fn(taos_collector_t *self, taos_collect_fn *fn); + +#endif // TAOS_COLLECTOR_H diff --git a/include/libs/monitorfw/taos_collector_registry.h b/include/libs/monitorfw/taos_collector_registry.h new file mode 100644 index 0000000000..81d1f8050c --- /dev/null +++ b/include/libs/monitorfw/taos_collector_registry.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_collector_registry.h + * @brief The collector registry registers collectors for metric exposition. + */ + +#ifndef TAOS_REGISTRY_H +#define TAOS_REGISTRY_H + +#include "taos_collector.h" +#include "taos_metric.h" + +/** + * @brief A taos_registry_t is responsible for registering metrics and briding them to the string exposition format + */ +typedef struct taos_collector_registry taos_collector_registry_t; + +/** + * @brief Initialize the default registry by calling taos_collector_registry_init within your program. You MUST NOT + * modify this value. + */ +extern taos_collector_registry_t *TAOS_COLLECTOR_REGISTRY_DEFAULT; + +/** + * @brief Initializes the default collector registry and enables metric collection on the executing process + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_default_init(void); + +/** + * @brief Constructs a taos_collector_registry_t* + * @param name The name of the collector registry. It MUST NOT be default. + * @return The constructed taos_collector_registry_t* + */ +taos_collector_registry_t *taos_collector_registry_new(const char *name); + +/** + * @brief Destroy a collector registry. You MUST set self to NULL after destruction. + * @param self The target taos_collector_registry_t* + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_destroy(taos_collector_registry_t *self); + +/** + * @brief Enable process metrics on the given collector registry + * @param self The target taos_collector_registry_t* + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_enable_process_metrics(taos_collector_registry_t *self); + +/** + * @brief Registers a metric with the default collector on TAOS_DEFAULT_COLLECTOR_REGISTRY + * + * The metric to be registered MUST NOT already be registered with the given . If so, the program will + * halt. It returns a taos_metric_t* to simplify metric creation and registration. Furthermore, + * TAOS_DEFAULT_COLLECTOR_REGISTRY must be registered via taos_collector_registry_default_init() prior to calling this + * function. The metric will be added to the default registry's default collector. + * + * @param metric The metric to register on TAOS_DEFAULT_COLLECTOR_REGISTRY* + * @return The registered taos_metric_t* + */ +taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metric); + +/** + * @brief Registers a metric with the default collector on TAOS_DEFAULT_COLLECTOR_REGISTRY. Returns an non-zero integer + * value on failure. + * + * See taos_collector_registry_must_register_metric. + * + * @param metric The metric to register on TAOS_DEFAULT_COLLECTOR_REGISTRY* + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_register_metric(taos_metric_t *metric); + +int taos_collector_registry_deregister_metric(const char *key); + +taos_metric_t *taos_collector_registry_get_metric(char* metric_name); + +/** + * @brief Register a collector with the given registry. Returns a non-zero integer value on failure. + * @param self The target taos_collector_registry_t* + * @param collector The taos_collector_t* to register onto the taos_collector_registry_t* as self + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_register_collector(taos_collector_registry_t *self, taos_collector_t *collector); + +/** + * @brief Returns a string in the default metric exposition format. The string MUST be freed to avoid unnecessary heap + * memory growth. + * + * Reference: https://prometheus.io/docs/instrumenting/exposition_formats/ + * + * @param self The target taos_collector_registry_t* + * @return The string int he default metric exposition format. + */ +const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts, char *format); + +int taos_collector_registry_clear_batch(taos_collector_registry_t *self); + +const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self, char *ts, char *format, char** prom_str); + +/** + *@brief Validates that the given metric name complies with the specification: + * + * Reference: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels + * + * Returns a non-zero integer value on failure. + * + * @param self The target taos_collector_registry_t* + * @param metric_name The metric name to validate + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_validate_metric_name(taos_collector_registry_t *self, const char *metric_name); + +#endif // TAOS_H diff --git a/include/libs/monitorfw/taos_counter.h b/include/libs/monitorfw/taos_counter.h new file mode 100644 index 0000000000..28a9eed41c --- /dev/null +++ b/include/libs/monitorfw/taos_counter.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_COUNTER_H +#define TAOS_COUNTER_H + +#include + +#include "taos_metric.h" + +/** + * @file taos_counter.h + * @brief https://prometheus.io/docs/concepts/metric_types/#counter + */ + +/** + * @brief A prometheus counter. + * + * References + * * See https://prometheus.io/docs/concepts/metric_types/#counter + */ +typedef taos_metric_t taos_counter_t; + +/** + * @brief Construct a taos_counter_t* + * @param name The name of the metric + * @param help The metric description + * @param label_key_count The number of labels associated with the given metric. Pass 0 if the metric does not + * require labels. + * @param label_keys A collection of label keys. The number of keys MUST match the value passed as label_key_count. If + * no labels are required, pass NULL. Otherwise, it may be convenient to pass this value as a + * literal. + * @return The constructed taos_counter_t* + * + * *Example* + * + * // An example with labels + * taos_counter_new("foo", "foo is a counter with labels", 2, (const char**) { "one", "two" }); + * + * // An example without labels + * taos_counter_new("foo", "foo is a counter without labels", 0, NULL); + */ +taos_counter_t *taos_counter_new(const char *name, const char *help, size_t label_key_count, const char **label_keys); + +/** + * @brief Destroys a taos_counter_t*. You must set self to NULL after destruction. A non-zero integer value will be + * returned on failure. + * @param self A taos_counter_t* + * @return A non-zero integer value upon failure. + */ +int taos_counter_destroy(taos_counter_t *self); + +/** + * @brief Increment the taos_counter_t by 1. A non-zero integer value will be returned on failure. + * @param self The target taos_counter_t* + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the counter's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_counter_inc(foo_counter, (const char**) { "bar", "bang" }); + ** + * // An example without labels + * taos_counter_inc(foo_counter, NULL); + */ +int taos_counter_inc(taos_counter_t *self, const char **label_values); + +/** + * @brief Add the value to the taos_counter_t*. A non-zero integer value will be returned on failure. + * @param self The target taos_counter_t* + * @param r_value The double to add to the taos_counter_t passed as self. The value MUST be greater than or equal to 0. + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the counter's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_counter_add(foo_counter, 22, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_counter_add(foo_counter, 22, NULL); + */ +int taos_counter_add(taos_counter_t *self, double r_value, const char **label_values); + +#endif // TAOS_COUNTER_H diff --git a/include/libs/monitorfw/taos_gauge.h b/include/libs/monitorfw/taos_gauge.h new file mode 100644 index 0000000000..a33661dfe4 --- /dev/null +++ b/include/libs/monitorfw/taos_gauge.h @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_gauge.h + * @brief https://prometheus.io/docs/concepts/metric_types/#gauge + */ + +#ifndef TAOS_GAUGE_H +#define TAOS_GAUGE_H + +#include + +#include "taos_metric.h" + +/** + * @brief A prometheus gauge. + * + * References + * * See https://prometheus.io/docs/concepts/metric_types/#gauge + */ +typedef taos_metric_t taos_gauge_t; + +/** + * @brief Constructs a taos_gauge_t* + * @param name The name of the metric + * @param help The metric description + * @param label_key_count The number of labels associated with the given metric. Pass 0 if the metric does not + * require labels. + * @param label_keys A collection of label keys. The number of keys MUST match the value passed as label_key_count. If + * no labels are required, pass NULL. Otherwise, it may be convenient to pass this value as a + * literal. + * @return The constructed taos_guage_t* + * + * // An example with labels + * taos_gauge_new("foo", "foo is a gauge with labels", 2, (const char**) { "one", "two" }); + * + * // An example without labels + * taos_gauge_new("foo", "foo is a gauge without labels", 0, NULL); + */ +taos_gauge_t *taos_gauge_new(const char *name, const char *help, size_t label_key_count, const char **label_keys); + +/** + * @brief Destroys a taos_gauge_t*. You must set self to NULL after destruction. A non-zero integer value will be + * returned on failure. + * @param self The target taos_gauge_t* + * @return A non-zero integer value upon failure + */ +int taos_gauge_destroy(taos_gauge_t *self); + +/** + * @brief Increment the taos_gauge_t* by 1. + * @param self The target taos_gauger_t* + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the gauge's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure + * *Example* + * + * // An example with labels + * taos_gauge_inc(foo_gauge, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_gauge_inc(foo_gauge, NULL); + */ +int taos_gauge_inc(taos_gauge_t *self, const char **label_values); + +/** + * @brief Decrement the taos_gauge_t* by 1. + * @param self The target taos_gauger_t* + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the gauge's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * *Example* + * + * // An example with labels + * taos_gauge_dec(foo_gauge, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_gauge_dec(foo_gauge, NULL); + */ +int taos_gauge_dec(taos_gauge_t *self, const char **label_values); + +/** + * @brief Add the value to the taos_gauge_t*. + * @param self The target taos_gauge_t* + * @param r_value The double to add to the taos_gauge_t passed as self. + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the gauge's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_gauge_add(foo_gauge 22, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_gauge_add(foo_gauge, 22, NULL); + */ +int taos_gauge_add(taos_gauge_t *self, double r_value, const char **label_values); + +/** + * @brief Subtract the value to the taos_gauge. A non-zero integer value will be returned on failure. + * @param self The target taos_gauge_t* + * @param r_value The double to add to the taos_gauge_t passed as self. + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the gauge's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_gauge_sub(foo_gauge 22, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_gauge_sub(foo_gauge, 22, NULL); + */ +int taos_gauge_sub(taos_gauge_t *self, double r_value, const char **label_values); + +/** + * @brief Set the value for the taos_gauge_t* + * @param self The target taos_gauge_t* + * @param r_value The double to which the taos_gauge_t* passed as self will be set + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the gauge's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_gauge_set(foo_gauge 22, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_gauge_set(foo_gauge, 22, NULL); + */ +int taos_gauge_set(taos_gauge_t *self, double r_value, const char **label_values); + +#endif // TAOS_GAUGE_H diff --git a/include/libs/monitorfw/taos_linked_list.h b/include/libs/monitorfw/taos_linked_list.h new file mode 100644 index 0000000000..1fff5e65a6 --- /dev/null +++ b/include/libs/monitorfw/taos_linked_list.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_LIST_H +#define TAOS_LIST_H + +#include + +struct taos_linked_list; +/** + * @brief Provides a generic linked list + */ +typedef struct taos_linked_list taos_linked_list_t; + +#endif // TAOS_LIST_H diff --git a/include/libs/monitorfw/taos_map.h b/include/libs/monitorfw/taos_map.h new file mode 100644 index 0000000000..fa554d569c --- /dev/null +++ b/include/libs/monitorfw/taos_map.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_MAP_H +#define TAOS_MAP_H + +struct taos_map; +typedef struct taos_map taos_map_t; + +struct taos_map_node; +typedef struct taos_map_node taos_map_node_t; + +#endif // TAOS_MAP_H diff --git a/include/libs/monitorfw/taos_metric.h b/include/libs/monitorfw/taos_metric.h new file mode 100644 index 0000000000..a377311fa0 --- /dev/null +++ b/include/libs/monitorfw/taos_metric.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_metric.h + * @brief Functions for retrieving metric samples from metrics given an ordered set of labels + */ + +#ifndef TAOS_METRIC_H +#define TAOS_METRIC_H + +#include "taos_metric_sample.h" + +struct taos_metric; +/** + * @brief A prometheus metric. + * + * Reference: https://prometheus.io/docs/concepts/data_model + */ +typedef struct taos_metric taos_metric_t; + +/** + * @brief Returns a taos_metric_sample_t*. The order of label_values is significant. + * + * You may use this function to cache metric samples to avoid sample lookup. Metric samples are stored in a hash map + * with O(1) lookups in average case; nonethless, caching metric samples and updating them directly might be + * preferrable in performance-sensitive situations. + * + * @param self The target taos_metric_t* + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the counter's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A taos_metric_sample_t* + */ +taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const char **label_values); + +#endif // TAOS_METRIC_H diff --git a/include/libs/monitorfw/taos_metric_sample.h b/include/libs/monitorfw/taos_metric_sample.h new file mode 100644 index 0000000000..0283d56ad9 --- /dev/null +++ b/include/libs/monitorfw/taos_metric_sample.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_metric_sample.h + * @brief Functions for interfacting with metric samples directly + */ + +#ifndef TAOS_METRIC_SAMPLE_H +#define TAOS_METRIC_SAMPLE_H + +struct taos_metric_sample; +/** + * @brief Contains the specific metric and value given the name and label set + * Reference: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels + */ +typedef struct taos_metric_sample taos_metric_sample_t; + +/** + * @brief Add the r_value to the sample. The value must be greater than or equal to zero. + * @param self The target taos_metric_sample_t* + * @param r_value The double to add to taos_metric_sample_t* provided by self + * @return Non-zero integer value upon failure + */ +int taos_metric_sample_add(taos_metric_sample_t *self, double r_value); + +/** + * @brief Subtract the r_value from the sample. + * + * This operation MUST be called a sample derived from a gauge metric. + * @param self The target taos_metric_sample_t* + * @param r_value The double to subtract from the taos_metric_sample_t* provided by self + * @return Non-zero integer value upon failure + */ +int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value); + +/** + * @brief Set the r_value of the sample. + * + * This operation MUST be called on a sample derived from a gauge metric. + * @param self The target taos_metric_sample_t* + * @param r_value The double which will be set to the taos_metric_sample_t* provided by self + * @return Non-zero integer value upon failure + */ +int taos_metric_sample_set(taos_metric_sample_t *self, double r_value); + +int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, double* old_value); + +#endif // TAOS_METRIC_SAMPLE_H diff --git a/include/libs/monitorfw/taos_monitor.h b/include/libs/monitorfw/taos_monitor.h new file mode 100644 index 0000000000..c8569dd0a0 --- /dev/null +++ b/include/libs/monitorfw/taos_monitor.h @@ -0,0 +1,132 @@ +/* +Copyright 2019 DigitalOcean Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/** + * @file taos_monitor.h + * @brief Include taos_monitor.h to include the entire public API + * @mainpage Welcome to the documentation site for prometheus-client-c! + * @tableofcontents + * @section Introduction + * + * prometheus-client-c is a small suite of Prometheus client libraries targeted for the C programming language. + * In this brief tutorial you will learn how to create and register metrics, update metric samples, and expose metrics + * over HTTP. + * + * @section Creating-and-Registering-Metrics Creating and Registering Metrics + * + * prometheus-client-c supports the following metric types: + * + * * [Counter](https://prometheus.io/docs/concepts/metric_types/#counter) + * * [Gauge](https://prometheus.io/docs/concepts/metric_types/#gauge) + * * [Histogram](https://prometheus.io/docs/concepts/metric_types/#histogram) + * + * To get started using one of the metric types, declare the metric at file scope. For example: + * + * @code{.c} + * + * #incldue "taos_monitor.h" + * + * taos_counter_t *my_counter; + * + * @endcode + * + * Next, create a metric initialization function. You can create the metric and register it with the default metric + * collector registry in one chain of functions. A metric collector is responsible for collecting metrics and returning + * them. A metric collector registry is declared in global scope and contains metric collectors. More on this later... + * + * To create a metric and register it with the default metric collector registry in one shot, you may chain the metric + * constructor into the taos_collector_registry_must_register_metric function. For example: + * + * @code{.c} + * + * void foo_metric_init(void) { + * my_counter = taos_collector_registry_must_register_metric(taos_counter_new("my_counter", "counts things", 0, NULL)); + * } + * + * @endcode + * + * The first argument to taos_counter_new is the counter name. The second argument is the counter description. The third + * argument is the number of metric labels. In this case, we will only have one metric sample for this metric so we pass + * 0 to specify that no labels will be used. The 4th argument is an array of strings storing the metric labels. Since we + * have none, we pass NULL. A call to foo_metric_init within the program's main function will initialize the metrics + * for the file we just created to the default prometheus metric collector registery called + * TAOS_COLLECTOR_REGISTRY_DEFAULT + * + * @section Updating-Metric-Sample-Values Updating Metric Sample Values + * + * Now that we have a metric configured for creation and registration, we can update our metric within any of the + * functions of the file in which it was declared. For example: + * + * @code{.c} + * + * void my_lib_do_something(void) { + * printf("I did a really important thing!\n"); + * taos_counter_inc(my_counter, NULL); + * } + * @endcode + * + * This function will increment the default metric sample for my_counter. Since we are not using metric labels, we pass + * NULL as the second argument. + * + * @section Program-Initialization Program Initialization + * + * At the start of the program's main function you need to do two things: + * + * * Initialize the default metric collector registry: + * + * @code{.c} + * + * taos_collector_registry_default_init(); + * + * @endcode + * + * * For each file containing prometheus metrics, call its corresponding metric initialization function + * + * @code{.c} + * + * foo_metric_init() + * + * @endcode + * + * After initialization is complete, you may proceed to do work and update your metrics. + * + * @section Metric-Exposition-Over-HTTP Metric Exposition Over HTTP + * + * @todo Describe how to use libpromhttp to expose metrics over HTTP + * + * @section Where-To-Go-From-Here Where to Go From Here? + * + * Take a look at the [Files](https://github.internal.digitalocean.com/pages/timeseries/prometheus-client-c/files.html) + * tab in this documentation site for more information about the public API available to you. Also, you can take a look + * at the examples directory at the + * [Github repository](https://github.internal.digitalocean.com/timeseries/prometheus-client-c) for inspiration. + */ + +#ifndef TAOS_INCLUDED +#define TAOS_INCLUDED + +#include "taos_alloc.h" +#include "taos_collector.h" +#include "taos_collector_registry.h" +#include "taos_counter.h" +#include "taos_linked_list.h" +#include "taos_map.h" +#include "taos_metric.h" +#include "taos_metric_sample.h" +#include "taos_monitor_util.h" +#include "taos_gauge.h" + +#endif // TAOS_INCLUDED \ No newline at end of file diff --git a/include/libs/monitorfw/taos_monitor_util.h b/include/libs/monitorfw/taos_monitor_util.h new file mode 100644 index 0000000000..a42d1ddcef --- /dev/null +++ b/include/libs/monitorfw/taos_monitor_util.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_monitor_util.h + * @brief Functions for retrieving metric samples from metrics given an ordered set of labels + */ + +#ifndef TAOS_MONITOR_UTIL_H +#define TAOS_MONITOR_UTIL_H + +#include "taos_metric.h" +#include "tjson.h" + +void taos_monitor_split_str(char** arr, char* str, const char* del); +void taos_monitor_split_str_metric(char** arr, taos_metric_t* metric, const char* del, char** buf); +char* taos_monitor_get_metric_name(taos_metric_t* metric); + +#endif // TAOS_MONITOR_UTIL_H \ No newline at end of file diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 64ce735843..5f3761d7b7 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -56,7 +56,6 @@ extern "C" { #define STREAM_EXEC_T_RESTART_ALL_TASKS (-4) #define STREAM_EXEC_T_STOP_ALL_TASKS (-5) #define STREAM_EXEC_T_RESUME_TASK (-6) -#define STREAM_EXEC_T_UPDATE_TASK_EPSET (-7) typedef struct SStreamTask SStreamTask; typedef struct SStreamQueue SStreamQueue; @@ -783,11 +782,14 @@ bool streamTaskIsAllUpstreamClosed(SStreamTask* pTask); bool streamTaskSetSchedStatusWait(SStreamTask* pTask); int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask); int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask); -int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt, bool metaLock); +int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt); int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event); -int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event); -void streamTaskRestoreStatus(SStreamTask* pTask); + +typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param); +int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param); +int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param); +int32_t streamTaskRestoreStatus(SStreamTask* pTask); int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp, SRpcHandleInfo* pRpcInfo, int32_t taskId); diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index 9fd00cefb4..48b7b8c56f 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -48,11 +48,13 @@ void atomic_store_8(int8_t volatile *ptr, int8_t val); void atomic_store_16(int16_t volatile *ptr, int16_t val); void atomic_store_32(int32_t volatile *ptr, int32_t val); void atomic_store_64(int64_t volatile *ptr, int64_t val); +double atomic_store_double(double volatile *ptr, double val); void atomic_store_ptr(void *ptr, void *val); int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val); int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val); int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val); int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val); +double atomic_exchange_double(double volatile *ptr, int64_t val); void *atomic_exchange_ptr(void *ptr, void *val); int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval); int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval); @@ -68,6 +70,7 @@ int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val); +double atomic_fetch_add_double(double volatile *ptr, double val); void *atomic_fetch_add_ptr(void *ptr, void *val); int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val); @@ -78,6 +81,7 @@ int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val); +double atomic_fetch_sub_double(double volatile *ptr, double val); void *atomic_fetch_sub_ptr(void *ptr, void *val); int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index c0c20a0fde..3a689fc63d 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -433,7 +433,7 @@ int32_t* taosGetErrno(); //mnode-compact #define TSDB_CODE_MND_INVALID_COMPACT_ID TAOS_DEF_ERROR_CODE(0, 0x04B1) - +#define TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x04B2) // vnode // #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x @@ -757,6 +757,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_COL_QUERY_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x266D) #define TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE TAOS_DEF_ERROR_CODE(0, 0x266E) #define TSDB_CODE_PAR_ORDERBY_AMBIGUOUS TAOS_DEF_ERROR_CODE(0, 0x266F) +#define TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT TAOS_DEF_ERROR_CODE(0, 0x2670) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/include/util/tbase58.h b/include/util/tbase58.h index e1b03f8a8f..ab62e1926e 100644 --- a/include/util/tbase58.h +++ b/include/util/tbase58.h @@ -22,6 +22,9 @@ extern "C" { #endif +#define TBASE_MAX_ILEN 4096 +#define TBASE_MAX_OLEN 5653 + uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen); char *base58_encode(const uint8_t *value, int32_t vlen); diff --git a/include/util/tdef.h b/include/util/tdef.h index 3fe3214ac4..e2d1beb5a5 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -187,6 +187,8 @@ typedef enum ELogicConditionType { LOGIC_COND_TYPE_NOT, } ELogicConditionType; +#define TSDB_INT32_ID_LEN 11 + #define TSDB_NAME_DELIMITER_LEN 1 #define TSDB_UNI_LEN 24 @@ -196,6 +198,8 @@ typedef enum ELogicConditionType { // ACCOUNT is a 32 bit positive integer // this is the length of its string representation, including the terminator zero #define TSDB_ACCT_ID_LEN 11 +#define TSDB_NODE_ID_LEN 11 +#define TSDB_VGROUP_ID_LEN 11 #define TSDB_MAX_COLUMNS 4096 #define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns @@ -547,6 +551,10 @@ enum { #define VNODE_TIMEOUT_SEC 60 #define MNODE_TIMEOUT_SEC 60 +#define MONITOR_TABLENAME_LEN 200 +#define MONITOR_TAG_NAME_LEN 100 +#define MONITOR_TAG_VALUE_LEN 300 +#define MONITOR_METRIC_NAME_LEN 100 #ifdef __cplusplus } #endif diff --git a/include/util/tjson.h b/include/util/tjson.h index 6922930c13..b9ea72b4bb 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -103,6 +103,7 @@ char* tjsonToUnformattedString(const SJson* pJson); SJson* tjsonParse(const char* pStr); bool tjsonValidateJson(const char* pJson); const char* tjsonGetError(); +void tjsonDeleteItemFromObject(const SJson* pJson, const char* pName); #ifdef __cplusplus } diff --git a/include/util/tlog.h b/include/util/tlog.h index 11ac0e1fae..af92e1864e 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -120,6 +120,7 @@ void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile); #define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }} #define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", DEBUG_TRACE, uDebugFlag, __VA_ARGS__); }} #define uDebugL(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLongString("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }} +#define uInfoL(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLongString("UTL ", DEBUG_INFO, uDebugFlag, __VA_ARGS__); }} #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); } #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); } diff --git a/include/util/tqueue.h b/include/util/tqueue.h index 9f09bd2930..eb887596d0 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -72,40 +72,6 @@ struct STaosQnode { char item[]; }; -struct STaosQueue { - STaosQnode *head; - STaosQnode *tail; - STaosQueue *next; // for queue set - STaosQset *qset; // for queue set - void *ahandle; // for queue set - FItem itemFp; - FItems itemsFp; - TdThreadMutex mutex; - int64_t memOfItems; - int32_t numOfItems; - int64_t threadId; - int64_t memLimit; - int64_t itemLimit; -}; - -struct STaosQset { - STaosQueue *head; - STaosQueue *current; - TdThreadMutex mutex; - tsem_t sem; - int32_t numOfQueues; - int32_t numOfItems; -}; - -struct STaosQall { - STaosQnode *current; - STaosQnode *start; - int32_t numOfItems; - int64_t memOfItems; - int32_t unAccessedNumOfItems; - int64_t unAccessMemOfItems; -}; - STaosQueue *taosOpenQueue(); void taosCloseQueue(STaosQueue *queue); void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp); @@ -140,6 +106,8 @@ int32_t taosGetQueueNumber(STaosQset *qset); int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, SQueueInfo *qinfo); int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *qinfo); void taosResetQsetThread(STaosQset *qset, void *pItem); +void taosQueueSetThreadId(STaosQueue *pQueue, int64_t threadId); +int64_t taosQueueGetThreadId(STaosQueue *pQueue); #ifdef __cplusplus } diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index 2cf170cf04..d3ce2eb23b 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -26,6 +26,8 @@ typedef struct SScalableBf { SArray *bfArray; // array of bloom filters uint32_t growth; uint64_t numBits; + uint32_t maxBloomFilters; + int8_t status; _hash_fn_t hashFn1; _hash_fn_t hashFn2; } SScalableBf; diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index f089247859..a17c27c297 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -20,7 +20,7 @@ target_include_directories( target_link_libraries( taos INTERFACE api - PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom geometry + PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry ) if(TD_DARWIN_ARM64) @@ -61,7 +61,7 @@ target_include_directories( target_link_libraries( taos_static INTERFACE api - PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom geometry + PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry ) if(${BUILD_TEST}) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 6cc2a62236..6650d4c8b3 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -355,6 +355,7 @@ SRequestObj* acquireRequest(int64_t rid); int32_t releaseRequest(int64_t rid); int32_t removeRequest(int64_t rid); void doDestroyRequest(void* p); +int64_t removeFromMostPrevReq(SRequestObj* pRequest); char* getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); @@ -397,6 +398,8 @@ void hbRemoveAppHbMrg(SAppHbMgr** pAppHbMgr); void destroyAllRequests(SHashObj* pRequests); void stopAllRequests(SHashObj* pRequests); +SAppInstInfo* getAppInstInfo(const char* clusterKey); + // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); void hbDeregisterConn(STscObj* pTscObj, SClientHbKey connKey); @@ -433,6 +436,21 @@ void freeQueryParam(SSyncQueryParam* param); int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effeciveUser, SParseSqlRes* pRes); #endif +void clientSlowQueryMonitorInit(const char* clusterKey); +void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost); + +void clientSQLReqMonitorInit(const char* clusterKey); + +enum { + MONITORSQLTYPESELECT = 0, + MONITORSQLTYPEINSERT = 1, + MONITORSQLTYPEDELETE = 2 +}; + +void sqlReqLog(int64_t rid, bool killed, int32_t code, int8_t type); + +void clientMonitorClose(const char* clusterKey); + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 87f8c6be67..6c20813118 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -18,6 +18,7 @@ #include "clientLog.h" #include "functionMgt.h" #include "os.h" +#include "osSleep.h" #include "query.h" #include "qworker.h" #include "scheduler.h" @@ -105,7 +106,15 @@ static void deregisterRequest(SRequestObj *pRequest) { atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); reqType = SLOW_LOG_TYPE_QUERY; - } + } + } + + if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) { + sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT); + } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { + sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPESELECT); + } else if (QUERY_NODE_DELETE_STMT == pRequest->stmtType) { + sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEDELETE); } if (duration >= (tsSlowLogThreshold * 1000000UL)) { @@ -114,6 +123,7 @@ static void deregisterRequest(SRequestObj *pRequest) { taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s", taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration, pRequest->sqlstr); + SlowQueryLog(pTscObj->id, pRequest->killed, pRequest->code, duration); } } @@ -224,6 +234,7 @@ void destroyAppInst(SAppInstInfo *pAppInfo) { taosThreadMutexLock(&appInfo.mutex); + clientMonitorClose(pAppInfo->instKey); hbRemoveAppHbMrg(&pAppInfo->pAppHbMgr); taosHashRemove(appInfo.pInstMap, pAppInfo->instKey, strlen(pAppInfo->instKey)); @@ -374,6 +385,33 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); } +/// return the most previous req ref id +int64_t removeFromMostPrevReq(SRequestObj* pRequest) { + int64_t mostPrevReqRefId = pRequest->self; + SRequestObj* pTmp = pRequest; + while (pTmp->relation.prevRefId) { + pTmp = acquireRequest(pTmp->relation.prevRefId); + if (pTmp) { + mostPrevReqRefId = pTmp->self; + releaseRequest(mostPrevReqRefId); + } else { + break; + } + } + removeRequest(mostPrevReqRefId); + return mostPrevReqRefId; +} + +void destroyNextReq(int64_t nextRefId) { + if (nextRefId) { + SRequestObj* pObj = acquireRequest(nextRefId); + if (pObj) { + releaseRequest(nextRefId); + releaseRequest(nextRefId); + } + } +} + void destroySubRequests(SRequestObj *pRequest) { int32_t reqIdx = -1; SRequestObj *pReqList[16] = {NULL}; @@ -424,7 +462,7 @@ void doDestroyRequest(void *p) { uint64_t reqId = pRequest->requestId; tscTrace("begin to destroy request %" PRIx64 " p:%p", reqId, pRequest); - destroySubRequests(pRequest); + int64_t nextReqRefId = pRequest->relation.nextRefId; taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self)); @@ -460,6 +498,7 @@ void doDestroyRequest(void *p) { taosMemoryFreeClear(pRequest->sqlstr); taosMemoryFree(pRequest); tscTrace("end to destroy request %" PRIx64 " p:%p", reqId, pRequest); + destroyNextReq(nextReqRefId); } void destroyRequest(SRequestObj *pRequest) { @@ -468,7 +507,7 @@ void destroyRequest(SRequestObj *pRequest) { } taos_stop_query(pRequest); - removeRequest(pRequest->self); + removeFromMostPrevReq(pRequest); } void taosStopQueryImpl(SRequestObj *pRequest) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index aa18ef1c94..94da1e1998 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -15,6 +15,7 @@ #include "cJSON.h" #include "clientInt.h" +#include "clientMonitor.h" #include "clientLog.h" #include "command.h" #include "scheduler.h" @@ -157,6 +158,9 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas tscDebug("new app inst mgr %p, user:%s, ip:%s, port:%d", p, user, epSet.epSet.eps[0].fqdn, epSet.epSet.eps[0].port); pInst = &p; + + clientSlowQueryMonitorInit(p->instKey); + clientSQLReqMonitorInit(p->instKey); } else { ASSERTS((*pInst) && (*pInst)->pAppHbMgr, "*pInst:%p, pAppHgMgr:%p", *pInst, (*pInst) ? (*pInst)->pAppHbMgr : NULL); // reset to 0 in case of conn with duplicated user key but its user has ever been dropped. @@ -166,9 +170,19 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas taosThreadMutexUnlock(&appInfo.mutex); taosMemoryFreeClear(key); + return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType); } +SAppInstInfo* getAppInstInfo(const char* clusterKey) { + SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey)); + if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) { + return *ppAppInstInfo; + } else { + return NULL; + } +} + void freeQueryParam(SSyncQueryParam* param) { if (param == NULL) return; tsem_destroy(¶m->sem); @@ -1852,7 +1866,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i char* pStart = p + len; for (int32_t i = 0; i < numOfCols; ++i) { - int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i]; + int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i]; if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) { int32_t* offset = (int32_t*)pStart; @@ -1951,8 +1965,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int char* pStart = p; char* pStart1 = p1; for (int32_t i = 0; i < numOfCols; ++i) { - int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i]; - int32_t colLen1 = (blockVersion == 1) ? htonl(colLength1[i]) : colLength1[i]; + int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i]; + int32_t colLen1 = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength1[i]) : colLength1[i]; if (ASSERT(colLen < dataLen)) { tscError("doConvertJson error: colLen:%d >= dataLen:%d", colLen, dataLen); return TSDB_CODE_TSC_INTERNAL_ERROR; @@ -2011,7 +2025,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int } colLen1 = len; totalLen += colLen1; - colLength1[i] = (blockVersion == 1) ? htonl(len) : len; + colLength1[i] = (blockVersion == BLOCK_VERSION_1) ? htonl(len) : len; } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { len = numOfRows * sizeof(int32_t); memcpy(pStart1, pStart, len); @@ -2100,7 +2114,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 char* pStart = p; for (int32_t i = 0; i < numOfCols; ++i) { - if(blockVersion == 1){ + if(blockVersion == BLOCK_VERSION_1){ colLength[i] = htonl(colLength[i]); } if (colLength[i] >= dataLen) { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index eacad0f18f..47adb40eaa 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -279,6 +279,7 @@ void taos_close_internal(void *taos) { STscObj *pTscObj = (STscObj *)taos; tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs); + // clientMonitorClose(pTscObj->pAppInfo->instKey); taosRemoveRef(clientConnRefPool, pTscObj->id); } @@ -1253,54 +1254,34 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { } void restartAsyncQuery(SRequestObj *pRequest, int32_t code) { - int32_t reqIdx = 0; - SRequestObj *pReqList[16] = {NULL}; - SRequestObj *pUserReq = NULL; - pReqList[0] = pRequest; - uint64_t tmpRefId = 0; - SRequestObj *pTmp = pRequest; - while (pTmp->relation.prevRefId) { - tmpRefId = pTmp->relation.prevRefId; - pTmp = acquireRequest(tmpRefId); - if (pTmp) { - pReqList[++reqIdx] = pTmp; - releaseRequest(tmpRefId); - } else { - tscError("prev req ref 0x%" PRIx64 " is not there", tmpRefId); + tscInfo("restart request: %s p: %p", pRequest->sqlstr, pRequest); + SRequestObj* pUserReq = pRequest; + acquireRequest(pRequest->self); + while (pUserReq) { + if (pUserReq->self == pUserReq->relation.userRefId || pUserReq->relation.userRefId == 0) { break; - } - } - - tmpRefId = pRequest->relation.nextRefId; - while (tmpRefId) { - pTmp = acquireRequest(tmpRefId); - if (pTmp) { - tmpRefId = pTmp->relation.nextRefId; - removeRequest(pTmp->self); - releaseRequest(pTmp->self); } else { - tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId); - break; + int64_t nextRefId = pUserReq->relation.nextRefId; + releaseRequest(pUserReq->self); + if (nextRefId) { + pUserReq = acquireRequest(nextRefId); + } } } - - for (int32_t i = reqIdx; i >= 0; i--) { - destroyCtxInRequest(pReqList[i]); - if (pReqList[i]->relation.userRefId == pReqList[i]->self || 0 == pReqList[i]->relation.userRefId) { - pUserReq = pReqList[i]; - } else { - removeRequest(pReqList[i]->self); - } - } - + bool hasSubRequest = pUserReq != pRequest || pRequest->relation.prevRefId != 0; if (pUserReq) { + destroyCtxInRequest(pUserReq); pUserReq->prevCode = code; memset(&pUserReq->relation, 0, sizeof(pUserReq->relation)); } else { - tscError("user req is missing"); + tscError("User req is missing"); + removeFromMostPrevReq(pRequest); return; } - + if (hasSubRequest) + removeFromMostPrevReq(pRequest); + else + releaseRequest(pUserReq->self); doAsyncQuery(pUserReq, true); } diff --git a/source/client/src/clientSqlMonitor.c b/source/client/src/clientSqlMonitor.c new file mode 100644 index 0000000000..572af7ff55 --- /dev/null +++ b/source/client/src/clientSqlMonitor.c @@ -0,0 +1,79 @@ + /* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "clientInt.h" +#include "clientMonitor.h" +#include "clientLog.h" +#include "tglobal.h" + +const char* selectMonitorName = "taos_sql_req:count"; +const char* selectMonitorHelp = "count for select sql"; +const int selectMonitorLabelCount = 4; +const char* selectMonitorLabels[] = {"cluster_id", "sql_type", "username", "result"}; + +static const char* defaultClusterID = ""; + +void clientSQLReqMonitorInit(const char* clusterKey) { + if (!tsEnableMonitor) return; + SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey); + SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); + clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter); + createClusterCounter(clusterKey, selectMonitorName, selectMonitorHelp, selectMonitorLabelCount, selectMonitorLabels); +} + +void clientSQLReqLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int8_t type) { + const char* typeStr; + switch (type) { + case MONITORSQLTYPEDELETE: + typeStr = "delete"; + break; + case MONITORSQLTYPEINSERT: + typeStr = "insert"; + break; + default: + typeStr = "select"; + break; + } + const char* selectMonitorLabelValues[] = {defaultClusterID, typeStr, user, resultStr(result)}; + taosClusterCounterInc(clusterKey, selectMonitorName, selectMonitorLabelValues); +} + +void sqlReqLog(int64_t rid, bool killed, int32_t code, int8_t type) { + if (!tsEnableMonitor) return; + SQL_RESULT_CODE result = SQL_RESULT_SUCCESS; + if (TSDB_CODE_SUCCESS != code) { + result = SQL_RESULT_FAILED; + } + // to do Distinguish active Kill events + // else if (killed) { + // result = SQL_RESULT_CANCEL; + // } + + STscObj* pTscObj = acquireTscObj(rid); + if (pTscObj != NULL) { + if (pTscObj->pAppInfo == NULL) { + tscLog("sqlReqLog, not found pAppInfo"); + } else { + clientSQLReqLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, type); + } + releaseTscObj(rid); + } else { + tscLog("sqlReqLog, not found rid"); + } +} + +void clientMonitorClose(const char* clusterKey) { + tscLog("clientMonitorClose, key:%s", clusterKey); + clusterMonitorClose(clusterKey); +} diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index b141921a77..0270ae9657 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -876,12 +876,13 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { STaosQall* qall = taosAllocateQall(); taosReadAllQitems(pTmq->delayedTask, qall); - if (qall->numOfItems == 0) { + int32_t numOfItems = taosQallItemSize(qall); + if (numOfItems == 0) { taosFreeQall(qall); return TSDB_CODE_SUCCESS; } - tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, qall->numOfItems); + tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, numOfItems); int8_t* pTaskType = NULL; taosGetQitem(qall, (void**)&pTaskType); @@ -1009,19 +1010,8 @@ int32_t tmq_unsubscribe(tmq_t* tmq) { } taosSsleep(2); // sleep 2s for hb to send offset and rows to server - int32_t rsp; - int32_t retryCnt = 0; tmq_list_t* lst = tmq_list_new(); - while (1) { - rsp = tmq_subscribe(tmq, lst); - if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) { - break; - } else { - retryCnt++; - taosMsleep(500); - } - } - + int32_t rsp = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); return rsp; } @@ -1271,10 +1261,9 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } int32_t retryCnt = 0; - while (syncAskEp(tmq) != 0) { - if (retryCnt++ > MAX_RETRY_COUNT) { + while ((code = syncAskEp(tmq)) != 0) { + if (retryCnt++ > MAX_RETRY_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes", tmq->consumerId); - code = TSDB_CODE_MND_CONSUMER_NOT_READY; goto FAIL; } @@ -1611,17 +1600,39 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { return pRspObj; } -SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { - SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); - pRspObj->resType = RES_TYPE__TMQ; +void changeByteEndian(char* pData){ + char* p = pData; + + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | + // version: + int32_t blockVersion = *(int32_t*)p; + ASSERT(blockVersion == BLOCK_VERSION_1); + *(int32_t*)p = BLOCK_VERSION_2; + + p += sizeof(int32_t); + p += sizeof(int32_t); + p += sizeof(int32_t); + int32_t cols = *(int32_t*)p; + p += sizeof(int32_t); + p += sizeof(int32_t); + p += sizeof(uint64_t); + // check fields + p += cols * (sizeof(int8_t) + sizeof(int32_t)); + + int32_t* colLength = (int32_t*)p; + for (int32_t i = 0; i < cols; ++i) { + colLength[i] = htonl(colLength[i]); + } +} + +static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj* pRspObj) { (*numOfRows) = 0; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); pRspObj->vgId = pWrapper->vgHandle->vgId; pRspObj->resIter = -1; - memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; @@ -1633,41 +1644,44 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, } // extract the rows in this data packet for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { - SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i); - int64_t rows = htobe64(pRetrieve->numOfRows); + void* pRetrieve = taosArrayGetP(pRspObj->rsp.blockData, i); + void* rawData = NULL; + int64_t rows = 0; + // deal with compatibility + if(*(int64_t*)pRetrieve == 0){ + rawData = ((SRetrieveTableRsp*)pRetrieve)->data; + rows = htobe64(((SRetrieveTableRsp*)pRetrieve)->numOfRows); + }else if(*(int64_t*)pRetrieve == 1){ + rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data; + rows = htobe64(((SRetrieveTableRspForTmq*)pRetrieve)->numOfRows); + } + pVg->numOfRows += rows; (*numOfRows) += rows; - - if (needTransformSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable - SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); - if (schema) { + changeByteEndian(rawData); + if (needTransformSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable + SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); + if(schema){ taosArrayPush(pRspObj->rsp.blockSchema, &schema); } } } +} +SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { + SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); + pRspObj->resType = RES_TYPE__TMQ; + memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); + tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, pRspObj); return pRspObj; } SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); pRspObj->resType = RES_TYPE__TMQ_METADATA; - tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); - tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); - pRspObj->vgId = pWrapper->vgHandle->vgId; - pRspObj->resIter = -1; memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp)); - pRspObj->resInfo.totalRows = 0; - pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; - - // extract the rows in this data packet - for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { - SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i); - int64_t rows = htobe64(pRetrieve->numOfRows); - pVg->numOfRows += rows; - (*numOfRows) += rows; - } + tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, (SMqRspObj*)pRspObj); return pRspObj; } @@ -1814,7 +1828,7 @@ static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal } static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { - tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, tmq->qall->numOfItems); + tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, taosQallItemSize(tmq->qall)); while (1) { SMqRspWrapper* pRspWrapper = NULL; @@ -2122,26 +2136,19 @@ int32_t tmq_consumer_close(tmq_t* tmq) { if (tmq->status == TMQ_CONSUMER_STATUS__READY) { // if auto commit is set, commit before close consumer. Otherwise, do nothing. if (tmq->autoCommit) { - int32_t rsp = tmq_commit_sync(tmq, NULL); - if (rsp != 0) { - return rsp; + int32_t code = tmq_commit_sync(tmq, NULL); + if (code != 0) { + return code; } } taosSsleep(2); // sleep 2s for hb to send offset and rows to server - int32_t retryCnt = 0; tmq_list_t* lst = tmq_list_new(); - while (1) { - int32_t rsp = tmq_subscribe(tmq, lst); - if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) { - break; - } else { - retryCnt++; - taosMsleep(500); - } - } - + int32_t code = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); + if (code != 0) { + return code; + } } else { tscInfo("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId); } diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c new file mode 100644 index 0000000000..b41343443d --- /dev/null +++ b/source/client/src/slowQueryMonitor.c @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "clientInt.h" +#include "clientMonitor.h" +#include "clientLog.h" +#include "tglobal.h" + +const char* slowQueryName = "taos_slow_sql:count"; +const char* slowQueryHelp = "slow query log when cost over than config duration"; +const int slowQueryLabelCount = 4; +const char* slowQueryLabels[] = {"cluster_id", "username", "result", "duration"}; +static const char* defaultClusterID = ""; + +const int64_t usInSeconds = 1000 * 1000; +const int64_t msInMinutes = 60 * 1000; + +static const char* getSlowQueryLableCostDesc(int64_t cost) { + if (cost >= 1000 * usInSeconds) { + return "1000s-"; + } else if (cost >= 100 * usInSeconds) { + return "100-1000s"; + } else if (cost >= 10 * usInSeconds) { + return "10-100s"; + } else if (cost >= 3 * usInSeconds) { + return "3-10s"; + } + return "0-3s"; +} + +void clientSlowQueryMonitorInit(const char* clusterKey) { + if (!tsEnableMonitor) return; + SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey); + SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); + clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter); + createClusterCounter(clusterKey, slowQueryName, slowQueryHelp, slowQueryLabelCount, slowQueryLabels); +} + +void clientSlowQueryLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int32_t cost) { + const char* slowQueryLabelValues[] = {defaultClusterID, user, resultStr(result), getSlowQueryLableCostDesc(cost)}; + taosClusterCounterInc(clusterKey, slowQueryName, slowQueryLabelValues); +} + +void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost) { + if (!tsEnableMonitor) return; + SQL_RESULT_CODE result = SQL_RESULT_SUCCESS; + if (TSDB_CODE_SUCCESS != code) { + result = SQL_RESULT_FAILED; + } + // to do Distinguish active Kill events + // else if (killed) { + // result = SQL_RESULT_CANCEL; + // } + + STscObj* pTscObj = acquireTscObj(rid); + if (pTscObj != NULL) { + if(pTscObj->pAppInfo == NULL) { + tscLog("SlowQueryLog, not found pAppInfo"); + } else { + clientSlowQueryLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, cost); + } + releaseTscObj(rid); + } else { + tscLog("SlowQueryLog, not found rid"); + } +} diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 91f0d1eef8..7c3847e4a1 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -23,6 +23,12 @@ TARGET_LINK_LIBRARIES( PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom geometry ) +ADD_EXECUTABLE(clientMonitorTest clientMonitorTests.cpp) +TARGET_LINK_LIBRARIES( + clientMonitorTest + PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor +) + TARGET_INCLUDE_DIRECTORIES( clientTest PUBLIC "${TD_SOURCE_DIR}/include/client/" @@ -41,7 +47,18 @@ TARGET_INCLUDE_DIRECTORIES( PRIVATE "${TD_SOURCE_DIR}/source/client/inc" ) +TARGET_INCLUDE_DIRECTORIES( + clientMonitorTest + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" +) + add_test( NAME smlTest COMMAND smlTest ) + +# add_test( +# NAME clientMonitorTest +# COMMAND clientMonitorTest +# ) diff --git a/source/client/test/clientMonitorTests.cpp b/source/client/test/clientMonitorTests.cpp new file mode 100644 index 0000000000..f7ddc1a5cd --- /dev/null +++ b/source/client/test/clientMonitorTests.cpp @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "clientInt.h" +#include "clientMonitor.h" +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "executor.h" +#include "taos.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(clientMonitorTest, monitorTest) { + const char* cluster1 = "cluster1"; + const char* cluster2 = "cluster2"; + SEpSet epSet; + clusterMonitorInit(cluster1, epSet, NULL); + const char* counterName1 = "slow_query"; + const char* counterName2 = "select_count"; + const char* help1 = "test for slowQuery"; + const char* help2 = "test for selectSQL"; + const char* lables[] = {"lable1"}; + taos_counter_t* c1 = createClusterCounter(cluster1, counterName1, help1, 1, lables); + ASSERT_TRUE(c1 != NULL); + taos_counter_t* c2 = createClusterCounter(cluster1, counterName2, help2, 1, lables); + ASSERT_TRUE(c2 != NULL); + ASSERT_TRUE(c1 != c2); + taos_counter_t* c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables); + ASSERT_TRUE(c21 == NULL); + clusterMonitorInit(cluster2, epSet, NULL); + c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables); + ASSERT_TRUE(c21 != NULL); + int i = 0; + while (i < 12) { + taosMsleep(10); + ++i; + } + clusterMonitorClose(cluster1); + clusterMonitorClose(cluster2); +} + +TEST(clientMonitorTest, sendTest) { + TAOS* taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); + ASSERT_TRUE(taos != NULL); + printf("connect taosd sucessfully.\n"); + + int64_t rid = *(int64_t *)taos; + SlowQueryLog(rid, false, -1, 1000); + int i = 0; + while (i < 20) { + SlowQueryLog(rid, false, 0, i * 1000); + taosMsleep(10); + ++i; + } + + taos_close(taos); +} diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 47eac317ec..25cc5d7c79 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -223,12 +223,12 @@ static const SSysDbTableSchema userTblDistSchema[] = { }; static const SSysDbTableSchema userUsersSchema[] = { - {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false}, - {.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false}, - {.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, - {.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, + {.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, + {.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, + {.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; GRANTS_SCHEMA; @@ -276,7 +276,7 @@ static const SSysDbTableSchema configSchema[] = { }; static const SSysDbTableSchema variablesSchema[] = { - {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "value", .bytes = TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "scope", .bytes = TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, @@ -304,7 +304,7 @@ static const SSysDbTableSchema subscriptionSchema[] = { static const SSysDbTableSchema vnodesSchema[] = { {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = true}, {.name = "status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, @@ -312,12 +312,12 @@ static const SSysDbTableSchema vnodesSchema[] = { }; static const SSysDbTableSchema userUserPrivilegesSchema[] = { - {.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "notes", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "notes", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema userViewsSchema[] = { @@ -349,21 +349,21 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = { }; static const SSysDbTableSchema useGrantsFullSchema[] = { - {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema useGrantsLogsSchema[] = { - {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema useMachinesSchema[] = { - {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysTableMeta infosMeta[] = { @@ -395,9 +395,9 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false}, {TSDB_INS_TABLE_COMPACTS, userCompactsSchema, tListLen(userCompactsSchema), false}, {TSDB_INS_TABLE_COMPACT_DETAILS, userCompactsDetailSchema, tListLen(userCompactsDetailSchema), false}, - {TSDB_INS_TABLE_GRANTS_FULL, useGrantsFullSchema, tListLen(useGrantsFullSchema), false}, - {TSDB_INS_TABLE_GRANTS_LOGS, useGrantsLogsSchema, tListLen(useGrantsLogsSchema), false}, - {TSDB_INS_TABLE_MACHINES, useMachinesSchema, tListLen(useMachinesSchema), false}, + {TSDB_INS_TABLE_GRANTS_FULL, useGrantsFullSchema, tListLen(useGrantsFullSchema), true}, + {TSDB_INS_TABLE_GRANTS_LOGS, useGrantsLogsSchema, tListLen(useGrantsLogsSchema), true}, + {TSDB_INS_TABLE_MACHINES, useMachinesSchema, tListLen(useMachinesSchema), true}, }; static const SSysDbTableSchema connectionsSchema[] = { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 9439c172c4..f4455be206 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -452,20 +452,21 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p } if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + int32_t newLen = pSource->varmeta.length; memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); - if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { - char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); + if (pColumnInfoData->varmeta.allocLen < newLen) { + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newLen); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } pColumnInfoData->pData = tmp; - pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; + pColumnInfoData->varmeta.allocLen = newLen; } - pColumnInfoData->varmeta.length = pSource->varmeta.length; + pColumnInfoData->varmeta.length = newLen; if (pColumnInfoData->pData != NULL && pSource->pData != NULL) { - memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); + memcpy(pColumnInfoData->pData, pSource->pData, newLen); } } else { memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows)); @@ -1687,7 +1688,29 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) { } static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { + if (n >= total || n == 0) return; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + if (pColInfoData->varmeta.length != 0) { + int32_t newLen = pColInfoData->varmeta.offset[n]; + if (-1 == newLen) { + for (int i = n - 1; i >= 0; --i) { + newLen = pColInfoData->varmeta.offset[i]; + if (newLen != -1) { + if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { + newLen += getJsonValueLen(pColInfoData->pData + newLen); + } else { + newLen += varDataTLen(pColInfoData->pData + newLen); + } + break; + } + } + } + if (newLen <= -1) { + uFatal("colDataKeepFirstNRows: newLen:%d old:%d", newLen, pColInfoData->varmeta.length); + } else { + pColInfoData->varmeta.length = newLen; + } + } // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); } @@ -2118,28 +2141,28 @@ _end: return TSDB_CODE_SUCCESS; } -void buildCtbNameAddGruopId(char* ctbName, uint64_t groupId){ +void buildCtbNameAddGroupId(char* ctbName, uint64_t groupId){ char tmp[TSDB_TABLE_NAME_LEN] = {0}; snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%"PRIu64, groupId); ctbName[TSDB_TABLE_NAME_LEN - strlen(tmp) - 1] = 0; // put groupId to the end strcat(ctbName, tmp); } -bool isAutoTableName(char* ctbName){ - return (strlen(ctbName) == 34 && ctbName[0] == 't' && ctbName[1] == '_'); -} +// auto stream subtable name starts with 't_', followed by the first segment of MD5 digest for group vals. +// the total length is fixed to be 34 bytes. +bool isAutoTableName(char* ctbName) { return (strlen(ctbName) == 34 && ctbName[0] == 't' && ctbName[1] == '_'); } -bool alreadyAddGroupId(char* ctbName){ +bool alreadyAddGroupId(char* ctbName) { size_t len = strlen(ctbName); size_t _location = len - 1; - while(_location > 0){ - if(ctbName[_location] < '0' || ctbName[_location] > '9'){ + while (_location > 0) { + if (ctbName[_location] < '0' || ctbName[_location] > '9') { break; } _location--; } - return ctbName[_location] == '_' && len - 1 - _location > 15; //15 means the min length of groupid + return ctbName[_location] == '_' && len - 1 - _location >= 15; // 15 means the min length of groupid } char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { @@ -2196,7 +2219,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { // todo extract method int32_t* version = (int32_t*)data; - *version = 2; + *version = BLOCK_VERSION_1; data += sizeof(int32_t); int32_t* actualLen = (int32_t*)data; @@ -2277,7 +2300,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { data += colSizes[col]; } -// colSizes[col] = htonl(colSizes[col]); + colSizes[col] = htonl(colSizes[col]); // uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type, // htonl(colSizes[col]), colSizes[col]); } @@ -2342,9 +2365,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { pStart += sizeof(int32_t) * numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { - if(version == 1){ - colLen[i] = htonl(colLen[i]); - } + colLen[i] = htonl(colLen[i]); ASSERT(colLen[i] >= 0); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index ee85a909e7..55df80ca44 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -94,6 +94,9 @@ char tsMonitorFqdn[TSDB_FQDN_LEN] = {0}; uint16_t tsMonitorPort = 6043; int32_t tsMonitorMaxLogs = 100; bool tsMonitorComp = false; +bool tsMonitorLogProtocol = false; +int32_t tsMonitorIntervalForBasic = 30; +bool tsMonitorForceV2 = true; // audit bool tsEnableAudit = true; @@ -535,6 +538,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { return -1; if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; return 0; } @@ -693,6 +698,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; @@ -1081,6 +1090,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsQueryMaxConcurrentTables = cfgGetItem(pCfg, "queryMaxConcurrentTables")->i64; tsMetaCacheMaxSize = cfgGetItem(pCfg, "metaCacheMaxSize")->i32; tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32; + tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; + tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) { return -1; } @@ -1155,6 +1166,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32; tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval; tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32; + tsMonitorLogProtocol = cfgGetItem(pCfg, "monitorLogProtocol")->bval; + tsMonitorIntervalForBasic = cfgGetItem(pCfg, "monitorIntervalForBasic")->i32; + tsMonitorForceV2 = cfgGetItem(pCfg, "monitorForceV2")->i32; tsEnableAudit = cfgGetItem(pCfg, "audit")->bval; tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval; @@ -1244,6 +1258,8 @@ static int32_t taosSetReleaseCfg(SConfig *pCfg) { return 0; } int32_t taosSetReleaseCfg(SConfig *pCfg); #endif +static void taosSetAllDebugFlag(SConfig *pCfg, int32_t flag); + int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { if (tsCfg == NULL) osDefaultInit(); @@ -1288,7 +1304,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi taosSetServerLogCfg(pCfg); } - taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32); + taosSetAllDebugFlag(pCfg, cfgGetItem(pCfg, "debugFlag")->i32); if (taosMulModeMkDir(tsLogDir, 0777, true) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -1366,7 +1382,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile taosSetSystemCfg(tsCfg); if (taosSetFileHandlesLimit() != 0) return -1; - taosSetAllDebugFlag(cfgGetItem(tsCfg, "debugFlag")->i32); + taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32); cfgDumpCfg(tsCfg, tsc, false); @@ -1459,7 +1475,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { } if (strncasecmp(name, "debugFlag", 9) == 0) { - taosSetAllDebugFlag(pItem->i32); + taosSetAllDebugFlag(pCfg, pItem->i32); return 0; } @@ -1533,7 +1549,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) { switch (lowcaseName[0]) { case 'd': { if (strcasecmp("debugFlag", name) == 0) { - taosSetAllDebugFlag(pItem->i32); + taosSetAllDebugFlag(pCfg, pItem->i32); matched = true; } break; @@ -1757,11 +1773,13 @@ static void taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag taosSetDebugFlag(pFlagPtr, name, flag); } -void taosSetAllDebugFlag(int32_t flag) { +void taosSetGlobalDebugFlag(int32_t flag) { taosSetAllDebugFlag(tsCfg, flag); } + +static void taosSetAllDebugFlag(SConfig *pCfg, int32_t flag) { if (flag <= 0) return; SArray *noNeedToSetVars = NULL; - SConfigItem *pItem = cfgGetItem(tsCfg, "debugFlag"); + SConfigItem *pItem = cfgGetItem(pCfg, "debugFlag"); if (pItem != NULL) { pItem->i32 = flag; noNeedToSetVars = pItem->array; @@ -1811,4 +1829,4 @@ int8_t taosGranted(int8_t type) { break; } return 0; -} \ No newline at end of file +} diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 467300352c..9138d7c983 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1009,19 +1009,19 @@ int32_t tDeserializeSCreateTagIdxReq(void *buf, int32_t bufLen, SCreateTagIndexR tDecoderClear(&decoder); return 0; } -int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; - tEndEncode(&encoder); +// int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); +// if (tStartEncode(&encoder) < 0) return -1; +// tEndEncode(&encoder); - if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; - if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -1035,6 +1035,7 @@ int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq * return 0; } + int32_t tSerializeSMCreateFullTextReq(void *buf, int32_t bufLen, SMCreateFullTextReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -1059,32 +1060,32 @@ void tFreeSMCreateFullTextReq(SMCreateFullTextReq *pReq) { // impl later return; } -int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; +// if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; - if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; - tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} -int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; +// tEndEncode(&encoder); +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } +// int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; +// if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) { SEncoder encoder = {0}; @@ -1436,19 +1437,15 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) { void tFreeSStatusRsp(SStatusRsp *pRsp) { taosArrayDestroy(pRsp->pDnodeEps); } -int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { +int32_t tSerializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1; - if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1; - if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1; + + if (tEncodeI32(&encoder, pReq->contLen) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->pCont) < 0) return -1; + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1456,25 +1453,67 @@ int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq return tlen; } -int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { +int32_t tDeserializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1; - if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1; - tEndDecode(&decoder); + if (tDecodeI32(&decoder, &pReq->contLen) < 0) return -1; + if (pReq->contLen > 0) { + pReq->pCont = taosMemoryMalloc(pReq->contLen + 1); + if (pReq->pCont == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->pCont) < 0) return -1; + } + + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; } +void tFreeSStatisReq(SStatisReq *pReq) { + taosMemoryFreeClear(pReq->pCont); +} + +// int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); + +// if (tStartEncode(&encoder) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1; +// if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1; +// tEndEncode(&encoder); + +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } + +// int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); + +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1; +// if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1; +// tEndDecode(&decoder); + +// tDecoderClear(&decoder); +// return 0; +// } + int32_t tSerializeSDropUserReq(void *buf, int32_t bufLen, SDropUserReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3280,18 +3319,6 @@ int32_t tSerializeSDnodeListReq(void *buf, int32_t bufLen, SDnodeListReq *pReq) return tlen; } -int32_t tDeserializeSDnodeListReq(void *buf, int32_t bufLen, SDnodeListReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->rowNum) < 0) return -1; - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; -} - int32_t tSerializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3305,17 +3332,17 @@ int32_t tSerializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) return tlen; } -int32_t tDeserializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSServerVerRsp(void *buf, int32_t bufLen, SServerVerRsp *pRsp) { SEncoder encoder = {0}; @@ -4233,17 +4260,17 @@ int32_t tSerializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq return tlen; } -int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) { if (tEncodeCStr(pEncoder, pInfo->name) < 0) return -1; @@ -4325,24 +4352,24 @@ int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { return tlen; } -int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->type) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1; - if (pReq->payloadLen > 0) { - pReq->payload = taosMemoryMalloc(pReq->payloadLen); - if (pReq->payload == NULL) return -1; - if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1; - } +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->type) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1; +// if (pReq->payloadLen > 0) { +// pReq->payload = taosMemoryMalloc(pReq->payloadLen); +// if (pReq->payload == NULL) return -1; +// if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1; +// } - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } void tFreeSShowReq(SShowReq *pReq) { taosMemoryFreeClear(pReq->payload); } @@ -4595,34 +4622,34 @@ void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) { taosArrayDestroy(pRsp->pIndexRsp); } -int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI64(&encoder, pRsp->showId) < 0) return -1; - if (tEncodeSTableMetaRsp(&encoder, &pRsp->tableMeta) < 0) return -1; - tEndEncode(&encoder); +// if (tStartEncode(&encoder) < 0) return -1; +// if (tEncodeI64(&encoder, pRsp->showId) < 0) return -1; +// if (tEncodeSTableMetaRsp(&encoder, &pRsp->tableMeta) < 0) return -1; +// tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } -int32_t tDeserializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI64(&decoder, &pRsp->showId) < 0) return -1; - if (tDecodeSTableMetaRsp(&decoder, &pRsp->tableMeta) < 0) return -1; +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI64(&decoder, &pRsp->showId) < 0) return -1; +// if (tDecodeSTableMetaRsp(&decoder, &pRsp->tableMeta) < 0) return -1; - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } -void tFreeSShowRsp(SShowRsp *pRsp) { tFreeSTableMetaRsp(&pRsp->tableMeta); } +// void tFreeSShowRsp(SShowRsp *pRsp) { tFreeSTableMetaRsp(&pRsp->tableMeta); } int32_t tSerializeSTableInfoReq(void *buf, int32_t bufLen, STableInfoReq *pReq) { int32_t headLen = sizeof(SMsgHead); @@ -4930,17 +4957,17 @@ int32_t tSerializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) { return tlen; } -int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1; - tEndDecode(&decoder); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1; +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) { SEncoder encoder = {0}; @@ -4955,17 +4982,17 @@ int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pR return tlen; } -int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1; - tEndDecode(&decoder); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1; +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } int32_t tEncodeSReplica(SEncoder *pEncoder, SReplica *pReplica) { if (tEncodeI32(pEncoder, pReplica->id) < 0) return -1; @@ -5212,11 +5239,11 @@ int32_t tDeserializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCo if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->finished) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -2; + if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -3; + if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -4; + if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -5; + if (tDecodeI32(&decoder, &pReq->finished) < 0) return -6; tEndDecode(&decoder); tDecoderClear(&decoder); @@ -5830,17 +5857,17 @@ int32_t tSerializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecom return tlen; } -int32_t tDeserializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecomeFollowerReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecomeFollowerReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; - tEndDecode(&decoder); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq *pReq) { SEncoder encoder = {0}; @@ -5889,38 +5916,38 @@ int32_t tDeserializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq return 0; } -int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; - if (tEncodeI8(&encoder, pReq->spi) < 0) return -1; - if (tEncodeI8(&encoder, pReq->encrypt) < 0) return -1; - if (tEncodeBinary(&encoder, pReq->secret, TSDB_PASSWORD_LEN) < 0) return -1; - if (tEncodeBinary(&encoder, pReq->ckey, TSDB_PASSWORD_LEN) < 0) return -1; - tEndEncode(&encoder); +// if (tStartEncode(&encoder) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->spi) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->encrypt) < 0) return -1; +// if (tEncodeBinary(&encoder, pReq->secret, TSDB_PASSWORD_LEN) < 0) return -1; +// if (tEncodeBinary(&encoder, pReq->ckey, TSDB_PASSWORD_LEN) < 0) return -1; +// tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } -int32_t tDeserializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->spi) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->encrypt) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->secret) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->ckey) < 0) return -1; - tEndDecode(&decoder); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; +// if (tDecodeI8(&decoder, &pReq->spi) < 0) return -1; +// if (tDecodeI8(&decoder, &pReq->encrypt) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->secret) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->ckey) < 0) return -1; +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSServerStatusRsp(void *buf, int32_t bufLen, SServerStatusRsp *pRsp) { SEncoder encoder = {0}; @@ -6958,30 +6985,30 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp * void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp) { taosArrayDestroy(pRsp->taskStatus); } -int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { - // SEncoder encoder = {0}; - // tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { +// // SEncoder encoder = {0}; +// // tEncoderInit(&encoder, buf, bufLen); - // if (tStartEncode(&encoder) < 0) return -1; - // if (pRsp->rspList) { - // int32_t num = taosArrayGetSize(pRsp->rspList); - // if (tEncodeI32(&encoder, num) < 0) return -1; - // for (int32_t i = 0; i < num; ++i) { - // SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i); - // if (tEncodeI32(&encoder, rsp->code) < 0) return -1; - // } - // } else { - // if (tEncodeI32(&encoder, 0) < 0) return -1; - // } - // tEndEncode(&encoder); +// // if (tStartEncode(&encoder) < 0) return -1; +// // if (pRsp->rspList) { +// // int32_t num = taosArrayGetSize(pRsp->rspList); +// // if (tEncodeI32(&encoder, num) < 0) return -1; +// // for (int32_t i = 0; i < num; ++i) { +// // SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i); +// // if (tEncodeI32(&encoder, rsp->code) < 0) return -1; +// // } +// // } else { +// // if (tEncodeI32(&encoder, 0) < 0) return -1; +// // } +// // tEndEncode(&encoder); - // int32_t tlen = encoder.pos; - // tEncoderClear(&encoder); - // reture tlen; - return 0; -} +// // int32_t tlen = encoder.pos; +// // tEncoderClear(&encoder); +// // reture tlen; +// return 0; +// } -int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { +//int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { // SDecoder decoder = {0}; // int32_t num = 0; // tDecoderInit(&decoder, buf, bufLen); @@ -7002,8 +7029,8 @@ int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatc // tEndDecode(&decoder); // tDecoderClear(&decoder); - return 0; -} +// return 0; +//} int tEncodeSVCreateTbBatchRsp(SEncoder *pCoder, const SVCreateTbBatchRsp *pRsp) { int32_t nRsps = taosArrayGetSize(pRsp->pArray); @@ -7138,15 +7165,15 @@ int32_t tEncodeSVDropTSmaReq(SEncoder *pCoder, const SVDropTSmaReq *pReq) { return 0; } -int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { - if (tStartDecode(pCoder) < 0) return -1; +// int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { +// if (tStartDecode(pCoder) < 0) return -1; - if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; - if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1; +// if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; +// if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1; - tEndDecode(pCoder); - return 0; -} +// tEndDecode(pCoder); +// return 0; +// } int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { int32_t headLen = sizeof(SMsgHead); @@ -7431,34 +7458,34 @@ int32_t tDeserializeSMDropStreamReq(void *buf, int32_t bufLen, SMDropStreamReq * void tFreeMDropStreamReq(SMDropStreamReq *pReq) { FREESQL(); } -int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; - if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; +// if (tStartEncode(&encoder) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; - tEndEncode(&encoder); +// tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } -int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStreamReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStreamReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; +// if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; - tEndDecode(&decoder); +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { if (NULL == pReq) { @@ -7858,37 +7885,37 @@ int32_t tDecodeSVDropStbReq(SDecoder *pCoder, SVDropStbReq *pReq) { return 0; } -static int32_t tEncodeSVSubmitBlk(SEncoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) { - if (tStartEncode(pCoder) < 0) return -1; +// static int32_t tEncodeSVSubmitBlk(SEncoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) { +// if (tStartEncode(pCoder) < 0) return -1; - if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1; - if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1; - if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1; - if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1; +// if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1; +// if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1; +// if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1; +// if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1; - if (flags & TD_AUTO_CREATE_TABLE) { - if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; - } +// if (flags & TD_AUTO_CREATE_TABLE) { +// if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; +// } - tEndEncode(pCoder); - return 0; -} +// tEndEncode(pCoder); +// return 0; +// } -static int32_t tDecodeSVSubmitBlk(SDecoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) { - if (tStartDecode(pCoder) < 0) return -1; +// static int32_t tDecodeSVSubmitBlk(SDecoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) { +// if (tStartDecode(pCoder) < 0) return -1; - if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1; - if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1; - if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1; - if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1; +// if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1; +// if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1; +// if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1; +// if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1; - if (flags & TD_AUTO_CREATE_TABLE) { - if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; - } +// if (flags & TD_AUTO_CREATE_TABLE) { +// if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; +// } - tEndDecode(pCoder); - return 0; -} +// tEndDecode(pCoder); +// return 0; +// } static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBlock) { if (tStartEncode(pEncoder) < 0) return -1; @@ -7912,78 +7939,78 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl return 0; } -static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { - if (tStartDecode(pDecoder) < 0) return -1; +// static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { +// if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1; - if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; - pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); - if (NULL == pBlock->tblFName) return -1; - if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; - if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; - if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; - if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1; +// if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1; +// if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; +// pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); +// if (NULL == pBlock->tblFName) return -1; +// if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; +// if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1; - int32_t meta = 0; - if (tDecodeI32(pDecoder, &meta) < 0) return -1; - if (meta) { - pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - if (NULL == pBlock->pMeta) return -1; - if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1; - } else { - pBlock->pMeta = NULL; - } +// int32_t meta = 0; +// if (tDecodeI32(pDecoder, &meta) < 0) return -1; +// if (meta) { +// pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); +// if (NULL == pBlock->pMeta) return -1; +// if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1; +// } else { +// pBlock->pMeta = NULL; +// } - tEndDecode(pDecoder); - return 0; -} +// tEndDecode(pDecoder); +// return 0; +// } -int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) { - int32_t nBlocks = taosArrayGetSize(pRsp->pArray); +// int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) { +// int32_t nBlocks = taosArrayGetSize(pRsp->pArray); - if (tStartEncode(pEncoder) < 0) return -1; +// if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1; - if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1; - if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1; - for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) { - if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1; - } +// if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1; +// if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1; +// if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1; +// for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) { +// if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1; +// } - tEndEncode(pEncoder); - return 0; -} +// tEndEncode(pEncoder); +// return 0; +// } -int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) { - if (tStartDecode(pDecoder) < 0) return -1; +// int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) { +// if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1; - if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1; - if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1; - pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks)); - if (pRsp->pBlocks == NULL) return -1; - for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) { - if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1; - } +// if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1; +// pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks)); +// if (pRsp->pBlocks == NULL) return -1; +// for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) { +// if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1; +// } - tEndDecode(pDecoder); - tDecoderClear(pDecoder); - return 0; -} +// tEndDecode(pDecoder); +// tDecoderClear(pDecoder); +// return 0; +// } -void tFreeSSubmitBlkRsp(void *param) { - if (NULL == param) { - return; - } +// void tFreeSSubmitBlkRsp(void *param) { +// if (NULL == param) { +// return; +// } - SSubmitBlkRsp *pRsp = (SSubmitBlkRsp *)param; +// SSubmitBlkRsp *pRsp = (SSubmitBlkRsp *)param; - taosMemoryFree(pRsp->tblFName); - if (pRsp->pMeta) { - taosMemoryFree(pRsp->pMeta->pSchemas); - taosMemoryFree(pRsp->pMeta); - } -} +// taosMemoryFree(pRsp->tblFName); +// if (pRsp->pMeta) { +// taosMemoryFree(pRsp->pMeta->pSchemas); +// taosMemoryFree(pRsp->pMeta); +// } +// } void tFreeSSubmitRsp(SSubmitRsp *pRsp) { if (NULL == pRsp) return; @@ -8157,23 +8184,23 @@ int32_t tDecodeSVAlterTbRsp(SDecoder *pDecoder, SVAlterTbRsp *pRsp) { return 0; } -int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) { - int32_t meta = 0; - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) { +// int32_t meta = 0; +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1; - if (tDecodeI32(&decoder, &meta) < 0) return -1; - if (meta) { - pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - if (NULL == pRsp->pMeta) return -1; - if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; - } - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1; +// if (tDecodeI32(&decoder, &meta) < 0) return -1; +// if (meta) { +// pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); +// if (NULL == pRsp->pMeta) return -1; +// if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; +// } +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } int32_t tEncodeSMAlterStbRsp(SEncoder *pEncoder, const SMAlterStbRsp *pRsp) { if (tStartEncode(pEncoder) < 0) return -1; @@ -8198,22 +8225,22 @@ int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp) { return 0; } -int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) { - int32_t meta = 0; - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) { +// int32_t meta = 0; +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &meta) < 0) return -1; - if (meta) { - pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - if (NULL == pRsp->pMeta) return -1; - if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; - } - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &meta) < 0) return -1; +// if (meta) { +// pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); +// if (NULL == pRsp->pMeta) return -1; +// if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; +// } +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) { if (NULL == pRsp) { @@ -8249,22 +8276,22 @@ int32_t tDecodeSMCreateStbRsp(SDecoder *pDecoder, SMCreateStbRsp *pRsp) { return 0; } -int32_t tDeserializeSMCreateStbRsp(void *buf, int32_t bufLen, SMCreateStbRsp *pRsp) { - int32_t meta = 0; - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSMCreateStbRsp(void *buf, int32_t bufLen, SMCreateStbRsp *pRsp) { +// int32_t meta = 0; +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &meta) < 0) return -1; - if (meta) { - pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - if (NULL == pRsp->pMeta) return -1; - if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; - } - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &meta) < 0) return -1; +// if (meta) { +// pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); +// if (NULL == pRsp->pMeta) return -1; +// if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; +// } +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } void tFreeSMCreateStbRsp(SMCreateStbRsp *pRsp) { if (NULL == pRsp) { diff --git a/source/common/src/trow.c b/source/common/src/trow.c index b91562be7a..6e9278c630 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -137,6 +137,7 @@ int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pVa return TSDB_CODE_SUCCESS; } +#if 0 int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { terrno = TSDB_CODE_INVALID_PARA; @@ -177,16 +178,19 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal } return TSDB_CODE_SUCCESS; } +#endif int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) { switch (bitmapMode) { case 0: tdGetBitmapValTypeII(pBitmap, colIdx, pValType); break; +#if 0 case -1: case 1: tdGetBitmapValTypeI(pBitmap, colIdx, pValType); break; +#endif default: terrno = TSDB_CODE_INVALID_PARA; return TSDB_CODE_FAILED; @@ -231,7 +235,7 @@ bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCe return true; } -#ifdef BUILD_NO_CALL +// #ifdef BUILD_NO_CALL const uint8_t tdVTypeByte[2][3] = {{ // 2 bits TD_VTYPE_NORM_BYTE_II, @@ -439,13 +443,15 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { return true; } -int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { +int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t rowType) { STColumn *pTColumn; SColVal *pColVal; int32_t nColVal = taosArrayGetSize(pArray); int32_t varDataLen = 0; + int32_t nonVarDataLen = 0; int32_t maxVarDataLen = 0; int32_t iColVal = 0; + int32_t nBound = 0; void *varBuf = NULL; bool isAlloc = false; @@ -458,6 +464,9 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { } else { pColVal = NULL; } + if (pColVal && !COL_VAL_IS_NONE(pColVal)) { + ++nBound; + } if (iColumn == 0) { ASSERT(pColVal && pColVal->cid == pTColumn->colId); @@ -484,16 +493,26 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { } } } + } else { + if(pColVal && COL_VAL_IS_VALUE(pColVal)) { + nonVarDataLen += TYPE_BYTES[pTColumn->type]; + } } } ++iColVal; } - if (!(*ppRow)) { - *ppRow = (STSRow *)taosMemoryCalloc( - 1, sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1)); - isAlloc = true; + int32_t rowTotalLen = 0; + if (rowType == TD_ROW_TP) { + rowTotalLen = sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1); + } else { + rowTotalLen = sizeof(STSRow) + sizeof(col_id_t) + varDataLen + nonVarDataLen + (nBound - 1) * sizeof(SKvRowIdx) + + TD_BITMAP_BYTES(nBound - 1); + } + if (!(*ppRow)) { + *ppRow = (STSRow *)taosMemoryCalloc(1, rowTotalLen); + isAlloc = true; } if (!(*ppRow)) { @@ -512,10 +531,11 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { } } - SRowBuilder rb = {0}; + SRowBuilder rb = {.rowType = rowType}; tdSRowInit(&rb, pTSchema->version); - tdSRowSetInfo(&rb, pTSchema->numOfCols, pTSchema->numOfCols, pTSchema->flen); + tdSRowSetInfo(&rb, pTSchema->numOfCols, nBound, pTSchema->flen); tdSRowResetBuf(&rb, *ppRow); + int32_t iBound = 0; iColVal = 0; for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; ++iColumn) { @@ -529,21 +549,28 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { valType = TD_VTYPE_NONE; } else if (COL_VAL_IS_NULL(pColVal)) { valType = TD_VTYPE_NULL; + ++iBound; } else if (IS_VAR_DATA_TYPE(pTColumn->type)) { varDataSetLen(varBuf, pColVal->value.nData); if (pColVal->value.nData != 0) { memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData); } val = varBuf; + ++iBound; } else { val = (const void *)&pColVal->value.val; + ++iBound; } } else { // pColVal = NULL; valType = TD_VTYPE_NONE; } - tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal); + if (TD_IS_TP_ROW(rb.pBuf)) { + tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal); + } else { + tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, rb.offset, iBound - 1); + } ++iColVal; } @@ -605,7 +632,7 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell return true; } - +#if 0 int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { if (!pBitmap || colIdx < 0) { terrno = TSDB_CODE_INVALID_PARA; @@ -655,6 +682,7 @@ int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { } return TSDB_CODE_SUCCESS; } +#endif int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) { #ifdef TD_SUPPORT_BITMAP @@ -761,11 +789,11 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) { - if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) { + if (colIdx < 1) { terrno = TSDB_CODE_INVALID_PARA; + ASSERTS(0, "colIdx is %" PRIi64, colIdx); return terrno; } - offset -= sizeof(SKvRowIdx); --colIdx; #ifdef TD_SUPPORT_BITMAP @@ -779,6 +807,7 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); pColIdx->colId = colId; pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN + pBuilder->offset += sizeof(SKvRowIdx); if (valType == TD_VTYPE_NORM) { char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); if (IS_VAR_DATA_TYPE(colType)) { @@ -830,7 +859,7 @@ int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const vo return 0; } - +#if 0 int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, int32_t allNullLen, int32_t boundNullLen) { if ((boundNullLen > 0) && (allNullLen > 0) && (nBoundCols > 0)) { @@ -866,6 +895,7 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou #endif return TSDB_CODE_SUCCESS; } +#endif int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; @@ -904,6 +934,7 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { TD_ROW_SET_LEN(pBuilder->pBuf, len); TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); + pBuilder->offset = 0; break; default: terrno = TSDB_CODE_INVALID_PARA; @@ -1040,10 +1071,12 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int case 0: tdSetBitmapValTypeII(pBitmap, colIdx, valType); break; +#if 0 case -1: case 1: tdSetBitmapValTypeI(pBitmap, colIdx, valType); break; +#endif default: terrno = TSDB_CODE_INVALID_PARA; return TSDB_CODE_FAILED; @@ -1056,7 +1089,6 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColVal) { STColumn *pTColumn = &pTSchema->columns[iCol]; SCellVal cv = {0}; - SValue value = {0}; ASSERT((pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) || (iCol > 0)); @@ -1084,5 +1116,4 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV memcpy(&pColVal->value.val, cv.val, tDataTypes[pTColumn->type].bytes); } } -} -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/source/common/test/CMakeLists.txt b/source/common/test/CMakeLists.txt index eb79e79afa..51c733e796 100644 --- a/source/common/test/CMakeLists.txt +++ b/source/common/test/CMakeLists.txt @@ -35,6 +35,10 @@ target_include_directories( PUBLIC "${TD_SOURCE_DIR}/include/common" PUBLIC "${TD_SOURCE_DIR}/include/util" ) +add_test( + NAME dataformatTest + COMMAND dataformatTest +) # tmsg test # add_executable(tmsgTest "") diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index b1083e4590..8bc7d47ebf 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#if 0 #include #include @@ -59,7 +58,6 @@ STSchema *genSTSchema(int16_t nCols) { case 1: { pSchema[i].type = TSDB_DATA_TYPE_INT; pSchema[i].bytes = TYPE_BYTES[pSchema[i].type]; - ; } break; case 2: { pSchema[i].type = TSDB_DATA_TYPE_BIGINT; @@ -123,7 +121,8 @@ STSchema *genSTSchema(int16_t nCols) { return pResult; } -// ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, c9 bool +// ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, +// c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) { if (!(*pArray)) { *pArray = taosArrayInit(nCols, sizeof(SColVal)); @@ -142,59 +141,76 @@ static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) { taosArrayPush(*pArray, &colVal); continue; } - switch (i) { case 0: - sscanf(data[i], "%" PRIi64, &colVal.value.ts); + colVal.type = TSDB_DATA_TYPE_TIMESTAMP; + sscanf(data[i], "%" PRIi64, &colVal.value.val); break; case 1: - sscanf(data[i], "%" PRIi32, &colVal.value.i32); + colVal.type = TSDB_DATA_TYPE_INT; + sscanf(data[i], "%" PRIi32, (int32_t *)&colVal.value.val); break; case 2: - sscanf(data[i], "%" PRIi64, &colVal.value.i64); + colVal.type = TSDB_DATA_TYPE_BIGINT; + sscanf(data[i], "%" PRIi64, &colVal.value.val); break; case 3: - sscanf(data[i], "%f", &colVal.value.f); + colVal.type = TSDB_DATA_TYPE_FLOAT; + sscanf(data[i], "%f", (float *)&colVal.value.val); break; case 4: - sscanf(data[i], "%lf", &colVal.value.d); + colVal.type = TSDB_DATA_TYPE_DOUBLE; + sscanf(data[i], "%lf", (double *)&colVal.value.val); break; case 5: { + colVal.type = TSDB_DATA_TYPE_BINARY; int16_t dataLen = strlen(data[i]) + 1; colVal.value.nData = dataLen < 10 ? dataLen : 10; colVal.value.pData = (uint8_t *)data[i]; } break; case 6: { + colVal.type = TSDB_DATA_TYPE_NCHAR; int16_t dataLen = strlen(data[i]) + 1; colVal.value.nData = dataLen < 40 ? dataLen : 40; colVal.value.pData = (uint8_t *)data[i]; // just for test, not real nchar } break; - case 7: - case 9: { + case 7: { + colVal.type = TSDB_DATA_TYPE_TINYINT; int32_t d8; sscanf(data[i], "%" PRId32, &d8); - colVal.value.i8 = (int8_t)d8; - } break; + colVal.value.val = (int8_t)d8; + } case 8: { + colVal.type = TSDB_DATA_TYPE_SMALLINT; int32_t d16; sscanf(data[i], "%" PRId32, &d16); - colVal.value.i16 = (int16_t)d16; + colVal.value.val = (int16_t)d16; + } break; + case 9: { + colVal.type = TSDB_DATA_TYPE_BOOL; + int32_t d8; + sscanf(data[i], "%" PRId32, &d8); + colVal.value.val = (int8_t)d8; } break; case 10: { + colVal.type = TSDB_DATA_TYPE_UTINYINT; uint32_t u8; sscanf(data[i], "%" PRId32, &u8); - colVal.value.u8 = (uint8_t)u8; + colVal.value.val = (uint8_t)u8; } break; case 11: { + colVal.type = TSDB_DATA_TYPE_USMALLINT; uint32_t u16; sscanf(data[i], "%" PRId32, &u16); - colVal.value.u16 = (uint16_t)u16; + colVal.value.val = (uint16_t)u16; } break; case 12: { - sscanf(data[i], "%" PRIu32, &colVal.value.u32); + colVal.type = TSDB_DATA_TYPE_UINT; + sscanf(data[i], "%" PRIu32, (uint32_t *)&colVal.value.val); } break; case 13: { - sscanf(data[i], "%" PRIu64, &colVal.value.u64); + colVal.type = TSDB_DATA_TYPE_UBIGINT; + sscanf(data[i], "%" PRIu64, (uint64_t *)&colVal.value.val); } break; default: ASSERT(0); @@ -215,25 +231,25 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) { } switch (type) { case TSDB_DATA_TYPE_BOOL: - printf("%s ", cv->value.i8 == 0 ? "false" : "true"); + printf("%s ", cv->value.val == 0 ? "false" : "true"); break; case TSDB_DATA_TYPE_TINYINT: - printf("%" PRIi8 " ", cv->value.i8); + printf("%" PRIi8 " ", *(int8_t *)&cv->value.val); break; case TSDB_DATA_TYPE_SMALLINT: - printf("%" PRIi16 " ", cv->value.i16); + printf("%" PRIi16 " ", *(int16_t *)&cv->value.val); break; case TSDB_DATA_TYPE_INT: - printf("%" PRIi32 " ", cv->value.i32); + printf("%" PRIi32 " ", *(int32_t *)&cv->value.val); break; case TSDB_DATA_TYPE_BIGINT: - printf("%" PRIi64 " ", cv->value.i64); + printf("%" PRIi64 " ", cv->value.val); break; case TSDB_DATA_TYPE_FLOAT: - printf("%f ", cv->value.f); + printf("%f ", *(float *)&cv->value.val); break; case TSDB_DATA_TYPE_DOUBLE: - printf("%lf ", cv->value.d); + printf("%lf ", *(double *)&cv->value.val); break; case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_GEOMETRY: { @@ -242,7 +258,7 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) { printf("%s ", tv); } break; case TSDB_DATA_TYPE_TIMESTAMP: - printf("%" PRIi64 " ", cv->value.i64); + printf("%" PRIi64 " ", cv->value.val); break; case TSDB_DATA_TYPE_NCHAR: { char tv[15] = {0}; @@ -250,16 +266,16 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) { printf("%s ", tv); } break; case TSDB_DATA_TYPE_UTINYINT: - printf("%" PRIu8 " ", cv->value.u8); + printf("%" PRIu8 " ", *(uint8_t *)&cv->value.val); break; case TSDB_DATA_TYPE_USMALLINT: - printf("%" PRIu16 " ", cv->value.u16); + printf("%" PRIu16 " ", *(uint16_t *)&cv->value.val); break; case TSDB_DATA_TYPE_UINT: - printf("%" PRIu32 " ", cv->value.u32); + printf("%" PRIu32 " ", *(uint32_t *)&cv->value.val); break; case TSDB_DATA_TYPE_UBIGINT: - printf("%" PRIu64 " ", cv->value.u64); + printf("%" PRIu64 " ", *(uint64_t *)&cv->value.val); break; case TSDB_DATA_TYPE_JSON: printf("JSON "); @@ -286,11 +302,11 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) { return 0; } -void debugPrintTSRow(STSRow2 *row, STSchema *pTSchema, const char *tags, int32_t ln) { - // printf("%s:%d %s:v%d:%d ", tags, ln, (row->flags & 0xf0) ? "KV" : "TP", row->sver, row->nData); +void debugPrintTSRow(STSRow *row, STSchema *pTSchema, const char *tags, int32_t ln) { + printf("%s:%d %s:v%d:len-%u ", tags, ln, (row->type) ? "KV" : "TP", row->sver, row->len); for (int16_t i = 0; i < pTSchema->numOfCols; ++i) { SColVal cv = {0}; - tTSRowGet(row, pTSchema, i, &cv); + tTSRowGetVal(row, pTSchema, i, &cv); debugPrintSColVal(&cv, pTSchema->columns[i].type); } printf("\n"); @@ -315,36 +331,36 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) { case TSDB_DATA_TYPE_TINYINT: { int32_t d8; sscanf(rawVal, "%" PRId32, &d8); - EXPECT_EQ(cv->value.i8, (int8_t)d8); + EXPECT_EQ((int8_t)cv->value.val, (int8_t)d8); } break; case TSDB_DATA_TYPE_SMALLINT: { int32_t d16; sscanf(rawVal, "%" PRId32, &d16); - EXPECT_EQ(cv->value.i16, (int16_t)d16); + EXPECT_EQ((int16_t)cv->value.val, (int16_t)d16); } break; case TSDB_DATA_TYPE_INT: { - sscanf(rawVal, "%" PRId32, &rawSVal.i32); - EXPECT_EQ(cv->value.i32, rawSVal.i32); + sscanf(rawVal, "%" PRId32, (int32_t *)&rawSVal.val); + EXPECT_EQ((int32_t)cv->value.val, (int32_t)rawSVal.val); } break; case TSDB_DATA_TYPE_BIGINT: { - sscanf(rawVal, "%" PRIi64, &rawSVal.i64); - EXPECT_EQ(cv->value.i64, rawSVal.i64); + sscanf(rawVal, "%" PRIi64, &rawSVal.val); + EXPECT_EQ(cv->value.val, rawSVal.val); } break; case TSDB_DATA_TYPE_FLOAT: { - sscanf(rawVal, "%f", &rawSVal.f); - EXPECT_FLOAT_EQ(cv->value.f, rawSVal.f); + sscanf(rawVal, "%f", (float *)&rawSVal.val); + EXPECT_FLOAT_EQ((float)cv->value.val, (float)rawSVal.val); } break; case TSDB_DATA_TYPE_DOUBLE: { - sscanf(rawVal, "%lf", &rawSVal.d); - EXPECT_DOUBLE_EQ(cv->value.d, rawSVal.d); + sscanf(rawVal, "%lf", (double *)&rawSVal.val); + EXPECT_DOUBLE_EQ((double)cv->value.val, (double)rawSVal.val); } break; case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_GEOMETRY: { EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData); } break; case TSDB_DATA_TYPE_TIMESTAMP: { - sscanf(rawVal, "%" PRIi64, &rawSVal.ts); - EXPECT_DOUBLE_EQ(cv->value.ts, rawSVal.ts); + sscanf(rawVal, "%" PRIi64, &rawSVal.val); + EXPECT_DOUBLE_EQ(cv->value.val, rawSVal.val); } break; case TSDB_DATA_TYPE_NCHAR: { EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData); // informal nchar comparsion @@ -352,20 +368,20 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) { case TSDB_DATA_TYPE_UTINYINT: { uint32_t u8; sscanf(rawVal, "%" PRIu32, &u8); - EXPECT_EQ(cv->value.u8, (uint8_t)u8); + EXPECT_EQ((uint8_t)cv->value.val, (uint8_t)u8); } break; case TSDB_DATA_TYPE_USMALLINT: { uint32_t u16; sscanf(rawVal, "%" PRIu32, &u16); - EXPECT_EQ(cv->value.u16, (uint16_t)u16); + EXPECT_EQ((uint16_t)cv->value.val, (uint16_t)u16); } break; case TSDB_DATA_TYPE_UINT: { - sscanf(rawVal, "%" PRIu32, &rawSVal.u32); - EXPECT_EQ(cv->value.u32, rawSVal.u32); + sscanf(rawVal, "%" PRIu32, (uint32_t *)&rawSVal.val); + EXPECT_EQ((uint32_t)cv->value.val, (uint32_t)rawSVal.val); } break; case TSDB_DATA_TYPE_UBIGINT: { - sscanf(rawVal, "%" PRIu64, &rawSVal.u64); - EXPECT_EQ(cv->value.u64, rawSVal.u64); + sscanf(rawVal, "%" PRIu64, (uint64_t *)&rawSVal.val); + EXPECT_EQ((uint64_t)cv->value.val, (uint64_t)rawSVal.val); } break; case TSDB_DATA_TYPE_JSON: printf("JSON "); @@ -395,36 +411,66 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) { return 0; } -static void checkTSRow(const char **data, STSRow2 *row, STSchema *pTSchema) { +static void checkTSRow(const char **data, STSRow *row, STSchema *pTSchema) { for (int16_t i = 0; i < pTSchema->numOfCols; ++i) { SColVal cv = {0}; - tTSRowGet(row, pTSchema, i, &cv); + tTSRowGetVal(row, pTSchema, i, &cv); checkSColVal(data[i], &cv, pTSchema->columns[i].type); } + + STSRowIter rowIter = {0}; + rowIter.pSchema = pTSchema; + tdSTSRowIterReset(&rowIter, row); + for (int32_t i = 0; i < pTSchema->numOfCols; ++i) { + STColumn *pCol = pTSchema->columns + i; + SColVal colVal = {0}; + SCellVal cv = {0}; + if (!tdSTSRowIterFetch(&rowIter, pCol->colId, pCol->type, &cv)) { + break; + } + + colVal.cid = pCol->colId; + colVal.type = pCol->type; + if (tdValTypeIsNone(cv.valType)) { + colVal.flag = CV_FLAG_NONE; + } else if (tdValTypeIsNull(cv.valType)) { + colVal.flag = CV_FLAG_NULL; + } else { + colVal.flag = CV_FLAG_VALUE; + + if (IS_VAR_DATA_TYPE(pCol->type)) { + colVal.value.nData = varDataLen(cv.val); + colVal.value.pData = (uint8_t *)varDataVal(cv.val); + } else { + memcpy(&colVal.value.val, cv.val, tDataTypes[pCol->type].bytes); + } + } + checkSColVal(data[i], &colVal, pCol->type); + } } TEST(testCase, AllNormTest) { - int16_t nCols = 14; - STSRowBuilder rb = {0}; - STSRow2 *row = nullptr; - SArray *pArray = taosArrayInit(nCols, sizeof(SColVal)); + int16_t nCols = 14; + STSRow *row = nullptr; + SArray *pArray = taosArrayInit(nCols, sizeof(SColVal)); EXPECT_NE(pArray, nullptr); STSchema *pTSchema = genSTSchema(nCols); EXPECT_NE(pTSchema, nullptr); - // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, - // c9 bool - char *data[14] = {"1653694220000", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"}; + // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, + // c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned + char *data[14] = { + "1653694220000", "10", "20", "10.1", "10.1", "binary10", "nchar10", "10", "10", "10", "10", "20", "30", "40"}; genTestData((const char **)&data, nCols, &pArray); - tTSRowNew(&rb, pArray, pTSchema, &row); - + tdSTSRowNew(pArray, pTSchema, &row, TD_ROW_TP); debugPrintTSRow(row, pTSchema, __func__, __LINE__); + tdSRowPrint(row, pTSchema, __func__); checkTSRow((const char **)&data, row, pTSchema); - tsRowBuilderClear(&rb); + taosMemoryFreeClear(row); taosArrayDestroy(pArray); taosMemoryFree(pTSchema); } @@ -433,7 +479,7 @@ TEST(testCase, AllNormTest) { TEST(testCase, NoneTest) { const static int nCols = 14; const static int nRows = 20; - STSRow2 *row = nullptr; + STSRow *row = nullptr; SArray *pArray = taosArrayInit(nCols, sizeof(SColVal)); EXPECT_NE(pArray, nullptr); @@ -442,42 +488,46 @@ TEST(testCase, NoneTest) { // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, // c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned - const char *data[nRows][nCols] = { - {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"}, - {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, - {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, - {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, - {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"}, - {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, - {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, - {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"}, - {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"}, - {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"}, - {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"}, - {"1653694220011", "-2147483648", "nu", "nu", "nu", "biy10", "nu", "nu", "32767", "no", "nu", "nu", "nu", "100"}, - {"1653694220012", "2147483647", "nu", "nu", "nu", "ary10", "nu", "nu", "-32768", "no", "nu", "nu", "nu", "100"}, - {"1653694220013", "no", "-9223372036854775818", "nu", "nu", "b1", "nu", "nu", "10", "no", "nu", "nu", "nu", "nu"}, - {"1653694220014", "no", "nu", "nu", "nu", "b0", "nu", "nu", "10", "no", "nu", "nu", "nu", "9223372036854775808"}, - {"1653694220015", "no", "nu", "nu", "nu", "binary30", "char4", "nu", "10", "no", "nu", "nu", "nu", - "18446744073709551615"}, - {"1653694220016", "2147483647", "nu", "nu", "nu", "bin50", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"}, - {"1653694220017", "2147483646", "0", "0", "0", "binary10", "0", "0", "0", "0", "255", "0", "0", "0"}, - {"1653694220018", "no", "-9223372036854775808", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", - "4294967295", "100"}, - {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu", - "no"}}; + const int8_t rowType[nRows] = {TD_ROW_TP, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, + TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP, TD_ROW_KV, TD_ROW_KV, + TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP}; - for (int r = 0; r < nRows; ++r) { - genTestData((const char **)&data[r], nCols, &pArray); - tTSRowNew(NULL, pArray, pTSchema, &row); - debugPrintTSRow(row, pTSchema, __func__, __LINE__); // debug print - checkTSRow((const char **)&data[r], row, pTSchema); // check - tTSRowFree(row); - taosArrayClear(pArray); +const char *data[nRows][nCols] = { + {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"}, + {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, + {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, + {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, + {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"}, + {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, + {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, + {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"}, + {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"}, + {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"}, + {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"}, + {"1653694220011", "-2147483648", "nu", "nu", "nu", "biy10", "nu", "nu", "32767", "no", "nu", "nu", "nu", "100"}, + {"1653694220012", "2147483647", "nu", "nu", "nu", "ary10", "nu", "nu", "-32768", "no", "nu", "nu", "nu", "100"}, + {"1653694220013", "no", "-9223372036854775818", "nu", "nu", "b1", "nu", "nu", "10", "no", "nu", "nu", "nu", "nu"}, + {"1653694220014", "no", "nu", "nu", "nu", "b0", "nu", "nu", "10", "no", "nu", "nu", "nu", "9223372036854775808"}, + {"1653694220015", "no", "nu", "nu", "nu", "binary30", "char4", "nu", "10", "no", "nu", "nu", "nu", + "18446744073709551615"}, + {"1653694220016", "2147483647", "nu", "nu", "nu", "bin50", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"}, + {"1653694220017", "2147483646", "0", "0", "0", "binary10", "0", "0", "0", "0", "255", "0", "0", "0"}, + {"1653694220018", "no", "-9223372036854775808", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", + "4294967295", "100"}, + {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu", + "no"}}; + +for (int r = 0; r < nRows; ++r) { + genTestData((const char **)&data[r], nCols, &pArray); + tdSTSRowNew(pArray, pTSchema, &row, rowType[r]); + debugPrintTSRow(row, pTSchema, __func__, __LINE__); // debug print + tdSRowPrint(row, pTSchema, __func__); + checkTSRow((const char **)&data[r], row, pTSchema); // check + taosMemoryFreeClear(row); + taosArrayClear(pArray); } taosArrayDestroy(pArray); taosMemoryFree(pTSchema); } -#endif #endif \ No newline at end of file diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 1508d88def..f9456f1729 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -68,7 +68,7 @@ static struct { int64_t startTime; } global = {0}; -static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetAllDebugFlag(143); } +static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetGlobalDebugFlag(143); } static void dmSetAssert(int32_t signum, void *sigInfo, void *context) { tsAssert = 1; } static void dmStopDnode(int signum, void *sigInfo, void *context) { diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 80502e2662..975246a10f 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -43,6 +43,7 @@ typedef struct SDnodeMgmt { GetMnodeLoadsFp getMnodeLoadsFp; GetQnodeLoadsFp getQnodeLoadsFp; int32_t statusSeq; + SendMonitorReportFp sendMonitorReportFpBasic; } SDnodeMgmt; // dmHandle.c diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index a3a29f6f77..aa5e87e8b1 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "dmInt.h" #include "systable.h" -#include "tgrant.h" extern SConfig *tsCfg; @@ -118,11 +117,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { req.memTotal = tsTotalMemoryKB * 1024; req.memAvail = req.memTotal - tsRpcQueueMemoryAllowed - 16 * 1024 * 1024; tstrncpy(req.dnodeEp, tsLocalEp, TSDB_EP_LEN); - char *machine = tGetMachineId(); - if (machine) { - tstrncpy(req.machineId, machine, TSDB_MACHINE_ID_LEN + 1); - taosMemoryFreeClear(machine); - } + tstrncpy(req.machineId, pMgmt->pData->machineId, TSDB_MACHINE_ID_LEN + 1); req.clusterCfg.statusInterval = tsStatusInterval; req.clusterCfg.checkTime = 0; @@ -365,12 +360,12 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { terrno = TSDB_CODE_INVALID_MSG; return -1; } - +#if 0 if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) { terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } - +#endif if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) { terrno = TSDB_CODE_INVALID_MSG; return -1; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index b9dd45f1c0..a651fbf060 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -65,6 +65,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->processDropNodeFp = pInput->processDropNodeFp; pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp; pMgmt->sendAuditRecordsFp = pInput->sendAuditRecordFp; + pMgmt->sendMonitorReportFpBasic = pInput->sendMonitorReportFpBasic; pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp; pMgmt->getVnodeLoadsLiteFp = pInput->getVnodeLoadsLiteFp; pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 7ca19d7725..cecb3d1581 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -75,6 +75,7 @@ static void *dmNotifyThreadFp(void *param) { static void *dmMonitorThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); + int64_t lastTimeForBasic = taosGetTimestampMs(); setThreadName("dnode-monitor"); static int32_t TRIM_FREQ = 20; @@ -85,6 +86,7 @@ static void *dmMonitorThreadFp(void *param) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; int64_t curTime = taosGetTimestampMs(); + if (curTime < lastTime) lastTime = curTime; float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsMonitorInterval) { @@ -96,6 +98,15 @@ static void *dmMonitorThreadFp(void *param) { taosMemoryTrim(0); } } + + if(tsMonitorForceV2){ + if (curTime < lastTimeForBasic) lastTimeForBasic = curTime; + float intervalForBasic = (curTime - lastTimeForBasic) / 1000.0f; + if (intervalForBasic >= tsMonitorIntervalForBasic) { + (*pMgmt->sendMonitorReportFpBasic)(); + lastTimeForBasic = curTime; + } + } } return NULL; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 531e6f4b3d..a3a5f1d971 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -191,6 +191,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_VIEW_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STATIS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_COMPACT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_CLUSTER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 1488df3cb1..1b1dcc9b54 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -87,7 +87,10 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; code = 0; _OVER: diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index a2f0b7aced..bfac0bab9d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -835,8 +835,11 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index be88e8b3fd..3dfc2bd96f 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -194,26 +194,26 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) while (pVnode->refCount > 0) taosMsleep(10); dInfo("vgId:%d, wait for vnode write queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue, - pVnode->pWriteW.queue->threadId); + taosQueueGetThreadId(pVnode->pWriteW.queue)); tMultiWorkerCleanup(&pVnode->pWriteW); dInfo("vgId:%d, wait for vnode sync queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue, - pVnode->pSyncW.queue->threadId); + taosQueueGetThreadId(pVnode->pSyncW.queue)); tMultiWorkerCleanup(&pVnode->pSyncW); dInfo("vgId:%d, wait for vnode sync rd queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue, - pVnode->pSyncRdW.queue->threadId); + taosQueueGetThreadId(pVnode->pSyncRdW.queue)); tMultiWorkerCleanup(&pVnode->pSyncRdW); dInfo("vgId:%d, wait for vnode apply queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue, - pVnode->pApplyW.queue->threadId); + taosQueueGetThreadId(pVnode->pApplyW.queue)); tMultiWorkerCleanup(&pVnode->pApplyW); dInfo("vgId:%d, wait for vnode query queue:%p is empty", pVnode->vgId, pVnode->pQueryQ); while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); dInfo("vgId:%d, wait for vnode fetch queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ, - pVnode->pFetchQ->threadId); + taosQueueGetThreadId(pVnode->pFetchQ)); while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); tqNotifyClose(pVnode->pImpl->pTq); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 8b80527447..a6abe5ab4d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -365,16 +365,16 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { } dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue, - pVnode->pWriteW.queue->threadId); + taosQueueGetThreadId(pVnode->pWriteW.queue)); dInfo("vgId:%d, sync-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue, - pVnode->pSyncW.queue->threadId); + taosQueueGetThreadId(pVnode->pSyncW.queue)); dInfo("vgId:%d, sync-rd-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue, - pVnode->pSyncRdW.queue->threadId); + taosQueueGetThreadId(pVnode->pSyncRdW.queue)); dInfo("vgId:%d, apply-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue, - pVnode->pApplyW.queue->threadId); + taosQueueGetThreadId(pVnode->pApplyW.queue)); dInfo("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ); dInfo("vgId:%d, fetch-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ, - pVnode->pFetchQ->threadId); + taosQueueGetThreadId(pVnode->pFetchQ)); dInfo("vgId:%d, stream-queue:%p is alloced", pVnode->vgId, pVnode->pStreamQ); return 0; } diff --git a/source/dnode/mgmt/node_mgmt/CMakeLists.txt b/source/dnode/mgmt/node_mgmt/CMakeLists.txt index 6b875db860..0cdc68345a 100644 --- a/source/dnode/mgmt/node_mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/node_mgmt/CMakeLists.txt @@ -1,7 +1,7 @@ aux_source_directory(src IMPLEMENT_SRC) add_library(dnode STATIC ${IMPLEMENT_SRC}) target_link_libraries( - dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode + dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw ) IF (TD_STORAGE) @@ -14,6 +14,10 @@ IF (TD_STORAGE) ENDIF () +IF (DEFINED GRANT_CFG_INCLUDE_DIR) + add_definitions(-DGRANTS_CFG) +ENDIF() + target_include_directories( dnode PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index bc6a4652e7..90e44e5acc 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -128,6 +128,7 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); // dmMonitor.c void dmSendMonitorReport(); void dmSendAuditRecords(); +void dmSendMonitorReportBasic(); void dmGetVnodeLoads(SMonVloadInfo *pInfo); void dmGetVnodeLoadsLite(SMonVloadInfo *pInfo); void dmGetMnodeLoads(SMonMloadInfo *pInfo); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index d173c83e97..12712d6585 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -394,6 +394,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) { .processDropNodeFp = dmProcessDropNodeReq, .sendMonitorReportFp = dmSendMonitorReport, .sendAuditRecordFp = auditSendRecordsInBatch, + .sendMonitorReportFpBasic = dmSendMonitorReportBasic, .getVnodeLoadsFp = dmGetVnodeLoads, .getVnodeLoadsLiteFp = dmGetVnodeLoadsLite, .getMnodeLoadsFp = dmGetMnodeLoads, diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 6cbf31b15f..049b1bdf84 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -22,6 +22,7 @@ #ifdef TD_TSZ #include "tcompression.h" #include "tglobal.h" +#include "tgrant.h" #endif static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { @@ -137,6 +138,16 @@ int32_t dmInitVars(SDnode *pDnode) { pData->rebootTime = taosGetTimestampMs(); pData->dropped = 0; pData->stopped = 0; + char *machineId = tGetMachineId(); + if (machineId) { + tstrncpy(pData->machineId, machineId, TSDB_MACHINE_ID_LEN + 1); + taosMemoryFreeClear(machineId); + } else { +#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG) + terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE; + return -1; +#endif + } pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); if (pData->dnodeHash == NULL) { diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index c42aa6a1ae..21e25f5535 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -25,8 +25,16 @@ static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); } +static void dmGetMonitorBasicInfoBasic(SDnode *pDnode, SMonBasicInfo *pInfo) { + pInfo->protocol = 2; + pInfo->dnode_id = pDnode->data.dnodeId; + pInfo->cluster_id = pDnode->data.clusterId; + tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); +} + static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { - pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f); + //pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f); + pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) /1000.0f; pInfo->has_mnode = pDnode->wrappers[MNODE].required; pInfo->has_qnode = pDnode->wrappers[QNODE].required; pInfo->has_snode = pDnode->wrappers[SNODE].required; @@ -44,6 +52,12 @@ static void dmGetDmMonitorInfo(SDnode *pDnode) { monSetDmInfo(&dmInfo); } +static void dmGetDmMonitorInfoBasic(SDnode *pDnode) { + SMonDmInfo dmInfo = {0}; + dmGetMonitorBasicInfoBasic(pDnode, &dmInfo.basic); + monSetDmInfo(&dmInfo); +} + static void dmGetMmMonitorInfo(SDnode *pDnode) { SMgmtWrapper *pWrapper = &pDnode->wrappers[MNODE]; if (dmMarkWrapper(pWrapper) == 0) { @@ -106,7 +120,17 @@ void dmSendMonitorReport() { dmGetVmMonitorInfo(pDnode); dmGetQmMonitorInfo(pDnode); dmGetSmMonitorInfo(pDnode); - monSendReport(); + monGenAndSendReport(); +} + +void dmSendMonitorReportBasic() { + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; + dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); + + SDnode *pDnode = dmInstance(); + dmGetDmMonitorInfoBasic(pDnode); + dmGetMmMonitorInfo(pDnode); + monGenAndSendReportBasic(); } //Todo: put this in seperate file in the future diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 1a31f08801..77760b16f4 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -345,6 +345,7 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.parent = pDnode; rpcInit.rfp = rpcRfp; rpcInit.compressSize = tsCompressMsgSize; + rpcInit.dfp = destroyAhandle; rpcInit.retryMinInterval = tsRedirectPeriod; rpcInit.retryStepFactor = tsRedirectFactor; diff --git a/source/dnode/mgmt/node_util/CMakeLists.txt b/source/dnode/mgmt/node_util/CMakeLists.txt index 5c670cbdd3..d882d784de 100644 --- a/source/dnode/mgmt/node_util/CMakeLists.txt +++ b/source/dnode/mgmt/node_util/CMakeLists.txt @@ -6,5 +6,5 @@ target_include_directories( PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor + node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor monitorfw ) \ No newline at end of file diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 4769ef8538..2175824acd 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -109,6 +109,7 @@ typedef struct { SMsgCb msgCb; bool validMnodeEps; int64_t ipWhiteVer; + char machineId[TSDB_MACHINE_ID_LEN + 1]; } SDnodeData; typedef struct { @@ -122,6 +123,7 @@ typedef struct { ProcessDropNodeFp processDropNodeFp; SendMonitorReportFp sendMonitorReportFp; SendAuditRecordsFp sendAuditRecordFp; + SendMonitorReportFp sendMonitorReportFpBasic; GetVnodeLoadsFp getVnodeLoadsFp; GetVnodeLoadsFp getVnodeLoadsLiteFp; GetMnodeLoadsFp getMnodeLoadsFp; diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 0e77ae9743..ceaf086dc1 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -23,9 +23,13 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser audit + mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser audit monitorfw ) +IF (DEFINED GRANT_CFG_INCLUDE_DIR) + add_definitions(-DGRANTS_CFG) +ENDIF() + IF (TD_GRANT) TARGET_LINK_LIBRARIES(mnode grant) ADD_DEFINITIONS(-D_GRANT) diff --git a/source/dnode/mnode/impl/inc/mndIndex.h b/source/dnode/mnode/impl/inc/mndIndex.h index 2d5479bc9b..3b9b68a1b5 100644 --- a/source/dnode/mnode/impl/inc/mndIndex.h +++ b/source/dnode/mnode/impl/inc/mndIndex.h @@ -19,16 +19,16 @@ int32_t mndGetTableIdx(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq); -int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); +int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); -int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); +int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); -int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); +int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); #ifdef __cplusplus } #endif -#endif /*_TD_MND_IDX_H_*/ \ No newline at end of file +#endif /*_TD_MND_IDX_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 1084340dc2..57fd187da3 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -86,6 +86,10 @@ typedef struct SOrphanTask { int32_t nodeId; } SOrphanTask; +typedef struct { + SMsgHead head; +} SMStreamHbRspMsg, SMStreamReqCheckpointRspMsg; + int32_t mndInitStream(SMnode *pMnode); void mndCleanupStream(SMnode *pMnode); SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName); @@ -119,6 +123,7 @@ int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj * int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList); int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); +int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream); SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream); void destroyStreamTaskIter(SStreamTaskIter *pIter); diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index 4e71684372..75b4531dbb 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -240,7 +240,7 @@ int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompac SSdbRaw *pVgRaw = mndCompactActionEncode(pCompact); if (pVgRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + if (mndTransAppendPrepareLog(pTrans, pVgRaw) != 0) { sdbFreeRaw(pVgRaw); return -1; } @@ -363,13 +363,15 @@ static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *p } static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "kill-compact"); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "kill-compact"); if (pTrans == NULL) { mError("compact:%" PRId32 ", failed to drop since %s" , pCompact->compactId, terrstr()); return -1; } mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId); + mndTransSetDbName(pTrans, pCompact->dbname, NULL); + SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); @@ -378,7 +380,7 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - void *pIter = NULL; + void *pIter = NULL; while (1) { SCompactDetailObj *pDetail = NULL; pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail); @@ -452,7 +454,7 @@ int32_t mndProcessKillCompactReq(SRpcMsg *pReq){ code = TSDB_CODE_ACTION_IN_PROGRESS; - char obj[MND_COMPACT_ID_LEN] = {0}; + char obj[TSDB_INT32_ID_LEN] = {0}; sprintf(obj, "%d", pCompact->compactId); auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen); @@ -488,13 +490,17 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c sdbRelease(pMnode->pSdb, pDetail); } - return -1; + return TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST; } int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){ SQueryCompactProgressRsp req = {0}; - if (tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req) != 0) { + int32_t code = 0; + code = tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req); + if (code != 0) { terrno = TSDB_CODE_INVALID_MSG; + mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d", + code, pReq->pCont, pReq->contLen); return -1; } @@ -502,10 +508,10 @@ int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){ req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished); SMnode *pMnode = pReq->info.node; - int32_t code = -1; - - if(mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req) != 0){ + code = mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req); + if(code != 0){ + terrno = code; mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished); return -1; @@ -612,15 +618,17 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { return 0; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-compact-progress"); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, NULL, "update-compact-progress"); if (pTrans == NULL) { mError("trans:%" PRId32 ", failed to create since %s" , pTrans->id, terrstr()); return -1; } mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id); - + SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId); + mndTransSetDbName(pTrans, pCompact->dbname, NULL); + pIter = NULL; while (1) { SCompactDetailObj *pDetail = NULL; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 077c0a9c2a..37c2d19bd4 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -41,7 +41,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew); -static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj); +static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw); static int32_t mndProcessCreateDbReq(SRpcMsg *pReq); static int32_t mndProcessAlterDbReq(SRpcMsg *pReq); @@ -256,17 +256,29 @@ _OVER: return pRow; } -static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj) { - SDbObj *pNewDb = pObj; +static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) { + SSdb *pSdb = pMnode->pSdb; + SSdbRow *pRow = NULL; + SDbObj *pNewDb = NULL; + int code = -1; + + pRow = mndDbActionDecode(pRaw); + if (pRow == NULL) goto _OVER; + pNewDb = sdbGetRowObj(pRow); + if (pNewDb == NULL) goto _OVER; SDbObj *pOldDb = sdbAcquire(pMnode->pSdb, SDB_DB, pNewDb->name); if (pOldDb != NULL) { mError("trans:%d, db name already in use. name: %s", pTrans->id, pNewDb->name); sdbRelease(pMnode->pSdb, pOldDb); - return -1; + goto _OVER; } - return 0; + code = 0; +_OVER: + if (pNewDb) mndDbActionDelete(pSdb, pNewDb); + taosMemoryFreeClear(pRow); + return code; } static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb) { @@ -884,10 +896,10 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { return terrno; } -static int32_t mndSetAlterDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { +static int32_t mndSetAlterDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { SSdbRaw *pRedoRaw = mndDbActionEncode(pOld); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { sdbFreeRaw(pRedoRaw); return -1; } @@ -943,7 +955,7 @@ static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *p mndTransSetDbName(pTrans, pOld->name, NULL); if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; - if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndSetAlterDbPrepareLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -1120,10 +1132,10 @@ _OVER: return code; } -static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { +static int32_t mndSetDropDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { SSdbRaw *pRedoRaw = mndDbActionEncode(pDb); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1; if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; return 0; @@ -1257,7 +1269,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { goto _OVER; } - if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto _OVER; + if (mndSetDropDbPrepareLogs(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER; /*if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ /*if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index cb8a24e675..79a5f5fd83 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -14,6 +14,8 @@ */ #define _DEFAULT_SOURCE +#include +#include "tjson.h" #include "mndDnode.h" #include "audit.h" #include "mndCluster.h" @@ -28,6 +30,7 @@ #include "mndVgroup.h" #include "tmisce.h" #include "tunit.h" +#include "taos_monitor.h" #define TSDB_DNODE_VER_NUMBER 2 #define TSDB_DNODE_RESERVE_SIZE 64 @@ -74,6 +77,7 @@ static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessStatusReq(SRpcMsg *pReq); static int32_t mndProcessNotifyReq(SRpcMsg *pReq); static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq); +static int32_t mndProcessStatisReq(SRpcMsg *pReq); static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); @@ -109,6 +113,7 @@ int32_t mndInitDnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq); mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq); mndSetMsgHandle(pMnode, TDMT_MND_RESTORE_DNODE, mndProcessRestoreDnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_STATIS, mndProcessStatisReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndRetrieveConfigs); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndCancelGetNextConfig); @@ -141,7 +146,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { memcpy(dnodeObj.machineId, machineId, TSDB_MACHINE_ID_LEN); taosMemoryFreeClear(machineId); } else { -#ifdef TD_ENTERPRISE +#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG) terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE; goto _OVER; #endif @@ -410,9 +415,6 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) { dInfo.ep.port = pDnode->port; dInfo.offlineReason = pDnode->offlineReason; tstrncpy(dInfo.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - tstrncpy(dInfo.active, pDnode->active, TSDB_ACTIVE_KEY_LEN); - tstrncpy(dInfo.connActive, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN); - tstrncpy(dInfo.machineId, pDnode->machineId, TSDB_MACHINE_ID_LEN + 1); sdbRelease(pSdb, pDnode); if (mndIsMnode(pMnode, pDnode->id)) { dInfo.isMnode = 1; @@ -502,6 +504,207 @@ static bool mndUpdateMnodeState(SMnodeObj *pObj, SMnodeLoad *pMload) { return stateChanged; } +static int32_t mndProcessStatisReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SStatisReq statisReq = {0}; + int32_t code = -1; + + char strClusterId[TSDB_CLUSTER_ID_LEN] = {0}; + sprintf(strClusterId, "%"PRId64, pMnode->clusterId); + + if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return code; + } + + if(tsMonitorLogProtocol){ + mInfo("process statis req,\n %s", statisReq.pCont); + } + + SJson* pJson = tjsonParse(statisReq.pCont); + + int32_t ts_size = tjsonGetArraySize(pJson); + + for(int32_t i = 0; i < ts_size; i++){ + SJson* item = tjsonGetArrayItem(pJson, i); + + SJson* tables = tjsonGetObjectItem(item, "tables"); + + int32_t tableSize = tjsonGetArraySize(tables); + for(int32_t i = 0; i < tableSize; i++){ + SJson* table = tjsonGetArrayItem(tables, i); + + char tableName[MONITOR_TABLENAME_LEN] = {0}; + tjsonGetStringValue(table, "name", tableName); + + SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups"); + + int32_t size = tjsonGetArraySize(metricGroups); + for(int32_t i = 0; i < size; i++){ + SJson* item = tjsonGetArrayItem(metricGroups, i); + + SJson* arrayTag = tjsonGetObjectItem(item, "tags"); + + int32_t tagSize = tjsonGetArraySize(arrayTag); + for(int32_t j = 0; j < tagSize; j++){ + SJson* item = tjsonGetArrayItem(arrayTag, j); + + char tagName[MONITOR_TAG_NAME_LEN] = {0}; + tjsonGetStringValue(item, "name", tagName); + + if(strncmp(tagName, "cluster_id", MONITOR_TAG_NAME_LEN) == 0) { + tjsonDeleteItemFromObject(item, "value"); + tjsonAddStringToObject(item, "value", strClusterId); + } + } + } + } + } + + char *pCont = tjsonToString(pJson); + monSendContent(pCont); + + if(pJson != NULL){ + tjsonDelete(pJson); + pJson = NULL; + } + + if(pCont != NULL){ + taosMemoryFree(pCont); + pCont = NULL; + } + + tFreeSStatisReq(&statisReq); + return 0; + +/* + SJson* pJson = tjsonParse(statisReq.pCont); + + int32_t ts_size = tjsonGetArraySize(pJson); + + for(int32_t i = 0; i < ts_size; i++){ + SJson* item = tjsonGetArrayItem(pJson, i); + + SJson* tables = tjsonGetObjectItem(item, "tables"); + + int32_t tableSize = tjsonGetArraySize(tables); + for(int32_t i = 0; i < tableSize; i++){ + SJson* table = tjsonGetArrayItem(tables, i); + + char tableName[MONITOR_TABLENAME_LEN] = {0}; + tjsonGetStringValue(table, "name", tableName); + + SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups"); + + int32_t size = tjsonGetArraySize(metricGroups); + for(int32_t i = 0; i < size; i++){ + SJson* item = tjsonGetArrayItem(metricGroups, i); + + SJson* arrayTag = tjsonGetObjectItem(item, "tags"); + + int32_t tagSize = tjsonGetArraySize(arrayTag); + + char** labels = taosMemoryMalloc(sizeof(char*) * tagSize); + char** sample_labels = taosMemoryMalloc(sizeof(char*) * tagSize); + + for(int32_t j = 0; j < tagSize; j++){ + SJson* item = tjsonGetArrayItem(arrayTag, j); + + *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); + tjsonGetStringValue(item, "name", *(labels + j)); + + *(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN); + tjsonGetStringValue(item, "value", *(sample_labels + j)); + if(strncmp(*(labels + j), "cluster_id", MONITOR_TAG_NAME_LEN) == 0) { + strncpy(*(sample_labels + j), strClusterId, MONITOR_TAG_VALUE_LEN); + } + } + + SJson* metrics = tjsonGetObjectItem(item, "metrics"); + + int32_t metricLen = tjsonGetArraySize(metrics); + for(int32_t j = 0; j < metricLen; j++){ + SJson *item = tjsonGetArrayItem(metrics, j); + + char name[MONITOR_METRIC_NAME_LEN] = {0}; + tjsonGetStringValue(item, "name", name); + + double value = 0; + tjsonGetDoubleValue(item, "value", &value); + + double type = 0; + tjsonGetDoubleValue(item, "type", &type); + + int32_t metricNameLen = strlen(name) + strlen(tableName) + 2; + char* metricName = taosMemoryMalloc(metricNameLen); + memset(metricName, 0, metricNameLen); + sprintf(metricName, "%s:%s", tableName, name); + + taos_metric_t* metric = taos_collector_registry_get_metric(metricName); + if(metric == NULL){ + if(type == 0){ + metric = taos_counter_new(metricName, "", tagSize, (const char**)labels); + } + if(type == 1){ + metric = taos_gauge_new(metricName, "", tagSize, (const char**)labels); + } + mTrace("fail to get metric from registry, new one metric:%p", metric); + + if(taos_collector_registry_register_metric(metric) == 1){ + if(type == 0){ + taos_counter_destroy(metric); + } + if(type == 1){ + taos_gauge_destroy(metric); + } + + metric = taos_collector_registry_get_metric(metricName); + + mTrace("fail to register metric, get metric from registry:%p", metric); + } + else{ + mTrace("succeed to register metric:%p", metric); + } + } + else{ + mTrace("get metric from registry:%p", metric); + } + + if(type == 0){ + taos_counter_add(metric, value, (const char**)sample_labels); + } + if(type == 1){ + taos_gauge_set(metric, value, (const char**)sample_labels); + } + + taosMemoryFreeClear(metricName); + } + + for(int32_t j = 0; j < tagSize; j++){ + taosMemoryFreeClear(*(labels + j)); + taosMemoryFreeClear(*(sample_labels + j)); + } + + taosMemoryFreeClear(sample_labels); + taosMemoryFreeClear(labels); + } + } + + } + + code = 0; + +_OVER: + if(pJson != NULL){ + tjsonDelete(pJson); + pJson = NULL; + } + + tFreeSStatisReq(&statisReq); + return code; + */ +} + static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStatusReq statusReq = {0}; diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index 041cc664e5..622ed0080f 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -331,10 +331,10 @@ SDbObj *mndAcquireDbByIdx(SMnode *pMnode, const char *idxName) { return mndAcquireDb(pMnode, db); } -int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { +int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1; if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; return 0; @@ -349,10 +349,10 @@ int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) return 0; } -int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { +int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { sdbFreeRaw(pRedoRaw); return -1; } @@ -482,10 +482,10 @@ _OVER: return code; } -int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { +int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1; if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; return 0; @@ -652,7 +652,7 @@ int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pSt mndTransSetSerial(pTrans); - if (mndSetCreateIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; + if (mndSetCreateIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; if (mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1) != 0) goto _OVER; @@ -771,7 +771,7 @@ static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *p if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; mndTransSetSerial(pTrans); - if (mndSetDropIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; + if (mndSetDropIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index a39e9e93c6..69a0bd477d 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -709,7 +709,8 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { int32_t code = syncProcessMsg(pMgmt->sync, pMsg); if (code != 0) { - mGError("vgId:1, failed to process sync msg:%p type:%s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); + mGError("vgId:1, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pMsg, TMSG_INFO(pMsg->msgType), + terrstr(), code); } return code; @@ -911,11 +912,14 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr if (pObj->id == pMnode->selfDnodeId) { pClusterInfo->first_ep_dnode_id = pObj->id; tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep)); - pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f; + //pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f; + pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode); // pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f); tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role)); + desc.syncState = TAOS_SYNC_STATE_LEADER; } else { tstrncpy(desc.role, syncStr(pObj->syncState), sizeof(desc.role)); + desc.syncState = pObj->syncState; } taosArrayPush(pClusterInfo->mnodes, &desc); sdbRelease(pSdb, pObj); @@ -946,6 +950,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr SMonVnodeDesc *pVnDesc = &desc.vnodes[i]; pVnDesc->dnode_id = pVgid->dnodeId; tstrncpy(pVnDesc->vnode_role, syncStr(pVgid->syncState), sizeof(pVnDesc->vnode_role)); + pVnDesc->syncState = pVgid->syncState; if (pVgid->syncState == TAOS_SYNC_STATE_LEADER) { tstrncpy(desc.status, "ready", sizeof(desc.status)); pClusterInfo->vgroups_alive++; @@ -985,8 +990,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 1000; pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed; if (pMnode->grant.expireTimeMS == 0) { - pGrantInfo->expire_time = INT32_MAX; - pGrantInfo->timeseries_total = INT32_MAX; + pGrantInfo->expire_time = 0; + pGrantInfo->timeseries_total = 0; } mndReleaseRpc(pMnode); diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index a7d8c81d3f..2f4ad69e2a 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -257,6 +257,7 @@ static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-qnode"); if (pTrans == NULL) goto _OVER; + mndTransSetSerial(pTrans); mInfo("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId); if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) goto _OVER; @@ -380,6 +381,7 @@ static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-qnode"); if (pTrans == NULL) goto _OVER; + mndTransSetSerial(pTrans); mInfo("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id); if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index cbc0ace75d..9aba428ff6 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -44,9 +44,8 @@ static bool hasCountWindowNode(SPhysiNode* pNode) { } } -static bool countWindowStreamTask(SSubplan* pPlan) { - SPhysiNode* pNode = pPlan->pNode; - return hasCountWindowNode(pNode); +static bool isCountWindowStreamTask(SSubplan* pPlan) { + return hasCountWindowNode((SPhysiNode*)pPlan->pNode); } int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, @@ -301,15 +300,16 @@ static int32_t doAddShuffleSinkTask(SMnode* pMnode, SStreamObj* pStream, SEpSet* } static int64_t getVgroupLastVer(const SArray* pList, int32_t vgId) { - for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) { + int32_t size = (int32_t) taosArrayGetSize(pList); + for (int32_t i = 0; i < size; ++i) { SVgroupVer* pVer = taosArrayGet(pList, i); if (pVer->vgId == vgId) { return pVer->ver; } } - mError("failed to find the vgId:%d for extract last version", vgId); - return -1; + mDebug("no data in vgId:%d for extract last version, set to be 0, total existed vgs:%d", vgId, size); + return 1; } static void streamTaskSetDataRange(SStreamTask* pTask, int64_t skey, SArray* pVerList, int32_t vgId) { @@ -342,13 +342,13 @@ static void streamTaskSetDataRange(SStreamTask* pTask, int64_t skey, SArray* pVe } } -static void haltInitialTaskStatus(SStreamTask* pTask, SSubplan* pPlan) { - bool hasCountWindowNode = countWindowStreamTask(pPlan); - bool isRelStreamTask = (pTask->hTaskInfo.id.taskId != 0); - if (hasCountWindowNode && isRelStreamTask) { +static void haltInitialTaskStatus(SStreamTask* pTask, SSubplan* pPlan, bool isFillhistoryTask) { + bool hasCountWindowNode = isCountWindowStreamTask(pPlan); + + if (hasCountWindowNode && (!isFillhistoryTask)) { SStreamStatus* pStatus = &pTask->status; - mDebug("s-task:0x%x status is set to %s from %s for count window agg task with fill-history option set", - pTask->id.taskId, streamTaskGetStatusStr(pStatus->taskStatus), streamTaskGetStatusStr(TASK_STATUS__HALT)); + mDebug("s-task:0x%x status set %s from %s for count window agg task with fill-history option set", + pTask->id.taskId, streamTaskGetStatusStr(TASK_STATUS__HALT), streamTaskGetStatusStr(pStatus->taskStatus)); pStatus->taskStatus = TASK_STATUS__HALT; } } @@ -398,15 +398,17 @@ static void setHTasksId(SStreamObj* pStream) { static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset, int64_t skey, SArray* pVerList, SVgObj* pVgroup, bool isFillhistory, bool useTriggerParam) { - // new stream task SStreamTask* pTask = buildSourceTask(pStream, pEpset, isFillhistory, useTriggerParam); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } + mDebug("doAddSourceTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory); - haltInitialTaskStatus(pTask, plan); + if (pStream->conf.fillHistory) { + haltInitialTaskStatus(pTask, plan, isFillhistory); + } streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId); @@ -415,6 +417,7 @@ static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStre terrno = code; return terrno; } + return TDB_CODE_SUCCESS; } @@ -470,6 +473,9 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream int code = doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, false, useTriggerParam); if (code != 0) { + mError("create stream task, code:%s", tstrerror(code)); + + // todo drop the added source tasks. sdbRelease(pSdb, pVgroup); return code; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 1275ba7962..4243ccb77c 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -257,6 +257,7 @@ static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode"); if (pTrans == NULL) goto _OVER; + mndTransSetSerial(pTrans); mInfo("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); @@ -383,6 +384,7 @@ static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode"); if (pTrans == NULL) goto _OVER; + mndTransSetSerial(pTrans); mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index aad1dc881b..7ee1b36916 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -617,10 +617,10 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { return 0; } -static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { +static int32_t mndSetCreateStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { sdbFreeRaw(pRedoRaw); return -1; } @@ -629,18 +629,6 @@ static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -static int32_t mndSetCreateStbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { - SSdbRaw *pUndoRaw = mndStbActionEncode(pStb); - if (pUndoRaw == NULL) return -1; - if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - sdbFreeRaw(pUndoRaw); - return -1; - } - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; - - return 0; -} - static int32_t mndSetCreateStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); if (pCommitRaw == NULL) return -1; @@ -913,8 +901,6 @@ _OVER: int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { mndTransSetDbName(pTrans, pDb->name, pStb->name); if (mndTransCheckConflict(pMnode, pTrans) != 0) return -1; - if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; - if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) return -1; if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb) != 0) return -1; @@ -1746,10 +1732,10 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO return 0; } -static int32_t mndSetAlterStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { +static int32_t mndSetAlterStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { sdbFreeRaw(pRedoRaw); return -1; } @@ -2155,7 +2141,7 @@ static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbOb mndTransSetRpcRsp(pTrans, pCont, contLen); } - if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -2192,11 +2178,11 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO if (mndGetIdxsByTagName(pMnode, pStb, pField0->name, &idxObj) == 0) { exist = true; } - if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (exist == true) { - if (mndSetDropIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; + if (mndSetDropIdxPrepareLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; if (mndSetDropIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; } @@ -2215,13 +2201,13 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO exist = true; } - if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (exist == true) { memcpy(idxObj.colName, nTagName, strlen(nTagName)); idxObj.colName[strlen(nTagName)] = 0; - if (mndSetAlterIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; + if (mndSetAlterIdxPrepareLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; if (mndSetAlterIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; } @@ -2354,10 +2340,10 @@ _OVER: return code; } -static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { +static int32_t mndSetDropStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { sdbFreeRaw(pRedoRaw); return -1; } @@ -2427,7 +2413,7 @@ static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *p mndTransSetDbName(pTrans, pDb->name, pStb->name); if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; - if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER; + if (mndSetDropStbPrepareLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndDropIdxsByStb(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; @@ -3547,7 +3533,7 @@ static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) { mndTransSetDbName(pTrans, pDb->name, pStb->name); if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; - if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndSetAlterStbRedoActions2(pMnode, pTrans, pDb, pStb, sql, len) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 9edac8fbde..6aab2424f7 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -29,7 +29,7 @@ #define MND_STREAM_MAX_NUM 60 -typedef struct SMStreamNodeCheckMsg { +typedef struct { int8_t placeHolder; // // to fix windows compile error, define place holder } SMStreamNodeCheckMsg; @@ -721,6 +721,13 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } + // add into buffer firstly + // to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already. + taosThreadMutexLock(&execInfo.lock); + mDebug("stream stream:%s tasks register into node list", createReq.name); + saveStreamTasksInfo(&streamObj, &execInfo); + taosThreadMutexUnlock(&execInfo.lock); + // execute creation if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -730,12 +737,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { mndTransDrop(pTrans); - taosThreadMutexLock(&execInfo.lock); - - mDebug("stream tasks register into node list"); - saveStreamTasksInfo(&streamObj, &execInfo); - taosThreadMutexUnlock(&execInfo.lock); - SName dbname = {0}; tNameFromString(&dbname, createReq.sourceDB, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -878,7 +879,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre int64_t ts = taosGetTimestampMs(); if (mndTrigger == 1 && (ts - pStream->checkpointFreq < tsStreamCheckpointInterval * 1000)) { // mWarn("checkpoint interval less than the threshold, ignore it"); - return -1; + return TSDB_CODE_SUCCESS; } bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHECKPOINT_NAME, lock); @@ -2180,5 +2181,16 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { mndReleaseStream(pMnode, pStream); taosThreadMutexUnlock(&execInfo.lock); + { + SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRspMsg)}; + rsp.pCont = rpcMallocCont(rsp.contLen); + SMsgHead* pHead = rsp.pCont; + pHead->vgId = htonl(req.nodeId); + + tmsgSendRsp(&rsp); + + pReq->info.handle = NULL; // disable auto rsp + } + return 0; } diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 23f5c5d620..14f3c533e3 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -65,7 +65,7 @@ static void addIntoCheckpointList(SArray* pList, const SFailedCheckpointInfo* pI taosArrayPush(pList, pInfo); } -int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { +int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, MND_STREAM_TASK_RESET_NAME, " reset from failed checkpoint"); if (pTrans == NULL) { return terrno; @@ -115,7 +115,7 @@ static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int64_t streamId, in } else { mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, transId:%d, create reset trans", pStream->name, pStream->uid, transId); - code = createStreamResetStatusTrans(pMnode, pStream); + code = mndCreateStreamResetStatusTrans(pMnode, pStream); } } @@ -222,11 +222,11 @@ int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo* info){ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamHbMsg req = {0}; - SArray *pFailedTasks = taosArrayInit(4, sizeof(SFailedCheckpointInfo)); - SArray *pOrphanTasks = taosArrayInit(3, sizeof(SOrphanTask)); + SArray *pFailedTasks = NULL; + SArray *pOrphanTasks = NULL; - if(grantCheckExpire(TSDB_GRANT_STREAMS) < 0){ - if(suspendAllStreams(pMnode, &pReq->info) < 0){ + if (grantCheckExpire(TSDB_GRANT_STREAMS) < 0) { + if (suspendAllStreams(pMnode, &pReq->info) < 0) { return -1; } } @@ -244,6 +244,9 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { mTrace("receive stream-meta hb from vgId:%d, active numOfTasks:%d", req.vgId, req.numOfTasks); + pFailedTasks = taosArrayInit(4, sizeof(SFailedCheckpointInfo)); + pOrphanTasks = taosArrayInit(3, sizeof(SOrphanTask)); + taosThreadMutexLock(&execInfo.lock); // extract stream task list @@ -349,5 +352,16 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { taosArrayDestroy(pFailedTasks); taosArrayDestroy(pOrphanTasks); + { + SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamHbRspMsg)}; + rsp.pCont = rpcMallocCont(rsp.contLen); + SMsgHead* pHead = rsp.pCont; + pHead->vgId = htonl(req.vgId); + + tmsgSendRsp(&rsp); + + pReq->info.handle = NULL; // disable auto rsp + } + return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index 5bfd3933b5..2241d93465 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -261,22 +261,30 @@ int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msg return mndTransAppendRedoAction(pTrans, &action); } +static bool identicalName(const char* pDb, const char* pParam, int32_t len) { + return (strlen(pDb) == len) && (strncmp(pDb, pParam, len) == 0); +} + int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t len) { - // data in the hash table will be removed automatically, no need to remove it here. - SStreamTransInfo *pTransInfo = taosHashGet(execInfo.transMgmt.pDBTrans, pDBName, len); - if (pTransInfo == NULL) { - return TSDB_CODE_SUCCESS; - } + void *pIter = NULL; - // not checkpoint trans, ignore - if (strcmp(pTransInfo->name, MND_STREAM_CHECKPOINT_NAME) != 0) { - mDebug("not checkpoint trans, not kill it, name:%s, transId:%d", pTransInfo->name, pTransInfo->transId); - return TSDB_CODE_SUCCESS; - } + while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) { + SStreamTransInfo *pTransInfo = (SStreamTransInfo *)pIter; + if (strcmp(pTransInfo->name, MND_STREAM_CHECKPOINT_NAME) != 0) { + continue; + } - char *pDupDBName = strndup(pDBName, len); - mndKillTransImpl(pMnode, pTransInfo->transId, pDupDBName); - taosMemoryFree(pDupDBName); + SStreamObj *pStream = mndGetStreamObj(pMnode, pTransInfo->streamId); + if (pStream != NULL) { + if (identicalName(pStream->sourceDb, pDBName, len)) { + mndKillTransImpl(pMnode, pTransInfo->transId, pStream->sourceDb); + } else if (identicalName(pStream->targetDb, pDBName, len)) { + mndKillTransImpl(pMnode, pTransInfo->transId, pStream->targetDb); + } + + mndReleaseStream(pMnode, pStream); + } + } return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 3cabce2201..1ae85a2cc6 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -231,6 +231,8 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT taosMemoryFree(pReq); return -1; } + + mDebug("set the resume action for trans:%d", pTrans->id); return 0; } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 0fc8dad420..a1490a3813 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -77,29 +77,16 @@ static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { static int32_t mndTransValidatePrepareAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { SSdbRaw *pRaw = pAction->pRaw; SSdb *pSdb = pMnode->pSdb; - SSdbRow *pRow = NULL; - void *pObj = NULL; - int code = -1; + int code = 0; if (pRaw->status != SDB_STATUS_CREATING) goto _OUT; - pRow = (pSdb->decodeFps[pRaw->type])(pRaw); - if (pRow == NULL) goto _OUT; - pObj = sdbGetRowObj(pRow); - if (pObj == NULL) goto _OUT; - SdbValidateFp validateFp = pSdb->validateFps[pRaw->type]; - code = 0; if (validateFp) { - code = validateFp(pMnode, pTrans, pObj); + code = validateFp(pMnode, pTrans, pRaw); } _OUT: - if (pRow) { - SdbDeleteFp deleteFp = pSdb->deleteFps[pRaw->type]; - if (deleteFp) (*deleteFp)(pSdb, pRow->pObj, false); - taosMemoryFreeClear(pRow); - } return code; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 1adc4ed4bf..04d74112d4 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -751,8 +751,8 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { pTrans->createdTime); int32_t code = mndSyncPropose(pMnode, pRaw, pTrans->id); if (code != 0) { - mError("trans:%d, failed to sync, errno:%s code:%s createTime:%" PRId64 " saved trans:%d", pTrans->id, terrstr(), - tstrerror(code), pTrans->createdTime, pMnode->syncMgmt.transId); + mError("trans:%d, failed to sync, errno:%s code:0x%x createTime:%" PRId64 " saved trans:%d", pTrans->id, terrstr(), + code, pTrans->createdTime, pMnode->syncMgmt.transId); sdbFreeRaw(pRaw); return -1; } @@ -1100,7 +1100,10 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->rawWritten) return 0; - if (topHalf) return TSDB_CODE_MND_TRANS_CTX_SWITCH; + if (topHalf) { + terrno = TSDB_CODE_MND_TRANS_CTX_SWITCH; + return TSDB_CODE_MND_TRANS_CTX_SWITCH; + } int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pAction->pRaw); if (code == 0 || terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { @@ -1214,7 +1217,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA if (numOfActions == 0) return 0; if ((code = mndTransExecSingleActions(pMnode, pTrans, pArray, topHalf)) != 0) { - return -1; + return code; } int32_t numOfExecuted = 0; @@ -1419,7 +1422,7 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool pTrans->stage = TRN_STAGE_COMMIT; mInfo("trans:%d, stage from redoAction to commit", pTrans->id); continueExec = true; - } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) { + } else if (code == TSDB_CODE_ACTION_IN_PROGRESS || code == TSDB_CODE_MND_TRANS_CTX_SWITCH) { mInfo("trans:%d, stage keep on redoAction since %s", pTrans->id, tstrerror(code)); continueExec = false; } else { diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5e5a3626a4..8a2313370a 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -2484,7 +2484,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false); - char objName[20] = {0}; + char objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(objName, "all", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index a5df9ad820..b0290191bc 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -31,10 +31,10 @@ #define VGROUP_VER_NUMBER 1 #define VGROUP_RESERVE_SIZE 64 -static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); -static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); -static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); -static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj); +static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); +static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); +static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); +static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw); static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); @@ -181,15 +181,28 @@ _OVER: return pRow; } -static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj) { - SVgObj *pVgroup = pObj; +static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) { + SSdb *pSdb = pMnode->pSdb; + SSdbRow *pRow = NULL; + SVgObj *pVgroup = NULL; + int code = -1; + + pRow = mndVgroupActionDecode(pRaw); + if (pRow == NULL) goto _OVER; + pVgroup = sdbGetRowObj(pRow); + if (pVgroup == NULL) goto _OVER; int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); if (maxVgId > pVgroup->vgId) { mError("trans:%d, vgroup id %d already in use. maxVgId:%d", pTrans->id, pVgroup->vgId, maxVgId); - return -1; + goto _OVER; } - return 0; + + code = 0; +_OVER: + if (pVgroup) mndVgroupActionDelete(pSdb, pVgroup); + taosMemoryFreeClear(pRow); + return code; } static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup) { diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp index e3bfdb5d6c..8d106b1ede 100644 --- a/source/dnode/mnode/impl/test/stream/stream.cpp +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -29,6 +29,9 @@ #include "../../inc/mndStream.h" namespace { + +static int64_t defStreamId = 999; + SRpcMsg buildHbReq() { SStreamHbMsg msg = {0}; msg.vgId = 1; @@ -40,7 +43,7 @@ SRpcMsg buildHbReq() { entry.nodeId = i + 1; entry.stage = 1; entry.id.taskId = i + 1; - entry.id.streamId = 999; + entry.id.streamId = defStreamId; if (i == 0) { entry.stage = 4; @@ -57,7 +60,7 @@ SRpcMsg buildHbReq() { entry.stage = 1; entry.id.taskId = 5; - entry.id.streamId = 999; + entry.id.streamId = defStreamId; entry.checkpointId = 1; entry.checkpointFailed = true; @@ -118,15 +121,16 @@ void setTask(SStreamTask* pTask, int32_t nodeId, int64_t streamId, int32_t taskI taosHashPut(pExecNode->pTaskMap, &id, sizeof(id), &entry, sizeof(entry)); taosArrayPush(pExecNode->pTaskList, &id); } + void initStreamExecInfo() { SStreamExecInfo* pExecNode = &execInfo; SStreamTask task = {0}; - setTask(&task, 1, 999, 1); - setTask(&task, 1, 999, 2); - setTask(&task, 1, 999, 3); - setTask(&task, 1, 999, 4); - setTask(&task, 2, 999, 5); + setTask(&task, 1, defStreamId, 1); + setTask(&task, 1, defStreamId, 2); + setTask(&task, 1, defStreamId, 3); + setTask(&task, 1, defStreamId, 4); + setTask(&task, 2, defStreamId, 5); } void initNodeInfo() { @@ -138,38 +142,112 @@ void initNodeInfo() { } } // namespace +class StreamTest : public testing::Test { // 继承了 testing::Test + protected: + + static void SetUpTestSuite() { + mndInitExecInfo(); + initStreamExecInfo(); + initNodeInfo(); + + std::cout<<"setup env for streamTest suite"<(taosMemoryCalloc(1, sizeof(SMnode))); + {// init sdb + SSdbOpt opt = {0}; + opt.path = pMnode->path; + opt.pMnode = pMnode; + opt.pWal = pMnode->pWal; + + pMnode->pSdb = sdbInit(&opt); + taosThreadMutexInit(&pMnode->syncMgmt.lock, NULL); + } + + SVgroupChangeInfo info; + info.pUpdateNodeList = taosArrayInit(4, sizeof(SNodeUpdateInfo)); + info.pDBMap = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); + + const char* pDbName = "test_db_name"; + int32_t len = strlen(pDbName); + + taosHashPut(info.pDBMap, pDbName, len, NULL, 0); + + killAllCheckpointTrans(pMnode, &info); + + SStreamObj stream; + memset(&stream, 0, sizeof(SStreamObj)); + + stream.uid = defStreamId; + stream.lock = 0; + stream.tasks = taosArrayInit(1, POINTER_BYTES); + stream.pHTasksList = taosArrayInit(1, POINTER_BYTES); + + SArray* pLevel = taosArrayInit(1, POINTER_BYTES); + SStreamTask* pTask = static_cast(taosMemoryCalloc(1, sizeof(SStreamTask))); + pTask->id.streamId = defStreamId; + pTask->id.taskId = 1; + pTask->exec.qmsg = (char*)taosMemoryMalloc(1); + taosThreadMutexInit(&pTask->lock, NULL); + + taosArrayPush(pLevel, &pTask); + + taosArrayPush(stream.tasks, &pLevel); + mndCreateStreamResetStatusTrans(pMnode, &stream); + + tFreeStreamObj(&stream); + sdbCleanup(pMnode->pSdb); + + taosMemoryFree(pMnode); + + taosArrayDestroy(info.pUpdateNodeList); + taosHashCleanup(info.pDBMap); +} + +TEST_F(StreamTest, plan_Test) { char* ast = "{\"NodeType\":\"101\",\"Name\":\"SelectStmt\",\"SelectStmt\":{\"Distinct\":false,\"Projections\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"9\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_1\",\"UserAlias\":\"_wstart\",\"Name\":\"_wstart\",\"Id\":\"89\",\"Type\":\"3505\",\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_2\",\"UserAlias\":\"sum(voltage)\",\"Name\":\"sum\",\"Id\":\"1\",\"Type\":\"14\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"4\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"voltage\",\"UserAlias\":\"voltage\",\"TableId\":\"6555383776122680534\",\"TableType\":\"1\",\"ColId\":\"3\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"meters\",\"TableAlias\":\"meters\",\"ColName\":\"voltage\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"4\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"#expr_3\",\"UserAlias\":\"groupid\",\"Name\":\"_group_key\",\"Id\":\"96\",\"Type\":\"3754\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"4\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"#expr_3\",\"UserAlias\":\"groupid\",\"TableId\":\"6555383776122680534\",\"TableType\":\"1\",\"ColId\":\"5\",\"ProjId\":\"0\",\"ColType\":\"2\",\"DbName\":\"test\",\"TableName\":\"meters\",\"TableAlias\":\"meters\",\"ColName\":\"groupid\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}}],\"From\":{\"NodeType\":\"6\",\"Name\":\"RealTable\",\"RealTable\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"0\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"DbName\":\"test\",\"tableName\":\"meters\",\"tableAlias\":\"meters\",\"MetaSize\":\"475\",\"Meta\":{\"VgId\":\"0\",\"TableType\":\"1\",\"Uid\":\"6555383776122680534\",\"Suid\":\"6555383776122680534\",\"Sversion\":\"1\",\"Tversion\":\"1\",\"ComInfo\":{\"NumOfTags\":\"2\",\"Precision\":\"0\",\"NumOfColumns\":\"4\",\"RowSize\":\"20\"},\"ColSchemas\":[{\"Type\":\"9\",\"ColId\":\"1\",\"bytes\":\"8\",\"Name\":\"ts\"},{\"Type\":\"6\",\"ColId\":\"2\",\"bytes\":\"4\",\"Name\":\"current\"},{\"Type\":\"4\",\"ColId\":\"3\",\"bytes\":\"4\",\"Name\":\"voltage\"},{\"Type\":\"6\",\"ColId\":\"4\",\"bytes\":\"4\",\"Name\":\"phase\"},{\"Type\":\"4\",\"ColId\":\"5\",\"bytes\":\"4\",\"Name\":\"groupid\"},{\"Type\":\"8\",\"ColId\":\"6\",\"bytes\":\"26\",\"Name\":\"location\"}]},\"VgroupsInfoSize\":\"1340\",\"VgroupsInfo\":{\"Num\":\"2\",\"Vgroups\":[{\"VgId\":\"2\",\"HashBegin\":\"0\",\"HashEnd\":\"2147483646\",\"EpSet\":{\"InUse\":\"0\",\"NumOfEps\":\"1\",\"Eps\":[{\"Fqdn\":\"localhost\",\"Port\":\"6030\"}]},\"NumOfTable\":\"0\"},{\"VgId\":\"3\",\"HashBegin\":\"2147483647\",\"HashEnd\":\"4294967295\",\"EpSet\":{\"InUse\":\"0\",\"NumOfEps\":\"1\",\"Eps\":[{\"Fqdn\":\"localhost\",\"Port\":\"6030\"}]},\"NumOfTable\":\"0\"}]}}},\"PartitionBy\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"4\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"groupid\",\"UserAlias\":\"groupid\",\"TableId\":\"6555383776122680534\",\"TableType\":\"1\",\"ColId\":\"5\",\"ProjId\":\"0\",\"ColType\":\"2\",\"DbName\":\"test\",\"TableName\":\"meters\",\"TableAlias\":\"meters\",\"ColName\":\"groupid\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"Window\":{\"NodeType\":\"14\",\"Name\":\"IntervalWindow\",\"IntervalWindow\":{\"Interval\":{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"115\",\"Bytes\":\"8\"},\"AliasName\":\"c804c3a15ebe05b5baf40ad5ee12be1f\",\"UserAlias\":\"2s\",\"LiteralSize\":\"2\",\"Literal\":\"2s\",\"Duration\":true,\"Translate\":true,\"NotReserved\":false,\"IsNull\":false,\"Unit\":\"115\",\"Datum\":\"2000\"}},\"TsPk\":{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"9\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"ts\",\"UserAlias\":\"ts\",\"TableId\":\"6555383776122680534\",\"TableType\":\"1\",\"ColId\":\"1\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"meters\",\"TableAlias\":\"meters\",\"ColName\":\"ts\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}}},\"StmtName\":\"0x1580095ba\",\"HasAggFuncs\":true}}"; // char* ast = "{\"NodeType\":\"101\",\"Name\":\"SelectStmt\",\"SelectStmt\":{\"Distinct\":false,\"Projections\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"9\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_1\",\"UserAlias\":\"wstart\",\"Name\":\"_wstart\",\"Id\":\"89\",\"Type\":\"3505\",\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"#expr_2\",\"UserAlias\":\"min(c1)\",\"Name\":\"min\",\"Id\":\"2\",\"Type\":\"8\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"c1\",\"UserAlias\":\"c1\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"2\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c1\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"3\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"2\"},\"AliasName\":\"#expr_3\",\"UserAlias\":\"max(c2)\",\"Name\":\"max\",\"Id\":\"3\",\"Type\":\"7\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"3\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"2\"},\"AliasName\":\"c2\",\"UserAlias\":\"c2\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"3\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c2\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"4\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_4\",\"UserAlias\":\"sum(c3)\",\"Name\":\"sum\",\"Id\":\"1\",\"Type\":\"14\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"4\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"c3\",\"UserAlias\":\"c3\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"4\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c3\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_5\",\"UserAlias\":\"first(c4)\",\"Name\":\"first\",\"Id\":\"33\",\"Type\":\"504\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"c4\",\"UserAlias\":\"c4\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"5\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c4\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}},{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"9\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"ts\",\"UserAlias\":\"ts\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"1\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"ts\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"11\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"#expr_6\",\"UserAlias\":\"last(c5)\",\"Name\":\"last\",\"Id\":\"36\",\"Type\":\"506\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"11\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"c5\",\"UserAlias\":\"c5\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"6\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c5\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}},{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"9\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"ts\",\"UserAlias\":\"ts\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"1\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"ts\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"12\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"2\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"7\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_7\",\"UserAlias\":\"apercentile(c6, 50)\",\"Name\":\"apercentile\",\"Id\":\"12\",\"Type\":\"1\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"12\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"2\"},\"AliasName\":\"c6\",\"UserAlias\":\"c6\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"7\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c6\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"c0c7c76d30bd3dcaefc96f40275bdc0a\",\"UserAlias\":\"50\",\"LiteralSize\":\"2\",\"Literal\":\"50\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"50\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"13\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"7\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_8\",\"UserAlias\":\"avg(c7)\",\"Name\":\"avg\",\"Id\":\"8\",\"Type\":\"2\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"13\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"c7\",\"UserAlias\":\"c7\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"8\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c7\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"14\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_9\",\"UserAlias\":\"count(c8)\",\"Name\":\"count\",\"Id\":\"0\",\"Type\":\"3\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"14\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"c8\",\"UserAlias\":\"c8\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"9\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c8\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"6\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"4\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"7\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_10\",\"UserAlias\":\"spread(c1)\",\"Name\":\"spread\",\"Id\":\"17\",\"Type\":\"11\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"c1\",\"UserAlias\":\"c1\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"2\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c1\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"7\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_11\",\"UserAlias\":\"stddev(c2)\",\"Name\":\"stddev\",\"Id\":\"4\",\"Type\":\"12\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"3\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"2\"},\"AliasName\":\"c2\",\"UserAlias\":\"c2\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"3\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c2\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"8\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_12\",\"UserAlias\":\"hyperloglog(c11)\",\"Name\":\"hyperloglog\",\"Id\":\"43\",\"Type\":\"17\",\"Parameters\":[{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"8\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"c11\",\"UserAlias\":\"c11\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"12\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"c11\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"10\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"26\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"#expr_13\",\"UserAlias\":\"timediff(1, 0, 1h)\",\"Name\":\"timediff\",\"Id\":\"81\",\"Type\":\"2501\",\"Parameters\":[{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"c4ca4238a0b923820dcc509a6f75849b\",\"UserAlias\":\"1\",\"LiteralSize\":\"1\",\"Literal\":\"1\",\"Duration\":false,\"Translate\":true,\"NotReserved\":false,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"1\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"cfcd208495d565ef66e7dff9f98764da\",\"UserAlias\":\"0\",\"LiteralSize\":\"1\",\"Literal\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":false,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"104\",\"Bytes\":\"8\"},\"AliasName\":\"7c68645d71b803bf0ba2f22519f73e08\",\"UserAlias\":\"1h\",\"LiteralSize\":\"2\",\"Literal\":\"1h\",\"Duration\":true,\"Translate\":true,\"NotReserved\":false,\"IsNull\":false,\"Unit\":\"104\",\"Datum\":\"3600000\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}},{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"1\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"Name\":\"cast\",\"Id\":\"77\",\"Type\":\"2000\",\"Parameters\":[{\"NodeType\":\"5\",\"Name\":\"Function\",\"Function\":{\"DataType\":{\"Type\":\"8\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"96\"},\"AliasName\":\"#expr_14\",\"UserAlias\":\"timezone()\",\"Name\":\"timezone\",\"Id\":\"84\",\"Type\":\"2503\",\"UdfBufSize\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"2\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":true,\"NotReserved\":true,\"IsNull\":false,\"Unit\":\"0\",\"Datum\":\"0\"}}],\"UdfBufSize\":\"0\"}}],\"From\":{\"NodeType\":\"6\",\"Name\":\"RealTable\",\"RealTable\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"0\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"DbName\":\"test\",\"tableName\":\"at_once_interval_ext_stb\",\"tableAlias\":\"at_once_interval_ext_stb\",\"MetaSize\":\"2008\",\"Meta\":{\"VgId\":\"0\",\"TableType\":\"1\",\"Uid\":\"5129202035162885657\",\"Suid\":\"5129202035162885657\",\"Sversion\":\"1\",\"Tversion\":\"1\",\"ComInfo\":{\"NumOfTags\":\"13\",\"Precision\":\"0\",\"NumOfColumns\":\"14\",\"RowSize\":\"85\"},\"ColSchemas\":[{\"Type\":\"9\",\"ColId\":\"1\",\"bytes\":\"8\",\"Name\":\"ts\"},{\"Type\":\"2\",\"ColId\":\"2\",\"bytes\":\"1\",\"Name\":\"c1\"},{\"Type\":\"3\",\"ColId\":\"3\",\"bytes\":\"2\",\"Name\":\"c2\"},{\"Type\":\"4\",\"ColId\":\"4\",\"bytes\":\"4\",\"Name\":\"c3\"},{\"Type\":\"5\",\"ColId\":\"5\",\"bytes\":\"8\",\"Name\":\"c4\"},{\"Type\":\"11\",\"ColId\":\"6\",\"bytes\":\"1\",\"Name\":\"c5\"},{\"Type\":\"12\",\"ColId\":\"7\",\"bytes\":\"2\",\"Name\":\"c6\"},{\"Type\":\"13\",\"ColId\":\"8\",\"bytes\":\"4\",\"Name\":\"c7\"},{\"Type\":\"14\",\"ColId\":\"9\",\"bytes\":\"8\",\"Name\":\"c8\"},{\"Type\":\"6\",\"ColId\":\"10\",\"bytes\":\"4\",\"Name\":\"c9\"},{\"Type\":\"7\",\"ColId\":\"11\",\"bytes\":\"8\",\"Name\":\"c10\"},{\"Type\":\"8\",\"ColId\":\"12\",\"bytes\":\"8\",\"Name\":\"c11\"},{\"Type\":\"10\",\"ColId\":\"13\",\"bytes\":\"26\",\"Name\":\"c12\"},{\"Type\":\"1\",\"ColId\":\"14\",\"bytes\":\"1\",\"Name\":\"c13\"},{\"Type\":\"2\",\"ColId\":\"15\",\"bytes\":\"1\",\"Name\":\"t1\"},{\"Type\":\"3\",\"ColId\":\"16\",\"bytes\":\"2\",\"Name\":\"t2\"},{\"Type\":\"4\",\"ColId\":\"17\",\"bytes\":\"4\",\"Name\":\"t3\"},{\"Type\":\"5\",\"ColId\":\"18\",\"bytes\":\"8\",\"Name\":\"t4\"},{\"Type\":\"11\",\"ColId\":\"19\",\"bytes\":\"1\",\"Name\":\"t5\"},{\"Type\":\"12\",\"ColId\":\"20\",\"bytes\":\"2\",\"Name\":\"t6\"},{\"Type\":\"13\",\"ColId\":\"21\",\"bytes\":\"4\",\"Name\":\"t7\"},{\"Type\":\"14\",\"ColId\":\"22\",\"bytes\":\"8\",\"Name\":\"t8\"},{\"Type\":\"6\",\"ColId\":\"23\",\"bytes\":\"4\",\"Name\":\"t9\"},{\"Type\":\"7\",\"ColId\":\"24\",\"bytes\":\"8\",\"Name\":\"t10\"},{\"Type\":\"8\",\"ColId\":\"25\",\"bytes\":\"8\",\"Name\":\"t11\"},{\"Type\":\"10\",\"ColId\":\"26\",\"bytes\":\"26\",\"Name\":\"t12\"},{\"Type\":\"1\",\"ColId\":\"27\",\"bytes\":\"1\",\"Name\":\"t13\"}]},\"VgroupsInfoSize\":\"1340\",\"VgroupsInfo\":{\"Num\":\"2\",\"Vgroups\":[{\"VgId\":\"14\",\"HashBegin\":\"0\",\"HashEnd\":\"2147483646\",\"EpSet\":{\"InUse\":\"0\",\"NumOfEps\":\"1\",\"Eps\":[{\"Fqdn\":\"localhost\",\"Port\":\"6030\"}]},\"NumOfTable\":\"0\"},{\"VgId\":\"15\",\"HashBegin\":\"2147483647\",\"HashEnd\":\"4294967295\",\"EpSet\":{\"InUse\":\"0\",\"NumOfEps\":\"1\",\"Eps\":[{\"Fqdn\":\"localhost\",\"Port\":\"6030\"}]},\"NumOfTable\":\"0\"}]}}},\"Tags\":[{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}},{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"0\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"1\"},\"AliasName\":\"\",\"UserAlias\":\"\",\"LiteralSize\":\"0\",\"Duration\":false,\"Translate\":false,\"NotReserved\":false,\"IsNull\":true,\"Unit\":\"0\"}}],\"Window\":{\"NodeType\":\"14\",\"Name\":\"IntervalWindow\",\"IntervalWindow\":{\"Interval\":{\"NodeType\":\"2\",\"Name\":\"Value\",\"Value\":{\"DataType\":{\"Type\":\"5\",\"Precision\":\"0\",\"Scale\":\"115\",\"Bytes\":\"8\"},\"AliasName\":\"1fd7635317edfeca9054894ac9ef9b5e\",\"UserAlias\":\"14s\",\"LiteralSize\":\"3\",\"Literal\":\"14s\",\"Duration\":true,\"Translate\":true,\"NotReserved\":false,\"IsNull\":false,\"Unit\":\"115\",\"Datum\":\"14000\"}},\"TsPk\":{\"NodeType\":\"1\",\"Name\":\"Column\",\"Column\":{\"DataType\":{\"Type\":\"9\",\"Precision\":\"0\",\"Scale\":\"0\",\"Bytes\":\"8\"},\"AliasName\":\"ts\",\"UserAlias\":\"ts\",\"TableId\":\"5129202035162885657\",\"TableType\":\"1\",\"ColId\":\"1\",\"ProjId\":\"0\",\"ColType\":\"1\",\"DbName\":\"test\",\"TableName\":\"at_once_interval_ext_stb\",\"TableAlias\":\"at_once_interval_ext_stb\",\"ColName\":\"ts\",\"DataBlockId\":\"0\",\"SlotId\":\"0\"}}}},\"StmtName\":\"0x150146d14\",\"HasAggFuncs\":true}}"; SNode * pAst = NULL; SQueryPlan *pPlan = NULL; if (taosCreateLog("taoslog", 10, "/etc/taos", NULL, NULL, NULL, NULL, 1) != 0) { - // ignore create log failed, only print - printf(" WARING: Create failed:%s. configDir\n", strerror(errno)); + // ignore create log failed, only print + printf(" WARING: Create failed:%s. configDir\n", strerror(errno)); } if (nodesStringToNode(ast, &pAst) < 0) { - ASSERT(0); + ASSERT(0); } - SPlanContext cxt = { 0 }; + SPlanContext cxt = {0}; cxt.pAstRoot = pAst; cxt.topicQuery = false; cxt.streamQuery = true; @@ -181,10 +259,11 @@ TEST(testCase, plan_Test) { // using ast and param to build physical plan if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { - ASSERT(0); + ASSERT(0); } + if (pAst != NULL) nodesDestroyNode(pAst); - nodesDestroyNode((SNode *)pPlan); + nodesDestroyNode((SNode*)pPlan); } #pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index 9d1125dad2..e2d4a1f6fc 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -106,7 +106,7 @@ typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj); typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj); typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc); typedef int32_t (*SdbDeployFp)(SMnode *pMnode); -typedef int32_t (*SdbValidateFp)(SMnode *pMnode, void *pTrans, void *pObj); +typedef int32_t (*SdbValidateFp)(SMnode *pMnode, void *pTrans, SSdbRaw *pRaw); typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw); typedef SSdbRaw *(*SdbEncodeFp)(void *pObj); typedef bool (*sdbTraverseFp)(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 9a017e7074..3bef5b595b 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -178,6 +178,12 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { return tqStreamTaskProcessCheckRsp(pSnode->pMeta, pMsg, true); case TDMT_STREAM_TASK_CHECKPOINT_READY: return tqStreamTaskProcessCheckpointReadyMsg(pSnode->pMeta, pMsg); + case TDMT_MND_STREAM_HEARTBEAT_RSP: + return tqStreamProcessStreamHbRsp(pSnode->pMeta, pMsg); + case TDMT_MND_STREAM_REQ_CHKPT_RSP: + return tqStreamProcessReqCheckpointRsp(pSnode->pMeta, pMsg); + case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP: + return tqStreamProcessCheckpointReadyRsp(pSnode->pMeta, pMsg); default: sndError("invalid snode msg:%d", pMsg->msgType); ASSERT(0); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index fb6a86843c..e610161544 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -47,6 +47,8 @@ #include "vnode.h" +#include "taos_monitor.h" + #ifdef __cplusplus extern "C" { #endif @@ -104,6 +106,20 @@ typedef struct SQueryNode SQueryNode; #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" +#define VNODE_METRIC_SQL_COUNT "taos_sql_req:count" + +#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type" +#define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id" +#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id" +#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep" +#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id" +#define VNODE_METRIC_TAG_NAME_USERNAME "username" +#define VNODE_METRIC_TAG_NAME_RESULT "result" + +#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows" +//#define VNODE_METRIC_TAG_VALUE_INSERT "insert" +//#define VNODE_METRIC_TAG_VALUE_DELETE "delete" + // vnd.h typedef int32_t (*_query_reseek_func_t)(void* pQHandle); struct SQueryNode { @@ -242,6 +258,9 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); int32_t tqScanWal(STQ* pTq); @@ -442,6 +461,13 @@ typedef struct SVCommitSched { int64_t maxWaitMs; } SVCommitSched; +typedef struct SVMonitorObj{ + char strClusterId[TSDB_CLUSTER_ID_LEN]; + char strDnodeId[TSDB_NODE_ID_LEN]; + char strVgId[TSDB_VGROUP_ID_LEN]; + taos_counter_t *insertCounter; +}SVMonitorObj; + struct SVnode { char* path; SVnodeCfg config; @@ -480,6 +506,7 @@ struct SVnode { int32_t blockSec; int64_t blockSeq; SQHandle* pQuery; + SVMonitorObj monitor; }; #define TD_VID(PVNODE) ((PVNODE)->config.vgId) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index a2c7a04ab3..4a978c8f41 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -1008,11 +1008,11 @@ void metaDropTables(SMeta *pMeta, SArray *tbUids) { } tSimpleHashPut(suidHash, &suid, sizeof(tb_uid_t), &nCtbDropped, sizeof(int64_t)); } - + /* if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { tsdbCacheDropTable(pMeta->pVnode->pTsdb, uid, suid, NULL); } - + */ metaDebug("batch drop table:%" PRId64, uid); } metaULock(pMeta); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6b7420b55f..47900d540c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -917,6 +917,22 @@ static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask } } +int32_t handleStep2Async(SStreamTask* pStreamTask, void* param) { + STQ* pTq = param; + + SStreamMeta* pMeta = pStreamTask->pMeta; + STaskId hId = pStreamTask->hTaskInfo.id; + SStreamTask* pTask = streamMetaAcquireTask(pStreamTask->pMeta, hId.streamId, hId.taskId); + if (pTask == NULL) { + // todo handle error + } + + doStartFillhistoryStep2(pTask, pStreamTask, pTq); + + streamMetaReleaseTask(pMeta, pTask); + return 0; +} + // this function should be executed by only one thread, so we set an sentinel to protect this function int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { SStreamScanHistoryReq* pReq = (SStreamScanHistoryReq*)pMsg->pCont; @@ -1007,37 +1023,27 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { // the following procedure should be executed, no matter status is stop/pause or not tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, pTask->execInfo.step1El); - if (pTask->info.fillHistory) { - SStreamTask* pStreamTask = NULL; + ASSERT(pTask->info.fillHistory == 1); - // 1. get the related stream task - pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); - if (pStreamTask == NULL) { - tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s", - pTask->streamTaskId.taskId, pTask->id.idStr); + // 1. get the related stream task + SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); + if (pStreamTask == NULL) { + tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s", + pTask->streamTaskId.taskId, pTask->id.idStr); tqDebug("s-task:%s fill-history task set status to be dropping and drop it", id); streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0); - atomic_store_32(&pTask->status.inScanHistorySentinel, 0); - streamMetaReleaseTask(pMeta, pTask); - return -1; - } - - ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE); - - code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); - if (code == TSDB_CODE_SUCCESS) { - doStartFillhistoryStep2(pTask, pStreamTask, pTq); - } else { - tqError("s-task:%s failed to halt s-task:%s, not launch step2", id, pStreamTask->id.idStr); - } - - streamMetaReleaseTask(pMeta, pStreamTask); - } else { - ASSERT(0); + atomic_store_32(&pTask->status.inScanHistorySentinel, 0); + streamMetaReleaseTask(pMeta, pTask); + return -1; } + ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE); + code = streamTaskHandleEventAsync(pStreamTask->status.pSM, TASK_EVENT_HALT, handleStep2Async, pTq); + + streamMetaReleaseTask(pMeta, pStreamTask); + atomic_store_32(&pTask->status.inScanHistorySentinel, 0); streamMetaReleaseTask(pMeta, pTask); return code; @@ -1175,7 +1181,11 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) return TSDB_CODE_SUCCESS; } } else { - ASSERT(status == TASK_STATUS__HALT); +// ASSERT(status == TASK_STATUS__HALT); + if (status != TASK_STATUS__HALT) { + tqError("s-task:%s should in halt status, let's halt it directly", pTask->id.idStr); +// streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT); + } } // check if the checkpoint msg already sent or not. @@ -1220,3 +1230,16 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { return tqStreamTaskProcessTaskResetReq(pTq->pStreamMeta, pMsg); } + +// this function is needed, do not try to remove it. +int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg) { + return tqStreamProcessStreamHbRsp(pTq->pStreamMeta, pMsg); +} + +int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg) { + return tqStreamProcessReqCheckpointRsp(pTq->pStreamMeta, pMsg); +} + +int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg) { + return tqStreamProcessCheckpointReadyRsp(pTq->pStreamMeta, pMsg); +} diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 42bd1bc59c..9ff4e3ba62 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -72,7 +72,7 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); memcpy(name, varDataVal(varTbName), varDataLen(varTbName)); if (newSubTableRule && !isAutoTableName(name) && !alreadyAddGroupId(name) && groupId != 0) { - buildCtbNameAddGruopId(name, groupId); + buildCtbNameAddGroupId(name, groupId); } } else if (stbFullName) { name = buildCtbNameByGroupId(stbFullName, groupId); @@ -185,23 +185,26 @@ void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDa !alreadyAddGroupId(pDataBlock->info.parTbName) && gid != 0) { pCreateTableReq->name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); strcpy(pCreateTableReq->name, pDataBlock->info.parTbName); - buildCtbNameAddGruopId(pCreateTableReq->name, gid); + buildCtbNameAddGroupId(pCreateTableReq->name, gid); +// tqDebug("gen name from:%s", pDataBlock->info.parTbName); } else { pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName); +// tqDebug("copy name:%s", pDataBlock->info.parTbName); } } else { pCreateTableReq->name = buildCtbNameByGroupId(stbFullName, gid); +// tqDebug("gen name from stbFullName:%s gid:%"PRId64, stbFullName, gid); } } static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, int64_t suid) { - tqDebug("s-task:%s build create table msg", pTask->id.idStr); - STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; int32_t rows = pDataBlock->info.rows; SArray* tagArray = taosArrayInit(4, sizeof(STagVal)); - ; + + tqDebug("s-task:%s build create %d table(s) msg", pTask->id.idStr, rows); + int32_t code = 0; SVCreateTbBatchReq reqs = {0}; @@ -670,7 +673,8 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat } else { if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) && !alreadyAddGroupId(dstTableName) && groupId != 0) { - buildCtbNameAddGruopId(dstTableName, groupId); + tqDebug("s-task:%s append groupId:%" PRId64 " for generated dstTable:%s", id, groupId, dstTableName); + buildCtbNameAddGroupId(dstTableName, groupId); } } diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 280c110711..73508202d9 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -28,8 +28,8 @@ static int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDurat // extract data blocks(submit/delete) from WAL, and add them into the input queue for all the sources tasks. int32_t tqScanWal(STQ* pTq) { - int32_t vgId = TD_VID(pTq->pVnode); SStreamMeta* pMeta = pTq->pStreamMeta; + int32_t vgId = pMeta->vgId; int64_t st = taosGetTimestampMs(); tqDebug("vgId:%d continue to check if data in wal are available, scanCounter:%d", vgId, pMeta->scanInfo.scanCounter); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 3b7fe3cfdd..9bfdd70477 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -142,8 +142,10 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); if (ppHTask == NULL || *ppHTask == NULL) { - tqError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already", - vgId, req.taskId); + tqError( + "vgId:%d failed to acquire fill-history task:0x%x when handling update, may have been dropped already, rel " + "stream task:0x%x", + vgId, (uint32_t)pTask->hTaskInfo.id.taskId, req.taskId); CLEAR_RELATED_FILLHISTORY_TASK(pTask); } else { tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr); @@ -485,6 +487,10 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe return code; } +typedef struct SMStreamCheckpointReadyRspMsg { + SMsgHead head; +}SMStreamCheckpointReadyRspMsg; + int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t vgId = pMeta->vgId; char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); @@ -513,6 +519,18 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) streamProcessCheckpointReadyMsg(pTask); streamMetaReleaseTask(pMeta, pTask); + + { // send checkpoint ready rsp + SRpcMsg rsp = {.code = 0, .info = pMsg->info, .contLen = sizeof(SMStreamCheckpointReadyRspMsg)}; + rsp.pCont = rpcMallocCont(rsp.contLen); + SMsgHead* pHead = rsp.pCont; + pHead->vgId = htonl(req.downstreamNodeId); + + tmsgSendRsp(&rsp); + + pMsg->info.handle = NULL; // disable auto rsp + } + return code; } @@ -596,23 +614,35 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) { SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; + int32_t vgId = pMeta->vgId; + STaskId hTaskId = {0}; - int32_t vgId = pMeta->vgId; tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); - if (pTask != NULL) { - // drop the related fill-history task firstly + streamMetaWLock(pMeta); + + STaskId id = {.streamId = pReq->streamId, .taskId = pReq->taskId}; + SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + if ((ppTask != NULL) && ((*ppTask) != NULL)) { + streamMetaAcquireOneTask(*ppTask); + SStreamTask* pTask = *ppTask; + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - STaskId* pHTaskId = &pTask->hTaskInfo.id; - streamMetaUnregisterTask(pMeta, pHTaskId->streamId, pHTaskId->taskId); - tqDebug("s-task:0x%x vgId:%d drop fill-history task:0x%x firstly", pReq->taskId, vgId, - (int32_t)pHTaskId->taskId); + hTaskId.streamId = pTask->hTaskInfo.id.streamId; + hTaskId.taskId = pTask->hTaskInfo.id.taskId; } + + streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt); streamMetaReleaseTask(pMeta, pTask); } - streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt, true); + streamMetaWUnLock(pMeta); + + // drop the related fill-history task firstly + if (hTaskId.taskId != 0 && hTaskId.streamId != 0) { + streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId); + tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId); + } // drop the stream task now streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId); @@ -849,7 +879,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg){ pHistoryTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId); if (pHistoryTask == NULL) { tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64 - ", it may have been dropped already", + ", it may have been dropped already", pMeta->vgId, pTask->hTaskInfo.id.taskId); streamMetaReleaseTask(pMeta, pTask); @@ -936,4 +966,19 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m int32_t tqStreamTasksGetTotalNum(SStreamMeta* pMeta) { return taosArrayGetSize(pMeta->pTaskList); +} + +static int32_t doProcessDummyRspMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + + return TSDB_CODE_SUCCESS; +} + +int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } + +int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } + +int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { + return doProcessDummyRspMsg(pMeta, pMsg); } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 1ef2a451a7..0848fd0076 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -789,25 +789,6 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, int8_ return code; } -static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, int8_t ltype) { - SLastCol *pLastCol = NULL; - - char *err = NULL; - size_t vlen = 0; - SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid}; - size_t klen = ROCKS_KEY_LEN; - char *value = NULL; - value = rocksdb_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, (char *)key, klen, &vlen, &err); - if (NULL != err) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); - rocksdb_free(err); - } - - pLastCol = tsdbCacheDeserialize(value); - - return pLastCol; -} - typedef struct { int idx; SLastKey key; @@ -1052,6 +1033,25 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols, int nCols, int16_t *slotIds); #ifdef BUILD_NO_CALL +static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, int8_t ltype) { + SLastCol *pLastCol = NULL; + + char *err = NULL; + size_t vlen = 0; + SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid}; + size_t klen = ROCKS_KEY_LEN; + char *value = NULL; + value = rocksdb_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, (char *)key, klen, &vlen, &err); + if (NULL != err) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); + rocksdb_free(err); + } + + pLastCol = tsdbCacheDeserialize(value); + + return pLastCol; +} + int32_t tsdbCacheGetSlow(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) { rocksdb_writebatch_t *wb = NULL; int32_t code = 0; @@ -1233,10 +1233,10 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr int16_t *lastSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); int16_t *lastrowColIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); int16_t *lastrowSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); - SArray* lastTmpColArray = NULL; - SArray* lastTmpIndexArray = NULL; - SArray* lastrowTmpColArray = NULL; - SArray* lastrowTmpIndexArray = NULL; + SArray *lastTmpColArray = NULL; + SArray *lastTmpIndexArray = NULL; + SArray *lastrowTmpColArray = NULL; + SArray *lastrowTmpIndexArray = NULL; int lastIndex = 0; int lastrowIndex = 0; @@ -1245,7 +1245,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr SIdxKey *idxKey = taosArrayGet(remainCols, i); slotIds[i] = pr->pSlotIds[idxKey->idx]; if (idxKey->key.ltype == CACHESCAN_RETRIEVE_LAST >> 3) { - if(NULL == lastTmpIndexArray) { + if (NULL == lastTmpIndexArray) { lastTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t)); } taosArrayPush(lastTmpIndexArray, &(i)); @@ -1253,7 +1253,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr lastSlotIds[lastIndex] = pr->pSlotIds[idxKey->idx]; lastIndex++; } else { - if(NULL == lastrowTmpIndexArray) { + if (NULL == lastrowTmpIndexArray) { lastrowTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t)); } taosArrayPush(lastrowTmpIndexArray, &(i)); @@ -1265,17 +1265,18 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr pTmpColArray = taosArrayInit(lastIndex + lastrowIndex, sizeof(SLastCol)); - if(lastTmpIndexArray != NULL) { + if (lastTmpIndexArray != NULL) { mergeLastCid(uid, pTsdb, &lastTmpColArray, pr, lastColIds, lastIndex, lastSlotIds); - for(int i = 0; i < taosArrayGetSize(lastTmpColArray); i++) { - taosArrayInsert(pTmpColArray, *(int32_t*)taosArrayGet(lastTmpIndexArray, i), taosArrayGet(lastTmpColArray, i)); + for (int i = 0; i < taosArrayGetSize(lastTmpColArray); i++) { + taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastTmpIndexArray, i), taosArrayGet(lastTmpColArray, i)); } } - if(lastrowTmpIndexArray != NULL) { + if (lastrowTmpIndexArray != NULL) { mergeLastRowCid(uid, pTsdb, &lastrowTmpColArray, pr, lastrowColIds, lastrowIndex, lastrowSlotIds); - for(int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) { - taosArrayInsert(pTmpColArray, *(int32_t*)taosArrayGet(lastrowTmpIndexArray, i), taosArrayGet(lastrowTmpColArray, i)); + for (int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) { + taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastrowTmpIndexArray, i), + taosArrayGet(lastrowTmpColArray, i)); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 86f58717e2..90a26d17dc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2543,7 +2543,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr while (1) { // only check here, since the iterate data in memory is very fast. if (pReader->code != TSDB_CODE_SUCCESS) { - tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr); return pReader->code; } @@ -2694,7 +2694,7 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { while (1) { if (pReader->code != TSDB_CODE_SUCCESS) { - tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr); return pReader->code; } @@ -2909,7 +2909,7 @@ static int32_t buildBlockFromBufferSeqForPreFileset(STsdbReader* pReader, int64_ while (1) { if (pReader->code != TSDB_CODE_SUCCESS) { - tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr); return pReader->code; } @@ -2951,7 +2951,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t en while (1) { if (pReader->code != TSDB_CODE_SUCCESS) { - tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr); return pReader->code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 6aff1c2930..81a136a0e3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -448,8 +448,8 @@ _exit: taosMemoryFree(reader[0]); reader[0] = NULL; } else { - tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), - sver, ever, type); + tsdbInfo("vgId:%d, tsdb snapshot incremental reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d", + TD_VID(tsdb->pVnode), sver, ever, type); } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index b7c22aa0e9..ae840eb6d2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -69,7 +69,8 @@ _exit: taosMemoryFree(reader[0]); reader[0] = NULL; } else { - tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), ever, type); + tsdbInfo("vgId:%d, tsdb snapshot raw reader opened. sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), ever, + type); } return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 946ce9d278..63ca7251f5 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -480,6 +480,32 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC vnodeRollback(pVnode); } + snprintf(pVnode->monitor.strClusterId, TSDB_CLUSTER_ID_LEN, "%"PRId64, pVnode->config.syncCfg.nodeInfo[0].clusterId); + snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%"PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId); + snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%"PRId32, pVnode->config.vgId); + + if(tsEnableMonitor && pVnode->monitor.insertCounter == NULL){ + taos_counter_t *counter = NULL; + counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT); + if(counter == NULL){ + int32_t label_count = 7; + const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID, + VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP, + VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME, + VNODE_METRIC_TAG_NAME_RESULT}; + counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql", + label_count, sample_labels); + vInfo("vgId:%d, new metric:%p",TD_VID(pVnode), counter); + if(taos_collector_registry_register_metric(counter) == 1){ + taos_counter_destroy(counter); + counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT); + vInfo("vgId:%d, get metric from registry:%p",TD_VID(pVnode), counter); + } + } + pVnode->monitor.insertCounter = counter; + vInfo("vgId:%d, succeed to set metric:%p",TD_VID(pVnode), counter); + } + return pVnode; _err: diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 4fc7a88494..d1c811858a 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "vnd.h" #include "tsdb.h" +#include "vnd.h" #define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType, tags) \ do { \ @@ -49,7 +49,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { // decode req if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) { code = TSDB_CODE_INVALID_MSG; - goto _exit; + goto _exit4; } metaRsp.dbId = pVnode->config.dbId; @@ -59,7 +59,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName); code = vnodeValidateTableHash(pVnode, tableFName); if (code) { - goto _exit; + goto _exit4; } // query meta @@ -67,7 +67,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { if (metaGetTableEntryByName(&mer1, infoReq.tbName) < 0) { code = terrno; - goto _exit; + goto _exit3; } metaRsp.tableType = mer1.me.type; @@ -81,7 +81,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaRsp.suid = mer1.me.uid; } else if (mer1.me.type == TSDB_CHILD_TABLE) { metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); - if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; + if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit2; strcpy(metaRsp.stbName, mer2.me.name); metaRsp.suid = mer2.me.uid; @@ -125,6 +125,12 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); _exit: + taosMemoryFree(metaRsp.pSchemas); +_exit2: + metaReaderClear(&mer2); +_exit3: + metaReaderClear(&mer1); +_exit4: rpcMsg.info = pMsg->info; rpcMsg.pCont = pRsp; rpcMsg.contLen = rspLen; @@ -141,9 +147,6 @@ _exit: *pMsg = rpcMsg; } - taosMemoryFree(metaRsp.pSchemas); - metaReaderClear(&mer2); - metaReaderClear(&mer1); return TSDB_CODE_SUCCESS; } @@ -706,5 +709,5 @@ void *vnodeGetIvtIdx(void *pVnode) { } int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid) { - return tsdbGetTableSchema(((SVnode*)pVnode)->pMeta, uid, pSchema, suid); + return tsdbGetTableSchema(((SVnode *)pVnode)->pMeta, uid, pSchema, suid); } diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index ed1dcc64c9..b4c8a26f5c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -172,6 +172,23 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader goto _err; } + // open tsdb snapshot raw reader + if (!pReader->tsdbRAWDone) { + ASSERT(pReader->sver == 0); + code = tsdbSnapRAWReaderOpen(pVnode->pTsdb, ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader); + if (code) goto _err; + } + + // check snapshot ever + SSnapshot snapshot = {0}; + vnodeGetSnapshot(pVnode, &snapshot); + if (ever != snapshot.lastApplyIndex) { + vError("vgId:%d, abort reader open due to vnode snapshot changed. ever:%" PRId64 ", commit ver:%" PRId64, + TD_VID(pVnode), ever, snapshot.lastApplyIndex); + code = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _err; + } + vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever); *ppReader = pReader; return code; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index e4d7b11176..e32d4b70e0 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -16,6 +16,7 @@ #include "audit.h" #include "cos.h" #include "tencode.h" +#include "tglobal.h" #include "tmsg.h" #include "tstrbuild.h" #include "vnd.h" @@ -30,13 +31,15 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, SRpcMsg *pOriginRpc); -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg); static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg); static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -542,10 +545,10 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg break; /* TSDB */ case TDMT_VND_SUBMIT: - if (vnodeProcessSubmitReq(pVnode, ver, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err; + if (vnodeProcessSubmitReq(pVnode, ver, pMsg->pCont, pMsg->contLen, pRsp, pMsg) < 0) goto _err; break; case TDMT_VND_DELETE: - if (vnodeProcessDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDeleteReq(pVnode, ver, pReq, len, pRsp, pMsg) < 0) goto _err; break; case TDMT_VND_BATCH_DEL: if (vnodeProcessBatchDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; @@ -796,6 +799,12 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) return tqProcessTaskScanHistory(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_CHECKPOINT_READY: return tqProcessTaskCheckpointReadyMsg(pVnode->pTq, pMsg); + case TDMT_MND_STREAM_HEARTBEAT_RSP: + return tqProcessStreamHbRsp(pVnode->pTq, pMsg); + case TDMT_MND_STREAM_REQ_CHKPT_RSP: + return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg); + case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP: + return tqProcessTaskCheckpointReadyRsp(pVnode->pTq, pMsg); default: vError("unknown msg type:%d in stream queue", pMsg->msgType); return TSDB_CODE_APP_ERROR; @@ -1480,7 +1489,8 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) { return code; } -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg) { int32_t code = 0; terrno = 0; @@ -1698,10 +1708,35 @@ _exit: atomic_add_fetch_64(&pVnode->statis.nInsert, pSubmitRsp->affectedRows); atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows); atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); + + if(tsEnableMonitor && pSubmitRsp->affectedRows > 0 && strlen(pOriginalMsg->info.conn.user) > 0){ + const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Success"}; + taos_counter_add(pVnode->monitor.insertCounter, pSubmitRsp->affectedRows, sample_labels); + } + if (code == 0) { atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); code = tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len); } + /* + if (code == 0) { + atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); + code = tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len); + + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Success"}; + taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); + } + else{ + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Failed"}; + taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); + } + */ // clear taosArrayDestroy(newTbUids); @@ -1957,7 +1992,8 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pRe return 0; } -static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg) { int32_t code = 0; SDecoder *pCoder = &(SDecoder){0}; SDeleteRes *pRes = &(SDeleteRes){0}; @@ -2000,6 +2036,21 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, in return code; _err: + /* + if(code == TSDB_CODE_SUCCESS){ + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_DELETE, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Success"}; + taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); + } + else{ + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_DELETE, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Failed"}; + taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); + } + */ + return code; } static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 5f4b7b8442..f9f2ae6b21 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -372,8 +372,8 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int32_t code = syncProcessMsg(pVnode->sync, pMsg); if (code != 0) { - vGError("vgId:%d, failed to process sync msg:%p type:%s since %s", pVnode->config.vgId, pMsg, - TMSG_INFO(pMsg->msgType), terrstr()); + vGError("vgId:%d, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pVnode->config.vgId, pMsg, + TMSG_INFO(pMsg->msgType), terrstr(), code); } return code; diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 9f812517c1..6196d30a58 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -8,6 +8,7 @@ add_subdirectory(qcom) add_subdirectory(nodes) add_subdirectory(catalog) add_subdirectory(audit) +add_subdirectory(monitorfw) add_subdirectory(scalar) add_subdirectory(function) diff --git a/source/libs/executor/src/countwindowoperator.c b/source/libs/executor/src/countwindowoperator.c index 3980e5ae4d..1f38264644 100644 --- a/source/libs/executor/src/countwindowoperator.c +++ b/source/libs/executor/src/countwindowoperator.c @@ -94,10 +94,10 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < pBlock->info.rows;) { - int32_t step = pInfo->windowSliding; SCountWindowResult* pBuffInfo = setCountWindowOutputBuff(pExprSup, &pInfo->countSup, &pInfo->pRow); int32_t prevRows = pBuffInfo->winRows; int32_t num = updateCountWindowInfo(i, pBlock->info.rows, pInfo->windowCount, &pBuffInfo->winRows); + int32_t step = num; if (prevRows == 0) { pInfo->pRow->win.skey = tsCols[i]; } @@ -118,6 +118,8 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) { if (prevRows <= pInfo->windowSliding) { if (pBuffInfo->winRows > pInfo->windowSliding) { step = pInfo->windowSliding - prevRows; + } else { + step = pInfo->windowSliding; } } else { step = 0; @@ -129,7 +131,7 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) { return code; } -static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock) { +static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SExecTaskInfo* pTaskInfo, SFilterInfo* pFilterInfo, SSDataBlock* pBlock) { SResultRow* pResultRow = NULL; for (int32_t i = 0; i < taosArrayGetSize(pCountSup->pWinStates); i++) { SCountWindowResult* pBuff = setCountWindowOutputBuff(pExprSup, pCountSup, &pResultRow); @@ -143,6 +145,7 @@ static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, S clearWinStateBuff(pBuff); clearResultRowInitFlag(pExprSup->pCtx, pExprSup->numOfExprs); } + doFilter(pBlock, pFilterInfo, NULL); } static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) { @@ -177,7 +180,7 @@ static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) { if (pInfo->groupId == 0) { pInfo->groupId = pBlock->info.id.groupId; } else if (pInfo->groupId != pBlock->info.id.groupId) { - buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pRes); + buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pOperator->exprSupp.pFilterInfo, pRes); pInfo->groupId = pBlock->info.id.groupId; } @@ -187,7 +190,7 @@ static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) { } } - buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pRes); + buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pOperator->exprSupp.pFilterInfo, pRes); return pRes->info.rows == 0 ? NULL : pRes; } @@ -246,6 +249,11 @@ SOperatorInfo* createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo pInfo->countSup.stateIndex = 0; + code = filterInitFromNode((SNode*)pCountWindowNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); setOperatorInfo(pOperator, "CountWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT, true, OP_NOT_OPENED, pInfo, diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 2534c5e9f0..831fd4e883 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1009,6 +1009,22 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) { pSup->deleteMark = INT64_MAX; pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData; pInfo->ignoreExpiredData = false; + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT) { + SStreamCountAggOperatorInfo* pInfo = pOperator->info; + STimeWindowAggSupp* pSup = &pInfo->twAggSup; + + ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0); + + qInfo("save stream param for state: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); + + pSup->calTriggerSaved = pSup->calTrigger; + pSup->deleteMarkSaved = pSup->deleteMark; + pSup->calTrigger = STREAM_TRIGGER_AT_ONCE; + pSup->deleteMark = INT64_MAX; + pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData; + pInfo->ignoreExpiredData = false; + qInfo("save stream task:%s, param for state: %d", GET_TASKID(pTaskInfo), pInfo->ignoreExpiredData); } // iterate operator tree diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index cc83ecd84e..691210e5ba 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -461,7 +461,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy _error: destroyIndefinitOperatorInfo(pInfo); taosMemoryFree(pOperator); - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + pTaskInfo->code = code; return NULL; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 693f325556..9024f7a341 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3424,12 +3424,6 @@ _error: // table merge scan operator // table merge scan operator -// TODO: limit / duration optimization -// TODO: get block from tsdReader function, with task killed, func_data all filter out, skip, finish -// TODO: error processing, memory freeing -// TODO: add log for error and perf -// TODO: tsdb reader open/close dynamically -// TODO: blockdata deep cleanup static int32_t subTblRowCompareFn(const void* pLeft, const void* pRight, void* param) { int32_t left = *(int32_t*)pLeft; @@ -3769,8 +3763,9 @@ static int32_t stopSubTablesTableMergeScan(STableMergeScanInfo* pInfo) { taosMemoryFree(pSubTblsInfo); pInfo->pSubTablesMergeInfo = NULL; + + taosMemoryTrim(0); } - taosMemoryTrim(0); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 706b4c5a01..f9c7b51316 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -99,6 +99,9 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, (void**)&pCurWin->winInfo.pStatePos, &size); } } + if (ts < pCurWin->winInfo.sessionWin.win.ekey) { + pBuffInfo->rebuildWindow = true; + } } else { code = pAggSup->stateStore.streamStateCountWinAddIfNotExist( pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size); @@ -115,8 +118,16 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, } } -static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowInfo* pWinInfo, TSKEY* pTs, int32_t start, int32_t rows, int32_t maxRows, - SSHashObj* pStDeleted, bool* pRebuild) { +static void removeCountResult(SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey) { + SSessionKey key = {0}; + getSessionHashKey(pKey, &key); + tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); +} + +static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowInfo* pWinInfo, TSKEY* pTs, + int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated, + SSHashObj* pStDeleted, bool* pRebuild) { SSessionKey sWinKey = pWinInfo->winInfo.sessionWin; int32_t num = 0; for (int32_t i = start; i < rows; i++) { @@ -148,6 +159,7 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI if (needDelState) { memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey)); + removeCountResult(pStUpdated, pAggSup->pResultRows, &sWinKey); if (pWinInfo->winInfo.pStatePos->needFree) { pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &sWinKey); } @@ -164,7 +176,7 @@ void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, ESt } else { pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, pKey); } - SSessionKey tmpKey = {0}; + SSessionKey tmpKey = {.groupId = pKey->groupId, .win.ekey = INT64_MIN, .win.skey = INT64_MIN}; int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); if (code != TSDB_CODE_SUCCESS) { pAggSup->stateStore.streamStateFreeCur(pCur); @@ -242,7 +254,8 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl setSessionWinOutputInfo(pStUpdated, &curWin.winInfo); slidingRows = *curWin.pWindowCount; if (!buffInfo.rebuildWindow) { - winRows = updateCountWindowInfo(pAggSup, &curWin, startTsCols, i, rows, pAggSup->windowCount, pStDeleted, &buffInfo.rebuildWindow); + winRows = updateCountWindowInfo(pAggSup, &curWin, startTsCols, i, rows, pAggSup->windowCount, pStUpdated, + pStDeleted, &buffInfo.rebuildWindow); } if (buffInfo.rebuildWindow) { SSessionKey range = {0}; @@ -274,6 +287,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + curWin.winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &key); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); @@ -331,7 +345,7 @@ int32_t doStreamCountEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper size_t keyLen = 0; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pInfo->streamAggSup.pResultRows, pIte, &iter)) != NULL) { - void* key = taosHashGetKey(pIte, &keyLen); + void* key = tSimpleHashGetKey(pIte, &keyLen); tlen += encodeSSessionKey(buf, key); tlen += encodeSResultWindowInfo(buf, pIte, pInfo->streamAggSup.resultRowSize); } @@ -424,7 +438,7 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, &key); } while (1) { - SSessionKey tmpKey = {0}; + SSessionKey tmpKey = {.groupId = gpDatas[i], .win.skey = INT64_MIN, .win.ekey = INT64_MIN}; int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void **)&pPos, &size); if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) { pAggSup->stateStore.streamStateFreeCur(pCur); diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index fde5f22aaa..10220426a3 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -1125,8 +1125,11 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { int64_t firstRowTs = *(int64_t*)tsCol->pData; if ((pOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs) || (pOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) { - continue; - } + if (bExtractedBlock) { + blockDataDestroy(pBlk); + } + continue; + } } if (pBlk != NULL) { @@ -1149,10 +1152,11 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { tSimpleHashClear(mUidBlk); code = sortBlocksToExtSource(pHandle, aBlkSort, pOrder, aExtSrc); if (code != TSDB_CODE_SUCCESS) { - tSimpleHashCleanup(mUidBlk); - taosArrayDestroy(aBlkSort); - taosArrayDestroy(aExtSrc); - return code; + for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) { + blockDataDestroy(taosArrayGetP(aBlkSort, i)); + } + taosArrayClear(aBlkSort); + break; } int64_t el = taosGetTimestampUs() - p; @@ -1165,6 +1169,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { szSort = 0; qDebug("source %zu created", taosArrayGetSize(aExtSrc)); } + if (pBlk == NULL) { break; } @@ -1180,6 +1185,9 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { } tSimpleHashCleanup(mUidBlk); + for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) { + blockDataDestroy(taosArrayGetP(aBlkSort, i)); + } taosArrayDestroy(aBlkSort); tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL); if (!tsortIsClosed(pHandle)) { @@ -1188,7 +1196,19 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { taosArrayDestroy(aExtSrc); tSimpleHashCleanup(mTableNumRows); pHandle->type = SORT_SINGLESOURCE_SORT; - return TSDB_CODE_SUCCESS; + return code; +} + +static void freeSSortSource(SSortSource* source) { + if (NULL == source) return; + if (source->param && !source->onlyRef) { + taosMemoryFree(source->param); + } + if (!source->onlyRef && source->src.pBlock) { + blockDataDestroy(source->src.pBlock); + source->src.pBlock = NULL; + } + taosMemoryFree(source); } static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { @@ -1223,14 +1243,7 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { code = blockDataMerge(pHandle->pDataBlock, pBlock); if (code != TSDB_CODE_SUCCESS) { - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); - } - if (!source->onlyRef && source->src.pBlock) { - blockDataDestroy(source->src.pBlock); - source->src.pBlock = NULL; - } - taosMemoryFree(source); + freeSSortSource(source); return code; } @@ -1240,15 +1253,7 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { int64_t p = taosGetTimestampUs(); code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); if (code != 0) { - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); - } - if (!source->onlyRef && source->src.pBlock) { - blockDataDestroy(source->src.pBlock); - source->src.pBlock = NULL; - } - - taosMemoryFree(source); + freeSSortSource(source); return code; } @@ -1257,16 +1262,13 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { if (pHandle->pqMaxRows > 0) blockDataKeepFirstNRows(pHandle->pDataBlock, pHandle->pqMaxRows); code = doAddToBuf(pHandle->pDataBlock, pHandle); if (code != TSDB_CODE_SUCCESS) { + freeSSortSource(source); return code; } } } - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); - } - - taosMemoryFree(source); + freeSSortSource(source); if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) { size_t size = blockDataGetSize(pHandle->pDataBlock); diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 1313221952..b7b9f1cc9f 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -76,8 +76,8 @@ char* idxInt2str(int64_t val, char* dst, int radix) { return dst - 1; } __compar_fn_t idxGetCompar(int8_t type) { - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || - type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_NCHAR || + type == TSDB_DATA_TYPE_GEOMETRY) { return (__compar_fn_t)strcmp; } return getComparFunc(type, 0); @@ -108,8 +108,8 @@ static FORCE_INLINE TExeCond tCompareEqual(void* a, void* b, int8_t type) { return tCompare(func, QUERY_TERM, a, b, type); } TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) { - if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || - dtype == TSDB_DATA_TYPE_VARBINARY || dtype == TSDB_DATA_TYPE_GEOMETRY) { + if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY || + dtype == TSDB_DATA_TYPE_GEOMETRY) { return tDoCompare(func, cmptype, a, b); } #if 1 @@ -290,6 +290,7 @@ int idxUidCompare(const void* a, const void* b) { uint64_t r = *(uint64_t*)b; return l - r; } +#ifdef BUILD_NO_CALL int32_t idxConvertData(void* src, int8_t type, void** dst) { int tlen = -1; switch (type) { @@ -372,6 +373,8 @@ int32_t idxConvertData(void* src, int8_t type, void** dst) { // indexMayFillNumbericData(*dst, tlen); return tlen; } +#endif + int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { if (src == NULL) { *dst = strndup(INDEX_DATA_NULL_STR, (int)strlen(INDEX_DATA_NULL_STR)); diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index 1ee835573b..eff74f93d2 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -175,4 +175,8 @@ IF(NOT TD_DARWIN) NAME idxFstUT COMMAND idxFstUT ) + add_test( + NAME idxFstTest + COMMAND idxFstTest + ) ENDIF () diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index b889a2209a..adf783d2eb 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -1,4 +1,5 @@ +#include #include #include #include @@ -14,6 +15,12 @@ #include "tutil.h" void* callback(void* s) { return s; } +class FstEnv : public ::testing::Test { + protected: + virtual void SetUp() {} + virtual void TearDown() {} +}; + static std::string fileName = TD_TMP_DIR_PATH "tindex.tindex"; class FstWriter { public: @@ -154,7 +161,7 @@ class FstReadMemory { int32_t _size; }; -#define L 100 +#define L 200 #define M 100 #define N 100 @@ -200,7 +207,7 @@ void checkMillonWriteAndReadOfFst() { FstWriter* fw = new FstWriter; Performance_fstWriteRecords(fw); delete fw; - FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024); + FstReadMemory* fr = new FstReadMemory(1024 * 8 * 1024); if (fr->init()) { printf("success to init fst read"); @@ -637,23 +644,31 @@ void iterTFileReader(char* path, char* uid, char* colName, char* ver) { tfileIteratorDestroy(iter); } -int main(int argc, char* argv[]) { - // tool to check all kind of fst test - // if (argc > 1) { validateTFile(argv[1]); } - // if (argc > 4) { - // path suid colName ver - // iterTFileReader(argv[1], argv[2], argv[3], argv[4]); - //} - checkFstCheckIterator1(); - // checkFstCheckIterator2(); - // checkFstCheckIteratorPrefix(); - // checkFstCheckIteratorRange1(); - // checkFstCheckIteratorRange2(); - // checkFstCheckIteratorRange3(); - // checkFstLongTerm(); - // checkFstPrefixSearch(); +// int main(int argc, char* argv[]) { +// // tool to check all kind of fst test +// // if (argc > 1) { validateTFile(argv[1]); } +// // if (argc > 4) { +// // path suid colName ver +// // iterTFileReader(argv[1], argv[2], argv[3], argv[4]); +// //} +// checkFstCheckIterator1(); +// // checkFstCheckIterator2(); +// // checkFstCheckIteratorPrefix(); +// // checkFstCheckIteratorRange1(); +// // checkFstCheckIteratorRange2(); +// // checkFstCheckIteratorRange3(); +// // checkFstLongTerm(); +// // checkFstPrefixSearch(); - // checkMillonWriteAndReadOfFst(); +// // checkMillonWriteAndReadOfFst(); - return 1; -} +// return 1; +// } +TEST_F(FstEnv, checkIterator1) { checkFstCheckIterator1(); } +TEST_F(FstEnv, checkItertor2) { checkFstCheckIterator2(); } +TEST_F(FstEnv, checkPrefix) { checkFstCheckIteratorPrefix(); } +TEST_F(FstEnv, checkRange1) { checkFstCheckIteratorRange1(); } +TEST_F(FstEnv, checkRange2) { checkFstCheckIteratorRange2(); } +TEST_F(FstEnv, checkRange3) { checkFstCheckIteratorRange3(); } +TEST_F(FstEnv, checkLongTerm) { checkFstLongTerm(); } +TEST_F(FstEnv, checkMillonWriteData) { checkMillonWriteAndReadOfFst(); } diff --git a/source/libs/monitor/CMakeLists.txt b/source/libs/monitor/CMakeLists.txt index 30dce7aaef..cc8f40fa4c 100644 --- a/source/libs/monitor/CMakeLists.txt +++ b/source/libs/monitor/CMakeLists.txt @@ -6,7 +6,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries(monitor os util common transport) +target_link_libraries(monitor os util common qcom transport monitorfw) if(${BUILD_TEST}) add_subdirectory(test) diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h index ae1af4ba62..7fc718393b 100644 --- a/source/libs/monitor/inc/monInt.h +++ b/source/libs/monitor/inc/monInt.h @@ -17,8 +17,10 @@ #define _TD_MONITOR_INT_H_ #include "monitor.h" +#include "query.h" #include "tjson.h" +#include "thash.h" typedef struct { int64_t curTime; @@ -44,8 +46,22 @@ typedef struct { SMonSmInfo smInfo; SMonQmInfo qmInfo; SMonBmInfo bmInfo; + SHashObj *metrics; } SMonitor; +void monGenClusterInfoTable(SMonInfo *pMonitor); +void monGenVgroupInfoTable(SMonInfo *pMonitor); +void monGenDnodeInfoTable(SMonInfo *pMonitor); +void monGenDnodeStatusInfoTable(SMonInfo *pMonitor); +void monGenDataDiskTable(SMonInfo *pMonitor); +void monGenLogDiskTable(SMonInfo *pMonitor); +void monGenMnodeRoleTable(SMonInfo *pMonitor); +void monGenVnodeRoleTable(SMonInfo *pMonitor); + +void monSendPromReport(); +void monInitMonitorFW(); +void monCleanupMonitorFW(); + #ifdef __cplusplus } #endif diff --git a/source/libs/monitor/src/clientMonitor.c b/source/libs/monitor/src/clientMonitor.c new file mode 100644 index 0000000000..0891932583 --- /dev/null +++ b/source/libs/monitor/src/clientMonitor.c @@ -0,0 +1,210 @@ +#include "clientMonitor.h" +#include "os.h" +#include "tmisce.h" +#include "ttime.h" +#include "ttimer.h" +#include "tglobal.h" + +SRWLatch monitorLock; +void* tmrClientMonitor; +tmr_h tmrStartHandle; +SHashObj* clusterMonitorInfoTable; + +static int interval = 30 * 1000; +static int sendBathchSize = 1; + +int32_t sendReport(ClientMonitor* pMonitor, char* pCont); +void generateClusterReport(ClientMonitor* pMonitor, bool send) { + char ts[50]; + sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + char* pCont = (char*)taos_collector_registry_bridge_new(pMonitor->registry, ts, "%" PRId64, NULL); + if(NULL == pCont) { + uError("generateClusterReport failed, get null content."); + return; + } + if (send && strlen(pCont) != 0) { + if (sendReport(pMonitor, pCont) == 0) { + taos_collector_registry_clear_batch(pMonitor->registry); + } + } + taosMemoryFreeClear(pCont); +} + +void reportSendProcess(void* param, void* tmrId) { + taosTmrReset(reportSendProcess, tsMonitorInterval * 1000, NULL, tmrClientMonitor, &tmrStartHandle); + taosRLockLatch(&monitorLock); + + static int index = 0; + index++; + ClientMonitor** ppMonitor = (ClientMonitor**)taosHashIterate(clusterMonitorInfoTable, NULL); + while (ppMonitor != NULL && *ppMonitor != NULL) { + ClientMonitor* pMonitor = *ppMonitor; + generateClusterReport(*ppMonitor, index == sendBathchSize); + ppMonitor = taosHashIterate(clusterMonitorInfoTable, ppMonitor); + } + + if (index == sendBathchSize) index = 0; + taosRUnLockLatch(&monitorLock); +} + +void monitorClientInitOnce() { + static int8_t init = 0; + if (atomic_exchange_8(&init, 1) == 0) { + uInfo("tscMonitorInit once."); + clusterMonitorInfoTable = + (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + + tmrClientMonitor = taosTmrInit(0, 0, 0, "MONITOR"); + tmrStartHandle = taosTmrStart(reportSendProcess, tsMonitorInterval * 1000, NULL, tmrClientMonitor); + if(tsMonitorInterval < 1){ + interval = 30 * 1000; + } else { + interval = tsMonitorInterval * 1000; + } + if (tsMonitorInterval < 10) { + sendBathchSize = (10 / sendBathchSize) + 1; + } + taosInitRWLatch(&monitorLock); + } +} + +void createMonitorClient(const char* clusterKey, SEpSet epSet, void* pTransporter) { + if (clusterKey == NULL || strlen(clusterKey) == 0) { + uError("createMonitorClient failed, clusterKey is NULL"); + return; + } + taosWLockLatch(&monitorLock); + if (taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)) == NULL) { + uInfo("createMonitorClient for %s.", clusterKey); + ClientMonitor* pMonitor = taosMemoryCalloc(1, sizeof(ClientMonitor)); + snprintf(pMonitor->clusterKey, sizeof(pMonitor->clusterKey), "%s", clusterKey); + pMonitor->registry = taos_collector_registry_new(clusterKey); + pMonitor->colector = taos_collector_new(clusterKey); + epsetAssign(&pMonitor->epSet, &epSet); + pMonitor->pTransporter = pTransporter; + + taos_collector_registry_register_collector(pMonitor->registry, pMonitor->colector); + pMonitor->counters = + (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + + taosHashPut(clusterMonitorInfoTable, clusterKey, strlen(clusterKey), &pMonitor, sizeof(ClientMonitor*)); + uInfo("createMonitorClient for %s finished %p.", clusterKey, pMonitor); + } + taosWUnLockLatch(&monitorLock); +} + +static int32_t monitorReportAsyncCB(void* param, SDataBuf* pMsg, int32_t code) { + static int32_t emptyRspNum = 0; + if (TSDB_CODE_SUCCESS != code) { + uError("found error in monitorReport send callback, code:%d, please check the network.", code); + } + if (pMsg) { + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); + } + return code; +} + +int32_t sendReport(ClientMonitor* pMonitor, char* pCont) { + SStatisReq sStatisReq; + sStatisReq.pCont = pCont; + sStatisReq.contLen = strlen(pCont); + + int tlen = tSerializeSStatisReq(NULL, 0, &sStatisReq); + if (tlen < 0) return 0; + void* buf = taosMemoryMalloc(tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + tSerializeSStatisReq(buf, tlen, &sStatisReq); + + SMsgSendInfo* pInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (pInfo == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pInfo->fp = monitorReportAsyncCB; + pInfo->msgInfo.pData = buf; + pInfo->msgInfo.len = tlen; + pInfo->msgType = TDMT_MND_STATIS; + // pInfo->param = taosMemoryMalloc(sizeof(int32_t)); + // *(int32_t*)pInfo->param = i; + pInfo->paramFreeFp = taosMemoryFree; + pInfo->requestId = tGenIdPI64(); + pInfo->requestObjRefId = 0; + + int64_t transporterId = 0; + return asyncSendMsgToServer(pMonitor->pTransporter, &pMonitor->epSet, &transporterId, pInfo); +} + +void clusterMonitorInit(const char* clusterKey, SEpSet epSet, void* pTransporter) { + monitorClientInitOnce(); + createMonitorClient(clusterKey, epSet, pTransporter); +} + +taos_counter_t* createClusterCounter(const char* clusterKey, const char* name, const char* help, size_t label_key_count, + const char** label_keys) { + ClientMonitor** ppMonitor = (ClientMonitor**)taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)); + + if (ppMonitor != NULL && *ppMonitor != NULL) { + ClientMonitor* pMonitor = *ppMonitor; + taos_counter_t** ppCounter = (taos_counter_t**)taosHashGet(pMonitor->counters, name, strlen(name)); + if (ppCounter != NULL && *ppCounter != NULL) { + taosHashRemove(pMonitor->counters, name, strlen(name)); + uInfo("createClusterCounter remove old counter: %s.", name); + } + + taos_counter_t* newCounter = taos_counter_new(name, help, label_key_count, label_keys); + if (newCounter != NULL) { + taos_collector_add_metric(pMonitor->colector, newCounter); + taosHashPut(pMonitor->counters, name, strlen(name), &newCounter, sizeof(taos_counter_t*)); + uInfo("createClusterCounter %s(%p):%s : %p.", pMonitor->clusterKey, pMonitor, name, newCounter); + return newCounter; + } else { + return NULL; + } + } else { + return NULL; + } + return NULL; +} + +int taosClusterCounterInc(const char* clusterKey, const char* counterName, const char** label_values) { + taosRLockLatch(&monitorLock); + ClientMonitor** ppMonitor = (ClientMonitor**)taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)); + + if (ppMonitor != NULL && *ppMonitor != NULL) { + ClientMonitor* pMonitor = *ppMonitor; + taos_counter_t** ppCounter = (taos_counter_t**)taosHashGet(pMonitor->counters, counterName, strlen(counterName)); + if (ppCounter != NULL && *ppCounter != NULL) { + taos_counter_inc(*ppCounter, label_values); + } else { + uError("taosClusterCounterInc not found pCounter %s:%s.", clusterKey, counterName); + } + } else { + uError("taosClusterCounterInc not found pMonitor %s.", clusterKey); + } + taosRUnLockLatch(&monitorLock); + return 0; +} + +void clusterMonitorClose(const char* clusterKey) { + taosWLockLatch(&monitorLock); + ClientMonitor** ppMonitor = (ClientMonitor**)taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)); + + if (ppMonitor != NULL && *ppMonitor != NULL) { + ClientMonitor* pMonitor = *ppMonitor; + uInfo("clusterMonitorClose valule:%p clusterKey:%s.", pMonitor, pMonitor->clusterKey); + taosHashCleanup(pMonitor->counters); + taos_collector_registry_destroy(pMonitor->registry); + taosMemoryFree(pMonitor); + taosHashRemove(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)); + } + taosWUnLockLatch(&monitorLock); +} + +const char* resultStr(SQL_RESULT_CODE code) { + static const char* result_state[] = {"Success", "Failed", "Cancel"}; + return result_state[code]; +} diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c new file mode 100644 index 0000000000..488b4d1d58 --- /dev/null +++ b/source/libs/monitor/src/monFramework.c @@ -0,0 +1,763 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "monitor.h" +#include "monInt.h" + +#include "thash.h" +#include "taos_monitor.h" +#include "thttp.h" +#include "ttime.h" +#include "tglobal.h" + +extern SMonitor tsMonitor; +extern char* tsMonUri; +extern char* tsMonFwUri; + +#define LEVEL_LEN 11 + +#define CLUSTER_TABLE "taosd_cluster_info" + +#define MASTER_UPTIME CLUSTER_TABLE":cluster_uptime" +#define DBS_TOTAL CLUSTER_TABLE":dbs_total" +#define TBS_TOTAL CLUSTER_TABLE":tbs_total" +#define STBS_TOTAL CLUSTER_TABLE":stbs_total" +#define VGROUPS_TOTAL CLUSTER_TABLE":vgroups_total" +#define VGROUPS_ALIVE CLUSTER_TABLE":vgroups_alive" +#define VNODES_TOTAL CLUSTER_TABLE":vnodes_total" +#define VNODES_ALIVE CLUSTER_TABLE":vnodes_alive" +#define DNODES_TOTAL CLUSTER_TABLE":dnodes_total" +#define DNODES_ALIVE CLUSTER_TABLE":dnodes_alive" +#define MNODES_TOTAL CLUSTER_TABLE":mnodes_total" +#define MNODES_ALIVE CLUSTER_TABLE":mnodes_alive" +#define CONNECTIONS_TOTAL CLUSTER_TABLE":connections_total" +#define TOPICS_TOTAL CLUSTER_TABLE":topics_total" +#define STREAMS_TOTAL CLUSTER_TABLE":streams_total" +#define EXPIRE_TIME CLUSTER_TABLE":grants_expire_time" +#define TIMESERIES_USED CLUSTER_TABLE":grants_timeseries_used" +#define TIMESERIES_TOTAL CLUSTER_TABLE":grants_timeseries_total" + +#define VGROUP_TABLE "taosd_vgroups_info" + +#define TABLES_NUM VGROUP_TABLE":tables_num" +#define STATUS VGROUP_TABLE":status" + +#define DNODE_TABLE "taosd_dnodes_info" + +#define UPTIME DNODE_TABLE":uptime" +#define CPU_ENGINE DNODE_TABLE":cpu_engine" +#define CPU_SYSTEM DNODE_TABLE":cpu_system" +#define CPU_CORE DNODE_TABLE":cpu_cores" +#define MEM_ENGINE DNODE_TABLE":mem_engine" +#define MEM_SYSTEM DNODE_TABLE":mem_free" +#define MEM_TOTAL DNODE_TABLE":mem_total" +#define DISK_ENGINE DNODE_TABLE":disk_engine" +#define DISK_USED DNODE_TABLE":disk_used" +#define DISK_TOTAL DNODE_TABLE":disk_total" +#define NET_IN DNODE_TABLE":system_net_in" +#define NET_OUT DNODE_TABLE":system_net_out" +#define IO_READ DNODE_TABLE":io_read" +#define IO_WRITE DNODE_TABLE":io_write" +#define IO_READ_DISK DNODE_TABLE":io_read_disk" +#define IO_WRITE_DISK DNODE_TABLE":io_write_disk" +//#define ERRORS DNODE_TABLE":errors" +#define VNODES_NUM DNODE_TABLE":vnodes_num" +#define MASTERS DNODE_TABLE":masters" +#define HAS_MNODE DNODE_TABLE":has_mnode" +#define HAS_QNODE DNODE_TABLE":has_qnode" +#define HAS_SNODE DNODE_TABLE":has_snode" +#define DNODE_LOG_ERROR DNODE_TABLE":error_log_count" +#define DNODE_LOG_INFO DNODE_TABLE":info_log_count" +#define DNODE_LOG_DEBUG DNODE_TABLE":debug_log_count" +#define DNODE_LOG_TRACE DNODE_TABLE":trace_log_count" + +#define DNODE_STATUS "taosd_dnodes_status:status" + +#define DATADIR_TABLE "taosd_dnodes_data_dirs" + +#define DNODE_DATA_AVAIL DATADIR_TABLE":avail" +#define DNODE_DATA_USED DATADIR_TABLE":used" +#define DNODE_DATA_TOTAL DATADIR_TABLE":total" + +#define LOGDIR_TABLE "taosd_dnodes_log_dirs" + +#define DNODE_LOG_AVAIL LOGDIR_TABLE":avail" +#define DNODE_LOG_USED LOGDIR_TABLE":used" +#define DNODE_LOG_TOTAL LOGDIR_TABLE":total" + +#define MNODE_ROLE "taosd_mnodes_info:role" +#define VNODE_ROLE "taosd_vnodes_info:role" + +void monInitMonitorFW(){ + taos_collector_registry_default_init(); + + tsMonitor.metrics = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + taos_gauge_t *gauge = NULL; + + int32_t dnodes_label_count = 3; + const char *dnodes_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep"}; + char *dnodes_gauges[] = {UPTIME, CPU_ENGINE, CPU_SYSTEM, CPU_CORE, MEM_ENGINE, MEM_SYSTEM, + MEM_TOTAL, DISK_ENGINE, DISK_USED, DISK_TOTAL, NET_IN, + NET_OUT, IO_READ, IO_WRITE, IO_READ_DISK, IO_WRITE_DISK, /*ERRORS,*/ + VNODES_NUM, MASTERS, HAS_MNODE, HAS_QNODE, HAS_SNODE, + DNODE_LOG_ERROR, DNODE_LOG_INFO, DNODE_LOG_DEBUG, DNODE_LOG_TRACE}; + for(int32_t i = 0; i < 25; i++){ + gauge= taos_gauge_new(dnodes_gauges[i], "", dnodes_label_count, dnodes_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, dnodes_gauges[i], strlen(dnodes_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } + + int32_t dnodes_data_label_count = 5; + const char *dnodes_data_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep", "data_dir_name", "data_dir_level"}; + char *dnodes_data_gauges[] = {DNODE_DATA_AVAIL, DNODE_DATA_USED, DNODE_DATA_TOTAL}; + for(int32_t i = 0; i < 3; i++){ + gauge= taos_gauge_new(dnodes_data_gauges[i], "", dnodes_data_label_count, dnodes_data_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, dnodes_data_gauges[i], strlen(dnodes_data_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } + + int32_t dnodes_log_label_count = 4; + const char *dnodes_log_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep", "data_dir_name"}; + char *dnodes_log_gauges[] = {DNODE_LOG_AVAIL, DNODE_LOG_USED, DNODE_LOG_TOTAL}; + for(int32_t i = 0; i < 3; i++){ + gauge= taos_gauge_new(dnodes_log_gauges[i], "", dnodes_log_label_count, dnodes_log_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, dnodes_log_gauges[i], strlen(dnodes_log_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } +} + +void monCleanupMonitorFW(){ + taosHashCleanup(tsMonitor.metrics); + taos_collector_registry_destroy(TAOS_COLLECTOR_REGISTRY_DEFAULT); + TAOS_COLLECTOR_REGISTRY_DEFAULT = NULL; +} + +void monGenClusterInfoTable(SMonInfo *pMonitor){ + SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; + SMonBasicInfo *pBasicInfo = &pMonitor->dmInfo.basic; + SMonGrantInfo *pGrantInfo = &pMonitor->mmInfo.grant; + + char *metric_names[] = {MASTER_UPTIME, DBS_TOTAL, TBS_TOTAL, STBS_TOTAL, VGROUPS_TOTAL, + VGROUPS_ALIVE, VNODES_TOTAL, VNODES_ALIVE, MNODES_TOTAL, MNODES_ALIVE, + CONNECTIONS_TOTAL, TOPICS_TOTAL, STREAMS_TOTAL, + DNODES_TOTAL, DNODES_ALIVE, EXPIRE_TIME, TIMESERIES_USED, + TIMESERIES_TOTAL}; + + for(int32_t i = 0; i < 18; i++){ + if(taos_collector_registry_deregister_metric(metric_names[i]) != 0){ + uError("failed to delete metric %s", metric_names[i]); + } + + taosHashRemove(tsMonitor.metrics, metric_names[i], strlen(metric_names[i])); + } + + if(pBasicInfo->cluster_id == 0) { + uError("failed to generate dnode info table since cluster_id is 0"); + return; + } + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + //cluster info + taos_gauge_t *gauge = NULL; + int32_t label_count =1; + const char *sample_labels1[] = {"cluster_id"}; + + for(int32_t i = 0; i < 18; i++){ + gauge= taos_gauge_new(metric_names[i], "", label_count, sample_labels1); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, metric_names[i], strlen(metric_names[i]), &gauge, sizeof(taos_gauge_t *)); + } + + char buf[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(buf, TSDB_CLUSTER_ID_LEN, "%"PRId64, pBasicInfo->cluster_id); + const char *sample_label_values[] = {buf}; + + taos_gauge_t **metric = NULL; + + metric = taosHashGet(tsMonitor.metrics, MASTER_UPTIME, strlen(MASTER_UPTIME)); + taos_gauge_set(*metric, pInfo->master_uptime, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, DBS_TOTAL, strlen(DBS_TOTAL)); + taos_gauge_set(*metric, pInfo->dbs_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, TBS_TOTAL, strlen(TBS_TOTAL)); + taos_gauge_set(*metric, pInfo->tbs_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, STBS_TOTAL, strlen(STBS_TOTAL)); + taos_gauge_set(*metric, pInfo->stbs_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, VGROUPS_TOTAL, strlen(VGROUPS_TOTAL)); + taos_gauge_set(*metric, pInfo->vgroups_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, VGROUPS_ALIVE, strlen(VGROUPS_ALIVE)); + taos_gauge_set(*metric, pInfo->vgroups_alive, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, VNODES_TOTAL, strlen(VNODES_TOTAL)); + taos_gauge_set(*metric, pInfo->vnodes_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, VNODES_ALIVE, strlen(VNODES_ALIVE)); + taos_gauge_set(*metric, pInfo->vnodes_alive, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, CONNECTIONS_TOTAL, strlen(CONNECTIONS_TOTAL)); + taos_gauge_set(*metric, pInfo->connections_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, TOPICS_TOTAL, strlen(TOPICS_TOTAL)); + taos_gauge_set(*metric, pInfo->topics_toal, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, STREAMS_TOTAL, strlen(STREAMS_TOTAL)); + taos_gauge_set(*metric, pInfo->streams_total, sample_label_values); + + //dnodes number + int32_t dnode_total = taosArrayGetSize(pInfo->dnodes); + int32_t dnode_alive = 0; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) { + SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i); + + if(strcmp(pDnodeDesc->status, "ready") == 0){ + dnode_alive++; + } + } + + metric = taosHashGet(tsMonitor.metrics, DNODES_TOTAL, strlen(DNODES_TOTAL)); + taos_gauge_set(*metric, dnode_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, DNODES_ALIVE, strlen(DNODES_ALIVE)); + taos_gauge_set(*metric, dnode_alive, sample_label_values); + + //mnodes number + int32_t mnode_total = taosArrayGetSize(pInfo->mnodes); + int32_t mnode_alive = 0; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->mnodes); ++i) { + + SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->mnodes, i); + + bool dnodeIsOnline = false; + for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) { + SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i); + + if(pDnodeDesc->dnode_id == pMnodeDesc->mnode_id){ + if(strcmp(pDnodeDesc->status, "ready") == 0){ + dnodeIsOnline = true; + } + } + } + + if(dnodeIsOnline){ + if(pMnodeDesc->syncState != 0){ + mnode_alive++; + } + } + } + + metric = taosHashGet(tsMonitor.metrics, MNODES_TOTAL, strlen(MNODES_TOTAL)); + taos_gauge_set(*metric, mnode_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, MNODES_ALIVE, strlen(MNODES_ALIVE)); + taos_gauge_set(*metric, mnode_alive, sample_label_values); + + //grant info + metric = taosHashGet(tsMonitor.metrics, EXPIRE_TIME, strlen(EXPIRE_TIME)); + taos_gauge_set(*metric, pGrantInfo->expire_time, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, TIMESERIES_USED, strlen(TIMESERIES_USED)); + taos_gauge_set(*metric, pGrantInfo->timeseries_used, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, TIMESERIES_TOTAL, strlen(TIMESERIES_TOTAL)); + taos_gauge_set(*metric, pGrantInfo->timeseries_total, sample_label_values); +} + +void monGenVgroupInfoTable(SMonInfo *pMonitor){ + if(taos_collector_registry_deregister_metric(TABLES_NUM) != 0){ + uError("failed to delete metric "TABLES_NUM); + } + + if(taos_collector_registry_deregister_metric(STATUS) != 0){ + uError("failed to delete metric "STATUS); + } + + if(pMonitor->dmInfo.basic.cluster_id == 0) return; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + int32_t vgroup_label_count = 3; + const char *vgroup_sample_labels[] = {"cluster_id", "vgroup_id", "database_name"}; + taos_gauge_t *tableNumGauge = taos_gauge_new(TABLES_NUM, "", vgroup_label_count, vgroup_sample_labels); + if(taos_collector_registry_register_metric(tableNumGauge) == 1){ + taos_counter_destroy(tableNumGauge); + } + taos_gauge_t *statusGauge = taos_gauge_new(STATUS, "", vgroup_label_count, vgroup_sample_labels); + if(taos_collector_registry_register_metric(statusGauge) == 1){ + taos_counter_destroy(statusGauge); + } + + char cluster_id[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%"PRId64, pMonitor->dmInfo.basic.cluster_id); + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->vgroups); ++i) { + SMonVgroupDesc *pVgroupDesc = taosArrayGet(pInfo->vgroups, i); + + char vgroup_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(vgroup_id, TSDB_NODE_ID_LEN, "%"PRId32, pVgroupDesc->vgroup_id); + + const char *sample_labels[] = {cluster_id, vgroup_id, pVgroupDesc->database_name}; + + taos_gauge_t **metric = NULL; + + taos_gauge_set(tableNumGauge, pVgroupDesc->tables_num, sample_labels); + + int32_t status = 0; + if(strcmp(pVgroupDesc->status, "ready") == 0){ + status = 1; + } + taos_gauge_set(statusGauge, status, sample_labels); + } +} + +void monGenDnodeInfoTable(SMonInfo *pMonitor) { + if(pMonitor->dmInfo.basic.cluster_id == 0) { + uError("failed to generate dnode info table since cluster_id is 0"); + return; + } + + char cluster_id[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%"PRId64, pMonitor->dmInfo.basic.cluster_id); + + char dnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMonitor->dmInfo.basic.dnode_id); + + const char *sample_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep}; + + taos_gauge_t **metric = NULL; + + //dnode info + SMonDnodeInfo *pInfo = &pMonitor->dmInfo.dnode; + SMonSysInfo *pSys = &pMonitor->dmInfo.sys; + SVnodesStat *pStat = &pMonitor->vmInfo.vstat; + SMonClusterInfo *pClusterInfo = &pMonitor->mmInfo.cluster; + + double interval = (pMonitor->curTime - pMonitor->lastTime) / 1000.0; + if (pMonitor->curTime - pMonitor->lastTime == 0) { + interval = 1; + } + + double cpu_engine = 0; + double mem_engine = 0; + double net_in = 0; + double net_out = 0; + double io_read = 0; + double io_write = 0; + double io_read_disk = 0; + double io_write_disk = 0; + + SMonSysInfo *sysArrays[6]; + sysArrays[0] = &pMonitor->dmInfo.sys; + sysArrays[1] = &pMonitor->mmInfo.sys; + sysArrays[2] = &pMonitor->vmInfo.sys; + sysArrays[3] = &pMonitor->qmInfo.sys; + sysArrays[4] = &pMonitor->smInfo.sys; + sysArrays[5] = &pMonitor->bmInfo.sys; + for (int32_t i = 0; i < 6; ++i) { + cpu_engine += sysArrays[i]->cpu_engine; + mem_engine += sysArrays[i]->mem_engine; + net_in += sysArrays[i]->net_in; + net_out += sysArrays[i]->net_out; + io_read += sysArrays[i]->io_read; + io_write += sysArrays[i]->io_write; + io_read_disk += sysArrays[i]->io_read_disk; + io_write_disk += sysArrays[i]->io_write_disk; + } + + double req_select_rate = pStat->numOfSelectReqs / interval; + double req_insert_rate = pStat->numOfInsertReqs / interval; + double req_insert_batch_rate = pStat->numOfBatchInsertReqs / interval; + double net_in_rate = net_in / interval; + double net_out_rate = net_out / interval; + double io_read_rate = io_read / interval; + double io_write_rate = io_write / interval; + double io_read_disk_rate = io_read_disk / interval; + double io_write_disk_rate = io_write_disk / interval; + + metric = taosHashGet(tsMonitor.metrics, UPTIME, strlen(UPTIME)); + taos_gauge_set(*metric, pInfo->uptime, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, CPU_ENGINE, strlen(CPU_ENGINE)); + taos_gauge_set(*metric, cpu_engine, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, CPU_SYSTEM, strlen(CPU_SYSTEM)); + taos_gauge_set(*metric, pSys->cpu_system, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, CPU_CORE, strlen(CPU_CORE)); + taos_gauge_set(*metric, pSys->cpu_cores, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, MEM_ENGINE, strlen(MEM_ENGINE)); + taos_gauge_set(*metric, mem_engine, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, MEM_SYSTEM, strlen(MEM_SYSTEM)); + taos_gauge_set(*metric, pSys->mem_system, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, MEM_TOTAL, strlen(MEM_TOTAL)); + taos_gauge_set(*metric, pSys->mem_total, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DISK_ENGINE, strlen(DISK_ENGINE)); + taos_gauge_set(*metric, pSys->disk_engine, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DISK_USED, strlen(DISK_USED)); + taos_gauge_set(*metric, pSys->disk_used, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DISK_TOTAL, strlen(DISK_TOTAL)); + taos_gauge_set(*metric, pSys->disk_total, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, NET_IN, strlen(NET_IN)); + taos_gauge_set(*metric, net_in_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, NET_OUT, strlen(NET_OUT)); + taos_gauge_set(*metric, net_out_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, IO_READ, strlen(IO_READ)); + taos_gauge_set(*metric, io_read_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, IO_WRITE, strlen(IO_WRITE)); + taos_gauge_set(*metric, io_write_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, IO_READ_DISK, strlen(IO_READ_DISK)); + taos_gauge_set(*metric, io_read_disk_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, IO_WRITE_DISK, strlen(IO_WRITE_DISK)); + taos_gauge_set(*metric, io_write_disk_rate, sample_labels); + + //metric = taosHashGet(tsMonitor.metrics, ERRORS, strlen(ERRORS)); + //taos_gauge_set(*metric, pStat->errors, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, VNODES_NUM, strlen(VNODES_NUM)); + taos_gauge_set(*metric, pStat->totalVnodes, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, MASTERS, strlen(MASTERS)); + taos_gauge_set(*metric, pStat->masterNum, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, HAS_MNODE, strlen(HAS_MNODE)); + taos_gauge_set(*metric, pInfo->has_mnode, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, HAS_QNODE, strlen(HAS_QNODE)); + taos_gauge_set(*metric, pInfo->has_qnode, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, HAS_SNODE, strlen(HAS_SNODE)); + taos_gauge_set(*metric, pInfo->has_snode, sample_labels); + + //log number + SMonLogs *logs[6]; + logs[0] = &pMonitor->log; + logs[1] = &pMonitor->mmInfo.log; + logs[2] = &pMonitor->vmInfo.log; + logs[3] = &pMonitor->smInfo.log; + logs[4] = &pMonitor->qmInfo.log; + logs[5] = &pMonitor->bmInfo.log; + + int32_t numOfErrorLogs = 0; + int32_t numOfInfoLogs = 0; + int32_t numOfDebugLogs = 0; + int32_t numOfTraceLogs = 0; + + for (int32_t j = 0; j < 6; j++) { + SMonLogs *pLog = logs[j]; + numOfErrorLogs += pLog->numOfErrorLogs; + numOfInfoLogs += pLog->numOfInfoLogs; + numOfDebugLogs += pLog->numOfDebugLogs; + numOfTraceLogs += pLog->numOfTraceLogs; + } + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_ERROR, strlen(DNODE_LOG_ERROR)); + taos_gauge_set(*metric, numOfErrorLogs, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_INFO, strlen(DNODE_LOG_INFO)); + taos_gauge_set(*metric, numOfInfoLogs, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_DEBUG, strlen(DNODE_LOG_DEBUG)); + taos_gauge_set(*metric, numOfDebugLogs, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_TRACE, strlen(DNODE_LOG_TRACE)); + taos_gauge_set(*metric, numOfTraceLogs, sample_labels); +} + +void monGenDnodeStatusInfoTable(SMonInfo *pMonitor){ + if(taos_collector_registry_deregister_metric(DNODE_STATUS) != 0){ + uError("failed to delete metric "DNODE_STATUS); + } + + if(pMonitor->dmInfo.basic.cluster_id == 0) { + uError("failed to generate dnode info table since cluster_id is 0"); + return; + } + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + taos_gauge_t *gauge = NULL; + + int32_t dnodes_label_count = 3; + const char *dnodes_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep"}; + + gauge= taos_gauge_new(DNODE_STATUS, "", dnodes_label_count, dnodes_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + + char cluster_id[TSDB_CLUSTER_ID_LEN]; + snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%"PRId64, pMonitor->dmInfo.basic.cluster_id); + + //dnodes status + + SMonClusterInfo *pClusterInfo = &pMonitor->mmInfo.cluster; + + for (int32_t i = 0; i < taosArrayGetSize(pClusterInfo->dnodes); ++i) { + SMonDnodeDesc *pDnodeDesc = taosArrayGet(pClusterInfo->dnodes, i); + + char dnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pDnodeDesc->dnode_id); + + const char *sample_labels[] = {cluster_id, dnode_id, pDnodeDesc->dnode_ep}; + + int32_t status = 0; + if(strcmp(pDnodeDesc->status, "ready") == 0){ + status = 1; + } + taos_gauge_set(gauge, status, sample_labels); + } +} + +void monGenDataDiskTable(SMonInfo *pMonitor){ + if(pMonitor->dmInfo.basic.cluster_id == 0) return; + + SMonDiskInfo *pInfo = &pMonitor->vmInfo.tfs; + + char cluster_id[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%" PRId64, pMonitor->dmInfo.basic.cluster_id); + + char dnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMonitor->dmInfo.basic.dnode_id); + + taos_gauge_t **metric = NULL; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->datadirs); ++i) { + SMonDiskDesc *pDatadirDesc = taosArrayGet(pInfo->datadirs, i); + + char level[LEVEL_LEN] = {0}; + snprintf(level, LEVEL_LEN, "%"PRId32, pDatadirDesc->level); + + const char *sample_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep, pDatadirDesc->name, level}; + + metric = taosHashGet(tsMonitor.metrics, DNODE_DATA_AVAIL, strlen(DNODE_DATA_AVAIL)); + taos_gauge_set(*metric, pDatadirDesc->size.avail, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_DATA_USED, strlen(DNODE_DATA_USED)); + taos_gauge_set(*metric, pDatadirDesc->size.used, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_DATA_TOTAL, strlen(DNODE_DATA_TOTAL)); + taos_gauge_set(*metric, pDatadirDesc->size.total, sample_labels); + } +} + +void monGenLogDiskTable(SMonInfo *pMonitor){ + if(pMonitor->dmInfo.basic.cluster_id == 0) return; + + SMonDiskDesc *pLogDesc = &pMonitor->dmInfo.dnode.logdir; + SMonDiskDesc *pTempDesc = &pMonitor->dmInfo.dnode.tempdir; + + char cluster_id[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%" PRId64, pMonitor->dmInfo.basic.cluster_id); + + char dnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMonitor->dmInfo.basic.dnode_id); + + taos_gauge_t **metric = NULL; + + const char *sample_log_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep, pLogDesc->name}; + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_AVAIL, strlen(DNODE_LOG_AVAIL)); + taos_gauge_set(*metric, pLogDesc->size.avail, sample_log_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_USED, strlen(DNODE_LOG_USED)); + taos_gauge_set(*metric, pLogDesc->size.used, sample_log_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_TOTAL, strlen(DNODE_LOG_TOTAL)); + taos_gauge_set(*metric, pLogDesc->size.total, sample_log_labels); + + const char *sample_temp_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep, pTempDesc->name}; + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_AVAIL, strlen(DNODE_LOG_AVAIL)); + taos_gauge_set(*metric, pTempDesc->size.avail, sample_temp_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_USED, strlen(DNODE_LOG_USED)); + taos_gauge_set(*metric, pTempDesc->size.used, sample_temp_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_TOTAL, strlen(DNODE_LOG_TOTAL)); + taos_gauge_set(*metric, pTempDesc->size.total, sample_temp_labels); +} + +void monGenMnodeRoleTable(SMonInfo *pMonitor){ + char *mnodes_role_gauges[] = {MNODE_ROLE}; + + for(int32_t i = 0; i < 1; i++){ + if(taos_collector_registry_deregister_metric(mnodes_role_gauges[i]) != 0){ + uError("failed to delete metric %s", mnodes_role_gauges[i]); + } + + taosHashRemove(tsMonitor.metrics, mnodes_role_gauges[i], strlen(mnodes_role_gauges[i])); + } + + SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + SMonBasicInfo *pBasicInfo = &pMonitor->dmInfo.basic; + if(pBasicInfo->cluster_id == 0) return; + + taos_gauge_t *gauge = NULL; + int32_t mnodes_role_label_count = 3; + const char *mnodes_role_sample_labels[] = {"cluster_id", "mnode_id", "mnode_ep"}; + for(int32_t i = 0; i < 1; i++){ + gauge= taos_gauge_new(mnodes_role_gauges[i], "", mnodes_role_label_count, mnodes_role_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, mnodes_role_gauges[i], strlen(mnodes_role_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } + + char buf[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(buf, TSDB_CLUSTER_ID_LEN, "%" PRId64, pBasicInfo->cluster_id); + + taos_gauge_t **metric = NULL; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->mnodes); ++i) { + + SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->mnodes, i); + + char mnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(mnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMnodeDesc->mnode_id); + + const char *sample_labels[] = {buf, mnode_id, pMnodeDesc->mnode_ep}; + + bool dnodeIsOnline = false; + for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) { + SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i); + + if(pDnodeDesc->dnode_id == pMnodeDesc->mnode_id){ + if(strcmp(pDnodeDesc->status, "ready") == 0){ + dnodeIsOnline = true; + } + } + } + + metric = taosHashGet(tsMonitor.metrics, MNODE_ROLE, strlen(MNODE_ROLE)); + + if(dnodeIsOnline){ + taos_gauge_set(*metric, pMnodeDesc->syncState, sample_labels); + } + else{ + taos_gauge_set(*metric, 0, sample_labels); + } + //metric = taosHashGet(tsMonitor.metrics, MNODE_ROLE, strlen(MNODE_ROLE)); + //taos_gauge_set(*metric, pMnodeDesc->syncState, sample_labels); + } +} + +void monGenVnodeRoleTable(SMonInfo *pMonitor){ + char *vnodes_role_gauges[] = {VNODE_ROLE}; + taos_gauge_t *gauge = NULL; + + for(int32_t i = 0; i < 1; i++){ + if(taos_collector_registry_deregister_metric(vnodes_role_gauges[i]) != 0){ + uError("failed to delete metric %s", vnodes_role_gauges[i]); + } + + taosHashRemove(tsMonitor.metrics, vnodes_role_gauges[i], strlen(vnodes_role_gauges[i])); + } + + SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + SMonBasicInfo *pBasicInfo = &pMonitor->dmInfo.basic; + if(pBasicInfo->cluster_id == 0) return; + + int32_t vnodes_role_label_count = 4; + const char *vnodes_role_sample_labels[] = {"cluster_id", "vgroup_id", "database_name", "dnode_id"}; + for(int32_t i = 0; i < 1; i++){ + gauge= taos_gauge_new(vnodes_role_gauges[i], "", vnodes_role_label_count, vnodes_role_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, vnodes_role_gauges[i], strlen(vnodes_role_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } + + char buf[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(buf, TSDB_CLUSTER_ID_LEN, "%" PRId64, pBasicInfo->cluster_id); + + taos_gauge_t **metric = NULL; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->vgroups); ++i) { + SMonVgroupDesc *pVgroupDesc = taosArrayGet(pInfo->vgroups, i); + + char vgroup_id[TSDB_VGROUP_ID_LEN] = {0}; + snprintf(vgroup_id, TSDB_VGROUP_ID_LEN, "%"PRId32, pVgroupDesc->vgroup_id); + + for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { + SMonVnodeDesc *pVnodeDesc = &pVgroupDesc->vnodes[j]; + if (pVnodeDesc->dnode_id <= 0) continue; + + char dnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pVnodeDesc->dnode_id); + + const char *sample_labels[] = {buf, vgroup_id, pVgroupDesc->database_name, dnode_id}; + + metric = taosHashGet(tsMonitor.metrics, VNODE_ROLE, strlen(VNODE_ROLE)); + taos_gauge_set(*metric, pVnodeDesc->syncState, sample_labels); + } + } +} + +void monSendPromReport() { + char ts[50] = {0}; + sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + + char* promStr = NULL; + char* pCont = (char *)taos_collector_registry_bridge_new(TAOS_COLLECTOR_REGISTRY_DEFAULT, ts, "%" PRId64, &promStr); + if(tsMonitorLogProtocol){ + if(pCont != NULL){ + uInfoL("report cont:\n%s", pCont); + uDebugL("report cont prom:\n%s", promStr); + } + else{ + uInfo("report cont is null"); + } + } + if (pCont != NULL) { + EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + uError("failed to send monitor msg"); + }else{ + taos_collector_registry_clear_batch(TAOS_COLLECTOR_REGISTRY_DEFAULT); + } + taosMemoryFreeClear(pCont); + } + if(promStr != NULL){ + taosMemoryFreeClear(promStr); + } +} \ No newline at end of file diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 56cf0a2b51..4ecc6f1261 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -18,9 +18,13 @@ #include "taoserror.h" #include "thttp.h" #include "ttime.h" +#include "taos_monitor.h" +#include "tglobal.h" -static SMonitor tsMonitor = {0}; -static char* tsMonUri = "/report"; +SMonitor tsMonitor = {0}; +char* tsMonUri = "/report"; +char* tsMonFwUri = "/general-metric"; +char* tsMonFwBasicUri = "/taosd-cluster-basic"; void monRecordLog(int64_t ts, ELogLevel level, const char *content) { taosThreadMutexLock(&tsMonitor.lock); @@ -108,6 +112,9 @@ int32_t monInit(const SMonCfg *pCfg) { tsLogFp = monRecordLog; tsMonitor.lastTime = taosGetTimestampMs(); taosThreadMutexInit(&tsMonitor.lock, NULL); + + monInitMonitorFW(); + return 0; } @@ -121,6 +128,8 @@ void monCleanup() { tFreeSMonQmInfo(&tsMonitor.qmInfo); tFreeSMonBmInfo(&tsMonitor.bmInfo); taosThreadMutexDestroy(&tsMonitor.lock); + + monCleanupMonitorFW(); } static void monCleanupMonitorInfo(SMonInfo *pMonitor) { @@ -186,6 +195,22 @@ static void monGenBasicJson(SMonInfo *pMonitor) { tjsonAddDoubleToObject(pJson, "protocol", pInfo->protocol); } +static void monGenBasicJsonBasic(SMonInfo *pMonitor) { + SMonBasicInfo *pInfo = &pMonitor->dmInfo.basic; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + SJson *pJson = pMonitor->pJson; + char buf[40] = {0}; + + sprintf(buf, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + tjsonAddStringToObject(pJson, "ts", buf); + tjsonAddDoubleToObject(pJson, "dnode_id", pInfo->dnode_id); + tjsonAddStringToObject(pJson, "dnode_ep", pInfo->dnode_ep); + snprintf(buf, sizeof(buf), "%" PRId64, pInfo->cluster_id); + tjsonAddStringToObject(pJson, "cluster_id", buf); + tjsonAddDoubleToObject(pJson, "protocol", pInfo->protocol); +} + static void monGenClusterJson(SMonInfo *pMonitor) { SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; @@ -244,6 +269,17 @@ static void monGenClusterJson(SMonInfo *pMonitor) { } } +static void monGenClusterJsonBasic(SMonInfo *pMonitor) { + SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + //tjsonAddStringToObject(pMonitor->pJson, "first_ep", pInfo->first_ep); + tjsonAddStringToObject(pMonitor->pJson, "first_ep", tsFirst); + tjsonAddDoubleToObject(pMonitor->pJson, "first_ep_dnode_id", pInfo->first_ep_dnode_id); + tjsonAddStringToObject(pMonitor->pJson, "cluster_version", pInfo->version); + //tjsonAddDoubleToObject(pMonitor->pJson, "monitor_interval", pInfo->monitor_interval); +} + static void monGenVgroupJson(SMonInfo *pMonitor) { SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; @@ -517,21 +553,11 @@ static void monGenLogJson(SMonInfo *pMonitor) { if (tjsonAddItemToArray(pSummaryJson, pLogTrace) != 0) tjsonDelete(pLogTrace); } -void monSendReport() { - SMonInfo *pMonitor = monCreateMonitorInfo(); - if (pMonitor == NULL) return; - - monGenBasicJson(pMonitor); - monGenClusterJson(pMonitor); - monGenVgroupJson(pMonitor); - monGenStbJson(pMonitor); - monGenGrantJson(pMonitor); - monGenDnodeJson(pMonitor); - monGenDiskJson(pMonitor); - monGenLogJson(pMonitor); - +void monSendReport(SMonInfo *pMonitor){ char *pCont = tjsonToString(pMonitor->pJson); - // uDebugL("report cont:%s\n", pCont); + if(tsMonitorLogProtocol){ + uInfoL("report cont:\n%s", pCont); + } if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; if (taosSendHttpReport(tsMonitor.cfg.server, tsMonUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { @@ -539,6 +565,83 @@ void monSendReport() { } taosMemoryFree(pCont); } +} + +void monGenAndSendReport() { + SMonInfo *pMonitor = monCreateMonitorInfo(); + if (pMonitor == NULL) return; + + if(!tsMonitorForceV2){ + monGenBasicJson(pMonitor); + monGenClusterJson(pMonitor); + monGenVgroupJson(pMonitor); + monGenStbJson(pMonitor); + monGenGrantJson(pMonitor); + monGenDnodeJson(pMonitor); + monGenDiskJson(pMonitor); + monGenLogJson(pMonitor); + + monSendReport(pMonitor); + } + else{ + monGenClusterInfoTable(pMonitor); + monGenVgroupInfoTable(pMonitor); + monGenDnodeInfoTable(pMonitor); + monGenDnodeStatusInfoTable(pMonitor); + monGenDataDiskTable(pMonitor); + monGenLogDiskTable(pMonitor); + monGenMnodeRoleTable(pMonitor); + monGenVnodeRoleTable(pMonitor); + + monSendPromReport(); + } monCleanupMonitorInfo(pMonitor); } + +void monSendReportBasic(SMonInfo *pMonitor){ + char *pCont = tjsonToString(pMonitor->pJson); + if(tsMonitorLogProtocol){ + if(pCont != NULL){ + uInfoL("report cont basic:\n%s", pCont); + } + else{ + uInfo("report cont basic is null"); + } + } + if (pCont != NULL) { + EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwBasicUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + uError("failed to send monitor msg"); + } + taosMemoryFree(pCont); + } +} + +void monGenAndSendReportBasic() { + SMonInfo *pMonitor = monCreateMonitorInfo(); + + monGenBasicJsonBasic(pMonitor); + monGenClusterJsonBasic(pMonitor); + + if (pMonitor->mmInfo.cluster.first_ep_dnode_id != 0) { + monSendReportBasic(pMonitor); + } + + monCleanupMonitorInfo(pMonitor); +} + +void monSendContent(char *pCont) { + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; + if(tsMonitorLogProtocol){ + if (pCont != NULL){ + uInfoL("report client cont:\n%s\n", pCont); + } + } + if (pCont != NULL) { + EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + uError("failed to send monitor msg"); + } + } +} \ No newline at end of file diff --git a/source/libs/monitor/test/monTest.cpp b/source/libs/monitor/test/monTest.cpp index 3f7b1b51da..2660cff216 100644 --- a/source/libs/monitor/test/monTest.cpp +++ b/source/libs/monitor/test/monTest.cpp @@ -283,10 +283,10 @@ TEST_F(MonitorTest, 01_Full) { tFreeSMonSmInfo(&smInfo); tFreeSMonQmInfo(&qmInfo); tFreeSMonBmInfo(&bmInfo); - monSendReport(); + monGenAndSendReport(); } TEST_F(MonitorTest, 02_Log) { AddLogInfo2(); - monSendReport(); + monGenAndSendReport(); } diff --git a/source/libs/monitorfw/CMakeLists.txt b/source/libs/monitorfw/CMakeLists.txt new file mode 100644 index 0000000000..610cd63985 --- /dev/null +++ b/source/libs/monitorfw/CMakeLists.txt @@ -0,0 +1,9 @@ +aux_source_directory(src MONITOR2_SRC) +add_library(monitorfw STATIC ${MONITOR2_SRC}) +target_include_directories( + monitorfw + PUBLIC "${TD_SOURCE_DIR}/include/libs/monitorfw" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries(monitorfw os util common transport) diff --git a/source/libs/monitorfw/inc/taos_assert.h b/source/libs/monitorfw/inc/taos_assert.h new file mode 100644 index 0000000000..d3226eed26 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_assert.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#ifndef TAOS_ASSERT_H +#define TAOS_ASSERT_H + +#ifdef TAOS_ASSERT_ENABLE +#define TAOS_ASSERT(i) assert(i); +#else +#define TAOS_ASSERT(i) +#endif // TAOS_TEST + +#endif // TAOS_ASSERT_H diff --git a/source/libs/monitorfw/inc/taos_collector_registry_i.h b/source/libs/monitorfw/inc/taos_collector_registry_i.h new file mode 100644 index 0000000000..ed5bb1fe19 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_collector_registry_i.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "taos_collector_registry_t.h" + +#ifndef TAOS_COLLECTOR_REGISTRY_I_INCLUDED +#define TAOS_COLLECTOR_REGISTRY_I_INCLUDED + +int taos_collector_registry_enable_custom_process_metrics(taos_collector_registry_t *self, + const char *process_limits_path, + const char *process_stats_path); + +#endif // TAOS_COLLECTOR_REGISTRY_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_collector_registry_t.h b/source/libs/monitorfw/inc/taos_collector_registry_t.h new file mode 100644 index 0000000000..8e8a881fca --- /dev/null +++ b/source/libs/monitorfw/inc/taos_collector_registry_t.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_REGISTRY_T_H +#define TAOS_REGISTRY_T_H + +#include +#include + +// Public +#include "taos_collector_registry.h" + +// Private +#include "taos_map_t.h" +#include "taos_metric_formatter_t.h" +#include "taos_string_builder_t.h" + +struct taos_collector_registry { + const char *name; + bool disable_process_metrics; /**< Disables the collection of process metrics */ + taos_map_t *collectors; /**< Map of collectors keyed by name */ + taos_string_builder_t *string_builder; /**< Enables string building */ + taos_metric_formatter_t *metric_formatter; /**< metric formatter for metric exposition on bridge call */ + pthread_rwlock_t *lock; /**< mutex for safety against concurrent registration */ + taos_string_builder_t *string_builder_batch; +}; + +#endif // TAOS_REGISTRY_T_H diff --git a/source/libs/monitorfw/inc/taos_collector_t.h b/source/libs/monitorfw/inc/taos_collector_t.h new file mode 100644 index 0000000000..264e8e9ad9 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_collector_t.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_COLLECTOR_T_H +#define TAOS_COLLECTOR_T_H + +#include "taos_collector.h" +#include "taos_map_t.h" +#include "taos_string_builder_t.h" + +struct taos_collector { + const char *name; + taos_map_t *metrics; + taos_collect_fn *collect_fn; + taos_string_builder_t *string_builder; + const char *proc_limits_file_path; + const char *proc_stat_file_path; +}; + +#endif // TAOS_COLLECTOR_T_H diff --git a/source/libs/monitorfw/inc/taos_errors.h b/source/libs/monitorfw/inc/taos_errors.h new file mode 100644 index 0000000000..ee2a894df3 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_errors.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define TAOS_STDIO_CLOSE_DIR_ERROR "failed to close dir" +#define TAOS_STDIO_OPEN_DIR_ERROR "failed to open dir" +#define TAOS_METRIC_INCORRECT_TYPE "incorrect metric type" +#define TAOS_METRIC_INVALID_LABEL_NAME "invalid label name" +#define TAOS_PTHREAD_RWLOCK_DESTROY_ERROR "failed to destroy the pthread_rwlock_t*" +#define TAOS_PTHREAD_RWLOCK_INIT_ERROR "failed to initialize the pthread_rwlock_t*" +#define TAOS_PTHREAD_RWLOCK_LOCK_ERROR "failed to lock the pthread_rwlock_t*" +#define TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR "failed to unlock the pthread_rwlock_t*" +#define TAOS_REGEX_REGCOMP_ERROR "failed to compile the regular expression" +#define TAOS_REGEX_REGEXEC_ERROR "failed to execute the regular expression" diff --git a/source/libs/monitorfw/inc/taos_linked_list_i.h b/source/libs/monitorfw/inc/taos_linked_list_i.h new file mode 100644 index 0000000000..ed11a76427 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_linked_list_i.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_LIST_I_INCLUDED +#define TAOS_LIST_I_INCLUDED + +// Private +#include "taos_linked_list_t.h" + +/** + * @brief API PRIVATE Returns a pointer to a taos_linked_list + */ +taos_linked_list_t *taos_linked_list_new(void); + +/** + * @brief API PRIVATE removes all nodes from the given taos_linked_list * + */ +int taos_linked_list_purge(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Destroys a taos_linked_list + */ +int taos_linked_list_destroy(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Append an item to the back of the list + */ +int taos_linked_list_append(taos_linked_list_t *self, void *item); + +/** + * @brief API PRIVATE Push an item onto the front of the list + */ +int taos_linked_list_push(taos_linked_list_t *self, void *item); + +/** + * @brief API PRIVATE Pop the first item off of the list + */ +void *taos_linked_list_pop(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Returns the item at the head of the list or NULL if not present + */ +void *taos_linked_list_first(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Returns the item at the tail of the list or NULL if not present + */ +void *taos_linked_list_last(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Removes an item from the linked list + */ +int taos_linked_list_remove(taos_linked_list_t *self, void *item); + +/** + * @brief API PRIVATE Compares two items within a linked list + */ +taos_linked_list_compare_t taos_linked_list_compare(taos_linked_list_t *self, void *item_a, void *node_b); + +/** + * @brief API PRIVATE Get the size + */ +size_t taos_linked_list_size(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Set the free_fn member on taos_linked_list + */ +int taos_linked_list_set_free_fn(taos_linked_list_t *self, taos_linked_list_free_item_fn free_fn); + +/** + * @brief API PRIVATE Set the compare_fn member on the taos_linked_list + */ +int taos_linked_list_set_compare_fn(taos_linked_list_t *self, taos_linked_list_compare_item_fn compare_fn); + +/** + * API PRIVATE + * @brief does nothing + */ +void taos_linked_list_no_op_free(void *item); + +#endif // TAOS_LIST_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_linked_list_t.h b/source/libs/monitorfw/inc/taos_linked_list_t.h new file mode 100644 index 0000000000..ccd82ffb61 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_linked_list_t.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_LIST_T_H +#define TAOS_LIST_T_H + +#include "taos_linked_list.h" + +typedef enum { TAOS_LESS = -1, TAOS_EQUAL = 0, TAOS_GREATER = 1 } taos_linked_list_compare_t; + +/** + * @brief API PRIVATE Frees an item in a taos_linked_list_node + */ +typedef void (*taos_linked_list_free_item_fn)(void *); + +/** + * @brief API PRIVATE Compares two items within a taos_linked_list + */ +typedef taos_linked_list_compare_t (*taos_linked_list_compare_item_fn)(void *item_a, void *item_b); + +/** + * @brief API PRIVATE A struct containing a generic item, represented as a void pointer, and next, a pointer to the + * next taos_linked_list_node* + */ +typedef struct taos_linked_list_node { + struct taos_linked_list_node *next; + void *item; +} taos_linked_list_node_t; + +/** + * @brief API PRIVATE A linked list comprised of taos_linked_list_node* instances + */ +struct taos_linked_list { + taos_linked_list_node_t *head; + taos_linked_list_node_t *tail; + size_t size; + taos_linked_list_free_item_fn free_fn; + taos_linked_list_compare_item_fn compare_fn; +}; + +#endif // TAOS_LIST_T_H diff --git a/source/libs/monitorfw/inc/taos_log.h b/source/libs/monitorfw/inc/taos_log.h new file mode 100644 index 0000000000..fecbda59e5 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_log.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#ifndef TAOS_LOG_H +#define TAOS_LOG_H + +//#define TAOS_LOG_ENABLE + +#ifdef TAOS_LOG_ENABLE +#define TAOS_LOG(msg) printf("monitor_log %s %s %s %s %d %s\n", __DATE__, __TIME__, __FILE__, __FUNCTION__, __LINE__, msg); +#else +#define TAOS_LOG(msg) +#endif // TAOS_LOG_ENABLE + +#endif // TAOS_LOG_H diff --git a/source/libs/monitorfw/inc/taos_map_i.h b/source/libs/monitorfw/inc/taos_map_i.h new file mode 100644 index 0000000000..55f248b96a --- /dev/null +++ b/source/libs/monitorfw/inc/taos_map_i.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_MAP_I_INCLUDED +#define TAOS_MAP_I_INCLUDED + +#include "taos_map_t.h" + +taos_map_t *taos_map_new(void); + +int taos_map_set_free_value_fn(taos_map_t *self, taos_map_node_free_value_fn free_value_fn); + +void *taos_map_get(taos_map_t *self, const char *key); + +void *taos_map_get_withoutlock(taos_map_t *self, const char *key); + +int taos_map_set(taos_map_t *self, const char *key, void *value); + +int taos_map_delete(taos_map_t *self, const char *key); + +int taos_map_destroy(taos_map_t *self); + +size_t taos_map_size(taos_map_t *self); + +taos_map_node_t *taos_map_node_new(const char *key, void *value, taos_map_node_free_value_fn free_value_fn); + +#endif // TAOS_MAP_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_map_t.h b/source/libs/monitorfw/inc/taos_map_t.h new file mode 100644 index 0000000000..6fcbda1366 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_map_t.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_MAP_T_H +#define TAOS_MAP_T_H + +#include + +// Public +#include "taos_map.h" + +// Private +#include "taos_linked_list_t.h" + +typedef void (*taos_map_node_free_value_fn)(void *); + +struct taos_map_node { + const char *key; + void *value; + taos_map_node_free_value_fn free_value_fn; +}; + +struct taos_map { + size_t size; /**< contains the size of the map */ + size_t max_size; /**< stores the current max_size */ + taos_linked_list_t *keys; /**< linked list containing containing all keys present */ + taos_linked_list_t **addrs; /**< Sequence of linked lists. Each list contains nodes with the same index */ + pthread_rwlock_t *rwlock; + taos_map_node_free_value_fn free_value_fn; +}; + +#endif // TAOS_MAP_T_H diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_custom_i.h b/source/libs/monitorfw/inc/taos_metric_formatter_custom_i.h new file mode 100644 index 0000000000..f3e4ebae75 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_formatter_custom_i.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_FORMATTER_CUSTOMV2_I_H +#define TAOS_METRIC_FORMATTER_CUSTOMV2_I_H + +#define ALLOW_FORBID_FUNC + +#include +#include "tjson.h" + +int taos_metric_formatter_load_sample_new(taos_metric_formatter_t *self, taos_metric_sample_t *sample, + char *ts, char *format, char *metricName, int32_t metric_type, + SJson *arrayMetricGroups); +int taos_metric_formatter_load_metric_new(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format, + SJson* tableArray); +int taos_metric_formatter_load_metrics_new(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts, + char *format, SJson* tableArray); +#endif // TAOS_METRIC_FORMATTER_CUSTOMV2_I_H \ No newline at end of file diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_i.h b/source/libs/monitorfw/inc/taos_metric_formatter_i.h new file mode 100644 index 0000000000..fba30d9eeb --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_formatter_i.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_FORMATTER_I_H +#define TAOS_METRIC_FORMATTER_I_H + +// Private +#include "taos_metric_formatter_t.h" +#include "taos_metric_t.h" + +/** + * @brief API PRIVATE taos_metric_formatter constructor + */ +taos_metric_formatter_t *taos_metric_formatter_new(); + +/** + * @brief API PRIVATE taos_metric_formatter destructor + */ +int taos_metric_formatter_destroy(taos_metric_formatter_t *self); + +/** + * @brief API PRIVATE Loads the help text + */ +int taos_metric_formatter_load_help(taos_metric_formatter_t *self, const char *name, const char *help); + +/** + * @brief API PRIVATE Loads the type text + */ +int taos_metric_formatter_load_type(taos_metric_formatter_t *self, const char *name, taos_metric_type_t metric_type); + +/** + * @brief API PRIVATE Loads the formatter with a metric sample L-value + * @param name The metric name + * @param suffix The metric suffix. This is applicable to Summary and Histogram metric types. + * @param label_count The number of labels for the given metric. + * @param label_keys An array of constant strings. + * @param label_values An array of constant strings. + * + * The number of const char **and taos_label_value must be the same. + */ +int taos_metric_formatter_load_l_value(taos_metric_formatter_t *metric_formatter, const char *name, const char *suffix, + size_t label_count, const char **label_keys, const char **label_values); + +/** + * @brief API PRIVATE Loads the formatter with a metric sample + */ +int taos_metric_formatter_load_sample(taos_metric_formatter_t *metric_formatter, taos_metric_sample_t *sample, + char *ts, char *format); + +/** + * @brief API PRIVATE Loads a metric in the string exposition format + */ +int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format); + +/** + * @brief API PRIVATE Loads the given metrics + */ +int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts, char *format); + +/** + * @brief API PRIVATE Clear the underlying string_builder + */ +int taos_metric_formatter_clear(taos_metric_formatter_t *self); + +/** + * @brief API PRIVATE Returns the string built by taos_metric_formatter + */ +char *taos_metric_formatter_dump(taos_metric_formatter_t *metric_formatter); + +#endif // TAOS_METRIC_FORMATTER_I_H diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_t.h b/source/libs/monitorfw/inc/taos_metric_formatter_t.h new file mode 100644 index 0000000000..0d7425aa59 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_formatter_t.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_FORMATTER_T_H +#define TAOS_METRIC_FORMATTER_T_H + +#include "taos_string_builder_t.h" + +typedef struct taos_metric_formatter { + taos_string_builder_t *string_builder; + taos_string_builder_t *err_builder; +} taos_metric_formatter_t; + +#endif // TAOS_METRIC_FORMATTER_T_H diff --git a/source/libs/monitorfw/inc/taos_metric_i.h b/source/libs/monitorfw/inc/taos_metric_i.h new file mode 100644 index 0000000000..e8ae799547 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_i.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Private +#include "taos_metric_t.h" + +#ifndef TAOS_METRIC_I_INCLUDED +#define TAOS_METRIC_I_INCLUDED + +/** + * @brief API PRIVATE Returns a *taos_metric + */ +taos_metric_t *taos_metric_new(taos_metric_type_t type, const char *name, const char *help, size_t label_key_count, + const char **label_keys); + +/** + * @brief API PRIVATE Destroys a *taos_metric + */ +int taos_metric_destroy(taos_metric_t *self); + +/** + * @brief API PRIVATE takes a generic item, casts to a *taos_metric_t and destroys it + */ +int taos_metric_destroy_generic(void *item); + +/** + * @brief API Private takes a generic item, casts to a *taos_metric_t and destroys it. Discards any errors. + */ +void taos_metric_free_generic(void *item); + +#endif // TAOS_METRIC_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_metric_sample_i.h b/source/libs/monitorfw/inc/taos_metric_sample_i.h new file mode 100644 index 0000000000..b5e90f1933 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_sample_i.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "taos_metric_sample_t.h" +#include "taos_metric_t.h" + +#ifndef TAOS_METRIC_SAMPLE_I_H +#define TAOS_METRIC_SAMPLE_I_H + +/** + * @brief API PRIVATE Return a taos_metric_sample_t* + * + * @param type The type of metric sample + * @param l_value The entire left value of the metric e.g metric_name{foo="bar"} + * @param r_value A double representing the value of the sample + */ +taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value); + +/** + * @brief API PRIVATE Destroy the taos_metric_sample** + */ +int taos_metric_sample_destroy(taos_metric_sample_t *self); + +/** + * @brief API PRIVATE A taos_linked_list_free_item_fn to enable item destruction within a linked list's destructor + */ +int taos_metric_sample_destroy_generic(void *); + +/** + * @brief API PRIVATE A taos_linked_list_free_item_fn to enable item destruction within a linked list's destructor. + * + * This function ignores any errors. + */ +void taos_metric_sample_free_generic(void *gen); + +#endif // TAOS_METRIC_SAMPLE_I_H diff --git a/source/libs/monitorfw/inc/taos_metric_sample_t.h b/source/libs/monitorfw/inc/taos_metric_sample_t.h new file mode 100644 index 0000000000..387512af9d --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_sample_t.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_SAMPLE_T_H +#define TAOS_METRIC_SAMPLE_T_H + +#include "taos_metric_sample.h" +#include "taos_metric_t.h" + +#if !defined(WINDOWS) +#define C11_ATOMIC +#else +#define DOUBLE_ATOMIC +#endif + +#ifdef C11_ATOMIC +#include +#endif + +struct taos_metric_sample { + taos_metric_type_t type; /**< type is the metric type for the sample */ + char *l_value; /**< l_value is the full metric name and label set represeted as a string */ +#ifdef C11_ATOMIC + _Atomic double r_value; /**< r_value is the value of the metric sample */ +#else +#ifdef DOUBLE_ATOMIC + double r_value; /**< r_value is the value of the metric sample */ +#else + int64_t r_value; +#endif +#endif +}; + +#endif // TAOS_METRIC_SAMPLE_T_H diff --git a/source/libs/monitorfw/inc/taos_metric_t.h b/source/libs/monitorfw/inc/taos_metric_t.h new file mode 100644 index 0000000000..da237aa814 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_t.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_T_H +#define TAOS_METRIC_T_H + +#include + +// Public +#include "taos_metric.h" + +// Private +#include "taos_map_i.h" +#include "taos_map_t.h" +#include "taos_metric_formatter_t.h" + +/** + * @brief API PRIVATE Contains metric type constants + */ +typedef enum taos_metric_type { TAOS_COUNTER, TAOS_GAUGE, TAOS_HISTOGRAM, TAOS_SUMMARY } taos_metric_type_t; + +/** + * @brief API PRIVATE Maps metric type constants to human readable string values + */ +extern char *taos_metric_type_map[4]; + +/** + * @brief API PRIVATE An opaque struct to users containing metric metadata; one or more metric samples; and a metric + * formatter for locating metric samples and exporting metric data + */ +struct taos_metric { + taos_metric_type_t type; /**< metric_type The type of metric */ + char *name; /**< name The name of the metric */ + const char *help; /**< help The help output for the metric */ + taos_map_t *samples; /**< samples Map comprised of samples for the given metric */ + size_t label_key_count; /**< label_keys_count The count of labe_keys*/ + taos_metric_formatter_t *formatter; /**< formatter The metric formatter */ + pthread_rwlock_t *rwlock; /**< rwlock Required for locking on certain non-atomic operations */ + const char **label_keys; /**< labels Array comprised of const char **/ +}; + +#endif // TAOS_METRIC_T_H diff --git a/source/libs/monitorfw/inc/taos_monitor_util_i.h b/source/libs/monitorfw/inc/taos_monitor_util_i.h new file mode 100644 index 0000000000..fe072204a3 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_monitor_util_i.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_MONITOR_UTIL_I_H +#define TAOS_MONITOR_UTIL_I_H + +#include + +void taos_monitor_split_str(char** arr, char* str, const char* del); +int taos_monitor_count_occurrences(char *str, char *toSearch); +void taos_monitor_strip(char *s); +bool taos_monitor_is_match(const SJson* tags, char** pairs, int32_t count); + +#endif // TAOS_MONITOR_UTIL_I_H \ No newline at end of file diff --git a/source/libs/monitorfw/inc/taos_string_builder_i.h b/source/libs/monitorfw/inc/taos_string_builder_i.h new file mode 100644 index 0000000000..142ca020ba --- /dev/null +++ b/source/libs/monitorfw/inc/taos_string_builder_i.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_STRING_BUILDER_I_H +#define TAOS_STRING_BUILDER_I_H + +#include + +#include "taos_string_builder_t.h" + +/** + * API PRIVATE + * @brief Constructor for taos_string_builder + */ +taos_string_builder_t *taos_string_builder_new(void); + +/** + * API PRIVATE + * @brief Destroys a taos_string_builder* + */ +int taos_string_builder_destroy(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Adds a string + */ +int taos_string_builder_add_str(taos_string_builder_t *self, const char *str); + +/** + * API PRIVATE + * @brief Adds a char + */ +int taos_string_builder_add_char(taos_string_builder_t *self, char c); + +/** + * API PRIVATE + * @brief Clear the string + */ +int taos_string_builder_clear(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Remove data from the end + */ +int taos_string_buillder_truncate(taos_string_builder_t *self, size_t len); + +/** + * API PRIVATE + * @brief Returns the length of the string + */ +size_t taos_string_builder_len(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Returns a copy of the string. The returned string must be deallocated when no longer needed. + */ +char *taos_string_builder_dump(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Getter for str member + */ +char *taos_string_builder_str(taos_string_builder_t *self); + +#endif // TAOS_STRING_BUILDER_I_H diff --git a/source/libs/monitorfw/inc/taos_string_builder_t.h b/source/libs/monitorfw/inc/taos_string_builder_t.h new file mode 100644 index 0000000000..edd3d574fa --- /dev/null +++ b/source/libs/monitorfw/inc/taos_string_builder_t.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_STRING_BUILDER_T_H +#define TAOS_STRING_BUILDER_T_H + +struct taos_string_builder; +/** + * @brief API PRIVATE A structure with functions responsible for building a string + */ +typedef struct taos_string_builder taos_string_builder_t; + +#endif // TAOS_STRING_BUILDER_T_H diff --git a/source/libs/monitorfw/src/taos_collector.c b/source/libs/monitorfw/src/taos_collector.c new file mode 100644 index 0000000000..17d324462c --- /dev/null +++ b/source/libs/monitorfw/src/taos_collector.c @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" +#include "taos_collector.h" +#include "taos_collector_registry.h" + +// Private +#include "taos_assert.h" +#include "taos_collector_t.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_metric_i.h" +#include "taos_string_builder_i.h" + +taos_map_t *taos_collector_default_collect(taos_collector_t *self) { return self->metrics; } + +taos_collector_t *taos_collector_new(const char *name) { + int r = 0; + taos_collector_t *self = (taos_collector_t *)taos_malloc(sizeof(taos_collector_t)); + self->name = taos_strdup(name); + self->metrics = taos_map_new(); + if (self->metrics == NULL) { + taos_collector_destroy(self); + return NULL; + } + r = taos_map_set_free_value_fn(self->metrics, &taos_metric_free_generic); + if (r) { + taos_collector_destroy(self); + return NULL; + } + self->collect_fn = &taos_collector_default_collect; + self->string_builder = taos_string_builder_new(); + if (self->string_builder == NULL) { + taos_collector_destroy(self); + return NULL; + } + self->proc_limits_file_path = NULL; + self->proc_stat_file_path = NULL; + return self; +} + +int taos_collector_destroy(taos_collector_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_map_destroy(self->metrics); + if (r) ret = r; + self->metrics = NULL; + + r = taos_string_builder_destroy(self->string_builder); + if (r) ret = r; + self->string_builder = NULL; + + taos_free((char *)self->name); + self->name = NULL; + taos_free(self); + self = NULL; + + return ret; +} + +int taos_collector_destroy_generic(void *gen) { + int r = 0; + taos_collector_t *self = (taos_collector_t *)gen; + r = taos_collector_destroy(self); + self = NULL; + return r; +} + +void taos_collector_free_generic(void *gen) { + taos_collector_t *self = (taos_collector_t *)gen; + taos_collector_destroy(self); +} + +int taos_collector_set_collect_fn(taos_collector_t *self, taos_collect_fn *fn) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->collect_fn = fn; + return 0; +} + +int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (taos_map_get(self->metrics, metric->name) != NULL) { + TAOS_LOG("metric already found in collector"); + return 1; + } + return taos_map_set(self->metrics, metric->name, metric); +} + +int taos_collector_remove_metric(taos_collector_t *self, const char* key){ + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + return taos_map_delete(self->metrics, key); +} + +taos_metric_t* taos_collector_get_metric(taos_collector_t *self, char *metric_name){ + TAOS_ASSERT(self != NULL); + if (self == NULL) return NULL; + return taos_map_get(self->metrics, metric_name); +} \ No newline at end of file diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c new file mode 100644 index 0000000000..c3ed0112c5 --- /dev/null +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +// Public +#include "taos_alloc.h" +#include "taos_collector.h" +#include "taos_collector_registry.h" + +// Private +#include "taos_assert.h" +#include "taos_collector_registry_t.h" +#include "taos_collector_t.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_metric_formatter_i.h" +#include "taos_metric_i.h" +#include "taos_metric_t.h" +#include "taos_string_builder_i.h" +#include "taos_metric_formatter_custom_i.h" + +#define ALLOW_FORBID_FUNC +#include "tjson.h" + +taos_collector_registry_t *TAOS_COLLECTOR_REGISTRY_DEFAULT; + +taos_collector_registry_t *taos_collector_registry_new(const char *name) { + int r = 0; + + taos_collector_registry_t *self = (taos_collector_registry_t *)taos_malloc(sizeof(taos_collector_registry_t)); + + self->disable_process_metrics = false; + + self->name = taos_strdup(name); + self->collectors = taos_map_new(); + taos_map_set_free_value_fn(self->collectors, &taos_collector_free_generic); + taos_map_set(self->collectors, "default", taos_collector_new("default")); + + self->metric_formatter = taos_metric_formatter_new(); + self->string_builder = taos_string_builder_new(); + self->string_builder_batch = taos_string_builder_new(); + self->lock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t)); + r = pthread_rwlock_init(self->lock, NULL); + if (r) { + TAOS_LOG("failed to initialize rwlock"); + return NULL; + } + return self; +} + +int taos_collector_registry_default_init(void) { + if (TAOS_COLLECTOR_REGISTRY_DEFAULT != NULL) return 0; + + TAOS_COLLECTOR_REGISTRY_DEFAULT = taos_collector_registry_new("default"); + //if (TAOS_COLLECTOR_REGISTRY_DEFAULT) { + // return taos_collector_registry_enable_process_metrics(TAOS_COLLECTOR_REGISTRY_DEFAULT); + //} + return 1; +} + +int taos_collector_registry_destroy(taos_collector_registry_t *self) { + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_map_destroy(self->collectors); + self->collectors = NULL; + if (r) ret = r; + + r = taos_metric_formatter_destroy(self->metric_formatter); + self->metric_formatter = NULL; + if (r) ret = r; + + r = taos_string_builder_destroy(self->string_builder); + self->string_builder = NULL; + if (r) ret = r; + + r = taos_string_builder_destroy(self->string_builder_batch); + self->string_builder_batch = NULL; + if (r) ret = r; + + r = pthread_rwlock_destroy(self->lock); + taos_free(self->lock); + self->lock = NULL; + if (r) ret = r; + + taos_free((char *)self->name); + self->name = NULL; + + taos_free(self); + self = NULL; + + return ret; +} + +int taos_collector_registry_register_metric(taos_metric_t *metric) { + TAOS_ASSERT(metric != NULL); + + taos_collector_t *default_collector = + (taos_collector_t *)taos_map_get(TAOS_COLLECTOR_REGISTRY_DEFAULT->collectors, "default"); + + if (default_collector == NULL) { + return 1; + } + + return taos_collector_add_metric(default_collector, metric); +} + +int taos_collector_registry_deregister_metric(const char *key) { + TAOS_ASSERT(metric != NULL); + + taos_collector_t *default_collector = + (taos_collector_t *)taos_map_get(TAOS_COLLECTOR_REGISTRY_DEFAULT->collectors, "default"); + + if (default_collector == NULL) { + return 1; + } + + return taos_collector_remove_metric(default_collector, key); +} + +taos_metric_t *taos_collector_registry_get_metric(char* metric_name){ + TAOS_ASSERT(metric != NULL); + + taos_collector_t *default_collector = + (taos_collector_t *)taos_map_get(TAOS_COLLECTOR_REGISTRY_DEFAULT->collectors, "default"); + + if (default_collector == NULL) { + return NULL; + } + + return taos_collector_get_metric(default_collector, metric_name); +} + +taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metric) { + int err = taos_collector_registry_register_metric(metric); + if (err != 0) { + //exit(err); + return NULL; + } + return metric; +} + +int taos_collector_registry_register_collector(taos_collector_registry_t *self, taos_collector_t *collector) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = pthread_rwlock_wrlock(self->lock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + return 1; + } + if (taos_map_get(self->collectors, collector->name) != NULL) { + TAOS_LOG("the given taos_collector_t* is already registered"); + int rr = pthread_rwlock_unlock(self->lock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return 1; + } + } + r = taos_map_set(self->collectors, collector->name, collector); + if (r) { + int rr = pthread_rwlock_unlock(self->lock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return r; + } + } + r = pthread_rwlock_unlock(self->lock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return 1; + } + return 0; +} + +int taos_collector_registry_validate_metric_name(taos_collector_registry_t *self, const char *metric_name) { + regex_t r; + int ret = 0; + ret = regcomp(&r, "^[a-zA-Z_:][a-zA-Z0-9_:]*$", REG_EXTENDED); + if (ret) { + TAOS_LOG(TAOS_REGEX_REGCOMP_ERROR); + regfree(&r); + return ret; + } + + ret = regexec(&r, metric_name, 0, NULL, 0); + if (ret) { + TAOS_LOG(TAOS_REGEX_REGEXEC_ERROR); + regfree(&r); + return ret; + } + regfree(&r); + return 0; +} +/* +const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts, char *format) { + taos_metric_formatter_clear(self->metric_formatter); + taos_metric_formatter_load_metrics(self->metric_formatter, self->collectors, ts, format); + char *out = taos_metric_formatter_dump(self->metric_formatter); + + int r = 0; + r = taos_string_builder_add_str(self->string_builder_batch, out); + if (r) return NULL; + taos_free(out); + + return taos_string_builder_str(self->string_builder_batch); +} +*/ +int taos_collector_registry_clear_batch(taos_collector_registry_t *self){ + return taos_string_builder_clear(self->string_builder_batch); +} + +const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self, char *ts, char *format, char** prom_str) { + taos_metric_formatter_clear(self->metric_formatter); + + SJson* pJson = tjsonCreateArray(); + SJson* item = tjsonCreateObject(); + tjsonAddItemToArray(pJson, item); + tjsonAddStringToObject(item, "ts", ts); + tjsonAddDoubleToObject(item, "protocol", 2); + SJson* array = tjsonCreateArray(); + tjsonAddItemToObject(item, "tables", array); + + if(taos_metric_formatter_load_metrics_new(self->metric_formatter, self->collectors, ts, format, array) != 0){ + TAOS_LOG("failed to load metrics"); + tjsonDelete(pJson); + return NULL; + } + + if(tjsonGetArraySize(array) == 0){ + tjsonDelete(pJson); + return NULL; + } + + //caller free this + //generate prom protocol for debug + if(prom_str != NULL){ + *prom_str = taos_metric_formatter_dump(self->metric_formatter); + } + + //add this result to batch cache, format in batch cache is {},{} + int r = 0; + char* old_str = taos_string_builder_str(self->string_builder_batch); + if(old_str[0] != '\0'){ + r = taos_string_builder_add_str(self->string_builder_batch, ","); + if (r) goto _OVER; + } + char * item_str = tjsonToString(item); + r = taos_string_builder_add_str(self->string_builder_batch, item_str); + taos_free(item_str); + if (r) goto _OVER;; + + //generate final array format result, ie, add [] to str in batch cache + taos_string_builder_t* tmp_builder = taos_string_builder_new(); + + r = taos_string_builder_add_str(tmp_builder, "["); + if (r) goto _OVER;; + + r = taos_string_builder_add_str(tmp_builder, taos_string_builder_str(self->string_builder_batch)); + if (r) goto _OVER;; + + r = taos_string_builder_add_str(tmp_builder, "]"); + if (r) goto _OVER;; + + //caller free this + char *data = taos_string_builder_dump(tmp_builder); + if (data == NULL) goto _OVER;; + r = taos_string_builder_clear(tmp_builder); + if (r) goto _OVER;; + + r = taos_string_builder_destroy(tmp_builder); + tmp_builder = NULL; + if (r) goto _OVER;; + + tjsonDelete(pJson); + return data; + +_OVER: + tjsonDelete(pJson); + + return NULL; +} diff --git a/source/libs/monitorfw/src/taos_counter.c b/source/libs/monitorfw/src/taos_counter.c new file mode 100644 index 0000000000..d522411b2b --- /dev/null +++ b/source/libs/monitorfw/src/taos_counter.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Public +#include "taos_counter.h" + +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_metric_i.h" +#include "taos_metric_sample_i.h" +#include "taos_metric_sample_t.h" +#include "taos_metric_t.h" + +taos_counter_t *taos_counter_new(const char *name, const char *help, size_t label_key_count, const char **label_keys) { + return (taos_counter_t *)taos_metric_new(TAOS_COUNTER, name, help, label_key_count, label_keys); +} + +int taos_counter_destroy(taos_counter_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + int r = 0; + r = taos_metric_destroy(self); + self = NULL; + return r; +} + +int taos_counter_inc(taos_counter_t *self, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_COUNTER) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_add(sample, 1.0); +} + +int taos_counter_add(taos_counter_t *self, double r_value, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_COUNTER) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_add(sample, r_value); +} diff --git a/source/libs/monitorfw/src/taos_gauge.c b/source/libs/monitorfw/src/taos_gauge.c new file mode 100644 index 0000000000..7793f4c464 --- /dev/null +++ b/source/libs/monitorfw/src/taos_gauge.c @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Public +#include "taos_gauge.h" + +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_metric_i.h" +#include "taos_metric_sample_i.h" +#include "taos_metric_sample_t.h" +#include "taos_metric_t.h" + +taos_gauge_t *taos_gauge_new(const char *name, const char *help, size_t label_key_count, const char **label_keys) { + return (taos_gauge_t *)taos_metric_new(TAOS_GAUGE, name, help, label_key_count, label_keys); +} + +int taos_gauge_destroy(taos_gauge_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + r = taos_metric_destroy(self); + self = NULL; + return r; +} +/* +int taos_gauge_inc(taos_gauge_t *self, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_add(sample, 1.0); +} + +int taos_gauge_dec(taos_gauge_t *self, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_sub(sample, 1.0); +} + +int taos_gauge_add(taos_gauge_t *self, double r_value, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_add(sample, r_value); +} + +int taos_gauge_sub(taos_gauge_t *self, double r_value, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_sub(sample, r_value); +} +*/ +int taos_gauge_set(taos_gauge_t *self, double r_value, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_set(sample, r_value); +} diff --git a/source/libs/monitorfw/src/taos_linked_list.c b/source/libs/monitorfw/src/taos_linked_list.c new file mode 100644 index 0000000000..675400a6fa --- /dev/null +++ b/source/libs/monitorfw/src/taos_linked_list.c @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_linked_list_i.h" +#include "taos_linked_list_t.h" +#include "taos_log.h" + +taos_linked_list_t *taos_linked_list_new(void) { + taos_linked_list_t *self = (taos_linked_list_t *)taos_malloc(sizeof(taos_linked_list_t)); + self->head = NULL; + self->tail = NULL; + self->free_fn = NULL; + self->compare_fn = NULL; + self->size = 0; + return self; +} + +int taos_linked_list_purge(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node = self->head; + while (node != NULL) { + taos_linked_list_node_t *next = node->next; + if (node->item != NULL) { + if (self->free_fn) { + (*self->free_fn)(node->item); + } else { + taos_free(node->item); + } + } + taos_free(node); + node = NULL; + node = next; + } + self->head = NULL; + self->tail = NULL; + self->size = 0; + return 0; +} + +int taos_linked_list_destroy(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + int ret = 0; + + r = taos_linked_list_purge(self); + if (r) ret = r; + taos_free(self); + self = NULL; + return ret; +} + +void *taos_linked_list_first(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self->head) { + return self->head->item; + } else { + return NULL; + } +} + +void *taos_linked_list_last(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self->tail) { + return self->tail->item; + } else { + return NULL; + } +} + +int taos_linked_list_append(taos_linked_list_t *self, void *item) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node = (taos_linked_list_node_t *)taos_malloc(sizeof(taos_linked_list_node_t)); + + node->item = item; + if (self->tail) { + self->tail->next = node; + } else { + self->head = node; + } + self->tail = node; + node->next = NULL; + self->size++; + return 0; +} + +int taos_linked_list_push(taos_linked_list_t *self, void *item) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node = (taos_linked_list_node_t *)taos_malloc(sizeof(taos_linked_list_node_t)); + + node->item = item; + node->next = self->head; + self->head = node; + if (self->tail == NULL) { + self->tail = node; + } + self->size++; + return 0; +} + +void *taos_linked_list_pop(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return NULL; + taos_linked_list_node_t *node = self->head; + void *item = NULL; + if (node != NULL) { + item = node->item; + self->head = node->next; + if (self->tail == node) { + self->tail = NULL; + } + if (node->item != NULL) { + if (self->free_fn) { + (*self->free_fn)(node->item); + } else { + taos_free(node->item); + } + } + node->item = NULL; + node = NULL; + self->size--; + } + return item; +} + +int taos_linked_list_remove(taos_linked_list_t *self, void *item) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node; + taos_linked_list_node_t *prev_node = NULL; +#ifdef TAOS_LOG_ENABLE + int32_t count = 0; + char tmp[200] = {0}; + + count = 0; + for (node = self->head; node != NULL; node = node->next) { + count++; + } + sprintf(tmp, "list count:%d", count); + TAOS_LOG(tmp); +#endif + + // Locate the node +#ifdef TAOS_LOG_ENABLE + count = 0; +#endif + for (node = self->head; node != NULL; node = node->next) { +#ifdef TAOS_LOG_ENABLE + count++; +#endif + if (self->compare_fn) { + if ((*self->compare_fn)(node->item, item) == TAOS_EQUAL) { + break; + } + } else { + if (node->item == item) { + break; + } + } + prev_node = node; + } + +#ifdef TAOS_LOG_ENABLE + sprintf(tmp, "remove item:%d", count); + TAOS_LOG(tmp); +#endif + + if (node == NULL) return 0; + + if (prev_node) { + prev_node->next = node->next; + } else { + self->head = node->next; + } + if (node->next == NULL) { + self->tail = prev_node; + } + + if (node->item != NULL) { + if (self->free_fn) { + (*self->free_fn)(node->item); + } else { + taos_free(node->item); + } + } + + node->item = NULL; + taos_free(node); + node = NULL; + self->size--; + +#ifdef TAOS_LOG_ENABLE + count = 0; + for (node = self->head; node != NULL; node = node->next) { + count++; + } + + sprintf(tmp, "list count:%d", count); + TAOS_LOG(tmp); +#endif + + return 0; +} + +taos_linked_list_compare_t taos_linked_list_compare(taos_linked_list_t *self, void *item_a, void *item_b) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->compare_fn) { + return (*self->compare_fn)(item_a, item_b); + } else { + return strcmp(item_a, item_b); + } +} + +size_t taos_linked_list_size(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + return self->size; +} + +int taos_linked_list_set_free_fn(taos_linked_list_t *self, taos_linked_list_free_item_fn free_fn) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->free_fn = free_fn; + return 0; +} + +int taos_linked_list_set_compare_fn(taos_linked_list_t *self, taos_linked_list_compare_item_fn compare_fn) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->compare_fn = compare_fn; + return 0; +} + +void taos_linked_list_no_op_free(void *item) {} diff --git a/source/libs/monitorfw/src/taos_map.c b/source/libs/monitorfw/src/taos_map.c new file mode 100644 index 0000000000..8f0b83884e --- /dev/null +++ b/source/libs/monitorfw/src/taos_map.c @@ -0,0 +1,422 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_linked_list_i.h" +#include "taos_linked_list_t.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_map_t.h" + +#define TAOS_MAP_INITIAL_SIZE 32 + +static void destroy_map_node_value_no_op(void *value) {} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// taos_map_node +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +taos_map_node_t *taos_map_node_new(const char *key, void *value, taos_map_node_free_value_fn free_value_fn) { + taos_map_node_t *self = taos_malloc(sizeof(taos_map_node_t)); + self->key = taos_strdup(key); + self->value = value; + self->free_value_fn = free_value_fn; + return self; +} + +int taos_map_node_destroy(taos_map_node_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + taos_free((void *)self->key); + self->key = NULL; + if (self->value != NULL) (*self->free_value_fn)(self->value); + self->value = NULL; + taos_free(self); + self = NULL; + return 0; +} + +void taos_map_node_free(void *item) { + taos_map_node_t *map_node = (taos_map_node_t *)item; + taos_map_node_destroy(map_node); +} + +taos_linked_list_compare_t taos_map_node_compare(void *item_a, void *item_b) { + taos_map_node_t *map_node_a = (taos_map_node_t *)item_a; + taos_map_node_t *map_node_b = (taos_map_node_t *)item_b; + + return strcmp(map_node_a->key, map_node_b->key); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// taos_map +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +taos_map_t *taos_map_new() { + int r = 0; + + taos_map_t *self = (taos_map_t *)taos_malloc(sizeof(taos_map_t)); + self->size = 0; + self->max_size = TAOS_MAP_INITIAL_SIZE; + + self->keys = taos_linked_list_new(); + if (self->keys == NULL) return NULL; + + // These each key will be allocated once by taos_map_node_new and used here as well to save memory. With that said + // we will only have to deallocate each key once. That will happen on taos_map_node_destroy. + r = taos_linked_list_set_free_fn(self->keys, taos_linked_list_no_op_free); + if (r) { + taos_map_destroy(self); + return NULL; + } + + self->addrs = taos_malloc(sizeof(taos_linked_list_t) * self->max_size); + self->free_value_fn = destroy_map_node_value_no_op; + + for (int i = 0; i < self->max_size; i++) { + self->addrs[i] = taos_linked_list_new(); + r = taos_linked_list_set_free_fn(self->addrs[i], taos_map_node_free); + if (r) { + taos_map_destroy(self); + return NULL; + } + r = taos_linked_list_set_compare_fn(self->addrs[i], taos_map_node_compare); + if (r) { + taos_map_destroy(self); + return NULL; + } + } + + self->rwlock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t)); + r = pthread_rwlock_init(self->rwlock, NULL); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_INIT_ERROR); + taos_map_destroy(self); + return NULL; + } + + return self; +} + +int taos_map_destroy(taos_map_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + int ret = 0; + + r = taos_linked_list_destroy(self->keys); + if (r) ret = r; + self->keys = NULL; + + for (size_t i = 0; i < self->max_size; i++) { + r = taos_linked_list_destroy(self->addrs[i]); + if (r) ret = r; + self->addrs[i] = NULL; + } + taos_free(self->addrs); + self->addrs = NULL; + + r = pthread_rwlock_destroy(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_DESTROY_ERROR) + ret = r; + } + + taos_free(self->rwlock); + self->rwlock = NULL; + taos_free(self); + self = NULL; + + return ret; +} + +static size_t taos_map_get_index_internal(const char *key, size_t *size, size_t *max_size) { + size_t index; + size_t a = 31415, b = 27183; + for (index = 0; *key != '\0'; key++, a = a * b % (*max_size - 1)) { + index = (a * index + *key) % *max_size; + } + return index; +} + +/** + * @brief API PRIVATE hash function that returns an array index from the given key and taos_map. + * + * The algorithm is based off of Horner's method. In a simpler version, you set the return value to 0. Next, for each + * character in the string, you add the integer value of the current character to the product of the prime number and + * the current return value, set the result to the return value, then finally return the return value. + * + * In this version of the algorithm, we attempt to achieve a probabily of key to index conversion collisions to + * 1/M (with M being the max_size of the map). This optimizes dispersion and consequently, evens out the performance + * for gets and sets for each item. Instead of using a fixed prime number, we generate a coefficient for each iteration + * through the loop. + * + * Reference: + * * Algorithms in C: Third Edition by Robert Sedgewick, p579 + */ +size_t taos_map_get_index(taos_map_t *self, const char *key) { + return taos_map_get_index_internal(key, &self->size, &self->max_size); +} + +static void *taos_map_get_internal(const char *key, size_t *size, size_t *max_size, taos_linked_list_t *keys, + taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn) { + size_t index = taos_map_get_index_internal(key, size, max_size); + taos_linked_list_t *list = addrs[index]; + taos_map_node_t *temp_map_node = taos_map_node_new(key, NULL, free_value_fn); + + for (taos_linked_list_node_t *current_node = list->head; current_node != NULL; current_node = current_node->next) { + taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item; + taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, temp_map_node); + if (result == TAOS_EQUAL) { + taos_map_node_destroy(temp_map_node); + temp_map_node = NULL; + return current_map_node->value; + } + } + taos_map_node_destroy(temp_map_node); + temp_map_node = NULL; + return NULL; +} + +void *taos_map_get(taos_map_t *self, const char *key) { + TAOS_ASSERT(self != NULL); + int r = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + NULL; + } + void *payload = + taos_map_get_internal(key, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn); + r = pthread_rwlock_unlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return NULL; + } + return payload; +} + +void *taos_map_get_withoutlock(taos_map_t *self, const char *key) { + TAOS_ASSERT(self != NULL); + int r = 0; + void *payload = + taos_map_get_internal(key, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn); + return payload; +} + +static int taos_map_set_internal(const char *key, void *value, size_t *size, size_t *max_size, taos_linked_list_t *keys, + taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn, + bool destroy_current_value) { + taos_map_node_t *map_node = taos_map_node_new(key, value, free_value_fn); + if (map_node == NULL) return 1; + + size_t index = taos_map_get_index_internal(key, size, max_size); + taos_linked_list_t *list = addrs[index]; + for (taos_linked_list_node_t *current_node = list->head; current_node != NULL; current_node = current_node->next) { + taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item; + taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, map_node); + if (result == TAOS_EQUAL) { + if (destroy_current_value) { + free_value_fn(current_map_node->value); + current_map_node->value = NULL; + } + taos_free((char *)current_map_node->key); + current_map_node->key = NULL; + taos_free(current_map_node); + current_map_node = NULL; + current_node->item = map_node; + return 0; + } + } + taos_linked_list_append(list, map_node); + taos_linked_list_append(keys, (char *)map_node->key); + (*size)++; + return 0; +} + +int taos_map_ensure_space(taos_map_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + + if (self->size <= self->max_size / 2) { + return 0; + } + + // Increase the max size + size_t new_max = self->max_size * 2; + size_t new_size = 0; + + // Create a new list of keys + taos_linked_list_t *new_keys = taos_linked_list_new(); + if (new_keys == NULL) return 1; + + r = taos_linked_list_set_free_fn(new_keys, taos_linked_list_no_op_free); + if (r) return r; + + // Create a new array of addrs + taos_linked_list_t **new_addrs = taos_malloc(sizeof(taos_linked_list_t) * new_max); + + // Initialize the new array + for (int i = 0; i < new_max; i++) { + new_addrs[i] = taos_linked_list_new(); + r = taos_linked_list_set_free_fn(new_addrs[i], taos_map_node_free); + if (r) return r; + r = taos_linked_list_set_compare_fn(new_addrs[i], taos_map_node_compare); + if (r) return r; + } + + // Iterate through each linked-list at each memory region in the map's backbone + for (int i = 0; i < self->max_size; i++) { + // Create a new map node for each node in the linked list and insert it into the new map. Afterwards, deallocate + // the old map node + taos_linked_list_t *list = self->addrs[i]; + taos_linked_list_node_t *current_node = list->head; + while (current_node != NULL) { + taos_map_node_t *map_node = (taos_map_node_t *)current_node->item; + r = taos_map_set_internal(map_node->key, map_node->value, &new_size, &new_max, new_keys, new_addrs, + self->free_value_fn, false); + if (r) return r; + + taos_linked_list_node_t *next = current_node->next; + taos_free(current_node); + current_node = NULL; + taos_free((void *)map_node->key); + map_node->key = NULL; + taos_free(map_node); + map_node = NULL; + current_node = next; + } + // We're done deallocating each map node in the linked list, so deallocate the linked-list object + taos_free(self->addrs[i]); + self->addrs[i] = NULL; + } + // Destroy the collection of keys in the map + taos_linked_list_destroy(self->keys); + self->keys = NULL; + + // Deallocate the backbone of the map + taos_free(self->addrs); + self->addrs = NULL; + + // Update the members of the current map + self->size = new_size; + self->max_size = new_max; + self->keys = new_keys; + self->addrs = new_addrs; + + return 0; +} + +int taos_map_set(taos_map_t *self, const char *key, void *value) { + TAOS_ASSERT(self != NULL); + int r = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + return r; + } + + r = taos_map_ensure_space(self); + if (r) { + int rr = 0; + rr = pthread_rwlock_unlock(self->rwlock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return r; + } + } + r = taos_map_set_internal(key, value, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn, + true); + if (r) { + int rr = 0; + rr = pthread_rwlock_unlock(self->rwlock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return r; + } + } + r = pthread_rwlock_unlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + } + return r; +} + +static int taos_map_delete_internal(const char *key, size_t *size, size_t *max_size, taos_linked_list_t *keys, + taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn) { + int r = 0; + size_t index = taos_map_get_index_internal(key, size, max_size); + taos_linked_list_t *list = addrs[index]; + taos_map_node_t *temp_map_node = taos_map_node_new(key, NULL, free_value_fn); + + for (taos_linked_list_node_t *current_node = list->head; current_node != NULL; current_node = current_node->next) { + taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item; + taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, temp_map_node); + if (result == TAOS_EQUAL) { + r = taos_linked_list_remove(keys, (char*)current_map_node->key); + if (r) return r; + + r = taos_linked_list_remove(list, current_node->item); + if (r) return r; + + (*size)--; + break; + } + } + r = taos_map_node_destroy(temp_map_node); + temp_map_node = NULL; + return r; +} + +int taos_map_delete(taos_map_t *self, const char *key) { + TAOS_ASSERT(self != NULL); + int r = 0; + int ret = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + ret = r; + } + r = taos_map_delete_internal(key, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn); + if (r) ret = r; + r = pthread_rwlock_unlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + ret = r; + } + return ret; +} + +int taos_map_set_free_value_fn(taos_map_t *self, taos_map_node_free_value_fn free_value_fn) { + TAOS_ASSERT(self != NULL); + self->free_value_fn = free_value_fn; + return 0; +} + +size_t taos_map_size(taos_map_t *self) { + TAOS_ASSERT(self != NULL); + return self->size; +} diff --git a/source/libs/monitorfw/src/taos_metric.c b/source/libs/monitorfw/src/taos_metric.c new file mode 100644 index 0000000000..4e9af35f34 --- /dev/null +++ b/source/libs/monitorfw/src/taos_metric.c @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_metric_formatter_i.h" +#include "taos_metric_i.h" +#include "taos_metric_sample_i.h" + +char *taos_metric_type_map[4] = {"counter", "gauge", "histogram", "summary"}; + +taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name, const char *help, + size_t label_key_count, const char **label_keys) { + int r = 0; + taos_metric_t *self = (taos_metric_t *)taos_malloc(sizeof(taos_metric_t)); + self->type = metric_type; + int len = strlen(name) + 1; + self->name = taos_malloc(len); + memset(self->name, 0, len); + strcpy(self->name, name); + //self->name = name; + self->help = help; + + const char **k = (const char **)taos_malloc(sizeof(const char *) * label_key_count); + + for (int i = 0; i < label_key_count; i++) { + if (strcmp(label_keys[i], "le") == 0) { + TAOS_LOG(TAOS_METRIC_INVALID_LABEL_NAME); + taos_metric_destroy(self); + return NULL; + } + if (strcmp(label_keys[i], "quantile") == 0) { + TAOS_LOG(TAOS_METRIC_INVALID_LABEL_NAME); + taos_metric_destroy(self); + return NULL; + } + k[i] = taos_strdup(label_keys[i]); + } + self->label_keys = k; + self->label_key_count = label_key_count; + self->samples = taos_map_new(); + + if (metric_type == TAOS_HISTOGRAM) { + + } else { + r = taos_map_set_free_value_fn(self->samples, &taos_metric_sample_free_generic); + if (r) { + taos_metric_destroy(self); + return NULL; + } + } + + self->formatter = taos_metric_formatter_new(); + if (self->formatter == NULL) { + taos_metric_destroy(self); + return NULL; + } + self->rwlock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t)); + r = pthread_rwlock_init(self->rwlock, NULL); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_INIT_ERROR); + return NULL; + } + return self; +} + +int taos_metric_destroy(taos_metric_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_map_destroy(self->samples); + self->samples = NULL; + if (r) ret = r; + + r = taos_metric_formatter_destroy(self->formatter); + self->formatter = NULL; + if (r) ret = r; + + r = pthread_rwlock_destroy(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_DESTROY_ERROR); + ret = r; + } + + taos_free(self->rwlock); + self->rwlock = NULL; + + for (int i = 0; i < self->label_key_count; i++) { + taos_free((void *)self->label_keys[i]); + self->label_keys[i] = NULL; + } + taos_free(self->label_keys); + self->label_keys = NULL; + + taos_free(self->name); + self->name = NULL; + + taos_free(self); + self = NULL; + + return ret; +} + +int taos_metric_destroy_generic(void *item) { + int r = 0; + taos_metric_t *self = (taos_metric_t *)item; + r = taos_metric_destroy(self); + self = NULL; + return r; +} + +void taos_metric_free_generic(void *item) { + taos_metric_t *self = (taos_metric_t *)item; + taos_metric_destroy(self); +} + +taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const char **label_values) { + TAOS_ASSERT(self != NULL); + int r = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + return NULL; + } + +#define TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK() \ + r = pthread_rwlock_unlock(self->rwlock); \ + if (r) TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); \ + return NULL; + + // Get l_value + r = taos_metric_formatter_load_l_value(self->formatter, self->name, NULL, self->label_key_count, self->label_keys, + label_values); + if (r) { + TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); + } + + // This must be freed before returning + const char *l_value = taos_metric_formatter_dump(self->formatter); + if (l_value == NULL) { + TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); + } + + // Get sample + taos_metric_sample_t *sample = (taos_metric_sample_t *)taos_map_get(self->samples, l_value); + if (sample == NULL) { + sample = taos_metric_sample_new(self->type, l_value, 0.0); + r = taos_map_set(self->samples, l_value, sample); + if (r) { + TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); + } + } + pthread_rwlock_unlock(self->rwlock); + taos_free((void *)l_value); + return sample; +} + diff --git a/source/libs/monitorfw/src/taos_metric_formatter.c b/source/libs/monitorfw/src/taos_metric_formatter.c new file mode 100644 index 0000000000..53012935ba --- /dev/null +++ b/source/libs/monitorfw/src/taos_metric_formatter.c @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_collector_t.h" +#include "taos_linked_list_t.h" +#include "taos_map_i.h" +#include "taos_metric_formatter_i.h" +#include "taos_metric_sample_t.h" +#include "taos_metric_t.h" +#include "taos_string_builder_i.h" + + +taos_metric_formatter_t *taos_metric_formatter_new() { + taos_metric_formatter_t *self = (taos_metric_formatter_t *)taos_malloc(sizeof(taos_metric_formatter_t)); + self->string_builder = taos_string_builder_new(); + if (self->string_builder == NULL) { + taos_metric_formatter_destroy(self); + return NULL; + } + self->err_builder = taos_string_builder_new(); + if (self->err_builder == NULL) { + taos_metric_formatter_destroy(self); + return NULL; + } + return self; +} + +int taos_metric_formatter_destroy(taos_metric_formatter_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_string_builder_destroy(self->string_builder); + self->string_builder = NULL; + if (r) ret = r; + + r = taos_string_builder_destroy(self->err_builder); + self->err_builder = NULL; + if (r) ret = r; + + taos_free(self); + self = NULL; + return ret; +} +/* +int taos_metric_formatter_load_help(taos_metric_formatter_t *self, const char *name, const char *help) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, "# HELP "); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, name); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, help); + if (r) return r; + + return taos_string_builder_add_char(self->string_builder, '\n'); +} + +int taos_metric_formatter_load_type(taos_metric_formatter_t *self, const char *name, taos_metric_type_t metric_type) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, "# TYPE "); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, name); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, taos_metric_type_map[metric_type]); + if (r) return r; + + return taos_string_builder_add_char(self->string_builder, '\n'); +} +*/ +int taos_metric_formatter_load_l_value(taos_metric_formatter_t *self, const char *name, const char *suffix, + size_t label_count, const char **label_keys, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, name); + if (r) return r; + + if (suffix != NULL) { + r = taos_string_builder_add_char(self->string_builder, '_'); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, suffix); + if (r) return r; + } + + if (label_count == 0) return 0; + + for (int i = 0; i < label_count; i++) { + if (i == 0) { + r = taos_string_builder_add_char(self->string_builder, '{'); + if (r) return r; + } + r = taos_string_builder_add_str(self->string_builder, (const char *)label_keys[i]); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, '='); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, '"'); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, (const char *)label_values[i]); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, '"'); + if (r) return r; + + if (i == label_count - 1) { + r = taos_string_builder_add_char(self->string_builder, '}'); + if (r) return r; + } else { + r = taos_string_builder_add_char(self->string_builder, ','); + if (r) return r; + } + } + return 0; +} +/* +int taos_metric_formatter_load_sample(taos_metric_formatter_t *self, taos_metric_sample_t *sample, + char *ts, char *format) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, sample->l_value); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + char buffer[50]; + sprintf(buffer, format, sample->r_value); + r = taos_string_builder_add_str(self->string_builder, buffer); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, ts); + if (r) return r; + + //taos_metric_sample_set(sample, 0); + + return taos_string_builder_add_char(self->string_builder, '\n'); +} +*/ +int taos_metric_formatter_clear(taos_metric_formatter_t *self) { + TAOS_ASSERT(self != NULL); + return taos_string_builder_clear(self->string_builder); +} + +char *taos_metric_formatter_dump(taos_metric_formatter_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + if (self == NULL) return NULL; + char *data = taos_string_builder_dump(self->string_builder); + if (data == NULL) return NULL; + r = taos_string_builder_clear(self->string_builder); + if (r) { + taos_free(data); + return NULL; + } + return data; +} +/* +int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_metric_formatter_load_help(self, metric->name, metric->help); + if (r) return r; + + r = taos_metric_formatter_load_type(self, metric->name, metric->type); + if (r) return r; + + for (taos_linked_list_node_t *current_node = metric->samples->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *key = (const char *)current_node->item; + if (metric->type == TAOS_HISTOGRAM) { + + } else { + taos_metric_sample_t *sample = (taos_metric_sample_t *)taos_map_get(metric->samples, key); + if (sample == NULL) return 1; + r = taos_metric_formatter_load_sample(self, sample, ts, format); + if (r) return r; + } + } + return taos_string_builder_add_char(self->string_builder, '\n'); +} + +int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts, char *format) { + TAOS_ASSERT(self != NULL); + int r = 0; + for (taos_linked_list_node_t *current_node = collectors->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *collector_name = (const char *)current_node->item; + taos_collector_t *collector = (taos_collector_t *)taos_map_get(collectors, collector_name); + if (collector == NULL) return 1; + + taos_map_t *metrics = collector->collect_fn(collector); + if (metrics == NULL) return 1; + + for (taos_linked_list_node_t *current_node = metrics->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *metric_name = (const char *)current_node->item; + taos_metric_t *metric = (taos_metric_t *)taos_map_get(metrics, metric_name); + if (metric == NULL) return 1; + r = taos_metric_formatter_load_metric(self, metric, ts, format); + if (r) return r; + } + } + return r; +} +*/ \ No newline at end of file diff --git a/source/libs/monitorfw/src/taos_metric_formatter_custom.c b/source/libs/monitorfw/src/taos_metric_formatter_custom.c new file mode 100644 index 0000000000..3b1318dfc0 --- /dev/null +++ b/source/libs/monitorfw/src/taos_metric_formatter_custom.c @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC + +#include +#include "taos_metric_formatter_i.h" +#include "taos_metric_sample_t.h" +#include "tjson.h" +#include "taos_monitor_util_i.h" +#include "taos_assert.h" +#include "tdef.h" +#include "taos_collector_t.h" +#include "taos_log.h" + +int taos_metric_formatter_load_sample_new(taos_metric_formatter_t *self, taos_metric_sample_t *sample, + char *ts, char *format, char *metricName, int32_t metric_type, + SJson *arrayMetricGroups) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + char* start = strstr(sample->l_value, "{"); + char* end = strstr(sample->l_value, "}"); + + int32_t len = end -start; + + char* keyvalues = taosMemoryMalloc(len); + memset(keyvalues, 0, len); + memcpy(keyvalues, start + 1, len - 1); + + int32_t count = taos_monitor_count_occurrences(keyvalues, ","); + + char** keyvalue = taosMemoryMalloc(sizeof(char*) * (count + 1)); + memset(keyvalue, 0, sizeof(char*) * (count + 1)); + taos_monitor_split_str(keyvalue, keyvalues, ","); + + char** arr = taosMemoryMalloc(sizeof(char*) * (count + 1) * 2); + memset(arr, 0, sizeof(char*) * (count + 1) * 2); + + bool isfound = true; + for(int32_t i = 0; i < count + 1; i++){ + char* str = *(keyvalue + i); + + char** pair = arr + i * 2; + taos_monitor_split_str(pair, str, "="); + + taos_monitor_strip(pair[1]); + } + + int32_t table_size = tjsonGetArraySize(arrayMetricGroups); + + SJson* item = NULL; + for(int32_t i = 0; i < table_size; i++){ + SJson *cur = tjsonGetArrayItem(arrayMetricGroups, i); + + SJson* tag = tjsonGetObjectItem(cur, "tags"); + + if(taos_monitor_is_match(tag, arr, count + 1)) { + item = cur; + break; + } + } + + SJson* metrics = NULL; + if(item == NULL) { + item = tjsonCreateObject(); + + SJson* arrayTag = tjsonCreateArray(); + for(int32_t i = 0; i < count + 1; i++){ + char** pair = arr + i * 2; + + char* key = *pair; + char* value = *(pair + 1); + + SJson* tag = tjsonCreateObject(); + tjsonAddStringToObject(tag, "name", key); + tjsonAddStringToObject(tag, "value", value); + + tjsonAddItemToArray(arrayTag, tag); + } + tjsonAddItemToObject(item, "tags", arrayTag); + + metrics = tjsonCreateArray(); + tjsonAddItemToObject(item, "metrics", metrics); + + tjsonAddItemToArray(arrayMetricGroups, item); + } + else{ + metrics = tjsonGetObjectItem(item, "metrics"); + } + + taosMemoryFreeClear(arr); + taosMemoryFreeClear(keyvalue); + taosMemoryFreeClear(keyvalues); + + SJson* metric = tjsonCreateObject(); + tjsonAddStringToObject(metric, "name", metricName); + + double old_value = 0; +#define USE_EXCHANGE +#ifdef USE_EXCHANGE + taos_metric_sample_exchange(sample, 0, &old_value); +#else + old_value = sample->r_value; + taos_metric_sample_set(sample, 0); +#endif + + tjsonAddDoubleToObject(metric, "value", old_value); + tjsonAddDoubleToObject(metric, "type", metric_type); + tjsonAddItemToArray(metrics, metric); + + return 0; +} + +int taos_metric_formatter_load_metric_new(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format, + SJson* tableArray) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + int32_t size = strlen(metric->name); + char* name = taosMemoryMalloc(size + 1); + memset(name, 0, size + 1); + memcpy(name, metric->name, size); + char* arr[2] = {0}; //arr[0] is table name, arr[1] is metric name + taos_monitor_split_str((char**)&arr, name, ":"); + + bool isFound = false; + SJson* table = NULL; + SJson* arrayMetricGroups = NULL; + + int32_t table_count = tjsonGetArraySize(tableArray); + for(int32_t i = 0; i < table_count; i++){ + SJson* table = tjsonGetArrayItem(tableArray, i); + + char tableName[MONITOR_TABLENAME_LEN] = {0}; + tjsonGetStringValue(table, "name", tableName); + if(strcmp(tableName, arr[0]) == 0){ + isFound = true; + arrayMetricGroups = tjsonGetObjectItem(table, "metric_groups"); + break; + } + } + + if(!isFound){ + table = tjsonCreateObject(); + + tjsonAddStringToObject(table, "name", arr[0]); + + arrayMetricGroups = tjsonCreateArray(); + tjsonAddItemToObject(table, "metric_groups", arrayMetricGroups); + } + + int32_t sample_count = 0; + for (taos_linked_list_node_t *current_node = metric->samples->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *key = (const char *)current_node->item; + if (metric->type == TAOS_HISTOGRAM) { + + } else { + taos_metric_sample_t *sample = (taos_metric_sample_t *)taos_map_get(metric->samples, key); + if (sample == NULL) return 1; + r = taos_metric_formatter_load_sample_new(self, sample, ts, format, arr[1], metric->type, arrayMetricGroups); + if (r) return r; + } + sample_count++; + } + + if(!isFound && sample_count > 0){ + tjsonAddItemToArray(tableArray, table); + } + else{ + if(table != NULL) tjsonDelete(table); + } + + taosMemoryFreeClear(name); + return r; +} + +int taos_metric_formatter_load_metrics_new(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts, + char *format, SJson* tableArray) { + TAOS_ASSERT(self != NULL); + int r = 0; + + for (taos_linked_list_node_t *current_node = collectors->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *collector_name = (const char *)current_node->item; + taos_collector_t *collector = (taos_collector_t *)taos_map_get(collectors, collector_name); + if (collector == NULL) return 1; + + taos_map_t *metrics = collector->collect_fn(collector); + if (metrics == NULL) return 1; + + //if(strcmp(collector->name, "custom") != 0 ){ + + r = pthread_rwlock_wrlock(metrics->rwlock); + if (r) { + TAOS_LOG("failed to lock"); + return r; + } + +#ifdef TAOS_LOG_ENABLE + int32_t count = 0; +#endif + for (taos_linked_list_node_t *current_node = metrics->keys->head; current_node != NULL; + current_node = current_node->next) { +#ifdef TAOS_LOG_ENABLE + count++; +#endif + const char *metric_name = (const char *)current_node->item; + taos_metric_t *metric = (taos_metric_t *)taos_map_get_withoutlock(metrics, metric_name); + if (metric == NULL) { +#ifdef TAOS_LOG_ENABLE + char tmp[200] = {0}; + sprintf(tmp, "fail to get metric(%d):%s", count, metric_name); + TAOS_LOG(tmp); +#endif + continue;; + } + r = taos_metric_formatter_load_metric_new(self, metric, ts, format, tableArray); + if (r) { + TAOS_LOG("failed to load metric"); + continue; + } + } + +#ifdef TAOS_LOG_ENABLE + char tmp[20] = {0}; + sprintf(tmp, "list count:%d", count); + TAOS_LOG(tmp); +#endif + r = pthread_rwlock_unlock(metrics->rwlock); + if (r) { + TAOS_LOG("failed to unlock"); + return r; + } + //} + //else{ + /* + for (taos_linked_list_node_t *current_node = metrics->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *metric_name = (const char *)current_node->item; + taos_metric_t *metric = (taos_metric_t *)taos_map_get(metrics, metric_name); + if (metric == NULL) return 1; + r = taos_metric_formatter_load_metric(self, metric, ts, format); + if (r) return r; + } + */ + //} + } + return r; +} \ No newline at end of file diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c new file mode 100644 index 0000000000..c6d817b513 --- /dev/null +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_metric_sample_i.h" +#include "taos_metric_sample_t.h" + +#ifdef C11_ATOMIC +#include +#else +#define ALLOW_FORBID_FUNC +#include "tdef.h" +#include "osAtomic.h" +#endif + +taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value) { + taos_metric_sample_t *self = (taos_metric_sample_t *)taos_malloc(sizeof(taos_metric_sample_t)); + self->type = type; + self->l_value = taos_strdup(l_value); + self->r_value = 0; + return self; +} + +int taos_metric_sample_destroy(taos_metric_sample_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + taos_free((void *)self->l_value); + self->l_value = NULL; + taos_free((void *)self); + self = NULL; + return 0; +} + +int taos_metric_sample_destroy_generic(void *gen) { + int r = 0; + + taos_metric_sample_t *self = (taos_metric_sample_t *)gen; + r = taos_metric_sample_destroy(self); + self = NULL; + return r; +} + +void taos_metric_sample_free_generic(void *gen) { + taos_metric_sample_t *self = (taos_metric_sample_t *)gen; + taos_metric_sample_destroy(self); +} + +int taos_metric_sample_add(taos_metric_sample_t *self, double r_value) { + TAOS_ASSERT(self != NULL); + if (r_value < 0) { + return 1; + } + +#ifdef C11_ATOMIC + /*_Atomic*/ double old = atomic_load(&self->r_value); + + for (;;) { + _Atomic double new = ATOMIC_VAR_INIT(old + r_value); + if (atomic_compare_exchange_weak(&self->r_value, &old, new)) { + return 0; + } + } +#else +#ifdef DOUBLE_ATOMIC + atomic_fetch_add_double(&self->r_value, r_value); +#else + atomic_fetch_add_64(&self->r_value, r_value); +#endif +#endif + + return 0; +} + +/* +int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { + TAOS_ASSERT(self != NULL); + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + +#ifdef C11_ATOMIC + ///_Atomic/ + double old = atomic_load(&self->r_value); + for (;;) { + _Atomic double new = ATOMIC_VAR_INIT(old - r_value); + if (atomic_compare_exchange_weak(&self->r_value, &old, new)) { + return 0; + } + } +#else +#ifdef DOUBLE_ATOMIC + atomic_fetch_sub_double(&self->r_value, r_value); +#else + atomic_fetch_sub_64(&self->r_value, r_value); +#endif +#endif + + return 0; +} +*/ + +int taos_metric_sample_set(taos_metric_sample_t *self, double r_value) { + if (self->type != TAOS_GAUGE && self->type != TAOS_COUNTER) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + +#ifdef C11_ATOMIC + atomic_store(&self->r_value, r_value); +#else +#ifdef DOUBLE_ATOMIC + atomic_store_double(&self->r_value, r_value); +#else + atomic_store_64(&self->r_value, r_value); +#endif +#endif + + return 0; +} + +int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, double* old_value) { + if (self->type != TAOS_GAUGE && self->type != TAOS_COUNTER) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + +#ifdef C11_ATOMIC + _Atomic double new = ATOMIC_VAR_INIT(r_value); + for (;;) { + /*_Atomic*/ double old = atomic_load(&self->r_value); + *old_value = old; + if (atomic_compare_exchange_weak(&self->r_value, &old, new)) { + return 0; + } + } +#else +#ifdef DOUBLE_ATOMIC + *old_value = atomic_exchange_double(&self->r_value, r_value); +#else + *old_value = atomic_exchange_64(&self->r_value, r_value); +#endif +#endif + + return 0; +} \ No newline at end of file diff --git a/source/libs/monitorfw/src/taos_monitor_util.c b/source/libs/monitorfw/src/taos_monitor_util.c new file mode 100644 index 0000000000..182402b3ff --- /dev/null +++ b/source/libs/monitorfw/src/taos_monitor_util.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +#define ALLOW_FORBID_FUNC +#include "tjson.h" +#include +#include +#include +#include "osMemory.h" +#include "tdef.h" + +#include "taos_metric_t.h" + +void taos_monitor_split_str(char** arr, char* str, const char* del) { + char *lasts; + char* s = strsep(&str, del); + while (s != NULL) { + *arr++ = s; + s = strsep(&str, del); + } +} + +void taos_monitor_split_str_metric(char** arr, taos_metric_t* metric, const char* del, char** buf) { + int32_t size = strlen(metric->name); + char* name = taosMemoryMalloc(size + 1); + memset(name, 0, size + 1); + memcpy(name, metric->name, size); + + char* s = strtok(name, del); + while (s != NULL) { + *arr++ = s; + s = strtok(NULL, del); + } + + *buf = name; +} + +const char* taos_monitor_get_metric_name(taos_metric_t* metric){ + return metric->name; +} + +int taos_monitor_count_occurrences(char *str, char *toSearch) { + int count = 0; + char *ptr = str; + while ((ptr = strstr(ptr, toSearch)) != NULL) { + count++; + ptr++; + } + return count; +} + +void taos_monitor_strip(char *s) +{ + size_t i; + size_t len = strlen(s); + size_t offset = 0; + for(i = 0; i < len; ++i){ + char c = s[i]; + if(c=='\"') ++offset; + else s[i-offset] = c; + } + s[len-offset] = '\0'; +} + +bool taos_monitor_is_match(const SJson* tags, char** pairs, int32_t count) { + int32_t size = tjsonGetArraySize(tags); + if(size != count) return false; + + for(int32_t i = 0; i < size; i++){ + SJson* item = tjsonGetArrayItem(tags, i); + + char item_name[MONITOR_TAG_NAME_LEN] = {0}; + tjsonGetStringValue(item, "name", item_name); + + char item_value[MONITOR_TAG_VALUE_LEN] = {0}; + tjsonGetStringValue(item, "value", item_value); + + bool isfound = false; + for(int32_t j = 0; j < count; j++){ + + char** pair = pairs + j * 2; + + char* key = *pair; + char* value = *(pair + 1); + + + if(strcmp(value, item_value) == 0 && strcmp(key, item_name) == 0){ + isfound = true; + break; + } + } + + if(!isfound) return false; + } + + return true; +} diff --git a/source/libs/monitorfw/src/taos_string_builder.c b/source/libs/monitorfw/src/taos_string_builder.c new file mode 100644 index 0000000000..0f3940cdab --- /dev/null +++ b/source/libs/monitorfw/src/taos_string_builder.c @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_string_builder_i.h" +#include "taos_string_builder_t.h" + +// The initial size of a string created via taos_string_builder +#define TAOS_STRING_BUILDER_INIT_SIZE 32 + +// taos_string_builder_init prototype declaration +int taos_string_builder_init(taos_string_builder_t *self); + +struct taos_string_builder { + char *str; /**< the target string */ + size_t allocated; /**< the size allocated to the string in bytes */ + size_t len; /**< the length of str */ + size_t init_size; /**< the initialize size of space to allocate */ +}; + +taos_string_builder_t *taos_string_builder_new(void) { + int r = 0; + + taos_string_builder_t *self = (taos_string_builder_t *)taos_malloc(sizeof(taos_string_builder_t)); + self->init_size = TAOS_STRING_BUILDER_INIT_SIZE; + r = taos_string_builder_init(self); + if (r) { + taos_string_builder_destroy(self); + return NULL; + } + + return self; +} + +int taos_string_builder_init(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->str = (char *)taos_malloc(self->init_size); + *self->str = '\0'; + self->allocated = self->init_size; + self->len = 0; + return 0; +} + +int taos_string_builder_destroy(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + taos_free(self->str); + self->str = NULL; + taos_free(self); + self = NULL; + return 0; +} + +/** + * @brief API PRIVATE Grows the size of the string given the value we want to add + * + * The method continuously shifts left until the new size is large enough to accommodate add_len. This private method + * is called in methods that need to add one or more characters to the underlying string. + */ +static int taos_string_builder_ensure_space(taos_string_builder_t *self, size_t add_len) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (add_len == 0 || self->allocated >= self->len + add_len + 1) return 0; + while (self->allocated < self->len + add_len + 1) self->allocated <<= 1; + self->str = (char *)taos_realloc(self->str, self->allocated); + return 0; +} + +int taos_string_builder_add_str(taos_string_builder_t *self, const char *str) { + TAOS_ASSERT(self != NULL); + int r = 0; + + if (self == NULL) return 1; + if (str == NULL || *str == '\0') return 0; + + size_t len = strlen(str); + r = taos_string_builder_ensure_space(self, len); + if (r) return r; + + memcpy(self->str + self->len, str, len); + self->len += len; + self->str[self->len] = '\0'; + return 0; +} + +int taos_string_builder_add_char(taos_string_builder_t *self, char c) { + TAOS_ASSERT(self != NULL); + int r = 0; + + if (self == NULL) return 1; + r = taos_string_builder_ensure_space(self, 1); + if (r) return r; + + self->str[self->len] = c; + self->len++; + self->str[self->len] = '\0'; + return 0; +} + +int taos_string_builder_truncate(taos_string_builder_t *self, size_t len) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (len >= self->len) return 0; + + self->len = len; + self->str[self->len] = '\0'; + return 0; +} + +int taos_string_builder_clear(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + taos_free(self->str); + self->str = NULL; + return taos_string_builder_init(self); +} + +size_t taos_string_builder_len(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + return self->len; +} + +char *taos_string_builder_dump(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + // +1 to accommodate \0 + char *out = (char *)taos_malloc((self->len + 1) * sizeof(char)); + memcpy(out, self->str, self->len + 1); + return out; +} + +char *taos_string_builder_str(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + return self->str; +} diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index a9907295b9..689886c366 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -698,7 +698,7 @@ static const char* jkScanLogicPlanTagCond = "TagCond"; static const char* jkScanLogicPlanGroupTags = "GroupTags"; static const char* jkScanLogicPlanOnlyMetaCtbIdx = "OnlyMetaCtbIdx"; static const char* jkScanLogicPlanFilesetDelimited = "FilesetDelimited"; -static const char* jkScanLogicPlanparaTablesSort = "paraTablesSort"; +static const char* jkScanLogicPlanParaTablesSort = "ParaTablesSort"; static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; @@ -747,7 +747,7 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddBoolToObject(pJson, jkScanLogicPlanFilesetDelimited, pNode->filesetDelimited); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddBoolToObject(pJson, jkScanLogicPlanFilesetDelimited, pNode->paraTablesSort); + code = tjsonAddBoolToObject(pJson, jkScanLogicPlanParaTablesSort, pNode->paraTablesSort); } return code; } @@ -800,7 +800,7 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { code = tjsonGetBoolValue(pJson, jkScanLogicPlanFilesetDelimited, &pNode->filesetDelimited); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkScanLogicPlanFilesetDelimited, &pNode->paraTablesSort); + code = tjsonGetBoolValue(pJson, jkScanLogicPlanParaTablesSort, &pNode->paraTablesSort); } return code; } @@ -1895,7 +1895,7 @@ static const char* jkTableScanPhysiPlanAssignBlockUid = "AssignBlockUid"; static const char* jkTableScanPhysiPlanIgnoreUpdate = "IgnoreUpdate"; static const char* jkTableScanPhysiPlanFilesetDelimited = "FilesetDelimited"; static const char* jkTableScanPhysiPlanNeedCountEmptyTable = "NeedCountEmptyTable"; -static const char* jkTableScanPhysiPlanparaTablesSort = "paraTablesSort"; +static const char* jkTableScanPhysiPlanParaTablesSort = "ParaTablesSort"; static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; @@ -1971,7 +1971,7 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanNeedCountEmptyTable, pNode->needCountEmptyTable); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanparaTablesSort, pNode->paraTablesSort); + code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanParaTablesSort, pNode->paraTablesSort); } return code; } @@ -2050,7 +2050,7 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanNeedCountEmptyTable, &pNode->needCountEmptyTable); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanparaTablesSort, &pNode->paraTablesSort); + code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanParaTablesSort, &pNode->paraTablesSort); } return code; } diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index c2cd4786db..4f523bf4f2 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -347,8 +347,11 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: + case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: case QUERY_NODE_SHOW_VNODES_STMT: case QUERY_NODE_SHOW_SCORES_STMT: + case QUERY_NODE_SHOW_USERS_STMT: + case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: case QUERY_NODE_SHOW_GRANTS_FULL_STMT: case QUERY_NODE_SHOW_GRANTS_LOGS_STMT: case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT: diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index cd38b67fe9..808ab71b92 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -720,7 +720,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate goto end; } fields += sizeof(int8_t) + sizeof(int32_t); - if (needChangeLength && version == 1) { + if (needChangeLength && version == BLOCK_VERSION_1) { pStart += htonl(colLength[j]); } else { pStart += colLength[j]; @@ -751,7 +751,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate goto end; } fields += sizeof(int8_t) + sizeof(int32_t); - if (needChangeLength && version == 1) { + if (needChangeLength && version == BLOCK_VERSION_1) { pStart += htonl(colLength[i]); } else { pStart += colLength[i]; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index abe50a27da..3a622eeea6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -373,6 +373,7 @@ static int32_t collectUseTable(const SName* pName, SHashObj* pTable) { return taosHashPut(pTable, fullName, strlen(fullName), pName, sizeof(SName)); } +#ifdef BUILD_NO_CALL static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) { #ifndef TD_ENTERPRISE return TSDB_CODE_PAR_TABLE_NOT_EXIST; @@ -396,6 +397,7 @@ static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCac } return code; } +#endif int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView) { int32_t code = TSDB_CODE_SUCCESS; @@ -774,9 +776,11 @@ static bool isAggFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)); } +#ifdef BUILD_NO_CALL static bool isSelectFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsSelectFunc(((SFunctionNode*)pNode)->funcId)); } +#endif static bool isWindowPseudoColumnFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsWindowPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); @@ -790,9 +794,11 @@ static bool isInterpPseudoColumnFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsInterpPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); } +#ifdef BUILD_NO_CALL static bool isTimelineFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsTimelineFunc(((SFunctionNode*)pNode)->funcId)); } +#endif static bool isImplicitTsFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsImplicitTsFunc(((SFunctionNode*)pNode)->funcId)); @@ -1672,11 +1678,11 @@ static int32_t dataTypeComp(const SDataType* l, const SDataType* r) { static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { if (isMultiResFunc(pOp->pLeft)) { - generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); + generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT, ((SExprNode*)(pOp->pLeft))->userAlias); return DEAL_RES_ERROR; } if (isMultiResFunc(pOp->pRight)) { - generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT, ((SExprNode*)(pOp->pRight))->userAlias); return DEAL_RES_ERROR; } @@ -4003,6 +4009,26 @@ static int32_t translateEventWindow(STranslateContext* pCxt, SSelectStmt* pSelec } static int32_t translateCountWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { + SCountWindowNode* pCountWin = (SCountWindowNode*)pSelect->pWindow; + if (pCountWin->windowCount <= 1) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Size of Count window must exceed 1."); + } + + if (pCountWin->windowSliding <= 0) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Size of Count window must exceed 0."); + } + + if (pCountWin->windowSliding > pCountWin->windowCount) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "sliding value no larger than the count value."); + } + + if (pCountWin->windowCount > INT32_MAX) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Size of Count window must less than 2147483647(INT32_MAX)."); + } if (QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TIMELINE_QUERY, @@ -7738,9 +7764,11 @@ static int32_t addSubtableInfoToCreateStreamQuery(STranslateContext* pCxt, STabl return code; } +#ifdef BUILD_NO_CALL static bool isEventWindowQuery(SSelectStmt* pSelect) { return NULL != pSelect->pWindow && QUERY_NODE_EVENT_WINDOW == nodeType(pSelect->pWindow); } +#endif static bool hasJsonTypeProjection(SSelectStmt* pSelect) { SNode* pProj = NULL; @@ -7820,29 +7848,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm if (pStmt->pOptions->ignoreExpired != 1) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Ignore expired data of Count window must be 1."); - } - - SCountWindowNode* pCountWin = (SCountWindowNode*)pSelect->pWindow; - if (pCountWin->windowCount <= 1) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Size of Count window must exceed 1."); } - - if (pCountWin->windowSliding <= 0) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Size of Count window must exceed 0."); - } - - if (pCountWin->windowSliding > pCountWin->windowCount) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "sliding value no larger than the count value."); - } - - if (pCountWin->windowCount > INT32_MAX) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Size of Count window must less than 2147483647(INT32_MAX)."); - } - } return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index dfe33ce55e..76d8022578 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -192,6 +192,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Out of memory"; case TSDB_CODE_PAR_ORDERBY_AMBIGUOUS: return "ORDER BY \"%s\" is ambiguous"; + case TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT: + return "Operator not supported multi result: %s"; default: return "Unknown error"; } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 390faab537..d7b31d2ac8 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -381,7 +381,7 @@ TEST_F(ParserSelectTest, semanticCheck) { // TSDB_CODE_PAR_WRONG_VALUE_TYPE run("SELECT timestamp '2010a' FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE); - run("SELECT LAST(*) + SUM(c1) FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE); + run("SELECT LAST(*) + SUM(c1) FROM t1", TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT); run("SELECT CEIL(LAST(ts, c1)) FROM t1", TSDB_CODE_FUNC_FUNTION_PARA_NUM); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index c5e84898ed..37cdc31ae0 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -16,6 +16,7 @@ #include "planInt.h" #include "filter.h" #include "functionMgt.h" +#include "tglobal.h" typedef struct SLogicPlanContext { SPlanContext* pPlanCxt; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index d9a7475f59..9664be1da2 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -274,7 +274,7 @@ static bool stbSplNeedSplitWindow(bool streamQuery, SLogicNode* pNode) { } } - if (WINDOW_TYPE_STATE == pWindow->winType) { + if (WINDOW_TYPE_STATE == pWindow->winType || WINDOW_TYPE_COUNT == pWindow->winType) { if (!streamQuery) { return stbSplHasMultiTbScan(streamQuery, pNode); } else { diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 276b668717..d129b0024f 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -57,7 +57,7 @@ typedef enum { #define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ #define SCHEDULE_DEFAULT_MAX_NODE_NUM 20 -#define SCH_DEFAULT_TASK_TIMEOUT_USEC 5000000 +#define SCH_DEFAULT_TASK_TIMEOUT_USEC 30000000 #define SCH_MAX_TASK_TIMEOUT_USEC 300000000 #define SCH_DEFAULT_MAX_RETRY_NUM 6 #define SCH_MIN_AYSNC_EXEC_NUM 3 diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 841066a4c9..795c21c234 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -33,7 +33,7 @@ int32_t schedulerInit() { schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_MAX_JOB_NUM; schMgmt.cfg.maxNodeTableNum = tsQueryMaxConcurrentTables; schMgmt.cfg.schPolicy = SCHEDULE_DEFAULT_POLICY; - schMgmt.cfg.enableReSchedule = true; + schMgmt.cfg.enableReSchedule = false; qDebug("schedule init, policy: %d, maxNodeTableNum: %" PRId64", reSchedule:%d", schMgmt.cfg.schPolicy, schMgmt.cfg.maxNodeTableNum, schMgmt.cfg.enableReSchedule); diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 03f70604b7..1dc1db8e9c 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -17,10 +17,14 @@ #define _STREAM_BACKEDN_ROCKSDB_H_ #include "rocksdb/c.h" -//#include "streamInt.h" +// #include "streamInt.h" #include "streamState.h" #include "tcommon.h" +#ifdef __cplusplus +extern "C" { +#endif + typedef struct SCfComparator { rocksdb_comparator_t** comp; int32_t numOfComp; @@ -244,11 +248,6 @@ int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId); int32_t taskDbBuildSnap(void* arg, SArray* pSnap); -// int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); - -// STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId); -// void taskDbDestroy(void* pDb, bool flush); - int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId); SBkdMgt* bkdMgtCreate(char* path); @@ -258,4 +257,10 @@ int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname); void bkdMgtDestroy(SBkdMgt* bm); int32_t taskDbGenChkpUploadData(void* arg, void* bkdMgt, int64_t chkpId, int8_t type, char** path, SArray* list); -#endif \ No newline at end of file + +uint32_t nextPow2(uint32_t x); +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 87f63b48ed..d0055d5400 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -99,6 +99,7 @@ void streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration); int32_t streamDispatchStreamBlock(SStreamTask* pTask); void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups); int32_t getNumOfDispatchBranch(SStreamTask* pTask); +void clearBufferedDispatchMsg(SStreamTask* pTask); int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBlock); SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg); diff --git a/source/libs/stream/inc/streamsm.h b/source/libs/stream/inc/streamsm.h index 22e1c4497b..47e0ce1b55 100644 --- a/source/libs/stream/inc/streamsm.h +++ b/source/libs/stream/inc/streamsm.h @@ -26,21 +26,21 @@ extern "C" { typedef int32_t (*__state_trans_fn)(SStreamTask*); typedef int32_t (*__state_trans_succ_fn)(SStreamTask*); -typedef struct SAttachedEventInfo { +typedef struct SFutureHandleEventInfo { ETaskStatus status; // required status that this event can be handled EStreamTaskEvent event; // the delayed handled event void* pParam; - void* pFn; -} SAttachedEventInfo; + __state_trans_user_fn callBackFn; +} SFutureHandleEventInfo; typedef struct STaskStateTrans { - bool autoInvokeEndFn; - SStreamTaskState state; - EStreamTaskEvent event; - SStreamTaskState next; - __state_trans_fn pAction; - __state_trans_succ_fn pSuccAction; - SAttachedEventInfo attachEvent; + bool autoInvokeEndFn; + SStreamTaskState state; + EStreamTaskEvent event; + SStreamTaskState next; + __state_trans_fn pAction; + __state_trans_succ_fn pSuccAction; + SFutureHandleEventInfo attachEvent; } STaskStateTrans; struct SStreamTaskSM { diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index f173157da6..910fd93989 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -2788,7 +2788,6 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) { STREAM_STATE_DEL_ROCKSDB(pState, "state", &maxStateKey); return pCur; } -#ifdef BUILD_NO_CALL SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { stDebug("streamStateGetCur_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; @@ -2838,7 +2837,6 @@ int32_t streamStateFuncDel_rocksdb(SStreamState* pState, const STupleKey* key) { STREAM_STATE_DEL_ROCKSDB(pState, "func", key); return 0; } -#endif // session cf int32_t streamStateSessionPut_rocksdb(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) { @@ -3432,7 +3430,6 @@ int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* SSessionKey tmpKey = *key; int32_t valSize = *pVLen; void* tmp = taosMemoryMalloc(valSize); - // tdbRealloc(NULL, valSize); if (!tmp) { return -1; } @@ -3506,13 +3503,11 @@ int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, voi return code; } -#ifdef BUILD_NO_CALL int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen) { int code = 0; STREAM_STATE_PUT_ROCKSDB(pState, "default", key, pVal, pVLen); return code; } -#endif int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** pVal, int32_t* pVLen) { int code = 0; STREAM_STATE_GET_ROCKSDB(pState, "default", key, pVal, pVLen); @@ -3535,10 +3530,10 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co if (pIter == NULL) { return -1; } - + size_t klen = 0; rocksdb_iter_seek(pIter, start, strlen(start)); while (rocksdb_iter_valid(pIter)) { - const char* key = rocksdb_iter_key(pIter, NULL); + const char* key = rocksdb_iter_key(pIter, &klen); int32_t vlen = 0; const char* vval = rocksdb_iter_value(pIter, (size_t*)&vlen); char* val = NULL; @@ -3700,6 +3695,8 @@ uint32_t nextPow2(uint32_t x) { x = x | (x >> 16); return x + 1; } + +#ifdef BUILD_NO_CALL int32_t copyFiles(const char* src, const char* dst) { int32_t code = 0; // opt later, just hard link @@ -3739,6 +3736,7 @@ _err: taosCloseDir(&pDir); return code >= 0 ? 0 : -1; } +#endif int32_t isBkdDataMeta(char* name, int32_t len) { const char* pCurrent = "CURRENT"; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 9383383dc0..dc790b5b2d 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -315,6 +315,16 @@ int32_t getNumOfDispatchBranch(SStreamTask* pTask) { : taosArrayGetSize(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos); } +void clearBufferedDispatchMsg(SStreamTask* pTask) { + SDispatchMsgInfo* pMsgInfo = &pTask->msgInfo; + if (pMsgInfo->pData != NULL) { + destroyDispatchMsg(pMsgInfo->pData, getNumOfDispatchBranch(pTask)); + } + + pMsgInfo->pData = NULL; + pMsgInfo->dispatchMsgType = 0; +} + static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pData) { int32_t code = 0; int32_t numOfBlocks = taosArrayGetSize(pData->blocks); @@ -575,7 +585,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S !isAutoTableName(pDataBlock->info.parTbName) && !alreadyAddGroupId(pDataBlock->info.parTbName) && groupId != 0){ - buildCtbNameAddGruopId(pDataBlock->info.parTbName, groupId); + buildCtbNameAddGroupId(pDataBlock->info.parTbName, groupId); } } else { buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName); @@ -678,8 +688,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { // todo deal with only partially success dispatch case atomic_store_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 0); if (terrno == TSDB_CODE_APP_IS_STOPPING) { // in case of this error, do not retry anymore - destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); - pTask->msgInfo.pData = NULL; + clearBufferedDispatchMsg(pTask); return code; } @@ -740,6 +749,8 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) { int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + ASSERT(dataStrLen > 0); + void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) return -1; @@ -907,7 +918,6 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, SStreamChkptReadyInfo info = {.upStreamTaskId = pInfo->taskId, .upstreamNodeEpset = pInfo->epSet}; initRpcMsg(&info.msg, TDMT_STREAM_TASK_CHECKPOINT_READY, buf, tlen + sizeof(SMsgHead)); - info.msg.info.noResp = 1; // refactor later. stDebug("s-task:%s (level:%d) prepare checkpoint ready msg to upstream s-task:0x%" PRIx64 ":0x%x (vgId:%d) idx:%d, vgId:%d", @@ -937,15 +947,12 @@ void streamClearChkptReadyMsg(SStreamTask* pTask) { // this message has been sent successfully, let's try next one. static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId) { stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData); - destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); - bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); if (delayDispatch) { pTask->chkInfo.dispatchCheckpointTrigger = true; } - pTask->msgInfo.pData = NULL; - pTask->msgInfo.dispatchMsgType = 0; + clearBufferedDispatchMsg(pTask); int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs; @@ -1085,7 +1092,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i } else { // this message has been sent successfully, let's try next one. pTask->msgInfo.retryCount = 0; - // transtate msg has been sent to downstream successfully. let's transfer the fill-history task state + // trans-state msg has been sent to downstream successfully. let's transfer the fill-history task state if (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__TRANS_STATE) { stDebug("s-task:%s dispatch transtate msgId:%d to downstream successfully, start to transfer state", id, msgId); ASSERT(pTask->info.fillHistory == 1); @@ -1094,6 +1101,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i if (code != TSDB_CODE_SUCCESS) { // todo: do nothing if error happens } + clearBufferedDispatchMsg(pTask); + // now ready for next data output atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL); } else { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index bac6022834..27cd98aac6 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -410,6 +410,12 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { return TSDB_CODE_SUCCESS; } +static int32_t haltCallback(SStreamTask* pTask, void* param) { + streamTaskOpenAllUpstreamInput(pTask); + streamTaskSendCheckpointReq(pTask); + return TSDB_CODE_SUCCESS; +} + int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { int32_t code = TSDB_CODE_SUCCESS; SStreamMeta* pMeta = pTask->pMeta; @@ -419,11 +425,12 @@ int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { int32_t level = pTask->info.taskLevel; if (level == TASK_LEVEL__AGG || level == TASK_LEVEL__SOURCE) { // do transfer task operator states. code = streamDoTransferStateToStreamTask(pTask); - } else { // no state transfer for sink tasks, and drop fill-history task, followed by opening inputQ of sink task. + } else { + // no state transfer for sink tasks, and drop fill-history task, followed by opening inputQ of sink task. SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); if (pStreamTask != NULL) { // halt the related stream sink task - code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); + code = streamTaskHandleEventAsync(pStreamTask->status.pSM, TASK_EVENT_HALT, haltCallback, NULL); if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s halt stream task:%s failed, code:%s not transfer state to stream task", pTask->id.idStr, pStreamTask->id.idStr, tstrerror(code)); @@ -432,9 +439,6 @@ int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { } else { stDebug("s-task:%s halt by related fill-history task:%s", pStreamTask->id.idStr, pTask->id.idStr); } - - streamTaskOpenAllUpstreamInput(pStreamTask); - streamTaskSendCheckpointReq(pStreamTask); streamMetaReleaseTask(pMeta, pStreamTask); } } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index ab519e2b4b..a072ee1f6f 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -70,7 +70,7 @@ static void streamMetaEnvInit() { streamTimerInit(); } -void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit);} +void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } void streamMetaCleanup() { taosCloseRef(streamBackendId); @@ -669,6 +669,13 @@ static void doRemoveIdFromList(SStreamMeta* pMeta, int32_t num, SStreamTaskId* i } } +static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + streamTaskSendCheckpointSourceRsp(pTask); + } + return 0; +} + int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { SStreamTask* pTask = NULL; @@ -687,7 +694,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t } // handle the dropping event - streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_DROPPING); + streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL); } else { stDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId); streamMetaWUnLock(pMeta); @@ -1104,14 +1111,14 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { .inputQUsed = SIZE_IN_MiB(streamQueueGetItemSize((*pTask)->inputq.queue)), }; - entry.inputRate = entry.inputQUsed * 100.0 / (2*STREAM_TASK_QUEUE_CAPACITY_IN_SIZE); + entry.inputRate = entry.inputQUsed * 100.0 / (2 * STREAM_TASK_QUEUE_CAPACITY_IN_SIZE); if ((*pTask)->info.taskLevel == TASK_LEVEL__SINK) { entry.sinkQuota = (*pTask)->outputInfo.pTokenBucket->quotaRate; entry.sinkDataSize = SIZE_IN_MiB((*pTask)->execInfo.sink.dataSize); } if ((*pTask)->chkInfo.checkpointingId != 0) { - entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId)? 1:0; + entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId) ? 1 : 0; entry.checkpointId = (*pTask)->chkInfo.checkpointingId; entry.chkpointTransId = (*pTask)->chkInfo.transId; @@ -1160,7 +1167,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { } tEncoderClear(&encoder); - SRpcMsg msg = {.info.noResp = 1}; + SRpcMsg msg = {0}; initRpcMsg(&msg, TDMT_MND_STREAM_HEARTBEAT, buf, tlen); pMeta->pHbInfo->hbCount += 1; @@ -1172,7 +1179,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { stDebug("vgId:%d no tasks and no mnd epset, not send stream hb to mnode", pMeta->vgId); } - _end: +_end: streamMetaClearHbMsg(&hbMsg); return TSDB_CODE_SUCCESS; } @@ -1304,28 +1311,28 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { } void streamMetaRLock(SStreamMeta* pMeta) { -// stTrace("vgId:%d meta-rlock", pMeta->vgId); + // stTrace("vgId:%d meta-rlock", pMeta->vgId); taosThreadRwlockRdlock(&pMeta->lock); } void streamMetaRUnLock(SStreamMeta* pMeta) { -// stTrace("vgId:%d meta-runlock", pMeta->vgId); + // stTrace("vgId:%d meta-runlock", pMeta->vgId); int32_t code = taosThreadRwlockUnlock(&pMeta->lock); if (code != TSDB_CODE_SUCCESS) { stError("vgId:%d meta-runlock failed, code:%d", pMeta->vgId, code); } else { -// stTrace("vgId:%d meta-runlock completed", pMeta->vgId); + // stTrace("vgId:%d meta-runlock completed", pMeta->vgId); } } void streamMetaWLock(SStreamMeta* pMeta) { -// stTrace("vgId:%d meta-wlock", pMeta->vgId); + // stTrace("vgId:%d meta-wlock", pMeta->vgId); taosThreadRwlockWrlock(&pMeta->lock); -// stTrace("vgId:%d meta-wlock completed", pMeta->vgId); + // stTrace("vgId:%d meta-wlock completed", pMeta->vgId); } void streamMetaWUnLock(SStreamMeta* pMeta) { -// stTrace("vgId:%d meta-wunlock", pMeta->vgId); + // stTrace("vgId:%d meta-wunlock", pMeta->vgId); taosThreadRwlockUnlock(&pMeta->lock); } @@ -1395,7 +1402,7 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) pMeta->sendMsgBeforeClosing = true; } - pMeta->role = (isLeader)? NODE_ROLE_LEADER:NODE_ROLE_FOLLOWER; + pMeta->role = (isLeader) ? NODE_ROLE_LEADER : NODE_ROLE_FOLLOWER; streamMetaWUnLock(pMeta); if (isLeader) { @@ -1531,8 +1538,8 @@ int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) { bool streamMetaAllTasksReady(const SStreamMeta* pMeta) { int32_t num = taosArrayGetSize(pMeta->pTaskList); - for(int32_t i = 0; i < num; ++i) { - STaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); + for (int32_t i = 0; i < num; ++i) { + STaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pTaskId, sizeof(*pTaskId)); if (ppTask == NULL) { continue; @@ -1633,7 +1640,7 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3 pStartInfo->elapsedTime = (pStartInfo->startTs != 0) ? pStartInfo->readyTs - pStartInfo->startTs : 0; stDebug("vgId:%d all %d task(s) check downstream completed, last completed task:0x%x (succ:%d) startTs:%" PRId64 - ", readyTs:%" PRId64 " total elapsed time:%.2fs", + ", readyTs:%" PRId64 " total elapsed time:%.2fs", pMeta->vgId, numOfTotal, taskId, ready, pStartInfo->startTs, pStartInfo->readyTs, pStartInfo->elapsedTime / 1000.0); diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index bd28d2bca9..723f04c499 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -156,6 +156,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pKey = *pDestWinKey; goto _end; } @@ -167,6 +168,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pKey = *pDestWinKey; goto _end; } @@ -380,6 +382,14 @@ static SStreamStateCur* seekKeyCurrentPrev_buff(SStreamFileState* pFileState, co (*pWins) = pWinStates; } + if (size > 0 && index == -1) { + SRowBuffPos* pPos = taosArrayGetP(pWinStates, 0); + SSessionKey* pWin = (SSessionKey*)pPos->pKey; + if (pWinKey->win.skey == pWin->win.skey) { + index = 0; + } + } + if (index >= 0) { pCur = createSessionStateCursor(pFileState); pCur->buffIndex = index; @@ -387,6 +397,7 @@ static SStreamStateCur* seekKeyCurrentPrev_buff(SStreamFileState* pFileState, co *pIndex = index; } } + return pCur; } @@ -488,6 +499,7 @@ SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SS void* pFileStore = getStateFileStore(pFileState); SStreamStateCur* pCur = streamStateSessionSeekKeyPrev_rocksdb(pFileStore, pWinKey); if (pCur) { + pCur->pStreamFileState = pFileState; SSessionKey key = {0}; void* pVal = NULL; int len = 0; @@ -665,6 +677,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *key = *pDestWinKey; goto _end; } @@ -678,6 +691,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *key = *pDestWinKey; goto _end; } @@ -736,6 +750,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C void* pRockVal = NULL; SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileStore, pKey->groupId); code = streamStateSessionGetKVByCur_rocksdb(pCur, pWinKey, &pRockVal, pVLen); + streamStateFreeCur(pCur); if (code == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code); if (code == TSDB_CODE_SUCCESS) { @@ -743,7 +758,6 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)( (char*)(pRockVal) + (valSize - sizeof(COUNT_TYPE)) ); if (inSessionWindow(pWinKey, startTs, gap) || (*pWinStateCout) < winCount) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen); - streamStateFreeCur(pCur); goto _end; } } @@ -751,7 +765,6 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C pWinKey->win.ekey = endTs; (*pVal) = createSessionWinBuff(pFileState, pWinKey, NULL, NULL); taosMemoryFree(pRockVal); - streamStateFreeCur(pCur); } else { (*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey); code = TSDB_CODE_FAILED; @@ -771,6 +784,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pWinKey = *pDestWinKey; goto _end; } @@ -799,6 +813,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pWinKey = *pDestWinKey; goto _end; } diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 6112a208c6..cb340ade32 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -385,7 +385,7 @@ int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) { void doProcessDownstreamReadyRsp(SStreamTask* pTask) { EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST; - streamTaskOnHandleEventSuccess(pTask->status.pSM, event); + streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); int64_t initTs = pTask->execInfo.init; int64_t startTs = pTask->execInfo.start; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index f3494377d6..356bc92b63 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -412,9 +412,7 @@ void tFreeStreamTask(SStreamTask* pTask) { pTask->pReadyMsgList = taosArrayDestroy(pTask->pReadyMsgList); if (pTask->msgInfo.pData != NULL) { - destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); - pTask->msgInfo.pData = NULL; - pTask->msgInfo.dispatchMsgType = 0; + clearBufferedDispatchMsg(pTask); } if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { @@ -494,6 +492,8 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i pRange->range.maxVer = ver; pRange->range.minVer = ver; } else { + // the initial value of processedVer/nextProcessVer/checkpointVer for stream task with related fill-history task + // is set at the mnode. if (pTask->info.fillHistory == 1) { pChkInfo->checkpointVer = pRange->range.maxVer; pChkInfo->processedVer = pRange->range.maxVer; @@ -502,6 +502,15 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i pChkInfo->checkpointVer = pRange->range.minVer - 1; pChkInfo->processedVer = pRange->range.minVer - 1; pChkInfo->nextProcessVer = pRange->range.minVer; + + { // for compatible purpose, remove it later + if (pRange->range.minVer == 0) { + pChkInfo->checkpointVer = 0; + pChkInfo->processedVer = 0; + pChkInfo->nextProcessVer = 1; + stDebug("s-task:%s update the processedVer to 0 from -1 due to compatible purpose", pTask->id.idStr); + } + } } } } @@ -636,8 +645,6 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpSet:%s", pTask->id.taskId, pDispatcher->taskId, nodeId, buf); } - } else { - // do nothing } } @@ -755,21 +762,13 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) { return status; } -int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt, bool metaLock) { - if (pTask == NULL) { - return TSDB_CODE_SUCCESS; - } - +int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) { SStreamMeta* pMeta = pTask->pMeta; STaskId sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId}; if (pTask->info.fillHistory == 0) { return TSDB_CODE_SUCCESS; } - if (metaLock) { - streamMetaWLock(pMeta); - } - SStreamTask** ppStreamTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &sTaskId, sizeof(sTaskId)); if (ppStreamTask != NULL) { stDebug("s-task:%s clear the related stream task:0x%x attr to fill-history task", pTask->id.idStr, @@ -787,10 +786,6 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt, bool taosThreadMutexUnlock(&(*ppStreamTask)->lock); } - if (metaLock) { - streamMetaWUnLock(pMeta); - } - return TSDB_CODE_SUCCESS; } @@ -858,8 +853,8 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc) pDst->chkpointTransId = pSrc->chkpointTransId; } -void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask) { - streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_PAUSE); +static int32_t taskPauseCallback(SStreamTask* pTask, void* param) { + SStreamMeta* pMeta = pTask->pMeta; int32_t num = atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1); stInfo("vgId:%d s-task:%s pause stream task. pause task num:%d", pMeta->vgId, pTask->id.idStr, num); @@ -871,24 +866,24 @@ void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask) { } stDebug("vgId:%d s-task:%s set pause flag and pause task", pMeta->vgId, pTask->id.idStr); + return TSDB_CODE_SUCCESS; +} + +void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask) { + streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_PAUSE, taskPauseCallback, NULL); } void streamTaskResume(SStreamTask* pTask) { SStreamTaskState prevState = *streamTaskGetStatus(pTask); - SStreamMeta* pMeta = pTask->pMeta; - if (prevState.state == TASK_STATUS__PAUSE || prevState.state == TASK_STATUS__HALT) { - streamTaskRestoreStatus(pTask); - - char* pNew = streamTaskGetStatus(pTask)->name; - if (prevState.state == TASK_STATUS__PAUSE) { - int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1); - stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, prevState.name, num); - } else { - stInfo("s-task:%s status:%s resume from %s", pTask->id.idStr, pNew, prevState.name); - } + SStreamMeta* pMeta = pTask->pMeta; + int32_t code = streamTaskRestoreStatus(pTask); + if (code == TSDB_CODE_SUCCESS) { + char* pNew = streamTaskGetStatus(pTask)->name; + int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1); + stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, prevState.name, num); } else { - stDebug("s-task:%s status:%s not in pause/halt status, no need to resume", pTask->id.idStr, prevState.name); + stInfo("s-task:%s status:%s no need to resume, paused task(s):%d", pTask->id.idStr, prevState.name, pMeta->numOfPausedTasks); } } @@ -923,9 +918,8 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) { } tEncoderClear(&encoder); - SRpcMsg msg = {.info.noResp = 1}; + SRpcMsg msg = {0}; initRpcMsg(&msg, TDMT_MND_STREAM_REQ_CHKPT, buf, tlen); - stDebug("s-task:%s vgId:%d build and send task checkpoint req", id, vgId); tmsgSendReq(&pTask->info.mnodeEpset, &msg); diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 83e71c42bc..6aa215586a 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -59,20 +59,23 @@ static int32_t streamTaskInitStatus(SStreamTask* pTask); static int32_t streamTaskKeepCurrentVerInWal(SStreamTask* pTask); static int32_t initStateTransferTable(); static void doInitStateTransferTable(void); -static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask); static STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStreamTaskEvent event, __state_trans_fn fn, __state_trans_succ_fn succFn, - SAttachedEventInfo* pEventInfo, bool autoInvoke); + SFutureHandleEventInfo* pEventInfo); static int32_t dummyFn(SStreamTask* UNUSED_PARAM(p)) { return TSDB_CODE_SUCCESS; } -static int32_t attachEvent(SStreamTask* pTask, SAttachedEventInfo* pEvtInfo) { +static int32_t attachWaitedEvent(SStreamTask* pTask, SFutureHandleEventInfo* pEvtInfo) { char* p = streamTaskGetStatus(pTask)->name; stDebug("s-task:%s status:%s attach event:%s required status:%s, since not allowed to handle it", pTask->id.idStr, p, GET_EVT_NAME(pEvtInfo->event), StreamTaskStatusList[pEvtInfo->status].name); - taosArrayPush(pTask->status.pSM->pWaitingEventList, pEvtInfo); + + SArray* pList = pTask->status.pSM->pWaitingEventList; + taosArrayPush(pList, pEvtInfo); + + stDebug("s-task:%s add into waiting list, total waiting events:%d", pTask->id.idStr, (int32_t)taosArrayGetSize(pList)); return 0; } @@ -85,18 +88,6 @@ int32_t streamTaskInitStatus(SStreamTask* pTask) { return 0; } -static int32_t streamTaskDoCheckpoint(SStreamTask* pTask) { - stDebug("s-task:%s start to do checkpoint", pTask->id.idStr); - return 0; -} - -int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask) { - if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - streamTaskSendCheckpointSourceRsp(pTask); - } - return 0; -} - int32_t streamTaskKeepCurrentVerInWal(SStreamTask* pTask) { if (!HAS_RELATED_FILLHISTORY_TASK(pTask)) { stError("s-task:%s no related fill-history task, since it may have been dropped already", pTask->id.idStr); @@ -170,9 +161,11 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr, pEventName, el, pSM->prev.state.name, pSM->current.name); - SAttachedEventInfo* pEvtInfo = taosArrayGet(pSM->pWaitingEventList, 0); + ASSERT(taosArrayGetSize(pSM->pWaitingEventList) == 1); - // OK, let's handle the attached event, since the task has reached the required status now + SFutureHandleEventInfo* pEvtInfo = taosArrayGet(pSM->pWaitingEventList, 0); + + // OK, let's handle the waiting event, since the task has reached the required status now if (pSM->current.state == pEvtInfo->status) { stDebug("s-task:%s handle the event:%s in waiting list, state:%s", pTask->id.idStr, GET_EVT_NAME(pEvtInfo->event), pSM->current.name); @@ -189,7 +182,7 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, code = pNextTrans->pAction(pSM->pTask); if (pNextTrans->autoInvokeEndFn) { - return streamTaskOnHandleEventSuccess(pSM, pNextTrans->event); + return streamTaskOnHandleEventSuccess(pSM, pNextTrans->event, pEvtInfo->callBackFn, pEvtInfo->pParam); } else { return code; } @@ -203,30 +196,61 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, return code; } -void streamTaskRestoreStatus(SStreamTask* pTask) { +static int32_t removeEventInWaitingList(SStreamTask* pTask, EStreamTaskEvent event) { SStreamTaskSM* pSM = pTask->status.pSM; + bool removed = false; taosThreadMutexLock(&pTask->lock); - ASSERT(pSM->pActiveTrans == NULL); - ASSERT(pSM->current.state == TASK_STATUS__PAUSE || pSM->current.state == TASK_STATUS__HALT); + int32_t num = taosArrayGetSize(pSM->pWaitingEventList); + for (int32_t i = 0; i < num; ++i) { + SFutureHandleEventInfo* pInfo = taosArrayGet(pSM->pWaitingEventList, i); + if (pInfo->event == event) { + taosArrayRemove(pSM->pWaitingEventList, i); + stDebug("s-task:%s pause event in waiting list not be handled yet, remove it from waiting list, remaining:%d", + pTask->id.idStr, pInfo->event); + removed = true; + break; + } + } - SStreamTaskState state = pSM->current; - pSM->current = pSM->prev.state; - - pSM->prev.state = state; - pSM->prev.evt = 0; - - pSM->startTs = taosGetTimestampMs(); - - if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { - stDebug("s-task:%s restore status, %s -> %s, and then handle waiting event", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); - doHandleWaitingEvent(pSM, "restore-pause/halt", pTask); - } else { - stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); + if (!removed) { + stDebug("s-task:%s failed to remove event:%s in waiting list", pTask->id.idStr, StreamTaskEventList[event].name); } taosThreadMutexUnlock(&pTask->lock); + return TSDB_CODE_SUCCESS; +} + +int32_t streamTaskRestoreStatus(SStreamTask* pTask) { + SStreamTaskSM* pSM = pTask->status.pSM; + int32_t code = 0; + + taosThreadMutexLock(&pTask->lock); + + if (pSM->current.state == TASK_STATUS__PAUSE && pSM->pActiveTrans == NULL) { + SStreamTaskState state = pSM->current; + pSM->current = pSM->prev.state; + + pSM->prev.state = state; + pSM->prev.evt = 0; + + pSM->startTs = taosGetTimestampMs(); + + if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { + stDebug("s-task:%s restore status, %s -> %s, and then handle waiting event", pTask->id.idStr, + pSM->prev.state.name, pSM->current.name); + doHandleWaitingEvent(pSM, "restore-pause/halt", pTask); + } else { + stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); + } + } else { + removeEventInWaitingList(pTask, TASK_EVENT_PAUSE); + code = -1; // failed to restore the status + } + + taosThreadMutexUnlock(&pTask->lock); + return code; } SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask) { @@ -242,7 +266,7 @@ SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask) { } pSM->pTask = pTask; - pSM->pWaitingEventList = taosArrayInit(4, sizeof(SAttachedEventInfo)); + pSM->pWaitingEventList = taosArrayInit(4, sizeof(SFutureHandleEventInfo)); if (pSM->pWaitingEventList == NULL) { taosMemoryFree(pSM); @@ -273,7 +297,7 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt const char* id = pTask->id.idStr; if (pTrans->attachEvent.event != 0) { - attachEvent(pTask, &pTrans->attachEvent); + attachWaitedEvent(pTask, &pTrans->attachEvent); taosThreadMutexUnlock(&pTask->lock); while (1) { @@ -303,7 +327,32 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt // todo handle error code; if (pTrans->autoInvokeEndFn) { - streamTaskOnHandleEventSuccess(pSM, event); + streamTaskOnHandleEventSuccess(pSM, event, NULL, NULL); + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t doHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskStateTrans* pTrans, __state_trans_user_fn callbackFn, void* param) { + SStreamTask* pTask = pSM->pTask; + if (pTrans->attachEvent.event != 0) { + SFutureHandleEventInfo info = pTrans->attachEvent; + info.pParam = param; + info.callBackFn = callbackFn; + + attachWaitedEvent(pTask, &info); + taosThreadMutexUnlock(&pTask->lock); + } else { // override current active trans + pSM->pActiveTrans = pTrans; + pSM->startTs = taosGetTimestampMs(); + taosThreadMutexUnlock(&pTask->lock); + + int32_t code = pTrans->pAction(pTask); + // todo handle error code; + + if (pTrans->autoInvokeEndFn) { + streamTaskOnHandleEventSuccess(pSM, event, callbackFn, param); } } @@ -349,6 +398,45 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) { return code; } +int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param) { + int32_t code = TSDB_CODE_SUCCESS; + SStreamTask* pTask = pSM->pTask; + STaskStateTrans* pTrans = NULL; + + while (1) { + taosThreadMutexLock(&pTask->lock); + + if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) { + EStreamTaskEvent evt = pSM->pActiveTrans->event; + taosThreadMutexUnlock(&pTask->lock); + + stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed", + pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt)); + taosMsleep(100); + } else { + // no active event trans exists, handle this event directly + pTrans = streamTaskFindTransform(pSM->current.state, event); + if (pTrans == NULL) { + stDebug("s-task:%s failed to handle event:%s, status:%s", pTask->id.idStr, GET_EVT_NAME(event), pSM->current.name); + taosThreadMutexUnlock(&pTask->lock); + return TSDB_CODE_STREAM_INVALID_STATETRANS; + } + + if (pSM->pActiveTrans != NULL) { + // currently in some state transfer procedure, not auto invoke transfer, quit from this procedure + stDebug("s-task:%s event:%s handle procedure quit, status %s -> %s failed, handle event %s now", + pTask->id.idStr, GET_EVT_NAME(pSM->pActiveTrans->event), pSM->current.name, + pSM->pActiveTrans->next.name, GET_EVT_NAME(event)); + } + + code = doHandleEventAsync(pSM, event, pTrans, callbackFn, param); + break; + } + } + + return code; +} + static void keepPrevInfo(SStreamTaskSM* pSM) { STaskStateTrans* pTrans = pSM->pActiveTrans; @@ -356,8 +444,9 @@ static void keepPrevInfo(SStreamTaskSM* pSM) { pSM->prev.evt = pTrans->event; } -int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event) { +int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param) { SStreamTask* pTask = pSM->pTask; + const char* id = pTask->id.idStr; // do update the task status taosThreadMutexLock(&pTask->lock); @@ -369,16 +458,16 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even s == TASK_STATUS__UNINIT || s == TASK_STATUS__READY); // the pSM->prev.evt may be 0, so print string is not appropriate. - stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", pTask->id.idStr, - GET_EVT_NAME(event), pSM->current.name, GET_EVT_NAME(pSM->prev.evt)); + stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", id, GET_EVT_NAME(event), + pSM->current.name, GET_EVT_NAME(pSM->prev.evt)); taosThreadMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INVALID_STATETRANS; } if (pTrans->event != event) { - stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", pTask->id.idStr, - GET_EVT_NAME(event), pSM->current.name, GET_EVT_NAME(pTrans->event)); + stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", id, GET_EVT_NAME(event), + pSM->current.name, GET_EVT_NAME(pTrans->event)); taosThreadMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INVALID_STATETRANS; } @@ -388,16 +477,31 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even pSM->current = pTrans->next; pSM->pActiveTrans = NULL; + // todo remove it // on success callback, add into lock if necessary, or maybe we should add an option for this? pTrans->pSuccAction(pTask); + taosThreadMutexUnlock(&pTask->lock); + + // todo: add parameter to control lock + // after handling the callback function assigned by invoker, go on handling the waiting tasks + if (callbackFn != NULL) { + stDebug("s-task:%s start to handle user-specified callback fn for event:%s", id, GET_EVT_NAME(pTrans->event)); + callbackFn(pSM->pTask, param); + + stDebug("s-task:%s handle user-specified callback fn for event:%s completed", id, GET_EVT_NAME(pTrans->event)); + } + + taosThreadMutexLock(&pTask->lock); + + // tasks in waiting list if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask); } else { taosThreadMutexUnlock(&pTask->lock); int64_t el = (taosGetTimestampMs() - pSM->startTs); - stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr, + stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", id, GET_EVT_NAME(pTrans->event), el, pSM->prev.state.name, pSM->current.name); } @@ -453,7 +557,7 @@ void streamTaskSetStatusReady(SStreamTask* pTask) { } STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStreamTaskEvent event, __state_trans_fn fn, - __state_trans_succ_fn succFn, SAttachedEventInfo* pEventInfo, bool autoInvoke) { + __state_trans_succ_fn succFn, SFutureHandleEventInfo* pEventInfo) { STaskStateTrans trans = {0}; trans.state = StreamTaskStatusList[current]; trans.next = StreamTaskStatusList[next]; @@ -468,7 +572,7 @@ STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStr trans.pAction = (fn != NULL) ? fn : dummyFn; trans.pSuccAction = (succFn != NULL) ? succFn : dummyFn; - trans.autoInvokeEndFn = autoInvoke; + trans.autoInvokeEndFn = (fn == NULL); return trans; } @@ -482,93 +586,93 @@ void doInitStateTransferTable(void) { streamTaskSMTrans = taosArrayInit(8, sizeof(STaskStateTrans)); // initialization event handle - STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT, streamTaskInitStatus, streamTaskOnNormalTaskReady, false, false); + STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT, streamTaskInitStatus, streamTaskOnNormalTaskReady, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, false, false); + trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, NULL); taosArrayPush(streamTaskSMTrans, &trans); // scan-history related event - trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); // halt stream task, from other task status - trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true); + trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true); + trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL); taosArrayPush(streamTaskSMTrans, &trans); - SAttachedEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT}; + SFutureHandleEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT}; - trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info, true); + trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true); + trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL); taosArrayPush(streamTaskSMTrans, &trans); // checkpoint related event - trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, streamTaskDoCheckpoint, NULL, true); + trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, streamTaskDoCheckpoint, NULL, true); + trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__READY, TASK_EVENT_CHECKPOINT_DONE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__READY, TASK_EVENT_CHECKPOINT_DONE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); // pause & resume related event handle - trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - info = (SAttachedEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE}; - trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); + info = (SFutureHandleEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE}; + trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); + trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); // resume is completed by restore status of state-machine // stop related event - trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); // dropping related event - trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); } //clang-format on \ No newline at end of file diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 454ed4297c..764bf6e026 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -22,7 +22,7 @@ #define DEFAULT_MAP_CAPACITY 131072 #define DEFAULT_MAP_SIZE (DEFAULT_MAP_CAPACITY * 100) #define ROWS_PER_MILLISECOND 1 -#define MAX_NUM_SCALABLE_BF 100000 +#define MAX_NUM_SCALABLE_BF 64 #define MIN_NUM_SCALABLE_BF 10 #define DEFAULT_PREADD_BUCKET 1 #define MAX_INTERVAL MILLISECOND_PER_MINUTE @@ -81,7 +81,9 @@ static int64_t adjustInterval(int64_t interval, int32_t precision) { static int64_t adjustWatermark(int64_t adjInterval, int64_t originInt, int64_t watermark) { if (watermark <= adjInterval) { watermark = TMAX(originInt / adjInterval, 1) * adjInterval; - } else if (watermark > MAX_NUM_SCALABLE_BF * adjInterval) { + } + + if (watermark > MAX_NUM_SCALABLE_BF * adjInterval) { watermark = MAX_NUM_SCALABLE_BF * adjInterval; } return watermark; diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt index c90e05bcf6..c472207b27 100644 --- a/source/libs/stream/test/CMakeLists.txt +++ b/source/libs/stream/test/CMakeLists.txt @@ -1,40 +1,104 @@ -MESSAGE(STATUS "build stream unit test") - -# GoogleTest requires at least C++11 -SET(CMAKE_CXX_STANDARD 11) -AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) # bloomFilterTest -ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp") -TARGET_LINK_LIBRARIES(streamUpdateTest - PUBLIC os util common gtest gtest_main stream executor index +#TARGET_LINK_LIBRARIES(streamUpdateTest + #PUBLIC os util common gtest gtest_main stream executor index + #) + +#TARGET_INCLUDE_DIRECTORIES( + #streamUpdateTest + #PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/" + #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" +#) + +#ADD_EXECUTABLE(checkpointTest checkpointTest.cpp) +#TARGET_LINK_LIBRARIES( + #checkpointTest + #PUBLIC os common gtest stream executor qcom index transport util +#) + +#TARGET_INCLUDE_DIRECTORIES( + #checkpointTest + #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" +#) + +#add_executable(backendTest "") + +#target_sources(backendTest + #PRIVATE + #"backendTest.cpp" +#) + +#TARGET_LINK_LIBRARIES( + #backendTest + #PUBLIC rocksdb + #PUBLIC os common gtest stream executor qcom index transport util +#) + +#TARGET_INCLUDE_DIRECTORIES( + #backendTest + #PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/" + #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" +#) + +#add_test( + #NAME streamUpdateTest + #COMMAND streamUpdateTest +#) + +#add_test( + #NAME checkpointTest + #COMMAND checkpointTest +#) +#add_test( + #NAME backendTest + #COMMAND backendTest +#) + + +#add_executable(backendTest "") + +#target_sources(backendTest + #PUBLIC + #"backendTest.cpp" +#) + +#target_include_directories( + #backendTest + #PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/" + #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" +#) + +#target_link_libraries( + #backendTest + #PUBLIC rocksdb + #PUBLIC os common gtest stream executor qcom index transport util +#) + + +MESSAGE(STATUS "build parser unit test") + +IF(NOT TD_DARWIN) + # GoogleTest requires at least C++11 + SET(CMAKE_CXX_STANDARD 11) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + + ADD_EXECUTABLE(backendTest ${SOURCE_LIST}) + TARGET_LINK_LIBRARIES( + backendTest + PUBLIC rocksdb + PUBLIC os common gtest stream executor qcom index transport util vnode ) -TARGET_INCLUDE_DIRECTORIES( - streamUpdateTest - PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/" - PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" -) + TARGET_INCLUDE_DIRECTORIES( + backendTest + PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/" + PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" + ) -ADD_EXECUTABLE(checkpointTest checkpointTest.cpp) -TARGET_LINK_LIBRARIES( - checkpointTest - PUBLIC os common gtest stream executor qcom index transport util -) - -TARGET_INCLUDE_DIRECTORIES( - checkpointTest - PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" -) - -add_test( - NAME streamUpdateTest - COMMAND streamUpdateTest -) - -add_test( - NAME checkpointTest - COMMAND checkpointTest -) \ No newline at end of file + ADD_TEST( + NAME backendTest + COMMAND backendTest + ) +ENDIF () \ No newline at end of file diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp new file mode 100644 index 0000000000..a949748eb5 --- /dev/null +++ b/source/libs/stream/test/backendTest.cpp @@ -0,0 +1,437 @@ +#include + +#include +#include +#include +#include +#include "streamBackendRocksdb.h" +#include "streamSnapshot.h" +#include "streamState.h" +#include "tstream.h" +#include "tstreamFileState.h" +#include "tstreamUpdate.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +class BackendEnv : public ::testing::Test { + protected: + virtual void SetUp() {} + virtual void TearDown() {} +}; + +void *backendCreate() { + const char *streamPath = "/tmp"; + void *p = NULL; + + // char *absPath = NULL; + // // SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(streamPath, -1, 2); + // STaskDbWrapper *p = taskDbOpen((char *)streamPath, (char *)"stream-backend", -1); + // ASSERT(p != NULL); + return p; +} + +SStreamState *stateCreate(const char *path) { + SStreamTask *pTask = (SStreamTask *)taosMemoryCalloc(1, sizeof(SStreamTask)); + pTask->ver = 1024; + pTask->id.streamId = 1023; + pTask->id.taskId = 1111111; + SStreamMeta *pMeta = streamMetaOpen((path), NULL, NULL, 0, 0, NULL); + pTask->pMeta = pMeta; + + SStreamState *p = streamStateOpen((char *)path, pTask, true, 32, 32 * 1024); + ASSERT(p != NULL); + return p; +} +void *backendOpen() { + streamMetaInit(); + const char *path = "/tmp/backend"; + SStreamState *p = stateCreate(path); + ASSERT(p != NULL); + + // write bacth + // default/state/fill/sess/func/parname/partag + int32_t size = 100; + std::vector tsArray; + for (int32_t i = 0; i < size; i++) { + int64_t ts = taosGetTimestampMs(); + SWinKey key; // = {.groupId = (uint64_t)(i), .ts = ts}; + key.groupId = (uint64_t)(i); + key.ts = ts; + const char *val = "value data"; + int32_t vlen = strlen(val); + streamStatePut_rocksdb(p, &key, (char *)val, vlen); + + tsArray.push_back(ts); + } + for (int32_t i = 0; i < size; i++) { + int64_t ts = tsArray[i]; + SWinKey key = {0}; //{.groupId = (uint64_t)(i), .ts = ts}; + key.groupId = (uint64_t)(i); + key.ts = ts; + + const char *val = "value data"; + int32_t len = 0; + char *newVal = NULL; + streamStateGet_rocksdb(p, &key, (void **)&newVal, &len); + ASSERT(len == strlen(val)); + } + int64_t ts = tsArray[0]; + SWinKey key = {0}; // {.groupId = (uint64_t)(0), .ts = ts}; + key.groupId = (uint64_t)(0); + key.ts = ts; + + streamStateDel_rocksdb(p, &key); + + streamStateClear_rocksdb(p); + + for (int i = 0; i < size; i++) { + int64_t ts = tsArray[i]; + SWinKey key = {0}; //{.groupId = (uint64_t)(i), .ts = ts}; + key.groupId = (uint64_t)(i); + key.ts = ts; + + const char *val = "value data"; + int32_t len = 0; + char *newVal = NULL; + int32_t code = streamStateGet_rocksdb(p, &key, (void **)&newVal, &len); + ASSERT(code != 0); + } + tsArray.clear(); + + for (int i = 0; i < size; i++) { + int64_t ts = taosGetTimestampMs(); + tsArray.push_back(ts); + + SWinKey key = {0}; //{.groupId = (uint64_t)(i), .ts = ts}; + key.groupId = (uint64_t)(i); + key.ts = ts; + + const char *val = "value data"; + int32_t vlen = strlen(val); + streamStatePut_rocksdb(p, &key, (char *)val, vlen); + } + + SWinKey winkey; + int32_t code = streamStateGetFirst_rocksdb(p, &key); + ASSERT(code == 0); + ASSERT(key.ts == tsArray[0]); + + SStreamStateCur *pCurr = streamStateSeekToLast_rocksdb(p); + ASSERT(pCurr != NULL); + streamStateFreeCur(pCurr); + + winkey.groupId = 0; + winkey.ts = tsArray[0]; + char *val = NULL; + int32_t len = 0; + + pCurr = streamStateSeekKeyNext_rocksdb(p, &winkey); + ASSERT(pCurr != NULL); + + streamStateFreeCur(pCurr); + + tsArray.clear(); + for (int i = 0; i < size; i++) { + int64_t ts = taosGetTimestampMs(); + tsArray.push_back(ts); + STupleKey key = {0}; + key.groupId = (uint64_t)(0); //= {.groupId = (uint64_t)(0), .ts = ts, .exprIdx = i}; + key.ts = ts; + key.exprIdx = i; + + const char *val = "Value"; + int32_t len = strlen(val); + streamStateFuncPut_rocksdb(p, &key, val, len); + } + for (int i = 0; i < size; i++) { + STupleKey key = {0}; //{.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i}; + key.groupId = (uint64_t)(0); + key.ts = tsArray[i]; + key.exprIdx = i; + + char *val = NULL; + int32_t len = 0; + streamStateFuncGet_rocksdb(p, &key, (void **)&val, &len); + ASSERT(len == strlen("Value")); + } + for (int i = 0; i < size; i++) { + STupleKey key = {0}; //{.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i}; + key.groupId = (uint64_t)(0); + key.ts = tsArray[i]; + key.exprIdx = i; + + char *val = NULL; + int32_t len = 0; + streamStateFuncDel_rocksdb(p, &key); + } + + // session put + tsArray.clear(); + + for (int i = 0; i < size; i++) { + SSessionKey key = {0}; //{.win = {.skey = i, .ekey = i}, .groupId = (uint64_t)(0)}; + key.win.skey = i; + key.win.ekey = i; + key.groupId = (uint64_t)(0); + tsArray.push_back(i); + + const char *val = "Value"; + int32_t len = strlen(val); + streamStateSessionPut_rocksdb(p, &key, val, len); + + char *pval = NULL; + ASSERT(0 == streamStateSessionGet_rocksdb(p, &key, (void **)&pval, &len)); + ASSERT(strncmp(pval, val, len) == 0); + } + + for (int i = 0; i < size; i++) { + SSessionKey key = {0}; //{.win = {.skey = tsArray[i], .ekey = tsArray[i]}, .groupId = (uint64_t)(0)}; + key.win.skey = tsArray[i]; + key.win.ekey = tsArray[i]; + key.groupId = (uint64_t)(0); + + const char *val = "Value"; + int32_t len = strlen(val); + + char *pval = NULL; + ASSERT(0 == streamStateSessionGet_rocksdb(p, &key, (void **)&pval, &len)); + ASSERT(strncmp(pval, val, len) == 0); + taosMemoryFreeClear(pval); + } + + pCurr = streamStateSessionSeekToLast_rocksdb(p, 0); + ASSERT(pCurr != NULL); + + { + SSessionKey key; + memset(&key, 0, sizeof(key)); + char *val = NULL; + int32_t vlen = 0; + code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen); + ASSERT(code == 0); + pCurr = streamStateSessionSeekKeyPrev_rocksdb(p, &key); + + code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen); + ASSERT(code == 0); + + ASSERT(key.groupId == 0 && key.win.ekey == tsArray[tsArray.size() - 2]); + + pCurr = streamStateSessionSeekKeyNext_rocksdb(p, &key); + code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen); + ASSERT(code == 0); + ASSERT(vlen == strlen("Value")); + ASSERT(key.groupId == 0 && key.win.skey == tsArray[tsArray.size() - 1]); + + ASSERT(0 == streamStateSessionAddIfNotExist_rocksdb(p, &key, 10, (void **)&val, &len)); + + ASSERT(0 == + streamStateStateAddIfNotExist_rocksdb(p, &key, (char *)"key", strlen("key"), NULL, (void **)&val, &len)); + } + for (int i = 0; i < size; i++) { + SSessionKey key = {0}; //{.win = {.skey = tsArray[i], .ekey = tsArray[i]}, .groupId = (uint64_t)(0)}; + key.win.skey = tsArray[i]; + key.win.ekey = tsArray[i]; + key.groupId = (uint64_t)(0); + + const char *val = "Value"; + int32_t len = strlen(val); + + char *pval = NULL; + ASSERT(0 == streamStateSessionDel_rocksdb(p, &key)); + } + + for (int i = 0; i < size; i++) { + SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]}; + key.groupId = (uint64_t)(i); + key.ts = tsArray[i]; + const char *val = "Value"; + int32_t vlen = strlen(val); + ASSERT(streamStateFillPut_rocksdb(p, &key, val, vlen) == 0); + } + for (int i = 0; i < size; i++) { + SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]}; + key.groupId = (uint64_t)(i); + key.ts = tsArray[i]; + char *val = NULL; + int32_t vlen = 0; + ASSERT(streamStateFillGet_rocksdb(p, &key, (void **)&val, &vlen) == 0); + taosMemoryFreeClear(val); + } + { + SWinKey key = {0}; //{.groupId = (uint64_t)(0), .ts = tsArray[0]}; + key.groupId = (uint64_t)(0); + key.ts = tsArray[0]; + SStreamStateCur *pCurr = streamStateFillGetCur_rocksdb(p, &key); + ASSERT(pCurr != NULL); + + char *val = NULL; + int32_t vlen = 0; + ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen)); + ASSERT(vlen == strlen("Value")); + streamStateFreeCur(pCurr); + + pCurr = streamStateFillSeekKeyNext_rocksdb(p, &key); + ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen)); + ASSERT(vlen == strlen("Value") && key.groupId == 1 && key.ts == tsArray[1]); + + key.groupId = 1; + key.ts = tsArray[1]; + + pCurr = streamStateFillSeekKeyPrev_rocksdb(p, &key); + ASSERT(pCurr != NULL); + ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen)); + + ASSERT(vlen == strlen("Value") && key.groupId == 0 && key.ts == tsArray[0]); + } + + for (int i = 0; i < size - 1; i++) { + SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]}; + key.groupId = (uint64_t)(i); + key.ts = tsArray[i]; + char *val = NULL; + int32_t vlen = 0; + ASSERT(streamStateFillDel_rocksdb(p, &key) == 0); + taosMemoryFreeClear(val); + } + streamStateSessionClear_rocksdb(p); + + for (int i = 0; i < size; i++) { + char tbname[TSDB_TABLE_NAME_LEN] = {0}; + sprintf(tbname, "%s_%d", "tbname", i); + ASSERT(0 == streamStatePutParName_rocksdb(p, i, tbname)); + } + for (int i = 0; i < size; i++) { + char *val = NULL; + ASSERT(0 == streamStateGetParName_rocksdb(p, i, (void **)&val)); + ASSERT(strncmp(val, "tbname", strlen("tbname")) == 0); + taosMemoryFree(val); + } + + for (int i = 0; i < size; i++) { + char tbname[TSDB_TABLE_NAME_LEN] = {0}; + sprintf(tbname, "%s_%d", "tbname", i); + ASSERT(0 == streamStatePutParName_rocksdb(p, i, tbname)); + } + for (int i = 0; i < size; i++) { + char *val = NULL; + ASSERT(0 == streamStateGetParName_rocksdb(p, i, (void **)&val)); + ASSERT(strncmp(val, "tbname", strlen("tbname")) == 0); + taosMemoryFree(val); + } + for (int i = 0; i < size; i++) { + char key[128] = {0}; + sprintf(key, "tbname_%d", i); + char val[128] = {0}; + sprintf(val, "val_%d", i); + code = streamDefaultPut_rocksdb(p, key, val, strlen(val)); + ASSERT(code == 0); + } + for (int i = 0; i < size; i++) { + char key[128] = {0}; + sprintf(key, "tbname_%d", i); + + char *val = NULL; + int32_t len = 0; + code = streamDefaultGet_rocksdb(p, key, (void **)&val, &len); + ASSERT(code == 0); + } + SArray *result = taosArrayInit(8, sizeof(void *)); + streamDefaultIterGet_rocksdb(p, "tbname", "tbname_99", result); + ASSERT(taosArrayGetSize(result) >= 0); + + return p; + // streamStateClose((SStreamState *)p, true); +} +TEST_F(BackendEnv, checkOpen) { + SStreamState *p = (SStreamState *)backendOpen(); + int64_t tsStart = taosGetTimestampMs(); + { + void *pBatch = streamStateCreateBatch(); + int32_t size = 0; + for (int i = 0; i < size; i++) { + char key[128] = {0}; + sprintf(key, "key_%d", i); + char val[128] = {0}; + sprintf(val, "val_%d", i); + streamStatePutBatch(p, "default", (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val, + (int32_t)(strlen(val)), tsStart + 100000); + } + streamStatePutBatch_rocksdb(p, pBatch); + streamStateDestroyBatch(pBatch); + } + { + void *pBatch = streamStateCreateBatch(); + int32_t size = 0; + char valBuf[256] = {0}; + for (int i = 0; i < size; i++) { + char key[128] = {0}; + sprintf(key, "key_%d", i); + char val[128] = {0}; + sprintf(val, "val_%d", i); + streamStatePutBatchOptimize(p, 0, (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val, + (int32_t)(strlen(val)), tsStart + 100000, (void *)valBuf); + } + streamStatePutBatch_rocksdb(p, pBatch); + streamStateDestroyBatch(pBatch); + } + // do checkpoint 2 + taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 2); + { + void *pBatch = streamStateCreateBatch(); + int32_t size = 0; + char valBuf[256] = {0}; + for (int i = 0; i < size; i++) { + char key[128] = {0}; + sprintf(key, "key_%d", i); + char val[128] = {0}; + sprintf(val, "val_%d", i); + streamStatePutBatchOptimize(p, 0, (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val, + (int32_t)(strlen(val)), tsStart + 100000, (void *)valBuf); + } + streamStatePutBatch_rocksdb(p, pBatch); + streamStateDestroyBatch(pBatch); + } + + taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 3); + + const char *path = "/tmp/backend/stream"; + const char *dump = "/tmp/backend/stream/dump"; + // taosMkDir(dump); + taosMulMkDir(dump); + SBkdMgt *mgt = bkdMgtCreate((char *)path); + SArray *result = taosArrayInit(4, sizeof(void *)); + bkdMgtGetDelta(mgt, p->pTdbState->idstr, 3, result, (char *)dump); + + bkdMgtDestroy(mgt); + streamStateClose((SStreamState *)p, true); + taosRemoveDir(path); +} + +TEST_F(BackendEnv, backendChkp) { const char *path = "/tmp"; } + +typedef struct BdKV { + uint32_t k; + uint32_t v; +} BdKV; + +BdKV kvDict[] = {{0, 2}, {1, 2}, {15, 16}, {31, 32}, {56, 64}, {100, 128}, + {200, 256}, {500, 512}, {1000, 1024}, {2000, 2048}, {3000, 4096}}; + +TEST_F(BackendEnv, backendUtil) { + for (int i = 0; i < sizeof(kvDict) / sizeof(kvDict[0]); i++) { + ASSERT_EQ(nextPow2((uint32_t)(kvDict[i].k)), kvDict[i].v); + } +} + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/source/libs/stream/test/checkpointTest.cpp b/source/libs/stream/test/checkpointTest.cpp index 0dc2cc13f5..0caad479e5 100644 --- a/source/libs/stream/test/checkpointTest.cpp +++ b/source/libs/stream/test/checkpointTest.cpp @@ -25,46 +25,49 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" +// tsSnodeAddress = ""; +// tsS3StreamEnabled = 0; + +#include "cos.h" #include "rsync.h" #include "streamInt.h" -#include "cos.h" -int main(int argc, char **argv) { - testing::InitGoogleTest(&argc, argv); +// int main(int argc, char **argv) { +// testing::InitGoogleTest(&argc, argv); - if (taosInitCfg("/etc/taos/", NULL, NULL, NULL, NULL, 0) != 0) { - printf("error"); - } - if (s3Init() < 0) { - return -1; - } - strcpy(tsSnodeAddress, "127.0.0.1"); - int ret = RUN_ALL_TESTS(); - s3CleanUp(); - return ret; -} +// if (taosInitCfg("/etc/taos/", NULL, NULL, NULL, NULL, 0) != 0) { +// printf("error"); +// } +// if (s3Init() < 0) { +// return -1; +// } +// strcpy(tsSnodeAddress, "127.0.0.1"); +// int ret = RUN_ALL_TESTS(); +// s3CleanUp(); +// return ret; +// } TEST(testCase, checkpointUpload_Test) { - stopRsync(); - startRsync(); + // stopRsync(); + // startRsync(); taosSsleep(5); char* id = "2013892036"; - uploadCheckpoint(id, "/root/offset/"); + // uploadCheckpoint(id, "/root/offset/"); } TEST(testCase, checkpointDownload_Test) { char* id = "2013892036"; - downloadCheckpoint(id, "/root/offset/download/"); + // downloadCheckpoint(id, "/root/offset/download/"); } TEST(testCase, checkpointDelete_Test) { char* id = "2013892036"; - deleteCheckpoint(id); + // deleteCheckpoint(id); } TEST(testCase, checkpointDeleteFile_Test) { char* id = "2013892036"; - deleteCheckpointFile(id, "offset-ver0"); + // deleteCheckpointFile(id, "offset-ver0"); } diff --git a/source/libs/stream/test/tstreamUpdateTest.cpp b/source/libs/stream/test/tstreamUpdateTest.cpp index 1b999e5fb0..59171876ff 100644 --- a/source/libs/stream/test/tstreamUpdateTest.cpp +++ b/source/libs/stream/test/tstreamUpdateTest.cpp @@ -14,10 +14,7 @@ class StreamStateEnv : public ::testing::Test { streamMetaInit(); backend = streamBackendInit(path, 0, 0); } - virtual void TearDown() { - streamMetaCleanup(); - // indexClose(index); - } + virtual void TearDown() { streamMetaCleanup(); } const char *path = TD_TMP_DIR_PATH "stream"; void *backend; @@ -50,6 +47,14 @@ bool equalSBF(SScalableBf *left, SScalableBf *right) { } TEST(TD_STREAM_UPDATE_TEST, update) { + const char *streamPath = "/tmp"; + + char *absPath = NULL; + void *p = NULL; + // SBackendWrapper *p = streamBackendInit(streamPath, -1, 2); + // p = taskDbOpen((char *)streamPath, (char *)"test", -1); + p = bkdMgtCreate((char *)streamPath); + // const int64_t interval = 20 * 1000; // const int64_t watermark = 10 * 60 * 1000; // SUpdateInfo *pSU = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index e06ea70f70..7ff6116137 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1343,7 +1343,7 @@ ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->raftCfg int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { int32_t ret = 0; if (syncIsInit()) { - taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, pSyncNode, syncEnv()->pTimerManager, + taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, (void*)pSyncNode->rid, syncEnv()->pTimerManager, &pSyncNode->pPingTimer); atomic_store_64(&pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); } else { @@ -1415,8 +1415,8 @@ void syncNodeResetElectTimer(SSyncNode* pSyncNode) { static int32_t syncNodeDoStartHeartbeatTimer(SSyncNode* pSyncNode) { int32_t ret = 0; if (syncIsInit()) { - taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, pSyncNode, syncEnv()->pTimerManager, - &pSyncNode->pHeartbeatTimer); + taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, (void*)pSyncNode->rid, + syncEnv()->pTimerManager, &pSyncNode->pHeartbeatTimer); atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); } else { sError("vgId:%d, start heartbeat timer error, sync env is stop", pSyncNode->vgId); @@ -2153,7 +2153,11 @@ int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex static void syncNodeEqPingTimer(void* param, void* tmrId) { if (!syncIsInit()) return; - SSyncNode* pNode = param; + int64_t rid = (int64_t)param; + SSyncNode* pNode = syncNodeAcquire(rid); + + if (pNode == NULL) return; + if (atomic_load_64(&pNode->pingTimerLogicClockUser) <= atomic_load_64(&pNode->pingTimerLogicClock)) { SRpcMsg rpcMsg = {0}; int32_t code = syncBuildTimeout(&rpcMsg, SYNC_TIMEOUT_PING, atomic_load_64(&pNode->pingTimerLogicClock), @@ -2173,7 +2177,8 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) { } _out: - taosTmrReset(syncNodeEqPingTimer, pNode->pingTimerMS, pNode, syncEnv()->pTimerManager, &pNode->pPingTimer); + taosTmrReset(syncNodeEqPingTimer, pNode->pingTimerMS, (void*)pNode->rid, syncEnv()->pTimerManager, + &pNode->pPingTimer); } } @@ -2224,7 +2229,11 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { if (!syncIsInit()) return; - SSyncNode* pNode = param; + int64_t rid = (int64_t)param; + SSyncNode* pNode = syncNodeAcquire(rid); + + if (pNode == NULL) return; + if (pNode->totalReplicaNum > 1) { if (atomic_load_64(&pNode->heartbeatTimerLogicClockUser) <= atomic_load_64(&pNode->heartbeatTimerLogicClock)) { SRpcMsg rpcMsg = {0}; @@ -2245,7 +2254,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { } _out: - taosTmrReset(syncNodeEqHeartbeatTimer, pNode->heartbeatTimerMS, pNode, syncEnv()->pTimerManager, + taosTmrReset(syncNodeEqHeartbeatTimer, pNode->heartbeatTimerMS, (void*)pNode->rid, syncEnv()->pTimerManager, &pNode->pHeartbeatTimer); } else { @@ -3385,4 +3394,4 @@ bool syncNodeCanChange(SSyncNode* pSyncNode) { return true; } -#endif \ No newline at end of file +#endif diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 3ee65f11dd..455128e6ec 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -316,7 +316,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) } // 3. Try to Recycle a page - if (!pPage && !pCache->lru.pLruPrev->isAnchor) { + if (!pPageH && !pPage && !pCache->lru.pLruPrev->isAnchor) { pPage = pCache->lru.pLruPrev; tdbPCacheRemovePageFromHash(pCache, pPage); tdbPCachePinPage(pCache, pPage); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 7483588593..c010e31320 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -256,21 +256,21 @@ void transAsyncPoolDestroy(SAsyncPool* pool); int transAsyncSend(SAsyncPool* pool, queue* mq); bool transAsyncPoolIsEmpty(SAsyncPool* pool); -#define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc) \ - do { \ - for (int i = 0; i < pool->nAsync; i++) { \ - uv_async_t* async = &(pool->asyncs[i]); \ - SAsyncItem* item = async->data; \ - while (!QUEUE_IS_EMPTY(&item->qmsg)) { \ - tTrace("destroy msg in async pool "); \ - queue* h = QUEUE_HEAD(&item->qmsg); \ - QUEUE_REMOVE(h); \ - msgType* msg = QUEUE_DATA(h, msgType, q); \ - if (msg != NULL) { \ - freeFunc(msg); \ - } \ - } \ - } \ +#define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc, param) \ + do { \ + for (int i = 0; i < pool->nAsync; i++) { \ + uv_async_t* async = &(pool->asyncs[i]); \ + SAsyncItem* item = async->data; \ + while (!QUEUE_IS_EMPTY(&item->qmsg)) { \ + tTrace("destroy msg in async pool "); \ + queue* h = QUEUE_HEAD(&item->qmsg); \ + QUEUE_REMOVE(h); \ + msgType* msg = QUEUE_DATA(h, msgType, q); \ + if (msg != NULL) { \ + freeFunc(msg, param); \ + } \ + } \ + } \ } while (0) #define ASYNC_CHECK_HANDLE(exh1, id) \ @@ -427,6 +427,8 @@ SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, void transDQCancel(SDelayQueue* queue, SDelayTask* task); bool transEpSetIsEqual(SEpSet* a, SEpSet* b); + +bool transEpSetIsEqual2(SEpSet* a, SEpSet* b); /* * init global func */ diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 6de10cbb9e..c4ca39c323 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -191,6 +191,15 @@ static void httpDestroyMsg(SHttpMsg* msg) { taosMemoryFree(msg->cont); taosMemoryFree(msg); } +static void httpDestroyMsgWrapper(void* cont, void* param) { + httpDestroyMsg((SHttpMsg*)cont); + // if (msg == NULL) return; + + // taosMemoryFree(msg->server); + // taosMemoryFree(msg->uri); + // taosMemoryFree(msg->cont); + // taosMemoryFree(msg); +} static void httpMayDiscardMsg(SHttpModule* http, SAsyncItem* item) { SHttpMsg *msg = NULL, *quitMsg = NULL; @@ -554,7 +563,7 @@ void transHttpEnvDestroy() { httpSendQuit(); taosThreadJoin(load->thread, NULL); - TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsg); + TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsgWrapper, NULL); transAsyncPoolDestroy(load->asyncPool); uv_loop_close(load->loop); taosMemoryFree(load->loop); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index e2b69dd145..6ae72eac14 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -219,6 +219,8 @@ static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, /// NULL,cliHandleUpdate}; static FORCE_INLINE void destroyCmsg(void* cmsg); + +static FORCE_INLINE void destroyCmsgWrapper(void* arg, void* param); static FORCE_INLINE void destroyCmsgAndAhandle(void* cmsg); static FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst); static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx); @@ -961,7 +963,7 @@ static void cliSendCb(uv_write_t* req, int status) { tTrace("%s conn %p send cost:%dus ", CONN_GET_INST_LABEL(pConn), pConn, (int)cost); } } - if (pMsg->msg.contLen == 0 && pMsg->msg.pCont != 0) { + if (pMsg != NULL && pMsg->msg.contLen == 0 && pMsg->msg.pCont != 0) { rpcFreeCont(pMsg->msg.pCont); pMsg->msg.pCont = 0; } @@ -1963,7 +1965,17 @@ static FORCE_INLINE void destroyCmsg(void* arg) { transFreeMsg(pMsg->msg.pCont); taosMemoryFree(pMsg); } - +static FORCE_INLINE void destroyCmsgWrapper(void* arg, void* param) { + SCliMsg* pMsg = arg; + if (pMsg == NULL) { + return; + } + if (param != NULL) { + SCliThrd* pThrd = param; + if (pThrd->destroyAhandleFp) (*pThrd->destroyAhandleFp)(pMsg->msg.info.ahandle); + } + destroyCmsg(pMsg); +} static FORCE_INLINE void destroyCmsgAndAhandle(void* param) { if (param == NULL) return; @@ -2057,7 +2069,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { taosThreadJoin(pThrd->thread, NULL); CLI_RELEASE_UV(pThrd->loop); taosThreadMutexDestroy(&pThrd->msgMtx); - TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsg); + TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsgWrapper, (void*)pThrd); transAsyncPoolDestroy(pThrd->asyncPool); transDQDestroy(pThrd->delayQueue, destroyCmsgAndAhandle); @@ -2224,7 +2236,7 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { EPSET_FORWARD_INUSE(&pCtx->epSet); } } else { - if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { + if (!transEpSetIsEqual2(&pCtx->epSet, &epSet)) { tDebug("epset not equal, retry new epset1"); transPrintEpSet(&pCtx->epSet); transPrintEpSet(&epSet); @@ -2251,7 +2263,7 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { EPSET_FORWARD_INUSE(&pCtx->epSet); } } else { - if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { + if (!transEpSetIsEqual2(&pCtx->epSet, &epSet)) { tDebug("epset not equal, retry new epset2"); transPrintEpSet(&pCtx->epSet); transPrintEpSet(&epSet); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 4619722743..27099c76fc 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -70,7 +70,7 @@ int32_t transDecompressMsg(char** msg, int32_t len) { char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead)); STransMsgHead* pNewHead = (STransMsgHead*)buf; int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), (char*)pNewHead->content, - len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen); + len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen); memcpy((char*)pNewHead, (char*)pHead, sizeof(STransMsgHead)); pNewHead->msgLen = htonl(oriLen + sizeof(STransMsgHead)); @@ -603,6 +603,17 @@ bool transEpSetIsEqual(SEpSet* a, SEpSet* b) { } return true; } +bool transEpSetIsEqual2(SEpSet* a, SEpSet* b) { + if (a->numOfEps != b->numOfEps) { + return false; + } + for (int i = 0; i < a->numOfEps; i++) { + if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { + return false; + } + } + return true; +} static void transInitEnv() { refMgt = transOpenRefMgt(50000, transDestroyExHandle); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index b324ca5f91..5a1ef31b7d 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -159,7 +159,7 @@ static void uvStartSendResp(SSvrMsg* msg); static void uvNotifyLinkBrokenToApp(SSvrConn* conn); -static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); +static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); static FORCE_INLINE SSvrConn* createConn(void* hThrd); static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/); static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn); @@ -671,7 +671,8 @@ static FORCE_INLINE void destroySmsg(SSvrMsg* smsg) { transFreeMsg(smsg->msg.pCont); taosMemoryFree(smsg); } -static void destroyAllConn(SWorkThrd* pThrd) { +static FORCE_INLINE void destroySmsgWrapper(void* smsg, void* param) { destroySmsg((SSvrMsg*)smsg); } +static void destroyAllConn(SWorkThrd* pThrd) { tTrace("thread %p destroy all conn ", pThrd); while (!QUEUE_IS_EMPTY(&pThrd->conn)) { queue* h = QUEUE_HEAD(&pThrd->conn); @@ -1394,7 +1395,7 @@ void destroyWorkThrd(SWorkThrd* pThrd) { } taosThreadJoin(pThrd->thread, NULL); SRV_RELEASE_UV(pThrd->loop); - TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsg); + TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsgWrapper, NULL); transAsyncPoolDestroy(pThrd->asyncPool); uvWhiteListDestroy(pThrd->pWhiteList); diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index a54c301de9..c891ee4579 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -16,6 +16,12 @@ #define ALLOW_FORBID_FUNC #include "os.h" +typedef union { + volatile int64_t i; + volatile double d; + //double d; +} double_number; + #ifdef WINDOWS // add @@ -339,6 +345,21 @@ void atomic_store_64(int64_t volatile* ptr, int64_t val) { #endif } +double atomic_store_double(double volatile *ptr, double val){ + for (;;) { + double_number old_num = {0}; + old_num.d = *ptr; // current old value + + double_number new_num = {0}; + new_num.d = val; + + double_number ret_num = {0}; + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + + if (ret_num.i == old_num.i) return ret_num.d; + } +} + void atomic_store_ptr(void* ptr, void* val) { #ifdef WINDOWS ((*(void* volatile*)(ptr)) = (void*)(val)); @@ -393,6 +414,23 @@ int64_t atomic_exchange_64(int64_t volatile* ptr, int64_t val) { #endif } +double atomic_exchange_double(double volatile *ptr, int64_t val){ + for (;;) { + double_number old_num = {0}; + old_num.d = *ptr; // current old value + + double_number new_num = {0}; + int64_t iNew = val; + + double_number ret_num = {0}; + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + + if (ret_num.i == old_num.i) { + return ret_num.d; + } + } +} + void* atomic_exchange_ptr(void* ptr, void* val) { #ifdef WINDOWS #ifdef _WIN64 @@ -551,6 +589,21 @@ int64_t atomic_fetch_add_64(int64_t volatile* ptr, int64_t val) { #endif } +double atomic_fetch_add_double(double volatile *ptr, double val){ + for (;;) { + double_number old_num = {0}; + old_num.d = *ptr; // current old value + + double_number new_num = {0}; + new_num.d = old_num.d + val; + + double_number ret_num = {0}; + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + + if (ret_num.i == old_num.i) return ret_num.d; + } +} + void* atomic_fetch_add_ptr(void* ptr, void* val) { #ifdef WINDOWS return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)); @@ -657,6 +710,21 @@ int64_t atomic_fetch_sub_64(int64_t volatile* ptr, int64_t val) { #endif } +double atomic_fetch_sub_double(double volatile *ptr, double val){ + for (;;) { + double_number old_num = {0}; + old_num.d = *ptr; // current old value + + double_number new_num = {0}; + new_num.d = old_num.d - val; + + double_number ret_num = {0}; + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + + if (ret_num.i == old_num.i) return ret_num.d; + } +} + void* atomic_fetch_sub_ptr(void* ptr, void* val) { #ifdef WINDOWS return interlocked_fetch_sub_ptr(ptr, val); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index bab9ba0cea..e6491639dc 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -1329,7 +1329,6 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { char *data = taosMemoryMalloc(compressSize); gzFile dstFp = NULL; - TdFilePtr pFile = NULL; TdFilePtr pSrcFile = NULL; pSrcFile = taosOpenFile(srcFileName, TD_FILE_READ | TD_FILE_STREAM); @@ -1369,8 +1368,8 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { } cmp_end: - if (pFile) { - taosCloseFile(&pFile); + if (fd >= 0) { + close(fd); } if (pSrcFile) { taosCloseFile(&pSrcFile); diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index fea7a4f63d..187461826a 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -563,9 +563,9 @@ int32_t taosGetCpuCores(float *numOfCores, bool physical) { } void taosGetCpuUsage(double *cpu_system, double *cpu_engine) { - static int64_t lastSysUsed = 0; - static int64_t lastSysTotal = 0; - static int64_t lastProcTotal = 0; + static int64_t lastSysUsed = -1; + static int64_t lastSysTotal = -1; + static int64_t lastProcTotal = -1; static int64_t curSysUsed = 0; static int64_t curSysTotal = 0; static int64_t curProcTotal = 0; @@ -580,12 +580,14 @@ void taosGetCpuUsage(double *cpu_system, double *cpu_engine) { curSysTotal = curSysUsed + sysCpu.idle; curProcTotal = procCpu.utime + procCpu.stime + procCpu.cutime + procCpu.cstime; - if (curSysTotal - lastSysTotal > 0 && curSysUsed >= lastSysUsed && curProcTotal >= lastProcTotal) { - if (cpu_system != NULL) { - *cpu_system = (curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100; - } - if (cpu_engine != NULL) { - *cpu_engine = (curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100; + if(lastSysUsed >= 0 && lastSysTotal >=0 && lastProcTotal >=0){ + if (curSysTotal - lastSysTotal > 0 && curSysUsed >= lastSysUsed && curProcTotal >= lastProcTotal) { + if (cpu_system != NULL) { + *cpu_system = (curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100; + } + if (cpu_engine != NULL) { + *cpu_engine = (curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100; + } } } @@ -821,19 +823,27 @@ int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int } void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { - static int64_t last_rchars = 0; - static int64_t last_wchars = 0; - static int64_t last_read_bytes = 0; - static int64_t last_write_bytes = 0; + static int64_t last_rchars = -1; + static int64_t last_wchars = -1; + static int64_t last_read_bytes = -1; + static int64_t last_write_bytes = -1; static int64_t cur_rchars = 0; static int64_t cur_wchars = 0; static int64_t cur_read_bytes = 0; static int64_t cur_write_bytes = 0; if (taosGetProcIO(&cur_rchars, &cur_wchars, &cur_read_bytes, &cur_write_bytes) == 0) { - *rchars = cur_rchars - last_rchars; - *wchars = cur_wchars - last_wchars; - *read_bytes = cur_read_bytes - last_read_bytes; - *write_bytes = cur_write_bytes - last_write_bytes; + if(last_rchars >=0 && last_wchars >=0 && last_read_bytes >=0 && last_write_bytes >= 0){ + *rchars = cur_rchars - last_rchars; + *wchars = cur_wchars - last_wchars; + *read_bytes = cur_read_bytes - last_read_bytes; + *write_bytes = cur_write_bytes - last_write_bytes; + } + else{ + *rchars = 0; + *wchars = 0; + *read_bytes = 0; + *write_bytes = 0; + } last_rchars = cur_rchars; last_wchars = cur_wchars; last_read_bytes = cur_read_bytes; @@ -900,13 +910,20 @@ int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { } void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) { - static int64_t last_receive_bytes = 0; - static int64_t last_transmit_bytes = 0; + static int64_t last_receive_bytes = -1; + static int64_t last_transmit_bytes = -1; int64_t cur_receive_bytes = 0; int64_t cur_transmit_bytes = 0; if (taosGetCardInfo(&cur_receive_bytes, &cur_transmit_bytes) == 0) { - *receive_bytes = cur_receive_bytes - last_receive_bytes; - *transmit_bytes = cur_transmit_bytes - last_transmit_bytes; + if(last_receive_bytes >= 0 && last_transmit_bytes >= 0){ + *receive_bytes = cur_receive_bytes - last_receive_bytes; + *transmit_bytes = cur_transmit_bytes - last_transmit_bytes; + } + else{ + *receive_bytes = 0; + *transmit_bytes = 0; + } + last_receive_bytes = cur_receive_bytes; last_transmit_bytes = cur_transmit_bytes; } else { diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 88eb51d500..d9686d77f8 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -89,12 +89,14 @@ static int32_t taosArrayResize(SArray* pArray) { int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { if (newCap > pArray->capacity) { float factor = BOUNDARY_BIG_FACTOR; - if(newCap * pArray->elemSize > BOUNDARY_SIZE){ + if (newCap * pArray->elemSize > BOUNDARY_SIZE) { factor = BOUNDARY_SMALL_FACTOR; } + size_t tsize = (pArray->capacity * factor); while (newCap > tsize) { - tsize = (tsize * factor); + size_t newSize = (tsize * factor); + tsize = (newSize == tsize) ? (tsize + 2) : newSize; } pArray->pData = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize); diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c index fa3ecadd14..5eb72879c3 100644 --- a/source/util/src/tbase58.c +++ b/source/util/src/tbase58.c @@ -18,24 +18,29 @@ #include #include -#define BASE_BUF_SIZE 256 +#define TBASE_BUF_SIZE 256 static const char *basis_58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; char *base58_encode(const uint8_t *value, int32_t vlen) { const uint8_t *pb = value; const uint8_t *pe = pb + vlen; - uint8_t buf[BASE_BUF_SIZE] = {0}; + uint8_t buf[TBASE_BUF_SIZE] = {0}; uint8_t *pbuf = &buf[0]; bool bfree = false; int32_t nz = 0, size = 0, len = 0; + if (vlen > TBASE_MAX_ILEN) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + while (pb != pe && *pb == 0) { ++pb; ++nz; } size = (pe - pb) * 69 / 50 + 1; - if (size > BASE_BUF_SIZE) { + if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -47,7 +52,7 @@ char *base58_encode(const uint8_t *value, int32_t vlen) { int32_t num = *pb; int32_t i = 0; for (int32_t j = (int32_t)size - 1; (num != 0 || i < len) && j >= 0; --j, ++i) { - num += ((int32_t)buf[j]) << 8; + num += ((int32_t)pbuf[j]) << 8; pbuf[j] = num % 58; num /= 58; } @@ -57,7 +62,7 @@ char *base58_encode(const uint8_t *value, int32_t vlen) { const uint8_t *pi = pbuf + (size - len); while (pi != pbuf + size && *pi == 0) ++pi; - uint8_t *result = taosMemoryCalloc(1, size + 1); + uint8_t *result = taosMemoryCalloc(1, nz + (pbuf + size - pi) + 1); if (!result) { terrno = TSDB_CODE_OUT_OF_MEMORY; if (bfree) taosMemoryFree(pbuf); @@ -82,20 +87,35 @@ static const signed char index_58[256] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { + const char *pb = value; const char *pe = value + inlen; - uint8_t buf[BASE_BUF_SIZE] = {0}; + uint8_t buf[TBASE_BUF_SIZE] = {0}; uint8_t *pbuf = &buf[0]; bool bfree = false; int32_t nz = 0, size = 0, len = 0; - while (*value && isspace(*value)) ++value; - while (*value == '1') { - ++nz; - ++value; + if (inlen > TBASE_MAX_OLEN) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; } - size = (int32_t)(pe - value) * 733 / 1000 + 1; - if (size > BASE_BUF_SIZE) { + while (pb != pe) { + if (*pb == 0) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + ++pb; + } + + pb = value; + while (pb != pe && *pb && isspace(*pb)) ++pb; + while (pb != pe && *pb == '1') { + ++nz; + ++pb; + } + + size = (int32_t)(pe - pb) * 733 / 1000 + 1; + if (size > TBASE_BUF_SIZE) { if (!(pbuf = taosMemoryCalloc(1, size))) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -103,9 +123,10 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { bfree = true; } - while (*value && !isspace(*value)) { - int32_t num = index_58[(uint8_t)*value]; + while (pb != pe && *pb && !isspace(*pb)) { + int32_t num = index_58[(uint8_t)*pb]; if (num == -1) { + terrno = TSDB_CODE_INVALID_PARA; if (bfree) taosMemoryFree(pbuf); return NULL; } @@ -116,18 +137,18 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) { num >>= 8; } len = i; - ++value; + ++pb; } - while (isspace(*value)) ++value; - if (*value != 0) { + while (pb != pe && isspace(*pb)) ++pb; + if (*pb != 0) { if (bfree) taosMemoryFree(pbuf); return NULL; } const uint8_t *it = pbuf + (size - len); while (it != pbuf + size && *it == 0) ++it; - uint8_t *result = taosMemoryCalloc(1, size + 1); + uint8_t *result = taosMemoryCalloc(1, nz + (pbuf + size - it) + 1); if (!result) { if (bfree) taosMemoryFree(pbuf); terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 8c73604727..1f3aaa3835 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -329,6 +329,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_VIEW_NOT_EXIST, "view not exists in db //mnode-compact TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COMPACT_ID, "Invalid compact id") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST, "compact detail doesn't exist") // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_OFFLINE, "Dnode is offline") @@ -619,6 +620,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Invalid stream quer TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VIEW_QUERY, "Invalid view query type") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_COL_QUERY_MISMATCH, "Columns number mismatch with query result") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE, "View name is conflict with table") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT, "Operator not supported multi result") //planner TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "Planner internal error") diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 0b7ad330a7..82993e8449 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -375,4 +375,8 @@ bool tjsonValidateJson(const char* jIn) { return true; } -const char* tjsonGetError() { return cJSON_GetErrorPtr(); } \ No newline at end of file +const char* tjsonGetError() { return cJSON_GetErrorPtr(); } + +void tjsonDeleteItemFromObject(const SJson* pJson, const char* pName) { + cJSON_DeleteItemFromObject((cJSON*)pJson, pName); +} \ No newline at end of file diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 2cc13be6ba..7a4eb09b99 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -21,6 +21,40 @@ int64_t tsRpcQueueMemoryAllowed = 0; int64_t tsRpcQueueMemoryUsed = 0; +struct STaosQueue { + STaosQnode *head; + STaosQnode *tail; + STaosQueue *next; // for queue set + STaosQset *qset; // for queue set + void *ahandle; // for queue set + FItem itemFp; + FItems itemsFp; + TdThreadMutex mutex; + int64_t memOfItems; + int32_t numOfItems; + int64_t threadId; + int64_t memLimit; + int64_t itemLimit; +}; + +struct STaosQset { + STaosQueue *head; + STaosQueue *current; + TdThreadMutex mutex; + tsem_t sem; + int32_t numOfQueues; + int32_t numOfItems; +}; + +struct STaosQall { + STaosQnode *current; + STaosQnode *start; + int32_t numOfItems; + int64_t memOfItems; + int32_t unAccessedNumOfItems; + int64_t unAccessMemOfItems; +}; + void taosSetQueueMemoryCapacity(STaosQueue *queue, int64_t cap) { queue->memLimit = cap; } void taosSetQueueCapacity(STaosQueue *queue, int64_t size) { queue->itemLimit = size; } @@ -497,6 +531,12 @@ int64_t taosQallUnAccessedMemSize(STaosQall *qall) { return qall->unAccessMemOfI void taosResetQitems(STaosQall *qall) { qall->current = qall->start; } int32_t taosGetQueueNumber(STaosQset *qset) { return qset->numOfQueues; } +void taosQueueSetThreadId(STaosQueue* pQueue, int64_t threadId) { + pQueue->threadId = threadId; +} + +int64_t taosQueueGetThreadId(STaosQueue *pQueue) { return pQueue->threadId; } + #if 0 void taosResetQsetThread(STaosQset *qset, void *pItem) { diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 3b4975b701..7af794546b 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -20,6 +20,9 @@ #define DEFAULT_GROWTH 2 #define DEFAULT_TIGHTENING_RATIO 0.5 +#define DEFAULT_MAX_BLOOMFILTERS 4 +#define SBF_INVALID -1 +#define SBF_VALID 0 static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate); @@ -32,6 +35,8 @@ SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) { if (pSBf == NULL) { return NULL; } + pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS; + pSBf->status = SBF_VALID; pSBf->numBits = 0; pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void *)); if (tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO) == NULL) { @@ -45,6 +50,9 @@ SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) { } int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + if (pSBf->status == SBF_INVALID) { + return TSDB_CODE_FAILED; + } int32_t size = taosArrayGetSize(pSBf->bfArray); SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); ASSERT(pNormalBf); @@ -52,6 +60,7 @@ int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t le pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); if (pNormalBf == NULL) { + pSBf->status = SBF_INVALID; return TSDB_CODE_OUT_OF_MEMORY; } } @@ -59,6 +68,9 @@ int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t le } int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + if (pSBf->status == SBF_INVALID) { + return TSDB_CODE_FAILED; + } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); int32_t size = taosArrayGetSize(pSBf->bfArray); @@ -74,6 +86,7 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); if (pNormalBf == NULL) { + pSBf->status = SBF_INVALID; return TSDB_CODE_OUT_OF_MEMORY; } } @@ -81,6 +94,9 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { } int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + if (pSBf->status == SBF_INVALID) { + return TSDB_CODE_FAILED; + } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); int32_t size = taosArrayGetSize(pSBf->bfArray); @@ -93,6 +109,10 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32 } static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate) { + if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) { + return NULL; + } + SBloomFilter *pNormalBf = tBloomFilterInit(expectedEntries, errorRate); if (pNormalBf == NULL) { return NULL; @@ -128,6 +148,8 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { } if (tEncodeU32(pEncoder, pSBf->growth) < 0) return -1; if (tEncodeU64(pEncoder, pSBf->numBits) < 0) return -1; + if (tEncodeU32(pEncoder, pSBf->maxBloomFilters) < 0) return -1; + if (tEncodeI8(pEncoder, pSBf->status) < 0) return -1; return 0; } @@ -150,6 +172,8 @@ SScalableBf *tScalableBfDecode(SDecoder *pDecoder) { } if (tDecodeU32(pDecoder, &pSBf->growth) < 0) goto _error; if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) goto _error; + if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) goto _error; + if (tDecodeI8(pDecoder, &pSBf->status) < 0) goto _error; return pSBf; _error: diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index eadd9a2413..f90509bec1 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -21,7 +21,7 @@ #include "tutil.h" static int32_t initForwardBackwardPtr(SSkipList *pSkipList); -static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur); +static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur); static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); static void tSkipListCorrectLevel(SSkipList *pSkipList); static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); @@ -131,12 +131,14 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { return pNode; } +#ifdef BUILD_NO_CALL + void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t iterate) { SSkipListNode *backward[MAX_SKIP_LIST_LEVEL] = {0}; SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; bool hasDup = false; - char *pKey = NULL; - char *pDataKey = NULL; + char * pKey = NULL; + char * pDataKey = NULL; int32_t compare = 0; tSkipListWLock(pSkipList); @@ -260,6 +262,7 @@ void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) { tSkipListCorrectLevel(pSkipList); tSkipListUnlock(pSkipList); } +#endif SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList) { if (pSkipList == NULL) return NULL; @@ -350,6 +353,7 @@ void *tSkipListDestroyIter(SSkipListIterator *iter) { return NULL; } +#ifdef BUILD_NO_CALL void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { if (pSkipList == NULL || pSkipList->level < nlevel || nlevel <= 0) { return; @@ -358,7 +362,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); int32_t id = 1; - char *prev = NULL; + char * prev = NULL; while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); @@ -392,6 +396,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { p = SL_NODE_GET_FORWARD_POINTER(p, nlevel - 1); } } +#endif static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, SSkipListNode *pNode, bool isForward) { for (int32_t i = 0; i < pNode->level; ++i) { @@ -460,7 +465,7 @@ static FORCE_INLINE int32_t tSkipListUnlock(SSkipList *pSkipList) { static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData) { int32_t compare = 0; bool hasDupKey = false; - char *pDataKey = pSkipList->keyFn(pData); + char * pDataKey = pSkipList->keyFn(pData); if (pSkipList->size == 0) { for (int32_t i = 0; i < pSkipList->maxLevel; i++) { @@ -516,6 +521,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, return hasDupKey; } +#ifdef BUILD_NO_CALL static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) { int32_t level = pNode->level; uint8_t dupMode = SL_DUP_MODE(pSkipList); @@ -540,6 +546,7 @@ static void tSkipListCorrectLevel(SSkipList *pSkipList) { pSkipList->level -= 1; } } +#endif UNUSED_FUNC static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, int32_t level) { // record link count in each level diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 3e591c7d7f..138d4bc1f4 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -417,9 +417,9 @@ _OVER: return NULL; } else { while (worker->pid <= 0) taosMsleep(10); - queue->threadId = worker->pid; - uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, - queue->threadId); + + taosQueueSetThreadId(queue, worker->pid); + uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, worker->pid); return queue; } } diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index f4f3880388..3514c578e9 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -100,3 +100,11 @@ add_test( NAME talgoTest COMMAND talgoTest ) + +# tbaseCodecTest +add_executable(tbaseCodecTest "tbaseCodecTest.cpp") +target_link_libraries(tbaseCodecTest os util common gtest_main) +add_test( + NAME tbaseCodecTest + COMMAND tbaseCodecTest +) diff --git a/source/util/test/tbaseCodecTest.cpp b/source/util/test/tbaseCodecTest.cpp new file mode 100644 index 0000000000..4c56979885 --- /dev/null +++ b/source/util/test/tbaseCodecTest.cpp @@ -0,0 +1,83 @@ +#include +#include + +#include +#include "os.h" +#include "osTime.h" +#include "taos.h" +#include "taoserror.h" +#include "tbase58.h" +#include "tglobal.h" + +using namespace std; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +static void checkBase58Codec(uint8_t *pRaw, int32_t rawLen, int32_t index) { + int64_t start = taosGetTimestampUs(); + char *pEnc = base58_encode((const uint8_t *)pRaw, rawLen); + ASSERT_NE(nullptr, pEnc); + + int32_t encLen = strlen(pEnc); + int64_t endOfEnc = taosGetTimestampUs(); + std::cout << "index:" << index << ", encLen is " << encLen << ", cost:" << endOfEnc - start << " us" << std::endl; + int32_t decLen = 0; + char *pDec = (char *)base58_decode((const char *)pEnc, encLen, &decLen); + std::cout << "index:" << index << ", decLen is " << decLen << ", cost:" << taosGetTimestampUs() - endOfEnc << " us" + << std::endl; + ASSERT_NE(nullptr, pDec); + ASSERT_EQ(rawLen, decLen); + ASSERT_LE(rawLen, encLen); + ASSERT_EQ(0, strncmp((char *)pRaw, pDec, rawLen)); + taosMemoryFreeClear(pDec); + taosMemoryFreeClear(pEnc); +} + +TEST(TD_BASE_CODEC_TEST, tbase58_test) { + const int32_t TEST_LEN_MAX = TBASE_MAX_ILEN; + const int32_t TEST_LEN_STEP = 10; + int32_t rawLen = 0; + uint8_t *pRaw = NULL; + + pRaw = (uint8_t *)taosMemoryCalloc(1, TEST_LEN_MAX); + ASSERT_NE(nullptr, pRaw); + + // 1. normal case + // string blend with char and '\0' + rawLen = TEST_LEN_MAX; + for (int32_t i = 0; i < TEST_LEN_MAX; i += 500) { + checkBase58Codec(pRaw, rawLen, i); + pRaw[i] = i & 127; + } + + // string without '\0' + for (int32_t i = 0; i < TEST_LEN_MAX; ++i) { + pRaw[i] = i & 127; + } + checkBase58Codec(pRaw, TEST_LEN_MAX, 0); + for (int32_t i = 0; i < TEST_LEN_MAX; i += 500) { + rawLen = i; + checkBase58Codec(pRaw, rawLen, i); + } + taosMemoryFreeClear(pRaw); + ASSERT_EQ(nullptr, pRaw); + + // 2. overflow case + char tmp[1]; + char *pEnc = base58_encode((const uint8_t *)tmp, TBASE_MAX_ILEN + 1); + ASSERT_EQ(nullptr, pEnc); + char *pDec = (char *)base58_decode((const char *)tmp, TBASE_MAX_OLEN + 1, NULL); + ASSERT_EQ(nullptr, pDec); + + taosMemoryFreeClear(pRaw); + ASSERT_EQ(nullptr, pRaw); +} \ No newline at end of file diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py index 6bcf547136..dfd8d95c9c 100644 --- a/tests/army/community/cluster/incSnapshot.py +++ b/tests/army/community/cluster/incSnapshot.py @@ -46,7 +46,9 @@ class TDTestCase(TBase): # clusterDnodes.starttaosd(1) # time.sleep(5) autoGen.insert_data(5000, True) - tdSql.execute(f"flush database {self.db}") + self.flushDb(True) + # wait flush operation over + time.sleep(5) # sql = 'show vnodes;' # while True: @@ -67,7 +69,6 @@ class TDTestCase(TBase): dirs = glob.glob(dnodesRootDir) for dir in dirs: if os.path.isdir(dir): - tdLog.debug("delete dir: %s " % (dnodesRootDir)) self.remove_directory(os.path.join(dir, "wal")) sc.dnodeStart(1) @@ -88,7 +89,7 @@ class TDTestCase(TBase): if bFinish: break - self.timestamp_step = 1 + self.timestamp_step = 1000 self.insert_rows = 6000 self.checkInsertCorrect() self.checkAggCorrect() diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index b21cbb1ad8..eef650cc77 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -25,6 +25,7 @@ from frame.cases import * from frame.sql import * from frame.caseBase import * from frame import * +from frame.srvCtl import * class TDTestCase(TBase): @@ -65,6 +66,21 @@ class TDTestCase(TBase): sql = f"select avg(dc) from {self.db}.{self.stb}" tdSql.checkFirstValue(sql, 200) + def alterReplica3(self): + sql = f"alter database {self.db} replica 3" + tdSql.execute(sql, show=True) + time.sleep(2) + sc.dnodeStop(2) + sc.dnodeStop(3) + time.sleep(5) + sc.dnodeStart(2) + sc.dnodeStart(3) + + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + def doAction(self): tdLog.info(f"do action.") self.flushDb() @@ -81,7 +97,7 @@ class TDTestCase(TBase): self.alterReplica(1) self.checkAggCorrect() self.compactDb() - self.alterReplica(3) + self.alterReplica3() vgids = self.getVGroup(self.db) selid = random.choice(vgids) diff --git a/tests/army/community/query/show.py b/tests/army/community/query/show.py new file mode 100644 index 0000000000..8b6844820e --- /dev/null +++ b/tests/army/community/query/show.py @@ -0,0 +1,152 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import random + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.autogen import * + + +class TDTestCase(TBase): + updatecfgDict = { + } + + def insertData(self): + tdLog.info(f"create table and insert data.") + self.stb = "stb" + self.db = "db" + self.childtable_count = 10 + self.insert_rows = 10000 + + self.autoGen = AutoGen(startTs = 1600000000000*1000*1000, batch=500, fillOne=True) + self.autoGen.create_db(self.db, 2, 3, "precision 'ns'") + self.autoGen.create_stable(stbname = self.stb, tag_cnt = 5, column_cnt = 20, binary_len = 10, nchar_len = 5) + self.autoGen.create_child(self.stb, "child", self.childtable_count) + self.autoGen.insert_data(self.insert_rows, True) + + tdLog.info("create view.") + tdSql.execute(f"use {self.db}") + sqls = [ + "create view viewc0c1 as select c0,c1 from stb ", + "create view viewc0c1c2 as select c0,c1,c2 from stb ", + "create view viewc0c3 as select c0,c3 from stb where c3=1", + "create view viewc0c4c5 as select c4,c5 from stb ", + "create view viewc0c6 as select c0,c1,c6 from stb ", + "create view viewc0c7 as select c0,c1 from stb ", + "create view viewc0c7c8 as select c0,c7,c8 from stb where c8>0", + "create view viewc0c3c1 as select c0,c3,c1 from stb ", + "create view viewc2c4 as select c2,c4 from stb ", + "create view viewc2c5 as select c2,c5 from stb ", + ] + tdSql.executes(sqls) + + def checkView(self): + tdLog.info(f"check view like.") + + # like + sql = f"show views like 'view%'" + tdSql.query(sql) + tdSql.checkRows(10) + + sql = f"show views like 'vie_c0c1c2'" + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0,0,"viewc0c1c2") + + sql = f"show views like '%c2c_'" + tdSql.query(sql) + tdSql.checkRows(2) + tdSql.checkData(0,0, "viewc2c4") + tdSql.checkData(1,0, "viewc2c5") + + sql = f"show views like '%' " + tdSql.query(sql) + tdSql.checkRows(10) + + # zero + sql = "show views like '_' " + tdSql.query(sql) + tdSql.checkRows(0) + sql = "show views like 'a%' " + tdSql.query(sql) + tdSql.checkRows(0) + + + def doQuery(self): + tdLog.info(f"do query.") + + # __group_key + sql = f"select count(*) from {self.stb} " + tdSql.query(sql) + # column index 1 value same with 2 + allRows = self.insert_rows * self.childtable_count + tdSql.checkFirstValue(sql, allRows) + + def checkShow(self): + # not support + sql = "show accounts;" + tdSql.error(sql) + + # check result + sql = "SHOW CLUSTER;" + tdSql.query(sql) + tdSql.checkRows(1) + sql = "SHOW COMPACTS;" + tdSql.query(sql) + tdSql.checkRows(0) + sql = "SHOW COMPACT 1;" + tdSql.query(sql) + tdSql.checkRows(0) + sql = "SHOW CLUSTER MACHINES;" + tdSql.query(sql) + tdSql.checkRows(1) + + # run to check crash + sqls = [ + "show scores;", + "SHOW CLUSTER VARIABLES", + "SHOW BNODES;", + ] + tdSql.executes(sqls) + + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check view + self.checkView() + + # do action + self.doQuery() + + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json index d7544a897c..747ac7c8ec 100644 --- a/tests/army/enterprise/s3/s3_basic.json +++ b/tests/army/enterprise/s3/s3_basic.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "connection_pool_size": 8, - "num_of_records_per_req": 2000, + "num_of_records_per_req": 4000, "prepared_rand": 1000, "thread_count": 2, "create_table_thread_count": 1, @@ -18,26 +18,27 @@ "drop": "yes", "vgroups": 2, "replica": 1, - "duration":"1d", - "keep": "3d,6d,30d" + "duration":"15d", + "flush_each_batch":"yes", + "keep": "60d,100d,200d" }, "super_tables": [ { "name": "stb", "child_table_exists": "no", - "childtable_count": 4, - "insert_rows": 1000000, + "childtable_count": 2, + "insert_rows": 2000000, "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"now-13d", + "start_timestamp":"now-90d", "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, { "type": "double", "name": "dc"}, - { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "tinyint", "name": "ti"}, { "type": "smallint", "name": "si" }, - { "type": "int", "name": "ic" }, + { "type": "int", "name": "ic" ,"max": 1,"min": 1}, { "type": "bigint", "name": "bi" }, { "type": "utinyint", "name": "uti"}, { "type": "usmallint", "name": "usi"}, diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index e7bc188ca5..e9173dda00 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -28,12 +28,12 @@ from frame import * from frame.eos import * # -# 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa +# 192.168.1.52 MINIO S3 # ''' s3EndPoint http://192.168.1.52:9000 -s3AccessKey MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa +s3AccessKey 'zOgllR6bSnw2Ah3mCNel:cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX' s3BucketName ci-bucket s3UploadDelaySec 60 ''' @@ -42,7 +42,7 @@ s3UploadDelaySec 60 class TDTestCase(TBase): updatecfgDict = { 's3EndPoint': 'http://192.168.1.52:9000', - 's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa', + 's3AccessKey': 'zOgllR6bSnw2Ah3mCNel:cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX', 's3BucketName': 'ci-bucket', 's3BlockSize': '10240', 's3BlockCacheSize': '320', @@ -58,8 +58,8 @@ class TDTestCase(TBase): tdSql.execute(f"use {self.db}") # come from s3_basic.json - self.childtable_count = 4 - self.insert_rows = 1000000 + self.childtable_count = 2 + self.insert_rows = 2000000 self.timestamp_step = 1000 def createStream(self, sname): @@ -78,14 +78,27 @@ class TDTestCase(TBase): self.trimDb(True) rootPath = sc.clusterRootPath() - cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data" + cmd = f"ls {rootPath}/dnode1/data2*/vnode/vnode*/tsdb/*.data" tdLog.info(cmd) loop = 0 - while len(eos.runRetList(cmd)) > 0 and loop < 40: - time.sleep(5) + rets = [] + while loop < 180: + time.sleep(3) + rets = eos.runRetList(cmd) + cnt = len(rets) + if cnt == 0: + tdLog.info("All data file upload to server over.") + break self.trimDb(True) + tdLog.info(f"loop={loop} no upload {cnt} data files wait 3s retry ...") + if loop == 0: + sc.dnodeStop(1) + time.sleep(2) + sc.dnodeStart(1) loop += 1 - tdLog.info(f"loop={loop} wait 5s...") + + if len(rets) > 0: + tdLog.exit(f"s3 can not upload all data to server. data files cnt={len(rets)} list={rets}") def checkStreamCorrect(self): sql = f"select count(*) from {self.db}.stm1" diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py index d1f02e7865..cf21977c75 100644 --- a/tests/army/frame/autogen.py +++ b/tests/army/frame/autogen.py @@ -14,15 +14,18 @@ import time # Auto Gen class # class AutoGen: - def __init__(self, fillOne=False): - self.ts = 1600000000000 - self.batch_size = 100 + def __init__(self, startTs = 1600000000000, step = 1000, batch = 100, fillOne=False): + self.startTs = startTs + self.ts = startTs + self.step = step + self.batch_size = batch + self.fillOne = fillOne seed = time.time() % 10000 random.seed(seed) - self.fillOne = fillOne # set start ts def set_start_ts(self, ts): + self.startTs = ts self.ts = ts # set batch size @@ -111,9 +114,9 @@ class AutoGen: return ''.join(random.choice(letters) for i in range(count)) # create db - def create_db(self, dbname, vgroups = 2, replica = 1): + def create_db(self, dbname, vgroups = 2, replica = 1, others=""): self.dbname = dbname - tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica}') + tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica} {others}') # create table or stable def create_stable(self, stbname, tag_cnt, column_cnt, binary_len, nchar_len): @@ -167,12 +170,12 @@ class AutoGen: def insert_data(self, cnt, bContinue=False): if not bContinue: - self.ts = 1600000000000 + self.ts = self.startTs - currTs = 1600000000000 + currTs = self.startTs for i in range(self.child_cnt): name = f"{self.child_name}{i}" - currTs = self.insert_data_child(name, cnt, self.batch_size, 1) + currTs = self.insert_data_child(name, cnt, self.batch_size, self.step) self.ts = currTs tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}") diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index 091856056b..3a9b0cdf4b 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -33,14 +33,14 @@ class srvCtl: # control server # - # start + # start idx base is 1 def dnodeStart(self, idx): if clusterDnodes.getModel() == 'cluster': return clusterDnodes.starttaosd(idx) return tdDnodes.starttaosd(idx) - # stop + # stop idx base is 1 def dnodeStop(self, idx): if clusterDnodes.getModel() == 'cluster': return clusterDnodes.stoptaosd(idx) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index cc15df33a1..c32790022a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -15,10 +15,11 @@ ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f community/query/function/test_func_elapsed.py ,,y,army,./pytest.sh python3 ./test.py -f community/query/fill/fill_desc.py -N 3 -L 3 -D 2 -,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py +,,n,army,python3 ./test.py -f community/query/show.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1 # @@ -49,6 +50,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/compact-col.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tms_memleak.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 3 @@ -236,7 +238,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db-removewal.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb-removewal.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 #,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3 @@ -1083,6 +1086,7 @@ ,,y,script,./test.sh -f tsim/parser/join_multivnode.sim ,,y,script,./test.sh -f tsim/parser/join.sim ,,y,script,./test.sh -f tsim/parser/last_cache.sim +,,y,script,./test.sh -f tsim/parser/last_both.sim ,,y,script,./test.sh -f tsim/parser/last_groupby.sim ,,y,script,./test.sh -f tsim/parser/lastrow.sim ,,y,script,./test.sh -f tsim/parser/lastrow2.sim @@ -1209,6 +1213,7 @@ ,,y,script,./test.sh -f tsim/stream/deleteState.sim ,,y,script,./test.sh -f tsim/stream/distributeInterval0.sim ,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim +,,y,script,./test.sh -f tsim/stream/distributeMultiLevelInterval0.sim ,,y,script,./test.sh -f tsim/stream/distributeSession0.sim ,,y,script,./test.sh -f tsim/stream/drop_stream.sim ,,y,script,./test.sh -f tsim/stream/event0.sim diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh index 01192763a1..9f526819c8 100644 --- a/tests/script/coverage_test.sh +++ b/tests/script/coverage_test.sh @@ -214,7 +214,7 @@ function lcovFunc { '*/clientJniConnector.c' '*/clientTmqConnector.c' '*/version.c' '*/build_version.cc'\ '*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' \ '*/shellAuto.c' '*/shellTire.c' '*/shellCommand.c'\ - '*/sql.c' '*/sql.y'\ + '*/sql.c' '*/sql.y' '*/smaSnapshot.c' '*/smaCommit.c'\ --branch-coverage --function-coverage -o coverage.info # generate result diff --git a/tests/script/tsim/parser/last_both.sim b/tests/script/tsim/parser/last_both.sim new file mode 100644 index 0000000000..e01a966744 --- /dev/null +++ b/tests/script/tsim/parser/last_both.sim @@ -0,0 +1,150 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======================== dnode1 start +$db = testdb +sql drop database if exists $db +sql create database $db cachemodel 'none' minrows 10 stt_trigger 1 +sql use $db + +sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int) +sql create table tb1 using st2 tags (1); +sql create table tb2 using st2 tags (2); +sql create table tb3 using st2 tags (3); +sql create table tb4 using st2 tags (4); +sql create table tb5 using st2 tags (1); +sql create table tb6 using st2 tags (2); +sql create table tb7 using st2 tags (3); +sql create table tb8 using st2 tags (4); +sql create table tb9 using st2 tags (5); +sql create table tba using st2 tags (5); +sql create table tbb using st2 tags (5); +sql create table tbc using st2 tags (5); +sql create table tbd using st2 tags (5); +sql create table tbe using st2 tags (5); +sql create table tbf using st2 tags (5); + +sql insert into tb9 values ("2021-05-09 10:12:26.000",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.001",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.002",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.003",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.004",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.005",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.006",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.007",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.008",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.009",28, 29, '30', -1005) +sql delete from tb9 where ts = "2021-05-09 10:12:26.000" +sql flush database $db + +sql insert into tb1 values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) +sql insert into tb1 values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000) +sql insert into tb1 values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000) + +sql insert into tb2 values ("2021-05-09 10:11:13",-1,-2.0,'-3', -1001) +sql insert into tb2 values ("2021-05-10 10:11:14",-4,-5.0, NULL, -2001) +sql insert into tb2 values ("2021-05-11 10:11:15",-6, -7, '-8', -3001) + +sql insert into tb3 values ("2021-05-09 10:12:17", 7, 8.0, '9' , -1002) +sql insert into tb3 values ("2021-05-09 10:12:17",10,11.0, NULL, -2002) +sql insert into tb3 values ("2021-05-09 10:12:18",12,NULL, NULL, -3002) + +sql insert into tb4 values ("2021-05-09 10:12:19",13,14.0,'15' , -1003) +sql insert into tb4 values ("2021-05-10 10:12:20",16,17.0, NULL, -2003) +sql insert into tb4 values ("2021-05-11 10:12:21",18,NULL, NULL, -3003) + +sql insert into tb5 values ("2021-05-09 10:12:22",19, 20, '21', -1004) +sql insert into tb6 values ("2021-05-11 10:12:23",22, 23, NULL, -2004) +sql insert into tb7 values ("2021-05-10 10:12:24",24,NULL, '25', -3004) +sql insert into tb8 values ("2021-05-11 10:12:25",26,NULL, '27', -4004) + +sql insert into tba values ("2021-05-10 10:12:27",31, 32, NULL, -2005) +sql insert into tbb values ("2021-05-10 10:12:28",33,NULL, '35', -3005) +sql insert into tbc values ("2021-05-11 10:12:29",36, 37, NULL, -4005) +sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL ) + +sql drop table tbf; +sql alter table st2 add column c1 int; +sql alter table st2 drop column c1; + +run tsim/parser/last_both_query.sim + +sql flush database $db +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +run tsim/parser/last_both_query.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +sql drop database if exists $db +sql create database $db minrows 10 stt_trigger 1 +sql use $db + +sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int) +sql create table tb1 using st2 tags (1); +sql create table tb2 using st2 tags (2); +sql create table tb3 using st2 tags (3); +sql create table tb4 using st2 tags (4); +sql create table tb5 using st2 tags (1); +sql create table tb6 using st2 tags (2); +sql create table tb7 using st2 tags (3); +sql create table tb8 using st2 tags (4); +sql create table tb9 using st2 tags (5); +sql create table tba using st2 tags (5); +sql create table tbb using st2 tags (5); +sql create table tbc using st2 tags (5); +sql create table tbd using st2 tags (5); +sql create table tbe using st2 tags (5); +sql create table tbf using st2 tags (5); + +sql insert into tb9 values ("2021-05-09 10:12:26.000",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.001",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.002",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.003",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.004",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.005",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.006",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.007",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.008",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.009",28, 29, '30', -1005) +sql delete from tb9 where ts = "2021-05-09 10:12:26.000" +sql flush database $db + +sql insert into tb1 values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) +sql insert into tb1 values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000) +sql insert into tb1 values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000) + +sql insert into tb2 values ("2021-05-09 10:11:13",-1,-2.0,'-3', -1001) +sql insert into tb2 values ("2021-05-10 10:11:14",-4,-5.0, NULL, -2001) +sql insert into tb2 values ("2021-05-11 10:11:15",-6, -7, '-8', -3001) + +sql insert into tb3 values ("2021-05-09 10:12:17", 7, 8.0, '9' , -1002) +sql insert into tb3 values ("2021-05-09 10:12:17",10,11.0, NULL, -2002) +sql insert into tb3 values ("2021-05-09 10:12:18",12,NULL, NULL, -3002) + +sql insert into tb4 values ("2021-05-09 10:12:19",13,14.0,'15' , -1003) +sql insert into tb4 values ("2021-05-10 10:12:20",16,17.0, NULL, -2003) +sql insert into tb4 values ("2021-05-11 10:12:21",18,NULL, NULL, -3003) + +sql insert into tb5 values ("2021-05-09 10:12:22",19, 20, '21', -1004) +sql insert into tb6 values ("2021-05-11 10:12:23",22, 23, NULL, -2004) +sql insert into tb7 values ("2021-05-10 10:12:24",24,NULL, '25', -3004) +sql insert into tb8 values ("2021-05-11 10:12:25",26,NULL, '27', -4004) + +sql insert into tba values ("2021-05-10 10:12:27",31, 32, NULL, -2005) +sql insert into tbb values ("2021-05-10 10:12:28",33,NULL, '35', -3005) +sql insert into tbc values ("2021-05-11 10:12:29",36, 37, NULL, -4005) +sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL ) + +sql drop table tbf +sql alter database $db cachemodel 'both' +sql alter database $db cachesize 2 +sleep 11000 + +run tsim/parser/last_both_query.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/last_both_query.sim b/tests/script/tsim/parser/last_both_query.sim new file mode 100644 index 0000000000..5f86412199 --- /dev/null +++ b/tests/script/tsim/parser/last_both_query.sim @@ -0,0 +1,496 @@ + +sql connect + +$db = testdb +sql use $db +print "test tb1" + +sql select last(ts) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi + +sql select last(f1) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != 6 then + print $data00 + return -1 +endi + +sql select last(*) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 3 then + print expect 3, actual: $data03 + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi + +sql select last(tb1.*,ts,f4) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 3 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data06 != @70-01-01 07:59:57.000@ then + return -1 +endi + +print "test tb2" +sql select last(ts) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi + +sql select last(f1) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != -6 then + print $data00 + return -1 +endi + +sql select last(*) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi +if $data01 != -6 then + return -1 +endi +if $data02 != -7.000000000 then + print $data02 + return -1 +endi +if $data03 != -8 then + return -1 +endi +if $data04 != @70-01-01 07:59:56.999@ then + if $data04 != @70-01-01 07:59:57.-01@ then + return -1 + endi +endi + +sql select last(tb2.*,ts,f4) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi +if $data01 != -6 then + return -1 +endi +if $data02 != -7.000000000 then + print $data02 + return -1 +endi +if $data03 != -8 then + return -1 +endi +if $data04 != @70-01-01 07:59:56.999@ then + if $data04 != @70-01-01 07:59:57.-01@ then + return -1 + endi +endi +if $data05 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi +if $data06 != @70-01-01 07:59:56.999@ then + if $data04 != @70-01-01 07:59:57.-01@ then + return -1 + endi +endi + +print "test tbd" +sql select last(*) from tbd +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:12:29.000@ then + print $data00 + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != NULL then + print $data02 + return -1 +endi +if $data03 != NULL then + return -1 +endi +if $data04 != NULL then + return -1 +endi + +print "test tbe" +sql select last(*) from tbe +if $rows != 0 then + return -1 +endi + +print "test stable" +sql select last(ts) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi + +sql select last(f1) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != 6 then + print $data00 + return -1 +endi + +sql select last(*) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 37.000000000 then + print expect 37.000000000 actual: $data02 + return -1 +endi +if $data03 != 27 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi + + +sql select last(st2.*,ts,f4) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 37.000000000 then + print expect 37.000000000, acutal: $data02 + return -1 +endi +if $data03 != 27 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data06 != @70-01-01 07:59:57.000@ then + return -1 +endi + +sql select last(*), id from st2 group by id order by id +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49 + +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 21 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data10 != @21-05-11 10:12:23.000@ then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data12 != 23.000000000 then + print $data02 + return -1 +endi +if $data13 != -8 then + return -1 +endi +if $data14 != @70-01-01 07:59:58.-04@ then + return -1 +endi +if $data15 != 2 then + return -1 +endi +if $data20 != @21-05-10 10:12:24.000@ then + return -1 +endi +if $data21 != 24 then + return -1 +endi +if $data22 != 11.000000000 then + print expect 11.000000000 actual: $data22 + return -1 +endi +if $data23 != 25 then + return -1 +endi +if $data24 != @70-01-01 07:59:57.-04@ then = + return -1 +endi +if $data25 != 3 then + return -1 +endi +if $data30 != @21-05-11 10:12:25.000@ then + return -1 +endi +if $data31 != 26 then + return -1 +endi +if $data32 != 17.000000000 then + print $data02 + return -1 +endi +if $data33 != 27 then + return -1 +endi +if $data34 != @70-01-01 07:59:56.-04@ then + return -1 +endi +if $data35 != 4 then + return -1 +endi +if $data40 != @21-05-11 10:12:29.000@ then + return -1 +endi +if $data41 != 36 then + return -1 +endi +if $data42 != 37.000000000 then + print $data02 + return -1 +endi +if $data43 != 35 then + return -1 +endi +if $data44 != @70-01-01 07:59:56.-05@ then + return -1 +endi +if $data45 != 5 then + return -1 +endi + +sql select last_row(*), id from st2 group by id order by id +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49 + +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != NULL then + print $data02 + return -1 +endi +if $data03 != NULL then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data10 != @21-05-11 10:12:23.000@ then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data12 != 23.000000000 then + print $data02 + return -1 +endi +if $data13 != NULL then + return -1 +endi +if $data14 != @70-01-01 07:59:58.-04@ then + return -1 +endi +if $data15 != 2 then + return -1 +endi +if $data20 != @21-05-10 10:12:24.000@ then + return -1 +endi +if $data21 != 24 then + return -1 +endi +if $data22 != NULL then + print expect NULL actual: $data22 + return -1 +endi +if $data23 != 25 then + return -1 +endi +if $data24 != @70-01-01 07:59:57.-04@ then = + return -1 +endi +if $data25 != 3 then + return -1 +endi +if $data30 != @21-05-11 10:12:25.000@ then + return -1 +endi +if $data31 != 26 then + return -1 +endi +if $data32 != NULL then + print $data02 + return -1 +endi +if $data33 != 27 then + return -1 +endi +if $data34 != @70-01-01 07:59:56.-04@ then + return -1 +endi +if $data35 != 4 then + return -1 +endi +if $data40 != @21-05-11 10:12:29.000@ then + return -1 +endi +#if $data41 != NULL then +# return -1 +#endi +#if $data42 != NULL then +# print $data02 +# return -1 +#endi +if $data43 != NULL then + return -1 +endi +#if $data44 != NULL then +# return -1 +#endi +if $data45 != 5 then + return -1 +endi + +print "test tbn" +sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) +sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) +sql insert into tbn values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000) +sql insert into tbn values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000) +sql insert into tbn values ("2021-05-13 10:10:12", NULL,NULL, NULL,NULL) + +sql select last(*) from tbn; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-13 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 3 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi + +sql alter table tbn add column c1 int; +sql alter table tbn drop column c1; diff --git a/tests/script/tsim/parser/last_cache_query.sim b/tests/script/tsim/parser/last_cache_query.sim index 6cd5309590..30196e0b62 100644 --- a/tests/script/tsim/parser/last_cache_query.sim +++ b/tests/script/tsim/parser/last_cache_query.sim @@ -386,3 +386,5 @@ if $data04 != @70-01-01 07:59:57.000@ then return -1 endi +sql alter table tbn add column c1 int; +sql alter table tbn drop column c1; diff --git a/tests/script/tsim/query/query_count0.sim b/tests/script/tsim/query/query_count0.sim index c3a75d635b..5b95d4fad7 100644 --- a/tests/script/tsim/query/query_count0.sim +++ b/tests/script/tsim/query/query_count0.sim @@ -175,5 +175,51 @@ if $data33 != 3 then goto loop3 endi +print step3 +print =============== create database +sql create database test3 vgroups 1; +sql use test3; + +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql insert into t1 values(1648791213000,0,1,3,1.0); +sql insert into t1 values(1648791213001,2,2,3,1.1); +sql insert into t1 values(1648791223002,0,3,3,2.1); +sql insert into t1 values(1648791223003,1,4,3,3.1); +sql insert into t1 values(1648791223004,1,5,3,4.1); +sql insert into t1 values(1648791223005,2,6,3,4.1); + +$loop_count = 0 +loop4: + +sleep 300 +print 1 sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 3; +sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 3; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $rows != 1 then + print ======rows=$rows + goto loop4 +endi + +print 1 sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 6; +sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 6; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 + +# row 0 +if $rows != 0 then + print ======rows=$rows + return -1 +endi + print query_count0 end system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/query_count1.sim b/tests/script/tsim/query/query_count1.sim index 0694ab062a..043b604263 100644 --- a/tests/script/tsim/query/query_count1.sim +++ b/tests/script/tsim/query/query_count1.sim @@ -79,5 +79,27 @@ if $data22 != 4 then goto loop3 endi + +print step2 +print =============== create database +sql create database test1 vgroups 1; +sql use test1; + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +#2~INT32_MAX +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(-1); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(0); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(1); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(2147483648); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(10, 0); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(10, -1); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(10, 11); + +sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(2); +sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(2147483647); + print query_count0 end system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/distributeMultiLevelInterval0.sim b/tests/script/tsim/stream/distributeMultiLevelInterval0.sim new file mode 100644 index 0000000000..784ab7f4a5 --- /dev/null +++ b/tests/script/tsim/stream/distributeMultiLevelInterval0.sim @@ -0,0 +1,267 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +system sh/cfg.sh -n dnode1 -c streamAggCnt -v 2 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + + + +print ===== step1 +sql drop stream if exists streams1; +sql drop database if exists test; +sql create database test vgroups 4; +sql use test; +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create table ts3 using st tags(3,2,2); +sql create table ts4 using st tags(4,2,2); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt1 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4 from st interval(10s); + +sleep 1000 + +sql insert into ts1 values(1648791213000,1,1,3,4.1); +sql insert into ts1 values(1648791223000,2,2,3,1.1); +sql insert into ts1 values(1648791233000,3,3,3,2.1); +sql insert into ts1 values(1648791243000,4,4,3,3.1); + +sql insert into ts2 values(1648791213000,1,5,3,4.1); +sql insert into ts2 values(1648791223000,2,6,3,1.1); +sql insert into ts2 values(1648791233000,3,7,3,2.1); +sql insert into ts2 values(1648791243000,4,8,3,3.1); + + +$loop_count = 0 +loop0: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 1000 +print 2 select * from streamt1; +sql select * from streamt1; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 + +if $rows != 4 then + print =====rows=$rows + goto loop0 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop0 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop0 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop0 +endi + +if $data31 != 2 then + print =====data31=$data31 + goto loop0 +endi + + +sql insert into ts1 values(1648791213000,1,9,3,4.1); + +$loop_count = 0 +loop1: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 1000 +print 2 select * from streamt1; +sql select * from streamt1; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 + +if $rows != 4 then + print =====rows=$rows + goto loop1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop1 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop1 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop1 +endi + +if $data31 != 2 then + print =====data31=$data31 + goto loop1 +endi + +sql delete from ts2 where ts = 1648791243000 ; + +$loop_count = 0 +loop2: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 1000 +print 2 select * from streamt1; +sql select * from streamt1; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 + +if $rows != 4 then + print =====rows=$rows + goto loop2 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop2 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop2 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop2 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop2 +endi + +sql delete from ts2 where ts = 1648791223000 ; + +$loop_count = 0 +loop3: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 1000 +print 2 select * from streamt1; +sql select * from streamt1; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 + +if $rows != 4 then + print =====rows=$rows + goto loop3 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop3 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop3 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop3 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop3 +endi + + +sql insert into ts1 values(1648791233001,3,9,3,2.1); + +$loop_count = 0 +loop4: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 1000 +print 2 select * from streamt1; +sql select * from streamt1; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 + +if $rows != 4 then + print =====rows=$rows + goto loop4 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop4 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop4 +endi + +if $data21 != 3 then + print =====data21=$data21 + goto loop4 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop4 +endi + +sql select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5, avg(d) from st interval(10s); + + +print ===== over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index 567d963c86..f193e375d9 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -99,8 +99,14 @@ sql select * from d2.stb2 sql select * from d2.ctb2 sql select * from d2.ntb2 -print =============== check show +$loop_cnt = 5 +$loop_idx = 0 +$loop_flag = 0 + +loop_check_sysinfo_0: +print =============== check show of sysinfo 0 sql_error show users +sql_error show user privileges sql_error show cluster sql_error select * from information_schema.ins_dnodes sql_error select * from information_schema.ins_mnodes @@ -129,6 +135,9 @@ sql show topics sql show subscriptions sql show functions sql_error show grants +sql_error show grants full; +sql_error show grants logs; +sql_error show cluster machines; sql show queries sql show connections sql show apps @@ -142,7 +151,7 @@ sql_error show dnode 1 variables; sql show variables; -print =============== check information_schema +print =============== check information_schema of sysinfo 0 sql show databases if $rows != 3 then return -1 @@ -155,7 +164,7 @@ sql_error select * from information_schema.ins_modules sql_error select * from information_schema.ins_qnodes sql_error select * from information_schema.ins_cluster sql_error select * from information_schema.ins_users -sql_error select * from information_schema.ins_user_privilege +sql_error select * from information_schema.ins_user_privileges sql select * from information_schema.ins_databases sql select * from information_schema.ins_functions sql select * from information_schema.ins_indexes @@ -166,11 +175,14 @@ sql select * from information_schema.ins_topics sql select * from information_schema.ins_subscriptions sql select * from information_schema.ins_streams sql_error select * from information_schema.ins_grants +sql_error select * from information_schema.ins_grants_full +sql_error select * from information_schema.ins_grants_logs +sql_error select * from information_schema.ins_machines sql_error select * from information_schema.ins_vgroups sql select * from information_schema.ins_configs sql_error select * from information_schema.ins_dnode_variables -print =============== check performance_schema +print =============== check performance_schema of sysinfo 0 sql use performance_schema; sql select * from performance_schema.perf_connections sql select * from performance_schema.perf_queries @@ -178,4 +190,114 @@ sql select * from performance_schema.perf_consumers sql select * from performance_schema.perf_trans sql select * from performance_schema.perf_apps +goto loop_check_switch + +loop_check_sysinfo_1: +print =============== check show of sysinfo 1 +sql show users +sql show user privileges +sql show cluster +sql select * from information_schema.ins_dnodes +sql select * from information_schema.ins_mnodes +sql show snodes +sql select * from information_schema.ins_qnodes +sql show dnodes +sql show snodes +sql show qnodes +sql show mnodes +sql show db.vgroups +sql_error show db.stables +sql_error show db.tables +sql show indexes from stb from db +sql show databases +sql show d2.vgroups +sql show d2.stables +sql show d2.tables +sql show indexes from stb2 from d2 +#sql_error show create database db +sql_error show create table db.stb; +sql_error show create table db.ctb; +sql_error show create table db.ntb; +sql show streams +sql show consumers +sql show topics +sql show subscriptions +sql show functions +sql show grants +sql show grants full; +sql show grants logs; +sql show cluster machines; +sql show queries +sql show connections +sql show apps +sql show transactions +sql show create database d2 +sql show create table d2.stb2; +sql show create table d2.ctb2; +sql show create table d2.ntb2; +sql show local variables; +sql show dnode 1 variables; +sql show variables; + + +print =============== check information_schema of sysinfo 1 +sql show databases +if $rows != 3 then + return -1 +endi + +sql use information_schema; +sql select * from information_schema.ins_dnodes +sql select * from information_schema.ins_mnodes +sql_error select * from information_schema.ins_modules +sql select * from information_schema.ins_qnodes +sql select * from information_schema.ins_cluster +sql select * from information_schema.ins_users +sql select * from information_schema.ins_user_privileges +sql select * from information_schema.ins_databases +sql select * from information_schema.ins_functions +sql select * from information_schema.ins_indexes +sql select * from information_schema.ins_stables +sql select * from information_schema.ins_tables +sql select * from information_schema.ins_tags +sql select * from information_schema.ins_topics +sql select * from information_schema.ins_subscriptions +sql select * from information_schema.ins_streams +sql select * from information_schema.ins_grants +sql select * from information_schema.ins_grants_full +sql select * from information_schema.ins_grants_logs +sql select * from information_schema.ins_machines +sql select * from information_schema.ins_vgroups +sql select * from information_schema.ins_configs +sql select * from information_schema.ins_dnode_variables + +print =============== check performance_schema of sysinfo 1 +sql use performance_schema; +sql select * from performance_schema.perf_connections +sql select * from performance_schema.perf_queries +sql select * from performance_schema.perf_consumers +sql select * from performance_schema.perf_trans +sql select * from performance_schema.perf_apps + +loop_check_switch: +if $loop_idx > $loop_cnt then + goto loop_check_end +endi +$loop_idx = $loop_idx + 1 + +if $loop_flag == 0 then + system taos -P7100 -s 'alter user sysinfo0 sysinfo 1' + sleep 3000 + $loop_flag = 1 + goto loop_check_sysinfo_1 +else + system taos -P7100 -s 'alter user sysinfo0 sysinfo 0' + sleep 3000 + $loop_flag = 0 + goto loop_check_sysinfo_0 +endi + +loop_check_end: +print =============== check end + #system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/0-others/com_alltypedata.json b/tests/system-test/0-others/com_alltypedata.json index 0e6d8e3a07..1499ca7670 100644 --- a/tests/system-test/0-others/com_alltypedata.json +++ b/tests/system-test/0-others/com_alltypedata.json @@ -22,7 +22,7 @@ "vgroups": 2, "replica": 1, "precision": "ms", - "stt_trigger": 8, + "stt_trigger": 1, "minRows": 100, "maxRows": 4096 }, diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index c936cf1ae4..8163177a3b 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -1,11 +1,13 @@ from urllib.parse import uses_relative import taos +import taosws import sys import os import time import platform import inspect from taos.tmq import Consumer +from taos.tmq import * from pathlib import Path from util.log import * @@ -17,7 +19,7 @@ from util.dnodes import TDDnode from util.cluster import * import subprocess -BASEVERSION = "3.0.2.3" +BASEVERSION = "3.2.0.0" class TDTestCase: def caseDescription(self): f''' @@ -30,7 +32,7 @@ class TDTestCase: self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 4; ;use deldata; + self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 1; ;use deldata; create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int); create table deldata.ct1 using deldata.stb1 tags ( 1 ); insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); @@ -104,8 +106,19 @@ class TDTestCase: print(f"{packageName} has been exists") os.system(f" cd {packagePath} && tar xvf {packageName} && cd {packageTPath} && ./install.sh -e no " ) tdDnodes.stop(1) - print(f"start taosd: rm -rf {dataPath}/* && nohup taosd -c {cPath} & ") - os.system(f"rm -rf {dataPath}/* && nohup taosd -c {cPath} & " ) + print(f"start taosd: rm -rf {dataPath}/* && nohup /usr/bin/taosd -c {cPath} & ") + os.system(f"rm -rf {dataPath}/* && nohup /usr/bin/taosd -c {cPath} & " ) + os.system(f"killall taosadapter" ) + os.system(f"cp /etc/taos/taosadapter.toml {cPath}/taosadapter.toml " ) + taosadapter_cfg = cPath + "/taosadapter.toml" + taosadapter_log_path = cPath + "/../log/" + print(f"taosadapter_cfg:{taosadapter_cfg},taosadapter_log_path:{taosadapter_log_path} ") + self.alter_string_in_file(taosadapter_cfg,"#path = \"/var/log/taos\"",f"path = \"{taosadapter_log_path}\"") + self.alter_string_in_file(taosadapter_cfg,"taosConfigDir = \"\"",f"taosConfigDir = \"{cPath}\"") + print("/usr/bin/taosadapter --version") + os.system(f" /usr/bin/taosadapter --version" ) + print(f" LD_LIBRARY_PATH=/usr/lib -c {taosadapter_cfg} 2>&1 & ") + os.system(f" LD_LIBRARY_PATH=/usr/lib /usr/bin/taosadapter -c {taosadapter_cfg} 2>&1 & " ) sleep(5) @@ -116,7 +129,24 @@ class TDTestCase: def is_list_same_as_ordered_list(self,unordered_list, ordered_list): sorted_list = sorted(unordered_list) return sorted_list == ordered_list - + + def alter_string_in_file(self,file,old_str,new_str): + """ + replace str in file + :param file + :param old_str + :param new_str + :return: + """ + file_data = "" + with open(file, "r", encoding="utf-8") as f: + for line in f: + if old_str in line: + line = line.replace(old_str,new_str) + file_data += line + with open(file,"w",encoding="utf-8") as f: + f.write(file_data) + def run(self): scriptsPath = os.path.dirname(os.path.realpath(__file__)) distro_id = distro.id() @@ -131,7 +161,7 @@ class TDTestCase: dbname = "test" stb = f"{dbname}.meters" self.installTaosd(bPath,cPath) - os.system("echo 'debugFlag 143' > /etc/taos/taos.cfg ") + # os.system(f"echo 'debugFlag 143' >> {cPath}/taos.cfg ") tableNumbers=100 recordNumbers1=100 recordNumbers2=1000 @@ -163,11 +193,46 @@ class TDTestCase: # os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ") # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ') # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show streams;" ') - os.system(f"sed -i 's/\/etc\/taos/{cPath}/' 0-others/tmqBasic.json ") + self.alter_string_in_file("0-others/tmqBasic.json", "/etc/taos/", cPath) # os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/tmqBasic.json -y ") os.system('LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists tmq_test_topic as select current,voltage,phase from test.meters where voltage <= 106 and current <= 5;" ') os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show topics;" ') + os.system(f" /usr/bin/taosadapter --version " ) + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + } + consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) + try: + consumer.subscribe(["tmq_test_topic"]) + except TmqError: + tdLog.exit(f"subscribe error") + + while True: + message = consumer.poll(timeout=1.0) + if message: + print("message") + id = message.vgroup() + topic = message.topic() + database = message.database() + + for block in message: + nrows = block.nrows() + ncols = block.ncols() + for row in block: + print(row) + values = block.fetchall() + print(nrows, ncols) + + consumer.commit(message) + else: + print("break") + break + + consumer.close() tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") @@ -184,7 +249,8 @@ class TDTestCase: os.system("pkill taosd") # make sure all the data are saved in disk. self.checkProcessPid("taosd") - + os.system("pkill taosadapter") # make sure all the data are saved in disk. + self.checkProcessPid("taosadapter") tdLog.printNoPrefix("==========step2:update new version ") self.buildTaosd(bPath) @@ -193,6 +259,7 @@ class TDTestCase: tdsql=tdCom.newTdSql() print(tdsql) cmd = f" LD_LIBRARY_PATH=/usr/lib taos -h localhost ;" + print(os.system(cmd)) if os.system(cmd) == 0: raise Exception("failed to execute system command. cmd: %s" % cmd) diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index 169c70e675..690de65dbb 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -267,6 +267,7 @@ class TDTestCase: updatecfgDict["monitorInterval"] = "5" updatecfgDict["monitorMaxLogs"] = "10" updatecfgDict["monitorComp"] = "1" + updatecfgDict["monitorForceV2"] = "0" updatecfgDict["audit"] = '0' diff --git a/tests/system-test/2-query/tms_memleak.py b/tests/system-test/2-query/tms_memleak.py new file mode 100644 index 0000000000..0d5cdd8272 --- /dev/null +++ b/tests/system-test/2-query/tms_memleak.py @@ -0,0 +1,51 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TD-] + ''' + return + + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.conn = conn + + def restartTaosd(self, index=1, dbname="db"): + tdDnodes.stop(index) + tdDnodes.startWithoutSleep(index) + tdSql.execute(f"use tms_memleak") + + def run(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists tms_memleak") + tdSql.execute("create database if not exists tms_memleak") + tdSql.execute('use tms_memleak') + + tdSql.execute('create table st(ts timestamp, f int) tags (t int);') + + tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 1)('2021-04-19 00:00:02', 2)('2021-04-19 00:00:03', 3)('2021-04-19 00:00:04', 4)") + + tdSql.execute("insert into ct2 using st tags(2) values('2021-04-20 00:00:01', 5)('2021-04-20 00:00:02', 6)('2021-04-20 00:00:03', 7)('2021-04-20 00:00:04', 8)") + + tdSql.execute("insert into ct3 using st tags(3) values('2021-04-21 00:00:01', 5)('2021-04-21 00:00:02', 6)('2021-04-21 00:00:03', 7)('2021-04-21 00:00:04', 8)") + + tdSql.execute("insert into ct4 using st tags(4) values('2021-04-22 00:00:01', 5)('2021-04-22 00:00:02', 6)('2021-04-22 00:00:03', 7)('2021-04-22 00:00:04', 8)") + + tdSql.query("select * from st order by ts limit 1 "); + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1); + + tdSql.execute('drop database tms_memleak') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py similarity index 78% rename from tests/system-test/7-tmq/tmqVnodeTransform.py rename to tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py index c2b002ead6..a853489c3f 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py @@ -122,135 +122,7 @@ class TDTestCase: tdLog.debug(f"redistributeSql:{redistributeSql}") tdSql.query(redistributeSql) tdLog.debug("redistributeSql ok") - - def tmqCase1(self): - tdLog.printNoPrefix("======== test case 1: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 1, - 'stbName': 'stb', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 1000, - 'batchNum': 10, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - topicNameList = ['topic1'] - # expectRowsList = [] - tmqCom.initConsumerTable() - - tdLog.info("create topics from stb with filter") - queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) - # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - # tdSql.query(queryString) - # expectRowsList.append(tdSql.getRows()) - - # init consume info, and start tmq_sim, then check consume result - tdLog.info("insert consume info to consume processor") - consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 - topicList = topicNameList[0] - ifcheckdata = 1 - ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - - tdLog.info("create ctb1") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) - - tmqCom.getStartConsumeNotifyFromTmqsim() - tmqCom.getStartCommitNotifyFromTmqsim() - - #restart dnode & remove wal - self.restartAndRemoveWal() - - # redistribute vgroup - self.redistributeVgroups(); - - tdLog.info("create ctb2") - paraDict['ctbPrefix'] = "ctbn" - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict) - pInsertThread.join() - pInsertThread1.join() - - expectRows = 1 - resultList = tmqCom.selectConsumeResult(expectRows) - - if expectrowcnt / 2 > resultList[0]: - tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) - tdLog.exit("%d tmq consume rows error!"%consumerId) - - # tmqCom.checkFileContent(consumerId, queryString) - - time.sleep(10) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 1 end ...... ") - - def tmqCase2(self): - tdLog.printNoPrefix("======== test case 2: ") - paraDict = {'dbName':'dbt'} - - ntbName = "ntb" - - topicNameList = ['topic2'] - tmqCom.initConsumerTable() - - sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName) - tdLog.info("create nomal table sql: %s"%sqlString) - tdSql.execute(sqlString) - - tdLog.info("create topics from nomal table") - queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - tdSql.query("flush database %s"%(paraDict['dbName'])) - #restart dnode & remove wal - self.restartAndRemoveWal() - - # redistribute vgroup - self.redistributeVgroups(); - - sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName) - tdLog.info("alter table sql: %s"%sqlString) - tdSql.error(sqlString) - expectRows = 0 - resultList = tmqCom.selectConsumeResult(expectRows) - time.sleep(1) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 2 end ...... ") - + def tmqCase3(self): tdLog.printNoPrefix("======== test case 3: ") paraDict = {'dbName': 'dbt', @@ -330,12 +202,90 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 3 end ...... ") + def tmqCaseDbname(self): + tdLog.printNoPrefix("======== test case 4 subscrib Dbname start: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic4'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from database ") + queryString = "database %s "%(paraDict['dbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(1) + # restart dnode & remove wal + self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(6) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ") + def run(self): - self.prepareTestEnv() - self.tmqCase1() - self.tmqCase2() self.prepareTestEnv() self.tmqCase3() + self.prepareTestEnv() + self.tmqCaseDbname() def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py new file mode 100644 index 0000000000..40879d5c66 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py @@ -0,0 +1,266 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 1000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + # tdLog.info("create ctb") + # tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + # ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + # tdLog.info("insert data") + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def restartAndRemoveWal(self): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + os.system('rm -rf ' + dataPath) + tdLog.debug("dataPath:%s"%dataPath) + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def redistributeVgroups(self): + dnodesList = [] + tdSql.query("show dnodes") + for result in tdSql.queryResult: + dnodesList.append(result[0]) + print("dnodeList:",dnodesList) + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodesList.remove(result[0]) + vnodeId = result[1] + print("its all data",dnodesList) + # if self.replicaVar == 1: + # redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) + # else: + redistributeSql = f"redistribute vgroup {vnodeId} " + for vgdnode in dnodesList: + redistributeSql += f"dnode {vgdnode} " + print(redistributeSql) + + tdLog.debug(f"redistributeSql:{redistributeSql}") + tdSql.query(redistributeSql) + tdLog.debug("redistributeSql ok") + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups(); + + tdLog.info("create ctb2") + paraDict['ctbPrefix'] = "ctbn" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 > resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName':'dbt'} + + ntbName = "ntb" + + topicNameList = ['topic2'] + tmqCom.initConsumerTable() + + sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName) + tdLog.info("create nomal table sql: %s"%sqlString) + tdSql.execute(sqlString) + + tdLog.info("create topics from nomal table") + queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query("flush database %s"%(paraDict['dbName'])) + #restart dnode & remove wal + self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups(); + + sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName) + tdLog.info("alter table sql: %s"%sqlString) + tdSql.error(sqlString) + expectRows = 0 + resultList = tmqCom.selectConsumeResult(expectRows) + time.sleep(1) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1() + self.tmqCase2() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index e9b9b9e944..847bbcf4be 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -105,7 +105,8 @@ SWords shellCommands[] = { {"create or replace aggregate function as outputtype bufsize language ", 0, 0, NULL}, {"create user pass sysinfo 0;", 0, 0, NULL}, {"create user pass sysinfo 1;", 0, 0, NULL}, -#ifdef TD_ENTERPRISE +#ifdef TD_ENTERPRISE + {"create view as select", 0, 0, NULL}, {"compact database ", 0, 0, NULL}, #endif {"describe ", 0, 0, NULL}, @@ -162,13 +163,20 @@ SWords shellCommands[] = { {"show create database \\G;", 0, 0, NULL}, {"show create stable \\G;", 0, 0, NULL}, {"show create table \\G;", 0, 0, NULL}, +#ifdef TD_ENTERPRISE + {"show create view \\G;", 0, 0, NULL}, +#endif {"show connections;", 0, 0, NULL}, + {"show compact", 0, 0, NULL}, + {"show compacts;", 0, 0, NULL}, {"show cluster;", 0, 0, NULL}, {"show cluster alive;", 0, 0, NULL}, + {"show cluster machines;", 0, 0, NULL}, {"show databases;", 0, 0, NULL}, {"show dnodes;", 0, 0, NULL}, {"show dnode variables;", 0, 0, NULL}, {"show functions;", 0, 0, NULL}, + {"show licences;", 0, 0, NULL}, {"show mnodes;", 0, 0, NULL}, {"show queries;", 0, 0, NULL}, // 80 @@ -185,6 +193,7 @@ SWords shellCommands[] = { {"show table distributed ", 0, 0, NULL}, {"show tags from ", 0, 0, NULL}, {"show tags from ", 0, 0, NULL}, + {"show table tags from ", 0, 0, NULL}, {"show topics;", 0, 0, NULL}, {"show transactions;", 0, 0, NULL}, {"show users;", 0, 0, NULL}, @@ -194,7 +203,10 @@ SWords shellCommands[] = { {"show vgroups;", 0, 0, NULL}, {"show consumers;", 0, 0, NULL}, {"show grants;", 0, 0, NULL}, + {"show grants full;", 0, 0, NULL}, + {"show grants logs;", 0, 0, NULL}, #ifdef TD_ENTERPRISE + {"show views;", 0, 0, NULL}, {"split vgroup ", 0, 0, NULL}, #endif {"insert into values(", 0, 0, NULL}, @@ -302,6 +314,20 @@ char* key_systable[] = { char* udf_language[] = {"\'Python\'", "\'C\'"}; +// global keys can tips on anywhere +char* global_keys[] = { + "tbname", + "now", + "_wstart", + "_wend", + "_wduration", + "_qstart", + "_qend", + "_qduration", + "_qtag", + "_isfilled" + }; + // // ------- global variant define --------- // @@ -341,8 +367,9 @@ bool waitAutoFill = false; #define WT_VAR_KEYSELECT 20 #define WT_VAR_SYSTABLE 21 #define WT_VAR_LANGUAGE 22 +#define WT_VAR_GLOBALKEYS 23 -#define WT_VAR_CNT 23 +#define WT_VAR_CNT 24 #define WT_TEXT 0xFF @@ -494,10 +521,12 @@ void showHelp() { show connections;\n\ show cluster;\n\ show cluster alive;\n\ + show cluster machines;\n\ show databases;\n\ show dnodes;\n\ show dnode variables;\n\ show functions;\n\ + show licences;\n\ show mnodes;\n\ show queries;\n\ show query ;\n\ @@ -513,6 +542,7 @@ void showHelp() { show table distributed ;\n\ show tags from \n\ show tags from \n\ + show table tags from \n\ show topics;\n\ show transactions;\n\ show users;\n\ @@ -522,6 +552,8 @@ void showHelp() { show vgroups;\n\ show consumers;\n\ show grants;\n\ + show grants full;\n\ + show grants logs;\n\ ----- T ----- \n\ trim database ;\n\ ----- U ----- \n\ @@ -534,8 +566,13 @@ void showHelp() { balance vgroup ;\n\ balance vgroup leader on \n\ compact database ; \n\ + crate view as select ...\n\ redistribute vgroup dnode ;\n\ - split vgroup ;"); + split vgroup ;\n\ + show compacts;\n\ + show compact \n\ + show views;\n\ + show create view ;"); #endif printf("\n\n"); @@ -699,6 +736,7 @@ bool shellAutoInit() { GenerateVarType(WT_VAR_KEYSELECT, key_select, sizeof(key_select) / sizeof(char*)); GenerateVarType(WT_VAR_SYSTABLE, key_systable, sizeof(key_systable) / sizeof(char*)); GenerateVarType(WT_VAR_LANGUAGE, udf_language, sizeof(udf_language) / sizeof(char*)); + GenerateVarType(WT_VAR_GLOBALKEYS, global_keys, sizeof(global_keys) / sizeof(char*)); return true; } @@ -1800,6 +1838,13 @@ bool matchEnd(TAOS* con, SShellCmd* cmd) { goto _return; } + // global keys + if (fillWithType(con, cmd, last, WT_VAR_GLOBALKEYS)) { + ret = true; + goto _return; + } + + _return: taosMemoryFree(ps); return ret; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 23424cea98..995d3d04ec 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -75,7 +75,7 @@ bool shellIsEmptyCommand(const char *cmd) { int32_t shellRunSingleCommand(char *command) { shellCmdkilled = false; - + if (shellIsEmptyCommand(command)) { return 0; } @@ -1019,7 +1019,7 @@ void shellReadHistory() { char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1); int32_t read_size = 0; - while ((read_size = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) { + while ((read_size = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) > 0) { line[read_size - 1] = '\0'; taosMemoryFree(pHistory->hist[pHistory->hend]); pHistory->hist[pHistory->hend] = taosStrdup(line); @@ -1315,7 +1315,7 @@ int32_t shellExecute() { shellSetConn(shell.conn, runOnce); shellReadHistory(); - if(shell.args.is_bi_mode) { + if(shell.args.is_bi_mode) { // need set bi mode printf("Set BI mode is true.\n"); #ifndef WEBSOCKET