diff --git a/.gitignore b/.gitignore index 4e47039628..07003bda4c 100644 --- a/.gitignore +++ b/.gitignore @@ -99,6 +99,7 @@ tests/examples/JDBC/JDBCDemo/.classpath tests/examples/JDBC/JDBCDemo/.project tests/examples/JDBC/JDBCDemo/.settings/ source/libs/parser/inc/sql.* +source/os/src/timezone/ tests/script/tmqResult.txt tests/system-test/case_to_run.txt tests/develop-test/case_to_run.txt @@ -162,4 +163,12 @@ geos_c.h source/libs/parser/src/sql.c include/common/ttokenauto.h !packaging/smokeTest/pytest_require.txt -tdengine-test-dir +tdengine-test-dir/ +localtime.c +private.h +strftime.c +tzdir.h +tzfile.h +coverage.info +taos +taosd diff --git a/cmake/cmake.define b/cmake/cmake.define index eb95feaf82..a794d927ad 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -131,7 +131,7 @@ IF(TD_WINDOWS) SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MTd") ENDIF() - SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO") + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO /FORCE:MULTIPLE") # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) # SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") diff --git a/cmake/libuv_CMakeLists.txt.in b/cmake/libuv_CMakeLists.txt.in index 3bfb52fe9b..1d59418462 100644 --- a/cmake/libuv_CMakeLists.txt.in +++ b/cmake/libuv_CMakeLists.txt.in @@ -2,7 +2,7 @@ # libuv ExternalProject_Add(libuv GIT_REPOSITORY https://github.com/libuv/libuv.git - GIT_TAG v1.48.0 + GIT_TAG v1.49.2 SOURCE_DIR "${TD_CONTRIB_DIR}/libuv" BINARY_DIR "${TD_CONTRIB_DIR}/libuv" CONFIGURE_COMMAND "" diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 13826a1a74..ef6ed4af1d 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 9bbda8309f..9a6a5329ae 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index b013d45911..17446d184d 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/deps/arm/dm_static/libdmodule.a b/deps/arm/dm_static/libdmodule.a index 06d509b51b..b197930062 100644 Binary files a/deps/arm/dm_static/libdmodule.a and b/deps/arm/dm_static/libdmodule.a differ diff --git a/deps/darwin/arm/dm_static/libdmodule.a b/deps/darwin/arm/dm_static/libdmodule.a index 2bede61caa..c4e1214bc5 100644 Binary files a/deps/darwin/arm/dm_static/libdmodule.a and b/deps/darwin/arm/dm_static/libdmodule.a differ diff --git a/deps/darwin/x64/dm_static/libdmodule.a b/deps/darwin/x64/dm_static/libdmodule.a index ae1abc5dfc..3613ce21c2 100644 Binary files a/deps/darwin/x64/dm_static/libdmodule.a and b/deps/darwin/x64/dm_static/libdmodule.a differ diff --git a/deps/mips/dm_static/libdmodule.a b/deps/mips/dm_static/libdmodule.a index b60727d440..ce437c6d1b 100644 Binary files a/deps/mips/dm_static/libdmodule.a and b/deps/mips/dm_static/libdmodule.a differ diff --git a/deps/win/x64/dm_static/dmodule.lib b/deps/win/x64/dm_static/dmodule.lib index 55afb81e3f..955b11b7c3 100644 Binary files a/deps/win/x64/dm_static/dmodule.lib and b/deps/win/x64/dm_static/dmodule.lib differ diff --git a/deps/x86/dm_static/libdmodule.a b/deps/x86/dm_static/libdmodule.a index f2d3f600ce..84fc6c382d 100644 Binary files a/deps/x86/dm_static/libdmodule.a and b/deps/x86/dm_static/libdmodule.a differ diff --git a/docs/en/06-advanced/05-data-in/07-mqtt.md b/docs/en/06-advanced/05-data-in/07-mqtt.md index 3f6eed5834..73ef3b534c 100644 --- a/docs/en/06-advanced/05-data-in/07-mqtt.md +++ b/docs/en/06-advanced/05-data-in/07-mqtt.md @@ -112,14 +112,14 @@ Fill in the example data from the MQTT message body in **Message Body**. JSON data supports JSONObject or JSONArray, and the json parser can parse the following data: -``` json +```json {"id": 1, "message": "hello-word"} {"id": 2, "message": "hello-word"} ``` or -``` json +```json [{"id": 1, "message": "hello-word"},{"id": 2, "message": "hello-word"}] ``` diff --git a/docs/en/06-advanced/05-data-in/08-kafka.md b/docs/en/06-advanced/05-data-in/08-kafka.md index bdd652a5e7..b3bdca4cf9 100644 --- a/docs/en/06-advanced/05-data-in/08-kafka.md +++ b/docs/en/06-advanced/05-data-in/08-kafka.md @@ -109,7 +109,7 @@ In addition, the [Kerberos](https://web.mit.edu/kerberos/) authentication servic After configuration, you can use the [kcat](https://github.com/edenhill/kcat) tool to verify Kafka topic consumption: -```bash +```shell kcat \ -b \ -G kcat \ @@ -171,14 +171,14 @@ Enter sample data from the Kafka message body in **Message Body**. JSON data supports JSONObject or JSONArray, and the following data can be parsed using a JSON parser: -``` json +```json {"id": 1, "message": "hello-word"} {"id": 2, "message": "hello-word"} ``` or -``` json +```json [{"id": 1, "message": "hello-word"},{"id": 2, "message": "hello-word"}] ``` diff --git a/docs/en/06-advanced/05-data-in/index.md b/docs/en/06-advanced/05-data-in/index.md index 59a3c6da7b..5221aa2002 100644 --- a/docs/en/06-advanced/05-data-in/index.md +++ b/docs/en/06-advanced/05-data-in/index.md @@ -83,7 +83,7 @@ Parsing is the process of parsing unstructured strings into structured data. The JSON parsing supports JSONObject or JSONArray. The following JSON sample data can automatically parse fields: `groupid`, `voltage`, `current`, `ts`, `inuse`, `location`. -``` json +```json {"groupid": 170001, "voltage": "221V", "current": 12.3, "ts": "2023-12-18T22:12:00", "inuse": true, "location": "beijing.chaoyang.datun"} {"groupid": 170001, "voltage": "220V", "current": 12.2, "ts": "2023-12-18T22:12:02", "inuse": true, "location": "beijing.chaoyang.datun"} {"groupid": 170001, "voltage": "216V", "current": 12.5, "ts": "2023-12-18T22:12:04", "inuse": false, "location": "beijing.chaoyang.datun"} @@ -91,7 +91,7 @@ JSON parsing supports JSONObject or JSONArray. The following JSON sample data ca Or -``` json +```json [{"groupid": 170001, "voltage": "221V", "current": 12.3, "ts": "2023-12-18T22:12:00", "inuse": true, "location": "beijing.chaoyang.datun"}, {"groupid": 170001, "voltage": "220V", "current": 12.2, "ts": "2023-12-18T22:12:02", "inuse": true, "location": "beijing.chaoyang.datun"}, {"groupid": 170001, "voltage": "216V", "current": 12.5, "ts": "2023-12-18T22:12:04", "inuse": false, "location": "beijing.chaoyang.datun"}] @@ -101,7 +101,7 @@ Subsequent examples will only explain using JSONObject. The following nested JSON data can automatically parse fields `groupid`, `data_voltage`, `data_current`, `ts`, `inuse`, `location_0_province`, `location_0_city`, `location_0_datun`, and you can also choose which fields to parse and set aliases for the parsed fields. -``` json +```json {"groupid": 170001, "data": { "voltage": "221V", "current": 12.3 }, "ts": "2023-12-18T22:12:00", "inuse": true, "location": [{"province": "beijing", "city":"chaoyang", "street": "datun"}]} ``` @@ -114,7 +114,7 @@ The following nested JSON data can automatically parse fields `groupid`, `data_v You can use **named capture groups** in regular expressions to extract multiple fields from any string (text) field. As shown in the figure, extract fields such as access IP, timestamp, and accessed URL from nginx logs. -``` re +```regex (?\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b)\s-\s-\s\[(?\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}\s\+\d{4})\]\s"(?[A-Z]+)\s(?[^\s"]+).*(?\d{3})\s(?\d+) ``` @@ -133,7 +133,7 @@ Custom rhai syntax scripts for parsing input data (refer to `https://rhai.rs/boo For example, for data reporting three-phase voltage values, which are entered into three subtables respectively, such data needs to be parsed -``` json +```json { "ts": "2024-06-27 18:00:00", "voltage": "220.1,220.3,221.1", @@ -164,7 +164,7 @@ The final parsing result is shown below: The parsed data may still not meet the data requirements of the target table. For example, the original data collected by a smart meter is as follows (in json format): -``` json +```json {"groupid": 170001, "voltage": "221V", "current": 12.3, "ts": "2023-12-18T22:12:00", "inuse": true, "location": "beijing.chaoyang.datun"} {"groupid": 170001, "voltage": "220V", "current": 12.2, "ts": "2023-12-18T22:12:02", "inuse": true, "location": "beijing.chaoyang.datun"} {"groupid": 170001, "voltage": "216V", "current": 12.5, "ts": "2023-12-18T22:12:04", "inuse": false, "location": "beijing.chaoyang.datun"} diff --git a/docs/en/07-develop/02-sql.md b/docs/en/07-develop/02-sql.md index 57376e615d..d32ff340f3 100644 --- a/docs/en/07-develop/02-sql.md +++ b/docs/en/07-develop/02-sql.md @@ -83,14 +83,14 @@ Next, create a supertable (STABLE) named `meters`, whose table structure include Create Database -```bash +```shell curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql' \ --data 'CREATE DATABASE IF NOT EXISTS power' ``` Create Table, specify the database as `power` in the URL -```bash +```shell curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql/power' \ --data 'CREATE STABLE IF NOT EXISTS meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))' ``` @@ -167,7 +167,7 @@ NOW is an internal system function, defaulting to the current time of the client Write data -```bash +```shell curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql' \ --data 'INSERT INTO power.d1001 USING power.meters TAGS(2,'\''California.SanFrancisco'\'') VALUES (NOW + 1a, 10.30000, 219, 0.31000) (NOW + 2a, 12.60000, 218, 0.33000) (NOW + 3a, 12.30000, 221, 0.31000) power.d1002 USING power.meters TAGS(3, '\''California.SanFrancisco'\'') VALUES (NOW + 1a, 10.30000, 218, 0.25000)' ``` @@ -247,7 +247,7 @@ Rust connector also supports using **serde** for deserializing to get structured Query Data -```bash +```shell curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql' \ --data 'SELECT ts, current, location FROM power.meters limit 100' ``` @@ -329,7 +329,7 @@ Below are code examples of setting reqId to execute SQL in various language conn Query data, specify reqId as 3 -```bash +```shell curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql?req_id=3' \ --data 'SELECT ts, current, location FROM power.meters limit 1' ``` diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index 795556c225..0e91dd09db 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -273,19 +273,19 @@ To better operate the above data structures, some convenience functions are prov Create table: -```bash +```shell create table battery(ts timestamp, vol1 float, vol2 float, vol3 float, deviceId varchar(16)); ``` Create custom function: -```bash +```shell create aggregate function max_vol as '/root/udf/libmaxvol.so' outputtype binary(64) bufsize 10240 language 'C'; ``` Use custom function: -```bash +```shell select max_vol(vol1, vol2, vol3, deviceid) from battery; ``` @@ -334,7 +334,7 @@ When developing UDFs in Python, you need to implement the specified interface fu The interface for scalar functions is as follows. -```Python +```python def process(input: datablock) -> tuple[output_type]: ``` @@ -347,7 +347,7 @@ The main parameters are as follows: The interface for aggregate functions is as follows. -```Python +```python def start() -> bytes: def reduce(inputs: datablock, buf: bytes) -> bytes def finish(buf: bytes) -> output_type: @@ -365,7 +365,7 @@ Finally, when all row data blocks have been processed, the finish function is ca The interfaces for initialization and destruction are as follows. -```Python +```python def init() def destroy() ``` @@ -381,7 +381,7 @@ Parameter description: The template for developing scalar functions in Python is as follows. -```Python +```python def init(): # initialization def destroy(): @@ -393,7 +393,7 @@ def process(input: datablock) -> tuple[output_type]: The template for developing aggregate functions in Python is as follows. -```Python +```python def init(): #initialization def destroy(): @@ -828,7 +828,7 @@ Through this example, we learned how to define aggregate functions and print cus
pybitand.py -```Python +```python {{#include tests/script/sh/pybitand.py}} ``` diff --git a/docs/en/08-operation/04-maintenance.md b/docs/en/08-operation/04-maintenance.md index 5e7fca1f08..970ee40d18 100644 --- a/docs/en/08-operation/04-maintenance.md +++ b/docs/en/08-operation/04-maintenance.md @@ -15,7 +15,7 @@ TDengine is designed for various writing scenarios, and many of these scenarios ### Syntax -```SQL +```sql COMPACT DATABASE db_name [start with 'XXXX'] [end with 'YYYY']; SHOW COMPACTS [compact_id]; KILL COMPACT compact_id; @@ -41,7 +41,7 @@ KILL COMPACT compact_id; When one or more nodes in a multi-replica cluster restart due to upgrades or other reasons, it may lead to an imbalance in the load among the various dnodes in the cluster. In extreme cases, all vgroup leaders may be located on the same dnode. To solve this problem, you can use the following commands, which were first released in version 3.0.4.0. It is recommended to use the latest version as much as possible. -```SQL +```sql balance vgroup leader; # Rebalance all vgroup leaders balance vgroup leader on ; # Rebalance a vgroup leader balance vgroup leader database ; # Rebalance all vgroup leaders within a database diff --git a/docs/en/08-operation/12-multi.md b/docs/en/08-operation/12-multi.md index f2f464be1c..1d0b8ad6cb 100644 --- a/docs/en/08-operation/12-multi.md +++ b/docs/en/08-operation/12-multi.md @@ -121,7 +121,7 @@ The cost of using object storage services is related to the amount of data store When the TSDB time-series data exceeds the time specified by the `s3_keeplocal` parameter, the related data files will be split into multiple file blocks, each with a default size of 512 MB (`s3_chunkpages * tsdb_pagesize`). Except for the last file block, which is retained on the local file system, the rest of the file blocks are uploaded to the object storage service. -```math +```text Upload Count = Data File Size / (s3_chunkpages * tsdb_pagesize) - 1 ``` @@ -135,7 +135,7 @@ During query operations, if data in object storage needs to be accessed, TSDB do Adjacent multiple data pages are downloaded as a single data block from object storage to reduce the number of downloads. The size of each data page is specified by the `tsdb_pagesize` parameter when creating the database, with a default of 4 KB. -```math +```text Download Count = Number of Data Blocks Needed for Query - Number of Cached Data Blocks ``` @@ -155,7 +155,7 @@ For deployment methods, please refer to the [Flexify](https://azuremarketplace.m In the configuration file /etc/taos/taos.cfg, add parameters for S3 access: -```cfg +```text s3EndPoint http //20.191.157.23,http://20.191.157.24,http://20.191.157.25 s3AccessKey FLIOMMNL0:uhRNdeZMLD4wo,ABCIOMMN:uhRNdeZMD4wog,DEFOMMNL049ba:uhRNdeZMLD4wogXd s3BucketName td-test diff --git a/docs/en/08-operation/14-user.md b/docs/en/08-operation/14-user.md index 3165f6ad32..5aa8b2e211 100644 --- a/docs/en/08-operation/14-user.md +++ b/docs/en/08-operation/14-user.md @@ -18,14 +18,14 @@ create user user_name pass'password' [sysinfo {1|0}] [createdb {1|0}] The parameters are explained as follows. - user_name: Up to 23 B long. -- password: Up to 128 B long, valid characters include letters and numbers as well as special characters other than single and double quotes, apostrophes, backslashes, and spaces, and it cannot be empty. +- password: The password must be between 8 and 16 characters long and include at least three types of characters from the following: uppercase letters, lowercase letters, numbers, and special characters. Special characters include `! @ # $ % ^ & * ( ) - _ + = [ ] { } : ; > < ? | ~ , .`. - sysinfo: Whether the user can view system information. 1 means they can view it, 0 means they cannot. System information includes server configuration information, various node information such as dnode, query node (qnode), etc., as well as storage-related information, etc. The default is to view system information. - createdb: Whether the user can create databases. 1 means they can create databases, 0 means they cannot. The default value is 0. // Supported starting from TDengine Enterprise version 3.3.2.0 -The following SQL can create a user named test with the password 123456 who can view system information. +The following SQL can create a user named test with the password abc123!@# who can view system information. ```sql -create user test pass '123456' sysinfo 1 +create user test pass 'abc123!@#' sysinfo 1 ``` ### Viewing Users diff --git a/docs/en/10-third-party/01-collection/09-emq-broker.md b/docs/en/10-third-party/01-collection/09-emq-broker.md index 24cb12997f..4f3eadcca6 100644 --- a/docs/en/10-third-party/01-collection/09-emq-broker.md +++ b/docs/en/10-third-party/01-collection/09-emq-broker.md @@ -140,7 +140,7 @@ Finally, click the "Create" button at the bottom left to save the rule. ## Write a Mock Test Program -```javascript +```js {{#include docs/examples/other/mock.js}} ``` diff --git a/docs/en/10-third-party/01-collection/11-kafka.md b/docs/en/10-third-party/01-collection/11-kafka.md index 2627e33b65..25ba7fd54a 100644 --- a/docs/en/10-third-party/01-collection/11-kafka.md +++ b/docs/en/10-third-party/01-collection/11-kafka.md @@ -95,7 +95,7 @@ curl http://localhost:8083/connectors If all components have started successfully, the following output will be displayed: -```txt +```text [] ``` @@ -181,7 +181,7 @@ If the above command is executed successfully, the following output will be disp Prepare a text file with test data, content as follows: -```txt title="test-data.txt" +```text title="test-data.txt" meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249000000 meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611250000000 meters,location=California.LosAngeles,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249000000 @@ -303,7 +303,7 @@ kafka-console-consumer.sh --bootstrap-server localhost:9092 --from-beginning --t Output: -```txt +```text ...... meters,location="California.SanFrancisco",groupid=2i32 current=10.3f32,voltage=219i32,phase=0.31f32 1538548685000000000 meters,location="California.SanFrancisco",groupid=2i32 current=12.6f32,voltage=218i32,phase=0.33f32 1538548695000000000 diff --git a/docs/en/10-third-party/03-visual/01-grafana.mdx b/docs/en/10-third-party/03-visual/01-grafana.md similarity index 99% rename from docs/en/10-third-party/03-visual/01-grafana.mdx rename to docs/en/10-third-party/03-visual/01-grafana.md index bd2d313def..ec857f7795 100644 --- a/docs/en/10-third-party/03-visual/01-grafana.mdx +++ b/docs/en/10-third-party/03-visual/01-grafana.md @@ -60,7 +60,7 @@ Click `Save & Test` to test, if successful, it will prompt: `TDengine Data sourc For users using Grafana version 7.x or configuring with [Grafana Provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/), you can use the installation script on the Grafana server to automatically install the plugin and add the data source Provisioning configuration file. -```sh +```shell bash -c "$(curl -fsSL \ https://raw.githubusercontent.com/taosdata/grafanaplugin/master/install.sh)" -- \ -a http://localhost:6041 \ @@ -77,7 +77,7 @@ Save the script and execute `./install.sh --help` to view detailed help document Use the [`grafana-cli` command line tool](https://grafana.com/docs/grafana/latest/administration/cli/) to install the plugin [installation](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation). -```bash +```shell grafana-cli plugins install tdengine-datasource # with sudo sudo -u grafana grafana-cli plugins install tdengine-datasource @@ -85,7 +85,7 @@ sudo -u grafana grafana-cli plugins install tdengine-datasource Alternatively, download the .zip file from [GitHub](https://github.com/taosdata/grafanaplugin/releases/tag/latest) or [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) to your local machine and unzip it into the Grafana plugins directory. Example command line download is as follows: -```bash +```shell GF_VERSION=3.5.1 # from GitHub wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/tdengine-datasource-$GF_VERSION.zip @@ -95,13 +95,13 @@ wget -O tdengine-datasource-$GF_VERSION.zip https://grafana.com/api/plugins/tden For CentOS 7.2 operating system, unzip the plugin package into the /var/lib/grafana/plugins directory and restart Grafana. -```bash +```shell sudo unzip tdengine-datasource-$GF_VERSION.zip -d /var/lib/grafana/plugins/ ``` If Grafana is running in a Docker environment, you can use the following environment variable to set up automatic installation of the TDengine data source plugin: -```bash +```shell GF_INSTALL_PLUGINS=tdengine-datasource ``` @@ -120,7 +120,7 @@ Click `Save & Test` to test, if successful, it will prompt: `TDengine Data sourc Refer to [Grafana containerized installation instructions](https://grafana.com/docs/grafana/next/setup-grafana/installation/docker/#install-plugins-in-the-docker-container). Use the following command to start a container and automatically install the TDengine plugin: -```bash +```shell docker run -d \ -p 3000:3000 \ --name=grafana \ diff --git a/docs/en/14-reference/01-components/01-taosd.md b/docs/en/14-reference/01-components/01-taosd.md index 436f7c522b..81bfd2ecad 100644 --- a/docs/en/14-reference/01-components/01-taosd.md +++ b/docs/en/14-reference/01-components/01-taosd.md @@ -28,68 +28,70 @@ After modifying configuration file parameters, you need to restart the *taosd* s ### Connection Related -|Parameter Name |Supported Version |Description| -|-----------------------|-------------------------|------------| -|firstEp | |Endpoint of the first dnode in the cluster that taosd actively connects to at startup, default value localhost:6030| -|secondEp | |Endpoint of the second dnode in the cluster that taosd tries to connect to if the firstEp is unreachable, no default value| -|fqdn | |The service address that taosd listens on, default is the first hostname configured on the server| -|serverPort | |The port that taosd listens on, default value 6030| -|compressMsgSize | |Whether to compress RPC messages; -1: do not compress any messages; 0: compress all messages; N (N>0): only compress messages larger than N bytes; default value -1| -|shellActivityTimer | |Duration in seconds for the client to send heartbeat to mnode, range 1-120, default value 3 | -|numOfRpcSessions | |Maximum number of connections supported by RPC, range 100-100000, default value 30000| -|numOfRpcThreads | |Number of threads for receiving and sending RPC data, range 1-50, default value is half of the CPU cores| -|numOfTaskQueueThreads | |Number of threads for client to process RPC messages, range 4-16, default value is half of the CPU cores| -|rpcQueueMemoryAllowed | |Maximum memory allowed for received RPC messages in dnode, in bytes, range 104857600-INT64_MAX, default value is 1/10 of server memory | -|resolveFQDNRetryTime | Cancelled after 3.x |Number of retries when FQDN resolution fails| +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +|firstEp | |Not supported |Endpoint of the first dnode in the cluster that taosd actively connects to at startup, default value localhost:6030| +|secondEp | |Not supported |Endpoint of the second dnode in the cluster that taosd tries to connect to if the firstEp is unreachable, no default value| +|fqdn | |Not supported |The service address that taosd listens on, default is the first hostname configured on the server| +|serverPort | |Not supported |The port that taosd listens on, default value 6030| +|compressMsgSize | |Supported, effective after restart|Whether to compress RPC messages; -1: do not compress any messages; 0: compress all messages; N (N>0): only compress messages larger than N bytes; default value -1| +|shellActivityTimer | |Supported, effective immediately |Duration in seconds for the client to send heartbeat to mnode, range 1-120, default value 3 | +|numOfRpcSessions | |Supported, effective after restart|Maximum number of connections supported by RPC, range 100-100000, default value 30000| +|numOfRpcThreads | |Supported, effective after restart|Number of threads for receiving and sending RPC data, range 1-50, default value is half of the CPU cores| +|numOfTaskQueueThreads | |Supported, effective after restart|Number of threads for client to process RPC messages, range 4-16, default value is half of the CPU cores| +|rpcQueueMemoryAllowed | |Supported, effective immediately |Maximum memory allowed for received RPC messages in dnode, in bytes, range 104857600-INT64_MAX, default value is 1/10 of server memory | +|resolveFQDNRetryTime | Cancelled after 3.x |Not supported |Number of retries when FQDN resolution fails| |timeToGetAvailableConn | Cancelled after 3.3.4.x |Maximum waiting time to get an available connection, range 10-50000000, in milliseconds, default value 500000| -|maxShellConns | Cancelled after 3.x |Maximum number of connections allowed| -|maxRetryWaitTime | |Maximum timeout for reconnection, default value is 10s| -|shareConnLimit |Added in 3.3.4.0 |Number of requests a connection can share, range 1-512, default value 10| -|readTimeout |Added in 3.3.4.0 |Minimum timeout for a single request, range 64-604800, in seconds, default value 900| +|maxShellConns | Cancelled after 3.x |Supported, effective after restart|Maximum number of connections allowed| +|maxRetryWaitTime | |Supported, effective after restart|Maximum timeout for reconnection, default value is 10s| +|shareConnLimit |Added in 3.3.4.0 |Supported, effective after restart|Number of requests a connection can share, range 1-512, default value 10| +|readTimeout |Added in 3.3.4.0 |Supported, effective after restart|Minimum timeout for a single request, range 64-604800, in seconds, default value 900| ### Monitoring Related -|Parameter Name|Supported Version|Description| -|-----------------------|----------|-| -|monitor | |Whether to collect and report monitoring data, 0: off; 1: on; default value 0| -|monitorFqdn | |The FQDN of the server where the taosKeeper service is located, default value none| -|monitorPort | |The port number listened to by the taosKeeper service, default value 6043| -|monitorInterval | |The time interval for recording system parameters (CPU/memory) in the monitoring database, in seconds, range 1-200000, default value 30| -|monitorMaxLogs | |Number of cached logs pending report| -|monitorComp | |Whether to use compression when reporting monitoring logs| -|monitorLogProtocol | |Whether to print monitoring logs| -|monitorForceV2 | |Whether to use V2 protocol for reporting| -|telemetryReporting | |Whether to upload telemetry, 0: do not upload, 1: upload, default value 1| -|telemetryServer | |Telemetry server address| -|telemetryPort | |Telemetry server port number| -|telemetryInterval | |Telemetry upload interval, in seconds, default 43200| -|crashReporting | |Whether to upload crash information; 0: do not upload, 1: upload; default value 1| +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +|monitor | |Supported, effective immediately |Whether to collect and report monitoring data, 0: off; 1: on; default value 0| +|monitorFqdn | |Supported, effective after restart|The FQDN of the server where the taosKeeper service is located, default value none| +|monitorPort | |Supported, effective after restart|The port number listened to by the taosKeeper service, default value 6043| +|monitorInterval | |Supported, effective immediately |The time interval for recording system parameters (CPU/memory) in the monitoring database, in seconds, range 1-200000, default value 30| +|monitorMaxLogs | |Supported, effective immediately |Number of cached logs pending report| +|monitorComp | |Supported, effective after restart|Whether to use compression when reporting monitoring logs| +|monitorLogProtocol | |Supported, effective immediately |Whether to print monitoring logs| +|monitorForceV2 | |Supported, effective immediately |Whether to use V2 protocol for reporting| +|telemetryReporting | |Supported, effective immediately |Whether to upload telemetry, 0: do not upload, 1: upload, default value 1| +|telemetryServer | |Not supported |Telemetry server address| +|telemetryPort | |Not supported |Telemetry server port number| +|telemetryInterval | |Supported, effective immediately |Telemetry upload interval, in seconds, default 43200| +|crashReporting | |Supported, effective immediately |Whether to upload crash information; 0: do not upload, 1: upload; default value 1| ### Query Related -|Parameter Name|Supported Version|Description| -|------------------------|----------|-| -|countAlwaysReturnValue | |Whether count/hyperloglog functions return a value when input data is empty or NULL; 0: return empty row, 1: return; default value 1; When this parameter is set to 1, if the query contains an INTERVAL clause or the query uses TSMA, and the corresponding group or window has empty or NULL data, the corresponding group or window will not return a query result; Note that this parameter should be consistent between client and server| -|tagFilterCache | |Whether to cache tag filter results| -|maxNumOfDistinctRes | |Maximum number of distinct results allowed to return, default value 100,000, maximum allowed value 100 million| -|queryBufferSize | |Not effective yet| -|queryRspPolicy | |Query response strategy| -|filterScalarMode | |Force scalar filter mode, 0: off; 1: on, default value 0| -|queryPlannerTrace | |Internal parameter, whether the query plan outputs detailed logs| -|queryNodeChunkSize | |Internal parameter, chunk size of the query plan| -|queryUseNodeAllocator | |Internal parameter, allocation method of the query plan| -|queryMaxConcurrentTables| |Internal parameter, concurrency number of the query plan| -|queryRsmaTolerance | |Internal parameter, tolerance time for determining which level of rsma data to query, in milliseconds| -|enableQueryHb | |Internal parameter, whether to send query heartbeat messages| -|pqSortMemThreshold | |Internal parameter, memory threshold for sorting| +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +|countAlwaysReturnValue | |Supported, effective immediately |Whether count/hyperloglog functions return a value when input data is empty or NULL; 0: return empty row, 1: return; default value 1; When this parameter is set to 1, if the query contains an INTERVAL clause or the query uses TSMA, and the corresponding group or window has empty or NULL data, the corresponding group or window will not return a query result; Note that this parameter should be consistent between client and server| +|tagFilterCache | |Not supported |Whether to cache tag filter results| +|queryBufferSize | |Supported, effective after restart|Not effective yet| +|queryRspPolicy | |Supported, effective immediately |Query response strategy| +|queryUseMemoryPool | |Not supported |Whether query will use memory pool to manage memory, default value: 1 (on); 0: off, 1: on| +|minReservedMemorySize | |Not supported |The minimum reserved system available memory size, all memory except reserved can be used for queries, unit: MB, default reserved size is 20% of system physical memory, value range 1024-1000000000| +|singleQueryMaxMemorySize| |Not supported |The memory limit that a single query can use on a single node (dnode), exceeding this limit will return an error, unit: MB, default value: 0 (no limit), value range 0-1000000000| +|filterScalarMode | |Not supported |Force scalar filter mode, 0: off; 1: on, default value 0| +|queryPlannerTrace | |Supported, effective immediately |Internal parameter, whether the query plan outputs detailed logs| +|queryNodeChunkSize | |Supported, effective immediately |Internal parameter, chunk size of the query plan| +|queryUseNodeAllocator | |Supported, effective immediately |Internal parameter, allocation method of the query plan| +|queryMaxConcurrentTables| |Not supported |Internal parameter, concurrency number of the query plan| +|queryRsmaTolerance | |Not supported |Internal parameter, tolerance time for determining which level of rsma data to query, in milliseconds| +|enableQueryHb | |Supported, effective immediately |Internal parameter, whether to send query heartbeat messages| +|pqSortMemThreshold | |Not supported |Internal parameter, memory threshold for sorting| ### Region Related -|Parameter Name|Supported Version|Description| -|-----------------|----------|-| -|timezone | |Time zone; defaults to dynamically obtaining the current time zone setting from the system| -|locale | |System locale information and encoding format, defaults to obtaining from the system| -|charset | |Character set encoding, defaults to obtaining from the system| +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +|timezone | |Not supported |Time zone; defaults to dynamically obtaining the current time zone setting from the system| +|locale | |Not supported |System locale information and encoding format, defaults to obtaining from the system| +|charset | |Not supported |Character set encoding, defaults to obtaining from the system| :::info @@ -167,152 +169,153 @@ The effective value of charset is UTF-8. ### Storage Related -|Parameter Name|Supported Version|Description| -|--------------------|----------|-| -|dataDir | |Directory for data files, all data files are written to this directory, default value /var/lib/taos| -|tempDir | |Specifies the directory for generating temporary files during system operation, default value /tmp| -|minimalDataDirGB | |Minimum space to be reserved in the time-series data storage directory specified by dataDir, in GB, default value 2| -|minimalTmpDirGB | |Minimum space to be reserved in the temporary file directory specified by tempDir, in GB, default value 1| -|minDiskFreeSize |After 3.1.1.0|When the available space on a disk is less than or equal to this threshold, the disk will no longer be selected for generating new data files, unit is bytes, range 52428800-1073741824, default value 52428800; Enterprise parameter| -|s3MigrateIntervalSec|After 3.3.4.3|Trigger cycle for automatic upload of local data files to S3, in seconds. Minimum: 600; Maximum: 100000. Default value 3600; Enterprise parameter| -|s3MigrateEnabled |After 3.3.4.3|Whether to automatically perform S3 migration, default value is 0, which means auto S3 migration is off, can be set to 1; Enterprise parameter| -|s3Accesskey |After 3.3.4.3|Colon-separated user SecretId:SecretKey, for example AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E; Enterprise parameter| -|s3Endpoint |After 3.3.4.3|COS service domain name in the user's region, supports http and https, the region of the bucket must match the endpoint, otherwise it cannot be accessed; Enterprise parameter| -|s3BucketName |After 3.3.4.3|Bucket name, followed by a hyphen and the AppId of the user registered COS service, where AppId is unique to COS, not present in AWS and Alibaba Cloud, needs to be part of the bucket name, separated by a hyphen; parameter values are string type, but do not need quotes; for example test0711-1309024725; Enterprise parameter| -|s3PageCacheSize |After 3.3.4.3|Number of S3 page cache pages, range 4-1048576, unit is pages, default value 4096; Enterprise parameter| -|s3UploadDelaySec |After 3.3.4.3|How long a data file remains unchanged before being uploaded to S3, range 1-2592000 (30 days), in seconds, default value 60; Enterprise parameter| -|cacheLazyLoadThreshold| |Internal parameter, cache loading strategy| +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +|dataDir | |Not supported |Directory for data files, all data files are written to this directory, default value /var/lib/taos| +|tempDir | |Not supported |Specifies the directory for generating temporary files during system operation, default value /tmp| +|minimalDataDirGB | |Not supported |Minimum space to be reserved in the time-series data storage directory specified by dataDir, in GB, default value 2| +|minimalTmpDirGB | |Not supported |Minimum space to be reserved in the temporary file directory specified by tempDir, in GB, default value 1| +|minDiskFreeSize |After 3.1.1.0|Supported, effective immediately |When the available space on a disk is less than or equal to this threshold, the disk will no longer be selected for generating new data files, unit is bytes, range 52428800-1073741824, default value 52428800; Enterprise parameter| +|s3MigrateIntervalSec|After 3.3.4.3|Supported, effective immediately |Trigger cycle for automatic upload of local data files to S3, in seconds. Minimum: 600; Maximum: 100000. Default value 3600; Enterprise parameter| +|s3MigrateEnabled |After 3.3.4.3|Supported, effective immediately |Whether to automatically perform S3 migration, default value is 0, which means auto S3 migration is off, can be set to 1; Enterprise parameter| +|s3Accesskey |After 3.3.4.3|Supported, effective after restart|Colon-separated user SecretId:SecretKey, for example AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E; Enterprise parameter| +|s3Endpoint |After 3.3.4.3|Supported, effective after restart|COS service domain name in the user's region, supports http and https, the region of the bucket must match the endpoint, otherwise it cannot be accessed; Enterprise parameter| +|s3BucketName |After 3.3.4.3|Supported, effective after restart|Bucket name, followed by a hyphen and the AppId of the user registered COS service, where AppId is unique to COS, not present in AWS and Alibaba Cloud, needs to be part of the bucket name, separated by a hyphen; parameter values are string type, but do not need quotes; for example test0711-1309024725; Enterprise parameter| +|s3PageCacheSize |After 3.3.4.3|Supported, effective after restart|Number of S3 page cache pages, range 4-1048576, unit is pages, default value 4096; Enterprise parameter| +|s3UploadDelaySec |After 3.3.4.3|Supported, effective immediately |How long a data file remains unchanged before being uploaded to S3, range 1-2592000 (30 days), in seconds, default value 60; Enterprise parameter| +|cacheLazyLoadThreshold| |Supported, effective immediately |Internal parameter, cache loading strategy| ### Cluster Related -|Parameter Name|Supported Version|Description| -|--------------------------|----------|-| -|supportVnodes | |Maximum number of vnodes supported by a dnode, range 0-4096, default value is twice the number of CPU cores + 5| -|numOfCommitThreads | |Maximum number of commit threads, range 0-1024, default value 4| -|numOfMnodeReadThreads | |Number of Read threads for mnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| -|numOfVnodeQueryThreads | |Number of Query threads for vnode, range 0-1024, default value is twice the number of CPU cores (not exceeding 16)| -|numOfVnodeFetchThreads | |Number of Fetch threads for vnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| -|numOfVnodeRsmaThreads | |Number of Rsma threads for vnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| -|numOfQnodeQueryThreads | |Number of Query threads for qnode, range 0-1024, default value is twice the number of CPU cores (not exceeding 16)| -|numOfSnodeSharedThreads | |Number of shared threads for snode, range 0-1024, default value is one quarter of the CPU cores (not less than 2, not exceeding 4)| -|numOfSnodeUniqueThreads | |Number of exclusive threads for snode, range 0-1024, default value is one quarter of the CPU cores (not less than 2, not exceeding 4)| -|ratioOfVnodeStreamThreads | |Ratio of stream computing using vnode threads, range 0.01-4, default value 4| -|ttlUnit | |Unit for ttl parameter, range 1-31572500, in seconds, default value 86400| -|ttlPushInterval | |Frequency of ttl timeout checks, range 1-100000, in seconds, default value 10| -|ttlChangeOnWrite | |Whether ttl expiration time changes with table modification; 0: no change, 1: change; default value 0| -|ttlBatchDropNum | |Number of subtables deleted in a batch for ttl, minimum value 0, default value 10000| -|retentionSpeedLimitMB | |Speed limit for data migration across different levels of disks, range 0-1024, in MB, default value 0, which means no limit| -|maxTsmaNum | |Maximum number of TSMAs that can be created in the cluster; range 0-3; default value 3| -|tmqMaxTopicNum | |Maximum number of topics that can be established for subscription; range 1-10000; default value 20| -|tmqRowSize | |Maximum number of records in a subscription data block, range 1-1000000, default value 4096| -|audit | |Audit feature switch; Enterprise parameter| -|auditInterval | |Time interval for reporting audit data; Enterprise parameter| -|auditCreateTable | |Whether to enable audit feature for creating subtables; Enterprise parameter| -|encryptAlgorithm | |Data encryption algorithm; Enterprise parameter| -|encryptScope | |Encryption scope; Enterprise parameter| -|enableWhiteList | |Switch for whitelist feature; Enterprise parameter| -|syncLogBufferMemoryAllowed| |Maximum memory allowed for sync log cache messages for a dnode, in bytes, range 104857600-INT64_MAX, default value is 1/10 of server memory, effective from versions 3.1.3.2/3.3.2.13| -|syncElectInterval | |Internal parameter, for debugging synchronization module| -|syncHeartbeatInterval | |Internal parameter, for debugging synchronization module| -|syncHeartbeatTimeout | |Internal parameter, for debugging synchronization module| -|syncSnapReplMaxWaitN | |Internal parameter, for debugging synchronization module| -|syncSnapReplMaxWaitN | |Internal parameter, for debugging synchronization module| -|arbHeartBeatIntervalSec | |Internal parameter, for debugging synchronization module| -|arbCheckSyncIntervalSec | |Internal parameter, for debugging synchronization module| -|arbSetAssignedTimeoutSec | |Internal parameter, for debugging synchronization module| -|mndSdbWriteDelta | |Internal parameter, for debugging mnode module| -|mndLogRetention | |Internal parameter, for debugging mnode module| -|skipGrant | |Internal parameter, for authorization checks| -|trimVDbIntervalSec | |Internal parameter, for deleting expired data| -|ttlFlushThreshold | |Internal parameter, frequency of ttl timer| -|compactPullupInterval | |Internal parameter, frequency of data reorganization timer| -|walFsyncDataSizeLimit | |Internal parameter, threshold for WAL to perform FSYNC| -|transPullupInterval | |Internal parameter, retry interval for mnode to execute transactions| -|mqRebalanceInterval | |Internal parameter, interval for consumer rebalancing| -|uptimeInterval | |Internal parameter, for recording system uptime| -|timeseriesThreshold | |Internal parameter, for usage statistics| -|udf | |Whether to start UDF service; 0: do not start, 1: start; default value 0 | -|udfdResFuncs | |Internal parameter, for setting UDF result sets| -|udfdLdLibPath | |Internal parameter, indicates the library path for loading UDF| +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +|supportVnodes | |Supported, effective immediately |Maximum number of vnodes supported by a dnode, range 0-4096, default value is twice the number of CPU cores + 5| +|numOfCommitThreads | |Supported, effective after restart|Maximum number of commit threads, range 0-1024, default value 4| +|numOfMnodeReadThreads | |Supported, effective after restart|Number of Read threads for mnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| +|numOfVnodeQueryThreads | |Supported, effective after restart|Number of Query threads for vnode, range 0-1024, default value is twice the number of CPU cores (not exceeding 16)| +|numOfVnodeFetchThreads | |Supported, effective after restart|Number of Fetch threads for vnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| +|numOfVnodeRsmaThreads | |Supported, effective after restart|Number of Rsma threads for vnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| +|numOfQnodeQueryThreads | |Supported, effective after restart|Number of Query threads for qnode, range 0-1024, default value is twice the number of CPU cores (not exceeding 16)| +|numOfSnodeSharedThreads | |Supported, effective after restart|Number of shared threads for snode, range 0-1024, default value is one quarter of the CPU cores (not less than 2, not exceeding 4)| +|numOfSnodeUniqueThreads | |Supported, effective after restart|Number of exclusive threads for snode, range 0-1024, default value is one quarter of the CPU cores (not less than 2, not exceeding 4)| +|ratioOfVnodeStreamThreads | |Supported, effective after restart|Ratio of stream computing using vnode threads, range 0.01-4, default value 4| +|ttlUnit | |Not supported |Unit for ttl parameter, range 1-31572500, in seconds, default value 86400| +|ttlPushInterval | |Supported, effective immediately |Frequency of ttl timeout checks, range 1-100000, in seconds, default value 10| +|ttlChangeOnWrite | |Supported, effective immediately |Whether ttl expiration time changes with table modification; 0: no change, 1: change; default value 0| +|ttlBatchDropNum | |Supported, effective immediately |Number of subtables deleted in a batch for ttl, minimum value 0, default value 10000| +|retentionSpeedLimitMB | |Supported, effective immediately |Speed limit for data migration across different levels of disks, range 0-1024, in MB, default value 0, which means no limit| +|maxTsmaNum | |Supported, effective immediately |Maximum number of TSMAs that can be created in the cluster; range 0-3; default value 3| +|tmqMaxTopicNum | |Supported, effective immediately |Maximum number of topics that can be established for subscription; range 1-10000; default value 20| +|tmqRowSize | |Supported, effective immediately |Maximum number of records in a subscription data block, range 1-1000000, default value 4096| +|audit | |Supported, effective immediately |Audit feature switch; Enterprise parameter| +|auditInterval | |Supported, effective immediately |Time interval for reporting audit data; Enterprise parameter| +|auditCreateTable | |Supported, effective immediately |Whether to enable audit feature for creating subtables; Enterprise parameter| +|encryptAlgorithm | |Not supported |Data encryption algorithm; Enterprise parameter| +|encryptScope | |Not supported |Encryption scope; Enterprise parameter| +|enableWhiteList | |Supported, effective immediately |Switch for whitelist feature; Enterprise parameter| +|syncLogBufferMemoryAllowed| |Supported, effective immediately |Maximum memory allowed for sync log cache messages for a dnode, in bytes, range 104857600-INT64_MAX, default value is 1/10 of server memory, effective from versions 3.1.3.2/3.3.2.13| +|syncElectInterval | |Not supported |Internal parameter, for debugging synchronization module| +|syncHeartbeatInterval | |Not supported |Internal parameter, for debugging synchronization module| +|syncHeartbeatTimeout | |Not supported |Internal parameter, for debugging synchronization module| +|syncSnapReplMaxWaitN | |Supported, effective immediately |Internal parameter, for debugging synchronization module| +|arbHeartBeatIntervalSec | |Supported, effective immediately |Internal parameter, for debugging synchronization module| +|arbCheckSyncIntervalSec | |Supported, effective immediately |Internal parameter, for debugging synchronization module| +|arbSetAssignedTimeoutSec | |Supported, effective immediately |Internal parameter, for debugging synchronization module| +|mndSdbWriteDelta | |Supported, effective immediately |Internal parameter, for debugging mnode module| +|mndLogRetention | |Supported, effective immediately |Internal parameter, for debugging mnode module| +|skipGrant | |Not supported |Internal parameter, for authorization checks| +|trimVDbIntervalSec | |Supported, effective immediately |Internal parameter, for deleting expired data| +|ttlFlushThreshold | |Supported, effective immediately |Internal parameter, frequency of ttl timer| +|compactPullupInterval | |Supported, effective immediately |Internal parameter, frequency of data reorganization timer| +|walFsyncDataSizeLimit | |Supported, effective immediately |Internal parameter, threshold for WAL to perform FSYNC| +|transPullupInterval | |Supported, effective immediately |Internal parameter, retry interval for mnode to execute transactions| +|mqRebalanceInterval | |Supported, effective immediately |Internal parameter, interval for consumer rebalancing| +|uptimeInterval | |Supported, effective immediately |Internal parameter, for recording system uptime| +|timeseriesThreshold | |Supported, effective immediately |Internal parameter, for usage statistics| +|udf | |Supported, effective after restart|Whether to start UDF service; 0: do not start, 1: start; default value 0 | +|udfdResFuncs | |Supported, effective after restart|Internal parameter, for setting UDF result sets| +|udfdLdLibPath | |Supported, effective after restart|Internal parameter, indicates the library path for loading UDF| ### Stream Computing Parameters -| Parameter Name | Supported Version | Description | -|-----------------------|----------|-| -| disableStream | | Switch to enable or disable stream computing | -| streamBufferSize | | Controls the size of the window state cache in memory, default value is 128MB | -| streamAggCnt | | Internal parameter, number of concurrent aggregation computations | -| checkpointInterval | | Internal parameter, checkpoint synchronization interval | -| concurrentCheckpoint | | Internal parameter, whether to check checkpoints concurrently | -| maxStreamBackendCache | | Internal parameter, maximum cache used by stream computing | -| streamSinkDataRate | | Internal parameter, used to control the write speed of stream computing results | +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +| disableStream | |Supported, effective immediately | Switch to enable or disable stream computing | +| streamBufferSize | |Supported, effective immediately | Controls the size of the window state cache in memory, default value is 128MB | +| streamAggCnt | |Not supported | Internal parameter, number of concurrent aggregation computations | +| checkpointInterval | |Supported, effective after restart| Internal parameter, checkpoint synchronization interval | +| concurrentCheckpoint | |Supported, effective immediately | Internal parameter, whether to check checkpoints concurrently | +| maxStreamBackendCache | |Supported, effective immediately | Internal parameter, maximum cache used by stream computing | +| streamSinkDataRate | |Supported, effective after restart| Internal parameter, used to control the write speed of stream computing results | ### Log Related -| Parameter Name | Supported Version | Description | -|----------------|----------|-| -| logDir | | Log file directory, operational logs will be written to this directory, default value /var/log/taos | -| minimalLogDirGB | | Stops writing logs when the available space on the disk where the log folder is located is less than this value, unit GB, default value 1 | -| numOfLogLines | | Maximum number of lines allowed in a single log file, default value 10,000,000 | -| asyncLog | | Log writing mode, 0: synchronous, 1: asynchronous, default value 1 | -| logKeepDays | | Maximum retention time for log files, unit: days, default value 0, which means unlimited retention, log files will not be renamed, nor will new log files be rolled out, but the content of the log files may continue to roll depending on the log file size setting; when set to a value greater than 0, when the log file size reaches the set limit, it will be renamed to taosdlog.yyy, where yyy is the timestamp of the last modification of the log file, and a new log file will be rolled out | -| slowLogThreshold| 3.3.3.0 onwards | Slow query threshold, queries taking longer than or equal to this threshold are considered slow, unit seconds, default value 3 | -| slowLogMaxLen | 3.3.3.0 onwards | Maximum length of slow query logs, range 1-16384, default value 4096 | -| slowLogScope | 3.3.3.0 onwards | Type of slow query records, range ALL/QUERY/INSERT/OTHERS/NONE, default value QUERY | -| slowLogExceptDb | 3.3.3.0 onwards | Specifies the database that does not report slow queries, only supports configuring one database | -| debugFlag | | Log switch for running logs, 131 (outputs error and warning logs), 135 (outputs error, warning, and debug logs), 143 (outputs error, warning, debug, and trace logs); default value 131 or 135 (depending on the module) | -| tmrDebugFlag | | Log switch for the timer module, range as above | -| uDebugFlag | | Log switch for the utility module, range as above | -| rpcDebugFlag | | Log switch for the rpc module, range as above | -| qDebugFlag | | Log switch for the query module, range as above | -| dDebugFlag | | Log switch for the dnode module, range as above | -| vDebugFlag | | Log switch for the vnode module, range as above | -| mDebugFlag | | Log switch for the mnode module, range as above | -| azDebugFlag | 3.3.4.3 onwards | Log switch for the S3 module, range as above | -| sDebugFlag | | Log switch for the sync module, range as above | -| tsdbDebugFlag | | Log switch for the tsdb module, range as above | -| tqDebugFlag | | Log switch for the tq module, range as above | -| fsDebugFlag | | Log switch for the fs module, range as above | -| udfDebugFlag | | Log switch for the udf module, range as above | -| smaDebugFlag | | Log switch for the sma module, range as above | -| idxDebugFlag | | Log switch for the index module, range as above | -| tdbDebugFlag | | Log switch for the tdb module, range as above | -| metaDebugFlag | | Log switch for the meta module, range as above | -| stDebugFlag | | Log switch for the stream module, range as above | -| sndDebugFlag | | Log switch for the snode module, range as above | +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +| logDir | |Not supported | Log file directory, operational logs will be written to this directory, default value /var/log/taos | +| minimalLogDirGB | |Not supported | Stops writing logs when the available space on the disk where the log folder is located is less than this value, unit GB, default value 1 | +| numOfLogLines | |Supported, effective immediately | Maximum number of lines allowed in a single log file, default value 10,000,000 | +| asyncLog | |Supported, effective immediately | Log writing mode, 0: synchronous, 1: asynchronous, default value 1 | +| logKeepDays | |Supported, effective immediately | Maximum retention time for log files, unit: days, default value 0, which means unlimited retention, log files will not be renamed, nor will new log files be rolled out, but the content of the log files may continue to roll depending on the log file size setting; when set to a value greater than 0, when the log file size reaches the set limit, it will be renamed to taosdlog.yyy, where yyy is the timestamp of the last modification of the log file, and a new log file will be rolled out | +| slowLogThreshold| 3.3.3.0 onwards |Supported, effective immediately | Slow query threshold, queries taking longer than or equal to this threshold are considered slow, unit seconds, default value 3 | +| slowLogMaxLen | 3.3.3.0 onwards |Supported, effective immediately | Maximum length of slow query logs, range 1-16384, default value 4096 | +| slowLogScope | 3.3.3.0 onwards |Supported, effective immediately | Type of slow query records, range ALL/QUERY/INSERT/OTHERS/NONE, default value QUERY | +| slowLogExceptDb | 3.3.3.0 onwards |Supported, effective immediately | Specifies the database that does not report slow queries, only supports configuring one database | +| debugFlag | |Supported, effective immediately | Log switch for running logs, 131 (outputs error and warning logs), 135 (outputs error, warning, and debug logs), 143 (outputs error, warning, debug, and trace logs); default value 131 or 135 (depending on the module) | +| tmrDebugFlag | |Supported, effective immediately | Log switch for the timer module, range as above | +| uDebugFlag | |Supported, effective immediately | Log switch for the utility module, range as above | +| rpcDebugFlag | |Supported, effective immediately | Log switch for the rpc module, range as above | +| qDebugFlag | |Supported, effective immediately | Log switch for the query module, range as above | +| dDebugFlag | |Supported, effective immediately | Log switch for the dnode module, range as above | +| vDebugFlag | |Supported, effective immediately | Log switch for the vnode module, range as above | +| mDebugFlag | |Supported, effective immediately | Log switch for the mnode module, range as above | +| azDebugFlag | 3.3.4.3 onwards |Supported, effective immediately | Log switch for the S3 module, range as above | +| sDebugFlag | |Supported, effective immediately | Log switch for the sync module, range as above | +| tsdbDebugFlag | |Supported, effective immediately | Log switch for the tsdb module, range as above | +| tqDebugFlag | |Supported, effective immediately | Log switch for the tq module, range as above | +| fsDebugFlag | |Supported, effective immediately | Log switch for the fs module, range as above | +| udfDebugFlag | |Supported, effective immediately | Log switch for the udf module, range as above | +| smaDebugFlag | |Supported, effective immediately | Log switch for the sma module, range as above | +| idxDebugFlag | |Supported, effective immediately | Log switch for the index module, range as above | +| tdbDebugFlag | |Supported, effective immediately | Log switch for the tdb module, range as above | +| metaDebugFlag | |Supported, effective immediately | Log switch for the meta module, range as above | +| stDebugFlag | |Supported, effective immediately | Log switch for the stream module, range as above | +| sndDebugFlag | |Supported, effective immediately | Log switch for the snode module, range as above | ### Debugging Related -| Parameter Name | Supported Version | Description | -|----------------------|-------------------|-------------| -| enableCoreFile | | Whether to generate a core file when crashing, 0: do not generate, 1: generate; default value is 1 | -| configDir | | Directory where the configuration files are located | -| scriptDir | | Directory for internal test tool scripts | -| assert | | Assertion control switch, default value is 0 | -| randErrorChance | | Internal parameter, used for random failure testing | -| randErrorDivisor | | Internal parameter, used for random failure testing | -| randErrorScope | | Internal parameter, used for random failure testing | -| safetyCheckLevel | | Internal parameter, used for random failure testing | -| experimental | | Internal parameter, used for some experimental features | -| simdEnable | After 3.3.4.3 | Internal parameter, used for testing SIMD acceleration | -| AVX512Enable | After 3.3.4.3 | Internal parameter, used for testing AVX512 acceleration | -| rsyncPort | | Internal parameter, used for debugging stream computing | -| snodeAddress | | Internal parameter, used for debugging stream computing | -| checkpointBackupDir | | Internal parameter, used for restoring snode data | -| enableAuditDelete | | Internal parameter, used for testing audit functions | -| slowLogThresholdTest | | Internal parameter, used for testing slow logs | +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +| enableCoreFile | |Supported, effective immediately | Whether to generate a core file when crashing, 0: do not generate, 1: generate; default value is 1 | +| configDir | |Not supported | Directory where the configuration files are located | +|forceReadConfig | |Not supported ||Force the use of parameters from the configuration file,default value: 0| +| scriptDir | |Not supported | Directory for internal test tool scripts | +| assert | |Not supported | Assertion control switch, default value is 0 | +| randErrorChance | |Supported, effective immediately | Internal parameter, used for random failure testing | +| randErrorDivisor | |Supported, effective immediately | Internal parameter, used for random failure testing | +| randErrorScope | |Supported, effective immediately | Internal parameter, used for random failure testing | +| safetyCheckLevel | |Supported, effective immediately | Internal parameter, used for random failure testing | +| experimental | |Supported, effective immediately | Internal parameter, used for some experimental features | +| simdEnable | After 3.3.4.3 |Not supported | Internal parameter, used for testing SIMD acceleration | +| AVX512Enable | After 3.3.4.3 |Not supported | Internal parameter, used for testing AVX512 acceleration | +| rsyncPort | |Not supported | Internal parameter, used for debugging stream computing | +| snodeAddress | |Supported, effective immediately | Internal parameter, used for debugging stream computing | +| checkpointBackupDir | |Supported, effective immediately | Internal parameter, used for restoring snode data | +| enableAuditDelete | |Not supported | Internal parameter, used for testing audit functions | +| slowLogThresholdTest | |Not supported | Internal parameter, used for testing slow logs | +| bypassFlag |After 3.3.4.5 |Supported, effective immediately | Internal parameter, used for short-circuit testing| ### Compression Parameters -| Parameter Name | Supported Version | Description | -|----------------|-------------------|-------------| -| fPrecision | | Sets the compression precision for float type floating numbers, range 0.1 ~ 0.00000001, default value 0.00000001, floating numbers smaller than this value will have their mantissa truncated | -| dPrecision | | Sets the compression precision for double type floating numbers, range 0.1 ~ 0.0000000000000001, default value 0.0000000000000001, floating numbers smaller than this value will have their mantissa truncated | -| lossyColumn | Before 3.3.0.0 | Enables TSZ lossy compression for float and/or double types; range float/double/none; default value none, indicating lossless compression is off | -| ifAdtFse | | When TSZ lossy compression is enabled, use the FSE algorithm instead of the HUFFMAN algorithm, FSE algorithm is faster in compression but slightly slower in decompression, choose this for faster compression speed; 0: off, 1: on; default value is 0 | -| maxRange | | Internal parameter, used for setting lossy compression | -| curRange | | Internal parameter, used for setting lossy compression | -| compressor | | Internal parameter, used for setting lossy compression | +|Parameter Name |Supported Version |Dynamic Modification|Description| +|-----------------------|-------------------------|--------------------|------------| +| fPrecision | |Supported, effective immediately | Sets the compression precision for float type floating numbers, range 0.1 ~ 0.00000001, default value 0.00000001, floating numbers smaller than this value will have their mantissa truncated | +| dPrecision | |Supported, effective immediately | Sets the compression precision for double type floating numbers, range 0.1 ~ 0.0000000000000001, default value 0.0000000000000001, floating numbers smaller than this value will have their mantissa truncated | +| lossyColumn | Before 3.3.0.0 |Not supported | Enables TSZ lossy compression for float and/or double types; range float/double/none; default value none, indicating lossless compression is off | +| ifAdtFse | |Supported, effective after restart| When TSZ lossy compression is enabled, use the FSE algorithm instead of the HUFFMAN algorithm, FSE algorithm is faster in compression but slightly slower in decompression, choose this for faster compression speed; 0: off, 1: on; default value is 0 | +| maxRange | |Supported, effective after restart| Internal parameter, used for setting lossy compression | +| curRange | |Supported, effective after restart| Internal parameter, used for setting lossy compression | +| compressor | |Supported, effective after restart| Internal parameter, used for setting lossy compression | **Additional Notes** diff --git a/docs/en/14-reference/01-components/02-taosc.md b/docs/en/14-reference/01-components/02-taosc.md index 46ee43ce81..7b8d3ae78c 100644 --- a/docs/en/14-reference/01-components/02-taosc.md +++ b/docs/en/14-reference/01-components/02-taosc.md @@ -10,107 +10,109 @@ The TDengine client driver provides all the APIs needed for application programm ### Connection Related -|Parameter Name|Supported Version|Description| -|----------------------|----------|-------------| -|firstEp | |At startup, the endpoint of the first dnode in the cluster to actively connect to, default value: hostname:6030, if the server's hostname cannot be obtained, it is assigned to localhost| -|secondEp | |At startup, if the firstEp cannot be connected, try to connect to the endpoint of the second dnode in the cluster, no default value| -|compressMsgSize | |Whether to compress RPC messages; -1: no messages are compressed; 0: all messages are compressed; N (N>0): only messages larger than N bytes are compressed; default value -1| -|shellActivityTimer | |The duration in seconds for the client to send heartbeats to mnode, range 1-120, default value 3| -|numOfRpcSessions | |Maximum number of connections supported by RPC, range 100-100000, default value 30000| -|numOfRpcThreads | |Number of threads for RPC to send and receive data, range 1-50, default value is half of the CPU cores| -|numOfTaskQueueThreads | |Number of threads for the client to handle RPC messages, range 4-16, default value is half of the CPU cores| -|timeToGetAvailableConn| Cancelled after 3.3.4.* |The longest waiting time to get an available connection, range 10-50000000, in milliseconds, default value 500000| -|useAdapter | |Internal parameter, whether to use taosadapter, affects CSV file import| -|shareConnLimit |Added in 3.3.4.0|Internal parameter, the number of queries a link can share, range 1-256, default value 10| -|readTimeout |Added in 3.3.4.0|Internal parameter, minimum timeout, range 64-604800, in seconds, default value 900| +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +|firstEp | |Supported, effective immediately |At startup, the endpoint of the first dnode in the cluster to actively connect to, default value: hostname:6030, if the server's hostname cannot be obtained, it is assigned to localhost| +|secondEp | |Supported, effective immediately |At startup, if the firstEp cannot be connected, try to connect to the endpoint of the second dnode in the cluster, no default value| +|compressMsgSize | |Supported, effective immediately |Whether to compress RPC messages; -1: no messages are compressed; 0: all messages are compressed; N (N>0): only messages larger than N bytes are compressed; default value -1| +|shellActivityTimer | |Not supported |The duration in seconds for the client to send heartbeats to mnode, range 1-120, default value 3| +|numOfRpcSessions | |Supported, effective immediately |Maximum number of connections supported by RPC, range 100-100000, default value 30000| +|numOfRpcThreads | |Not supported |Number of threads for RPC to send and receive data, range 1-50, default value is half of the CPU cores| +|numOfTaskQueueThreads | |Not supported |Number of threads for the client to handle RPC messages, range 4-16, default value is half of the CPU cores| +|timeToGetAvailableConn| Cancelled after 3.3.4.* |Not supported |The longest waiting time to get an available connection, range 10-50000000, in milliseconds, default value 500000| +|useAdapter | |Supported, effective immediately |Internal parameter, whether to use taosadapter, affects CSV file import| +|shareConnLimit |Added in 3.3.4.0|Not supported |Internal parameter, the number of queries a link can share, range 1-256, default value 10| +|readTimeout |Added in 3.3.4.0|Not supported |Internal parameter, minimum timeout, range 64-604800, in seconds, default value 900| ### Query Related -|Parameter Name|Supported Version|Description| -|---------------------------------|---------|-| -|countAlwaysReturnValue | |Whether the count/hyperloglog function returns a value when the input data is empty or NULL; 0: returns an empty row, 1: returns; default value 1; when this parameter is set to 1, if the query contains an INTERVAL clause or the query uses TSMA, and the corresponding group or window has empty or NULL data, the corresponding group or window will not return a query result; note that this parameter should be consistent between client and server| -|keepColumnName | |Automatically sets the alias to the column name (excluding the function name) when querying with Last, First, LastRow functions without specifying an alias, thus the order by clause will automatically refer to the column corresponding to the function; 1: automatically sets the alias to the column name (excluding the function name), 0: does not automatically set an alias; default value: 0| -|multiResultFunctionStarReturnTags|After 3.3.3.0|When querying a supertable, whether last(\*)/last_row(\*)/first(\*) returns tag columns; when querying basic tables, subtables, it is not affected by this parameter; 0: does not return tag columns, 1: returns tag columns; default value: 0; when this parameter is set to 0, last(\*)/last_row(\*)/first(\*) only returns the ordinary columns of the supertable; when set to 1, it returns both the ordinary columns and tag columns of the supertable| -|metaCacheMaxSize | |Specifies the maximum size of metadata cache for a single client, in MB; default value -1, meaning unlimited| -|maxTsmaCalcDelay | |The allowable delay for tsma calculation by the client during query, range 600s - 86400s, i.e., 10 minutes - 1 day; default value: 600 seconds| -|tsmaDataDeleteMark | |The retention time for intermediate results of historical data calculated by TSMA, in milliseconds; range >= 3600000, i.e., at least 1h; default value: 86400000, i.e., 1d | -|queryPolicy | |Execution strategy for query statements, 1: only use vnode, do not use qnode; 2: subtasks without scan operators are executed on qnode, subtasks with scan operators are executed on vnode; 3: vnode only runs scan operators, all other operators are executed on qnode; default value: 1| -|queryTableNotExistAsEmpty | |Whether to return an empty result set when the queried table does not exist; false: returns an error; true: returns an empty result set; default value false| -|querySmaOptimize | |Optimization strategy for sma index, 0: do not use sma index, always query from original data; 1: use sma index, directly query from pre-calculated results for eligible statements; default value: 0| -|queryPlannerTrace | |Internal parameter, whether the query plan outputs detailed logs| -|queryNodeChunkSize | |Internal parameter, chunk size of the query plan| -|queryUseNodeAllocator | |Internal parameter, allocation method of the query plan| -|queryMaxConcurrentTables | |Internal parameter, concurrency number of the query plan| -|enableQueryHb | |Internal parameter, whether to send query heartbeat messages| -|minSlidingTime | |Internal parameter, minimum allowable value for sliding| -|minIntervalTime | |Internal parameter, minimum allowable value for interval| +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +|countAlwaysReturnValue | |Supported, effective immediately |Whether the count/hyperloglog function returns a value when the input data is empty or NULL; 0: returns an empty row, 1: returns; default value 1; when this parameter is set to 1, if the query contains an INTERVAL clause or the query uses TSMA, and the corresponding group or window has empty or NULL data, the corresponding group or window will not return a query result; note that this parameter should be consistent between client and server| +|keepColumnName | |Supported, effective immediately |Automatically sets the alias to the column name (excluding the function name) when querying with Last, First, LastRow functions without specifying an alias, thus the order by clause will automatically refer to the column corresponding to the function; 1: automatically sets the alias to the column name (excluding the function name), 0: does not automatically set an alias; default value: 0| +|multiResultFunctionStarReturnTags|After 3.3.3.0|Supported, effective immediately |When querying a supertable, whether last(\*)/last_row(\*)/first(\*) returns tag columns; when querying basic tables, subtables, it is not affected by this parameter; 0: does not return tag columns, 1: returns tag columns; default value: 0; when this parameter is set to 0, last(\*)/last_row(\*)/first(\*) only returns the ordinary columns of the supertable; when set to 1, it returns both the ordinary columns and tag columns of the supertable| +|metaCacheMaxSize | |Supported, effective immediately |Specifies the maximum size of metadata cache for a single client, in MB; default value -1, meaning unlimited| +|maxTsmaCalcDelay | |Supported, effective immediately |The allowable delay for tsma calculation by the client during query, range 600s - 86400s, i.e., 10 minutes - 1 day; default value: 600 seconds| +|tsmaDataDeleteMark | |Supported, effective immediately |The retention time for intermediate results of historical data calculated by TSMA, in milliseconds; range >= 3600000, i.e., at least 1h; default value: 86400000, i.e., 1d | +|queryPolicy | |Supported, effective immediately |Execution strategy for query statements, 1: only use vnode, do not use qnode; 2: subtasks without scan operators are executed on qnode, subtasks with scan operators are executed on vnode; 3: vnode only runs scan operators, all other operators are executed on qnode; default value: 1| +|queryTableNotExistAsEmpty | |Supported, effective immediately |Whether to return an empty result set when the queried table does not exist; false: returns an error; true: returns an empty result set; default value false| +|querySmaOptimize | |Supported, effective immediately |Optimization strategy for sma index, 0: do not use sma index, always query from original data; 1: use sma index, directly query from pre-calculated results for eligible statements; default value: 0| +|queryPlannerTrace | |Supported, effective immediately |Internal parameter, whether the query plan outputs detailed logs| +|queryNodeChunkSize | |Supported, effective immediately |Internal parameter, chunk size of the query plan| +|queryUseNodeAllocator | |Supported, effective immediately |Internal parameter, allocation method of the query plan| +|queryMaxConcurrentTables | |Not supported |Internal parameter, concurrency number of the query plan| +|enableQueryHb | |Supported, effective immediately |Internal parameter, whether to send query heartbeat messages| +|minSlidingTime | |Supported, effective immediately |Internal parameter, minimum allowable value for sliding| +|minIntervalTime | |Supported, effective immediately |Internal parameter, minimum allowable value for interval| ### Writing Related -| Parameter Name | Supported Version | Description | -|---------------------------------|-------------------|-------------| -| smlChildTableName | | Key for custom child table name in schemaless, no default value | -| smlAutoChildTableNameDelimiter | | Delimiter between schemaless tags, concatenated as the child table name, no default value | -| smlTagName | | Default tag name when schemaless tag is empty, default value "_tag_null" | -| smlTsDefaultName | | Configuration for setting the time column name in schemaless auto table creation, default value "_ts" | -| smlDot2Underline | | Converts dots in supertable names to underscores in schemaless | -| maxInsertBatchRows | | Internal parameter, maximum number of rows per batch insert | +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +| smlChildTableName | |Supported, effective immediately | Key for custom child table name in schemaless, no default value | +| smlAutoChildTableNameDelimiter | |Supported, effective immediately | Delimiter between schemaless tags, concatenated as the child table name, no default value | +| smlTagName | |Supported, effective immediately | Default tag name when schemaless tag is empty, default value "_tag_null" | +| smlTsDefaultName | |Supported, effective immediately | Configuration for setting the time column name in schemaless auto table creation, default value "_ts" | +| smlDot2Underline | |Supported, effective immediately | Converts dots in supertable names to underscores in schemaless | +| maxInsertBatchRows | |Supported, effective immediately | Internal parameter, maximum number of rows per batch insert | ### Region Related -| Parameter Name | Supported Version | Description | -|----------------|-------------------|-------------| -| timezone | | Time zone; defaults to dynamically obtaining the current system time zone setting | -| locale | | System locale and encoding format, defaults to system settings | -| charset | | Character set encoding, defaults to system settings | +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +| timezone | |Supported, effective immediately | Time zone; defaults to dynamically obtaining the current system time zone setting | +| locale | |Supported, effective immediately | System locale and encoding format, defaults to system settings | +| charset | |Supported, effective immediately | Character set encoding, defaults to system settings | ### Storage Related -| Parameter Name | Supported Version | Description | -|-----------------|-------------------|-------------| -| tempDir | | Specifies the directory for generating temporary files during operation, default on Linux platform is /tmp | -| minimalTmpDirGB | | Minimum space required to be reserved in the directory specified by tempDir, in GB, default value: 1 | +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +| tempDir | |Supported, effective immediately | Specifies the directory for generating temporary files during operation, default on Linux platform is /tmp | +| minimalTmpDirGB | |Supported, effective immediately | Minimum space required to be reserved in the directory specified by tempDir, in GB, default value: 1 | ### Log Related -| Parameter Name | Supported Version | Description | -|------------------|-------------------|-------------| -| logDir | | Log file directory, operational logs will be written to this directory, default value: /var/log/taos | -| minimalLogDirGB | | Stops writing logs when the disk space available in the log directory is less than this value, in GB, default value: 1 | -| numOfLogLines | | Maximum number of lines allowed in a single log file, default value: 10,000,000 | -| asyncLog | | Log writing mode, 0: synchronous, 1: asynchronous, default value: 1 | -| logKeepDays | | Maximum retention time for log files, in days, default value: 0, meaning unlimited retention. Log files will not be renamed, nor will new log files be rolled out, but the content of the log files may continue to roll depending on the log file size setting; when set to a value greater than 0, the log file will be renamed to taoslogx.yyy, where yyy is the timestamp of the last modification of the log file, and a new log file will be rolled out | -| debugFlag | | Log switch for running logs, 131 (output error and warning logs), 135 (output error, warning, and debug logs), 143 (output error, warning, debug, and trace logs); default value 131 or 135 (depending on the module) | -| tmrDebugFlag | | Log switch for the timer module, value range as above | -| uDebugFlag | | Log switch for the utility module, value range as above | -| rpcDebugFlag | | Log switch for the rpc module, value range as above | -| jniDebugFlag | | Log switch for the jni module, value range as above | -| qDebugFlag | | Log switch for the query module, value range as above | -| cDebugFlag | | Log switch for the client module, value range as above | -| simDebugFlag | | Internal parameter, log switch for the test tool, value range as above | -| tqClientDebugFlag| After 3.3.4.3 | Log switch for the client module, value range as above | +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +| logDir | |Not supported | Log file directory, operational logs will be written to this directory, default value: /var/log/taos | +| minimalLogDirGB | |Supported, effective immediately | Stops writing logs when the disk space available in the log directory is less than this value, in GB, default value: 1 | +| numOfLogLines | |Supported, effective immediately | Maximum number of lines allowed in a single log file, default value: 10,000,000 | +| asyncLog | |Supported, effective immediately | Log writing mode, 0: synchronous, 1: asynchronous, default value: 1 | +| logKeepDays | |Supported, effective immediately | Maximum retention time for log files, in days, default value: 0, meaning unlimited retention. Log files will not be renamed, nor will new log files be rolled out, but the content of the log files may continue to roll depending on the log file size setting; when set to a value greater than 0, the log file will be renamed to taoslogx.yyy, where yyy is the timestamp of the last modification of the log file, and a new log file will be rolled out | +| debugFlag | |Supported, effective immediately | Log switch for running logs, 131 (output error and warning logs), 135 (output error, warning, and debug logs), 143 (output error, warning, debug, and trace logs); default value 131 or 135 (depending on the module) | +| tmrDebugFlag | |Supported, effective immediately | Log switch for the timer module, value range as above | +| uDebugFlag | |Supported, effective immediately | Log switch for the utility module, value range as above | +| rpcDebugFlag | |Supported, effective immediately | Log switch for the rpc module, value range as above | +| jniDebugFlag | |Supported, effective immediately | Log switch for the jni module, value range as above | +| qDebugFlag | |Supported, effective immediately | Log switch for the query module, value range as above | +| cDebugFlag | |Supported, effective immediately | Log switch for the client module, value range as above | +| simDebugFlag | |Supported, effective immediately | Internal parameter, log switch for the test tool, value range as above | +| tqClientDebugFlag| After 3.3.4.3 |Supported, effective immediately | Log switch for the client module, value range as above | ### Debugging Related -| Parameter Name | Supported Version | Description | -|------------------|-------------------|-------------| -| crashReporting | | Whether to upload crash to telemetry, 0: do not upload, 1: upload; default value: 1 | -| enableCoreFile | | Whether to generate a core file when crashing, 0: do not generate, 1: generate; default value: 1 | -| assert | | Assertion control switch, default value: 0 | -| configDir | | Directory for configuration files | -| scriptDir | | Internal parameter, directory for test cases | -| randErrorChance | After 3.3.3.0 | Internal parameter, used for random failure testing | -| randErrorDivisor | After 3.3.3.0 | Internal parameter, used for random failure testing | -| randErrorScope | After 3.3.3.0 | Internal parameter, used for random failure testing | -| safetyCheckLevel | After 3.3.3.0 | Internal parameter, used for random failure testing | -| simdEnable | After 3.3.4.3 | Internal parameter, used for testing SIMD acceleration | -| AVX512Enable | After 3.3.4.3 | Internal parameter, used for testing AVX512 acceleration | +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +| crashReporting | |Supported, effective immediately | Whether to upload crash to telemetry, 0: do not upload, 1: upload; default value: 1 | +| enableCoreFile | |Supported, effective immediately | Whether to generate a core file when crashing, 0: do not generate, 1: generate; default value: 1 | +| assert | |Not supported | Assertion control switch, default value: 0 | +| configDir | |Not supported | Directory for configuration files | +| scriptDir | |Not supported | Internal parameter, directory for test cases | +| randErrorChance | After 3.3.3.0 |Not supported | Internal parameter, used for random failure testing | +| randErrorDivisor | After 3.3.3.0 |Not supported | Internal parameter, used for random failure testing | +| randErrorScope | After 3.3.3.0 |Not supported | Internal parameter, used for random failure testing | +| safetyCheckLevel | After 3.3.3.0 |Not supported | Internal parameter, used for random failure testing | +| simdEnable | After 3.3.4.3 |Not supported | Internal parameter, used for testing SIMD acceleration | +| AVX512Enable | After 3.3.4.3 |Not supported | Internal parameter, used for testing AVX512 acceleration | +| bypassFlag |After 3.3.4.5 |Supported, effective immediately | Internal parameter, used for short-circuit testing| + ### SHELL Related -|Parameter Name|Supported Version|Description| -|-----------------|----------|-| -|enableScience | |Whether to enable scientific notation for displaying floating numbers; 0: do not enable, 1: enable; default value: 1| +|Parameter Name|Supported Version|Dynamic Modification|Description| +|----------------------|----------|--------------------|-------------| +|enableScience | |Not supported |Whether to enable scientific notation for displaying floating numbers; 0: do not enable, 1: enable; default value: 1| ## API diff --git a/docs/en/14-reference/01-components/04-taosx.md b/docs/en/14-reference/01-components/04-taosx.md index b53d9f6777..8ce16f471d 100644 --- a/docs/en/14-reference/01-components/04-taosx.md +++ b/docs/en/14-reference/01-components/04-taosx.md @@ -31,7 +31,7 @@ The following parameter descriptions and examples use `` as a placehold In command line mode, taosX uses DSN to represent a data source (source or destination), a typical DSN is as follows: -```bash +```shell # url-like [+]://[[:@]:][/][?=[&=]] |------|------------|---|-----------|-----------|------|------|----------|-----------------------| @@ -390,7 +390,7 @@ You can view the log files or use the `journalctl` command to view the logs of ` The command to view logs under Linux using `journalctl` is as follows: -```bash +```shell journalctl -u taosx [-f] ``` @@ -572,7 +572,7 @@ uint32_t len: The binary length of this string (excluding `\0`). **Return Value**: -``` c +```c struct parser_resp_t { int e; // 0 if success. void* p; // Success if contains. @@ -589,7 +589,7 @@ When creation is successful, e = 0, p is the parser object. Parse the input payload and return the result in JSON format [u8]. The returned JSON will be fully decoded using the default JSON parser (expanding the root array and all objects). -``` c +```c const char* parser_mutate( void* parser, const uint8_t* in_ptr, uint32_t in_len, diff --git a/docs/en/14-reference/01-components/05-taosx-agent.md b/docs/en/14-reference/01-components/05-taosx-agent.md index 9353d45de3..3e8b4f4d63 100644 --- a/docs/en/14-reference/01-components/05-taosx-agent.md +++ b/docs/en/14-reference/01-components/05-taosx-agent.md @@ -26,7 +26,7 @@ The default configuration file for `Agent` is located at `/etc/taos/agent.toml`, As shown below: -```TOML +```toml # taosX service endpoint # #endpoint = "http://localhost:6055" @@ -83,7 +83,7 @@ You don't need to be confused about how to set up the configuration file. Read a On Linux systems, the `Agent` can be started with the Systemd command: -```bash +```shell systemctl start taosx-agent ``` @@ -95,6 +95,6 @@ You can view the log files or use the `journalctl` command to view the logs of t The command to view logs with `journalctl` on Linux is as follows: -```bash +```shell journalctl -u taosx-agent [-f] ``` diff --git a/docs/en/14-reference/01-components/06-taoskeeper.md b/docs/en/14-reference/01-components/06-taoskeeper.md index 570cf7305f..78556d888a 100644 --- a/docs/en/14-reference/01-components/06-taoskeeper.md +++ b/docs/en/14-reference/01-components/06-taoskeeper.md @@ -143,13 +143,13 @@ For details on TDengine monitoring configuration, please refer to: [TDengine Mon After installation, please use the `systemctl` command to start the taoskeeper service process. -```bash +```shell systemctl start taoskeeper ``` Check if the service is working properly: -```bash +```shell systemctl status taoskeeper ``` @@ -261,7 +261,7 @@ Query OK, 14 row(s) in set (0.006542s) You can view the most recent report record of a supertable, such as: -``` shell +```shell taos> select last_row(*) from taosd_dnodes_info; last_row(_ts) | last_row(disk_engine) | last_row(system_net_in) | last_row(vnodes_num) | last_row(system_net_out) | last_row(uptime) | last_row(has_mnode) | last_row(io_read_disk) | last_row(error_log_count) | last_row(io_read) | last_row(cpu_cores) | last_row(has_qnode) | last_row(has_snode) | last_row(disk_total) | last_row(mem_engine) | last_row(info_log_count) | last_row(cpu_engine) | last_row(io_write_disk) | last_row(debug_log_count) | last_row(disk_used) | last_row(mem_total) | last_row(io_write) | last_row(masters) | last_row(cpu_system) | last_row(trace_log_count) | last_row(mem_free) | ====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== diff --git a/docs/en/14-reference/01-components/07-explorer.md b/docs/en/14-reference/01-components/07-explorer.md index 726b07d050..a9a2ebb356 100644 --- a/docs/en/14-reference/01-components/07-explorer.md +++ b/docs/en/14-reference/01-components/07-explorer.md @@ -14,7 +14,7 @@ taosExplorer does not require separate installation. Starting from TDengine vers Before starting taosExplorer, please make sure the content in the configuration file is correct. -```TOML +```toml # This is an automatically generated configuration file for Explorer in [TOML](https://toml.io/) format. # # Here is a full list of available options. @@ -148,7 +148,7 @@ Description: Then start taosExplorer, you can directly execute taos-explorer in the command line or use the systemctl command: -```bash +```shell systemctl start taos-explorer # Linux sc.exe start taos-explorer # Windows ``` diff --git a/docs/en/14-reference/01-components/12-tdinsight.mdx b/docs/en/14-reference/01-components/12-tdinsight.md similarity index 71% rename from docs/en/14-reference/01-components/12-tdinsight.mdx rename to docs/en/14-reference/01-components/12-tdinsight.md index 463ceaaca6..c9464760e7 100644 --- a/docs/en/14-reference/01-components/12-tdinsight.mdx +++ b/docs/en/14-reference/01-components/12-tdinsight.md @@ -171,7 +171,37 @@ Metric details: 5. **Writes**: Total number of writes 6. **Other**: Total number of other requests -There are also line charts for the above categories. +There are also line charts for the above categories. + +### Automatic import of preconfigured alert rules + +After summarizing user experience, 14 commonly used alert rules are sorted out. These alert rules can monitor key indicators of the TDengine cluster and report alerts, such as abnormal and exceeded indicators. +Starting from TDengine-Server 3.3.4.3 (TDengine-datasource 3.6.3), TDengine Datasource supports automatic import of preconfigured alert rules. You can import 14 alert rules to Grafana (version 11 or later) with one click. +In the TDengine-datasource setting interface, turn on the "Load Tengine Alert" switch, click the "Save & test" button, the plugin will automatically load the mentioned 14 alert rules. The rules will be placed in the Grafana alerts directory. If not required, turn off the "Load TDengine Alert" switch, and click the button next to "Clear TDengine Alert" to clear all the alert rules imported into this data source. + +After importing, click on "Alert rules" on the left side of the Grafana interface to view all current alert rules. By configuring contact points, users can receive alert notifications. + +The specific configuration of the 14 alert rules is as follows: + +| alert rule| Rule threshold| Behavior when no data | Data scanning interval |Duration | SQL | +| ------ | --------- | ---------------- | ----------- |------- |----------------------| +|CPU load of dnode node|average > 80%|Trigger alert|5 minutes|5 minutes |`select now(), dnode_id, last(cpu_system) as cup_use from log.taosd_dnodes_info where _ts >= (now- 5m) and _ts < now partition by dnode_id having first(_ts) > 0 `| +|Memory of dnode node |average > 60%|Trigger alert|5 minutes|5 minutes|`select now(), dnode_id, last(mem_engine) / last(mem_total) * 100 as taosd from log.taosd_dnodes_info where _ts >= (now- 5m) and _ts 80%|Trigger alert|5 minutes|5 minutes|`select now(), dnode_id, data_dir_level, data_dir_name, last(used) / last(total) * 100 as used from log.taosd_dnodes_data_dirs where _ts >= (now - 5m) and _ts < now partition by dnode_id, data_dir_level, data_dir_name`| +|Authorization expires |< 60天|Trigger alert|1 day|0 0 seconds|`select now(), cluster_id, last(grants_expire_time) / 86400 as expire_time from log.taosd_cluster_info where _ts >= (now - 24h) and _ts < now partition by cluster_id having first(_ts) > 0 `| +|The used measurement points has reached the authorized number|>= 90%|Trigger alert|1 day|0 seconds|`select now(), cluster_id, CASE WHEN max(grants_timeseries_total) > 0.0 THEN max(grants_timeseries_used) /max(grants_timeseries_total) * 100.0 ELSE 0.0 END AS result from log.taosd_cluster_info where _ts >= (now - 30s) and _ts < now partition by cluster_id having timetruncate(first(_ts), 1m) > 0`| +|Number of concurrent query requests | > 100|Do not trigger alert|1 minute|0 seconds|`select now() as ts, count(*) as slow_count from performance_schema.perf_queries`| +|Maximum time for slow query execution (no time window) |> 300秒|Do not trigger alert|1 minute|0 seconds|`select now() as ts, count(*) as slow_count from performance_schema.perf_queries where exec_usec>300000000`| +|dnode offline |total != alive|Trigger alert|30 seconds|0 seconds|`select now(), cluster_id, last(dnodes_total) - last(dnodes_alive) as dnode_offline from log.taosd_cluster_info where _ts >= (now -30s) and _ts < now partition by cluster_id having first(_ts) > 0`| +|vnode offline |total != alive|Trigger alert|30 seconds|0 seconds|`select now(), cluster_id, last(vnodes_total) - last(vnodes_alive) as vnode_offline from log.taosd_cluster_info where _ts >= (now - 30s) and _ts < now partition by cluster_id having first(_ts) > 0 `| +|Number of data deletion requests |> 0|Do not trigger alert|30 seconds|0 seconds|``select now(), count(`count`) as `delete_count` from log.taos_sql_req where sql_type = 'delete' and _ts >= (now -30s) and _ts < now``| +|Adapter RESTful request fail |> 5|Do not trigger alert|30 seconds|0 seconds|``select now(), sum(`fail`) as `Failed` from log.adapter_requests where req_type=0 and ts >= (now -30s) and ts < now``| +|Adapter WebSocket request fail |> 5|Do not trigger alert|30 seconds|0 seconds|``select now(), sum(`fail`) as `Failed` from log.adapter_requests where req_type=1 and ts >= (now -30s) and ts < now``| +|Dnode data reporting is missing |< 3|Trigger alert|180 seconds|0 seconds|`select now(), cluster_id, count(*) as dnode_report from log.taosd_cluster_info where _ts >= (now -180s) and _ts < now partition by cluster_id having timetruncate(first(_ts), 1h) > 0`| +|Restart dnode |max(update_time) > last(update_time)|Trigger alert|90 seconds|0 seconds|`select now(), dnode_id, max(uptime) - last(uptime) as dnode_restart from log.taosd_dnodes_info where _ts >= (now - 90s) and _ts < now partition by dnode_id`| + +TDengine users can modify and improve these alert rules according to their own business needs. In Grafana 7.5 and below versions, the Dashboard and Alert rules functions are combined, while in subsequent new versions, the two functions are separated. To be compatible with Grafana7.5 and below versions, an Alert Used Only panel has been added to the TDinsight panel, which is only required for Grafana7.5 and below versions. + ## Upgrade @@ -248,13 +278,13 @@ The new version of the plugin uses the Grafana unified alerting feature, the `-E Assuming you start the TDengine database on the host `tdengine` with HTTP API port `6041`, user `root1`, and password `pass5ord`. Execute the script: -```bash +```shell ./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord ``` If you want to monitor multiple TDengine clusters, you need to set up multiple TDinsight dashboards. Setting up a non-default TDinsight requires some changes: the `-n` `-i` `-t` options need to be changed to non-default names, and if using the built-in SMS alert feature, `-N` and `-L` should also be changed. -```bash +```shell sudo ./TDengine.sh -n TDengine-Env1 -a http://another:6041 -u root -p taosdata -i tdinsight-env1 -t 'TDinsight Env1' ``` diff --git a/docs/en/14-reference/02-tools/08-taos-cli.md b/docs/en/14-reference/02-tools/08-taos-cli.md index 1dd77de186..9309da831d 100644 --- a/docs/en/14-reference/02-tools/08-taos-cli.md +++ b/docs/en/14-reference/02-tools/08-taos-cli.md @@ -10,7 +10,7 @@ The TDengine command line program (hereinafter referred to as TDengine CLI) is t To enter the TDengine CLI, simply execute `taos` in the terminal. -```bash +```shell taos ``` @@ -81,7 +81,7 @@ There are many other parameters: Example: -```bash +```shell taos -h h1.taos.com -s "use db; show tables;" ``` diff --git a/docs/en/14-reference/02-tools/10-taosbenchmark.md b/docs/en/14-reference/02-tools/10-taosbenchmark.md index c41cff49e1..09227f210b 100644 --- a/docs/en/14-reference/02-tools/10-taosbenchmark.md +++ b/docs/en/14-reference/02-tools/10-taosbenchmark.md @@ -28,7 +28,7 @@ taosBenchmark supports comprehensive performance testing for TDengine, and the T Execute the following command to quickly experience taosBenchmark performing a write performance test on TDengine based on the default configuration. -```bash +```shell taosBenchmark ``` @@ -38,7 +38,7 @@ When running without parameters, taosBenchmark by default connects to the TDengi When running taosBenchmark using command line parameters and controlling its behavior, the `-f ` parameter cannot be used. All configuration parameters must be specified through the command line. Below is an example of using command line mode to test the write performance of taosBenchmark. -```bash +```shell taosBenchmark -I stmt -n 200 -t 100 ``` @@ -50,7 +50,7 @@ The taosBenchmark installation package includes examples of configuration files, Use the following command line to run taosBenchmark and control its behavior through a configuration file. -```bash +```shell taosBenchmark -f ``` diff --git a/docs/en/14-reference/03-taos-sql/03-table.md b/docs/en/14-reference/03-taos-sql/03-table.md index 3c047eb19e..b7a8bb5b1c 100644 --- a/docs/en/14-reference/03-taos-sql/03-table.md +++ b/docs/en/14-reference/03-taos-sql/03-table.md @@ -170,7 +170,7 @@ ALTER TABLE [db_name.]tb_name alter_table_clause alter_table_clause: { alter_table_options - | SET tag tag_name = new_tag_value,tag_name2=new_tag2_value... + | SET tag tag_name = new_tag_value, tag_name2=new_tag2_value ... } alter_table_options: @@ -194,7 +194,7 @@ alter_table_option: { ### Modify Subtable Tag Value ```sql -ALTER TABLE tb_name SET TAG tag_name1=new_tag_value1,tag_name2=new_tag_value2...; +ALTER TABLE tb_name SET TAG tag_name1=new_tag_value1, tag_name2=new_tag_value2 ...; ``` ### Modify Table Lifespan diff --git a/docs/en/14-reference/03-taos-sql/06-select.md b/docs/en/14-reference/03-taos-sql/06-select.md index 5c71697110..c33fef95fb 100644 --- a/docs/en/14-reference/03-taos-sql/06-select.md +++ b/docs/en/14-reference/03-taos-sql/06-select.md @@ -210,19 +210,19 @@ However, renaming individual columns is not supported for `first(*)`, `last(*)`, Retrieve all subtable names and related tag information from a supertable: -```mysql +```sql SELECT TAGS TBNAME, location FROM meters; ``` It is recommended that users query the subtable tag information of supertables using the INS_TAGS system table under INFORMATION_SCHEMA, for example, to get all subtable names and tag values of the supertable meters: -```mysql +```sql SELECT table_name, tag_name, tag_type, tag_value FROM information_schema.ins_tags WHERE stable_name='meters'; ``` Count the number of subtables under a supertable: -```mysql +```sql SELECT COUNT(*) FROM (SELECT DISTINCT TBNAME FROM meters); ``` @@ -385,7 +385,7 @@ SELECT CURRENT_USER(); ### Syntax -```txt +```text WHERE (column|tbname) match/MATCH/nmatch/NMATCH _regex_ ``` @@ -403,7 +403,7 @@ The length of the regular match string cannot exceed 128 bytes. You can set and ### Syntax -```txt +```text CASE value WHEN compare_value THEN result [WHEN compare_value THEN result ...] [ELSE result] END CASE WHEN condition THEN result [WHEN condition THEN result ...] [ELSE result] END ``` @@ -493,7 +493,7 @@ SELECT ... FROM (SELECT ... FROM ...) ...; ## UNION ALL Clause -```txt title=Syntax +```text title=Syntax SELECT ... UNION ALL SELECT ... [UNION ALL SELECT ...] diff --git a/docs/en/14-reference/03-taos-sql/08-delete-data.mdx b/docs/en/14-reference/03-taos-sql/08-delete-data.md similarity index 100% rename from docs/en/14-reference/03-taos-sql/08-delete-data.mdx rename to docs/en/14-reference/03-taos-sql/08-delete-data.md diff --git a/docs/en/14-reference/03-taos-sql/10-function.md b/docs/en/14-reference/03-taos-sql/10-function.md index fa6b8d333a..4eb7b083fb 100644 --- a/docs/en/14-reference/03-taos-sql/10-function.md +++ b/docs/en/14-reference/03-taos-sql/10-function.md @@ -417,7 +417,7 @@ MOD(expr1, expr2) **Example**: -``` sql +```sql taos> select mod(10,3); mod(10,3) | ============================ @@ -454,7 +454,7 @@ RAND([seed]) **Example**: -``` sql +```sql taos> select rand(); rand() | ============================ diff --git a/docs/en/14-reference/03-taos-sql/21-node.md b/docs/en/14-reference/03-taos-sql/21-node.md index 1da0117a89..76cb68e9be 100644 --- a/docs/en/14-reference/03-taos-sql/21-node.md +++ b/docs/en/14-reference/03-taos-sql/21-node.md @@ -41,38 +41,28 @@ If there is a single replica on the node and the node is offline, to forcibly de ALTER DNODE dnode_id dnode_option ALTER ALL DNODES dnode_option - -dnode_option: { - 'resetLog' - | 'balance' 'value' - | 'monitor' 'value' - | 'debugFlag' 'value' - | 'monDebugFlag' 'value' - | 'vDebugFlag' 'value' - | 'mDebugFlag' 'value' - | 'cDebugFlag' 'value' - | 'httpDebugFlag' 'value' - | 'qDebugflag' 'value' - | 'sdbDebugFlag' 'value' - | 'uDebugFlag' 'value' - | 'tsdbDebugFlag' 'value' - | 'sDebugflag' 'value' - | 'rpcDebugFlag' 'value' - | 'dDebugFlag' 'value' - | 'mqttDebugFlag' 'value' - | 'wDebugFlag' 'value' - | 'tmrDebugFlag' 'value' - | 'cqDebugFlag' 'value' -} ``` -The modifiable configuration items in the syntax above are configured in the same way as in the dnode configuration file, the difference being that modifications are dynamic, take immediate effect, and do not require restarting the dnode. +For configuration parameters that support dynamic modification, you can use the ALTER DNODE or ALTER ALL DNODES syntax to modify the values of configuration parameters in a dnode. Starting from version 3.3.4.0, the modified configuration parameters will be automatically persisted and will remain effective even after the database service is restarted. -`value` is the value of the parameter, which needs to be in string format. For example, to change the log output level of dnode 1 to debug: +To check whether a configuration parameter supports dynamic modification, please refer to the following page: [taosd Reference](../01-components/01-taosd.md) + +The value is the parameter's value and needs to be in character format. For example, to change the log output level of dnode 1 to debug: ```sql ALTER DNODE 1 'debugFlag' '143'; ``` +### Additional Notes: +Configuration parameters in a dnode are divided into global configuration parameters and local configuration parameters. You can check the category field in SHOW VARIABLES or SHOW DNODE dnode_id VARIABLE to determine whether a configuration parameter is a global configuration parameter or a local configuration parameter: + +Local configuration parameters: You can use ALTER DNODE or ALTER ALL DNODES to update the local configuration parameters of a specific dnode or all dnodes. +Global configuration parameters: Global configuration parameters require consistency across all dnodes, so you can only use ALTER ALL DNODES to update the global configuration parameters of all dnodes. +There are three cases for whether a configuration parameter can be dynamically modified: + +Supports dynamic modification, effective immediately +Supports dynamic modification, effective after restart +Does not support dynamic modification +For configuration parameters that take effect after a restart, you can see the modified values through SHOW VARIABLES or SHOW DNODE dnode_id VARIABLE, but you need to restart the database service to make them effective. ## Add Management Node @@ -136,18 +126,12 @@ If the client is also considered as part of the cluster in a broader sense, the ```sql ALTER LOCAL local_option - -local_option: { - 'resetLog' - | 'rpcDebugFlag' 'value' - | 'tmrDebugFlag' 'value' - | 'cDebugFlag' 'value' - | 'uDebugFlag' 'value' - | 'debugFlag' 'value' -} ``` -The parameters in the syntax above are used in the same way as in the configuration file for the client, but do not require a restart of the client, and the changes take effect immediately. +You can use the above syntax to modify the client's configuration parameters, and there is no need to restart the client. The changes take effect immediately. + +To check whether a configuration parameter supports dynamic modification, please refer to the following page:[taosc Reference](../01-components/02-taosc.md) + ## View Client Configuration diff --git a/docs/en/14-reference/03-taos-sql/22-meta.md b/docs/en/14-reference/03-taos-sql/22-meta.md index 271f52d2c0..3a17353b02 100644 --- a/docs/en/14-reference/03-taos-sql/22-meta.md +++ b/docs/en/14-reference/03-taos-sql/22-meta.md @@ -318,7 +318,7 @@ Configuration parameters for each dnode in the system. Users with SYSINFO attrib Note: Users with SYSINFO property set to 0 cannot view this table. | # | **Column Name** | **Data Type** | **Description** | -| ---- | :-------------: | -------------- | ------------------------------------ | +|:-----|:----------------|:---------------|:-------------------------------------| | 1 | user_name | VARCHAR(24) | Username | | 2 | privilege | VARCHAR(10) | Permission description | | 3 | db_name | VARCHAR(65) | Database name | @@ -328,17 +328,32 @@ Note: Users with SYSINFO property set to 0 cannot view this table. ## INS_DISK_USAGE -| # | **Column Name** | **Data type** | **Description**| -| --- | :----------: | ------------ | ------------------------| -| 1 | db_name | VARCHAR(32) | Database name -| 2 | vgroup_id | INT | vgroup ID -| 3 | wal | BIGINT | WAL file size, in KB -| 4 | data1 | BIGINT | Data file size on primary storage, in KB -| 5 | data2 | BIGINT | Data file size on secondary storage, in KB -| 6 | data3 | BIGINT | Data file size on tertiary storage, in KB -| 7 | cache_rdb | BIGINT | Size of last/last_row files, in KB -| 8 | table_meta | BIGINT | Size of meta files, in KB -| 9 | s3 | BIGINT | Size occupied on S3, in KB -| 10 | raw_data | BIGINT | Estimated size of raw data, in KB +| # | **Column Name** | **Data type** | **Description**| +|:----|:-----------|:-----------|:--------------------| +| 1 | db_name | VARCHAR(32) | Database name | +| 2 | vgroup_id | INT | vgroup ID | +| 3 | wal | BIGINT | WAL file size, in KB | +| 4 | data1 | BIGINT | Data file size on primary storage, in KB | +| 5 | data2 | BIGINT | Data file size on secondary storage, in KB | +| 6 | data3 | BIGINT | Data file size on tertiary storage, in KB | +| 7 | cache_rdb | BIGINT | Size of last/last_row files, in KB | +| 8 | table_meta | BIGINT | Size of meta files, in KB | +| 9 | s3 | BIGINT | Size occupied on S3, in KB | +| 10 | raw_data | BIGINT | Estimated size of raw data, in KB | note: + +## INS_FILESETS + +Provides information about file sets. + +| # | **Column** | **Data Type** | **Description** | ** | +| --- | :-----------: | ------------- | ---------------------------------------------------- | +| 1 | db_name | VARCHAR(65) | Database name | +| 2 | vgroup_id | INT | Vgroup ID | +| 3 | fileset_id | INT | File set ID | +| 4 | start_time | TIMESTAMP | Start time of the time range covered by the file set | +| 5 | end_time | TIMESTAMP | End time of the time range covered by the file set | +| 6 | total_size | BIGINT | Total size of the file set | +| 7 | last_compact | TIMESTAMP | Time of the last compaction | +| 8 | shold_compact | bool | Whether the file set should be compacted | diff --git a/docs/en/14-reference/03-taos-sql/25-user.md b/docs/en/14-reference/03-taos-sql/25-user.md index 780fd92040..04313a5da3 100644 --- a/docs/en/14-reference/03-taos-sql/25-user.md +++ b/docs/en/14-reference/03-taos-sql/25-user.md @@ -13,16 +13,16 @@ CREATE USER user_name PASS 'password' [SYSINFO {1|0}] [CREATEDB {1|0}]; The username can be up to 23 bytes long. -The password can be up to 31 bytes long. The password can include letters, numbers, and special characters except for single quotes, double quotes, backticks, backslashes, and spaces, and it cannot be an empty string. +The password must be between 8 and 16 characters long and include at least three types of characters from the following: uppercase letters, lowercase letters, numbers, and special characters. Special characters include `! @ # $ % ^ & * ( ) - _ + = [ ] { } : ; > < ? | ~ , .`. `SYSINFO` indicates whether the user can view system information. `1` means they can view, `0` means they have no permission to view. System information includes service configuration, dnode, vnode, storage, etc. The default value is `1`. `CREATEDB` indicates whether the user can create databases. `1` means they can create databases, `0` means they have no permission to create databases. The default value is `0`. // Supported starting from TDengine Enterprise version 3.3.2.0 -In the example below, we create a user with the password `123456` who can view system information. +In the example below, we create a user with the password `abc123!@#` who can view system information. ```sql -taos> create user test pass '123456' sysinfo 1; +taos> create user test pass 'abc123!@#' sysinfo 1; Query OK, 0 of 0 rows affected (0.001254s) ``` diff --git a/docs/en/14-reference/03-taos-sql/32-compress.md b/docs/en/14-reference/03-taos-sql/32-compress.md index 8bb2ae31e3..30b107b632 100644 --- a/docs/en/14-reference/03-taos-sql/32-compress.md +++ b/docs/en/14-reference/03-taos-sql/32-compress.md @@ -29,13 +29,13 @@ In this document, it specifically refers to the internal levels of the second-le - Default compression algorithms list and applicable range for each data type | Data Type |Available Encoding Algorithms | Default Encoding Algorithm | Available Compression Algorithms | Default Compression Algorithm | Default Compression Level | -|:------------------------------------:|:----------------:|:-----------:|:--------------------:|:----:|:------:| -| int/uint | simple8b | simple8b | lz4/zlib/zstd/xz | lz4 | medium | -| tinyint/untinyint/smallint/usmallint | simple8b | simple8b | lz4/zlib/zstd/xz | zlib | medium | -| bigint/ubigint/timestamp | simple8b/delta-i | delta-i | lz4/zlib/zstd/xz | lz4 | medium | -| float/double | delta-d | delta-d | lz4/zlib/zstd/xz/tsz | lz4 | medium | -| binary/nchar | disabled | disabled | lz4/zlib/zstd/xz | zstd | medium | -| bool | bit-packing | bit-packing | lz4/zlib/zstd/xz | zstd | medium | +|:------------------------------------:|:-------------------------:|:-----------:|:--------------------:|:----:|:------:| +| int/uint | disabled/simple8b | simple8b | lz4/zlib/zstd/xz | lz4 | medium | +| tinyint/untinyint/smallint/usmallint | disabled/simple8b | simple8b | lz4/zlib/zstd/xz | zlib | medium | +| bigint/ubigint/timestamp | disabled/simple8b/delta-i | delta-i | lz4/zlib/zstd/xz | lz4 | medium | +| float/double | disabled/delta-d | delta-d | lz4/zlib/zstd/xz/tsz | lz4 | medium | +| binary/nchar | disabled | disabled | lz4/zlib/zstd/xz | zstd | medium | +| bool | disabled/bit-packing | bit-packing | lz4/zlib/zstd/xz | zstd | medium | ## SQL Syntax diff --git a/docs/en/14-reference/05-connector/10-cpp.mdx b/docs/en/14-reference/05-connector/10-cpp.md similarity index 100% rename from docs/en/14-reference/05-connector/10-cpp.mdx rename to docs/en/14-reference/05-connector/10-cpp.md diff --git a/docs/en/14-reference/05-connector/14-java.mdx b/docs/en/14-reference/05-connector/14-java.md similarity index 100% rename from docs/en/14-reference/05-connector/14-java.mdx rename to docs/en/14-reference/05-connector/14-java.md diff --git a/docs/en/14-reference/05-connector/20-go.mdx b/docs/en/14-reference/05-connector/20-go.md similarity index 99% rename from docs/en/14-reference/05-connector/20-go.mdx rename to docs/en/14-reference/05-connector/20-go.md index b6313ae3b1..3342066a3d 100644 --- a/docs/en/14-reference/05-connector/20-go.mdx +++ b/docs/en/14-reference/05-connector/20-go.md @@ -108,7 +108,7 @@ For the source code of the example programs, please refer to: [Example Programs] The Data Source Name has a generic format, similar to [PEAR DB](http://pear.php.net/manual/en/package.database.db.intro-dsn.php), but without the type prefix (brackets indicate optional): -``` text +```text [username[:password]@][protocol[(address)]]/[dbname][?param1=value1&...¶mN=valueN] ``` diff --git a/docs/en/14-reference/05-connector/26-rust.mdx b/docs/en/14-reference/05-connector/26-rust.md similarity index 100% rename from docs/en/14-reference/05-connector/26-rust.mdx rename to docs/en/14-reference/05-connector/26-rust.md diff --git a/docs/en/14-reference/05-connector/30-python.mdx b/docs/en/14-reference/05-connector/30-python.md similarity index 100% rename from docs/en/14-reference/05-connector/30-python.mdx rename to docs/en/14-reference/05-connector/30-python.md diff --git a/docs/en/14-reference/05-connector/35-node.mdx b/docs/en/14-reference/05-connector/35-node.md similarity index 100% rename from docs/en/14-reference/05-connector/35-node.mdx rename to docs/en/14-reference/05-connector/35-node.md diff --git a/docs/en/14-reference/05-connector/40-csharp.mdx b/docs/en/14-reference/05-connector/40-csharp.md similarity index 100% rename from docs/en/14-reference/05-connector/40-csharp.mdx rename to docs/en/14-reference/05-connector/40-csharp.md diff --git a/docs/en/14-reference/05-connector/43-r-lang.mdx b/docs/en/14-reference/05-connector/43-r-lang.md similarity index 100% rename from docs/en/14-reference/05-connector/43-r-lang.mdx rename to docs/en/14-reference/05-connector/43-r-lang.md diff --git a/docs/en/14-reference/05-connector/45-php.mdx b/docs/en/14-reference/05-connector/45-php.md similarity index 100% rename from docs/en/14-reference/05-connector/45-php.mdx rename to docs/en/14-reference/05-connector/45-php.md diff --git a/docs/en/14-reference/05-connector/50-odbc.mdx b/docs/en/14-reference/05-connector/50-odbc.md similarity index 100% rename from docs/en/14-reference/05-connector/50-odbc.mdx rename to docs/en/14-reference/05-connector/50-odbc.md diff --git a/docs/en/14-reference/05-connector/60-rest-api.mdx b/docs/en/14-reference/05-connector/60-rest-api.md similarity index 99% rename from docs/en/14-reference/05-connector/60-rest-api.mdx rename to docs/en/14-reference/05-connector/60-rest-api.md index a63022f547..88e53f7618 100644 --- a/docs/en/14-reference/05-connector/60-rest-api.mdx +++ b/docs/en/14-reference/05-connector/60-rest-api.md @@ -21,7 +21,7 @@ Below is an example using the `curl` tool in an Ubuntu environment (please confi The following example lists all databases, please replace `h1.tdengine.com` and 6041 (default value) with the actual running TDengine service FQDN and port number: -```bash +```shell curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" \ -d "select name, ntables, status from information_schema.ins_databases;" \ h1.tdengine.com:6041/rest/sql @@ -100,13 +100,13 @@ The BODY of the HTTP request contains a complete SQL statement. The data table i Use `curl` to initiate an HTTP Request with custom authentication as follows: -```bash +```shell curl -L -H "Authorization: Basic " -d "" :/rest/sql/[db_name][?tz=timezone[&req_id=req_id][&row_with_meta=true]] ``` Or, -```bash +```shell curl -L -u username:password -d "" :/rest/sql/[db_name][?tz=timezone[&req_id=req_id][&row_with_meta=true]] ``` @@ -279,7 +279,7 @@ Column types use the following strings: Prepare data -```bash +```shell create database demo use demo create table t(ts timestamp,c1 varbinary(20),c2 geometry(100)) @@ -288,7 +288,7 @@ insert into t values(now,'\x7f8290','point(100 100)') Execute query -```bash +```shell curl --location 'http://:/rest/sql' \ --header 'Content-Type: text/plain' \ --header 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' \ @@ -428,7 +428,7 @@ Data Query Return Example HTTP requests need to include an authorization code ``, used for identity verification. The authorization code is usually provided by the administrator and can be simply obtained by sending an `HTTP GET` request as follows: -```bash +```shell curl http://:/rest/login// ``` @@ -440,7 +440,7 @@ Here, `fqdn` is the FQDN or IP address of the TDengine database, `port` is the p Example of obtaining an authorization code: -```bash +```shell curl http://192.168.0.1:6041/rest/login/root/taosdata ``` @@ -457,7 +457,7 @@ Return value: - Query all records of table d1001 in the demo database: - ```bash + ```shell curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "select * from demo.d1001" 192.168.0.1:6041/rest/sql curl -L -H "Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04" -d "select * from demo.d1001" 192.168.0.1:6041/rest/sql ``` @@ -509,7 +509,7 @@ Return value: - Create database demo: - ```bash + ```shell curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "create database demo" 192.168.0.1:6041/rest/sql curl -L -H "Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04" -d "create database demo" 192.168.0.1:6041/rest/sql ``` @@ -560,7 +560,7 @@ Return value: #### TDengine 2.x response codes and message bodies -```JSON +```json { "status": "succ", "head": [ @@ -624,7 +624,7 @@ Return value: #### TDengine 3.0 Response Codes and Message Body -```JSON +```json { "code": 0, "column_meta": [ diff --git a/docs/en/14-reference/09-error-code.md b/docs/en/14-reference/09-error-code.md index 84504395b5..1d3ea3f9a1 100644 --- a/docs/en/14-reference/09-error-code.md +++ b/docs/en/14-reference/09-error-code.md @@ -72,6 +72,8 @@ This document details the server error codes that may be encountered when using | 0x80000133 | Invalid operation | Invalid or unsupported operation | 1. Modify to confirm the current operation is legal and supported, check parameter validity 2. If the problem persists, preserve the scene and logs, report issue on github | | 0x80000134 | Invalid value | Invalid value | Preserve the scene and logs, report issue on github | | 0x80000135 | Invalid fqdn | Invalid FQDN | Check if the configured or input FQDN value is correct | +| 0x8000013C | Invalid disk id | Invalid disk id | Check users whether the mounted disk is invalid or use the parameter diskIDCheckEnabled to skip the disk check. | + ## tsc @@ -129,7 +131,7 @@ This document details the server error codes that may be encountered when using | 0x80000350 | User already exists | Create user, duplicate creation | Confirm if the operation is correct | | 0x80000351 | Invalid user | User does not exist | Confirm if the operation is correct | | 0x80000352 | Invalid user format | Incorrect format | Confirm if the operation is correct | -| 0x80000353 | Invalid password format | Incorrect format | Confirm if the operation is correct | +| 0x80000353 | Invalid password format | The password must be between 8 and 16 characters long and include at least three types of characters from the following: uppercase letters, lowercase letters, numbers, and special characters. | Confirm the format of the password string | | 0x80000354 | Can not get user from conn | Internal error | Report issue | | 0x80000355 | Too many users | (Enterprise only) Exceeding user limit | Adjust configuration | | 0x80000357 | Authentication failure | Incorrect password | Confirm if the operation is correct | @@ -251,6 +253,7 @@ This document details the server error codes that may be encountered when using | 0x80000529 | Vnode is stopped | Vnode is closed | Report issue | | 0x80000530 | Duplicate write request | Duplicate write request, internal error | Report issue | | 0x80000531 | Vnode query is busy | Query is busy | Report issue | +| 0x80000540 | Vnode already exist but Dbid not match | Internal error | Report issue | ## tsdb @@ -283,6 +286,9 @@ This document details the server error codes that may be encountered when using | 0x80000729 | Task message error | Query message error | Preserve the scene and logs, report issue on GitHub | | 0x8000072B | Task status error | Subquery status error | Preserve the scene and logs, report issue on GitHub | | 0x8000072F | Job not exist | Query JOB no longer exists | Preserve the scene and logs, report issue on GitHub | +| 0x80000739 | Query memory upper limit is reached | Single query memory upper limit is reached | Modify memory upper limit size or optimize SQL | +| 0x8000073A | Query memory exhausted | Query memory in dnode is exhausted | Limit concurrent queries or add more physical memory | +| 0x8000073B | Timeout for long time no fetch | Query without fetch for a long time | Correct application to fetch data asap | ## grant diff --git a/docs/en/27-train-faq/index.md b/docs/en/27-train-faq/index.md index 3b3f7a8e17..ca6cd91714 100644 --- a/docs/en/27-train-faq/index.md +++ b/docs/en/27-train-faq/index.md @@ -90,7 +90,7 @@ Batch insertion. Each insert statement can insert multiple records into one tabl When inserting nchar type data containing Chinese characters on Windows, first ensure that the system's regional settings are set to China (this can be set in the Control Panel). At this point, the `taos` client in cmd should already be working properly; if developing a Java application in an IDE, such as Eclipse or IntelliJ, ensure that the file encoding in the IDE is set to GBK (which is the default encoding type for Java), then initialize the client configuration when creating the Connection, as follows: -```JAVA +```java Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.LOCALE_KEY, "UTF-8"); @@ -145,7 +145,7 @@ Version 3.0 of TDengine includes a standalone component developed in Go called ` The Go language version requirement is 1.14 or higher. If there are Go compilation errors, often due to issues accessing Go mod in China, they can be resolved by setting Go environment variables: -```sh +```shell go env -w GO111MODULE=on go env -w GOPROXY=https://goproxy.cn,direct ``` @@ -196,7 +196,7 @@ Here are the solutions: 1. Create a file /Library/LaunchDaemons/limit.maxfiles.plist, write the following content (the example changes limit and maxfiles to 100,000, modify as needed): -```plist +```xml @@ -286,4 +286,14 @@ This connection only reports the most basic information that does not involve an This feature is an optional configuration item, which is enabled by default in the open-source version. The specific parameter is telemetryReporting, as explained in the [official documentation](../tdengine-reference/components/taosd/). You can disable this parameter at any time by modifying telemetryReporting to 0 in taos.cfg, then restarting the database service. Code located at: [https://github.com/taosdata/TDengine/blob/62e609c558deb764a37d1a01ba84bc35115a85a4/source/dnode/mnode/impl/src/mndTelem.c](https://github.com/taosdata/TDengine/blob/62e609c558deb764a37d1a01ba84bc35115a85a4/source/dnode/mnode/impl/src/mndTelem.c). -Additionally, for the highly secure enterprise version, TDengine Enterprise, this parameter will not be operational. +Additionally, for the highly secure enterprise version, TDengine Enterprise, this parameter will not be operational. + +### 31 What should I do if I encounter 'Sync leader is unreachable' when connecting to the cluster for the first time? + +Reporting this error indicates that the first connection to the cluster was successful, but the IP address accessed for the first time was not the leader of mnode. An error occurred when the client attempted to establish a connection with the leader. The client searches for the leader node through EP, which specifies the fqdn and port number. There are two common reasons for this error: + +- The ports of other dnodes in the cluster are not open +- The client's hosts file is not configured correctly + +Therefore, first, check whether all ports on the server and cluster (default 6030 for native connections and 6041 for HTTP connections) are open; Next, check if the client's hosts file has configured the fqdn and IP information for all dnodes in the cluster. +If the issue still cannot be resolved, it is necessary to contact Taos technical personnel for support. diff --git a/docs/examples/node/websocketexample/sql_example.js b/docs/examples/node/websocketexample/sql_example.js index c120b84d99..7e78bb6c64 100644 --- a/docs/examples/node/websocketexample/sql_example.js +++ b/docs/examples/node/websocketexample/sql_example.js @@ -26,7 +26,6 @@ async function createDbAndTable() { let conf = new taos.WSConfig(dsn); conf.setUser('root'); conf.setPwd('taosdata'); - conf.setDb('power'); wsSql = await taos.sqlConnect(conf); console.log("Connected to " + dsn + " successfully."); // create database diff --git a/docs/examples/node/websocketexample/tmq_example.js b/docs/examples/node/websocketexample/tmq_example.js index 4f7f099c31..4ea3db55c3 100644 --- a/docs/examples/node/websocketexample/tmq_example.js +++ b/docs/examples/node/websocketexample/tmq_example.js @@ -40,7 +40,6 @@ async function prepare() { let conf = new taos.WSConfig('ws://localhost:6041'); conf.setUser('root'); conf.setPwd('taosdata'); - conf.setDb('power'); const createDB = `CREATE DATABASE IF NOT EXISTS ${db}`; const createStable = `CREATE STABLE IF NOT EXISTS ${db}.${stable} (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);`; diff --git a/docs/examples/node/websocketexample/tmq_seek_example.js b/docs/examples/node/websocketexample/tmq_seek_example.js index be4d8ddfa4..67286f34b1 100644 --- a/docs/examples/node/websocketexample/tmq_seek_example.js +++ b/docs/examples/node/websocketexample/tmq_seek_example.js @@ -34,10 +34,10 @@ async function createConsumer() { } async function prepare() { - let conf = new taos.WSConfig('ws://192.168.1.98:6041'); + let conf = new taos.WSConfig('ws://localhost:6041'); conf.setUser('root'); conf.setPwd('taosdata'); - conf.setDb('power'); + const createDB = `CREATE DATABASE IF NOT EXISTS ${db}`; const createStable = `CREATE STABLE IF NOT EXISTS ${db}.${stable} (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);`; diff --git a/docs/zh/05-basic/03-query.md b/docs/zh/05-basic/03-query.md index b6f172829a..f3c9eb099e 100644 --- a/docs/zh/05-basic/03-query.md +++ b/docs/zh/05-basic/03-query.md @@ -303,13 +303,12 @@ Query OK, 5 row(s) in set (0.016812s) #### FILL 子句 -FILL 子句，用于指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种： 1. 不进行填充：NONE（默认填充模式）。 -2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL（VALUE， 1.23）。这里需要注意，最终填充的值受由相应列的类型决定，如 FILL（VALUE， 1.23），相应列为 INT 类型，则填充值为 1。 -3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL（PREV）。 -4. NULL 填充：使用 NULL 填充数据。例如：FILL（NULL）。 -5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL（LINEAR）。 -6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL（NEXT）。 +2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如 FILL(VALUE, 1.23)，相应列为 INT 类型，则填充值为 1, 若查询列表中有多列需要 FILL, 则需要给每一个 FILL 列指定 VALUE, 如 `SELECT _wstart, min(c1), max(c1) FROM ... FILL(VALUE, 0, 0)`, 注意, SELECT 表达式中只有包含普通列时才需要指定 FILL VALUE, 如 `_wstart`, `_wstart+1a`, `now`, `1+1` 以及使用 partition by 时的 partition key (如 tbname)都不需要指定 VALUE, 如 `timediff(last(ts), _wstart)` 则需要指定VALUE。 +3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。 +4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。 +5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。 +6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。 以上填充模式中，除了 NONE 模式默认不填充值之外，其他模式在查询的整个时间范围内如果没有数据 FILL 子句将被忽略，即不产生填充数据，查询结果为空。这种行为在部分模式（PREV、NEXT、LINEAR）下具有合理性，因为在这些模式下没有数据意味着无法产生填充数值。 diff --git a/docs/zh/06-advanced/06-TDgpt/02-management.md b/docs/zh/06-advanced/06-TDgpt/02-management.md index ef1206fc04..b37c39944f 100644 --- a/docs/zh/06-advanced/06-TDgpt/02-management.md +++ b/docs/zh/06-advanced/06-TDgpt/02-management.md @@ -4,11 +4,10 @@ sidebar_label: "安装部署" --- ### 环境准备 -使用 TDgpt 的高级时序数据分析功能需要在 TDengine 集群中安装部署 AI node（Anode）。Anode 可以运行在 Linux/Windows/MacOS 等平台上，同时需要 3.10 或以上版本的 Python 环境支持。 +使用 TDgpt 的高级时序数据分析功能需要在 TDengine 集群中安装部署 AI node（Anode）。Anode 运行在 Linux 平台上，并需要 3.10 或以上版本的 Python 环境支持。 > 部署 Anode 需要 TDengine Enterprise 3.3.4.3 及以后版本，请首先确认搭配 Anode 使用的 TDengine 能够支持 Anode。 ### 安装及卸载 -不同操作系统上安装及部署 Anode 有一些差异，主要是卸载操作、安装路径、服务启停等方面。本文以 Linux 系统为例，说明安装部署的流程。 使用 Linux 环境下的安装包 TDengine-enterprise-anode-1.x.x.tar.gz 可进行 Anode 的安装部署工作，命令如下： ```bash @@ -37,7 +36,7 @@ systemctl status taosanoded |/usr/local/taos/taosanode/bin|可执行文件目录| |/usr/local/taos/taosanode/resource|资源文件目录，链接到文件夹 /var/lib/taos/taosanode/resource/| |/usr/local/taos/taosanode/lib|库文件目录| -|/var/lib/taos/taosanode/model/|模型文件目录，链接到文件夹 /var/lib/taos/taosanode/model| +|/usr/local/taos/taosanode/model/|模型文件目录，链接到文件夹 /var/lib/taos/taosanode/model| |/var/log/taos/taosanode/|日志文件目录| |/etc/taos/taosanode.ini|配置文件| @@ -64,7 +63,7 @@ pidfile = /usr/local/taos/taosanode/taosanode.pid # conflict with systemctl, so do NOT uncomment this # daemonize = /var/log/taos/taosanode/taosanode.log -# log directory +# uWSGI log files logto = /var/log/taos/taosanode/taosanode.log # wWSGI monitor port @@ -74,7 +73,7 @@ stats = 127.0.0.1:8387 virtualenv = /usr/local/taos/taosanode/venv/ [taosanode] -# default app log file +# default taosanode log file app-log = /var/log/taos/taosanode/taosanode.app.log # model storage directory diff --git a/docs/zh/06-advanced/06-TDgpt/03-preprocess.md b/docs/zh/06-advanced/06-TDgpt/03-preprocess.md index 9efd2bdf11..b63cae0740 100644 --- a/docs/zh/06-advanced/06-TDgpt/03-preprocess.md +++ b/docs/zh/06-advanced/06-TDgpt/03-preprocess.md @@ -12,7 +12,7 @@ import wndata from './pic/white-noise-data.png' 预处理流程 TDgpt 首先对输入数据进行白噪声检查（White Noise Data check）, 检查通过以后针对预测分析，还要进行输入（历史）数据的重采样和时间戳对齐处理（异常检测跳过数据重采样和时间戳对齐步骤）。 -预处理完成以后，再进行预测或异常检测操作。预处理过程部署于预测或异常检测处理逻辑的一部分。 +预处理完成以后，再进行预测或异常检测操作。预处理过程不属于预测或异常检测处理逻辑的一部分。 ### 白噪声检查 diff --git a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md index c7388ab9c0..3981fff8c6 100644 --- a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md +++ b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md @@ -14,7 +14,7 @@ description: 预测算法 ```bash taos> select * from foo; - ts | k | + ts | i32 | ======================================== 2020-01-01 00:00:12.681 | 13 | 2020-01-01 00:00:13.727 | 14 | @@ -42,7 +42,7 @@ algo=expr1 ``` 1. `column_expr`：预测的时序数据列。与异常检测相同，只支持数值类型列输入。 -2. `options`：异常检测函数的参数，使用规则与 anomaly_window 相同。预测支持 `conf`, `every`, `rows`, `start`, `rows` 几个控制参数，其含义如下： +2. `options`：预测函数的参数。字符串类型，其中使用 K=V 方式调用算法及相关参数。采用逗号分隔的 K=V 字符串表示，其中的字符串不需要使用单引号、双引号、或转义号等符号，不能使用中文及其他宽字符。预测支持 `conf`, `every`, `rows`, `start`, `rows` 几个控制参数，其含义如下： ### 参数说明 diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md index 954076c8fd..0584c87311 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md @@ -34,7 +34,8 @@ return { ```python import numpy as np -from service import AbstractForecastService +from taosanalytics.service import AbstractForecastService + # 算法实现类名称 需要以下划线 "_" 开始，并以 Service 结束 class _MyForecastService(AbstractForecastService): @@ -51,12 +52,12 @@ class _MyForecastService(AbstractForecastService): super().__init__() def execute(self): - """ 算法逻辑的核心实现""" + """ 算法逻辑的核心实现""" res = [] """这个预测算法固定返回 1 作为预测值，预测值的数量是用户通过 self.fc_rows 指定""" ts_list = [self.start_ts + i * self.time_step for i in range(self.fc_rows)] - res.app(ts_list) # 设置预测结果时间戳列 + res.append(ts_list) # 设置预测结果时间戳列 """生成全部为 1 的预测结果 """ res_list = [1] * self.fc_rows @@ -64,18 +65,18 @@ class _MyForecastService(AbstractForecastService): """检查用户输入，是否要求返回预测置信区间上下界""" if self.return_conf: - """对于没有计算预测置信区间上下界的算法，直接返回预测值作为上下界即可""" - bound_list = [1] * self.fc_rows - res.append(bound_list) # 预测结果置信区间下界 - res.append(bound_list) # 预测结果执行区间上界 + """对于没有计算预测置信区间上下界的算法，直接返回预测值作为上下界即可""" + bound_list = [1] * self.fc_rows + res.append(bound_list) # 预测结果置信区间下界 + res.append(bound_list) # 预测结果执行区间上界 """返回结果""" - return { "res": res, "mse": 0} + return {"res": res, "mse": 0} - def set_params(self, params): - """该算法无需任何输入参数，直接重载父类该函数，不处理算法参数设置逻辑""" - pass + """该算法无需任何输入参数，直接调用父类函数，不处理算法参数设置逻辑""" + return super().set_params(params) + ``` 将该文件保存在 `./taosanalytics/algo/fc/` 目录下，然后重启 taosanode 服务。在 TDengine 命令行接口中执行 `SHOW ANODES FULL` 能够看到新加入的算法。应用就可以通过 SQL 语句调用该预测算法。 diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md index 5b49db330e..c48ce42836 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md @@ -16,7 +16,7 @@ sidebar_label: "异常检测" ```python import numpy as np -from service import AbstractAnomalyDetectionService +from taosanalytics.service import AbstractAnomalyDetectionService # 算法实现类名称 需要以下划线 "_" 开始，并以 Service 结束 class _MyAnomalyDetectionService(AbstractAnomalyDetectionService): diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/index.md b/docs/zh/06-advanced/06-TDgpt/06-dev/index.md index 072a66c7d3..bcd972df8e 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/index.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/index.md @@ -19,24 +19,25 @@ Anode的主要目录结构如下图所示 ```bash . +├── bin ├── cfg -├── model -│   └── ad_autoencoder -├── release -├── script -└── taosanalytics - ├── algo - │   ├── ad - │   └── fc - ├── misc - └── test +├── lib +│   └── taosanalytics +│   ├── algo +│   │   ├── ad +│   │   └── fc +│   ├── misc +│   └── test +├── log -> /var/log/taos/taosanode +├── model -> /var/lib/taos/taosanode/model +└── venv -> /var/lib/taos/taosanode/venv ``` |目录|说明| |---|---| |taosanalytics| 源代码目录，其下包含了算法具体保存目录 algo，放置杂项目录 misc，单元测试和集成测试目录 test。 algo 目录下 ad 保存异常检测算法代码，fc 目录保存预测算法代码| -|script|是安装脚本和发布脚本放置目录| +|venv| Python 虚拟环境| |model|放置针对数据集完成的训练模型| |cfg|配置文件目录| @@ -63,7 +64,8 @@ Anode采用算法自动加载模式，因此只识别符合命名约定的 Pytho ```SQL --- algo 后面的参数 name 即为类属性 `name` -SELECT COUNT(*) FROM foo ANOMALY_WINDOW(col_name, 'algo=name') +SELECT COUNT(*) +FROM foo ANOMALY_WINDOW(col_name, 'algo=name') ``` ## 添加具有模型的分析算法 @@ -76,19 +78,10 @@ SELECT COUNT(*) FROM foo ANOMALY_WINDOW(col_name, 'algo=name') ```bash . -├── cfg -├── model -│   └── ad_autoencoder -│   ├── ad_autoencoder_foo.dat -│   └── ad_autoencoder_foo.info -├── release -├── script -└── taosanalytics - ├── algo - │   ├── ad - │   └── fc - ├── misc - └── test +└── model + └── ad_autoencoder + ├── ad_autoencoder_foo.dat + └── ad_autoencoder_foo.info ``` diff --git a/docs/zh/08-operation/04-maintenance.md b/docs/zh/08-operation/04-maintenance.md index 88122fed69..9ef165179d 100644 --- a/docs/zh/08-operation/04-maintenance.md +++ b/docs/zh/08-operation/04-maintenance.md @@ -18,6 +18,7 @@ TDengine 面向多种写入场景，而很多写入场景下，TDengine 的存 ```SQL COMPACT DATABASE db_name [start with 'XXXX'] [end with 'YYYY']； +COMPACT [db_name.]VGROUPS IN (vgroup_id1, vgroup_id2, ...) [start with 'XXXX'] [end with 'YYYY']； SHOW COMPACTS [compact_id]； KILL COMPACT compact_id； ``` @@ -25,6 +26,7 @@ KILL COMPACT compact_id； ### 效果 - 扫描并压缩指定的 DB 中所有 VGROUP 中 VNODE 的所有数据文件 +- 扫描并压缩 DB 中指定的 VGROUP 列表中 VNODE 的所有数据文件, 若 db_name 为空，则默认为当前数据库 - COMPCAT 会删除被删除数据以及被删除的表的数据 - COMPACT 会合并多个 STT 文件 - 可通过 start with 关键字指定 COMPACT 数据的起始时间 diff --git a/docs/zh/08-operation/12-multi.md b/docs/zh/08-operation/12-multi.md index 1e81a7ff1e..d18957d7d2 100644 --- a/docs/zh/08-operation/12-multi.md +++ b/docs/zh/08-operation/12-multi.md @@ -68,19 +68,19 @@ dataDir /mnt/data6 2 0 在配置文件 /etc/taos/taos.cfg 中，添加用于 S3 访问的参数： -|参数名称 | 参数含义 | -|:-------------|:-----------------------------------------------| -|s3EndPoint | 用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 的保持一致，否则无法访问。 | -|s3AccessKey |冒号分隔的用户 SecretId:SecretKey。例如：AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E | -|s3BucketName | 存储桶名称，减号后面是用户注册 COS 服务的 AppId。其中 AppId 是 COS 特有，AWS 和阿里云都没有，配置时需要作为 bucket name 的一部分，使用减号分隔。参数值均为字符串类型，但不需要引号。例如：test0711-1309024725 | -|s3UploadDelaySec | data 文件持续多长时间不再变动后上传至 s3，单位：秒。最小值：1；最大值：2592000 (30天），默认值 60 秒 | -|s3PageCacheSize |s3 page cache 缓存页数目，单位：页。最小值：4；最大值：1024*1024\*1024。 ，默认值 4096| -|s3MigrateIntervalSec | 本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600 | -|s3MigrateEnabled | 是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1。 | +| 参数名称 | 参数含义 | +|:---------------------|:-----------------------------------------------| +| s3EndPoint | 用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 的保持一致，否则无法访问。 | +| s3AccessKey | 冒号分隔的用户 SecretId:SecretKey。例如：AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E | +| s3BucketName | 存储桶名称，减号后面是用户注册 COS 服务的 AppId。其中 AppId 是 COS 特有，AWS 和阿里云都没有，配置时需要作为 bucket name 的一部分，使用减号分隔。参数值均为字符串类型，但不需要引号。例如：test0711-1309024725 | +| s3UploadDelaySec | data 文件持续多长时间不再变动后上传至 s3，单位：秒。最小值：1；最大值：2592000（30天），默认值 60 秒 | +| s3PageCacheSize | S3 page cache 缓存页数目，单位：页。最小值：4；最大值：1024*1024*1024。 ，默认值 4096| +| s3MigrateIntervalSec | 本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600 | +| s3MigrateEnabled | 是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1。 | ### 检查配置参数可用性 -在 taos.cfg 中完成对 s3 的配置后，通过 taosd 命令的 checks3 参数可以检查所配置的 S3 服务是否可用： +在 taos.cfg 中完成对 S3 的配置后，通过 taosd 命令的 checks3 参数可以检查所配置的 S3 服务是否可用： ``` taosd --checks3 @@ -106,11 +106,11 @@ s3migrate database ; 详细的 DB 参数见下表： -| # | 参数 | 默认值 | 最小值 | 最大值 | 描述 | -| :--- | :----------- | :----- | :----- | :------ | :----------------------------------------------------------- | -| 1 | s3_keeplocal | 365 | 1 | 365000 | 数据在本地保留的天数，即 data 文件在本地磁盘保留多长时间后可以上传到 S3。默认单位：天，支持 m（分钟）、h（小时）和 d（天）三个单位 | -| 2 | s3_chunkpages | 131072 | 131072 | 1048576 | 上传对象的大小阈值，与 tsdb_pagesize 参数一样，不可修改，单位为 TSDB 页 | -| 3 | s3_compact | 1 | 0 | 1 | TSDB 文件组首次上传 S3 时，是否自动进行 compact 操作。 | +| # | 参数 | 默认值 | 最小值 | 最大值 | 描述 | +|:--|:--------------|:-------|:------ |:------- | :----------------------------------------------------------- | +| 1 | s3_keeplocal | 365 | 1 | 365000 | 数据在本地保留的天数，即 data 文件在本地磁盘保留多长时间后可以上传到 S3。默认单位：天，支持 m（分钟）、h（小时）和 d（天）三个单位 | +| 2 | s3_chunkpages | 131072 | 131072 | 1048576 | 上传对象的大小阈值，与 tsdb_pagesize 参数一样，不可修改，单位为 TSDB 页 | +| 3 | s3_compact | 1 | 0 | 1 | TSDB 文件组首次上传 S3 时，是否自动进行 compact 操作 | ### 对象存储读写次数估算 @@ -168,10 +168,10 @@ s3BucketName td-test 用户界面同 S3，不同的地方在于下面三个参数的配置： -| # | 参数 | 示例值 | 描述 | -| :--- | :----------- | :--------------------------------------- | :----------------------------------------------------------- | -| 1 | s3EndPoint | https://fd2d01c73.blob.core.windows.net | Blob URL | -| 2 | s3AccessKey | fd2d01c73:veUy/iRBeWaI2YAerl+AStw6PPqg== | 冒号分隔的用户 accountId:accountKey | -| 3 | s3BucketName | test-container | Container name | +| # | 参数 | 示例值 | 描述 | +|:--|:-------------|:-----------------------------------------|:----------------------------------| +| 1 | s3EndPoint | https://fd2d01c73.blob.core.windows.net | Blob URL | +| 2 | s3AccessKey | fd2d01c73:veUy/iRBeWaI2YAerl+AStw6PPqg== | 冒号分隔的用户 accountId:accountKey | +| 3 | s3BucketName | test-container | Container name | 其中 fd2d01c73 是账户 ID；微软 Blob 存储服务只支持 Https 协议，不支持 Http。 diff --git a/docs/zh/08-operation/14-user.md b/docs/zh/08-operation/14-user.md index dd8f94892c..a894570b46 100644 --- a/docs/zh/08-operation/14-user.md +++ b/docs/zh/08-operation/14-user.md @@ -16,15 +16,15 @@ create user user_name pass'password' [sysinfo {1|0}] [createdb {1|0}] ``` 相关参数说明如下。 -- user_name：最长为 23 B。 -- password：最长为 128 B，合法字符包括字母和数字以及单双引号、撇号、反斜杠和空格以外的特殊字符，且不可以为空。 +- user_name：用户名最长不超过 23 个字节。 +- password：密码长度必须为 8 到 16 位，并且至少包含大写字母、小写字母、数字、特殊字符中的三类。特殊字符包括 `! @ # $ % ^ & * ( ) - _ + = [ ] { } : ; > < ? | ~ , .`。 - sysinfo ：用户是否可以查看系统信息。1 表示可以查看，0 表示不可以查看。系统信息包括服务端配置信息、服务端各种节点信息，如 dnode、查询节点（qnode）等，以及与存储相关的信息等。默认为可以查看系统信息。 - createdb：用户是否可以创建数据库。1 表示可以创建，0 表示不可以创建。缺省值为 0。// 从 TDengine 企业版 3.3.2.0 开始支持 -如下 SQL 可以创建密码为 123456 且可以查看系统信息的用户 test。 +如下 SQL 可以创建密码为 abc123!@# 且可以查看系统信息的用户 test。 ```sql -create user test pass '123456' sysinfo 1 +create user test pass 'abc123!@#' sysinfo 1 ``` ### 查看用户 diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index 435a89c845..da86fadf2b 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -26,65 +26,67 @@ taosd 命令行参数如下 ::: ### 连接相关 -|参数名称|支持版本|参数含义| -|-----------------------|-------------------------|------------| -|firstEp | |taosd 启动时，主动连接的集群中首个 dnode 的 end point，默认值 localhost:6030| -|secondEp | |taosd 启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，无默认值| -|fqdn | |taosd 监听的服务地址，默认为所在服务器上配置的第一个 hostname| -|serverPort | |taosd 监听的端口，默认值 6030| -|compressMsgSize | |是否对 RPC 消息进行压缩；-1：所有消息都不压缩；0：所有消息都压缩；N (N>0)：只有大于 N 个字节的消息才压缩；默认值 -1| -|shellActivityTimer | |客户端向 mnode 发送心跳的时长，单位为秒，取值范围 1-120，默认值 3 | -|numOfRpcSessions | |RPC 支持的最大连接数，取值范围 100-100000，默认值 30000| -|numOfRpcThreads | |RPC 收发数据线程数目，取值范围1-50,默认值为 CPU 核数的一半| -|numOfTaskQueueThreads | |客户端处理 RPC 消息的线程数取值, 范围4-16,默认值为 CPU 核数的一半| -|rpcQueueMemoryAllowed | |dnode允许的已经收到的RPC消息占用的内存最大值，单位 bytes，取值范围 104857600-INT64_MAX，默认值为服务器内存的 1/10 | -|resolveFQDNRetryTime | 3.x 之后取消 |FQDN 解析失败时的重试次数| -|timeToGetAvailableConn | 3.3.4.x之后取消 |获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，默认值 500000| -|maxShellConns | 3.x 后取消 |允许创建的最大链接数| -|maxRetryWaitTime | |重连最大超时时间, 默认值是 10s| -|shareConnLimit |3.3.4.0 新增 |一个链接可以共享的请求的数目，取值范围 1-512，默认值 10| -|readTimeout |3.3.4.0 新增 |单个请求最小超时时间，取值范围 64-604800，单位为秒，默认值 900| +|参数名称|支持版本|动态修改|参数含义| +|-----------------------|-------------------------|-------------------------|------------| +|firstEp | |不支持动态修改 |taosd 启动时，主动连接的集群中首个 dnode 的 end point，默认值 localhost:6030| +|secondEp | |不支持动态修改 |taosd 启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，无默认值| +|fqdn | |不支持动态修改 |taosd 监听的服务地址，默认为所在服务器上配置的第一个 hostname| +|serverPort | |不支持动态修改 |taosd 监听的端口，默认值 6030| +|compressMsgSize | |支持动态修改 重启生效 |是否对 RPC 消息进行压缩；-1：所有消息都不压缩；0：所有消息都压缩；N (N>0)：只有大于 N 个字节的消息才压缩；默认值 -1| +|shellActivityTimer | |支持动态修改 立即生效 |客户端向 mnode 发送心跳的时长，单位为秒，取值范围 1-120，默认值 3 | +|numOfRpcSessions | |支持动态修改 重启生效 |RPC 支持的最大连接数，取值范围 100-100000，默认值 30000| +|numOfRpcThreads | |支持动态修改 重启生效 |RPC 收发数据线程数目，取值范围1-50,默认值为 CPU 核数的一半| +|numOfTaskQueueThreads | |支持动态修改 重启生效 |客户端处理 RPC 消息的线程数取值, 范围4-16,默认值为 CPU 核数的一半| +|rpcQueueMemoryAllowed | |支持动态修改 立即生效 |dnode允许的已经收到的RPC消息占用的内存最大值，单位 bytes，取值范围 104857600-INT64_MAX，默认值为服务器内存的 1/10 | +|resolveFQDNRetryTime | 3.x 之后取消 |不支持动态修改 |FQDN 解析失败时的重试次数| +|timeToGetAvailableConn | 3.3.4.x之后取消 |支持动态修改 重启生效 |获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，默认值 500000| +|maxShellConns | 3.x 后取消 |支持动态修改 重启生效 |允许创建的最大链接数| +|maxRetryWaitTime | |支持动态修改 重启生效 |重连最大超时时间, 默认值是 10s| +|shareConnLimit |3.3.4.0 新增 |支持动态修改 重启生效 |一个链接可以共享的请求的数目，取值范围 1-512，默认值 10| +|readTimeout |3.3.4.0 新增 |支持动态修改 重启生效 |单个请求最小超时时间，取值范围 64-604800，单位为秒，默认值 900| ### 监控相关 -|参数名称|支持版本|参数含义| -|-----------------------|----------|-| -|monitor | |是否收集监控数据并上报，0：关闭；1:打开；默认值 0| -|monitorFqdn | |taosKeeper 服务所在服务器的 FQDN，默认值 无| -|monitorPort | |taosKeeper 服务所监听的端口号，默认值 6043| -|monitorInterval | |监控数据库记录系统参数（CPU/内存）的时间间隔，单位是秒，取值范围 1-200000 ，默认值 30| -|monitorMaxLogs | |缓存的待上报日志条数| -|monitorComp | |是否采用压缩方式上报监控日志时| -|monitorLogProtocol | |是否打印监控日志| -|monitorForceV2 | |是否使用 V2 版本协议上报| -|telemetryReporting | |是否上传 telemetry，0：不上传，1：上传，默认值 1| -|telemetryServer | |telemetry 服务器地址| -|telemetryPort | |telemetry 服务器端口编号| -|telemetryInterval | |telemetry 上传时间间隔，单位为秒，默认 43200| -|crashReporting | |是否上传 crash 信息；0：不上传，1：上传；默认值 1| +|参数名称|支持版本|动态修改|参数含义| +|-----------------------|----------|-------------------------|-| +|monitor | |支持动态修改 立即生效 |是否收集监控数据并上报，0：关闭；1:打开；默认值 0| +|monitorFqdn | |支持动态修改 重启生效 |taosKeeper 服务所在服务器的 FQDN，默认值 无| +|monitorPort | |支持动态修改 重启生效 |taosKeeper 服务所监听的端口号，默认值 6043| +|monitorInterval | |支持动态修改 立即生效 |监控数据库记录系统参数（CPU/内存）的时间间隔，单位是秒，取值范围 1-200000 ，默认值 30| +|monitorMaxLogs | |支持动态修改 立即生效 |缓存的待上报日志条数| +|monitorComp | |支持动态修改 重启生效 |是否采用压缩方式上报监控日志时| +|monitorLogProtocol | |支持动态修改 立即生效 |是否打印监控日志| +|monitorForceV2 | |支持动态修改 立即生效 |是否使用 V2 版本协议上报| +|telemetryReporting | |支持动态修改 立即生效 |是否上传 telemetry，0：不上传，1：上传，默认值 1| +|telemetryServer | |不支持动态修改 |telemetry 服务器地址| +|telemetryPort | |不支持动态修改 |telemetry 服务器端口编号| +|telemetryInterval | |支持动态修改 立即生效 |telemetry 上传时间间隔，单位为秒，默认 43200| +|crashReporting | |支持动态修改 立即生效 |是否上传 crash 信息；0：不上传，1：上传；默认值 1| ### 查询相关 -|参数名称|支持版本|参数含义| -|------------------------|----------|-| -|countAlwaysReturnValue | |count/hyperloglog 函数在输入数据为空或者 NULL 的情况下是否返回值；0：返回空行，1：返回；默认值 1；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了 TSMA 时，且相应的组或窗口内数据为空或者 NULL，对应的组或窗口将不返回查询结果；注意此参数客户端和服务端值应保持一致| -|tagFilterCache | |是否缓存标签过滤结果| -|maxNumOfDistinctRes | |允许返回的 distinct 结果最大行数，默认值 10 万，最大允许值 1 亿| -|queryBufferSize | |暂不生效| -|queryRspPolicy | |查询响应策略| -|filterScalarMode | |强制使用标量过滤模式，0：关闭；1：开启，默认值 0| -|queryPlannerTrace | |内部参数，查询计划是否输出详细日志| -|queryNodeChunkSize | |内部参数，查询计划的块大小| -|queryUseNodeAllocator | |内部参数，查询计划的分配方法| -|queryMaxConcurrentTables| |内部参数，查询计划的并发数目| -|queryRsmaTolerance | |内部参数，用于判定查询哪一级 rsma 数据时的容忍时间，单位为毫秒| -|enableQueryHb | |内部参数，是否发送查询心跳消息| -|pqSortMemThreshold | |内部参数，排序使用的内存阈值| +|参数名称|支持版本|动态修改|参数含义| +|------------------------|----------|-------------------------|-| +|countAlwaysReturnValue | |支持动态修改 立即生效 |count/hyperloglog 函数在输入数据为空或者 NULL 的情况下是否返回值；0：返回空行，1：返回；默认值 1；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了 TSMA 时，且相应的组或窗口内数据为空或者 NULL，对应的组或窗口将不返回查询结果；注意此参数客户端和服务端值应保持一致| +|tagFilterCache | |不支持动态修改 |是否缓存标签过滤结果| +|queryBufferSize | |支持动态修改 重启生效 |暂不生效| +|queryRspPolicy | |支持动态修改 立即生效 |查询响应策略| +|queryUseMemoryPool | |不支持动态修改 |查询是否使用内存池管理内存，默认值：1（打开）; 0: 关闭，1: 打开| +|minReservedMemorySize | |不支持动态修改 |最小预留的系统可用内存数量，除预留外的内存都可以被用于查询，单位：MB，默认预留大小为系统物理内存的 20%，取值范围 1024 - 1000000000| +|singleQueryMaxMemorySize| |不支持动态修改 |单个查询在单个节点(dnode)上可以使用的内存上限，超过该上限将返回错误，单位：MB，默认值：0（无上限），取值范围 0 - 1000000000| +|filterScalarMode | |不支持动态修改 |强制使用标量过滤模式，0：关闭；1：开启，默认值 0| +|queryPlannerTrace | |支持动态修改 立即生效 |内部参数，查询计划是否输出详细日志| +|queryNodeChunkSize | |支持动态修改 立即生效 |内部参数，查询计划的块大小| +|queryUseNodeAllocator | |支持动态修改 立即生效 |内部参数，查询计划的分配方法| +|queryMaxConcurrentTables| |不支持动态修改 |内部参数，查询计划的并发数目| +|queryRsmaTolerance | |不支持动态修改 |内部参数，用于判定查询哪一级 rsma 数据时的容忍时间，单位为毫秒| +|enableQueryHb | |支持动态修改 立即生效 |内部参数，是否发送查询心跳消息| +|pqSortMemThreshold | |不支持动态修改 |内部参数，排序使用的内存阈值| ### 区域相关 -|参数名称|支持版本|参数含义| -|-----------------|----------|-| -|timezone | |时区；缺省从系统中动态获取当前的时区设置| -|locale | |系统区位信息及编码格式，缺省从系统中获取| -|charset | |字符集编码，缺省从系统中获取| +|参数名称|支持版本|动态修改|参数含义| +|-----------------|----------|-------------------------|-| +|timezone | |不支持动态修改 |时区；缺省从系统中动态获取当前的时区设置| +|locale | |不支持动态修改 |系统区位信息及编码格式，缺省从系统中获取| +|charset | |不支持动态修改 |字符集编码，缺省从系统中获取| :::info 1. 为应对多时区的数据写入和查询问题，TDengine 采用 Unix 时间戳(Unix Timestamp)来记录和存储时间戳。Unix 时间戳的特点决定了任一时刻不论在任何时区，产生的时间戳均一致。需要注意的是，Unix 时间戳是在客户端完成转换和记录。为了确保客户端其他形式的时间转换为正确的 Unix 时间戳，需要设置正确的时区。 @@ -162,159 +164,161 @@ charset 的有效值是 UTF-8。 ::: ### 存储相关 -|参数名称|支持版本|参数含义| -|--------------------|----------|-| -|dataDir | |数据文件目录，所有的数据文件都将写入该目录，默认值 /var/lib/taos| -|tempDir | |指定所有系统运行过程中的临时文件生成的目录，默认值 /tmp| -|minimalDataDirGB | |dataDir 指定的时序数据存储目录所需要保留的最小空间，单位 GB，默认值 2| -|minimalTmpDirGB | |tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，默认值 1| -|minDiskFreeSize |3.1.1.0 后|当某块磁盘上的可用空间小于等于这个阈值时，该磁盘将不再被选择用于生成新的数据文件，单位为字节，取值范围 52428800-1073741824，默认值为 52428800；企业版参数| -|s3MigrateIntervalSec|3.3.4.3 后|本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600；企业版参数| -|s3MigrateEnabled |3.3.4.3 后|是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1；企业版参数| -|s3Accesskey |3.3.4.3 后|冒号分隔的用户 SecretId:SecretKey，例如 AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E；企业版参数| -|s3Endpoint |3.3.4.3 后|用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 保持一致，否则无法访问；企业版参数| -|s3BucketName |3.3.4.3 后|存储桶名称，减号后面是用户注册 COS 服务的 AppId，其中 AppId 是 COS 特有，AWS 和阿里云都没有，配置时需要作为 bucket name 的一部分，使用减号分隔；参数值均为字符串类型，但不需要引号；例如 test0711-1309024725；企业版参数| -|s3PageCacheSize |3.3.4.3 后|S3 page cache 缓存页数目，取值范围 4-1048576，单位为页，默认值 4096；企业版参数| -|s3UploadDelaySec |3.3.4.3 后|data 文件持续多长时间不再变动后上传至 S3，取值范围 1-2592000 (30天），单位为秒，默认值 60；企业版参数| -|cacheLazyLoadThreshold| |内部参数，缓存的装载策略| +|参数名称|支持版本|动态修改|参数含义| +|--------------------|----------|-------------------------|-| +|dataDir | |不支持动态修改 |数据文件目录，所有的数据文件都将写入该目录，默认值 /var/lib/taos| +|diskIDCheckEnabled | |不支持动态修改 |在 3.3.4.3 后，在重启 dnode 时增加了检查 dataDir 所在磁盘 id 是否发生改变，0:进行检查，1:不进行检查；默认值：1| +|tempDir | |不支持动态修改 |指定所有系统运行过程中的临时文件生成的目录，默认值 /tmp| +|minimalDataDirGB | |不支持动态修改 |dataDir 指定的时序数据存储目录所需要保留的最小空间，单位 GB，默认值 2| +|minimalTmpDirGB | |不支持动态修改 |tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，默认值 1| +|minDiskFreeSize |3.1.1.0 后|支持动态修改 立即生效 |当某块磁盘上的可用空间小于等于这个阈值时，该磁盘将不再被选择用于生成新的数据文件，单位为字节，取值范围 52428800-1073741824，默认值为 52428800；企业版参数| +|s3MigrateIntervalSec|3.3.4.3 后|支持动态修改 立即生效 |本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600；企业版参数| +|s3MigrateEnabled |3.3.4.3 后|支持动态修改 立即生效 |是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1；企业版参数| +|s3Accesskey |3.3.4.3 后|支持动态修改 重启生效 |冒号分隔的用户 SecretId:SecretKey，例如 AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E；企业版参数| +|s3Endpoint |3.3.4.3 后|支持动态修改 重启生效 |用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 保持一致，否则无法访问；企业版参数| +|s3BucketName |3.3.4.3 后|支持动态修改 重启生效 |存储桶名称，减号后面是用户注册 COS 服务的 AppId，其中 AppId 是 COS 特有，AWS 和阿里云都没有，配置时需要作为 bucket name 的一部分，使用减号分隔；参数值均为字符串类型，但不需要引号；例如 test0711-1309024725；企业版参数| +|s3PageCacheSize |3.3.4.3 后|支持动态修改 重启生效 |S3 page cache 缓存页数目，取值范围 4-1048576，单位为页，默认值 4096；企业版参数| +|s3UploadDelaySec |3.3.4.3 后|支持动态修改 立即生效 |data 文件持续多长时间不再变动后上传至 S3，取值范围 1-2592000 (30天），单位为秒，默认值 60；企业版参数| +|cacheLazyLoadThreshold| |支持动态修改 立即生效 |内部参数，缓存的装载策略| ### 集群相关 -|参数名称|支持版本|参数含义| -|--------------------------|----------|-| -|supportVnodes | |dnode 支持的最大 vnode 数目，取值范围 0-4096，默认值 CPU 核数的 2 倍 + 5| -|numOfCommitThreads | |落盘线程的最大数量，取值范围 0-1024，默认值为 4| -|numOfMnodeReadThreads | |mnode 的 Read 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| -|numOfVnodeQueryThreads | |vnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）| -|numOfVnodeFetchThreads | |vnode 的 Fetch 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| -|numOfVnodeRsmaThreads | |vnode 的 Rsma 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| -|numOfQnodeQueryThreads | |qnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）| -|numOfSnodeSharedThreads | |snode 的共享线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）| -|numOfSnodeUniqueThreads | |snode 的独占线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）| -|ratioOfVnodeStreamThreads | |流计算使用 vnode 线程的比例，取值范围 0.01-4，默认值 4| -|ttlUnit | |ttl 参数的单位，取值范围 1-31572500，单位为秒，默认值 86400| -|ttlPushInterval | |ttl 检测超时频率，取值范围 1-100000，单位为秒，默认值 10| -|ttlChangeOnWrite | |ttl 到期时间是否伴随表的修改操作改变；0：不改变，1：改变；默认值为 0| -|ttlBatchDropNum | |ttl 一批删除子表的数目，最小值为 0，默认值 10000| -|retentionSpeedLimitMB | |数据在不同级别硬盘上迁移时的速度限制，取值范围 0-1024，单位 MB，默认值 0，表示不限制| -|maxTsmaNum | |集群内可创建的TSMA个数；取值范围 0-3；默认值 3| -|tmqMaxTopicNum | |订阅最多可建立的 topic 数量；取值范围 1-10000；默认值为 20| -|tmqRowSize | |订阅数据块的最大记录条数，取值范围 1-1000000，默认值 4096| -|audit | |审计功能开关；企业版参数| -|auditInterval | |审计数据上报的时间间隔；企业版参数| -|auditCreateTable | |是否针对创建子表开启申计功能；企业版参数| -|encryptAlgorithm | |数据加密算法；企业版参数| -|encryptScope | |加密范围；企业版参数| -|enableWhiteList | |白名单功能开关；企业版参数| -|syncLogBufferMemoryAllowed| |一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围 104857600-INT64_MAX，默认值 服务器内存的 1/10，3.1.3.2/3.3.2.13 版本开始生效 | -|syncElectInterval | |内部参数，用于同步模块调试| -|syncHeartbeatInterval | |内部参数，用于同步模块调试| -|syncHeartbeatTimeout | |内部参数，用于同步模块调试| -|syncSnapReplMaxWaitN | |内部参数，用于同步模块调试| -|syncSnapReplMaxWaitN | |内部参数，用于同步模块调试| -|arbHeartBeatIntervalSec | |内部参数，用于同步模块调试| -|arbCheckSyncIntervalSec | |内部参数，用于同步模块调试| -|arbSetAssignedTimeoutSec | |内部参数，用于同步模块调试| -|mndSdbWriteDelta | |内部参数，用于 mnode 模块调试| -|mndLogRetention | |内部参数，用于 mnode 模块调试| -|skipGrant | |内部参数，用于授权检查| -|trimVDbIntervalSec | |内部参数，用于删除过期数据| -|ttlFlushThreshold | |内部参数，ttl 定时器的频率| -|compactPullupInterval | |内部参数，数据重整定时器的频率| -|walFsyncDataSizeLimit | |内部参数，WAL 进行 FSYNC 的阈值| -|transPullupInterval | |内部参数，mnode 执行事务的重试间隔| -|mqRebalanceInterval | |内部参数，消费者再平衡的时间间隔| -|uptimeInterval | |内部参数，用于记录系统启动时间| -|timeseriesThreshold | |内部参数，用于统计用量| -|udf | |是否启动 UDF 服务；0：不启动，1：启动；默认值为 0 | -|udfdResFuncs | |内部参数，用于 UDF 结果集设置| -|udfdLdLibPath | |内部参数，表示 UDF 装载的库路径| +|参数名称|支持版本|动态修改|参数含义| +|--------------------------|----------|-------------------------|-| +|supportVnodes | |支持动态修改 立即生效 |dnode 支持的最大 vnode 数目，取值范围 0-4096，默认值 CPU 核数的 2 倍 + 5| +|numOfCommitThreads | |支持动态修改 重启生效 |落盘线程的最大数量，取值范围 0-1024，默认值为 4| +|numOfMnodeReadThreads | |支持动态修改 重启生效 |mnode 的 Read 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| +|numOfVnodeQueryThreads | |支持动态修改 重启生效 |vnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）| +|numOfVnodeFetchThreads | |支持动态修改 重启生效 |vnode 的 Fetch 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| +|numOfVnodeRsmaThreads | |支持动态修改 重启生效 |vnode 的 Rsma 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| +|numOfQnodeQueryThreads | |支持动态修改 重启生效 |qnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）| +|numOfSnodeSharedThreads | |支持动态修改 重启生效 |snode 的共享线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）| +|numOfSnodeUniqueThreads | |支持动态修改 重启生效 |snode 的独占线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）| +|ratioOfVnodeStreamThreads | |支持动态修改 重启生效 |流计算使用 vnode 线程的比例，取值范围 0.01-4，默认值 4| +|ttlUnit | |不支持动态修改 |ttl 参数的单位，取值范围 1-31572500，单位为秒，默认值 86400| +|ttlPushInterval | |支持动态修改 立即生效 |ttl 检测超时频率，取值范围 1-100000，单位为秒，默认值 10| +|ttlChangeOnWrite | |支持动态修改 立即生效 |ttl 到期时间是否伴随表的修改操作改变；0：不改变，1：改变；默认值为 0| +|ttlBatchDropNum | |支持动态修改 立即生效 |ttl 一批删除子表的数目，最小值为 0，默认值 10000| +|retentionSpeedLimitMB | |支持动态修改 立即生效 |数据在不同级别硬盘上迁移时的速度限制，取值范围 0-1024，单位 MB，默认值 0，表示不限制| +|maxTsmaNum | |支持动态修改 立即生效 |集群内可创建的TSMA个数；取值范围 0-3；默认值 3| +|tmqMaxTopicNum | |支持动态修改 立即生效 |订阅最多可建立的 topic 数量；取值范围 1-10000；默认值为 20| +|tmqRowSize | |支持动态修改 立即生效 |订阅数据块的最大记录条数，取值范围 1-1000000，默认值 4096| +|audit | |支持动态修改 立即生效 |审计功能开关；企业版参数| +|auditInterval | |支持动态修改 立即生效 |审计数据上报的时间间隔；企业版参数| +|auditCreateTable | |支持动态修改 立即生效 |是否针对创建子表开启申计功能；企业版参数| +|encryptAlgorithm | |不支持动态修改 |数据加密算法；企业版参数| +|encryptScope | |不支持动态修改 |加密范围；企业版参数| +|enableWhiteList | |支持动态修改 立即生效 |白名单功能开关；企业版参数| +|syncLogBufferMemoryAllowed| |支持动态修改 立即生效 |一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围 104857600-INT64_MAX，默认值 服务器内存的 1/10，3.1.3.2/3.3.2.13 版本开始生效 | +|syncElectInterval | |不支持动态修改 |内部参数，用于同步模块调试| +|syncHeartbeatInterval | |不支持动态修改 |内部参数，用于同步模块调试| +|syncHeartbeatTimeout | |不支持动态修改 |内部参数，用于同步模块调试| +|syncSnapReplMaxWaitN | |支持动态修改 立即生效 |内部参数，用于同步模块调试| +|arbHeartBeatIntervalSec | |支持动态修改 立即生效 |内部参数，用于同步模块调试| +|arbCheckSyncIntervalSec | |支持动态修改 立即生效 |内部参数，用于同步模块调试| +|arbSetAssignedTimeoutSec | |支持动态修改 立即生效 |内部参数，用于同步模块调试| +|mndSdbWriteDelta | |支持动态修改 立即生效 |内部参数，用于 mnode 模块调试| +|mndLogRetention | |支持动态修改 立即生效 |内部参数，用于 mnode 模块调试| +|skipGrant | |不支持动态修改 |内部参数，用于授权检查| +|trimVDbIntervalSec | |支持动态修改 立即生效 |内部参数，用于删除过期数据| +|ttlFlushThreshold | |支持动态修改 立即生效 |内部参数，ttl 定时器的频率| +|compactPullupInterval | |支持动态修改 立即生效 |内部参数，数据重整定时器的频率| +|walFsyncDataSizeLimit | |支持动态修改 立即生效 |内部参数，WAL 进行 FSYNC 的阈值| +|transPullupInterval | |支持动态修改 立即生效 |内部参数，mnode 执行事务的重试间隔| +|mqRebalanceInterval | |支持动态修改 立即生效 |内部参数，消费者再平衡的时间间隔| +|uptimeInterval | |支持动态修改 立即生效 |内部参数，用于记录系统启动时间| +|timeseriesThreshold | |支持动态修改 立即生效 |内部参数，用于统计用量| +|udf | |支持动态修改 重启生效 |是否启动 UDF 服务；0：不启动，1：启动；默认值为 0 | +|udfdResFuncs | |支持动态修改 重启生效 |内部参数，用于 UDF 结果集设置| +|udfdLdLibPath | |支持动态修改 重启生效 |内部参数，表示 UDF 装载的库路径| ### 流计算参数 -|参数名称|支持版本|参数含义| -|-----------------------|----------|-| -|disableStream | |流计算的启动开关| -|streamBufferSize | |控制内存中窗口状态缓存的大小，默认值为 128MB| -|streamAggCnt | |内部参数，并发进行聚合计算的数目| -|checkpointInterval | |内部参数，checkponit 同步间隔| -|concurrentCheckpoint | |内部参数，是否并发检查 checkpoint| -|maxStreamBackendCache | |内部参数，流计算使用的最大缓存| -|streamSinkDataRate | |内部参数，用于控制流计算结果的写入速度| +|参数名称|支持版本|动态修改|参数含义| +|-----------------------|-------------------------|----------|-| +|disableStream | |支持动态修改 重启生效 |流计算的启动开关| +|streamBufferSize | |支持动态修改 重启生效 |控制内存中窗口状态缓存的大小，默认值为 128MB| +|streamAggCnt | |不支持动态修改 |内部参数，并发进行聚合计算的数目| +|checkpointInterval | |支持动态修改 重启生效 |内部参数，checkponit 同步间隔| +|concurrentCheckpoint | |支持动态修改 立即生效 |内部参数，是否并发检查 checkpoint| +|maxStreamBackendCache | |支持动态修改 立即生效 |内部参数，流计算使用的最大缓存| +|streamSinkDataRate | |支持动态修改 重启生效 |内部参数，用于控制流计算结果的写入速度| ### 日志相关 -|参数名称|支持版本|参数含义| -|----------------|----------|-| -|logDir | |日志文件目录，运行日志将写入该目录，默认值 /var/log/taos| -|minimalLogDirGB | |日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位 GB，默认值 1| -|numOfLogLines | |单个日志文件允许的最大行数，默认值 10,000,000| -|asyncLog | |日志写入模式，0：同步，1：异步，默认值 1| -|logKeepDays | |日志文件的最长保存时间，单位：天，默认值 0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taosdlog.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件| -|slowLogThreshold|3.3.3.0 后|慢查询门限值，大于等于门限值认为是慢查询，单位秒，默认值 3 | -|slowLogMaxLen |3.3.3.0 后|慢查询日志最大长度，取值范围 1-16384，默认值 4096| -|slowLogScope |3.3.3.0 后|慢查询记录类型，取值范围 ALL/QUERY/INSERT/OTHERS/NONE，默认值 QUERY| -|slowLogExceptDb |3.3.3.0 后|指定的数据库不上报慢查询，仅支持配置换一个数据库| -|debugFlag | |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块）| -|tmrDebugFlag | |定时器模块的日志开关，取值范围同上| -|uDebugFlag | |共用功能模块的日志开关，取值范围同上| -|rpcDebugFlag | |rpc 模块的日志开关，取值范围同上| -|qDebugFlag | |query 模块的日志开关，取值范围同上| -|dDebugFlag | |dnode 模块的日志开关，取值范围同上| -|vDebugFlag | |vnode 模块的日志开关，取值范围同上| -|mDebugFlag | |mnode 模块的日志开关，取值范围同上| -|azDebugFlag |3.3.4.3 后|S3 模块的日志开关，取值范围同上| -|sDebugFlag | |sync 模块的日志开关，取值范围同上| -|tsdbDebugFlag | |tsdb 模块的日志开关，取值范围同上| -|tqDebugFlag | |tq 模块的日志开关，取值范围同上| -|fsDebugFlag | |fs 模块的日志开关，取值范围同上| -|udfDebugFlag | |udf 模块的日志开关，取值范围同上| -|smaDebugFlag | |sma 模块的日志开关，取值范围同上| -|idxDebugFlag | |index 模块的日志开关，取值范围同上| -|tdbDebugFlag | |tdb 模块的日志开关，取值范围同上| -|metaDebugFlag | |meta 模块的日志开关，取值范围同上| -|stDebugFlag | |stream 模块的日志开关，取值范围同上| -|sndDebugFlag | |snode 模块的日志开关，取值范围同上| +|参数名称|支持版本|动态修改|参数含义| +|----------------|-------------------------|----------|-| +|logDir | |不支持动态修改 |日志文件目录，运行日志将写入该目录，默认值 /var/log/taos| +|minimalLogDirGB | ｜不支持动态修改 |日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位 GB，默认值 1| +|numOfLogLines | ｜支持动态修改 立即生效 |单个日志文件允许的最大行数，默认值 10,000,000| +|asyncLog | |支持动态修改 立即生效 |日志写入模式，0：同步，1：异步，默认值 1| +|logKeepDays | ｜支持动态修改 立即生效 |日志文件的最长保存时间，单位：天，默认值 0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taosdlog.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件| +|slowLogThreshold|3.3.3.0 后|支持动态修改 立即生效 |慢查询门限值，大于等于门限值认为是慢查询，单位秒，默认值 3 | +|slowLogMaxLen |3.3.3.0 后|支持动态修改 立即生效 |慢查询日志最大长度，取值范围 1-16384，默认值 4096| +|slowLogScope |3.3.3.0 后|支持动态修改 立即生效 |慢查询记录类型，取值范围 ALL/QUERY/INSERT/OTHERS/NONE，默认值 QUERY| +|slowLogExceptDb |3.3.3.0 后|支持动态修改 立即生效 |指定的数据库不上报慢查询，仅支持配置换一个数据库| +|debugFlag | |支持动态修改 立即生效 |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块）| +|tmrDebugFlag | |支持动态修改 立即生效 |定时器模块的日志开关，取值范围同上| +|uDebugFlag | |支持动态修改 立即生效 |共用功能模块的日志开关，取值范围同上| +|rpcDebugFlag | |支持动态修改 立即生效 |rpc 模块的日志开关，取值范围同上| +|qDebugFlag | |支持动态修改 立即生效 |query 模块的日志开关，取值范围同上| +|dDebugFlag | |支持动态修改 立即生效 |dnode 模块的日志开关，取值范围同上| +|vDebugFlag | |支持动态修改 立即生效 |vnode 模块的日志开关，取值范围同上| +|mDebugFlag | |支持动态修改 立即生效 |mnode 模块的日志开关，取值范围同上| +|azDebugFlag |3.3.4.3 后|支持动态修改 立即生效 |S3 模块的日志开关，取值范围同上| +|sDebugFlag | |支持动态修改 立即生效 |sync 模块的日志开关，取值范围同上| +|tsdbDebugFlag | |支持动态修改 立即生效 |tsdb 模块的日志开关，取值范围同上| +|tqDebugFlag | |支持动态修改 立即生效 |tq 模块的日志开关，取值范围同上| +|fsDebugFlag | |支持动态修改 立即生效 |fs 模块的日志开关，取值范围同上| +|udfDebugFlag | |支持动态修改 立即生效 |udf 模块的日志开关，取值范围同上| +|smaDebugFlag | |支持动态修改 立即生效 |sma 模块的日志开关，取值范围同上| +|idxDebugFlag | |支持动态修改 立即生效 |index 模块的日志开关，取值范围同上| +|tdbDebugFlag | |支持动态修改 立即生效 |tdb 模块的日志开关，取值范围同上| +|metaDebugFlag | |支持动态修改 立即生效 |meta 模块的日志开关，取值范围同上| +|stDebugFlag | |支持动态修改 立即生效 |stream 模块的日志开关，取值范围同上| +|sndDebugFlag | |支持动态修改 立即生效 |snode 模块的日志开关，取值范围同上| ### 调试相关 -|参数名称|支持版本|参数含义| -|--------------------|----------|-| -|enableCoreFile | |crash 时是否生成 core 文件，0：不生成，1：生成；默认值 1| -|configDir | |配置文件所在目录| -|scriptDir | |内部测试工具的脚本目录| -|assert | |断言控制开关，默认值 0| -|randErrorChance | |内部参数，用于随机失败测试| -|randErrorDivisor | |内部参数，用于随机失败测试| -|randErrorScope | |内部参数，用于随机失败测试| -|safetyCheckLevel | |内部参数，用于随机失败测试| -|experimental | |内部参数，用于一些实验特性| -|simdEnable |3.3.4.3 后|内部参数，用于测试 SIMD 加速| -|AVX512Enable |3.3.4.3 后|内部参数，用于测试 AVX512 加速| -|rsyncPort | |内部参数，用于调试流计算| -|snodeAddress | |内部参数，用于调试流计算| -|checkpointBackupDir | |内部参数，用于恢复 snode 数据| -|enableAuditDelete | |内部参数，用于测试审计功能| -|slowLogThresholdTest| |内部参数，用于测试慢日志| -|bypassFlag |3.3.4.5 后|内部参数，用于短路测试，0：正常写入，1：写入消息在 taos 客户端发送 RPC 消息前返回，2：写入消息在 taosd 服务端收到 RPC 消息后返回，4：写入消息在 taosd 服务端写入内存缓存前返回，8：写入消息在 taosd 服务端数据落盘前返回；默认值 0| +|参数名称|支持版本|动态修改|参数含义| +|--------------------|-------------------------|----------|-| +|enableCoreFile | |支持动态修改 立即生效 |crash 时是否生成 core 文件，0：不生成，1：生成；默认值 1| +|configDir | |不支持动态修改 |配置文件所在目录| +|forceReadConfig | |不支持动态修改 |强制使用配置文件中的参数，0:使用持久化的配置参数，1:使用配置文件中的配置参数；默认值 0| +|scriptDir | |不支持动态修改 |内部测试工具的脚本目录| +|assert | |不支持动态修改 |断言控制开关，默认值 0| +|randErrorChance | |支持动态修改 立即生效 |内部参数，用于随机失败测试| +|randErrorDivisor | |支持动态修改 立即生效 |内部参数，用于随机失败测试| +|randErrorScope | |支持动态修改 立即生效 |内部参数，用于随机失败测试| +|safetyCheckLevel | |支持动态修改 立即生效 |内部参数，用于随机失败测试| +|experimental | |支持动态修改 立即生效 |内部参数，用于一些实验特性| +|simdEnable |3.3.4.3 后|不支持动态修改 |内部参数，用于测试 SIMD 加速| +|AVX512Enable |3.3.4.3 后|不支持动态修改 |内部参数，用于测试 AVX512 加速| +|rsyncPort | |不支持动态修改 |内部参数，用于调试流计算| +|snodeAddress | |支持动态修改 重启生效 |内部参数，用于调试流计算| +|checkpointBackupDir | |支持动态修改 重启生效 |内部参数，用于恢复 snode 数据| +|enableAuditDelete | |不支持动态修改 |内部参数，用于测试审计功能| +|slowLogThresholdTest| |不支持动态修改 |内部参数，用于测试慢日志| +|bypassFlag |3.3.4.5 后|支持动态修改 立即生效 |内部参数，用于短路测试，0：正常写入，1：写入消息在 taos 客户端发送 RPC 消息前返回，2：写入消息在 taosd 服务端收到 RPC 消息后返回，4：写入消息在 taosd 服务端写入内存缓存前返回，8：写入消息在 taosd 服务端数据落盘前返回；默认值 0| ### 压缩参数 -|参数名称|支持版本|参数含义| -|------------|----------|-| -|fPrecision | |设置 float 类型浮点数压缩精度 ，取值范围 0.1 ~ 0.00000001 ，默认值 0.00000001 , 小于此值的浮点数尾数部分将被截断| -|dPrecision | |设置 double 类型浮点数压缩精度 , 取值范围 0.1 ~ 0.0000000000000001 ， 默认值 0.0000000000000001 ， 小于此值的浮点数尾数部分将被截取| -|lossyColumn |3.3.0.0 前|对 float 和/或 double 类型启用 TSZ 有损压缩；取值范围 float/double/none；默认值 none，表示关闭无损压缩| -|ifAdtFse | |在启用 TSZ 有损压缩时，使用 FSE 算法替换 HUFFMAN 算法，FSE 算法压缩速度更快，但解压稍慢，追求压缩速度可选用此算法；0：关闭，1：打开；默认值为 0| -|maxRange | |内部参数，用于有损压缩设置| -|curRange | |内部参数，用于有损压缩设置| -|compressor | |内部参数，用于有损压缩设置| +|参数名称|支持版本|动态修改|参数含义| +|------------|----------|-------------------------|-| +|fPrecision | |支持动态修改 立即生效 |设置 float 类型浮点数压缩精度 ，取值范围 0.1 ~ 0.00000001 ，默认值 0.00000001 , 小于此值的浮点数尾数部分将被截断| +|dPrecision | |支持动态修改 立即生效 |设置 double 类型浮点数压缩精度 , 取值范围 0.1 ~ 0.0000000000000001 ， 默认值 0.0000000000000001 ， 小于此值的浮点数尾数部分将被截取| +|lossyColumn |3.3.0.0 前|不支持动态修改 |对 float 和/或 double 类型启用 TSZ 有损压缩；取值范围 float/double/none；默认值 none，表示关闭无损压缩| +|ifAdtFse | |支持动态修改 重启生效 |在启用 TSZ 有损压缩时，使用 FSE 算法替换 HUFFMAN 算法，FSE 算法压缩速度更快，但解压稍慢，追求压缩速度可选用此算法；0：关闭，1：打开；默认值为 0| +|maxRange | |支持动态修改 重启生效 |内部参数，用于有损压缩设置| +|curRange | |支持动态修改 重启生效 |内部参数，用于有损压缩设置| +|compressor | |支持动态修改 重启生效 |内部参数，用于有损压缩设置| **补充说明** -1. 在 3.2.0.0 ~ 3.3.0.0（不包含）版本生效，启用该参数后不能回退到升级前的版本 -2. TSZ 压缩算法是通过数据预测技术完成的压缩，所以更适合有规律变化的数据 -3. TSZ 压缩时间会更长一些，如果您的服务器 CPU 空闲多，存储空间小的情况下适合选用 -4. 示例：对 float 和 double 类型都启用有损压缩 +1. 在 3.4.0.0 之后，所有配置参数都将被持久化到本地存储，重启数据库服务后，将默认使用持久化的配置参数列表；如果您希望继续使用 config 文件中配置的参数，需设置 forceReadConfig 为 1。 +2. 在 3.2.0.0 ~ 3.3.0.0（不包含）版本生效，启用该参数后不能回退到升级前的版本 +3. TSZ 压缩算法是通过数据预测技术完成的压缩，所以更适合有规律变化的数据 +4. TSZ 压缩时间会更长一些，如果您的服务器 CPU 空闲多，存储空间小的情况下适合选用 +5. 示例：对 float 和 double 类型都启用有损压缩 ```shell lossyColumns float|double ``` -5. 配置需重启服务生效，重启如果在 taosd 日志中看到以下内容，表明配置已生效： +6. 配置需重启服务生效，重启如果在 taosd 日志中看到以下内容，表明配置已生效： ```sql 02/22 10:49:27.607990 00002933 UTL lossyColumns float|double ``` diff --git a/docs/zh/14-reference/01-components/02-taosc.md b/docs/zh/14-reference/01-components/02-taosc.md index 6b6f621aaf..46b8854d27 100755 --- a/docs/zh/14-reference/01-components/02-taosc.md +++ b/docs/zh/14-reference/01-components/02-taosc.md @@ -9,101 +9,102 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在 ## 配置参数 ### 连接相关 -|参数名称|支持版本|参数含义| -|----------------------|----------|-------------| -|firstEp | |启动时，主动连接的集群中首个 dnode 的 endpoint，缺省值：hostname:6030，若无法获取该服务器的 hostname，则赋值为 localhost| -|secondEp | |启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，没有缺省值| -|compressMsgSize | |是否对 RPC 消息进行压缩；-1：所有消息都不压缩；0：所有消息都压缩；N (N>0)：只有大于 N 个字节的消息才压缩；缺省值 -1| -|shellActivityTimer | |客户端向 mnode 发送心跳的时长，单位为秒，取值范围 1-120，默认值 3| -|numOfRpcSessions | |RPC 支持的最大连接数，取值范围 100-100000，缺省值 30000| -|numOfRpcThreads | |RPC 收发数据线程数目，取值范围1-50,默认值为 CPU 核数的一半| -|numOfTaskQueueThreads | |客户端处理 RPC消息的线程数, 范围4-16,默认值为 CPU 核数的一半| -|timeToGetAvailableConn| 3.3.4.*之后取消 |获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，缺省值 500000| -|useAdapter | |内部参数，是否使用 taosadapter，影响 CSV 文件导入| -|shareConnLimit |3.3.4.0 新增|内部参数，一个链接可以共享的查询数目，取值范围 1-256，默认值 10| -|readTimeout |3.3.4.0 新增|内部参数，最小超时时间，取值范围 64-604800，单位为秒，默认值 900| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|firstEp | |支持动态修改 立即生效 |启动时，主动连接的集群中首个 dnode 的 endpoint，缺省值：hostname:6030，若无法获取该服务器的 hostname，则赋值为 localhost| +|secondEp | |支持动态修改 立即生效 |启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，没有缺省值| +|serverPort | |支持动态修改 立即生效 |taosd 监听的端口，默认值 6030| +|compressMsgSize | |支持动态修改 立即生效 |是否对 RPC 消息进行压缩；-1：所有消息都不压缩；0：所有消息都压缩；N (N>0)：只有大于 N 个字节的消息才压缩；缺省值 -1| +|shellActivityTimer | |不支持动态修改 |客户端向 mnode 发送心跳的时长，单位为秒，取值范围 1-120，默认值 3| +|numOfRpcSessions | |支持动态修改 立即生效 |RPC 支持的最大连接数，取值范围 100-100000，缺省值 30000| +|numOfRpcThreads | |不支持动态修改 |RPC 收发数据线程数目，取值范围1-50,默认值为 CPU 核数的一半| +|numOfTaskQueueThreads | |不支持动态修改 |客户端处理 RPC消息的线程数, 范围4-16,默认值为 CPU 核数的一半| +|timeToGetAvailableConn| 3.3.4.*之后取消 |不支持动态修改 |获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，缺省值 500000| +|useAdapter | |支持动态修改 立即生效 |内部参数，是否使用 taosadapter，影响 CSV 文件导入| +|shareConnLimit |3.3.4.0 新增|不支持动态修改 |内部参数，一个链接可以共享的查询数目，取值范围 1-256，默认值 10| +|readTimeout |3.3.4.0 新增|不支持动态修改 |内部参数，最小超时时间，取值范围 64-604800，单位为秒，默认值 900| ### 查询相关 -|参数名称|支持版本|参数含义| -|---------------------------------|---------|-| -|countAlwaysReturnValue | |count/hyperloglog 函数在输入数据为空或者 NULL 的情况下是否返回值；0：返回空行，1：返回；默认值 1；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了 TSMA 时，且相应的组或窗口内数据为空或者 NULL，对应的组或窗口将不返回查询结果；注意此参数客户端和服务端值应保持一致| -|keepColumnName | |Last、First、LastRow 函数查询且未指定别名时，自动设置别名为列名（不含函数名），因此 order by 子句如果引用了该列名将自动引用该列对应的函数；1：表示自动设置别名为列名(不包含函数名)，0：表示不自动设置别名；缺省值：0| -|multiResultFunctionStarReturnTags|3.3.3.0 后|查询超级表时，last(\*)/last_row(\*)/first(\*) 是否返回标签列；查询普通表、子表时，不受该参数影响；0：不返回标签列，1：返回标签列；缺省值：0；该参数设置为 0 时，last(\*)/last_row(\*)/first(\*) 只返回超级表的普通列；为 1 时，返回超级表的普通列和标签列| -|metaCacheMaxSize | |指定单个客户端元数据缓存大小的最大值，单位 MB；缺省值 -1，表示无限制| -|maxTsmaCalcDelay | |查询时客户端可允许的 tsma 计算延迟，若 tsma 的计算延迟大于配置值，则该 TSMA 将不会被使用；取值范围 600s - 86400s，即 10 分钟 - 1 小时；缺省值：600 秒| -|tsmaDataDeleteMark | |TSMA 计算的历史数据中间结果保存时间，单位为毫秒；取值范围 >= 3600000，即大于等于1h；缺省值：86400000，即 1d | -|queryPolicy | |查询语句的执行策略，1：只使用 vnode，不使用 qnode；2：没有扫描算子的子任务在 qnode 执行，带扫描算子的子任务在 vnode 执行；3：vnode 只运行扫描算子，其余算子均在 qnode 执行；缺省值：1| -|queryTableNotExistAsEmpty | |查询表不存在时是否返回空结果集；false：返回错误；true：返回空结果集；缺省值 false| -|querySmaOptimize | |sma index 的优化策略，0：表示不使用 sma index，永远从原始数据进行查询；1：表示使用 sma index，对符合的语句，直接从预计算的结果进行查询；缺省值：0| -|queryPlannerTrace | |内部参数，查询计划是否输出详细日志| -|queryNodeChunkSize | |内部参数，查询计划的块大小| -|queryUseNodeAllocator | |内部参数，查询计划的分配方法| -|queryMaxConcurrentTables | |内部参数，查询计划的并发数目| -|enableQueryHb | |内部参数，是否发送查询心跳消息| -|minSlidingTime | |内部参数，sliding 的最小允许值| -|minIntervalTime | |内部参数，interval 的最小允许值| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|countAlwaysReturnValue | |支持动态修改 立即生效 |count/hyperloglog 函数在输入数据为空或者 NULL 的情况下是否返回值；0：返回空行，1：返回；默认值 1；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了 TSMA 时，且相应的组或窗口内数据为空或者 NULL，对应的组或窗口将不返回查询结果；注意此参数客户端和服务端值应保持一致| +|keepColumnName | |支持动态修改 立即生效 |Last、First、LastRow 函数查询且未指定别名时，自动设置别名为列名（不含函数名），因此 order by 子句如果引用了该列名将自动引用该列对应的函数；1：表示自动设置别名为列名(不包含函数名)，0：表示不自动设置别名；缺省值：0| +|multiResultFunctionStarReturnTags|3.3.3.0 后|支持动态修改 立即生效 |查询超级表时，last(\*)/last_row(\*)/first(\*) 是否返回标签列；查询普通表、子表时，不受该参数影响；0：不返回标签列，1：返回标签列；缺省值：0；该参数设置为 0 时，last(\*)/last_row(\*)/first(\*) 只返回超级表的普通列；为 1 时，返回超级表的普通列和标签列| +|metaCacheMaxSize | |支持动态修改 立即生效 |指定单个客户端元数据缓存大小的最大值，单位 MB；缺省值 -1，表示无限制| +|maxTsmaCalcDelay | |支持动态修改 立即生效 |查询时客户端可允许的 tsma 计算延迟，若 tsma 的计算延迟大于配置值，则该 TSMA 将不会被使用；取值范围 600s - 86400s，即 10 分钟 - 1 小时；缺省值：600 秒| +|tsmaDataDeleteMark | |支持动态修改 立即生效 |TSMA 计算的历史数据中间结果保存时间，单位为毫秒；取值范围 >= 3600000，即大于等于1h；缺省值：86400000，即 1d | +|queryPolicy | |支持动态修改 立即生效 |查询语句的执行策略，1：只使用 vnode，不使用 qnode；2：没有扫描算子的子任务在 qnode 执行，带扫描算子的子任务在 vnode 执行；3：vnode 只运行扫描算子，其余算子均在 qnode 执行；缺省值：1| +|queryTableNotExistAsEmpty | |支持动态修改 立即生效 |查询表不存在时是否返回空结果集；false：返回错误；true：返回空结果集；缺省值 false| +|querySmaOptimize | |支持动态修改 立即生效 |sma index 的优化策略，0：表示不使用 sma index，永远从原始数据进行查询；1：表示使用 sma index，对符合的语句，直接从预计算的结果进行查询；缺省值：0| +|queryPlannerTrace | |支持动态修改 立即生效 |内部参数，查询计划是否输出详细日志| +|queryNodeChunkSize | |支持动态修改 立即生效 |内部参数，查询计划的块大小| +|queryUseNodeAllocator | |支持动态修改 立即生效 |内部参数，查询计划的分配方法| +|queryMaxConcurrentTables | |不支持动态修改 |内部参数，查询计划的并发数目| +|enableQueryHb | |支持动态修改 立即生效 |内部参数，是否发送查询心跳消息| +|minSlidingTime | |支持动态修改 立即生效 |内部参数，sliding 的最小允许值| +|minIntervalTime | |支持动态修改 立即生效 |内部参数，interval 的最小允许值| ### 写入相关 -|参数名称|支持版本|参数含义| -|------------------------------|----------|-| -|smlChildTableName | |schemaless 自定义的子表名的 key，无缺省值| -|smlAutoChildTableNameDelimiter| |schemaless tag 之间的连接符，连起来作为子表名，无缺省值| -|smlTagName | |schemaless tag 为空时默认的 tag 名字，缺省值 "_tag_null"| -|smlTsDefaultName | |schemaless 自动建表的时间列名字通过该配置设置，缺省值 "_ts"| -|smlDot2Underline | |schemaless 把超级表名中的 dot 转成下划线| -|maxInsertBatchRows | |内部参数，一批写入的最大条数| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|smlChildTableName | |支持动态修改 立即生效 |schemaless 自定义的子表名的 key，无缺省值| +|smlAutoChildTableNameDelimiter| |支持动态修改 立即生效 |schemaless tag 之间的连接符，连起来作为子表名，无缺省值| +|smlTagName | |支持动态修改 立即生效 |schemaless tag 为空时默认的 tag 名字，缺省值 "_tag_null"| +|smlTsDefaultName | |支持动态修改 立即生效 |schemaless 自动建表的时间列名字通过该配置设置，缺省值 "_ts"| +|smlDot2Underline | |支持动态修改 立即生效 |schemaless 把超级表名中的 dot 转成下划线| +|maxInsertBatchRows | |支持动态修改 立即生效 |内部参数，一批写入的最大条数| ### 区域相关 -|参数名称|支持版本|参数含义| -|-----------------|----------|-| -|timezone | |时区；缺省从系统中动态获取当前的时区设置| -|locale | |系统区位信息及编码格式，缺省从系统中获取| -|charset | |字符集编码，缺省从系统中获取| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|timezone | |支持动态修改 立即生效 |时区；缺省从系统中动态获取当前的时区设置| +|locale | |支持动态修改 立即生效 |系统区位信息及编码格式，缺省从系统中获取| +|charset | |支持动态修改 立即生效 |字符集编码，缺省从系统中获取| ### 存储相关 -|参数名称|支持版本|参数含义| -|-----------------|----------|-| -|tempDir | |指定所有运行过程中的临时文件生成的目录，Linux 平台默认值为 /tmp| -|minimalTmpDirGB | |tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，缺省值：1| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|tempDir | |支持动态修改 立即生效 |指定所有运行过程中的临时文件生成的目录，Linux 平台默认值为 /tmp| +|minimalTmpDirGB | |支持动态修改 立即生效 |tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，缺省值：1| ### 日志相关 -|参数名称|支持版本|参数含义| -|-----------------|----------|-| -|logDir | |日志文件目录，运行日志将写入该目录，缺省值：/var/log/taos| -|minimalLogDirGB | |日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位 GB，缺省值：1| -|numOfLogLines | |单个日志文件允许的最大行数，缺省值：10,000,000| -|asyncLog | |日志写入模式，0：同步，1：异步，缺省值：1| -|logKeepDays | |日志文件的最长保存时间，单位：天，缺省值：0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taoslogx.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件| -|debugFlag | |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块）| -|tmrDebugFlag | |定时器模块的日志开关，取值范围同上| -|uDebugFlag | |共用功能模块的日志开关，取值范围同上| -|rpcDebugFlag | |rpc 模块的日志开关，取值范围同上| -|jniDebugFlag | |jni 模块的日志开关，取值范围同上| -|qDebugFlag | |query 模块的日志开关，取值范围同上| -|cDebugFlag | |客户端模块的日志开关，取值范围同上| -|simDebugFlag | |内部参数，测试工具的日志开关，取值范围同上| -|tqClientDebugFlag|3.3.4.3 后|客户端模块的日志开关，取值范围同上| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|logDir | |不支持动态修改 |日志文件目录，运行日志将写入该目录，缺省值：/var/log/taos| +|minimalLogDirGB | |支持动态修改 立即生效 |日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位 GB，缺省值：1| +|numOfLogLines | |支持动态修改 立即生效 |单个日志文件允许的最大行数，缺省值：10,000,000| +|asyncLog | |支持动态修改 立即生效 |日志写入模式，0：同步，1：异步，缺省值：1| +|logKeepDays | |支持动态修改 立即生效 |日志文件的最长保存时间，单位：天，缺省值：0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taoslogx.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件| +|debugFlag | |支持动态修改 立即生效 |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块）| +|tmrDebugFlag | |支持动态修改 立即生效 |定时器模块的日志开关，取值范围同上| +|uDebugFlag | |支持动态修改 立即生效 |共用功能模块的日志开关，取值范围同上| +|rpcDebugFlag | |支持动态修改 立即生效 |rpc 模块的日志开关，取值范围同上| +|jniDebugFlag | |支持动态修改 立即生效 |jni 模块的日志开关，取值范围同上| +|qDebugFlag | |支持动态修改 立即生效 |query 模块的日志开关，取值范围同上| +|cDebugFlag | |支持动态修改 立即生效 |客户端模块的日志开关，取值范围同上| +|simDebugFlag | |支持动态修改 立即生效 |内部参数，测试工具的日志开关，取值范围同上| +|tqClientDebugFlag|3.3.4.3 后|支持动态修改 立即生效 |客户端模块的日志开关，取值范围同上| ### 调试相关 -|参数名称|支持版本|参数含义| -|-----------------|-----------|-| -|crashReporting | |是否上传 crash 到 telemetry，0：不上传，1：上传；缺省值：1| -|enableCoreFile | |crash 时是否生成 core 文件，0：不生成，1：生成；缺省值：1| -|assert | |断言控制开关，缺省值：0| -|configDir | |配置文件所在目录| -|scriptDir | |内部参数，测试用例的目录| -|randErrorChance |3.3.3.0 后|内部参数，用于随机失败测试| -|randErrorDivisor |3.3.3.0 后|内部参数，用于随机失败测试| -|randErrorScope |3.3.3.0 后|内部参数，用于随机失败测试| -|safetyCheckLevel |3.3.3.0 后|内部参数，用于随机失败测试| -|simdEnable |3.3.4.3 后|内部参数，用于测试 SIMD 加速| -|AVX512Enable |3.3.4.3 后|内部参数，用于测试 AVX512 加速| -|bypassFlag |3.3.4.5 后|内部参数，用于短路测试，0：正常写入，1：写入消息在 taos 客户端发送 RPC 消息前返回，2：写入消息在 taosd 服务端收到 RPC 消息后返回，4：写入消息在 taosd 服务端写入内存缓存前返回，8：写入消息在 taosd 服务端数据落盘前返回；缺省值：0| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|crashReporting | |支持动态修改 立即生效 |是否上传 crash 到 telemetry，0：不上传，1：上传；缺省值：1| +|enableCoreFile | |支持动态修改 立即生效 |crash 时是否生成 core 文件，0：不生成，1：生成；缺省值：1| +|assert | |不支持动态修改 |断言控制开关，缺省值：0| +|configDir | |不支持动态修改 |配置文件所在目录| +|scriptDir | |不支持动态修改 |内部参数，测试用例的目录| +|randErrorChance |3.3.3.0 后|不支持动态修改 |内部参数，用于随机失败测试| +|randErrorDivisor |3.3.3.0 后|不支持动态修改 |内部参数，用于随机失败测试| +|randErrorScope |3.3.3.0 后|不支持动态修改 |内部参数，用于随机失败测试| +|safetyCheckLevel |3.3.3.0 后|不支持动态修改 |内部参数，用于随机失败测试| +|simdEnable |3.3.4.3 后|不支持动态修改 |内部参数，用于测试 SIMD 加速| +|AVX512Enable |3.3.4.3 后|不支持动态修改 |内部参数，用于测试 AVX512 加速| +|bypassFlag |3.3.4.5 后|支持动态修改 立即生效 |内部参数，用于短路测试，0：正常写入，1：写入消息在 taos 客户端发送 RPC 消息前返回，2：写入消息在 taosd 服务端收到 RPC 消息后返回，4：写入消息在 taosd 服务端写入内存缓存前返回，8：写入消息在 taosd 服务端数据落盘前返回；缺省值：0| ### SHELL 相关 -|参数名称|支持版本|参数含义| -|-----------------|----------|-| -|enableScience | |是否开启科学计数法显示浮点数；0：不开始，1：开启；缺省值：1| +|参数名称|支持版本|动态修改|参数含义| +|----------------------|----------|-------------------------|-------------| +|enableScience | |不支持动态修改 |是否开启科学计数法显示浮点数；0：不开始，1：开启；缺省值：1| ## API diff --git a/docs/zh/14-reference/02-tools/08-taos-cli.md b/docs/zh/14-reference/02-tools/08-taos-cli.md index adecc5f760..5b204da4c2 100644 --- a/docs/zh/14-reference/02-tools/08-taos-cli.md +++ b/docs/zh/14-reference/02-tools/08-taos-cli.md @@ -54,7 +54,7 @@ taos> SET MAX_BINARY_DISPLAY_WIDTH ; - -h HOST: 要连接的 TDengine 服务端所在服务器的 FQDN, 默认为连接本地服务 - -P PORT: 指定服务端所用端口号 - -u USER: 连接时使用的用户名 -- -p PASSWORD: 连接服务端时使用的密码 +- -p PASSWORD: 连接服务端时使用的密码，特殊字符如 `! & ( ) < > ; |` 需使用字符 `\` 进行转义处理 - -?, --help: 打印出所有命令行参数 还有更多其他参数： diff --git a/docs/zh/14-reference/03-taos-sql/02-database.md b/docs/zh/14-reference/03-taos-sql/02-database.md index 9856089713..32df6c60c1 100644 --- a/docs/zh/14-reference/03-taos-sql/02-database.md +++ b/docs/zh/14-reference/03-taos-sql/02-database.md @@ -64,7 +64,8 @@ database_option: { - DURATION：数据文件存储数据的时间跨度。可以使用加单位的表示形式，如 DURATION 100h、DURATION 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。不加时间单位时默认单位为天，如 DURATION 50 表示 50 天。 - MAXROWS：文件块中记录的最大条数，默认为 4096 条。 - MINROWS：文件块中记录的最小条数，默认为 100 条。 -- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 3 倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据从而释放存储空间。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。了解更多，请点击 [关于主键时间戳](https://docs.taosdata.com/reference/taos-sql/insert/) +- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 3 倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据从而释放存储空间。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](../../operation/planning/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。了解更多，请点击 [关于主键时间戳](https://docs.taosdata.com/reference/taos-sql/insert/) + - KEEP_TIME_OFFSET：自 3.2.0.0 版本生效。删除或迁移保存时间超过 KEEP 值的数据的延迟执行时间，默认值为 0 (小时)。在数据文件保存时间超过 KEEP 后，删除或迁移操作不会立即执行，而会额外等待本参数指定的时间间隔，以实现与业务高峰期错开的目的。 - STT_TRIGGER：表示落盘文件触发文件合并的个数。开源版本固定为 1，企业版本可设置范围为 1 到 16。对于少表高频写入场景，此参数建议使用默认配置；而对于多表低频写入场景，此参数建议配置较大的值。 - SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。 @@ -135,11 +136,11 @@ alter_database_option: { 1. 如何查看 cachesize? -通过 select * from information_schema.ins_databases; 可以查看这些 cachesize 的具体值。 +通过 select * from information_schema.ins_databases; 可以查看这些 cachesize 的具体值（单位为 MB）。。 2. 如何查看 cacheload? -通过 show \.vgroups; 可以查看 cacheload +通过 show \.vgroups; 可以查看 cacheload（单位为字节）。 3. 判断 cachesize 是否够用 @@ -232,6 +233,3 @@ SHOW db_name.disk_info; 该命令本质上等同于 `select sum(data1 + data2 + data3)/sum(raw_data), sum(data1 + data2 + data3) from information_schema.ins_disk_usage where db_name="dbname"` - - - diff --git a/docs/zh/14-reference/03-taos-sql/03-table.md b/docs/zh/14-reference/03-taos-sql/03-table.md index a9f9ddb517..ca2170db8b 100644 --- a/docs/zh/14-reference/03-taos-sql/03-table.md +++ b/docs/zh/14-reference/03-taos-sql/03-table.md @@ -171,7 +171,7 @@ ALTER TABLE [db_name.]tb_name alter_table_clause alter_table_clause: { alter_table_options - | SET TAG tag_name = new_tag_value,tag_name2=new_tag2_value... + | SET TAG tag_name = new_tag_value, tag_name2=new_tag2_value ... } alter_table_options: @@ -195,7 +195,7 @@ alter_table_option: { ### 修改子表标签值 ``` -ALTER TABLE tb_name SET TAG tag_name1=new_tag_value1,tag_name2=new_tag_value2...; +ALTER TABLE tb_name SET TAG tag_name1=new_tag_value1, tag_name2=new_tag_value2 ...; ``` ### 修改表生命周期 diff --git a/docs/zh/14-reference/03-taos-sql/08-delete-data.mdx b/docs/zh/14-reference/03-taos-sql/08-delete-data.mdx index 3f9c431338..ea03552795 100644 --- a/docs/zh/14-reference/03-taos-sql/08-delete-data.mdx +++ b/docs/zh/14-reference/03-taos-sql/08-delete-data.mdx @@ -6,7 +6,7 @@ title: "删除数据" 删除数据是 TDengine 提供的根据指定时间段删除指定表或超级表中数据记录的功能，方便用户清理由于设备故障等原因产生的异常数据。 -**注意**：删除数据并不会立即释放该表所占用的磁盘空间，而是把该表的数据标记为已删除，在查询时这些数据将不会再出现，但释放磁盘空间会延迟到系统自动或用户手动进行数据重整时。 +**注意**：删除数据并不会立即释放该表所占用的磁盘空间，而是把该表的数据标记为已删除，在查询时这些数据将不会再出现，但释放磁盘空间会延迟到系统自动清理（建库参数 keep 生效）或用户手动进行数据重整时（企业版功能 compact）。 **语法：** diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index 1d96db0969..7799e6f50e 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -1817,7 +1817,7 @@ ignore_null_values: { } ``` -**功能说明**：返回指定时间截面指定列的记录值或插值。ignore_null_values 参数的值可以是 0 或 1，为 1 时表示忽略 NULL 值, 缺省值为0。 +**功能说明**：返回指定时间截面指定列的记录值或插值。ignore_null_values 参数的值可以是 0 或 1，为 1 时表示忽略 NULL 值, 缺省值为 0。 **返回数据类型**：同字段类型。 @@ -1838,9 +1838,9 @@ ignore_null_values: { - INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(3.0.2.0 版本以后支持)。 - INTERP 可以与伪列 _isfilled 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(3.0.3.0 版本以后支持)。 - INTERP 对于带复合主键的表的查询，若存在相同时间戳的数据，则只有对应的复合主键最小的数据参与运算。 -- INTERP 查询支持NEAR FILL模式, 即当需要FILL时, 使用距离当前时间点最近的数据进行插值, 当前后时间戳与当前时间断面一样近时, FILL 前一行的值. 此模式在流计算中和窗口查询中不支持。例如: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', '2023-01-01 00:10:00') FILL(NEAR)。(3.3.4.9版本及以后支持)。 -- INTERP 只有在使用FILL PREV/NEXT/NEAR 模式时才可以使用伪列 `_irowts_origin`。`_irowts_origin`在3.3.4.9版本及以后支持。 -- INTERP `RANEG`子句支持时间范围的扩展(3.3.4.9版本及以后支持), 如`RANGE('2023-01-01 00:00:00', 10s)`表示在时间点'2023-01-01 00:00:00'查找前后10s的数据进行插值, FILL PREV/NEXT/NEAR分别表示从时间点向前/向后/前后查找数据, 若时间点周围没有数据, 则使用FILL指定的值进行插值, 因此此时FILL子句必须指定值。例如: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', 10s) FILL(PREV, 1). 目前仅支持时间点和时间范围的组合, 不支持时间区间和时间范围的组合, 即不支持RANGE('2023-01-01 00:00:00', '2023-02-01 00:00:00', 1h)。所指定的时间范围规则与EVERY类似, 单位不能是年或月, 值不能为0, 不能带引号。使用该扩展时, 不支持除FILL PREV/NEXT/NEAR外的其他FILL模式, 且不能指定EVERY子句。 +- INTERP 查询支持 NEAR FILL 模式, 即当需要 FILL 时, 使用距离当前时间点最近的数据进行插值, 当前后时间戳与当前时间断面一样近时, FILL 前一行的值. 此模式在流计算中和窗口查询中不支持。例如: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', '2023-01-01 00:10:00') FILL(NEAR)(3.3.4.9 版本及以后支持)。 +- INTERP 只有在使用 FILL PREV/NEXT/NEAR 模式时才可以使用伪列 `_irowts_origin`。`_irowts_origin`在 3.3.4.9 版本及以后支持。 +- INTERP `RANGE`子句支持时间范围的扩展(3.3.4.9 版本及以后支持), 如`RANGE('2023-01-01 00:00:00', 10s)`表示在时间点 '2023-01-01 00:00:00' 查找前后 10s 的数据进行插值, FILL PREV/NEXT/NEAR 分别表示从时间点向前/向后/前后查找数据, 若时间点周围没有数据, 则使用 FILL 指定的值进行插值, 因此此时 FILL 子句必须指定值。例如: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', 10s) FILL(PREV, 1)。目前仅支持时间点和时间范围的组合, 不支持时间区间和时间范围的组合, 即不支持 RANGE('2023-01-01 00:00:00', '2023-02-01 00:00:00', 1h)。所指定的时间范围规则与 EVERY 类似, 单位不能是年或月, 值不能为 0, 不能带引号。使用该扩展时, 不支持除FILL PREV/NEXT/NEAR外的其他 FILL 模式, 且不能指定 EVERY 子句。 ### LAST diff --git a/docs/zh/14-reference/03-taos-sql/12-distinguished.md b/docs/zh/14-reference/03-taos-sql/12-distinguished.md index 4945e37779..ef8faff85a 100644 --- a/docs/zh/14-reference/03-taos-sql/12-distinguished.md +++ b/docs/zh/14-reference/03-taos-sql/12-distinguished.md @@ -124,7 +124,7 @@ INTERVAL 子句允许使用 AUTO 关键字来指定窗口偏移量，此时如 ```sql -- 有起始时间限制，从 '2018-10-03 14:38:05' 切分时间窗口 -SELECT COUNT(*) FROM meters WHERE _rowts >= '2018-10-03 14:38:05' INTERVAL (1m， AUTO); +SELECT COUNT(*) FROM meters WHERE _rowts >= '2018-10-03 14:38:05' INTERVAL (1m, AUTO); -- 无起始时间限制，不生效，仍以 0 为偏移量 SELECT COUNT(*) FROM meters WHERE _rowts < '2018-10-03 15:00:00' INTERVAL (1m, AUTO); diff --git a/docs/zh/14-reference/03-taos-sql/21-node.md b/docs/zh/14-reference/03-taos-sql/21-node.md index e3a672790c..f86dc6a86c 100644 --- a/docs/zh/14-reference/03-taos-sql/21-node.md +++ b/docs/zh/14-reference/03-taos-sql/21-node.md @@ -42,32 +42,11 @@ DROP DNODE dnode_id [force] [unsafe] ALTER DNODE dnode_id dnode_option ALTER ALL DNODES dnode_option - -dnode_option: { - 'resetLog' - | 'balance' 'value' - | 'monitor' 'value' - | 'debugFlag' 'value' - | 'monDebugFlag' 'value' - | 'vDebugFlag' 'value' - | 'mDebugFlag' 'value' - | 'cDebugFlag' 'value' - | 'httpDebugFlag' 'value' - | 'qDebugflag' 'value' - | 'sdbDebugFlag' 'value' - | 'uDebugFlag' 'value' - | 'tsdbDebugFlag' 'value' - | 'sDebugflag' 'value' - | 'rpcDebugFlag' 'value' - | 'dDebugFlag' 'value' - | 'mqttDebugFlag' 'value' - | 'wDebugFlag' 'value' - | 'tmrDebugFlag' 'value' - | 'cqDebugFlag' 'value' -} ``` -上面语法中的这些可修改配置项其配置方式与 dnode 配置文件中的配置方式相同，区别是修改是动态的立即生效，且不需要重启 dnode。 +对于支持动态修改的配置参数，您可以使用 ALTER DNODE 或 ALTER ALL DNODES 语法修改 dnode 中配置参数的值，自 3.3.4.0 后，修改的配置参数将自动持久化，即便数据库服务重启后仍然生效。 + +对于一个配置参数是否支持动态修改，请您参考以下页面：[taosd 参考手册](../01-components/01-taosd.md) value 是参数的值，需要是字符格式。如修改 dnode 1 的日志输出级别为 debug： @@ -75,6 +54,18 @@ value 是参数的值，需要是字符格式。如修改 dnode 1 的日志输 ALTER DNODE 1 'debugFlag' '143'; ``` +### 补充说明： +配置参数在 dnode 中被分为全局配置参数与局部配置参数，您可以查看 SHOW VARIABLES 或 SHOW DNODE dnode_id VARIABLE 中的 category 字段来确认配置参数属于全局配置参数还是局部配置参数： +1. 局部配置参数：您可以使用 ALTER DNODE 或 ALTER ALL DNODES 来更新某一个 dnode 或全部 dnodes 的局部配置参数。 +2. 全局配置参数：全局配置参数要求各个 dnode 保持一致，所以您只可以使用 ALTER ALL DNODES 来更新全部 dnodes 的全局配置参数。 + +配置参数是否可以动态修改，有以下三种情况： +1. 支持动态修改 立即生效 +2. 支持动态修改 重启生效 +3. 不支持动态修改 + +对于重启后生效的配置参数，您可以通过 SHOW VARIABLES 或 SHOW DNODE dnode_id VARIABLE 看到修改后的值，但是需要重启数据库服务才使其生效。 + ## 添加管理节点 ```sql @@ -137,18 +128,12 @@ SHOW CLUSTER ALIVE; ```sql ALTER LOCAL local_option - -local_option: { - 'resetLog' - | 'rpcDebugFlag' 'value' - | 'tmrDebugFlag' 'value' - | 'cDebugFlag' 'value' - | 'uDebugFlag' 'value' - | 'debugFlag' 'value' -} ``` -上面语法中的参数与在配置文件中配置客户端的用法相同，但不需要重启客户端，修改后立即生效。 +您可以使用以上语法更该客户端的配置参数，并且不需要重启客户端，修改后立即生效。 + +对于一个配置参数是否支持动态修改，请您参考以下页面：[taosc 参考手册](../01-components/02-taosc.md) + ## 查看客户端配置 diff --git a/docs/zh/14-reference/03-taos-sql/22-meta.md b/docs/zh/14-reference/03-taos-sql/22-meta.md index 74b327cbac..b269453ed2 100644 --- a/docs/zh/14-reference/03-taos-sql/22-meta.md +++ b/docs/zh/14-reference/03-taos-sql/22-meta.md @@ -303,24 +303,24 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 ## INS_STREAMS -| # | **列名** | **数据类型** | **说明** | -| --- | :----------: | ------------ | -------------------------------------------------------------------------------------------------------------------- | -| 1 | stream_name | VARCHAR(64) | 流计算名称 | -| 2 | create_time | TIMESTAMP | 创建时间 | -| 3 | sql | VARCHAR(1024) | 创建流计算时提供的 SQL 语句 | -| 4 | status | VARCHAR(20) | 流当前状态 | -| 5 | source_db | VARCHAR(64) | 源数据库 | -| 6 | target_db | VARCHAR(64) | 目的数据库 | -| 7 | target_table | VARCHAR(192) | 流计算写入的目标表 | -| 8 | watermark | BIGINT | watermark，详见 SQL 手册流式计算。需要注意，`watermark` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 9 | trigger | INT | 计算结果推送模式，详见 SQL 手册流式计算。需要注意，`trigger` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| # | **列名** | **数据类型** | **说明** | +|:----|:-----------|:------------|:--------| +| 1 | stream_name | VARCHAR(64) | 流计算名称 | +| 2 | create_time | TIMESTAMP | 创建时间 | +| 3 | sql | VARCHAR(1024) | 创建流计算时提供的 SQL 语句 | +| 4 | status | VARCHAR(20) | 流当前状态 | +| 5 | source_db | VARCHAR(64) | 源数据库 | +| 6 | target_db | VARCHAR(64) | 目的数据库 | +| 7 | target_table | VARCHAR(192) | 流计算写入的目标表 | +| 8 | watermark | BIGINT | watermark，详见 SQL 手册流式计算。需要注意，`watermark` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 9 | trigger | INT | 计算结果推送模式，详见 SQL 手册流式计算。需要注意，`trigger` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。| ## INS_USER_PRIVILEGES 注：SYSINFO 属性为 0 的用户不能查看此表。 -| # | **列名** | **数据类型** | **说明** | -| --- | :----------: | ------------ | -------------------------------------------------------------------------------------------------------------------- | +| # | **列名** | **数据类型** | **说明** | +|:----|:-----------|:------------|:--------| | 1 | user_name | VARCHAR(24) | 用户名 | 2 | privilege | VARCHAR(10) | 权限描述 | 3 | db_name | VARCHAR(65) | 数据库名称 @@ -329,17 +329,31 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 ## INS_DISK_USAGE -| # | **列名** | **数据类型** | **说明** | -| --- | :----------: | ------------ | -------------------------------------------------------------------------------------------------------------------- | -| 1 | db_name | VARCHAR(32) | 数据库名称 -| 2 | vgroup_id | INT | vgroup 的 ID -| 3 | wal | BIGINT | wal 文件大小, 单位为 K -| 4 | data1 | BIGINT | 一级存储上数据文件的大小,单位为KB -| 5 | data2 | BIGINT | 二级存储上数据文件的大小,单位为 KB -| 6 | data3 | BIGINT | 三级存储上数据文件的大小, 单位为KB -| 7 | cache_rdb | BIGINT | last/last_row 文件的大小,单位为KB -| 8 | table_meta | BIGINT | meta 文件的大小, 单位为KB -| 9 | s3 | BIGINT | s3 上占用的大小, 单位为KB -| 10 | raw_data | BIGINT | 预估的原始数据的大小, 单位为KB +| # | **列名** | **数据类型** | **说明** | +|:----|:-----------|:------------|:--------| +| 1 | db_name | VARCHAR(32) | 数据库名称 | +| 2 | vgroup_id | INT | vgroup 的 ID | +| 3 | wal | BIGINT | wal 文件大小, 单位为 K | +| 4 | data1 | BIGINT | 一级存储上数据文件的大小,单位为KB | +| 5 | data2 | BIGINT | 二级存储上数据文件的大小,单位为 KB | +| 6 | data3 | BIGINT | 三级存储上数据文件的大小, 单位为KB | +| 7 | cache_rdb | BIGINT | last/last_row 文件的大小,单位为KB | +| 8 | table_meta | BIGINT | meta 文件的大小, 单位为KB | +| 9 | s3 | BIGINT | s3 上占用的大小, 单位为KB | +| 10 | raw_data | BIGINT | 预估的原始数据的大小, 单位为KB | +## INS_FILESETS + +提供当前数据存储的文件组的相关信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :-----------: | ------------ | --------------------------------------- | +| 1 | db_name | VARCHAR(65) | 数据库名 | +| 2 | vgroup_id | INT | vgroup id | +| 3 | fileset_id | INT | 文件组 id | +| 4 | start_time | TIMESTAMP | 文件组的覆盖数据的开始时间 | +| 5 | end_time | TIMESTAMP | 文件组的覆盖数据的结束时间 | +| 6 | total_size | BIGINT | 文件组的总大小 | +| 7 | last_compact | TIMESTAMP | 最后一次压缩的时间 | +| 8 | shold_compact | bool | 是否需要压缩，true：需要，false：不需要 | diff --git a/docs/zh/14-reference/03-taos-sql/25-user.md b/docs/zh/14-reference/03-taos-sql/25-user.md index bb99bc94a2..3ddde297de 100644 --- a/docs/zh/14-reference/03-taos-sql/25-user.md +++ b/docs/zh/14-reference/03-taos-sql/25-user.md @@ -14,16 +14,16 @@ CREATE USER user_name PASS 'password' [SYSINFO {1|0}] [CREATEDB {1|0}]; 用户名最长不超过 23 个字节。 -密码最长不超过 31 个字节。密码可以包含字母、数字以及除单引号、双引号、反引号、反斜杠和空格以外的特殊字符，密码不能为空字符串。 +密码长度必须为 8 到 16 位，并且至少包含大写字母、小写字母、数字、特殊字符中的三类。特殊字符包括 `! @ # $ % ^ & * ( ) - _ + = [ ] { } : ; > < ? | ~ , .`。 `SYSINFO` 表示该用户是否能够查看系统信息。`1` 表示可以查看，`0` 表示无权查看。系统信息包括服务配置、dnode、vnode、存储等信息。缺省值为 `1`。 `CREATEDB` 表示该用户是否能够创建数据库。`1` 表示可以创建，`0` 表示无权创建。缺省值为 `0`。// 从 TDengine 企业版 3.3.2.0 开始支持 -在下面的示例中，我们创建一个密码为 `123456` 且可以查看系统信息的用户。 +在下面的示例中，我们创建一个密码为 `abc123!@#` 且可以查看系统信息的用户。 ```sql -taos> create user test pass '123456' sysinfo 1; +taos> create user test pass 'abc123!@#' sysinfo 1; Query OK, 0 of 0 rows affected (0.001254s) ``` diff --git a/docs/zh/14-reference/03-taos-sql/32-compress.md b/docs/zh/14-reference/03-taos-sql/32-compress.md index 2ef6e9e06a..60d03e0cc0 100644 --- a/docs/zh/14-reference/03-taos-sql/32-compress.md +++ b/docs/zh/14-reference/03-taos-sql/32-compress.md @@ -30,13 +30,13 @@ description: 可配置压缩算法 - 各个数据类型的默认压缩算法列表和适用范围 | 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法 | 压缩算法默认值 |压缩等级默认值| -|:------------------------------------:|:----------------:|:-----------:|:--------------------:|:----:|:------:| -| int/uint | simple8b | simple8b | lz4/zlib/zstd/xz | lz4 | medium | -| tinyint/untinyint/smallint/usmallint | simple8b | simple8b | lz4/zlib/zstd/xz | zlib | medium | -| bigint/ubigint/timestamp | simple8b/delta-i | delta-i | lz4/zlib/zstd/xz | lz4 | medium | -| float/double | delta-d | delta-d | lz4/zlib/zstd/xz/tsz | lz4 | medium | -| binary/nchar | disabled | disabled | lz4/zlib/zstd/xz | zstd | medium | -| bool | bit-packing | bit-packing | lz4/zlib/zstd/xz | zstd | medium | +|:------------------------------------:|:-------------------------:|:-----------:|:--------------------:|:----:|:------:| +| int/uint | disabled/simple8b | simple8b | lz4/zlib/zstd/xz | lz4 | medium | +| tinyint/untinyint/smallint/usmallint | disabled/simple8b | simple8b | lz4/zlib/zstd/xz | zlib | medium | +| bigint/ubigint/timestamp | disabled/simple8b/delta-i | delta-i | lz4/zlib/zstd/xz | lz4 | medium | +| float/double | disabled/delta-d | delta-d | lz4/zlib/zstd/xz/tsz | lz4 | medium | +| binary/nchar | disabled | disabled | lz4/zlib/zstd/xz | zstd | medium | +| bool | disabled/bit-packing | bit-packing | lz4/zlib/zstd/xz | zstd | medium | ## SQL 语法 diff --git a/docs/zh/14-reference/05-connector/10-cpp.mdx b/docs/zh/14-reference/05-connector/10-cpp.mdx index b434ff98a0..a836234af2 100644 --- a/docs/zh/14-reference/05-connector/10-cpp.mdx +++ b/docs/zh/14-reference/05-connector/10-cpp.mdx @@ -680,14 +680,14 @@ TDengine 客户端驱动的版本号与 TDengine 服务端的版本号是一一 - **接口说明**：清理运行环境，应用退出前应调用。 - `int taos_options(TSDB_OPTION option, const void * arg, ...)` - - **接口说明**：设置客户端选项，目前支持区域设置（`TSDB_OPTION_LOCALE`）、字符集设置（`TSDB_OPTION_CHARSET`）、时区设置（`TSDB_OPTION_TIMEZONE`）、配置文件路径设置（`TSDB_OPTION_CONFIGDIR`）。区域设置、字符集、时区默认为操作系统当前设置。 + - **接口说明**：设置客户端选项，支持区域设置（`TSDB_OPTION_LOCALE`）、字符集设置（`TSDB_OPTION_CHARSET`）、时区设置（`TSDB_OPTION_TIMEZONE`）、配置文件路径设置（`TSDB_OPTION_CONFIGDIR`）。区域设置、字符集、时区默认为操作系统当前设置。 - **参数说明**： - `option`：[入参] 设置项类型。 - `arg`：[入参] 设置项值。 - **返回值**：`0`：成功，`-1`：失败。 - `int taos_options_connection(TAOS *taos, TSDB_OPTION_CONNECTION option, const void *arg, ...)` - - **接口说明**：设置客户端每个连接选项，目前支持字符集设置（`TSDB_OPTION_CONNECTION_CHARSET`）、时区设置（`TSDB_OPTION_CONNECTION_TIMEZONE`）、用户 IP 设置（`TSDB_OPTION_CONNECTION_USER_IP`）、用户 APP 设置（`TSDB_OPTION_CONNECTION_USER_APP`）。 + - **接口说明**：设置客户端连接选项，目前支持字符集设置（`TSDB_OPTION_CONNECTION_CHARSET`）、时区设置（`TSDB_OPTION_CONNECTION_TIMEZONE`）、用户 IP 设置（`TSDB_OPTION_CONNECTION_USER_IP`）、用户 APP 设置（`TSDB_OPTION_CONNECTION_USER_APP`）。 - **参数说明**： - `taos`: [入参] taos_connect 返回的连接句柄。 - `option`：[入参] 设置项类型。 @@ -700,7 +700,7 @@ TDengine 客户端驱动的版本号与 TDengine 服务端的版本号是一一 - 同样参数多次调用该接口，以后面的为准，可以作为修改的方法。 - TSDB_OPTION_CONNECTION_CLEAR 选项用于重置所有连接选项。 - 时区和字符集重置后，使用系统的设置，user ip 和 user app 重置后为空。 - - 连接选项的值都是 string 类型，user app 参数值最大为 23，超过该值会被截断；其他参数非法时报错。 + - 连接选项的值都是 string 类型，user app 参数值最大长度为 23，超过该长度会被截断；其他参数非法时报错。 - 时区配置找不到时区文件或者不能按照规范解释时，默认为 UTC，和操作系统时区规则相同，详见 tzset 函数说明。可通过 select timezone() 查看当前连接的时区。 - 时区和字符集只在 client 侧起作用，对于在服务端的相关行为不起作用。 - 时区文件使用操作系统时区文件，可以自行更新操作系统时区文件。如果设置时区报错，请检查是否有时区文件或路径（mac:/var/db/timezone/zoneinfo, linux:/usr/share/zoneinfo）是否正确。 diff --git a/docs/zh/14-reference/05-connector/14-java.mdx b/docs/zh/14-reference/05-connector/14-java.mdx index e8554ae668..be04a9e2dc 100644 --- a/docs/zh/14-reference/05-connector/14-java.mdx +++ b/docs/zh/14-reference/05-connector/14-java.mdx @@ -172,7 +172,7 @@ WKB规范请参考[Well-Known Binary (WKB)](https://libgeos.org/specifications/w **原因**：程序没有找到依赖的本地函数库 taos。 -**解决方法**：Windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，Linux 下将建立如下软链 `ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可，macOS 下需要建立软链 `ln -s /usr/local/lib/libtaos.dylib`。 +**解决方法**：Windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，Linux 下将建立如下软链 `ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可，macOS 下需要建立软链 `ln -s /usr/local/lib/libtaos.dylib /usr/lib/libtaos.dylib`。 3. java.lang.UnsatisfiedLinkError: taos.dll Can't load AMD 64 bit on a IA 32-bit platform diff --git a/docs/zh/14-reference/09-error-code.md b/docs/zh/14-reference/09-error-code.md index 00e6a72b6f..2bebe2406b 100644 --- a/docs/zh/14-reference/09-error-code.md +++ b/docs/zh/14-reference/09-error-code.md @@ -75,6 +75,7 @@ description: TDengine 服务端的错误码列表和详细说明 | 0x80000133 | Invalid operation | 无效的或不支持的操作 | 1. 修改确认当前操作为合法有效支持的操作，检查参数有效性 2. 如果问题还未解决，保留现场和日志，github上报issue | | 0x80000134 | Invalid value | 无效值 | 保留现场和日志，github上报issue | | 0x80000135 | Invalid fqdn | 无效FQDN | 检查配置或输入的FQDN值是否正确 | +| 0x8000013C | Invalid disk id | 不合法的disk id | 建议用户检查挂载磁盘是否失效或者使用参数 diskIDCheckEnabled 来跳过磁盘检查 | @@ -136,7 +137,7 @@ description: TDengine 服务端的错误码列表和详细说明 | 0x80000350 | User already exists | Create user, 重复创建 | 确认操作是否正确 | | 0x80000351 | Invalid user | 用户不存在 | 确认操作是否正确 | | 0x80000352 | Invalid user format | 格式不正确 | 确认操作是否正确 | -| 0x80000353 | Invalid password format | 格式不正确 | 确认操作是否正确 | +| 0x80000353 | Invalid password format | 密码长度必须为 8 到 16 位，并且至少包含大写字母、小写字母、数字、特殊字符中的三类 | 确认密码字符串的格式 | | 0x80000354 | Can not get user from conn | 内部错误 | 上报issue | | 0x80000355 | Too many users | （仅企业版）用户数量超限 | 调整配置 | | 0x80000357 | Authentication failure | 密码不正确 | 确认操作是否正确 | @@ -261,6 +262,7 @@ description: TDengine 服务端的错误码列表和详细说明 | 0x80000529 | Vnode is stopped | Vnode 已经关闭 | 上报问题 | | 0x80000530 | Duplicate write request | 重复写入请求，内部错误 | 上报问题 | | 0x80000531 | Vnode query is busy | 查询忙碌 | 上报问题 | +| 0x80000540 | Vnode already exist but Dbid not match | 内部错误 | 上报问题 | ## tsdb @@ -294,6 +296,9 @@ description: TDengine 服务端的错误码列表和详细说明 | 0x80000729 | Task message error | 查询消息错误 | 保留现场和日志，github上报issue | | 0x8000072B | Task status error | 子查询状态错误 | 保留现场和日志，github上报issue | | 0x8000072F | Job not exist | 查询JOB已经不存在 | 保留现场和日志，github上报issue | +| 0x80000739 | Query memory upper limit is reached | 单个查询达到内存使用上限 | 设置合理的内存上限或调整 SQL 语句 | +| 0x8000073A | Query memory exhausted | dnode查询内存到达使用上限 | 设置合理的内存上限或调整并发查询量或增大系统内存 | +| 0x8000073B | Timeout for long time no fetch | 查询被长时间中断未恢复 | 调整应用实现尽快 fetch 数据 | ## grant diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md index af8468411c..ece7c9f309 100644 --- a/docs/zh/27-train-faq/01-faq.md +++ b/docs/zh/27-train-faq/01-faq.md @@ -280,4 +280,21 @@ TDinsight插件中展示的数据是通过taosKeeper和taosAdapter服务收集 https://docs.taosdata.com/reference/components/taosd/#%E7%9B%91%E6%8E%A7%E7%9B%B8%E5%85%B3 您可以随时关闭该参数，只需要在taos.cfg 中修改telemetryReporting为 0，然后重启数据库服务即可。 代码位于:https://github.com/taosdata/TDengine/blob/62e609c558deb764a37d1a01ba84bc35115a85a4/source/dnode/mnode/impl/src/mndTelem.c -此外，对于安全性要求极高的企业版 TDengine Enterprise 来说，此参数不会工作。 +此外，对于安全性要求极高的企业版 TDengine Enterprise 来说，此参数不会工作。 +### 31 第一次连接集群时遇到“Sync leader is unreachable”怎么办？ +报这个错，说明第一次向集群的连接是成功的，但第一次访问的IP不是mnode的leader节点，客户端试图与leader建立连接时发生错误。客户端通过EP，也就是指定的fqdn与端口号寻找leader节点，常见的报错原因有两个： + +- 集群中其他节点的端口没有打开 +- 客户端的hosts未正确配置 + +因此用户首先要检查服务端，集群的所有端口（原生连接默认6030，http连接默认6041）有无打开；其次是客户端的hosts文件中是否配置了集群所有节点的fqdn与IP信息。 +如仍无法解决，则需要联系涛思技术人员支持。 + +### 32 同一台服务器，数据库的数据目录 dataDir 不变，为什么原有数据库丢失且集群 ID 发生了变化？ +背景知识：TDengine 服务端进程（taosd）在启动时，若数据目录（dataDir，该目录在配置文件 taos.cfg 中指定）下不存在有效的数据文件子目录（如 mnode、dnode 和 vnode 等），则会自动创建这些目录。在创建新的 mnode 目录的同时，会分配一个新的集群 ID，从而产生一个新的集群。 + +原因分析：taosd 的数据目录 dataDir 可以指向多个不同的挂载点。如果这些挂载点未在 fstab 文件中配置自动挂载，服务器重启后，dataDir 将仅作为一个本地磁盘的普通目录存在，而未能按预期指向挂载的磁盘。此时，若 taosd 服务启动，它将在 dataDir 下新建目录，从而产生一个新的集群。 + +问题影响：服务器重启后，原有数据库丢失（注：并非真正丢失，只是原有的数据磁盘未挂载，暂时看不到）且集群 ID 发生变化，导致无法访问原有数据库。对于企业版用户，如果已针对集群 ID 进行授权，还会发现集群服务器的机器码未变，但原有的授权已失效。如果未针对该问题进行监控或者未及时发现并进行处理，则用户不会注意到原有数据库已经丢失，从而造成损失，增加运维成本。 + +问题解决：应在 fstab 文件中配置 dataDir 目录的自动挂载，确保 dataDir 始终指向预期的挂载点和目录，此时，再重启服务器，会找回原有的数据库和集群。在后续的版本中，我们将开发一个功能，使 taosd 在检测到启动前后 dataDir 发生变化时，在启动阶段退出，同时提供相应的错误提示。 \ No newline at end of file diff --git a/include/common/streamMsg.h b/include/common/streamMsg.h index 3db92ba58d..d410bd17e0 100644 --- a/include/common/streamMsg.h +++ b/include/common/streamMsg.h @@ -188,6 +188,7 @@ void tCleanupStreamHbMsg(SStreamHbMsg* pMsg); typedef struct { SMsgHead head; int32_t msgId; + SEpSet mndEpset; } SMStreamHbRspMsg; int32_t tEncodeStreamHbRsp(SEncoder* pEncoder, const SMStreamHbRspMsg* pRsp); diff --git a/include/common/systable.h b/include/common/systable.h index 71d29dd318..bd8ba76f4f 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -62,6 +62,7 @@ extern "C" { #define TSDB_INS_TABLE_ENCRYPTIONS "ins_encryptions" #define TSDB_INS_TABLE_TSMAS "ins_tsmas" #define TSDB_INS_DISK_USAGE "ins_disk_usage" +#define TSDB_INS_TABLE_FILESETS "ins_filesets" #define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" #define TSDB_PERFS_TABLE_SMAS "perf_smas" diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 5125c1caef..e6333d2ddc 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -19,6 +19,7 @@ #include "tarray.h" #include "tconfig.h" #include "tdef.h" +#include "tmsg.h" #ifdef __cplusplus extern "C" { @@ -30,6 +31,9 @@ extern "C" { #define SLOW_LOG_TYPE_OTHERS 0x4 #define SLOW_LOG_TYPE_ALL 0x7 +#define GLOBAL_CONFIG_FILE_VERSION 1 +#define LOCAL_CONFIG_FILE_VERSION 1 + typedef enum { DND_CA_SM4 = 1, } EEncryptAlgor; @@ -41,6 +45,8 @@ typedef enum { DND_CS_MNODE_WAL = 8, } EEncryptScope; +extern SConfig *tsCfg; + // cluster extern char tsFirst[]; extern char tsSecond[]; @@ -49,6 +55,9 @@ extern char tsLocalEp[]; extern char tsVersionName[]; extern uint16_t tsServerPort; extern int32_t tsVersion; +extern int32_t tsForceReadConfig; +extern int32_t tsdmConfigVersion; +extern int32_t tsConfigInited; extern int32_t tsStatusInterval; extern int32_t tsNumOfSupportVnodes; extern char tsEncryptAlgorithm[]; @@ -70,12 +79,23 @@ extern int32_t tsTagFilterResCacheSize; extern int32_t tsBypassFlag; // queue & threads +extern int32_t tsQueryMinConcurrentTaskNum; +extern int32_t tsQueryMaxConcurrentTaskNum; +extern int32_t tsQueryConcurrentTaskNum; +extern int32_t tsSingleQueryMaxMemorySize; +extern int8_t tsQueryUseMemoryPool; +extern int8_t tsMemPoolFullFunc; +// extern int32_t tsQueryBufferPoolSize; +extern int32_t tsMinReservedMemorySize; +extern int64_t tsCurrentAvailMemorySize; +extern int8_t tsNeedTrim; +extern int32_t tsQueryNoFetchTimeoutSec; +extern int32_t tsNumOfQueryThreads; extern int32_t tsNumOfRpcThreads; extern int32_t tsNumOfRpcSessions; extern int32_t tsShareConnLimit; extern int32_t tsReadTimeout; extern int32_t tsTimeToGetAvailableConn; -extern int32_t tsKeepAliveIdle; extern int32_t tsNumOfCommitThreads; extern int32_t tsNumOfTaskQueueThreads; extern int32_t tsNumOfMnodeQueryThreads; @@ -92,6 +112,9 @@ extern int32_t tsNumOfSnodeWriteThreads; extern int64_t tsQueueMemoryAllowed; extern int32_t tsRetentionSpeedLimitMB; +extern const char *tsAlterCompactTaskKeywords; +extern int32_t tsNumOfCompactThreads; + // sync raft extern int32_t tsElectInterval; extern int32_t tsHeartbeatInterval; @@ -155,7 +178,7 @@ extern bool tsEnableCrashReport; extern char *tsTelemUri; extern char *tsClientCrashReportUri; extern char *tsSvrCrashReportUri; -extern int8_t tsSafetyCheckLevel; +extern int32_t tsSafetyCheckLevel; enum { TSDB_SAFETY_CHECK_LEVELL_NEVER = 0, TSDB_SAFETY_CHECK_LEVELL_NORMAL = 1, @@ -244,6 +267,7 @@ extern int64_t tsmaDataDeleteMark; extern int64_t tsWalFsyncDataSizeLimit; // internal +extern bool tsDiskIDCheckEnabled; extern int32_t tsTransPullupInterval; extern int32_t tsCompactPullupInterval; extern int32_t tsMqRebalanceInterval; @@ -259,7 +283,7 @@ extern int32_t tsS3MigrateIntervalSec; extern bool tsS3MigrateEnabled; extern int32_t tsGrantHBInterval; extern int32_t tsUptimeInterval; - +extern bool tsUpdateCacheBatch; extern bool tsDisableStream; extern int64_t tsStreamBufferSize; extern int tsStreamAggCnt; @@ -289,6 +313,16 @@ void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); int8_t taosGranted(int8_t type); int32_t taosSetSlowLogScope(char *pScopeStr, int32_t *pScope); +int32_t taosPersistGlobalConfig(SArray *array, const char *path, int32_t version); +int32_t taosPersistLocalConfig(const char *path); +int32_t localConfigSerialize(SArray *array, char **serialized); +int32_t tSerializeSConfigArray(SEncoder *pEncoder, SArray *array); +int32_t tDeserializeSConfigArray(SDecoder *pDecoder, SArray *array); +int32_t setAllConfigs(SConfig *pCfg); +void printConfigNotMatch(SArray *array); + +int32_t compareSConfigItemArrays(SArray *mArray, const SArray *dArray, SArray *diffArray); +bool isConifgItemLazyMode(SConfigItem *item); #ifdef __cplusplus } #endif diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7abf38fdcf..0b6a8b3f1b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -162,6 +162,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_ANODE, TSDB_MGMT_TABLE_ANODE_FULL, TSDB_MGMT_TABLE_USAGE, + TSDB_MGMT_TABLE_FILESETS, TSDB_MGMT_TABLE_MAX, } EShowType; @@ -310,6 +311,7 @@ typedef enum ENodeType { QUERY_NODE_DESCRIBE_STMT, QUERY_NODE_RESET_QUERY_CACHE_STMT, QUERY_NODE_COMPACT_DATABASE_STMT, + QUERY_NODE_COMPACT_VGROUPS_STMT, QUERY_NODE_CREATE_FUNCTION_STMT, QUERY_NODE_DROP_FUNCTION_STMT, QUERY_NODE_CREATE_STREAM_STMT, @@ -402,6 +404,7 @@ typedef enum ENodeType { QUERY_NODE_CREATE_TSMA_STMT, QUERY_NODE_SHOW_CREATE_TSMA_STMT, QUERY_NODE_DROP_TSMA_STMT, + QUERY_NODE_SHOW_FILESETS_STMT, // logic plan node QUERY_NODE_LOGIC_PLAN_SCAN = 1000, @@ -681,7 +684,7 @@ typedef struct { int32_t tsSlowLogThreshold; int32_t tsSlowLogMaxLen; int32_t tsSlowLogScope; - int32_t tsSlowLogThresholdTest; //Obsolete + int32_t tsSlowLogThresholdTest; // Obsolete char tsSlowLogExceptDb[TSDB_DB_NAME_LEN]; } SMonitorParas; @@ -1347,6 +1350,11 @@ typedef struct { int8_t withArbitrator; int8_t encryptAlgorithm; char dnodeListStr[TSDB_DNODE_LIST_LEN]; + // 1. add auto-compact parameters + int32_t compactInterval; // minutes + int32_t compactStartTime; // minutes + int32_t compactEndTime; // minutes + int8_t compactTimeOffset; // hour } SCreateDbReq; int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); @@ -1378,6 +1386,11 @@ typedef struct { int32_t sqlLen; char* sql; int8_t withArbitrator; + // 1. add auto-compact parameters + int32_t compactInterval; + int32_t compactStartTime; + int32_t compactEndTime; + int8_t compactTimeOffset; } SAlterDbReq; int32_t tSerializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq); @@ -1510,6 +1523,10 @@ typedef struct { int32_t s3ChunkSize; int32_t s3KeepLocal; int8_t s3Compact; + int8_t compactTimeOffset; + int32_t compactInterval; + int32_t compactStartTime; + int32_t compactEndTime; int32_t tsdbPageSize; int32_t walRetentionPeriod; int32_t walRollPeriod; @@ -1617,6 +1634,7 @@ typedef struct { STimeWindow timeRange; int32_t sqlLen; char* sql; + SArray* vgroupIds; } SCompactDbReq; int32_t tSerializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq); @@ -1827,6 +1845,16 @@ int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); void tFreeSStatusReq(SStatusReq* pReq); +typedef struct { + int32_t forceReadConfig; + int32_t cver; + SArray* array; +} SConfigReq; + +int32_t tSerializeSConfigReq(void* buf, int32_t bufLen, SConfigReq* pReq); +int32_t tDeserializeSConfigReq(void* buf, int32_t bufLen, SConfigReq* pReq); +void tFreeSConfigReq(SConfigReq* pReq); + typedef struct { int32_t dnodeId; char machineId[TSDB_MACHINE_ID_LEN + 1]; @@ -1904,6 +1932,18 @@ int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp); int32_t tDeserializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp); void tFreeSStatusRsp(SStatusRsp* pRsp); +typedef struct { + int32_t forceReadConfig; + int32_t isConifgVerified; + int32_t isVersionVerified; + int32_t cver; + SArray* array; +} SConfigRsp; + +int32_t tSerializeSConfigRsp(void* buf, int32_t bufLen, SConfigRsp* pRsp); +int32_t tDeserializeSConfigRsp(void* buf, int32_t bufLen, SConfigRsp* pRsp); +void tFreeSConfigRsp(SConfigRsp* pRsp); + typedef struct { int32_t reserved; } SMTimerReq; @@ -2002,6 +2042,8 @@ typedef struct { int32_t dnodeId; int32_t numberFileset; int32_t finished; + int32_t progress; + int64_t remainingTime; } SQueryCompactProgressRsp; int32_t tSerializeSQueryCompactProgressRsp(void* buf, int32_t bufLen, SQueryCompactProgressRsp* pReq); @@ -2209,6 +2251,7 @@ typedef struct { char name[TSDB_CONFIG_OPTION_LEN + 1]; char value[TSDB_CONFIG_PATH_LEN + 1]; char scope[TSDB_CONFIG_SCOPE_LEN + 1]; + char category[TSDB_CONFIG_CATEGORY_LEN + 1]; char info[TSDB_CONFIG_INFO_LEN + 1]; } SVariablesInfo; @@ -2417,8 +2460,9 @@ int32_t tDeserializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq void tFreeSMCfgDnodeReq(SMCfgDnodeReq* pReq); typedef struct { - char config[TSDB_DNODE_CONFIG_LEN]; - char value[TSDB_DNODE_VALUE_LEN]; + char config[TSDB_DNODE_CONFIG_LEN]; + char value[TSDB_DNODE_VALUE_LEN]; + int32_t version; } SDCfgDnodeReq; int32_t tSerializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq); @@ -2758,7 +2802,7 @@ int32_t tDeserializeSResFetchReq(void* buf, int32_t bufLen, SResFetchReq* pReq); typedef struct { SMsgHead header; - uint64_t sId; + uint64_t clientId; } SSchTasksStatusReq; typedef struct { @@ -2788,7 +2832,7 @@ typedef struct SQueryNodeEpId { typedef struct { SMsgHead header; - uint64_t sId; + uint64_t clientId; SQueryNodeEpId epId; SArray* taskAction; // SArray } SSchedulerHbReq; diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 0c61aa5a51..6752287ed1 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -47,7 +47,7 @@ typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype); typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg); typedef void (*SendRspFp)(SRpcMsg* pMsg); typedef void (*RegisterBrokenLinkArgFp)(struct SRpcMsg* pMsg); -typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type); +typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type, int32_t status); typedef void (*ReportStartup)(const char* name, const char* desc); typedef struct { @@ -76,7 +76,7 @@ int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg); int32_t tmsgSendSyncReq(const SEpSet* epSet, SRpcMsg* pMsg); void tmsgSendRsp(SRpcMsg* pMsg); void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg); -void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type); +void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type, int32_t code); void tmsgReportStartup(const char* name, const char* desc); bool tmsgUpdateDnodeInfo(int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port); void tmsgUpdateDnodeEpSet(SEpSet* epset); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index c22a3da5ad..93bfe306b6 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -260,6 +260,7 @@ TD_DEF_MSG_TYPE(TDMT_MND_STREAM_TASK_RESET, "stream-reset-tasks", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_UPDATE_DNODE_INFO, "update-dnode-info", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_AUDIT, "audit", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CONFIG, "init-config", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8 diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 4abf8c27fd..780cdbf34e 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -17,10 +17,10 @@ #define _TD_MND_H_ #include "monitor.h" +#include "sync.h" #include "tmsg.h" #include "tmsgcb.h" #include "trpc.h" -#include "sync.h" #ifdef __cplusplus extern "C" { @@ -73,7 +73,7 @@ int32_t mndStart(SMnode *pMnode); */ void mndStop(SMnode *pMnode); -int32_t mndIsCatchUp(SMnode *pMnode); +int32_t mndIsCatchUp(SMnode *pMnode); ESyncRole mndGetRole(SMnode *pMnode); int64_t mndGetTerm(SMnode *pMnode); @@ -109,7 +109,7 @@ int64_t mndGetRoleTimeMs(SMnode *pMnode); * @param pMsg The request msg. * @return int32_t 0 for success, -1 for failure. */ -int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo* pQueueInfo); +int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo *pQueueInfo); int32_t mndProcessSyncMsg(SRpcMsg *pMsg); int32_t mndPreProcessQueryMsg(SRpcMsg *pMsg); void mndPostProcessQueryMsg(SRpcMsg *pMsg); diff --git a/include/libs/crypt/crypt.h b/include/libs/crypt/crypt.h index c294877a57..5f981b7ac8 100644 --- a/include/libs/crypt/crypt.h +++ b/include/libs/crypt/crypt.h @@ -21,19 +21,19 @@ extern "C" { #endif -typedef struct SCryptOpts{ - int32_t len; - char* source; - char* result; - int32_t unitLen; - unsigned char key[17]; -}SCryptOpts; +typedef struct SCryptOpts { + int32_t len; + char* source; + char* result; + int32_t unitLen; + char key[17]; +} SCryptOpts; -int32_t CBC_Decrypt(SCryptOpts *opts); -int32_t CBC_Encrypt(SCryptOpts *opts); +int32_t CBC_Decrypt(SCryptOpts* opts); +int32_t CBC_Encrypt(SCryptOpts* opts); #ifdef __cplusplus } #endif -#endif // _CRYPT_H_ \ No newline at end of file +#endif // _CRYPT_H_ \ No newline at end of file diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 130de8c030..9267a74bb7 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -29,6 +29,9 @@ extern "C" { #define DS_BUF_FULL 2 #define DS_BUF_EMPTY 3 +#define DS_FLAG_USE_MEMPOOL (1 << 0) + + struct SSDataBlock; typedef struct SDeleterRes { @@ -84,7 +87,7 @@ typedef struct SOutputData { * @param pHandle output * @return error code */ -int32_t dsCreateDataSinker(void* pSinkManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void* pParam, const char* id); +int32_t dsCreateDataSinker(void* pSinkManager, SDataSinkNode** ppDataSink, DataSinkHandle* pHandle, void* pParam, const char* id); int32_t dsDataSinkGetCacheSize(SDataSinkStat* pStat); @@ -131,6 +134,9 @@ void dsScheduleProcess(void* ahandle, void* pItem); */ void dsDestroyDataSinker(DataSinkHandle handle); +int32_t dsGetSinkFlags(DataSinkHandle handle, uint64_t* pFlags); + + #ifdef __cplusplus } #endif diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 99bf81b08d..883c5f7b99 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -58,6 +58,7 @@ typedef struct { struct SStorageAPI api; void* pWorkerCb; + bool localExec; } SReadHandle; // in queue mode, data streams are seperated by msg @@ -167,6 +168,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds); +int32_t qExecutorInit(void); void qResetTaskCode(qTaskInfo_t tinfo); void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo); diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 52e740b3df..6721d72996 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -171,6 +171,8 @@ typedef union { typedef void (*TsdReaderNotifyCbFn)(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param); +struct SFileSetReader; + typedef struct TsdReader { int32_t (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables); @@ -191,6 +193,13 @@ typedef struct TsdReader { void (*tsdSetFilesetDelimited)(void* pReader); void (*tsdSetSetNotifyCb)(void* pReader, TsdReaderNotifyCbFn notifyFn, void* param); + + // for fileset query + int32_t (*fileSetReaderOpen)(void *pVnode, struct SFileSetReader **ppReader); + int32_t (*fileSetReadNext)(struct SFileSetReader *); + int32_t (*fileSetGetEntryField)(struct SFileSetReader *, const char *, void *); + void (*fileSetReaderClose)(struct SFileSetReader **); + } TsdReader; typedef struct SStoreCacheReader { @@ -431,7 +440,7 @@ typedef struct SStateStore { int32_t (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp, void* pFile, TSKEY delMark, const char* id, int64_t ckId, int8_t type, struct SStreamFileState** ppFileState); - + int32_t (*streamStateGroupPut)(SStreamState* pState, int64_t groupId, void* value, int32_t vLen); SStreamStateCur* (*streamStateGroupGetCur)(SStreamState* pState); void (*streamStateGroupCurNext)(SStreamStateCur* pCur); diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 82d5565581..8eb30b8184 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -42,11 +42,12 @@ extern "C" { #define SHOW_CREATE_VIEW_RESULT_FIELD1_LEN (TSDB_VIEW_FNAME_LEN + 4 + VARSTR_HEADER_SIZE) #define SHOW_CREATE_VIEW_RESULT_FIELD2_LEN (TSDB_MAX_ALLOWED_SQL_LEN + VARSTR_HEADER_SIZE) -#define SHOW_LOCAL_VARIABLES_RESULT_COLS 4 +#define SHOW_LOCAL_VARIABLES_RESULT_COLS 5 #define SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_PATH_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE) -#define SHOW_LOCAL_VARIABLES_RESULT_FIELD4_LEN (TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE) +#define SHOW_LOCAL_VARIABLES_RESULT_FIELD4_LEN (TSDB_CONFIG_CATEGORY_LEN + VARSTR_HEADER_SIZE) +#define SHOW_LOCAL_VARIABLES_RESULT_FIELD5_LEN (TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE) #define COMPACT_DB_RESULT_COLS 3 #define COMPACT_DB_RESULT_FIELD1_LEN 32 @@ -110,6 +111,16 @@ typedef struct SDatabaseOptions { SValueNode* s3KeepLocalStr; int8_t s3Compact; int8_t withArbitrator; + // for auto-compact + int8_t compactTimeOffset; // hours + int32_t compactInterval; // minutes + int32_t compactStartTime; // minutes + int32_t compactEndTime; // minutes + SValueNode* pCompactTimeOffsetNode; + SValueNode* pCompactIntervalNode; + SNodeList* pCompactTimeRangeList; + // for cache + SDbCfgInfo* pDbCfg; } SDatabaseOptions; typedef struct SCreateDatabaseStmt { @@ -159,6 +170,14 @@ typedef struct SCompactDatabaseStmt { SNode* pEnd; } SCompactDatabaseStmt; +typedef struct SCompactVgroupsStmt { + ENodeType type; + SNode* pDbName; + SNodeList* vgidList; + SNode* pStart; + SNode* pEnd; +} SCompactVgroupsStmt; + typedef struct STableOptions { ENodeType type; bool commentNull; diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index ad9ac26121..3e95f1e286 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -631,7 +631,7 @@ typedef struct SDownstreamSourceNode { SQueryNodeAddr addr; uint64_t clientId; uint64_t taskId; - uint64_t schedId; + uint64_t sId; int32_t execId; int32_t fetchMsgType; bool localExec; diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index cb4e359727..6e73e94b01 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -113,6 +113,11 @@ int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64 int32_t qWorkerDbgEnableDebug(char *option); +void qWorkerRetireJob(uint64_t jobId, uint64_t clientId, int32_t errCode); + +void qWorkerRetireJobs(int64_t retireSize, int32_t errCode); + + #ifdef __cplusplus } #endif diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index cfa3f44f7f..7b3fb5bb79 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -171,7 +171,7 @@ void *rpcReallocCont(void *ptr, int64_t contLen); int32_t rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); int32_t rpcSendResponse(const SRpcMsg *pMsg); int32_t rpcRegisterBrokenLinkArg(SRpcMsg *msg); -int32_t rpcReleaseHandle(void *handle, int8_t type); // just release conn to rpc instance, no close sock +int32_t rpcReleaseHandle(void *handle, int8_t type, int32_t code); // just release conn to rpc instance, no close sock // These functions will not be called in the child process int32_t rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx); diff --git a/include/os/os.h b/include/os/os.h index 4314148685..a9c5ce5679 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -110,6 +110,7 @@ extern "C" { #include "osLz4.h" #include "osMath.h" #include "osMemory.h" +#include "osMemPool.h" #include "osRand.h" #include "osSemaphore.h" #include "osSignal.h" diff --git a/include/os/osFile.h b/include/os/osFile.h index 1c397f3042..7bd99644d3 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -83,6 +83,8 @@ int32_t taosUnLockFile(TdFilePtr pFile); int32_t taosUmaskFile(int32_t maskVal); int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *atime); +int32_t taosGetFileDiskID(const char *path, int64_t *diskid); +bool taosCheckFileDiskID(const char *path, int64_t *actDiskID, int64_t expDiskID); int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno); int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int64_t *mtime); bool taosCheckExistFile(const char *pathname); diff --git a/include/os/osLocale.h b/include/os/osLocale.h index d7a3540887..78d5f948b0 100644 --- a/include/os/osLocale.h +++ b/include/os/osLocale.h @@ -28,9 +28,9 @@ extern "C" { #define setlocale SETLOCALE_FUNC_TAOS_FORBID #endif -char *taosCharsetReplace(char *charsetstr); -void taosGetSystemLocale(char *outLocale, char *outCharset); -int32_t taosSetSystemLocale(const char *inLocale); +char *taosCharsetReplace(char *charsetstr); +void taosGetSystemLocale(char *outLocale, char *outCharset); +int32_t taosSetSystemLocale(const char *inLocale); #ifdef __cplusplus } diff --git a/include/os/osMemPool.h b/include/os/osMemPool.h new file mode 100644 index 0000000000..78cdf15b49 --- /dev/null +++ b/include/os/osMemPool.h @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef _TD_OS_MEMPOOL_H_ +#define _TD_OS_MEMPOOL_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define MEMPOOL_MAX_CHUNK_SIZE (1 << 30) +#define MEMPOOL_MIN_CHUNK_SIZE (1 << 20) + +typedef enum MemPoolEvictPolicy { + E_EVICT_ALL = 1, + E_EVICT_NONE, + E_EVICT_AUTO, + E_EVICT_MAX_VALUE, // no used +} MemPoolEvictPolicy; + +typedef struct SMemPoolJob { + uint64_t jobId; + uint64_t clientId; + + int32_t remainSession; + + int64_t allocMemSize; + int64_t maxAllocMemSize; +} SMemPoolJob; + +typedef struct SMPStatItem { + int64_t inErr; + int64_t exec; + int64_t succ; + int64_t fail; +} SMPStatItem; + +typedef struct SMPStatItemExt { + int64_t inErr; + int64_t exec; + int64_t succ; + int64_t fail; + int64_t origExec; + int64_t origSucc; + int64_t origFail; +} SMPStatItemExt; + +typedef struct SMPMemoryStat { + SMPStatItem memMalloc; + SMPStatItem memCalloc; + SMPStatItemExt memRealloc; + SMPStatItem memStrdup; + SMPStatItem memStrndup; + SMPStatItem memFree; + SMPStatItem memTrim; + + SMPStatItem chunkMalloc; + SMPStatItem chunkRecycle; + SMPStatItem chunkReUse; + SMPStatItem chunkFree; +} SMPMemoryStat; + +typedef struct SMPStatDetail { + SMPMemoryStat times; + SMPMemoryStat bytes; +} SMPStatDetail; + + +typedef void (*mpDecConcSessionNum)(void); +typedef void (*mpIncConcSessionNum)(void); +typedef void (*mpSetConcSessionNum)(int32_t); +typedef void (*mpReserveFailFp)(int64_t, int32_t); +typedef void (*mpReserveReachFp)(uint64_t, uint64_t, int32_t); +typedef void (*mpCfgUpdate)(void*, void*); + +typedef struct SMemPoolCallBack { + //mpDecConcSessionNum decSessFp; + //mpIncConcSessionNum incSessFp; + //mpSetConcSessionNum setSessFp; + mpReserveFailFp failFp; + mpReserveReachFp reachFp; + //mpCfgUpdate cfgUpdateFp; +} SMemPoolCallBack; + + +typedef struct SMemPoolCfg { + //bool reserveMode; + int64_t reserveSize; + //int32_t *upperLimitSize; //MB + //int64_t retireUnitSize; + int32_t *jobQuota; //MB + int32_t chunkSize; + int32_t threadNum; + MemPoolEvictPolicy evicPolicy; + SMemPoolCallBack cb; +} SMemPoolCfg; + +#define MEMPOOL_GET_ALLOC_SIZE(_dstat) ((_dstat)->bytes.memMalloc.succ + (_dstat)->bytes.memCalloc.succ + (_dstat)->bytes.memRealloc.succ + (_dstat)->bytes.memStrdup.succ + (_dstat)->bytes.memStrndup.succ) +#define MEMPOOL_GET_FREE_SIZE(_dstat) ((_dstat)->bytes.memRealloc.origSucc + (_dstat)->bytes.memFree.succ) +#define MEMPOOL_GET_USED_SIZE(_dstat) (MEMPOOL_GET_ALLOC_SIZE(_dstat) - MEMPOOL_GET_FREE_SIZE(_dstat)) + + +int32_t taosMemPoolOpen(char* poolName, SMemPoolCfg* cfg, void** poolHandle); +void *taosMemPoolMalloc(void* poolHandle, void* session, int64_t size, char* fileName, int32_t lineNo); +void *taosMemPoolCalloc(void* poolHandle, void* session, int64_t num, int64_t size, char* fileName, int32_t lineNo); +void *taosMemPoolRealloc(void* poolHandle, void* session, void *ptr, int64_t size, char* fileName, int32_t lineNo); +char *taosMemPoolStrdup(void* poolHandle, void* session, const char *ptr, char* fileName, int32_t lineNo); +char *taosMemPoolStrndup(void* poolHandle, void* session, const char *ptr, int64_t size, char* fileName, int32_t lineNo); +void taosMemPoolFree(void* poolHandle, void* session, void *ptr, char* fileName, int32_t lineNo); +int64_t taosMemPoolGetMemorySize(void* poolHandle, void* session, void *ptr, char* fileName, int32_t lineNo); +int32_t taosMemPoolTrim(void* poolHandle, void* session, int32_t size, char* fileName, int32_t lineNo, bool* trimed); +void *taosMemPoolMallocAlign(void* poolHandle, void* session, uint32_t alignment, int64_t size, char* fileName, int32_t lineNo); +void taosMemPoolClose(void* poolHandle); +void taosMemPoolModDestroy(void); +void taosAutoMemoryFree(void *ptr); +int32_t taosMemPoolInitSession(void* poolHandle, void** ppSession, void* pJob, char *sessionId); +void taosMemPoolDestroySession(void* poolHandle, void* session); +int32_t taosMemPoolCallocJob(uint64_t jobId, uint64_t cId, void** ppJob); +void taosMemPoolCfgUpdate(void* poolHandle, SMemPoolCfg* pCfg); +void taosMemPoolPrintStat(void* poolHandle, void* session, char* procName); +int32_t taosMemPoolTryLockPool(void* poolHandle, bool readLock); +void taosMemPoolUnLockPool(void* poolHandle, bool readLock); +void taosMemPoolGetUsedSizeBegin(void* poolHandle, int64_t* usedSize, bool* needEnd); +void taosMemPoolGetUsedSizeEnd(void* poolHandle); +int32_t taosMemPoolGetSessionStat(void* session, SMPStatDetail** ppStat, int64_t* allocSize, int64_t* maxAllocSize); +void taosMemPoolSchedTrim(void); +int32_t taosMemoryPoolInit(mpReserveFailFp, mpReserveReachFp); + + +#define taosMemPoolFreeClear(ptr) \ + do { \ + if (ptr) { \ + taosMemPoolFree((void *)ptr); \ + (ptr) = NULL; \ + } \ + } while (0) + + +#ifndef BUILD_TEST +extern void* gMemPoolHandle; +extern threadlocal void* threadPoolSession; +extern threadlocal bool threadPoolEnabled; +extern int8_t tsMemPoolFullFunc; + + +#define taosEnableMemPoolUsage(_session) do { threadPoolSession = _session; tsEnableRandErr = true;} while (0) +#define taosDisableMemPoolUsage() do { threadPoolSession = NULL; tsEnableRandErr = false;} while (0) + +#define taosSaveDisableMemPoolUsage(_enable, _randErr) do { (_enable) = threadPoolEnabled; (_randErr) = tsEnableRandErr; threadPoolEnabled = false; tsEnableRandErr = false;} while (0) +#define taosRestoreEnableMemPoolUsage(_enable, _randErr) do { threadPoolEnabled = (_enable); tsEnableRandErr = (_randErr);} while (0) + + +#define taosMemoryMalloc(_size) ((threadPoolEnabled && threadPoolSession) ? (taosMemPoolMalloc(gMemPoolHandle, threadPoolSession, _size, (char*)__FILE__, __LINE__)) : (taosMemMalloc(_size))) +#define taosMemoryCalloc(_num, _size) ((threadPoolEnabled && threadPoolSession) ? (taosMemPoolCalloc(gMemPoolHandle, threadPoolSession, _num, _size, (char*)__FILE__, __LINE__)) : (taosMemCalloc(_num, _size))) +#define taosMemoryRealloc(_ptr, _size) ((threadPoolEnabled && threadPoolSession) ? (taosMemPoolRealloc(gMemPoolHandle, threadPoolSession, _ptr, _size, (char*)__FILE__, __LINE__)) : (taosMemRealloc(_ptr, _size))) +#define taosStrdup(_ptr) ((threadPoolEnabled && threadPoolSession) ? (taosMemPoolStrdup(gMemPoolHandle, threadPoolSession, _ptr, (char*)__FILE__, __LINE__)) : (taosStrdupi(_ptr))) +#define taosStrndup(_ptr, _size) ((threadPoolEnabled && threadPoolSession) ? (taosMemPoolStrndup(gMemPoolHandle, threadPoolSession, _ptr, _size, (char*)__FILE__, __LINE__)) : (taosStrndupi(_ptr, _size))) +#define taosMemoryFree(_ptr) ((threadPoolEnabled && threadPoolSession) ? (taosMemPoolFree(gMemPoolHandle, threadPoolSession, _ptr, (char*)__FILE__, __LINE__)) : (taosMemFree(_ptr))) +#define taosMemorySize(_ptr) ((threadPoolEnabled && threadPoolSession) ? (taosMemPoolGetMemorySize(gMemPoolHandle, threadPoolSession, _ptr, (char*)__FILE__, __LINE__)) : (taosMemSize(_ptr))) +#define taosMemoryTrim(_size, _trimed) ((threadPoolEnabled && threadPoolSession) ? (taosMemPoolTrim(gMemPoolHandle, threadPoolSession, _size, (char*)__FILE__, __LINE__, _trimed)) : (taosMemTrim(_size, _trimed))) +#define taosMemoryMallocAlign(_alignment, _size) ((threadPoolEnabled && threadPoolSession) ? (taosMemPoolMallocAlign(gMemPoolHandle, threadPoolSession, _alignment, _size, (char*)__FILE__, __LINE__)) : (taosMemMallocAlign(_alignment, _size))) +#else +#define taosEnableMemoryPoolUsage(_pool, _session) +#define taosDisableMemoryPoolUsage() +#define taosSaveDisableMemoryPoolUsage() +#define taosRestoreEnableMemoryPoolUsage() + +#define taosMemoryMalloc(_size) taosMemMalloc(_size) +#define taosMemoryCalloc(_num, _size) taosMemCalloc(_num, _size) +#define taosMemoryRealloc(_ptr, _size) taosMemRealloc(_ptr, _size) +#define taosStrdup(_ptr) taosStrdupi(_ptr) +#define taosStrndup(_ptr, _size) taosStrndupi(_ptr, _size) +#define taosMemoryFree(_ptr) taosMemFree(_ptr) +#define taosMemorySize(_ptr) taosMemSize(_ptr) +#define taosMemoryTrim(_size, _trimed) taosMemTrim(_size, _trimed) +#define taosMemoryMallocAlign(_alignment, _size) taosMemMallocAlign(_alignment, _size) + +#endif + +#define taosMemoryFreeClear(ptr) \ + do { \ + if (ptr) { \ + taosMemoryFree((void *)ptr); \ + (ptr) = NULL; \ + } \ + } while (0) + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_OS_MEMPOOL_H_*/ diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 0cb4534d83..51e57820b9 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -36,35 +36,32 @@ extern "C" { #endif // ifndef ALLOW_FORBID_FUNC #endif // if !defined(WINDOWS) -// #define taosMemoryMalloc malloc -// #define taosMemoryCalloc calloc -// #define taosMemoryRealloc realloc -// #define taosMemoryFree free - int32_t taosMemoryDbgInit(); int32_t taosMemoryDbgInitRestore(); -void *taosMemoryMalloc(int64_t size); -void *taosMemoryCalloc(int64_t num, int64_t size); -void *taosMemoryRealloc(void *ptr, int64_t size); -char *taosStrdup(const char *ptr); -void taosMemoryFree(void *ptr); -int64_t taosMemorySize(void *ptr); +void *taosMemMalloc(int64_t size); +void *taosMemCalloc(int64_t num, int64_t size); +void *taosMemRealloc(void *ptr, int64_t size); +char *taosStrdupi(const char *ptr); +char *taosStrndupi(const char *ptr, int64_t size); +void taosMemFree(void *ptr); +int64_t taosMemSize(void *ptr); void taosPrintBackTrace(); -void taosMemoryTrim(int32_t size); -void *taosMemoryMallocAlign(uint32_t alignment, int64_t size); +int32_t taosMemTrim(int32_t size, bool* trimed); +void *taosMemMallocAlign(uint32_t alignment, int64_t size); #define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n)) #define TAOS_MEMCPY(_d, _s, _n) ((void)memcpy(_d, _s, _n)) #define TAOS_MEMMOVE(_d, _s, _n) ((void)memmove(_d, _s, _n)) -#define taosMemoryFreeClear(ptr) \ +#define taosMemFreeClear(ptr) \ do { \ if (ptr) { \ - taosMemoryFree((void *)ptr); \ + taosMemFree((void *)ptr); \ (ptr) = NULL; \ } \ } while (0) +#include "osMemPool.h" #define TAOS_MEMORY_REALLOC(ptr, len) \ do { \ void *tmp = taosMemoryRealloc(ptr, (len)); \ diff --git a/include/os/osString.h b/include/os/osString.h index c30437ee2c..63c7aca58c 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -25,7 +25,7 @@ typedef int32_t TdUcs4; #if !defined(DISALLOW_NCHAR_WITHOUT_ICONV)// && defined(DARWIN) #include "iconv.h" #else -typedef void *iconv_t; +typedef void *iconv_t; #endif typedef enum { M2C = 0, C2M, CM_NUM } ConvType; @@ -67,29 +67,35 @@ typedef struct { #ifdef strndup #undef strndup #endif -#define strndup STR_TO_F_FUNC_TAOS_FORBID +#define strndup STR_TO_F_FUNC_TAOS_FORBID #endif #define tstrncpy(dst, src, size) \ do { \ (void)strncpy((dst), (src), (size)); \ - (dst)[(size) - 1] = 0; \ + (dst)[(size)-1] = 0; \ } while (0) int64_t tsnprintf(char *dst, int64_t size, const char *format, ...); -#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src)) +#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src)) #define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size)) -#define TAOS_STRCAT(_dst, _src) ((void)strcat(_dst, _src)) -#define TAOS_STRNCAT(_dst, _src, len) ((void)strncat(_dst, _src, len)) +#define TAOS_STRCAT(_dst, _src) ((void)strcat(_dst, _src)) +#define TAOS_STRNCAT(_dst, _src, len) ((void)strncat(_dst, _src, len)) char *tstrdup(const char *src); +char *tstrndup(const char *str, int64_t size); int32_t taosUcs4len(TdUcs4 *ucs4); int32_t taosStr2int64(const char *str, int64_t *val); -int32_t taosStr2int16(const char *str, int16_t *val); int32_t taosStr2int32(const char *str, int32_t *val); +int32_t taosStr2int16(const char *str, int16_t *val); int32_t taosStr2int8(const char *str, int8_t *val); +int32_t taosStr2Uint64(const char *str, uint64_t *val); +int32_t taosStr2Uint32(const char *str, uint32_t *val); +int32_t taosStr2Uint16(const char *str, uint16_t *val); +int32_t taosStr2Uint8(const char *str, uint8_t *val); + iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt); void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetCxt); int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt); @@ -121,10 +127,9 @@ double taosStr2Double(const char *str, char **pEnd); float taosStr2Float(const char *str, char **pEnd); int32_t taosHex2Ascii(const char *z, uint32_t n, void **data, uint32_t *size); int32_t taosAscii2Hex(const char *z, uint32_t n, void **data, uint32_t *size); -char *taosStrndup(const char *s, int n); -//int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size); -bool isHex(const char* z, uint32_t n); -bool isValidateHex(const char* z, uint32_t n); +// int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size); +bool isHex(const char *z, uint32_t n); +bool isValidateHex(const char *z, uint32_t n); #ifdef __cplusplus } diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index d292b326d5..1f4dfe3809 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -45,6 +45,7 @@ int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma, ch int32_t taosGetTotalMemory(int64_t *totalKB); int32_t taosGetProcMemory(int64_t *usedKB); int32_t taosGetSysMemory(int64_t *usedKB); +int32_t taosGetSysAvailMemory(int64_t *availSize); int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); int32_t taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes); void taosSetDefaultProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index a5942606eb..64ef0b3829 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -159,6 +159,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_SOCKET_ERROR TAOS_DEF_ERROR_CODE(0, 0x0139) #define TSDB_CODE_UNSUPPORT_OS TAOS_DEF_ERROR_CODE(0, 0x013A) #define TSDB_CODE_TIME_ERROR TAOS_DEF_ERROR_CODE(0, 0x013B) +#define TSDB_CODE_INVALID_DISK_ID TAOS_DEF_ERROR_CODE(0, 0x013C) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) @@ -480,6 +481,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_DNODE_INVALID_EN_WHITELIST TAOS_DEF_ERROR_CODE(0, 0x0428) #define TSDB_CODE_DNODE_INVALID_MONITOR_PARAS TAOS_DEF_ERROR_CODE(0, 0x0429) #define TSDB_CODE_MNODE_STOPPED TAOS_DEF_ERROR_CODE(0, 0x042A) +#define TSDB_CODE_DNODE_INVALID_COMPACT_TASKS TAOS_DEF_ERROR_CODE(0, 0x042B) // anode #define TSDB_CODE_MND_ANODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0430) @@ -565,6 +567,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_VND_ARB_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0537) // internal #define TSDB_CODE_VND_WRITE_DISABLED TAOS_DEF_ERROR_CODE(0, 0x0538) // internal #define TSDB_CODE_VND_TTL_FLUSH_INCOMPLETION TAOS_DEF_ERROR_CODE(0, 0x0539) // internal +#define TSDB_CODE_VND_ALREADY_EXIST_BUT_NOT_MATCH TAOS_DEF_ERROR_CODE(0, 0x0540) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) @@ -643,6 +646,9 @@ int32_t taosGetErrSize(); #define TSDB_CODE_QRY_FILTER_RANGE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0736) #define TSDB_CODE_QRY_FILTER_INVALID_TYPE TAOS_DEF_ERROR_CODE(0, 0x0737) #define TSDB_CODE_QRY_TASK_SUCC_TO_PARTSUSS TAOS_DEF_ERROR_CODE(0, 0x0738) +#define TSDB_CODE_QRY_REACH_QMEM_THRESHOLD TAOS_DEF_ERROR_CODE(0, 0x0739) +#define TSDB_CODE_QRY_QUERY_MEM_EXHAUSTED TAOS_DEF_ERROR_CODE(0, 0x073A) +#define TSDB_CODE_QRY_NO_FETCH_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x073B) // grant #define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) @@ -899,6 +905,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_PAR_INVALID_ANOMALY_WIN_COL TAOS_DEF_ERROR_CODE(0, 0x2683) #define TSDB_CODE_PAR_INVALID_ANOMALY_WIN_OPT TAOS_DEF_ERROR_CODE(0, 0x2684) #define TSDB_CODE_PAR_INVALID_FORECAST_CLAUSE TAOS_DEF_ERROR_CODE(0, 0x2685) +#define TSDB_CODE_PAR_INVALID_VGID_LIST TAOS_DEF_ERROR_CODE(0, 0x2686) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 947ef67902..b1bd144d77 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -53,19 +53,30 @@ typedef enum { } ECfgDataType; typedef enum { CFG_SCOPE_SERVER, CFG_SCOPE_CLIENT, CFG_SCOPE_BOTH } ECfgScopeType; +typedef enum { CFG_CATEGORY_GLOBAL, CFG_CATEGORY_LOCAL } ECfgCategoryType; +typedef enum { CFG_ALTER_LOCAL, CFG_ALTER_DNODE, CFG_ALTER_ALL_DNODES } CfgAlterType; typedef enum { CFG_DYN_NONE = 0, CFG_DYN_SERVER = 1, CFG_DYN_CLIENT = 2, - CFG_DYN_BOTH = 3, + CFG_DYN_SERVER_LAZY = 3, + CFG_DYN_CLIENT_LAZY = 4, + CFG_DYN_BOTH_LAZY = 5, + CFG_DYN_BOTH = 6, #ifdef TD_ENTERPRISE CFG_DYN_ENT_SERVER = CFG_DYN_SERVER, CFG_DYN_ENT_CLIENT = CFG_DYN_CLIENT, + CFG_DYN_ENT_SERVER_LAZY = CFG_DYN_SERVER_LAZY, + CFG_DYN_ENT_CLIENT_LAZY = CFG_DYN_CLIENT_LAZY, + CFG_DYN_ENT_BOTH_LAZY = CFG_DYN_BOTH_LAZY, CFG_DYN_ENT_BOTH = CFG_DYN_BOTH, #else CFG_DYN_ENT_SERVER = CFG_DYN_NONE, CFG_DYN_ENT_CLIENT = CFG_DYN_NONE, + CFG_DYN_ENT_SERVER_LAZY = CFG_DYN_NONE, + CFG_DYN_ENT_CLIENT_LAZY = CFG_DYN_NONE, + CFG_DYN_ENT_BOTH_LAZY = CFG_DYN_NONE, CFG_DYN_ENT_BOTH = CFG_DYN_NONE, #endif } ECfgDynType; @@ -75,6 +86,7 @@ typedef struct SConfigItem { ECfgDataType dtype; int8_t scope; int8_t dynScope; + int8_t category; char *name; union { bool bval; @@ -99,17 +111,21 @@ typedef struct { const char *value; } SConfigPair; -typedef struct SConfig SConfig; +typedef struct SConfig SConfig; typedef struct SConfigIter SConfigIter; int32_t cfgInit(SConfig **ppCfg); int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const void *sourceStr); -int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs); // SConfigPair +int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs); // SConfigPair +int32_t cfgUpdateFromArray(SConfig *pCfg, SArray *pArgs); // SConfigItem void cfgCleanup(SConfig *pCfg); int32_t cfgGetSize(SConfig *pCfg); SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName); int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype, bool lock); -int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer); +int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem **ppItem, const char *name, const char *value, ECfgSrcType stype, + bool lock); +int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer, + CfgAlterType alterType); int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter); SConfigItem *cfgNextIter(SConfigIter *pIter); @@ -118,15 +134,16 @@ void cfgLock(SConfig *pCfg); void cfgUnLock(SConfig *pCfg); // clang-format off -int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope, int8_t dynScope); -int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, int8_t dynScope); -int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, int8_t dynScope); -int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope, int8_t dynScope); -int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope); -int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope); -int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope); -int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope); -int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope); +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddInt32Ex(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope,int8_t category); +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope,int8_t category); // clang-format on const char *cfgStypeStr(ECfgSrcType type); @@ -134,12 +151,17 @@ const char *cfgDtypeStr(ECfgDataType type); int32_t cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); int32_t cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); +int32_t cfgDumpItemCategory(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump); int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl); +SArray *taosGetLocalCfg(SConfig *pCfg); +SArray *taosGetGlobalCfg(SConfig *pCfg); +void taosSetLocalCfg(SConfig *pCfg, SArray *pArray); +void taosSetGlobalCfg(SConfig *pCfg, SArray *pArray); #ifdef __cplusplus } #endif diff --git a/include/util/tdef.h b/include/util/tdef.h index c69d4f8f19..2ee84b42bd 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -190,45 +190,22 @@ typedef enum EOperatorType { } EOperatorType; static const EOperatorType OPERATOR_ARRAY[] = { - OP_TYPE_ADD, - OP_TYPE_SUB, - OP_TYPE_MULTI, - OP_TYPE_DIV, - OP_TYPE_REM, + OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV, OP_TYPE_REM, - OP_TYPE_MINUS, + OP_TYPE_MINUS, - OP_TYPE_BIT_AND, - OP_TYPE_BIT_OR, + OP_TYPE_BIT_AND, OP_TYPE_BIT_OR, - OP_TYPE_GREATER_THAN, - OP_TYPE_GREATER_EQUAL, - OP_TYPE_LOWER_THAN, - OP_TYPE_LOWER_EQUAL, - OP_TYPE_EQUAL, - OP_TYPE_NOT_EQUAL, - OP_TYPE_IN, - OP_TYPE_NOT_IN, - OP_TYPE_LIKE, - OP_TYPE_NOT_LIKE, - OP_TYPE_MATCH, - OP_TYPE_NMATCH, + OP_TYPE_GREATER_THAN, OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_EQUAL, OP_TYPE_EQUAL, + OP_TYPE_NOT_EQUAL, OP_TYPE_IN, OP_TYPE_NOT_IN, OP_TYPE_LIKE, OP_TYPE_NOT_LIKE, OP_TYPE_MATCH, OP_TYPE_NMATCH, - OP_TYPE_IS_NULL, - OP_TYPE_IS_NOT_NULL, - OP_TYPE_IS_TRUE, - OP_TYPE_IS_FALSE, - OP_TYPE_IS_UNKNOWN, - OP_TYPE_IS_NOT_TRUE, - OP_TYPE_IS_NOT_FALSE, - OP_TYPE_IS_NOT_UNKNOWN, - //OP_TYPE_COMPARE_MAX_VALUE, + OP_TYPE_IS_NULL, OP_TYPE_IS_NOT_NULL, OP_TYPE_IS_TRUE, OP_TYPE_IS_FALSE, OP_TYPE_IS_UNKNOWN, OP_TYPE_IS_NOT_TRUE, + OP_TYPE_IS_NOT_FALSE, OP_TYPE_IS_NOT_UNKNOWN, + // OP_TYPE_COMPARE_MAX_VALUE, - OP_TYPE_JSON_GET_VALUE, - OP_TYPE_JSON_CONTAINS, + OP_TYPE_JSON_GET_VALUE, OP_TYPE_JSON_CONTAINS, - OP_TYPE_ASSIGN -}; + OP_TYPE_ASSIGN}; #define OP_TYPE_CALC_MAX OP_TYPE_BIT_OR @@ -318,6 +295,8 @@ typedef enum ELogicConditionType { #define TSDB_MAX_JSON_KEY_LEN 256 #define TSDB_AUTH_LEN 16 +#define TSDB_PASSWORD_MIN_LEN 8 +#define TSDB_PASSWORD_MAX_LEN 16 #define TSDB_PASSWORD_LEN 32 #define TSDB_USET_PASSWORD_LEN 129 #define TSDB_VERSION_LEN 32 @@ -529,6 +508,15 @@ typedef enum ELogicConditionType { #define TSDB_MIN_TABLE_TTL 0 #define TSDB_DEFAULT_TABLE_TTL 0 +#define TSDB_DEFAULT_COMPACT_INTERVAL 0 +#define TSDB_MIN_COMPACT_INTERVAL 10 // unit minute +#define TSDB_MAX_COMPACT_INTERVAL TSDB_MAX_KEEP // unit minute +#define TSDB_DEFAULT_COMPACT_START_TIME 0 +#define TSDB_DEFAULT_COMPACT_END_TIME 0 +#define TSDB_MIN_COMPACT_TIME_OFFSET 0 +#define TSDB_MAX_COMPACT_TIME_OFFSET 23 +#define TSDB_DEFAULT_COMPACT_TIME_OFFSET 0 + #define TSDB_MIN_EXPLAIN_RATIO 0 #define TSDB_MAX_EXPLAIN_RATIO 1 #define TSDB_DEFAULT_EXPLAIN_RATIO 0.001 @@ -603,6 +591,7 @@ enum { ENCRYPT_KEY_STAT_UNKNOWN = 0, ENCRYPT_KEY_STAT_UNSET, ENCRYPT_KEY_STAT_SE typedef struct { char dir[TSDB_FILENAME_LEN]; int32_t level; + int64_t diskId; int32_t primary; int8_t disable; // disable create new file } SDiskCfg; @@ -638,12 +627,13 @@ enum { RAND_ERR_MEMORY = 1, RAND_ERR_FILE = 2, RAND_ERR_NETWORK = 4 }; #define VNODE_HANDLE -3 #define CLIENT_HANDLE -5 -#define TSDB_CONFIG_OPTION_LEN 32 -#define TSDB_CONFIG_VALUE_LEN 64 -#define TSDB_CONFIG_SCOPE_LEN 8 -#define TSDB_CONFIG_NUMBER 16 -#define TSDB_CONFIG_PATH_LEN 4096 -#define TSDB_CONFIG_INFO_LEN 64 +#define TSDB_CONFIG_OPTION_LEN 32 +#define TSDB_CONFIG_VALUE_LEN 64 +#define TSDB_CONFIG_SCOPE_LEN 8 +#define TSDB_CONFIG_NUMBER 16 +#define TSDB_CONFIG_PATH_LEN 4096 +#define TSDB_CONFIG_INFO_LEN 64 +#define TSDB_CONFIG_CATEGORY_LEN 8 #define QUERY_ID_SIZE 20 #define QUERY_OBJ_ID_SIZE 18 @@ -674,6 +664,8 @@ typedef enum { ANAL_ALGO_TYPE_END, } EAnalAlgoType; +#define MIN_RESERVE_MEM_SIZE 1024 // MB + #ifdef __cplusplus } #endif diff --git a/include/util/tencode.h b/include/util/tencode.h index 270d9e75df..854b2db433 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -88,6 +88,7 @@ static int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val); static int32_t tEncodeI64v(SEncoder* pCoder, int64_t val); static int32_t tEncodeFloat(SEncoder* pCoder, float val); static int32_t tEncodeDouble(SEncoder* pCoder, double val); +static int32_t tEncodeBool(SEncoder* pCoder, bool val); static int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len); static int32_t tEncodeBinaryEx(SEncoder* pCoder, const uint8_t* val, uint32_t len); static int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t len); @@ -116,6 +117,7 @@ static int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val); static int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val); static int32_t tDecodeFloat(SDecoder* pCoder, float* val); static int32_t tDecodeDouble(SDecoder* pCoder, double* val); +static int32_t tDecodeBool(SDecoder* pCoder, bool* val); static int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len); static int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len); static int32_t tDecodeCStr(SDecoder* pCoder, char** val); @@ -205,6 +207,8 @@ static FORCE_INLINE int32_t tEncodeDouble(SEncoder* pCoder, double val) { return tEncodeU64(pCoder, v.ui); } +static int32_t tEncodeBool(SEncoder* pCoder, bool val) { return tEncodeU8(pCoder, val ? 1 : 0); } + static FORCE_INLINE int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len) { TAOS_CHECK_RETURN(tEncodeU32v(pCoder, len)); if (len) { @@ -391,6 +395,15 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) { return 0; } +static int32_t tDecodeBool(SDecoder* pCoder, bool* val) { + uint8_t v; + TAOS_CHECK_RETURN(tDecodeU8(pCoder, &v)); + if (val) { + *val = v ? true : false; + } + return 0; +} + static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) { uint32_t length = 0; diff --git a/include/util/theap.h b/include/util/theap.h index b795db6aea..abef2ab391 100644 --- a/include/util/theap.h +++ b/include/util/theap.h @@ -87,6 +87,8 @@ BoundedQueue* createBoundedQueue(uint32_t maxSize, pq_comp_fn fn, FDelete delete void taosBQSetFn(BoundedQueue* q, pq_comp_fn fn); +void taosBQClear(BoundedQueue* q); + void destroyBoundedQueue(BoundedQueue* q); /* diff --git a/include/util/tlockfree.h b/include/util/tlockfree.h index 24e83f23ca..8c884dafae 100644 --- a/include/util/tlockfree.h +++ b/include/util/tlockfree.h @@ -72,6 +72,7 @@ void taosInitRWLatch(SRWLatch *pLatch); void taosWLockLatch(SRWLatch *pLatch); void taosWUnLockLatch(SRWLatch *pLatch); void taosRLockLatch(SRWLatch *pLatch); +int32_t taosRTryLockLatch(SRWLatch *pLatch); void taosRUnLockLatch(SRWLatch *pLatch); int32_t taosWTryLockLatch(SRWLatch *pLatch); diff --git a/include/util/tlog.h b/include/util/tlog.h index 78ca6ffe15..d0e42e3660 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -77,6 +77,7 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc); void taosCloseLog(); void taosResetLog(); void taosDumpData(uint8_t *msg, int32_t len); +void taosSetNoNewFile(); void taosPrintLog(const char *flags, int32_t level, int32_t dflag, const char *format, ...) #ifdef __GNUC__ diff --git a/include/util/tpagedbuf.h b/include/util/tpagedbuf.h index 71cee62d2e..045716d3ca 100644 --- a/include/util/tpagedbuf.h +++ b/include/util/tpagedbuf.h @@ -50,7 +50,7 @@ typedef struct SDiskbasedBufStatis { * @param handle * @return */ -int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id, +int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int64_t inMemBufSize, const char* id, const char* dir); /** diff --git a/include/util/ttimer.h b/include/util/ttimer.h index 4111a8ca28..53a8f0a19f 100644 --- a/include/util/ttimer.h +++ b/include/util/ttimer.h @@ -39,6 +39,8 @@ bool taosTmrStop(tmr_h tmrId); bool taosTmrStopA(tmr_h *tmrId); +bool taosTmrIsStopped(tmr_h* timerId); + bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void *param, void *handle, tmr_h *pTmrId); #ifdef __cplusplus diff --git a/include/util/tutil.h b/include/util/tutil.h index d88363bf37..01977e27b6 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -227,6 +227,11 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, #define TAOS_UNUSED(expr) (void)(expr) +bool taosIsBigChar(char c); +bool taosIsSmallChar(char c); +bool taosIsNumberChar(char c); +bool taosIsSpecialChar(char c); + #ifdef __cplusplus } #endif diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index bc80cb2673..2543a1f3ec 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -47,11 +47,12 @@ enum { RES_TYPE__TMQ_BATCH_META, }; -#define SHOW_VARIABLES_RESULT_COLS 4 +#define SHOW_VARIABLES_RESULT_COLS 5 #define SHOW_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) #define SHOW_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE) -#define SHOW_VARIABLES_RESULT_FIELD4_LEN (TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE) +#define SHOW_VARIABLES_RESULT_FIELD4_LEN (TSDB_CONFIG_CATEGORY_LEN + VARSTR_HEADER_SIZE) +#define SHOW_VARIABLES_RESULT_FIELD5_LEN (TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE) #define TD_RES_QUERY(res) (*(int8_t*)(res) == RES_TYPE__QUERY) #define TD_RES_TMQ(res) (*(int8_t*)(res) == RES_TYPE__TMQ) diff --git a/source/client/inc/clientStmt2.h b/source/client/inc/clientStmt2.h index 64abf31bc1..01cf76965a 100644 --- a/source/client/inc/clientStmt2.h +++ b/source/client/inc/clientStmt2.h @@ -150,6 +150,7 @@ typedef struct { SStmtExecInfo exec; SStmtBindInfo bInfo; + char *db; int64_t reqid; int32_t errCode; tsem_t asyncQuerySem; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index a3ffa7f3c2..c58d3bd4df 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -689,7 +689,7 @@ void doDestroyRequest(void *p) { SRequestObj *pRequest = (SRequestObj *)p; uint64_t reqId = pRequest->requestId; - tscTrace("begin to destroy request %" PRIx64 " p:%p", reqId, pRequest); + tscDebug("begin to destroy request 0x%" PRIx64 " p:%p", reqId, pRequest); int64_t nextReqRefId = pRequest->relation.nextRefId; @@ -731,7 +731,7 @@ void doDestroyRequest(void *p) { taosMemoryFreeClear(pRequest->effectiveUser); taosMemoryFreeClear(pRequest->sqlstr); taosMemoryFree(pRequest); - tscTrace("end to destroy request %" PRIx64 " p:%p", reqId, pRequest); + tscDebug("end to destroy request %" PRIx64 " p:%p", reqId, pRequest); destroyNextReq(nextReqRefId); } @@ -955,7 +955,7 @@ void taos_init_imp(void) { taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (NULL == appInfo.pInstMap || NULL == appInfo.pInstMapByClusterId) { (void)printf("failed to allocate memory when init appInfo\n"); - tscInitRes = TSDB_CODE_OUT_OF_MEMORY; + tscInitRes = terrno; return; } taosHashSetFreeFp(appInfo.pInstMap, destroyAppInst); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 23b3b0315f..b3e288c816 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -595,6 +595,7 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { } SAppInstInfo *pInst = pAppHbMgr->pAppInstInfo; + if (code != 0) { pInst->onlineDnodes = pInst->totalDnodes ? 0 : -1; tscDebug("hb rsp error %s, update server status %d/%d", tstrerror(code), pInst->onlineDnodes, pInst->totalDnodes); @@ -1338,7 +1339,6 @@ static void *hbThreadFunc(void *param) { } void *buf = taosMemoryMalloc(tlen); if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); break; @@ -1352,7 +1352,6 @@ static void *hbThreadFunc(void *param) { SMsgSendInfo *pInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); taosMemoryFree(buf); @@ -1364,7 +1363,6 @@ static void *hbThreadFunc(void *param) { pInfo->msgType = TDMT_MND_HEARTBEAT; pInfo->param = taosMemoryMalloc(sizeof(int32_t)); if (pInfo->param == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); taosMemoryFree(buf); @@ -1372,7 +1370,7 @@ static void *hbThreadFunc(void *param) { break; } *(int32_t *)pInfo->param = i; - pInfo->paramFreeFp = taosMemoryFree; + pInfo->paramFreeFp = taosAutoMemoryFree; pInfo->requestId = generateRequestId(); pInfo->requestObjRefId = 0; @@ -1458,7 +1456,7 @@ int32_t appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key, SAppHbMgr **pAppHbMg (*pAppHbMgr)->reportBytes = 0; (*pAppHbMgr)->key = taosStrdup(key); if ((*pAppHbMgr)->key == NULL) { - TSC_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + TSC_ERR_JRET(terrno); } // init app info diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index e5e304bc79..c2a199e9c1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -126,7 +126,7 @@ int32_t taos_connect_internal(const char* ip, const char* user, const char* pass char* key = getClusterKey(user, secretEncrypt, ip, port); if (NULL == key) { - TSC_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + TSC_ERR_RET(terrno); } tscInfo("connecting to server, numOfEps:%d inUse:%d user:%s db:%s key:%s", epSet.epSet.numOfEps, epSet.epSet.inUse, user, db, key); @@ -1820,11 +1820,10 @@ int32_t doProcessMsgFromServerImpl(SRpcMsg* pMsg, SEpSet* pEpSet) { .handleRefId = pMsg->info.refId, .pEpSet = pEpSet}; - if (pMsg->code != TSDB_CODE_OUT_OF_MEMORY && pMsg->contLen > 0) { + if (pMsg->contLen > 0) { buf.pData = taosMemoryCalloc(1, pMsg->contLen); if (buf.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - pMsg->code = TSDB_CODE_OUT_OF_MEMORY; + pMsg->code = terrno; } else { (void)memcpy(buf.pData, pMsg->pCont, pMsg->contLen); } @@ -2957,7 +2956,6 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t s SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); if (NULL == param) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } int32_t code = tsem_init(¶m->sem, 0, 0); @@ -2998,7 +2996,6 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly, SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); if (param == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } int32_t code = tsem_init(¶m->sem, 0, 0); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 8d19cd6237..f576c44b09 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -292,7 +292,6 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t)); if (NULL == rid) { tscError("out of memory when taos connect to %s:%u, user:%s db:%s", ip, port, user, db); - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } *rid = pObj->id; @@ -424,7 +423,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa void *pReq = taosMemoryMalloc(msgLen); if (pReq == NULL) { - fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL); + fp(param, terrno, taos, 0, NULL); releaseTscObj(connId); return; } @@ -438,7 +437,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa SFetchWhiteListInfo *pParam = taosMemoryMalloc(sizeof(SFetchWhiteListInfo)); if (pParam == NULL) { - fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL); + fp(param, terrno, taos, 0, NULL); taosMemoryFree(pReq); releaseTscObj(connId); return; @@ -449,7 +448,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa pParam->userParam = param; SMsgSendInfo *pSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pSendInfo == NULL) { - fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL); + fp(param, terrno, taos, 0, NULL); taosMemoryFree(pParam); taosMemoryFree(pReq); releaseTscObj(connId); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index e182cd97ee..58ba39864d 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -15,9 +15,10 @@ #include "catalog.h" #include "clientInt.h" -#include "clientMonitor.h" #include "clientLog.h" +#include "clientMonitor.h" #include "cmdnodes.h" +#include "command.h" #include "os.h" #include "query.h" #include "systable.h" @@ -26,7 +27,6 @@ #include "tglobal.h" #include "tname.h" #include "tversion.h" -#include "command.h" extern SClientHbMgr clientHbMgr; @@ -40,7 +40,7 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) { setErrno(pRequest, code); if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) { - if (removeMeta(pRequest->pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type)) != 0){ + if (removeMeta(pRequest->pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type)) != 0) { tscError("failed to remove meta data for table"); } } @@ -50,7 +50,7 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) { if (pRequest->body.queryFp != NULL) { doRequestCallback(pRequest, code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -104,7 +104,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { SEpSet dstEpSet = connectRsp.epSet; if (srcEpSet.numOfEps == 1) { if (rpcSetDefaultAddr(pTscObj->pAppInfo->pTransporter, srcEpSet.eps[srcEpSet.inUse].fqdn, - dstEpSet.eps[dstEpSet.inUse].fqdn) != 0){ + dstEpSet.eps[dstEpSet.inUse].fqdn) != 0) { tscError("failed to set default addr for rpc"); } updateEpSet = 0; @@ -146,14 +146,15 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { pTscObj->authVer = connectRsp.authVer; pTscObj->whiteListInfo.ver = connectRsp.whiteListVer; - if(taosHashGet(appInfo.pInstMapByClusterId, &connectRsp.clusterId, LONG_BYTES) == NULL){ - if(taosHashPut(appInfo.pInstMapByClusterId, &connectRsp.clusterId, LONG_BYTES, &pTscObj->pAppInfo, POINTER_BYTES) != 0){ + if (taosHashGet(appInfo.pInstMapByClusterId, &connectRsp.clusterId, LONG_BYTES) == NULL) { + if (taosHashPut(appInfo.pInstMapByClusterId, &connectRsp.clusterId, LONG_BYTES, &pTscObj->pAppInfo, + POINTER_BYTES) != 0) { tscError("failed to put appInfo into appInfo.pInstMapByClusterId"); - }else{ + } else { MonitorSlowLogData data = {0}; data.clusterId = pTscObj->pAppInfo->clusterId; data.type = SLOW_LOG_READ_BEGINNIG; - (void)monitorPutData2MonitorQueue(data); // ignore + (void)monitorPutData2MonitorQueue(data); // ignore monitorClientSlowQueryInit(connectRsp.clusterId); monitorClientSQLReqInit(connectRsp.clusterId); } @@ -162,7 +163,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { (void)taosThreadMutexLock(&clientHbMgr.lock); SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, pTscObj->appHbMgrIdx); if (pAppHbMgr) { - if (hbRegisterConn(pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType) != 0){ + if (hbRegisterConn(pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType) != 0) { tscError("0x%" PRIx64 " failed to register conn to hbMgr", pRequest->requestId); } } else { @@ -176,10 +177,10 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { pTscObj->pAppInfo->numOfConns); End: - if (code != 0){ + if (code != 0) { setErrno(pRequest, code); } - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } @@ -196,7 +197,7 @@ EXIT: SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pRequest) { SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if(pMsgSendInfo == NULL) return pMsgSendInfo; + if (pMsgSendInfo == NULL) return pMsgSendInfo; pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->requestId = pRequest->requestId; pMsgSendInfo->param = pRequest; @@ -227,11 +228,11 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) { .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; char dbFName[TSDB_DB_FNAME_LEN]; (void)snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_INFORMATION_SCHEMA_DB); - if (catalogRefreshDBVgInfo(pCatalog, &conn, dbFName) != 0){ + if (catalogRefreshDBVgInfo(pCatalog, &conn, dbFName) != 0) { tscError("0x%" PRIx64 " failed to refresh db vg info", pRequest->requestId); } (void)snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_PERFORMANCE_SCHEMA_DB); - if (catalogRefreshDBVgInfo(pCatalog, &conn, dbFName) != 0){ + if (catalogRefreshDBVgInfo(pCatalog, &conn, dbFName) != 0) { tscError("0x%" PRIx64 " failed to refresh db vg info", pRequest->requestId); } } @@ -240,7 +241,7 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) { if (pRequest->body.queryFp) { doRequestCallback(pRequest, code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -252,19 +253,19 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || TSDB_CODE_MND_DB_IN_DROPPING == code) { SUseDbRsp usedbRsp = {0}; - if (tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp) != 0){ + if (tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp) != 0) { tscError("0x%" PRIx64 " deserialize SUseDbRsp failed", pRequest->requestId); } struct SCatalog* pCatalog = NULL; if (usedbRsp.vgVersion >= 0) { // cached in local int64_t clusterId = pRequest->pTscObj->pAppInfo->clusterId; - int32_t code1 = catalogGetHandle(clusterId, &pCatalog); + int32_t code1 = catalogGetHandle(clusterId, &pCatalog); if (code1 != TSDB_CODE_SUCCESS) { tscWarn("0x%" PRIx64 "catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->requestId, clusterId, tstrerror(code1)); } else { - if (catalogRemoveDB(pCatalog, usedbRsp.db, usedbRsp.uid) != 0){ + if (catalogRemoveDB(pCatalog, usedbRsp.db, usedbRsp.uid) != 0) { tscError("0x%" PRIx64 "catalogRemoveDB failed, db:%s, uid:%" PRId64, pRequest->requestId, usedbRsp.db, usedbRsp.uid); } @@ -282,7 +283,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { doRequestCallback(pRequest, pRequest->code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -291,7 +292,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { } SUseDbRsp usedbRsp = {0}; - if (tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp) != 0){ + if (tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp) != 0) { tscError("0x%" PRIx64 " deserialize SUseDbRsp failed", pRequest->requestId); } @@ -309,7 +310,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { tscTrace("db:%s, usedbRsp received, numOfVgroups:%d", usedbRsp.db, usedbRsp.vgNum); for (int32_t i = 0; i < usedbRsp.vgNum; ++i) { SVgroupInfo* pInfo = taosArrayGet(usedbRsp.pVgroupInfos, i); - if (pInfo == NULL){ + if (pInfo == NULL) { continue; } tscTrace("vgId:%d, numOfEps:%d inUse:%d ", pInfo->vgId, pInfo->epSet.numOfEps, pInfo->epSet.inUse); @@ -319,7 +320,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { } SName name = {0}; - if(tNameFromString(&name, usedbRsp.db, T_NAME_ACCT | T_NAME_DB) != TSDB_CODE_SUCCESS) { + if (tNameFromString(&name, usedbRsp.db, T_NAME_ACCT | T_NAME_DB) != TSDB_CODE_SUCCESS) { tscError("0x%" PRIx64 " failed to parse db name:%s", pRequest->requestId, usedbRsp.db); } @@ -338,7 +339,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId, tstrerror(code1)); } else { - if (catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup) != 0){ + if (catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup) != 0) { tscError("0x%" PRIx64 " failed to update db vg info, db:%s, dbId:%" PRId64, pRequest->requestId, output.db, output.dbId); } @@ -350,7 +351,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { tFreeSUsedbRsp(&usedbRsp); char db[TSDB_DB_NAME_LEN] = {0}; - if(tNameGetDbName(&name, db) != TSDB_CODE_SUCCESS) { + if (tNameGetDbName(&name, db) != TSDB_CODE_SUCCESS) { tscError("0x%" PRIx64 " failed to get db name since %s", pRequest->requestId, tstrerror(code)); } @@ -362,7 +363,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { if (pRequest->body.queryFp != NULL) { doRequestCallback(pRequest, pRequest->code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -387,7 +388,7 @@ int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) { SMCreateStbRsp createRsp = {0}; SDecoder coder = {0}; tDecoderInit(&coder, pMsg->pData, pMsg->len); - if (pMsg->len > 0){ + if (pMsg->len > 0) { code = tDecodeSMCreateStbRsp(&coder, &createRsp); // pMsg->len == 0 if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); @@ -419,7 +420,7 @@ int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) { doRequestCallback(pRequest, code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -432,13 +433,13 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) { setErrno(pRequest, code); } else { SDropDbRsp dropdbRsp = {0}; - if (tDeserializeSDropDbRsp(pMsg->pData, pMsg->len, &dropdbRsp) != 0){ + if (tDeserializeSDropDbRsp(pMsg->pData, pMsg->len, &dropdbRsp) != 0) { tscError("0x%" PRIx64 " deserialize SDropDbRsp failed", pRequest->requestId); } struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (TSDB_CODE_SUCCESS == code) { - if (catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid) != 0){ + if (catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid) != 0) { tscError("0x%" PRIx64 " failed to remove db:%s", pRequest->requestId, dropdbRsp.db); } STscObj* pTscObj = pRequest->pTscObj; @@ -451,7 +452,7 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) { (void)snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_INFORMATION_SCHEMA_DB); if (catalogRefreshDBVgInfo(pCatalog, &conn, dbFName) != TSDB_CODE_SUCCESS) { tscError("0x%" PRIx64 " failed to refresh db vg info, db:%s", pRequest->requestId, dbFName); - } + } (void)snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_PERFORMANCE_SCHEMA_DB); if (catalogRefreshDBVgInfo(pCatalog, &conn, dbFName) != 0) { tscError("0x%" PRIx64 " failed to refresh db vg info, db:%s", pRequest->requestId, dbFName); @@ -465,7 +466,7 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) { if (pRequest->body.queryFp != NULL) { doRequestCallback(pRequest, code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -480,7 +481,7 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) { SMAlterStbRsp alterRsp = {0}; SDecoder coder = {0}; tDecoderInit(&coder, pMsg->pData, pMsg->len); - if (pMsg->len > 0){ + if (pMsg->len > 0) { code = tDecodeSMAlterStbRsp(&coder, &alterRsp); // pMsg->len == 0 if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); @@ -512,7 +513,7 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) { doRequestCallback(pRequest, code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -520,8 +521,8 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) { } static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { - int32_t code = 0; - int32_t line = 0; + int32_t code = 0; + int32_t line = 0; SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); TSDB_CHECK_NULL(pBlock, code, line, END, terrno); pBlock->info.hasVarCol = true; @@ -545,6 +546,10 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD4_LEN; TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD5_LEN; + TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); + int32_t numOfCfg = taosArrayGetSize(pVars); code = blockDataEnsureCapacity(pBlock, numOfCfg); TSDB_CHECK_CODE(code, line, END); @@ -574,6 +579,13 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { code = colDataSetVal(pColInfo, i, scope, false); TSDB_CHECK_CODE(code, line, END); + char category[TSDB_CONFIG_CATEGORY_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(category, pInfo->category, TSDB_CONFIG_CATEGORY_LEN + VARSTR_HEADER_SIZE); + pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + TSDB_CHECK_NULL(pColInfo, code, line, END, terrno); + code = colDataSetVal(pColInfo, i, category, false); + TSDB_CHECK_CODE(code, line, END); + char info[TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(info, pInfo->info, TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); @@ -605,7 +617,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { code = terrno; - goto _exit; + goto _exit; } (*pRsp)->useconds = 0; @@ -617,7 +629,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeBufSize, SHOW_VARIABLES_RESULT_COLS); - if(len < 0) { + if (len < 0) { uError("buildShowVariablesRsp error, len:%d", len); code = terrno; goto _exit; @@ -639,11 +651,11 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { return TSDB_CODE_SUCCESS; _exit: - if(*pRsp) { + if (*pRsp) { taosMemoryFree(*pRsp); *pRsp = NULL; } - if(pBlock) { + if (pBlock) { blockDataDestroy(pBlock); pBlock = NULL; } @@ -677,7 +689,7 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) { if (pRequest->body.queryFp != NULL) { doRequestCallback(pRequest, code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } @@ -685,8 +697,8 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) { } static int32_t buildCompactDbBlock(SCompactDbRsp* pRsp, SSDataBlock** block) { - int32_t code = 0; - int32_t line = 0; + int32_t code = 0; + int32_t line = 0; SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); TSDB_CHECK_NULL(pBlock, code, line, END, terrno); pBlock->info.hasVarCol = true; @@ -772,7 +784,7 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS); int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeBufSize, COMPACT_DB_RESULT_COLS); - if(len < 0) { + if (len < 0) { uError("buildRetriveTableRspForCompactDb error, len:%d", len); code = terrno; goto _exit; @@ -794,24 +806,23 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr return TSDB_CODE_SUCCESS; _exit: - if(*pRsp) { + if (*pRsp) { taosMemoryFree(*pRsp); *pRsp = NULL; } - if(pBlock) { + if (pBlock) { blockDataDestroy(pBlock); pBlock = NULL; } return code; } - int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); } else { - SCompactDbRsp rsp = {0}; + SCompactDbRsp rsp = {0}; SRetrieveTableRsp* pRes = NULL; code = tDeserializeSCompactDbRsp(pMsg->pData, pMsg->len, &rsp); if (TSDB_CODE_SUCCESS == code) { @@ -830,9 +841,9 @@ int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pEpSet); if (pRequest->body.queryFp != NULL) { - pRequest->body.queryFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, code); + pRequest->body.queryFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, code); } else { - if (tsem_post(&pRequest->body.rspSem) != 0){ + if (tsem_post(&pRequest->body.rspSem) != 0) { tscError("failed to post semaphore"); } } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 76aec26ff4..830144c12a 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1348,7 +1348,7 @@ end: destroyRequest(pRequest); tDecoderClear(&coder); qDestroyQuery(pQuery); - taosArrayDestroyP(pTagList, taosMemoryFree); + taosArrayDestroyP(pTagList, NULL); return code; } @@ -1563,7 +1563,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { req.source = TD_REQ_FROM_TAOX; tEncodeSize(tEncodeSVAlterTbReq, &req, tlen, code); if (code != 0) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } tlen += sizeof(SMsgHead); @@ -1577,7 +1577,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { code = tEncodeSVAlterTbReq(&coder, &req); if (code != 0) { tEncoderClear(&coder); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } tEncoderClear(&coder); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index cc96d61b18..376dea895c 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -444,7 +444,7 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) } clearColValArraySml(info->currTableDataCtx->pValues); - taosArrayClearP(info->escapedStringList, taosMemoryFree); + taosArrayClearP(info->escapedStringList, NULL); if (unlikely(ret != TSDB_CODE_SUCCESS)) { uError("SML:0x%" PRIx64 " %s smlBuildCol error:%d", info->id, __FUNCTION__, ret); return ret; @@ -1270,7 +1270,7 @@ void smlDestroyInfo(SSmlHandle *info) { taosArrayDestroy(info->valueJsonArray); taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); - taosArrayDestroyP(info->escapedStringList, taosMemoryFree); + taosArrayDestroyP(info->escapedStringList, NULL); if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { @@ -1536,7 +1536,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { SML_CHECK_NULL(info->lines); } - taosArrayClearP(info->escapedStringList, taosMemoryFree); + taosArrayClearP(info->escapedStringList, NULL); if(info->protocol == TSDB_SML_JSON_PROTOCOL) { taosMemoryFreeClear(info->preLine.tags); } diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index b54f4e0beb..4ae02bd7f7 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -134,7 +134,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { void* data = NULL; uint32_t size = 0; if(taosHex2Ascii(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN, &data, &size) < 0){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (size + VARSTR_HEADER_SIZE > TSDB_MAX_VARBINARY_LEN) { diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 9cad219614..6fb923ae38 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -300,7 +300,7 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { STableDataCxt** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (!pSrc) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } STableDataCxt* pDst = NULL; @@ -355,7 +355,7 @@ int32_t stmtParseSql(STscStmt* pStmt) { STableDataCxt** pSrc = (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pSrc || NULL == *pSrc) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } STableDataCxt* pTableCtx = *pSrc; @@ -845,13 +845,11 @@ TAOS_STMT* stmtInit(STscObj* taos, int64_t reqid, TAOS_STMT_OPTIONS* pOptions) { pStmt = taosMemoryCalloc(1, sizeof(STscStmt)); if (NULL == pStmt) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == pStmt->sql.pTableCache) { - terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pStmt); return NULL; } @@ -876,13 +874,11 @@ TAOS_STMT* stmtInit(STscObj* taos, int64_t reqid, TAOS_STMT_OPTIONS* pOptions) { pStmt->sql.siInfo.mgmtEpSet = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); pStmt->sql.siInfo.pTableHash = tSimpleHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY)); if (NULL == pStmt->sql.siInfo.pTableHash) { - terrno = TSDB_CODE_OUT_OF_MEMORY; (void)stmtClose(pStmt); return NULL; } pStmt->sql.siInfo.pTableCols = taosArrayInit(STMT_TABLE_COLS_NUM, POINTER_BYTES); if (NULL == pStmt->sql.siInfo.pTableCols) { - terrno = TSDB_CODE_OUT_OF_MEMORY; (void)stmtClose(pStmt); return NULL; } @@ -948,7 +944,7 @@ int stmtPrepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { int32_t stmtInitStbInterlaceTableInfo(STscStmt* pStmt) { STableDataCxt** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (!pSrc) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } STableDataCxt* pDst = NULL; diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index f03879f199..dba6aa3beb 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -75,6 +75,10 @@ static int32_t stmtCreateRequest(STscStmt2* pStmt) { if (pStmt->reqid != 0) { pStmt->reqid++; } + if (pStmt->db != NULL) { + taosMemoryFreeClear(pStmt->exec.pRequest->pDb); + pStmt->exec.pRequest->pDb = strdup(pStmt->db); + } if (TSDB_CODE_SUCCESS == code) { pStmt->exec.pRequest->syncQuery = true; pStmt->exec.pRequest->isStmtBind = true; @@ -109,7 +113,7 @@ static int32_t stmtSwitchStatus(STscStmt2* pStmt, STMT_STATUS newStatus) { } break; case STMT_SETTAGS: - if (STMT_STATUS_NE(SETTBNAME) && STMT_STATUS_NE(FETCH_FIELDS)) { + if (STMT_STATUS_EQ(INIT)) { code = TSDB_CODE_TSC_STMT_API_ERROR; } break; @@ -275,7 +279,7 @@ static int32_t stmtParseSql(STscStmt2* pStmt) { STableDataCxt** pSrc = (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pSrc || NULL == *pSrc) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } STableDataCxt* pTableCtx = *pSrc; @@ -295,7 +299,7 @@ static int32_t stmtParseSql(STscStmt2* pStmt) { if (NULL == pStmt->sql.pBindInfo) { pStmt->sql.pBindInfo = taosMemoryMalloc(pTableCtx->boundColsInfo.numOfBound * sizeof(*pStmt->sql.pBindInfo)); if (NULL == pStmt->sql.pBindInfo) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -412,6 +416,7 @@ static void stmtFreeTbCols(void* buf) { static int32_t stmtCleanSQLInfo(STscStmt2* pStmt) { STMT_DLOG_E("start to free SQL info"); + taosMemoryFreeClear(pStmt->db); taosMemoryFree(pStmt->sql.pBindInfo); taosMemoryFree(pStmt->sql.queryRes.fields); taosMemoryFree(pStmt->sql.queryRes.userFields); @@ -540,7 +545,7 @@ static int32_t stmtGetFromCache(STscStmt2* pStmt) { if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } pStmt->exec.pCurrBlock = pNewBlock; @@ -630,7 +635,7 @@ static int32_t stmtGetFromCache(STscStmt2* pStmt) { if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } pStmt->exec.pCurrBlock = pNewBlock; @@ -766,13 +771,11 @@ TAOS_STMT2* stmtInit2(STscObj* taos, TAOS_STMT2_OPTION* pOptions) { pStmt = taosMemoryCalloc(1, sizeof(STscStmt2)); if (NULL == pStmt) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == pStmt->sql.pTableCache) { - terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pStmt); return NULL; } @@ -798,7 +801,6 @@ TAOS_STMT2* stmtInit2(STscObj* taos, TAOS_STMT2_OPTION* pOptions) { pStmt->sql.siInfo.mgmtEpSet = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); pStmt->sql.siInfo.pTableHash = tSimpleHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY)); if (NULL == pStmt->sql.siInfo.pTableHash) { - terrno = TSDB_CODE_OUT_OF_MEMORY; (void)stmtClose(pStmt); return NULL; } @@ -845,6 +847,7 @@ static int stmtSetDbName2(TAOS_STMT2* stmt, const char* dbName) { STMT_DLOG("start to set dbName: %s", dbName); + pStmt->db = taosStrdup(dbName); STMT_ERR_RET(stmtCreateRequest(pStmt)); // The SQL statement specifies a database name, overriding the previously specified database @@ -894,7 +897,7 @@ int stmtPrepare2(TAOS_STMT2* stmt, const char* sql, unsigned long length) { static int32_t stmtInitStbInterlaceTableInfo(STscStmt2* pStmt) { STableDataCxt** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (!pSrc) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } STableDataCxt* pDst = NULL; @@ -995,6 +998,19 @@ int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && + STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + pStmt->bInfo.needParse = false; + } + STMT_ERR_RET(stmtCreateRequest(pStmt)); + + if (pStmt->bInfo.needParse) { + STMT_ERR_RET(stmtParseSql(pStmt)); + } + if (pStmt->sql.stbInterlaceMode && NULL == pStmt->sql.siInfo.pDataCtx) { + STMT_ERR_RET(stmtInitStbInterlaceTableInfo(pStmt)); + } + SBoundColInfo* tags_info = (SBoundColInfo*)pStmt->bInfo.boundTags; if (tags_info->numOfBound <= 0 || tags_info->numOfCols <= 0) { tscWarn("no tags or cols bound in sql, will not bound tags"); @@ -1096,6 +1112,7 @@ static int stmtFetchStbColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIEL STMT_ERR_RET(qBuildStmtStbColFields(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.preCtbname, fieldNum, fields)); if (pStmt->bInfo.tbType == TSDB_SUPER_TABLE) { pStmt->bInfo.needParse = true; + qDestroyStmtDataBlock(*pDataBlock); if (taosHashRemove(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)) != 0) { tscError("get fileds %s remove exec blockHash fail", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -1199,7 +1216,7 @@ static int32_t stmtCacheBlock(STscStmt2* pStmt) { STableDataCxt** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (!pSrc) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } STableDataCxt* pDst = NULL; @@ -1266,7 +1283,7 @@ static int32_t stmtBackupQueryFields(STscStmt2* pStmt) { pRes->fields = taosMemoryMalloc(size); pRes->userFields = taosMemoryMalloc(size); if (NULL == pRes->fields || NULL == pRes->userFields) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } (void)memcpy(pRes->fields, pStmt->exec.pRequest->body.resInfo.fields, size); (void)memcpy(pRes->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size); @@ -1284,7 +1301,7 @@ static int32_t stmtRestoreQueryFields(STscStmt2* pStmt) { if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { pStmt->exec.pRequest->body.resInfo.fields = taosMemoryMalloc(size); if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } (void)memcpy(pStmt->exec.pRequest->body.resInfo.fields, pRes->fields, size); } @@ -1292,7 +1309,7 @@ static int32_t stmtRestoreQueryFields(STscStmt2* pStmt) { if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { pStmt->exec.pRequest->body.resInfo.userFields = taosMemoryMalloc(size); if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + STMT_ERR_RET(terrno); } (void)memcpy(pStmt->exec.pRequest->body.resInfo.userFields, pRes->userFields, size); } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 902e581bfd..b254f6eb8b 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -45,7 +45,7 @@ tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); \ if (FUNC(&decoder, DATA) < 0) { \ tDecoderClear(&decoder); \ - code = TSDB_CODE_OUT_OF_MEMORY; \ + code = terrno; \ goto END;\ }\ tDecoderClear(&decoder);\ @@ -534,7 +534,7 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) { void tmq_list_destroy(tmq_list_t* list) { if (list == NULL) return; SArray* container = &list->container; - taosArrayDestroyP(container, taosMemoryFree); + taosArrayDestroyP(container, NULL); } int32_t tmq_list_get_size(const tmq_list_t* list) { @@ -659,7 +659,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse pMsgSendInfo->requestId = generateRequestId(); pMsgSendInfo->requestObjRefId = 0; pMsgSendInfo->param = pParam; - pMsgSendInfo->paramFreeFp = taosMemoryFree; + pMsgSendInfo->paramFreeFp = taosAutoMemoryFree; pMsgSendInfo->fp = tmqCommitCb; pMsgSendInfo->msgType = TDMT_VND_TMQ_COMMIT_OFFSET; @@ -1385,7 +1385,7 @@ static int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; sendInfo->param = pParam; - sendInfo->paramFreeFp = taosMemoryFree; + sendInfo->paramFreeFp = taosAutoMemoryFree; sendInfo->fp = askEpCb; sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; @@ -1552,7 +1552,7 @@ static void tmqMgmtInit(void) { tmqMgmt.timer = taosTmrInit(1000, 100, 360000, "TMQ"); if (tmqMgmt.timer == NULL) { - tmqInitRes = TSDB_CODE_OUT_OF_MEMORY; + tmqInitRes = terrno; } tmqMgmt.rsetId = taosOpenRef(10000, tmqFreeImpl); @@ -1886,7 +1886,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } END: - taosArrayDestroyP(req.topicNames, taosMemoryFree); + taosArrayDestroyP(req.topicNames, NULL); return code; } @@ -2177,7 +2177,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p sendInfo->requestId = req.reqId; sendInfo->requestObjRefId = 0; sendInfo->param = pParam; - sendInfo->paramFreeFp = taosMemoryFree; + sendInfo->paramFreeFp = taosAutoMemoryFree; sendInfo->fp = tmqPollCb; sendInfo->msgType = TDMT_VND_TMQ_CONSUME; @@ -2520,7 +2520,7 @@ int32_t tmq_unsubscribe(tmq_t* tmq) { tmq_list_t* lst = tmq_list_new(); if (lst == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto END; } code = tmq_subscribe(tmq, lst); @@ -2701,10 +2701,12 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { tqErrorC("failed to allocate memory for sync commit"); return terrno; } - if (tsem2_init(&pInfo->sem, 0, 0) != 0) { + + code = tsem2_init(&pInfo->sem, 0, 0); + if (code != 0) { tqErrorC("failed to init sem for sync commit"); taosMemoryFree(pInfo); - return TSDB_CODE_OUT_OF_MEMORY; + return code; } pInfo->code = 0; @@ -2778,9 +2780,10 @@ int32_t tmq_commit_offset_sync(tmq_t* tmq, const char* pTopicName, int32_t vgId, return terrno; } - if (tsem2_init(&pInfo->sem, 0, 0) != 0) { + code = tsem2_init(&pInfo->sem, 0, 0); + if (code != 0) { taosMemoryFree(pInfo); - return TSDB_CODE_OUT_OF_MEMORY; + return code; } pInfo->code = 0; @@ -2847,6 +2850,7 @@ end: } } + int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pResInfo) { SMqRspObj* pRspObj = (SMqRspObj*)res; SMqDataRsp* data = &pRspObj->dataRsp; @@ -2966,9 +2970,10 @@ static int32_t tmCommittedCb(void* param, SDataBuf* pMsg, int32_t code) { if (pMsg) { SDecoder decoder = {0}; tDecoderInit(&decoder, (uint8_t*)pMsg->pData, pMsg->len); - if (tDecodeMqVgOffset(&decoder, &pParam->vgOffset) < 0) { + int32_t err = tDecodeMqVgOffset(&decoder, &pParam->vgOffset); + if (err < 0) { tOffsetDestroy(&pParam->vgOffset.offset); - code = TSDB_CODE_OUT_OF_MEMORY; + code = err; goto end; } tDecoderClear(&decoder); @@ -3261,8 +3266,9 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a code = terrno; goto end; } - if (tsem2_init(&pCommon->rsp, 0, 0) != 0) { - code = TSDB_CODE_OUT_OF_MEMORY; + + code = tsem2_init(&pCommon->rsp, 0, 0); + if (code != 0) { goto end; } (void)taosThreadMutexInit(&pCommon->mutex, 0); @@ -3296,7 +3302,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req); if (msgSize < 0) { taosMemoryFree(pParam); - code = TSDB_CODE_OUT_OF_MEMORY; + code = msgSize; goto end; } @@ -3307,10 +3313,11 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a goto end; } - if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) { + msgSize = tSerializeSMqPollReq(msg, msgSize, &req); + if (msgSize < 0) { taosMemoryFree(msg); taosMemoryFree(pParam); - code = TSDB_CODE_OUT_OF_MEMORY; + code = msgSize; goto end; } @@ -3326,7 +3333,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a sendInfo->requestId = req.reqId; sendInfo->requestObjRefId = 0; sendInfo->param = pParam; - sendInfo->paramFreeFp = taosMemoryFree; + sendInfo->paramFreeFp = taosAutoMemoryFree; sendInfo->fp = tmqGetWalInfoCb; sendInfo->msgType = TDMT_VND_TMQ_VG_WALINFO; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index fa0871b812..5a4393bfed 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -300,7 +300,13 @@ void* doConsumeData(void* param) { int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc > 1) { - numOfThreads = atoi(argv[1]); + //numOfThreads = atoi(argv[1]); + int32_t code = taosStr2int32(argv[1], &numOfThreads); + if (code != 0) { + return code; + } + + } numOfThreads = TMAX(numOfThreads, 1); diff --git a/source/common/src/cos.c b/source/common/src/cos.c index de4f65f2ad..35ab328815 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -261,7 +261,7 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro for (int i = 0; i < error->extraDetailsCount; i++) { if (elen - len > 0) { len += tsnprintf(&(cbd->err_msg[len]), elen - len, " %s: %s\n", error->extraDetails[i].name, - error->extraDetails[i].value); + error->extraDetails[i].value); } } } @@ -742,9 +742,9 @@ upload: TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } n = tsnprintf(buf, sizeof(buf), - "%d" - "%s", - i + 1, manager.etags[i]); + "%d" + "%s", + i + 1, manager.etags[i]); size += growbuffer_append(&(manager.gb), buf, n); } size += growbuffer_append(&(manager.gb), "", strlen("")); @@ -908,10 +908,10 @@ upload: int n; for (int i = 0; i < cp.part_num; ++i) { n = tsnprintf(buf, sizeof(buf), - "%d" - "%s", - // i + 1, manager.etags[i]); - cp.parts[i].index + 1, cp.parts[i].etag); + "%d" + "%s", + // i + 1, manager.etags[i]); + cp.parts[i].index + 1, cp.parts[i].etag); size += growbuffer_append(&(manager.gb), buf, n); } size += growbuffer_append(&(manager.gb), "", strlen("")); @@ -1916,7 +1916,8 @@ void s3EvictCache(const char *path, long object_size) { } long s3Size(const char *object_name) { - long size = 0; + int32_t code = 0; + long size = 0; cos_pool_t *p = NULL; int is_cname = 0; @@ -1941,7 +1942,10 @@ long s3Size(const char *object_name) { if (cos_status_is_ok(s)) { char *content_length_str = (char *)apr_table_get(resp_headers, COS_CONTENT_LENGTH); if (content_length_str != NULL) { - size = atol(content_length_str); + code = taosStr2Int64(content_length_str, &size); + if (code != 0) { + cos_warn_log("parse content length failed since %s\n", tstrerror(code)); + } } cos_warn_log("head object succeeded: %ld\n", size); } else { diff --git a/source/common/src/msg/streamMsg.c b/source/common/src/msg/streamMsg.c index c92ab52ac1..54b17b14d1 100644 --- a/source/common/src/msg/streamMsg.c +++ b/source/common/src/msg/streamMsg.c @@ -403,7 +403,7 @@ _exit: } void tCleanupStreamDispatchReq(SStreamDispatchReq* pReq) { - taosArrayDestroyP(pReq->data, taosMemoryFree); + taosArrayDestroyP(pReq->data, NULL); taosArrayDestroy(pReq->dataLen); } @@ -651,6 +651,7 @@ int32_t tEncodeStreamHbRsp(SEncoder* pEncoder, const SMStreamHbRspMsg* pRsp) { TAOS_CHECK_EXIT(tStartEncode(pEncoder)); TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->msgId)); + TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pRsp->mndEpset)); tEndEncode(pEncoder); _exit: @@ -663,6 +664,7 @@ int32_t tDecodeStreamHbRsp(SDecoder* pDecoder, SMStreamHbRspMsg* pRsp) { TAOS_CHECK_EXIT(tStartDecode(pDecoder)); TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->msgId)); + TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pRsp->mndEpset)); tEndDecode(pDecoder); _exit: diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index 842b290f46..2193c7983f 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#include "tglobal.h" #include "tmsg.h" #undef TD_MSG_NUMBER_ @@ -76,7 +77,7 @@ static int32_t tSerializeSMonitorParas(SEncoder *encoder, const SMonitorParas *p TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogScope)); TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogMaxLen)); TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThreshold)); - TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThresholdTest)); //Obsolete + TAOS_CHECK_RETURN(tEncodeI32(encoder, pMonitorParas->tsSlowLogThresholdTest)); // Obsolete TAOS_CHECK_RETURN(tEncodeCStr(encoder, pMonitorParas->tsSlowLogExceptDb)); return 0; } @@ -87,7 +88,7 @@ static int32_t tDeserializeSMonitorParas(SDecoder *decoder, SMonitorParas *pMoni TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogScope)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogMaxLen)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThreshold)); - TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThresholdTest)); //Obsolete + TAOS_CHECK_RETURN(tDecodeI32(decoder, &pMonitorParas->tsSlowLogThresholdTest)); // Obsolete TAOS_CHECK_RETURN(tDecodeCStrTo(decoder, pMonitorParas->tsSlowLogExceptDb)); return 0; } @@ -1608,6 +1609,99 @@ _exit: void tFreeSStatusReq(SStatusReq *pReq) { taosArrayDestroy(pReq->pVloads); } +int32_t tSerializeSConfigReq(void *buf, int32_t bufLen, SConfigReq *pReq) { + SEncoder encoder = {0}; + int32_t code = 0; + int32_t lino; + int32_t tlen; + tEncoderInit(&encoder, buf, bufLen); + TAOS_CHECK_EXIT(tStartEncode(&encoder)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->cver)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->forceReadConfig)); + if (pReq->forceReadConfig) { + TAOS_CHECK_EXIT(tSerializeSConfigArray(&encoder, pReq->array)); + } + tEndEncode(&encoder); +_exit: + if (code) { + tlen = code; + } else { + tlen = encoder.pos; + } + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSConfigReq(void *buf, int32_t bufLen, SConfigReq *pReq) { + SDecoder decoder = {0}; + int32_t code = 0; + int32_t lino; + tDecoderInit(&decoder, buf, bufLen); + TAOS_CHECK_EXIT(tStartDecode(&decoder)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->cver)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->forceReadConfig)); + if (pReq->forceReadConfig) { + pReq->array = taosArrayInit(128, sizeof(SConfigItem)); + if (pReq->array == NULL) { + TAOS_CHECK_EXIT(terrno); + } + TAOS_CHECK_EXIT(tDeserializeSConfigArray(&decoder, pReq->array)); + } + tEndDecode(&decoder); +_exit: + tDecoderClear(&decoder); + return code; +} + +void tFreeSConfigReq(SConfigReq *pReq) { taosArrayDestroy(pReq->array); } + +int32_t tSerializeSConfigRsp(void *buf, int32_t bufLen, SConfigRsp *pRsp) { + SEncoder encoder = {0}; + int32_t code = 0; + int32_t lino; + int32_t tlen; + tEncoderInit(&encoder, buf, bufLen); + TAOS_CHECK_EXIT(tStartEncode(&encoder)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->forceReadConfig)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->isConifgVerified)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->isVersionVerified)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->cver)); + if ((!pRsp->isConifgVerified) || (!pRsp->isVersionVerified)) { + TAOS_CHECK_EXIT(tSerializeSConfigArray(&encoder, pRsp->array)); + } + tEndEncode(&encoder); +_exit: + if (code) { + tlen = code; + } else { + tlen = encoder.pos; + } + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSConfigRsp(void *buf, int32_t bufLen, SConfigRsp *pRsp) { + SDecoder decoder = {0}; + int32_t code = 0; + int32_t lino; + tDecoderInit(&decoder, buf, bufLen); + TAOS_CHECK_EXIT(tStartDecode(&decoder)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pRsp->forceReadConfig)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pRsp->isConifgVerified)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pRsp->isVersionVerified)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pRsp->cver)); + if ((!pRsp->isConifgVerified) || (!pRsp->isVersionVerified)) { + pRsp->array = taosArrayInit(128, sizeof(SConfigItem)); + TAOS_CHECK_EXIT(tDeserializeSConfigArray(&decoder, pRsp->array)); + } +_exit: + tEndDecode(&decoder); + tDecoderClear(&decoder); + return code; +} + +void tFreeSConfigRsp(SConfigRsp *pRsp) { taosArrayDestroy(pRsp->array); } + int32_t tSerializeSDnodeInfoReq(void *buf, int32_t bufLen, SDnodeInfoReq *pReq) { int32_t code = 0, lino = 0; int32_t tlen = 0; @@ -3160,6 +3254,7 @@ int32_t tSerializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) tEncoderInit(&encoder, buf, bufLen); TAOS_CHECK_EXIT(tStartEncode(&encoder)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->version)); TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->config)); TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->value)); tEndEncode(&encoder); @@ -3181,6 +3276,7 @@ int32_t tDeserializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq tDecoderInit(&decoder, buf, bufLen); TAOS_CHECK_EXIT(tStartDecode(&decoder)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->version)); TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->config)); TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->value)); tEndDecode(&decoder); @@ -3943,6 +4039,12 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->s3Compact)); TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->dnodeListStr)); + // auto-compact parameters + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactInterval)); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactStartTime)); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactEndTime)); + TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->compactTimeOffset)); + tEndEncode(&encoder); _exit: @@ -4010,30 +4112,44 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->tsdbPageSize)); - pReq->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET; if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->keepTimeOffset)); + } else { + pReq->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET; } DECODESQL(); - pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; - pReq->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; - pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; - pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; - pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT; if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->withArbitrator)); TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->encryptAlgorithm)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->s3ChunkSize)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->s3KeepLocal)); TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->s3Compact)); + } else { + pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; + pReq->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; + pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; + pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; + pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT; } if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->dnodeListStr)); } + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactInterval)); + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactStartTime)); + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactEndTime)); + TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->compactTimeOffset)); + } else { + pReq->compactInterval = TSDB_DEFAULT_COMPACT_INTERVAL; + pReq->compactStartTime = TSDB_DEFAULT_COMPACT_START_TIME; + pReq->compactEndTime = TSDB_DEFAULT_COMPACT_END_TIME; + pReq->compactTimeOffset = TSDB_DEFAULT_COMPACT_TIME_OFFSET; + } + tEndDecode(&decoder); _exit: @@ -4084,6 +4200,11 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { ENCODESQL(); TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->withArbitrator)); + // auto compact config + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactInterval)); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactStartTime)); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactEndTime)); + TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->compactTimeOffset)); tEndEncode(&encoder); _exit: @@ -4151,6 +4272,19 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->withArbitrator)); } + + // auto compact config + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactInterval)); + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactStartTime)); + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactEndTime)); + TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->compactTimeOffset)); + } else { + pReq->compactInterval = TSDB_DEFAULT_COMPACT_INTERVAL; + pReq->compactStartTime = TSDB_DEFAULT_COMPACT_START_TIME; + pReq->compactEndTime = TSDB_DEFAULT_COMPACT_END_TIME; + pReq->compactTimeOffset = TSDB_DEFAULT_COMPACT_TIME_OFFSET; + } tEndDecode(&decoder); _exit: @@ -4527,6 +4661,17 @@ int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->timeRange.skey)); TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->timeRange.ekey)); ENCODESQL(); + + // encode vgroup list + int32_t numOfVgroups = taosArrayGetSize(pReq->vgroupIds); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, numOfVgroups)); + if (numOfVgroups > 0) { + for (int32_t i = 0; i < numOfVgroups; ++i) { + int64_t vgid = *(int64_t *)taosArrayGet(pReq->vgroupIds, i); + TAOS_CHECK_EXIT(tEncodeI64v(&encoder, vgid)); + } + } + tEndEncode(&encoder); _exit: @@ -4550,6 +4695,26 @@ int32_t tDeserializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->timeRange.skey)); TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->timeRange.ekey)); DECODESQL(); + + // decode vgroup list + if (!tDecodeIsEnd(&decoder)) { + int32_t numOfVgroups = 0; + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &numOfVgroups)); + if (numOfVgroups > 0) { + pReq->vgroupIds = taosArrayInit(numOfVgroups, sizeof(int64_t)); + if (NULL == pReq->vgroupIds) { + TAOS_CHECK_EXIT(terrno); + } + + for (int32_t i = 0; i < numOfVgroups; ++i) { + int64_t vgid; + TAOS_CHECK_EXIT(tDecodeI64v(&decoder, &vgid)); + if (taosArrayPush(pReq->vgroupIds, &vgid) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + } + } + } tEndDecode(&decoder); _exit: @@ -4557,7 +4722,11 @@ _exit: return code; } -void tFreeSCompactDbReq(SCompactDbReq *pReq) { FREESQL(); } +void tFreeSCompactDbReq(SCompactDbReq *pReq) { + FREESQL(); + taosArrayDestroy(pReq->vgroupIds); + pReq->vgroupIds = NULL; +} int32_t tSerializeSCompactDbRsp(void *buf, int32_t bufLen, SCompactDbRsp *pRsp) { SEncoder encoder = {0}; @@ -5249,6 +5418,10 @@ int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp) { TAOS_CHECK_RETURN(tEncodeI32(encoder, pRsp->s3KeepLocal)); TAOS_CHECK_RETURN(tEncodeI8(encoder, pRsp->s3Compact)); TAOS_CHECK_RETURN(tEncodeI8(encoder, pRsp->hashMethod)); + TAOS_CHECK_RETURN(tEncodeI32v(encoder, pRsp->compactInterval)); + TAOS_CHECK_RETURN(tEncodeI32v(encoder, pRsp->compactStartTime)); + TAOS_CHECK_RETURN(tEncodeI32v(encoder, pRsp->compactEndTime)); + TAOS_CHECK_RETURN(tEncodeI8(encoder, pRsp->compactTimeOffset)); return 0; } @@ -5324,27 +5497,40 @@ int32_t tDeserializeSDbCfgRspImpl(SDecoder *decoder, SDbCfgRsp *pRsp) { } TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->schemaless)); TAOS_CHECK_RETURN(tDecodeI16(decoder, &pRsp->sstTrigger)); - pRsp->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET; if (!tDecodeIsEnd(decoder)) { TAOS_CHECK_RETURN(tDecodeI32(decoder, &pRsp->keepTimeOffset)); + } else { + pRsp->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET; } - pRsp->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; - pRsp->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; - pRsp->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; - pRsp->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; - pRsp->s3Compact = TSDB_DEFAULT_S3_COMPACT; if (!tDecodeIsEnd(decoder)) { TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->withArbitrator)); TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->encryptAlgorithm)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pRsp->s3ChunkSize)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pRsp->s3KeepLocal)); TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->s3Compact)); + } else { + pRsp->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; + pRsp->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; + pRsp->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; + pRsp->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; + pRsp->s3Compact = TSDB_DEFAULT_S3_COMPACT; } if (!tDecodeIsEnd(decoder)) { TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->hashMethod)); } else { pRsp->hashMethod = 1; // default value } + if (!tDecodeIsEnd(decoder)) { + TAOS_CHECK_RETURN(tDecodeI32v(decoder, &pRsp->compactInterval)); + TAOS_CHECK_RETURN(tDecodeI32v(decoder, &pRsp->compactStartTime)); + TAOS_CHECK_RETURN(tDecodeI32v(decoder, &pRsp->compactEndTime)); + TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->compactTimeOffset)); + } else { + pRsp->compactInterval = TSDB_DEFAULT_COMPACT_INTERVAL; + pRsp->compactStartTime = TSDB_DEFAULT_COMPACT_START_TIME; + pRsp->compactEndTime = TSDB_DEFAULT_COMPACT_END_TIME; + pRsp->compactTimeOffset = TSDB_DEFAULT_COMPACT_TIME_OFFSET; + } return 0; } @@ -5624,6 +5810,7 @@ int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) { TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->name)); TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->value)); TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->scope)); + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->category)); return 0; } @@ -5631,6 +5818,7 @@ int32_t tDecodeSVariablesInfo(SDecoder *pDecoder, SVariablesInfo *pInfo) { TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->name)); TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->value)); TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->scope)); + TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->category)); return 0; } @@ -6800,6 +6988,9 @@ int32_t tSerializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryComp TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->dnodeId)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->numberFileset)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->finished)); + // 1. add progress and remaining time + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->progress)); + TAOS_CHECK_EXIT(tEncodeI64v(&encoder, pReq->remainingTime)); tEndEncode(&encoder); @@ -6824,6 +7015,14 @@ int32_t tDeserializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCo TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->dnodeId)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->numberFileset)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->finished)); + // 1. decode progress and remaining time + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->progress)); + TAOS_CHECK_EXIT(tDecodeI64v(&decoder, &pReq->remainingTime)); + } else { + pReq->progress = 0; + pReq->remainingTime = 0; + } tEndDecode(&decoder); _exit: @@ -9295,7 +9494,7 @@ int32_t tSerializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *pR tEncoderInit(&encoder, buf, bufLen); TAOS_CHECK_EXIT(tStartEncode(&encoder)); - TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->sId)); + TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->epId.nodeId)); TAOS_CHECK_EXIT(tEncodeU16(&encoder, pReq->epId.ep.port)); TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->epId.ep.fqdn)); @@ -9342,7 +9541,7 @@ int32_t tDeserializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq * tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen); TAOS_CHECK_EXIT(tStartDecode(&decoder)); - TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->sId)); + TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->epId.nodeId)); TAOS_CHECK_EXIT(tDecodeU16(&decoder, &pReq->epId.ep.port)); TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->epId.ep.fqdn)); @@ -10182,7 +10381,7 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { char name[TSDB_COL_NAME_LEN] = {0}; char *tmp = NULL; TAOS_CHECK_EXIT(tDecodeCStr(pCoder, &tmp)); - strncpy(name, tmp, TSDB_COL_NAME_LEN - 1); + tstrncpy(name, tmp, TSDB_COL_NAME_LEN); if (taosArrayPush(pReq->ctb.tagName, name) == NULL) { TAOS_CHECK_EXIT(terrno); } @@ -11252,11 +11451,11 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { static void tDeleteMqDataRspCommon(SMqDataRsp *pRsp) { taosArrayDestroy(pRsp->blockDataLen); pRsp->blockDataLen = NULL; - taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); + taosArrayDestroyP(pRsp->blockData, NULL); pRsp->blockData = NULL; taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper); pRsp->blockSchema = NULL; - taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree); + taosArrayDestroyP(pRsp->blockTbName, NULL); pRsp->blockTbName = NULL; tOffsetDestroy(&pRsp->reqOffset); tOffsetDestroy(&pRsp->rspOffset); @@ -11318,7 +11517,7 @@ void tDeleteSTaosxRsp(SMqDataRsp *pRsp) { taosArrayDestroy(pRsp->createTableLen); pRsp->createTableLen = NULL; - taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree); + taosArrayDestroyP(pRsp->createTableReq, NULL); pRsp->createTableReq = NULL; } @@ -12218,7 +12417,7 @@ void setFieldWithOptions(SFieldWithOptions *fieldWithOptions, SField *field) { fieldWithOptions->bytes = field->bytes; fieldWithOptions->flags = field->flags; fieldWithOptions->type = field->type; - strncpy(fieldWithOptions->name, field->name, TSDB_COL_NAME_LEN); + tstrncpy(fieldWithOptions->name, field->name, TSDB_COL_NAME_LEN); } int32_t tSerializeTableTSMAInfoReq(void *buf, int32_t bufLen, const STableTSMAInfoReq *pReq) { SEncoder encoder = {0}; @@ -12877,7 +13076,7 @@ _exit: void tDeleteMqBatchMetaRsp(SMqBatchMetaRsp *pRsp) { taosMemoryFreeClear(pRsp->pMetaBuff); - taosArrayDestroyP(pRsp->batchMetaReq, taosMemoryFree); + taosArrayDestroyP(pRsp->batchMetaReq, NULL); taosArrayDestroy(pRsp->batchMetaLen); pRsp->batchMetaReq = NULL; pRsp->batchMetaLen = NULL; diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 0028622ff5..cb08046399 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -329,6 +329,7 @@ static const SSysDbTableSchema variablesSchema[] = { {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "value", .bytes = TSDB_CONFIG_PATH_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "scope", .bytes = TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "category", .bytes = TSDB_CONFIG_CATEGORY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "info", .bytes = TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; @@ -398,6 +399,8 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = { {.name = "number_fileset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "finished", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, + {.name = "progress(%)", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "remain_time(s)", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, }; static const SSysDbTableSchema anodesSchema[] = { @@ -414,6 +417,18 @@ static const SSysDbTableSchema anodesFullSchema[] = { {.name = "algo", .bytes = TSDB_ANALYTIC_ALGO_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; +static const SSysDbTableSchema filesetsFullSchema[] = { + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "fileset_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, + {.name = "end_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, + {.name = "total_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, + {.name = "last_compact", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, + {.name = "shold_compact", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL, .sysInfo = false}, + // {.name = "details", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, +}; + static const SSysDbTableSchema tsmaSchema[] = { {.name = "tsma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, @@ -505,6 +520,7 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_ANODES, anodesSchema, tListLen(anodesSchema), true}, {TSDB_INS_TABLE_ANODES_FULL, anodesFullSchema, tListLen(anodesFullSchema), true}, {TSDB_INS_DISK_USAGE, diskUsageSchema, tListLen(diskUsageSchema), false}, + {TSDB_INS_TABLE_FILESETS, filesetsFullSchema, tListLen(filesetsFullSchema), false}, }; static const SSysDbTableSchema connectionsSchema[] = { diff --git a/source/common/src/tcol.c b/source/common/src/tcol.c index 55a4b21208..41075b48e9 100644 --- a/source/common/src/tcol.c +++ b/source/common/src/tcol.c @@ -267,7 +267,7 @@ bool checkColumnEncode(char encode[TSDB_CL_COMPRESS_OPTION_LEN]) { } bool checkColumnEncodeOrSetDefault(uint8_t type, char encode[TSDB_CL_COMPRESS_OPTION_LEN]) { if (0 == strlen(encode)) { - strncpy(encode, getDefaultEncodeStr(type), TSDB_CL_COMPRESS_OPTION_LEN); + tstrncpy(encode, getDefaultEncodeStr(type), TSDB_CL_COMPRESS_OPTION_LEN); return true; } return checkColumnEncode(encode) && validColEncode(type, columnEncodeVal(encode)); @@ -284,7 +284,7 @@ bool checkColumnCompress(char compress[TSDB_CL_COMPRESS_OPTION_LEN]) { } bool checkColumnCompressOrSetDefault(uint8_t type, char compress[TSDB_CL_COMPRESS_OPTION_LEN]) { if (0 == strlen(compress)) { - strncpy(compress, getDefaultCompressStr(type), TSDB_CL_COMPRESS_OPTION_LEN); + tstrncpy(compress, getDefaultCompressStr(type), TSDB_CL_COMPRESS_OPTION_LEN); return true; } @@ -306,7 +306,7 @@ bool checkColumnLevel(char level[TSDB_CL_COMPRESS_OPTION_LEN]) { } bool checkColumnLevelOrSetDefault(uint8_t type, char level[TSDB_CL_COMPRESS_OPTION_LEN]) { if (0 == strlen(level)) { - strncpy(level, getDefaultLevelStr(type), TSDB_CL_COMPRESS_OPTION_LEN); + tstrncpy(level, getDefaultLevelStr(type), TSDB_CL_COMPRESS_OPTION_LEN); return true; } return checkColumnLevel(level) && validColCompressLevel(type, columnLevelVal(level)); @@ -330,7 +330,7 @@ void setColLevel(uint32_t* compress, uint8_t level) { int32_t setColCompressByOption(uint8_t type, uint8_t encode, uint16_t compressType, uint8_t level, bool check, uint32_t* compress) { - if(compress == NULL) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; + if (compress == NULL) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; if (check && !validColEncode(type, encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; setColEncode(compress, encode); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index a38842735c..c1ab7ccff0 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1434,7 +1434,7 @@ static void debugPrintTagVal(int8_t type, const void *val, int32_t vlen, const c case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: { char tmpVal[32] = {0}; - strncpy(tmpVal, val, vlen > 31 ? 31 : vlen); + tstrncpy(tmpVal, val, vlen > 31 ? 31 : vlen); printf("%s:%d type:%d vlen:%d, val:\"%s\"\n", tag, ln, (int32_t)type, vlen, tmpVal); } break; case TSDB_DATA_TYPE_FLOAT: diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index e43bdc4687..8529c5b690 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -15,23 +15,29 @@ #define _DEFAULT_SOURCE #include "tglobal.h" +#include "cJSON.h" #include "defines.h" #include "os.h" #include "osString.h" #include "tconfig.h" #include "tgrant.h" +#include "tjson.h" #include "tlog.h" #include "tmisce.h" #include "tunit.h" +#include "tutil.h" + #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" #endif +#define CONFIG_PATH_LEN (TSDB_FILENAME_LEN + 12) +#define CONFIG_FILE_LEN (CONFIG_PATH_LEN + 32) + // GRANT_CFG_DECLARE; SConfig *tsCfg = NULL; - // cluster char tsFirst[TSDB_EP_LEN] = {0}; char tsSecond[TSDB_EP_LEN] = {0}; @@ -40,6 +46,9 @@ char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port char tsVersionName[16] = "community"; uint16_t tsServerPort = 6030; int32_t tsVersion = 30000000; +int32_t tsForceReadConfig = 0; +int32_t tsdmConfigVersion = -1; +int32_t tsConfigInited = 0; int32_t tsStatusInterval = 1; // second int32_t tsNumOfSupportVnodes = 256; char tsEncryptAlgorithm[16] = {0}; @@ -54,13 +63,28 @@ char tsEncryptKey[17] = {0}; int32_t tsMaxShellConns = 50000; int32_t tsShellActivityTimer = 3; // second +// memory pool +int8_t tsMemPoolFullFunc = 0; +int8_t tsQueryUseMemoryPool = 1; +int32_t tsQueryBufferPoolSize = 0; // MB +int32_t tsSingleQueryMaxMemorySize = 0; // MB +int32_t tsMinReservedMemorySize = 0; // MB +int64_t tsCurrentAvailMemorySize = 0; +int8_t tsNeedTrim = 0; + // queue & threads +int32_t tsQueryMinConcurrentTaskNum = 1; +int32_t tsQueryMaxConcurrentTaskNum = 0; +int32_t tsQueryConcurrentTaskNum = 0; +int32_t tsQueryNoFetchTimeoutSec = 3600 * 5; + int32_t tsNumOfRpcThreads = 1; int32_t tsNumOfRpcSessions = 30000; int32_t tsShareConnLimit = 10; int32_t tsReadTimeout = 900; int32_t tsTimeToGetAvailableConn = 500000; +int32_t tsNumOfQueryThreads = 0; int32_t tsNumOfCommitThreads = 2; int32_t tsNumOfTaskQueueThreads = 16; int32_t tsNumOfMnodeQueryThreads = 16; @@ -78,6 +102,9 @@ int32_t tsMaxStreamBackendCache = 128; // M int32_t tsPQSortMemThreshold = 16; // M int32_t tsRetentionSpeedLimitMB = 0; // unlimited +const char *tsAlterCompactTaskKeywords = "max_compact_tasks"; +int32_t tsNumOfCompactThreads = 2; + // sync raft int32_t tsElectInterval = 25 * 1000; int32_t tsHeartbeatInterval = 1000; @@ -138,9 +165,9 @@ bool tsEnableCrashReport = false; #else bool tsEnableCrashReport = true; #endif -char *tsClientCrashReportUri = "/ccrashreport"; -char *tsSvrCrashReportUri = "/dcrashreport"; -int8_t tsSafetyCheckLevel = TSDB_SAFETY_CHECK_LEVELL_NORMAL; +char *tsClientCrashReportUri = "/ccrashreport"; +char *tsSvrCrashReportUri = "/dcrashreport"; +int32_t tsSafetyCheckLevel = TSDB_SAFETY_CHECK_LEVELL_NORMAL; // schemaless bool tsSmlDot2Underline = true; @@ -279,6 +306,7 @@ int32_t tsTtlFlushThreshold = 100; /* maximum number of dirty items in memory. int32_t tsTtlBatchDropNum = 10000; // number of tables dropped per batch // internal +bool tsDiskIDCheckEnabled = false; int32_t tsTransPullupInterval = 2; int32_t tsCompactPullupInterval = 10; int32_t tsMqRebalanceInterval = 2; @@ -301,6 +329,8 @@ bool tsFilterScalarMode = false; int tsResolveFQDNRetryTime = 100; // seconds int tsStreamAggCnt = 100000; +bool tsUpdateCacheBatch = true; + int8_t tsS3EpNum = 0; char tsS3Endpoint[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {""}; char tsS3AccessKey[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {""}; @@ -327,7 +357,7 @@ bool tsExperimental = true; int32_t tsMaxTsmaNum = 3; int32_t tsMaxTsmaCalcDelay = 600; int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24; // in ms, default to 1d -void* pTimezoneNameMap = NULL; +void *pTimezoneNameMap = NULL; #define TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, pName) \ if ((pItem = cfgGetItem(pCfg, pName)) == NULL) { \ @@ -457,9 +487,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { TAOS_RETURN(TSDB_CODE_SUCCESS); } -struct SConfig *taosGetCfg() { - return tsCfg; -} +struct SConfig *taosGetCfg() { return tsCfg; } static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { @@ -524,43 +552,71 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input } int32_t taosAddClientLogCfg(SConfig *pCfg) { - TAOS_CHECK_RETURN(cfgAddDir(pCfg, "configDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddDir(pCfg, "scriptDir", configDir, CFG_SCOPE_CLIENT, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); + TAOS_CHECK_RETURN(cfgAddDir(pCfg, "configDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddDir(pCfg, "scriptDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "asyncLog", tsAsyncLog, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "simDebugFlag", simDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebugFlag", tqClientDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_SERVER)); + cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, CFG_SCOPE_BOTH, + CFG_DYN_ENT_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "asyncLog", tsAsyncLog, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "simDebugFlag", simDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebugFlag", tqClientDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_SERVER, + CFG_CATEGORY_LOCAL)); TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t taosAddServerLogCfg(SConfig *pCfg) { - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "azDebugFlag", azDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "stDebugFlag", stDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "sndDebugFlag", sndDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "azDebugFlag", azDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebug", tqClientDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, + CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "stDebugFlag", stDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "sndDebugFlag", sndDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL)); TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -568,121 +624,152 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { char defaultFqdn[TSDB_FQDN_LEN] = {0}; int32_t defaultServerPort = 6030; if (taosGetFqdn(defaultFqdn) != 0) { - (void)strcpy(defaultFqdn, "localhost"); + tstrncpy(defaultFqdn, "localhost", TSDB_FQDN_LEN); } - TAOS_CHECK_RETURN(cfgAddString(pCfg, "firstEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); + cfgAddBool(pCfg, "forceReadConfig", tsForceReadConfig, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "firstEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_CLIENT, + CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, CFG_SCOPE_CLIENT, CFG_DYN_ENT_CLIENT)); + cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH, + CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN( - cfgAddBool(pCfg, "queryTableNotExistAsEmpty", tsQueryTbNotExistAsEmpty, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableScience", tsEnableScience, CFG_SCOPE_CLIENT, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); + cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, CFG_SCOPE_CLIENT, CFG_DYN_ENT_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryTableNotExistAsEmpty", tsQueryTbNotExistAsEmpty, CFG_SCOPE_CLIENT, + CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "enableScience", tsEnableScience, CFG_SCOPE_CLIENT, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, + CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, CFG_SCOPE_CLIENT, - CFG_DYN_CLIENT)); + CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, + CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( - cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "smlChildTableName", tsSmlChildTableName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); + cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "smlChildTableName", tsSmlChildTableName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, + CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddString(pCfg, "smlAutoChildTableNameDelimiter", tsSmlAutoChildTableNameDelimiter, - CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT, - CFG_DYN_CLIENT) != 0); + CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_SERVER, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, - CFG_SCOPE_CLIENT, CFG_DYN_NONE)); + cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); + cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( - cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "safetyCheckLevel", tsSafetyCheckLevel, 0, 5, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "bypassFlag", tsBypassFlag, 0, INT32_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); + cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, + CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, + CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT, + CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL) != 0); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_SERVER, + CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, + CFG_SCOPE_CLIENT, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT, + CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "safetyCheckLevel", tsSafetyCheckLevel, 0, 5, CFG_SCOPE_BOTH, CFG_DYN_BOTH, + CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "bypassFlag", tsBypassFlag, 0, INT32_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); tsNumOfRpcThreads = tsNumOfCores / 2; - tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, TSDB_MAX_RPC_THREADS); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_BOTH_LAZY, + CFG_CATEGORY_GLOBAL)); tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 100000); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, + CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); tsShareConnLimit = TRANGE(tsShareConnLimit, 1, 512); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "shareConnLimit", tsShareConnLimit, 1, 512, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "shareConnLimit", tsShareConnLimit, 1, 512, CFG_SCOPE_BOTH, CFG_DYN_BOTH_LAZY, + CFG_CATEGORY_GLOBAL)); tsReadTimeout = TRANGE(tsReadTimeout, 64, 24 * 3600 * 7); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "readTimeout", tsReadTimeout, 64, 24 * 3600 * 7, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "readTimeout", tsReadTimeout, 64, 24 * 3600 * 7, CFG_SCOPE_BOTH, + CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 10000000); - TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH, + CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); tsNumOfTaskQueueThreads = tsNumOfCores * 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 16); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, + CFG_DYN_CLIENT_LAZY, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - + cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, - CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN( - cfgAddInt32(pCfg, "maxTsmaCalcDelay", tsMaxTsmaCalcDelay, 600, 86400, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); + CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, + CFG_DYN_BOTH, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxTsmaCalcDelay", tsMaxTsmaCalcDelay, 600, 86400, CFG_SCOPE_CLIENT, + CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsmaDataDeleteMark", tsmaDataDeleteMark, 60 * 60 * 1000, INT64_MAX, - CFG_SCOPE_CLIENT, CFG_DYN_CLIENT)); + CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t taosAddSystemCfg(SConfig *pCfg) { SysNameInfo info = taosGetSysNameInfo(); - TAOS_CHECK_RETURN(cfgAddTimezone(pCfg, "timezone", tsTimezoneStr, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "assert", tsAssert, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableCoreFile", tsEnableCoreFile, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddTimezone(pCfg, "timezone", tsTimezoneStr, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH, CFG_DYN_BOTH, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "enableCoreFile", tsEnableCoreFile, CFG_SCOPE_BOTH, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN( + cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "simdEnable", tsSIMDEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "AVX512Enable", tsAVX512Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "simdEnable", tsSIMDEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "AVX512Enable", tsAVX512Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN( + cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); #if !defined(_ALPINE) - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN( + cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); #endif - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "os sysname", info.sysname, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "os nodename", info.nodename, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "os release", info.release, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "os version", info.version, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "os machine", info.machine, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN( + cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE, + CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "os sysname", info.sysname, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "os nodename", info.nodename, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "os release", info.release, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "os version", info.version, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "os machine", info.machine, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "version", td_version, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "compatible_version", td_compatible_version, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "gitinfo", td_gitinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "buildinfo", td_buildinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "version", td_version, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "compatible_version", td_compatible_version, CFG_SCOPE_BOTH, CFG_DYN_NONE, + CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "gitinfo", td_gitinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "buildinfo", td_buildinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -690,6 +777,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfCommitThreads = tsNumOfCores / 2; tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4); + tsNumOfCompactThreads = tsNumOfCommitThreads; + tsNumOfCompactThreads = TRANGE(tsNumOfCompactThreads, 2, 4); + tsNumOfSupportVnodes = tsNumOfCores * 2 + 5; tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); @@ -721,136 +811,145 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsLogBufferMemoryAllowed = TRANGE(tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); // clang-format off - TAOS_CHECK_RETURN(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "retentionSpeedLimitMB", tsRetentionSpeedLimitMB, 0, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxCompactConcurrency", tsNumOfCompactThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "retentionSpeedLimitMB", tsRetentionSpeedLimitMB, 0, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryUseMemoryPool", tsQueryUseMemoryPool, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL) != 0); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "memPoolFullFunc", tsMemPoolFullFunc, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL) != 0); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "singleQueryMaxMemorySize", tsSingleQueryMaxMemorySize, 0, 1000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL) != 0); + //TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferPoolSize", tsQueryBufferPoolSize, 0, 1000000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0); + TAOS_CHECK_RETURN(cfgAddInt32Ex(pCfg, "minReservedMemorySize", 0, 1024, 1000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL) != 0); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryNoFetchTimeoutSec", tsQueryNoFetchTimeoutSec, 60, 1000000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_LOCAL) != 0); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 4, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 4, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "syncLogBufferMemoryAllowed", tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbCheckSyncIntervalSec", tsArbCheckSyncIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbSetAssignedTimeoutSec", tsArbSetAssignedTimeoutSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "syncLogBufferMemoryAllowed", tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbCheckSyncIntervalSec", tsArbCheckSyncIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "arbSetAssignedTimeoutSec", tsArbSetAssignedTimeoutSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 86400, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 1, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogMaxLen", tsSlowLogMaxLen, 1, 16384, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "slowLogScope", tsSlowLogScopeString, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "slowLogExceptDb", tsSlowLogExceptDb, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 86400, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 1, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "slowLogMaxLen", tsSlowLogMaxLen, 1, 16384, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "slowLogScope", tsSlowLogScopeString, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "slowLogExceptDb", tsSlowLogExceptDb, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableAuditDelete", tsEnableAuditDelete, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_SERVER, CFG_DYN_BOTH,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableAuditDelete", tsEnableAuditDelete, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "rsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "snodeAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_SERVER, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "rsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "snodeAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxTsmaNum", tsMaxTsmaNum, 0, 3, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "diskIDCheckEnabled", tsDiskIDCheckEnabled, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlBatchDropNum", tsTtlBatchDropNum, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlFlushThreshold", tsTtlFlushThreshold, -1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "s3MigrateEnabled", tsS3MigrateEnabled, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxTsmaNum", tsMaxTsmaNum, 0, 3, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlBatchDropNum", tsTtlBatchDropNum, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlFlushThreshold", tsTtlFlushThreshold, -1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "s3MigrateEnabled", tsS3MigrateEnabled, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1800, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "concurrentCheckpoint", tsMaxConcurrentCheckpoint, 1, 10, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1800, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "concurrentCheckpoint", tsMaxConcurrentCheckpoint, 1, 10, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + SConfigItem *pItem = NULL; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "fPrecision"); + tsFPrecision = pItem->fval; + TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER, CFG_DYN_NONE)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey[0], CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint[0], CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - - TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey[0], CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint[0], CFG_SCOPE_SERVER, CFG_DYN_NONE)); - TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE)); - - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL)); // min free disk space used to check if the disk is full [50MB, 1GB] - TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); + TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); // clang-format on @@ -923,6 +1022,14 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } + pItem = cfgGetItem(pCfg, "maxCompactConcurrency"); + if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { + tsNumOfCompactThreads = numOfCores / 2; + tsNumOfCompactThreads = TRANGE(tsNumOfCompactThreads, 2, 4); + pItem->i32 = tsNumOfCompactThreads; + pItem->stype = stype; + } + pItem = cfgGetItem(pCfg, "numOfMnodeReadThreads"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { tsNumOfMnodeReadThreads = numOfCores / 8; @@ -1198,6 +1305,9 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "fqdn"); tstrncpy(tsLocalFqdn, pItem->str, TSDB_FQDN_LEN); + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "forceReadConfig"); + tsForceReadConfig = pItem->bval; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "serverPort"); tsServerPort = (uint16_t)pItem->i32; (void)snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); @@ -1353,9 +1463,6 @@ static int32_t taosSetSystemCfg(SConfig *pCfg) { tsEnableCoreFile = pItem->bval; taosSetCoreDump(tsEnableCoreFile); - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "assert"); - tsAssert = pItem->bval; - // todo tsVersion = 30000000; @@ -1411,6 +1518,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfCommitThreads"); tsNumOfCommitThreads = pItem->i32; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "maxCompactConcurrency"); + tsNumOfCompactThreads = pItem->i32; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "retentionSpeedLimitMB"); tsRetentionSpeedLimitMB = pItem->i32; @@ -1542,6 +1652,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "maxTsmaNum"); tsMaxTsmaNum = pItem->i32; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "diskIDCheckEnabled"); + tsDiskIDCheckEnabled = pItem->bval; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "transPullupInterval"); tsTransPullupInterval = pItem->i32; @@ -1692,6 +1805,21 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "experimental"); tsExperimental = pItem->bval; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "queryUseMemoryPool"); + tsQueryUseMemoryPool = pItem->bval; + + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "memPoolFullFunc"); + tsMemPoolFullFunc = pItem->bval; + + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "singleQueryMaxMemorySize"); + tsSingleQueryMaxMemorySize = pItem->i32; + + // TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "queryBufferPoolSize"); + // tsQueryBufferPoolSize = pItem->i32; + + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minReservedMemorySize"); + tsMinReservedMemorySize = pItem->i32; + // GRANT_CFG_GET; TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -1768,7 +1896,7 @@ _exit: int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs) { int32_t code = TSDB_CODE_SUCCESS; - + int32_t lino = -1; if (tsCfg == NULL) code = osDefaultInit(); if (code != 0) { (void)printf("failed to init os since %s\n", tstrerror(code)); @@ -1776,10 +1904,14 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char * SConfig *pCfg = NULL; TAOS_CHECK_RETURN(cfgInit(&pCfg)); - - TAOS_CHECK_GOTO(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE), NULL, _exit); - TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "debugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER), NULL, _exit); - TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER), NULL, _exit); + TAOS_CHECK_GOTO(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL), &lino, + _exit); + TAOS_CHECK_GOTO( + cfgAddInt32(pCfg, "debugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL), &lino, + _exit); + TAOS_CHECK_GOTO( + cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL), &lino, + _exit); if ((code = taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) { (void)printf("failed to load cfg since %s\n", tstrerror(code)); @@ -1791,7 +1923,7 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char * goto _exit; } - TAOS_CHECK_GOTO(taosSetTfsCfg(pCfg), NULL, _exit); + TAOS_CHECK_GOTO(taosSetTfsCfg(pCfg), &lino, _exit); SConfigItem *pItem = NULL; if ((pItem = cfgGetItem(pCfg, "dDebugFlag")) == NULL) { @@ -1808,7 +1940,8 @@ _exit: static int32_t taosCheckGlobalCfg() { uint32_t ipv4 = 0; - int32_t code = taosGetIpv4FromFqdn(tsLocalFqdn, &ipv4); + uInfo("start to check global tsLocalFqdn:%s, tsServerPort:%u", tsLocalFqdn, tsServerPort); + int32_t code = taosGetIpv4FromFqdn(tsLocalFqdn, &ipv4); if (code) { uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, tstrerror(code)); TAOS_RETURN(TSDB_CODE_RPC_FQDN_ERROR); @@ -1829,6 +1962,210 @@ static int32_t cfgInitWrapper(SConfig **pCfg) { TAOS_RETURN(TSDB_CODE_SUCCESS); } +int32_t setAllConfigs(SConfig *pCfg) { + int32_t code = 0; + int32_t lino = -1; + TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosUpdateServerCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosSetServerCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosSetReleaseCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosSetTfsCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosSetS3Cfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosSetSystemCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosSetFileHandlesLimit(), &lino, _exit); +_exit: + TAOS_RETURN(code); +} + +int32_t cfgDeserialize(SArray *array, char *buf, bool isGlobal) { + int32_t code = TSDB_CODE_SUCCESS; + + cJSON *pRoot = cJSON_Parse(buf); + if (pRoot == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (isGlobal) { + cJSON *pItem = cJSON_GetObjectItem(pRoot, "version"); + if (pItem == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + tsdmConfigVersion = pItem->valueint; + } + + int32_t sz = taosArrayGetSize(array); + cJSON *configs = cJSON_GetObjectItem(pRoot, "configs"); + if (configs == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + for (int i = 0; i < sz; i++) { + SConfigItem *pItem = (SConfigItem *)taosArrayGet(array, i); + cJSON *pJson = cJSON_GetObjectItem(configs, pItem->name); + if (pJson == NULL) { + continue; + } + if (strcasecmp(pItem->name, "dataDir") == 0) { + if (!tsDiskIDCheckEnabled) { + continue; + } + int sz = cJSON_GetArraySize(pJson); + cJSON *filed = NULL; + // check disk id for each dir + for (int j = 0; j < sz; j++) { + cJSON *diskCfgJson = cJSON_GetArrayItem(pJson, j); + if (diskCfgJson == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + filed = cJSON_GetObjectItem(diskCfgJson, "dir"); + if (filed == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + char *dir = cJSON_GetStringValue(filed); + filed = cJSON_GetObjectItem(diskCfgJson, "disk_id"); + if (filed == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + int64_t actDiskID = 0; + int64_t expDiskID = taosStr2Int64(cJSON_GetStringValue(filed), NULL, 10); + if ((code = taosGetFileDiskID(dir, &actDiskID)) != 0) { + uError("failed to get disk id for dir:%s, since %s", dir, tstrerror(code)); + goto _exit; + } + if (actDiskID != expDiskID) { + uError("failed to check disk id for dir:%s, actDiskID%" PRId64 ", expDiskID%" PRId64, dir, actDiskID, + expDiskID); + code = TSDB_CODE_INVALID_DISK_ID; + goto _exit; + } + } + continue; + } + pItem->stype = CFG_STYPE_CFG_FILE; + switch (pItem->dtype) { + { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + pItem->bval = cJSON_IsTrue(pJson); + break; + case CFG_DTYPE_INT32: + pItem->i32 = pJson->valueint; + break; + case CFG_DTYPE_INT64: + pItem->i64 = atoll(cJSON_GetStringValue(pJson)); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + pItem->fval = taosStr2Float(cJSON_GetStringValue(pJson), NULL); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + taosMemoryFree(pItem->str); + pItem->str = taosStrdup(pJson->valuestring); + if (pItem->str == NULL) { + code = terrno; + goto _exit; + } + break; + } + } + } +_exit: + if (code != TSDB_CODE_SUCCESS) { + uError("failed to deserialize config since %s", tstrerror(code)); + } + cJSON_Delete(pRoot); + return code; +} + +int32_t readCfgFile(const char *path, bool isGlobal) { + int32_t code = 0; + char filename[CONFIG_FILE_LEN] = {0}; + SArray *array = NULL; + if (isGlobal) { + array = taosGetGlobalCfg(tsCfg); + snprintf(filename, sizeof(filename), "%s%sdnode%sconfig%sglobal.json", path, TD_DIRSEP, TD_DIRSEP, TD_DIRSEP); + } else { + array = taosGetLocalCfg(tsCfg); + snprintf(filename, sizeof(filename), "%s%sdnode%sconfig%slocal.json", path, TD_DIRSEP, TD_DIRSEP, TD_DIRSEP); + } + uInfo("start to read config file:%s", filename); + + if (!taosCheckExistFile(filename)) { + uInfo("config file:%s does not exist", filename); + TAOS_RETURN(TSDB_CODE_SUCCESS); + } + int64_t fileSize = 0; + char *buf = NULL; + if (taosStatFile(filename, &fileSize, NULL, NULL) < 0) { + code = terrno; + uError("failed to stat file:%s , since %s", filename, tstrerror(code)); + TAOS_RETURN(code); + } + TdFilePtr pFile = taosOpenFile(filename, TD_FILE_READ); + if (pFile == NULL) { + code = terrno; + uError("failed to open file:%s , since %s", filename, tstrerror(code)); + goto _exit; + } + buf = (char *)taosMemoryMalloc(fileSize + 1); + if (buf == NULL) { + code = terrno; + uError("failed to malloc memory for file:%s , since %s", filename, tstrerror(code)); + goto _exit; + } + if (taosReadFile(pFile, buf, fileSize) != fileSize) { + code = terrno; + uError("failed to read file:%s , config since %s", filename, tstrerror(code)); + goto _exit; + } + buf[fileSize] = 0; + code = cfgDeserialize(array, buf, isGlobal); + if (code != TSDB_CODE_SUCCESS) { + uError("failed to deserialize config from %s since %s", filename, tstrerror(code)); + goto _exit; + } + +_exit: + if (code != TSDB_CODE_SUCCESS) { + uError("failed to read config from %s since %s", filename, tstrerror(code)); + } + taosMemoryFree(buf); + (void)taosCloseFile(&pFile); + TAOS_RETURN(code); +} + +int32_t tryLoadCfgFromDataDir(SConfig *pCfg) { + int32_t code = 0; + SConfigItem *pItem = NULL; + TAOS_CHECK_GET_CFG_ITEM(tsCfg, pItem, "forceReadConfig"); + tsForceReadConfig = pItem->i32; + if (!tsForceReadConfig) { + uInfo("load config from tsDataDir:%s", tsDataDir); + code = readCfgFile(tsDataDir, false); + if (code != TSDB_CODE_SUCCESS) { + uError("failed to read local config from %s since %s", tsDataDir, tstrerror(code)); + TAOS_RETURN(code); + } + code = readCfgFile(tsDataDir, true); + if (code != TSDB_CODE_SUCCESS) { + uError("failed to read global config from %s since %s", tsDataDir, tstrerror(code)); + TAOS_RETURN(code); + } + } + TAOS_RETURN(code); +} + int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { if (tsCfg != NULL) TAOS_RETURN(TSDB_CODE_SUCCESS); @@ -1864,6 +2201,11 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile TAOS_RETURN(code); } + if (!tsc) { + TAOS_CHECK_GOTO(taosSetTfsCfg(tsCfg), &lino, _exit); + TAOS_CHECK_GOTO(tryLoadCfgFromDataDir(tsCfg), &lino, _exit); + } + if (tsc) { TAOS_CHECK_GOTO(taosSetClientLogCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit); @@ -1873,7 +2215,6 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile TAOS_CHECK_GOTO(taosUpdateServerCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetServerCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetReleaseCfg(tsCfg), &lino, _exit); - TAOS_CHECK_GOTO(taosSetTfsCfg(tsCfg), &lino, _exit); TAOS_CHECK_GOTO(taosSetS3Cfg(tsCfg), &lino, _exit); } @@ -1888,7 +2229,6 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile TAOS_CHECK_GOTO(taosSetAllDebugFlag(tsCfg, pItem->i32), &lino, _exit); cfgDumpCfg(tsCfg, tsc, false); - TAOS_CHECK_GOTO(taosCheckGlobalCfg(), &lino, _exit); _exit: @@ -1950,6 +2290,15 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, uInfo("%s set from %f to %f", optName, *pVar, flag); *pVar = flag; } break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: { + char *pVar = pOptions[d].optionVar; + tstrncpy(pVar, pItem->str, strlen(pItem->str)); + uInfo("%s set to %s", optName, pVar); + } break; default: code = TSDB_CODE_INVALID_CFG; break; @@ -1963,6 +2312,7 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = -1; if (strcasecmp(name, "resetlog") == 0) { // trigger, no item in cfg @@ -1974,7 +2324,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { cfgLock(pCfg); SConfigItem *pItem = cfgGetItem(pCfg, name); - if (!pItem || (pItem->dynScope & CFG_DYN_SERVER) == 0) { + if (!pItem || (pItem->dynScope == CFG_DYN_CLIENT)) { uError("failed to config:%s, not support", name); code = TSDB_CODE_INVALID_CFG; goto _exit; @@ -1994,7 +2344,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { if (strcasecmp("slowLogScope", name) == 0) { int32_t scope = 0; - TAOS_CHECK_GOTO(taosSetSlowLogScope(pItem->str, &scope), NULL, _exit); + TAOS_CHECK_GOTO(taosSetSlowLogScope(pItem->str, &scope), &lino, _exit); tsSlowLogScope = scope; code = TSDB_CODE_SUCCESS; goto _exit; @@ -2008,25 +2358,46 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { { // 'bool/int32_t/int64_t/float/double' variables with general modification function static OptionNameAndVar debugOptions[] = { - {"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, {"mDebugFlag", &mDebugFlag}, - {"wDebugFlag", &wDebugFlag}, {"azDebugFlag", &azDebugFlag}, {"sDebugFlag", &sDebugFlag}, - {"tsdbDebugFlag", &tsdbDebugFlag}, {"tqDebugFlag", &tqDebugFlag}, {"fsDebugFlag", &fsDebugFlag}, - {"udfDebugFlag", &udfDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, - {"tdbDebugFlag", &tdbDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag}, - {"smaDebugFlag", &smaDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag}, - {"metaDebugFlag", &metaDebugFlag}, {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, - {"tqClientDebugFlag", &tqClientDebugFlag}, + {"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, + {"mDebugFlag", &mDebugFlag}, {"wDebugFlag", &wDebugFlag}, + {"azDebugFlag", &azDebugFlag}, {"sDebugFlag", &sDebugFlag}, + {"tsdbDebugFlag", &tsdbDebugFlag}, {"tqDebugFlag", &tqDebugFlag}, + {"fsDebugFlag", &fsDebugFlag}, {"udfDebugFlag", &udfDebugFlag}, + {"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, + {"tdbDebugFlag", &tdbDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, + {"uDebugFlag", &uDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, + {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag}, + {"metaDebugFlag", &metaDebugFlag}, {"stDebugFlag", &stDebugFlag}, + {"sndDebugFlag", &sndDebugFlag}, {"tqClientDebugFlag", &tqClientDebugFlag}, }; static OptionNameAndVar options[] = {{"audit", &tsEnableAudit}, {"asynclog", &tsAsyncLog}, {"disableStream", &tsDisableStream}, {"enableWhiteList", &tsEnableWhiteList}, + {"statusInterval", &tsStatusInterval}, {"telemetryReporting", &tsEnableTelem}, {"monitor", &tsEnableMonitor}, {"monitorInterval", &tsMonitorInterval}, + {"monitorComp", &tsMonitorComp}, + {"monitorForceV2", &tsMonitorForceV2}, + {"monitorLogProtocol", &tsMonitorLogProtocol}, + {"monitorMaxLogs", &tsMonitorMaxLogs}, + {"auditCreateTable", &tsEnableAuditCreateTable}, + {"auditInterval", &tsAuditInterval}, {"slowLogThreshold", &tsSlowLogThreshold}, + {"compressMsgSize", &tsCompressMsgSize}, + {"compressor", &tsCompressor}, + {"dPrecision", &tsDPrecision}, + {"fPrecision", &tsFPrecision}, + {"maxRange", &tsMaxRange}, + {"maxTsmaNum", &tsMaxTsmaNum}, + {"queryRsmaTolerance", &tsQueryRsmaTolerance}, + {"uptimeInterval", &tsUptimeInterval}, + {"slowLogMaxLen", &tsSlowLogMaxLen}, + {"slowLogScope", &tsSlowLogScope}, + {"slowLogExceptDb", &tsSlowLogExceptDb}, {"mndSdbWriteDelta", &tsMndSdbWriteDelta}, {"minDiskFreeSize", &tsMinDiskFreeSize}, @@ -2034,9 +2405,25 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"randErrorDivisor", &tsRandErrDivisor}, {"randErrorScope", &tsRandErrScope}, {"syncLogBufferMemoryAllowed", &tsLogBufferMemoryAllowed}, + {"resolveFQDNRetryTime", &tsResolveFQDNRetryTime}, + {"syncHeartbeatInterval", &tsHeartbeatInterval}, + {"syncHeartbeatTimeout", &tsHeartbeatTimeout}, + {"syncSnapReplMaxWaitN", &tsSnapReplMaxWaitN}, + {"walFsyncDataSizeLimit", &tsWalFsyncDataSizeLimit}, + + {"numOfCores", &tsNumOfCores}, + + {"enableCoreFile", &tsEnableCoreFile}, + + {"telemetryInterval", &tsTelemInterval}, {"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold}, {"checkpointInterval", &tsStreamCheckpointInterval}, + {"concurrentCheckpoint", &tsMaxConcurrentCheckpoint}, + + {"retentionSpeedLimitMB", &tsRetentionSpeedLimitMB}, + {"ttlChangeOnWrite", &tsTtlChangeOnWrite}, + {"logKeepDays", &tsLogKeepDays}, {"maxStreamBackendCache", &tsMaxStreamBackendCache}, {"mqRebalanceInterval", &tsMqRebalanceInterval}, @@ -2051,17 +2438,30 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"ttlBatchDropNum", &tsTtlBatchDropNum}, {"ttlFlushThreshold", &tsTtlFlushThreshold}, {"ttlPushInterval", &tsTtlPushIntervalSec}, + {"ttlUnit", &tsTtlUnit}, + {"s3Accesskey", &tsS3AccessKey}, + {"s3BucketName", &tsS3BucketName}, + {"s3Endpoint", &tsS3Endpoint}, {"s3MigrateIntervalSec", &tsS3MigrateIntervalSec}, {"s3MigrateEnabled", &tsS3MigrateEnabled}, //{"s3BlockSize", &tsS3BlockSize}, {"s3BlockCacheSize", &tsS3BlockCacheSize}, {"s3PageCacheSize", &tsS3PageCacheSize}, {"s3UploadDelaySec", &tsS3UploadDelaySec}, + {"mndLogRetention", &tsMndLogRetention}, {"supportVnodes", &tsNumOfSupportVnodes}, {"experimental", &tsExperimental}, - {"maxTsmaNum", &tsMaxTsmaNum}, + + {"numOfRpcSessions", &tsNumOfRpcSessions}, + {"rpcQueueMemoryAllowed", &tsQueueMemoryAllowed}, + {"shellActivityTimer", &tsShellActivityTimer}, + {"readTimeout", &tsReadTimeout}, {"safetyCheckLevel", &tsSafetyCheckLevel}, - {"bypassFlag", &tsBypassFlag}}; + {"bypassFlag", &tsBypassFlag}, + {"arbHeartBeatIntervalSec", &tsArbHeartBeatIntervalSec}, + {"arbCheckSyncIntervalSec", &tsArbCheckSyncIntervalSec}, + {"arbSetAssignedTimeoutSec", &tsArbSetAssignedTimeoutSec}, + {"queryNoFetchTimeoutSec", &tsQueryNoFetchTimeoutSec}}; if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) { code = taosCfgSetOption(options, tListLen(options), pItem, false); @@ -2078,12 +2478,13 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { int32_t lino = 0; if (strcasecmp("charset", name) == 0 || strcasecmp("timezone", name) == 0) { - goto _out; + goto _out; } + cfgLock(pCfg); SConfigItem *pItem = cfgGetItem(pCfg, name); - if ((pItem == NULL) || (pItem->dynScope & CFG_DYN_CLIENT) == 0) { + if ((pItem == NULL) || pItem->dynScope == CFG_DYN_SERVER) { uError("failed to config:%s, not support", name); code = TSDB_CODE_INVALID_CFG; goto _out; @@ -2283,7 +2684,6 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { }; static OptionNameAndVar options[] = {{"asyncLog", &tsAsyncLog}, - {"assert", &tsAssert}, {"compressMsgSize", &tsCompressMsgSize}, {"countAlwaysReturnValue", &tsCountAlwaysReturnValue}, {"crashReporting", &tsEnableCrashReport}, @@ -2291,7 +2691,6 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { {"keepColumnName", &tsKeepColumnName}, {"logKeepDays", &tsLogKeepDays}, {"maxInsertBatchRows", &tsMaxInsertBatchRows}, - {"maxRetryWaitTime", &tsMaxRetryWaitTime}, {"minSlidingTime", &tsMinSlidingTime}, {"minIntervalTime", &tsMinIntervalTime}, {"numOfLogLines", &tsNumOfLogLines}, @@ -2301,17 +2700,12 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { {"queryPlannerTrace", &tsQueryPlannerTrace}, {"queryNodeChunkSize", &tsQueryNodeChunkSize}, {"queryUseNodeAllocator", &tsQueryUseNodeAllocator}, - {"randErrorChance", &tsRandErrChance}, - {"randErrorDivisor", &tsRandErrDivisor}, - {"randErrorScope", &tsRandErrScope}, {"smlDot2Underline", &tsSmlDot2Underline}, - {"shellActivityTimer", &tsShellActivityTimer}, {"useAdapter", &tsUseAdapter}, - {"experimental", &tsExperimental}, {"multiResultFunctionStarReturnTags", &tsMultiResultFunctionStarReturnTags}, {"maxTsmaCalcDelay", &tsMaxTsmaCalcDelay}, {"tsmaDataDeleteMark", &tsmaDataDeleteMark}, - {"safetyCheckLevel", &tsSafetyCheckLevel}, + {"numOfRpcSessions", &tsNumOfRpcSessions}, {"bypassFlag", &tsBypassFlag}}; if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) { @@ -2426,3 +2820,406 @@ int8_t taosGranted(int8_t type) { } return 0; } + +int32_t globalConfigSerialize(int32_t version, SArray *array, char **serialized) { + char buf[30]; + cJSON *json = cJSON_CreateObject(); + if (json == NULL) goto _exit; + if (cJSON_AddNumberToObject(json, "file_version", GLOBAL_CONFIG_FILE_VERSION) == NULL) goto _exit; + if (cJSON_AddNumberToObject(json, "version", version) == NULL) goto _exit; + int sz = taosArrayGetSize(array); + + cJSON *cField = cJSON_CreateObject(); + if (cField == NULL) goto _exit; + + if (!cJSON_AddItemToObject(json, "configs", cField)) goto _exit; + + // cjson only support int32_t or double + // string are used to prohibit the loss of precision + for (int i = 0; i < sz; i++) { + SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); + switch (item->dtype) { + { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + if (cJSON_AddBoolToObject(cField, item->name, item->bval) == NULL) goto _exit; + break; + case CFG_DTYPE_INT32: + if (cJSON_AddNumberToObject(cField, item->name, item->i32) == NULL) goto _exit; + break; + case CFG_DTYPE_INT64: + (void)sprintf(buf, "%" PRId64, item->i64); + if (cJSON_AddStringToObject(cField, item->name, buf) == NULL) goto _exit; + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + (void)sprintf(buf, "%f", item->fval); + if (cJSON_AddStringToObject(cField, item->name, buf) == NULL) goto _exit; + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + if (cJSON_AddStringToObject(cField, item->name, item->str) == NULL) goto _exit; + break; + } + } + } + char *pSerialized = tjsonToString(json); +_exit: + if (terrno != TSDB_CODE_SUCCESS) { + uError("failed to serialize global config since %s", tstrerror(terrno)); + } + cJSON_Delete(json); + *serialized = pSerialized; + return terrno; +} + +int32_t localConfigSerialize(SArray *array, char **serialized) { + char buf[30]; + cJSON *json = cJSON_CreateObject(); + if (json == NULL) goto _exit; + + int sz = taosArrayGetSize(array); + + cJSON *cField = cJSON_CreateObject(); + if (cField == NULL) goto _exit; + if (cJSON_AddNumberToObject(json, "file_version", LOCAL_CONFIG_FILE_VERSION) == NULL) goto _exit; + if (!cJSON_AddItemToObject(json, "configs", cField)) goto _exit; + + // cjson only support int32_t or double + // string are used to prohibit the loss of precision + for (int i = 0; i < sz; i++) { + SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); + if (strcasecmp(item->name, "dataDir") == 0) { + int32_t sz = taosArrayGetSize(item->array); + cJSON *dataDirs = cJSON_CreateArray(); + if (!cJSON_AddItemToObject(cField, item->name, dataDirs)) { + uError("failed to serialize global config since %s", tstrerror(terrno)); + goto _exit; + } + for (int j = 0; j < sz; j++) { + SDiskCfg *disk = (SDiskCfg *)taosArrayGet(item->array, j); + cJSON *dataDir = cJSON_CreateObject(); + if (dataDir == NULL) goto _exit; + if (!cJSON_AddItemToArray(dataDirs, dataDir)) { + uError("failed to serialize global config since %s", tstrerror(terrno)); + goto _exit; + } + if (cJSON_AddStringToObject(dataDir, "dir", disk->dir) == NULL) goto _exit; + if (cJSON_AddNumberToObject(dataDir, "level", disk->level) == NULL) goto _exit; + if (disk->diskId == 0) { + if (taosGetFileDiskID(disk->dir, &disk->diskId) != 0) { + uError("failed to get disk id for %s", disk->dir); + goto _exit; + } + } + (void)sprintf(buf, "%" PRId64, disk->diskId); + if (cJSON_AddStringToObject(dataDir, "disk_id", buf) == NULL) goto _exit; + if (cJSON_AddNumberToObject(dataDir, "primary", disk->primary) == NULL) goto _exit; + if (cJSON_AddNumberToObject(dataDir, "disable", disk->disable) == NULL) goto _exit; + } + continue; + } + switch (item->dtype) { + { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + if (cJSON_AddBoolToObject(cField, item->name, item->bval) == NULL) goto _exit; + break; + case CFG_DTYPE_INT32: + if (cJSON_AddNumberToObject(cField, item->name, item->i32) == NULL) goto _exit; + break; + case CFG_DTYPE_INT64: + (void)sprintf(buf, "%" PRId64, item->i64); + if (cJSON_AddStringToObject(cField, item->name, buf) == NULL) goto _exit; + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + (void)sprintf(buf, "%f", item->fval); + if (cJSON_AddStringToObject(cField, item->name, buf) == NULL) goto _exit; + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + if (cJSON_AddStringToObject(cField, item->name, item->str) == NULL) goto _exit; + break; + } + } + } + char *pSerialized = tjsonToString(json); +_exit: + if (terrno != TSDB_CODE_SUCCESS) { + uError("failed to serialize local config since %s", tstrerror(terrno)); + } + cJSON_Delete(json); + *serialized = pSerialized; + return terrno; +} + +int32_t taosPersistGlobalConfig(SArray *array, const char *path, int32_t version) { + int32_t code = 0; + int32_t lino = 0; + char *buffer = NULL; + TdFilePtr pFile = NULL; + char filepath[CONFIG_FILE_LEN] = {0}; + char filename[CONFIG_FILE_LEN] = {0}; + snprintf(filepath, sizeof(filepath), "%s%sconfig", path, TD_DIRSEP); + snprintf(filename, sizeof(filename), "%s%sconfig%sglobal.json", path, TD_DIRSEP, TD_DIRSEP); + + TAOS_CHECK_GOTO(taosMkDir(filepath), &lino, _exit); + + TdFilePtr pConfigFile = + taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); + + if (pConfigFile == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + uError("failed to open file:%s since %s", filename, tstrerror(code)); + TAOS_RETURN(code); + } + char *serialized = NULL; + TAOS_CHECK_GOTO(globalConfigSerialize(version, array, &serialized), &lino, _exit); + + if (taosWriteFile(pConfigFile, serialized, strlen(serialized)) < 0) { + lino = __LINE__; + code = TAOS_SYSTEM_ERROR(errno); + uError("failed to write file:%s since %s", filename, tstrerror(code)); + goto _exit; + } + +_exit: + if (code != TSDB_CODE_SUCCESS) { + uError("failed to persist global config at line:%d, since %s", lino, tstrerror(code)); + } + (void)taosCloseFile(&pConfigFile); + taosMemoryFree(serialized); + return code; +} + +int32_t taosPersistLocalConfig(const char *path) { + int32_t code = 0; + int32_t lino = 0; + char *buffer = NULL; + TdFilePtr pFile = NULL; + char filepath[CONFIG_FILE_LEN] = {0}; + char filename[CONFIG_FILE_LEN] = {0}; + snprintf(filepath, sizeof(filepath), "%s%sconfig", path, TD_DIRSEP); + snprintf(filename, sizeof(filename), "%s%sconfig%slocal.json", path, TD_DIRSEP, TD_DIRSEP); + + // TODO(beryl) need to check if the file is existed + TAOS_CHECK_GOTO(taosMkDir(filepath), &lino, _exit); + + TdFilePtr pConfigFile = + taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); + + if (pConfigFile == NULL) { + code = TAOS_SYSTEM_ERROR(terrno); + uError("failed to open file:%s since %s", filename, tstrerror(code)); + TAOS_RETURN(code); + } + + char *serialized = NULL; + TAOS_CHECK_GOTO(localConfigSerialize(taosGetLocalCfg(tsCfg), &serialized), &lino, _exit); + if (taosWriteFile(pConfigFile, serialized, strlen(serialized)) < 0) { + lino = __LINE__; + code = TAOS_SYSTEM_ERROR(terrno); + uError("failed to write file:%s since %s", filename, tstrerror(code)); + goto _exit; + } + +_exit: + if (code != TSDB_CODE_SUCCESS) { + uError("failed to persist local config at line:%d, since %s", lino, tstrerror(code)); + } + (void)taosCloseFile(&pConfigFile); + taosMemoryFree(serialized); + return code; +} + +int32_t tSerializeSConfigArray(SEncoder *pEncoder, SArray *array) { + int32_t code = 0; + int32_t lino = 0; + int32_t sz = taosArrayGetSize(array); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, sz)); + for (int i = 0; i < sz; i++) { + SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); + TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, item->name)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, item->dtype)); + switch (item->dtype) { + { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + TAOS_CHECK_EXIT(tEncodeBool(pEncoder, item->bval)); + break; + case CFG_DTYPE_INT32: + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, item->i32)); + break; + case CFG_DTYPE_INT64: + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, item->i64)); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + TAOS_CHECK_EXIT(tEncodeFloat(pEncoder, item->fval)); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, item->str)); + break; + } + } + } +_exit: + return code; +} + +int32_t tDeserializeSConfigArray(SDecoder *pDecoder, SArray *array) { + int32_t code = 0; + int32_t lino = 0; + int32_t sz = 0; + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &sz)); + for (int i = 0; i < sz; i++) { + SConfigItem item = {0}; + TAOS_CHECK_EXIT(tDecodeCStr(pDecoder, &item.name)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, (int32_t *)&item.dtype)); + switch (item.dtype) { + { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + TAOS_CHECK_EXIT(tDecodeBool(pDecoder, &item.bval)); + break; + case CFG_DTYPE_INT32: + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &item.i32)); + break; + case CFG_DTYPE_INT64: + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &item.i64)); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + TAOS_CHECK_EXIT(tDecodeFloat(pDecoder, &item.fval)); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + TAOS_CHECK_EXIT(tDecodeCStr(pDecoder, &item.str)); + break; + } + } + if (taosArrayPush(array, &item) == NULL) { + code = terrno; + goto _exit; + } + } +_exit: + if (code != TSDB_CODE_SUCCESS) { + uError("failed to deserialize SConfigItem at line:%d, since %s", lino, tstrerror(code)); + } + return code; +} + +bool compareSConfigItem(const SConfigItem *item1, const SConfigItem *item2) { + switch (item1->dtype) { + case CFG_DTYPE_BOOL: + if (item1->bval != item2->bval) return false; + break; + case CFG_DTYPE_FLOAT: + if (item1->fval != item2->fval) return false; + break; + case CFG_DTYPE_INT32: + if (item1->i32 != item2->i32) return false; + break; + case CFG_DTYPE_INT64: + if (item1->i64 != item2->i64) return false; + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + if (strcmp(item1->str, item2->str) != 0) return false; + break; + default: + return false; + } + return true; +} + +int32_t compareSConfigItemArrays(SArray *mArray, const SArray *dArray, SArray *diffArray) { + int32_t code = 0; + int32_t msz = taosArrayGetSize(mArray); + int32_t dsz = taosArrayGetSize(dArray); + + if (msz != dsz) { + return TSDB_CODE_FAILED; + } + + for (int i = 0; i < msz; i++) { + SConfigItem *mItem = (SConfigItem *)taosArrayGet(mArray, i); + SConfigItem *dItem = (SConfigItem *)taosArrayGet(dArray, i); + if (!compareSConfigItem(mItem, dItem)) { + code = TSDB_CODE_FAILED; + if (taosArrayPush(diffArray, mItem) == NULL) { + code = terrno; + return code; + } + } + } + + return code; +} + +void printConfigNotMatch(SArray *array) { + uError( + "The global configuration parameters in the configuration file do not match those in the cluster. Please " + "turn off the forceReadConfigFile option or modify the global configuration parameters that are not " + "configured."); + int32_t sz = taosArrayGetSize(array); + for (int i = 0; i < sz; i++) { + SConfigItem *item = (SConfigItem *)taosArrayGet(array, i); + switch (item->dtype) { + { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + uError("config %s in cluster value is:%d", item->name, item->bval); + break; + case CFG_DTYPE_INT32: + uError("config %s in cluster value is:%d", item->name, item->i32); + break; + case CFG_DTYPE_INT64: + uError("config %s in cluster value is:%" PRId64, item->name, item->i64); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + uError("config %s in cluster value is:%f", item->name, item->fval); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + uError("config %s in cluster value is:%s", item->name, item->str); + break; + } + } + } +} + +bool isConifgItemLazyMode(SConfigItem *item) { + if (item->dynScope == CFG_DYN_CLIENT_LAZY || item->dynScope == CFG_DYN_SERVER_LAZY || + item->dynScope == CFG_DYN_BOTH_LAZY) { + return true; + } + return false; +} \ No newline at end of file diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index aa1804c1e9..144a1542cb 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -14,20 +14,23 @@ */ #define _DEFAULT_SOURCE -#include "tmisce.h" #include "tdatablock.h" #include "tglobal.h" #include "tjson.h" +#include "tmisce.h" int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) { pEp->port = 0; memset(pEp->fqdn, 0, TSDB_FQDN_LEN); - strncpy(pEp->fqdn, ep, TSDB_FQDN_LEN - 1); + tstrncpy(pEp->fqdn, ep, TSDB_FQDN_LEN); char* temp = strchr(pEp->fqdn, ':'); if (temp) { *temp = 0; - pEp->port = atoi(temp + 1); + pEp->port = taosStr2UInt16(temp + 1, NULL, 10); + if (pEp->port < 0) { + return TSDB_CODE_INVALID_PARA; + } } if (pEp->port == 0) { @@ -282,7 +285,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { locked = 1; while ((pItem = cfgNextIter(pIter)) != NULL) { -_start: + _start: col = startCol; // GRANT_CFG_SKIP; @@ -297,12 +300,11 @@ _start: TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, name, false), NULL, _exit); - char value[TSDB_CONFIG_PATH_LEN + VARSTR_HEADER_SIZE] = {0}; - int32_t valueLen = 0; - + char value[TSDB_CONFIG_PATH_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t valueLen = 0; SDiskCfg* pDiskCfg = NULL; if (strcasecmp(pItem->name, "dataDir") == 0 && exSize > 0) { - char* buf = &value[VARSTR_HEADER_SIZE]; + char* buf = &value[VARSTR_HEADER_SIZE]; pDiskCfg = taosArrayGet(pItem->array, index); valueLen = tsnprintf(buf, TSDB_CONFIG_PATH_LEN, "%s", pDiskCfg->dir); index++; @@ -331,11 +333,23 @@ _start: } TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, scope, false), NULL, _exit); + char category[TSDB_CONFIG_CATEGORY_LEN + VARSTR_HEADER_SIZE] = {0}; + TAOS_CHECK_GOTO(cfgDumpItemCategory(pItem, &category[VARSTR_HEADER_SIZE], TSDB_CONFIG_CATEGORY_LEN, &valueLen), + NULL, _exit); + varDataSetLen(category, valueLen); + + pColInfo = taosArrayGet(pBlock->pDataBlock, col++); + if (pColInfo == NULL) { + code = terrno; + TAOS_CHECK_GOTO(code, NULL, _exit); + } + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, category, false), NULL, _exit); + char info[TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE] = {0}; if (strcasecmp(pItem->name, "dataDir") == 0 && pDiskCfg) { char* buf = &info[VARSTR_HEADER_SIZE]; valueLen = tsnprintf(buf, TSDB_CONFIG_INFO_LEN, "level %d primary %d disabled %" PRIi8, pDiskCfg->level, - pDiskCfg->primary, pDiskCfg->disable); + pDiskCfg->primary, pDiskCfg->disable); } else { valueLen = 0; } @@ -352,7 +366,7 @@ _start: if (index > 0 && index <= exSize) { goto _start; } -} + } pBlock->info.rows = numOfRows; _exit: if (locked) cfgUnLock(pConf); diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 9ced37eb38..6bd64fb5e6 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -81,7 +81,7 @@ SName* tNameDup(const SName* name) { } int32_t tNameGetDbName(const SName* name, char* dst) { - strncpy(dst, name->dbname, tListLen(name->dbname)); + tstrncpy(dst, name->dbname, tListLen(name->dbname)); return 0; } diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index ebe246da4c..ddaf1d3c13 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -26,6 +26,7 @@ #endif #include "dmUtil.h" #include "tcs.h" +#include "qworker.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -279,7 +280,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { printf("ERROR: Encrypt key overflow, it should be at most %d characters\n", ENCRYPT_KEY_LEN); return TSDB_CODE_INVALID_CFG; } - tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN); + tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN + 1); } else { printf("'-y' requires a parameter\n"); return TSDB_CODE_INVALID_CFG; @@ -477,6 +478,13 @@ int mainWindows(int argc, char **argv) { taosCleanupArgs(); return code; } + + if ((code = taosMemoryPoolInit(qWorkerRetireJobs, qWorkerRetireJob)) != 0) { + dError("failed to init memPool, error:0x%x", code); + taosCloseLog(); + taosCleanupArgs(); + return code; + } if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL) { dError("failed to init conv"); diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index cbf1959e75..2108a097ee 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -28,6 +28,7 @@ typedef struct SDnodeMgmt { const char *path; const char *name; TdThread statusThread; + TdThread configThread; TdThread statusInfoThread; TdThread notifyThread; TdThread monitorThread; @@ -50,6 +51,7 @@ typedef struct SDnodeMgmt { // dmHandle.c SArray *dmGetMsgHandles(); void dmSendStatusReq(SDnodeMgmt *pMgmt); +void dmSendConfigReq(SDnodeMgmt *pMgmt); void dmUpdateStatusInfo(SDnodeMgmt *pMgmt); void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq); int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); @@ -64,6 +66,7 @@ int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); // dmWorker.c int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmStartStatusThread(SDnodeMgmt *pMgmt); +int32_t dmStartConfigThread(SDnodeMgmt *pMgmt); int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt); void dmStopStatusThread(SDnodeMgmt *pMgmt); void dmStopStatusInfoThread(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index f00222161a..ccc6439b5d 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -20,6 +20,7 @@ #include "systable.h" #include "tanalytics.h" #include "tchecksum.h" +#include "tutil.h" extern SConfig *tsCfg; @@ -114,7 +115,7 @@ static void dmMayShouldUpdateAnalFunc(SDnodeMgmt *pMgmt, int64_t newVer) { .pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_RETRIEVE_ANAL_ALGO, - .info.ahandle = (void *)0x9527, + .info.ahandle = 0, .info.refId = 0, .info.noResp = 0, .info.handle = 0, @@ -197,7 +198,8 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { req.clusterCfg.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold; tstrncpy(req.clusterCfg.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN); char timestr[32] = "1970-01-01 00:00:00.00"; - if (taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, NULL) != 0) { + if (taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, NULL) != + 0) { dError("failed to parse time since %s", tstrerror(code)); } memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); @@ -282,6 +284,135 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { dmProcessStatusRsp(pMgmt, &rpcRsp); } +static void dmProcessConfigRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { + const STraceId *trace = &pRsp->info.traceId; + int32_t code = 0; + SConfigRsp configRsp = {0}; + bool needStop = false; + + if (pRsp->code != 0) { + if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->pData->dropped && pMgmt->pData->dnodeId > 0) { + dGInfo("dnode:%d, set to dropped since not exist in mnode", pMgmt->pData->dnodeId); + pMgmt->pData->dropped = 1; + if (dmWriteEps(pMgmt->pData) != 0) { + dError("failed to write dnode file"); + } + dInfo("dnode will exit since it is in the dropped state"); + (void)raise(SIGINT); + } + } else { + bool needUpdate = false; + if (pRsp->pCont != NULL && pRsp->contLen > 0 && + tDeserializeSConfigRsp(pRsp->pCont, pRsp->contLen, &configRsp) == 0) { + // Try to use cfg file in current dnode. + if (configRsp.forceReadConfig) { + if (configRsp.isConifgVerified) { + uInfo("force read config and check config verified"); + code = taosPersistGlobalConfig(taosGetGlobalCfg(tsCfg), pMgmt->path, configRsp.cver); + if (code != TSDB_CODE_SUCCESS) { + dError("failed to persist global config since %s", tstrerror(code)); + goto _exit; + } + needUpdate = true; + } else { + // log the difference configurations + printConfigNotMatch(configRsp.array); + needStop = true; + goto _exit; + } + } + // Try to use cfg from mnode sdb. + if (!configRsp.isVersionVerified) { + uInfo("config version not verified, update config"); + needUpdate = true; + code = taosPersistGlobalConfig(configRsp.array, pMgmt->path, configRsp.cver); + if (code != TSDB_CODE_SUCCESS) { + dError("failed to persist global config since %s", tstrerror(code)); + goto _exit; + } + } + } + if (needUpdate) { + code = cfgUpdateFromArray(tsCfg, configRsp.array); + if (code != TSDB_CODE_SUCCESS) { + dError("failed to update config since %s", tstrerror(code)); + goto _exit; + } + code = setAllConfigs(tsCfg); + if (code != TSDB_CODE_SUCCESS) { + dError("failed to set all configs since %s", tstrerror(code)); + goto _exit; + } + } + code = taosPersistLocalConfig(pMgmt->path); + if (code != TSDB_CODE_SUCCESS) { + dError("failed to persist local config since %s", tstrerror(code)); + } + tsConfigInited = 1; + } +_exit: + tFreeSConfigRsp(&configRsp); + rpcFreeCont(pRsp->pCont); + if (needStop) { + dmStop(); + } +} + +void dmSendConfigReq(SDnodeMgmt *pMgmt) { + int32_t code = 0; + SConfigReq req = {0}; + + req.cver = tsdmConfigVersion; + req.forceReadConfig = tsForceReadConfig; + req.array = taosGetGlobalCfg(tsCfg); + dDebug("send config req to mnode, configVersion:%d", req.cver); + + int32_t contLen = tSerializeSConfigReq(NULL, 0, &req); + if (contLen < 0) { + dError("failed to serialize status req since %s", tstrerror(contLen)); + return; + } + + void *pHead = rpcMallocCont(contLen); + if (pHead == NULL) { + dError("failed to malloc cont since %s", tstrerror(contLen)); + return; + } + contLen = tSerializeSConfigReq(pHead, contLen, &req); + if (contLen < 0) { + rpcFreeCont(pHead); + dError("failed to serialize status req since %s", tstrerror(contLen)); + return; + } + + SRpcMsg rpcMsg = {.pCont = pHead, + .contLen = contLen, + .msgType = TDMT_MND_CONFIG, + .info.ahandle = 0, + .info.notFreeAhandle = 1, + .info.refId = 0, + .info.noResp = 0, + .info.handle = 0}; + SRpcMsg rpcRsp = {0}; + + SEpSet epSet = {0}; + int8_t epUpdated = 0; + (void)dmGetMnodeEpSet(pMgmt->pData, &epSet); + + dDebug("send status req to mnode, statusSeq:%d, begin to send rpc msg", pMgmt->statusSeq); + code = + rpcSendRecvWithTimeout(pMgmt->msgCb.statusRpc, &epSet, &rpcMsg, &rpcRsp, &epUpdated, tsStatusInterval * 5 * 1000); + if (code != 0) { + dError("failed to send status req since %s", tstrerror(code)); + return; + } + if (rpcRsp.code != 0) { + dError("failed to send config req since %s", tstrerror(rpcRsp.code)); + return; + } + dmProcessConfigRsp(pMgmt, &rpcRsp); +} + void dmUpdateStatusInfo(SDnodeMgmt *pMgmt) { SMonVloadInfo vinfo = {0}; dDebug("begin to get vnode loads"); @@ -344,6 +475,27 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return 0; } +extern void tsdbAlterMaxCompactTasks(); +static int32_t dmAlterMaxCompactTask(const char *value) { + int32_t max_compact_tasks; + char *endptr = NULL; + + max_compact_tasks = taosStr2Int32(value, &endptr, 10); + if (endptr == value || endptr[0] != '\0') { + return TSDB_CODE_INVALID_MSG; + } + + if (max_compact_tasks != tsNumOfCompactThreads) { + dInfo("alter max compact tasks from %d to %d", tsNumOfCompactThreads, max_compact_tasks); + tsNumOfCompactThreads = max_compact_tasks; +#ifdef TD_ENTERPRISE + tsdbAlterMaxCompactTasks(); +#endif + } + + return TSDB_CODE_SUCCESS; +} + int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t code = 0; SDCfgDnodeReq cfgReq = {0}; @@ -351,20 +503,46 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return TSDB_CODE_INVALID_MSG; } + if (strncmp(cfgReq.config, tsAlterCompactTaskKeywords, strlen(tsAlterCompactTaskKeywords) + 1) == 0) { + return dmAlterMaxCompactTask(cfgReq.value); + } + dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value); - SConfig *pCfg = taosGetCfg(); + SConfig *pCfg = taosGetCfg(); + SConfigItem *pItem = NULL; - code = cfgSetItem(pCfg, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_SERVER_CMD, true); + code = cfgGetAndSetItem(pCfg, &pItem, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_SERVER_CMD, true); if (code != 0) { if (strncasecmp(cfgReq.config, "resetlog", strlen("resetlog")) == 0) { - code = 0; + TAOS_CHECK_RETURN(taosCfgDynamicOptions(pCfg, cfgReq.config, true)); + return TSDB_CODE_SUCCESS; } else { return code; } } + if (pItem == NULL) { + return TSDB_CODE_CFG_NOT_FOUND; + } + if (!isConifgItemLazyMode(pItem)) { + TAOS_CHECK_RETURN(taosCfgDynamicOptions(pCfg, cfgReq.config, true)); + } - return taosCfgDynamicOptions(pCfg, cfgReq.config, true); + if (pItem->category == CFG_CATEGORY_GLOBAL) { + code = taosPersistGlobalConfig(taosGetGlobalCfg(pCfg), pMgmt->path, tsdmConfigVersion); + if (code != TSDB_CODE_SUCCESS) { + dError("failed to persist global config since %s", tstrerror(code)); + } + } else { + code = taosPersistLocalConfig(pMgmt->path); + if (code != TSDB_CODE_SUCCESS) { + dError("failed to persist local config since %s", tstrerror(code)); + } + } + if (cfgReq.version > 0) { + tsdmConfigVersion = cfgReq.version; + } + return code; } int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index fb7d891c67..b58c1a216d 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -23,6 +23,10 @@ static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) { if ((code = dmStartStatusThread(pMgmt)) != 0) { return code; } + + if ((code = dmStartConfigThread(pMgmt)) != 0) { + return code; + } if ((code = dmStartStatusInfoThread(pMgmt)) != 0) { return code; } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 7f802f3837..e55a78075a 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -47,6 +47,25 @@ static void *dmStatusThreadFp(void *param) { return NULL; } +static void *dmConfigThreadFp(void *param) { + SDnodeMgmt *pMgmt = param; + int64_t lastTime = taosGetTimestampMs(); + setThreadName("dnode-config"); + while (1) { + taosMsleep(200); + if (pMgmt->pData->dropped || pMgmt->pData->stopped || tsConfigInited) break; + + int64_t curTime = taosGetTimestampMs(); + if (curTime < lastTime) lastTime = curTime; + float interval = (curTime - lastTime) / 1000.0f; + if (interval >= tsStatusInterval) { + dmSendConfigReq(pMgmt); + lastTime = curTime; + } + } + return NULL; +} + static void *dmStatusInfoThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); @@ -202,9 +221,12 @@ static void *dmMonitorThreadFp(void *param) { trimCount = (trimCount + 1) % TRIM_FREQ; if (trimCount == 0) { - taosMemoryTrim(0); + taosMemoryTrim(0, NULL); } } + if (atomic_val_compare_exchange_8(&tsNeedTrim, 1, 0)) { + taosMemoryTrim(0, NULL); + } } return NULL; @@ -309,6 +331,22 @@ int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) { return 0; } +int32_t dmStartConfigThread(SDnodeMgmt *pMgmt) { + int32_t code = 0; + TdThreadAttr thAttr; + (void)taosThreadAttrInit(&thAttr); + (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_DETACHED); + if (taosThreadCreate(&pMgmt->configThread, &thAttr, dmConfigThreadFp, pMgmt) != 0) { + code = TAOS_SYSTEM_ERROR(errno); + dError("failed to create config thread since %s", tstrerror(code)); + return code; + } + + (void)taosThreadAttrDestroy(&thAttr); + tmsgReportStartup("config-status", "initialized"); + return 0; +} + int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt) { int32_t code = 0; TdThreadAttr thAttr; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 0d804eadf0..8e595f76c9 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -200,6 +200,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_NOTIFY, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index e8e7a75889..cfdc09f29e 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -220,6 +220,8 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { return code; } + tsNumOfQueryThreads += tsNumOfMnodeQueryThreads; + SSingleWorkerCfg fCfg = { .min = tsNumOfMnodeFetchThreads, .max = tsNumOfMnodeFetchThreads, diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index 65c2bb9bf3..ab5b70079a 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -104,8 +104,8 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) { int32_t code = 0; SSingleWorkerCfg queryCfg = { - .min = tsNumOfVnodeQueryThreads, - .max = tsNumOfVnodeQueryThreads, + .min = tsNumOfQnodeQueryThreads, + .max = tsNumOfQnodeQueryThreads, .name = "qnode-query", .fp = (FItem)qmProcessQueue, .param = pMgmt, @@ -117,6 +117,8 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) { return code; } + tsNumOfQueryThreads += tsNumOfQnodeQueryThreads; + SSingleWorkerCfg fetchCfg = { .min = tsNumOfQnodeFetchThreads, .max = tsNumOfQnodeFetchThreads, diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index fd30555c3b..83043b4393 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -186,7 +186,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { #if defined(TD_ENTERPRISE) pCfg->tsdbCfg.encryptAlgorithm = pCreate->encryptAlgorithm; if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) { - strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN + 1); } #else pCfg->tsdbCfg.encryptAlgorithm = 0; @@ -202,7 +202,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { #if defined(TD_ENTERPRISE) pCfg->walCfg.encryptAlgorithm = pCreate->encryptAlgorithm; if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) { - strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN + 1); } #else pCfg->walCfg.encryptAlgorithm = 0; @@ -211,7 +211,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { #if defined(TD_ENTERPRISE) pCfg->tdbEncryptAlgorithm = pCreate->encryptAlgorithm; if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) { - strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); } #else pCfg->tdbEncryptAlgorithm = 0; @@ -378,12 +378,11 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId); - if (vnodeCreate(path, &vnodeCfg, diskPrimary, pMgmt->pTfs) < 0) { - dError("vgId:%d, failed to create vnode since %s", req.vgId, terrstr()); + if ((code = vnodeCreate(path, &vnodeCfg, diskPrimary, pMgmt->pTfs)) < 0) { + dError("vgId:%d, failed to create vnode since %s", req.vgId, tstrerror(code)); vmReleaseVnode(pMgmt, pVnode); vmCleanPrimaryDisk(pMgmt, req.vgId); (void)tFreeSCreateVnodeReq(&req); - code = terrno != 0 ? terrno : -1; return code; } @@ -884,7 +883,7 @@ int32_t vmProcessArbHeartBeatReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { size_t size = taosArrayGetSize(arbHbReq.hbMembers); arbHbRsp.dnodeId = pMgmt->pData->dnodeId; - strncpy(arbHbRsp.arbToken, arbHbReq.arbToken, TSDB_ARB_TOKEN_SIZE); + tstrncpy(arbHbRsp.arbToken, arbHbReq.arbToken, TSDB_ARB_TOKEN_SIZE); arbHbRsp.hbMembers = taosArrayInit(size, sizeof(SVArbHbRspMember)); if (arbHbRsp.hbMembers == NULL) { goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 6a61cc3859..537c1f6297 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -94,7 +94,7 @@ static void vmUnRegisterCreatingState(SVnodeMgmt *pMgmt, int32_t vgId) { dTrace("vgId:%d, remove from creating Hash", vgId); r = taosHashRemove(pMgmt->creatingHash, &vgId, sizeof(int32_t)); if (r != 0) { - dError("vgId:%d, failed to remove vnode from hash", vgId); + dError("vgId:%d, failed to remove vnode from creatingHash", vgId); } (void)taosThreadRwlockUnlock(&pMgmt->lock); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 9c436a3dfa..c22adec9b4 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -456,6 +456,8 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { pQPool->max = tsNumOfVnodeQueryThreads; if ((code = tQueryAutoQWorkerInit(pQPool)) != 0) return code; + tsNumOfQueryThreads += tsNumOfVnodeQueryThreads; + SAutoQWorkerPool *pStreamPool = &pMgmt->streamPool; pStreamPool->name = "vnode-stream"; pStreamPool->ratio = tsRatioOfVnodeStreamThreads; diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 6d4ebe424a..1708a0c8d9 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -220,6 +220,7 @@ void dmCleanup() { dInfo("dnode env is cleaned up"); + taosMemPoolClose(gMemPoolHandle); taosCleanupCfg(); taosCloseLog(); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index f11e1508a8..85d891c8d8 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -180,7 +180,7 @@ int32_t dmInitVars(SDnode *pDnode) { //code = taosGetCryptKey(tsAuthCode, pData->machineId, tsCryptKey); code = 0; - strncpy(tsEncryptKey, tsAuthCode, 16); + tstrncpy(tsEncryptKey, tsAuthCode, 16); if (code != 0) { if(code == -1){ @@ -221,6 +221,7 @@ int32_t dmInitVars(SDnode *pDnode) { } extern SMonVloadInfo tsVinfo; + void dmClearVars(SDnode *pDnode) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 5f396a520a..acd95d4b43 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -368,7 +368,9 @@ static inline int32_t dmSendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { (void)rpcRegisterBrokenLinkArg(pMsg); } -static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { (void)rpcReleaseHandle(pHandle, type); } +static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type, int32_t status) { + (void)rpcReleaseHandle(pHandle, type, status); +} static bool rpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_MNODE_NOT_FOUND || diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 1da13f72cd..d5006e1e4d 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -230,7 +230,7 @@ static int32_t dmWriteCheckCodeFile(char *file, char *realfile, char *key, bool } SCryptOpts opts; - strncpy(opts.key, key, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, key, ENCRYPT_KEY_LEN + 1); opts.len = len; opts.source = DM_KEY_INDICATOR; opts.result = result; @@ -349,7 +349,7 @@ static int32_t dmCompareEncryptKey(char *file, char *key, bool toLogFile) { } SCryptOpts opts = {0}; - strncpy(opts.key, key, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, key, ENCRYPT_KEY_LEN + 1); opts.len = len; opts.source = content; opts.result = result; @@ -551,7 +551,7 @@ int32_t dmGetEncryptKey() { goto _OVER; } - strncpy(tsEncryptKey, encryptKey, ENCRYPT_KEY_LEN); + tstrncpy(tsEncryptKey, encryptKey, ENCRYPT_KEY_LEN + 1); taosMemoryFreeClear(encryptKey); tsEncryptionKeyChksum = taosCalcChecksum(0, tsEncryptKey, strlen(tsEncryptKey)); tsEncryptionKeyStat = ENCRYPT_KEY_STAT_LOADED; diff --git a/source/dnode/mgmt/test/sut/src/server.cpp b/source/dnode/mgmt/test/sut/src/server.cpp index 2218504df4..e7dc45a0f8 100644 --- a/source/dnode/mgmt/test/sut/src/server.cpp +++ b/source/dnode/mgmt/test/sut/src/server.cpp @@ -17,6 +17,7 @@ void* serverLoop(void* param) { TestServer* server = (TestServer*)param; + cfgInit(&tsCfg); if (dmInit() != 0) { return NULL; diff --git a/source/dnode/mnode/impl/inc/mndCompactDetail.h b/source/dnode/mnode/impl/inc/mndCompactDetail.h index 601af3b64b..e99923e226 100644 --- a/source/dnode/mnode/impl/inc/mndCompactDetail.h +++ b/source/dnode/mnode/impl/inc/mndCompactDetail.h @@ -25,19 +25,17 @@ extern "C" { int32_t mndInitCompactDetail(SMnode *pMnode); void mndCleanupCompactDetail(SMnode *pMnode); -void tFreeCompactDetailObj(SCompactDetailObj *pCompact); -int32_t tSerializeSCompactDetailObj(void *buf, int32_t bufLen, const SCompactDetailObj *pObj); -int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailObj *pObj); +void tFreeCompactDetailObj(SCompactDetailObj *pCompact); -SSdbRaw* mndCompactDetailActionEncode(SCompactDetailObj *pCompact); -SSdbRow* mndCompactDetailActionDecode(SSdbRaw *pRaw); +SSdbRaw *mndCompactDetailActionEncode(SCompactDetailObj *pCompact); +SSdbRow *mndCompactDetailActionDecode(SSdbRaw *pRaw); int32_t mndCompactDetailActionInsert(SSdb *pSdb, SCompactDetailObj *pCompact); int32_t mndCompactDetailActionDelete(SSdb *pSdb, SCompactDetailObj *pCompact); int32_t mndCompactDetailActionUpdate(SSdb *pSdb, SCompactDetailObj *pOldCompact, SCompactDetailObj *pNewCompact); -int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SVgObj *pVgroup, - SVnodeGid *pVgid, int32_t index); +int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj *pCompact, SVgObj *pVgroup, + SVnodeGid *pVgid, int32_t index); int32_t mndRetrieveCompactDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); diff --git a/include/util/tmempool.h b/source/dnode/mnode/impl/inc/mndConfig.h similarity index 53% rename from include/util/tmempool.h rename to source/dnode/mnode/impl/inc/mndConfig.h index 7a5aca7b34..bbfa3f4a65 100644 --- a/include/util/tmempool.h +++ b/source/dnode/mnode/impl/inc/mndConfig.h @@ -12,24 +12,28 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef _TD_UTIL_MEMPOOL_H_ -#define _TD_UTIL_MEMPOOL_H_ -#include "os.h" +#ifndef _TD_MND_CONFIG_H_ +#define _TD_MND_CONFIG_H_ + +#include "mndInt.h" #ifdef __cplusplus extern "C" { #endif +int32_t mndInitConfig(SMnode *pMnode); -typedef void *mpool_h; - -mpool_h taosMemPoolInit(int32_t maxNum, int32_t blockSize); -char *taosMemPoolMalloc(mpool_h handle); -void taosMemPoolFree(mpool_h handle, char *p); -void taosMemPoolCleanUp(mpool_h handle); +SSdbRaw *mnCfgActionEncode(SConfigObj *pCfg); +SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw); +static int32_t mndCfgActionInsert(SSdb *pSdb, SConfigObj *obj); +static int32_t mndCfgActionDelete(SSdb *pSdb, SConfigObj *obj); +static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigObj *oldItem, SConfigObj *newObj); +static int32_t mndCfgActionDeploy(SMnode *pMnode); +static int32_t mndCfgActionPrepare(SMnode *pMnode); +static int32_t mndProcessConfigReq(SRpcMsg *pReq); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_MEMPOOL_H_*/ +#endif /*_TD_MND_ARBGROUP_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 65708628b3..e3d2ad6d34 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -106,8 +106,8 @@ typedef enum { TRN_CONFLICT_GLOBAL = 1, TRN_CONFLICT_DB = 2, TRN_CONFLICT_DB_INSIDE = 3, -// TRN_CONFLICT_TOPIC = 4, -// TRN_CONFLICT_TOPIC_INSIDE = 5, + // TRN_CONFLICT_TOPIC = 4, + // TRN_CONFLICT_TOPIC_INSIDE = 5, TRN_CONFLICT_ARBGROUP = 6, TRN_CONFLICT_TSMA = 7, } ETrnConflct; @@ -316,6 +316,25 @@ typedef struct { TdThreadMutex mutex; } SArbGroup; +typedef struct { + char name[CFG_NAME_MAX_LEN]; + ECfgDataType dtype; + union { + bool bval; + float fval; + int32_t i32; + int64_t i64; + char* str; + }; +} SConfigObj; + +int32_t tEncodeSConfigObj(SEncoder* pEncoder, const SConfigObj* pObj); +int32_t tDecodeSConfigObj(SDecoder* pDecoder, SConfigObj* pObj); +SConfigObj* mndInitConfigObj(SConfigItem* pItem); +SConfigObj* mndInitConfigVersion(); +int32_t mndUpdateObj(SConfigObj* pObj, const char* name, char* value); +void tFreeSConfigObj(SConfigObj* obj); + typedef struct { int32_t maxUsers; int32_t maxDbs; @@ -423,6 +442,10 @@ typedef struct { int8_t s3Compact; int8_t withArbitrator; int8_t encryptAlgorithm; + int8_t compactTimeOffset; // hour + int32_t compactInterval; // minute + int32_t compactStartTime; // minute + int32_t compactEndTime; // minute } SDbCfg; typedef struct { @@ -489,8 +512,8 @@ typedef struct { int64_t dstTbUid; int8_t intervalUnit; int8_t slidingUnit; - int8_t timezone; // int8_t is not enough, timezone is unit of second - int32_t dstVgId; // for stream + int8_t timezone; // int8_t is not enough, timezone is unit of second + int32_t dstVgId; // for stream int64_t interval; int64_t offset; int64_t sliding; @@ -649,12 +672,12 @@ typedef struct { int32_t maxPollIntervalMs; } SMqConsumerObj; -int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, - char *topic, SCMSubscribeReq *subscribe, SMqConsumerObj** ppConsumer); -void tClearSMqConsumerObj(SMqConsumerObj* pConsumer); -void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer); -int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer); -void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer, int8_t sver); +int32_t tNewSMqConsumerObj(int64_t consumerId, char* cgroup, int8_t updateType, char* topic, SCMSubscribeReq* subscribe, + SMqConsumerObj** ppConsumer); +void tClearSMqConsumerObj(SMqConsumerObj* pConsumer); +void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer); +int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer); +void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer, int8_t sver); typedef struct { int32_t vgId; @@ -693,11 +716,11 @@ typedef struct { char* qmsg; // SubPlanToString } SMqSubscribeObj; -int32_t tNewSubscribeObj(const char *key, SMqSubscribeObj **ppSub); -int32_t tCloneSubscribeObj(const SMqSubscribeObj* pSub, SMqSubscribeObj **ppSub); -void tDeleteSubscribeObj(SMqSubscribeObj* pSub); -int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub); -void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub, int8_t sver); +int32_t tNewSubscribeObj(const char* key, SMqSubscribeObj** ppSub); +int32_t tCloneSubscribeObj(const SMqSubscribeObj* pSub, SMqSubscribeObj** ppSub); +void tDeleteSubscribeObj(SMqSubscribeObj* pSub); +int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub); +void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub, int8_t sver); // typedef struct { // int32_t epoch; @@ -846,6 +869,8 @@ typedef struct { int64_t startTime; int32_t newNumberFileset; int32_t newFinished; + int32_t progress; + int64_t remainingTime; } SCompactDetailObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c index 0192044e67..3879e8e6e2 100644 --- a/source/dnode/mnode/impl/src/mndArbGroup.c +++ b/source/dnode/mnode/impl/src/mndArbGroup.c @@ -1315,7 +1315,7 @@ static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock continue; } char dbNameInGroup[TSDB_DB_FNAME_LEN]; - strncpy(dbNameInGroup, pVgObj->dbName, TSDB_DB_FNAME_LEN); + tstrncpy(dbNameInGroup, pVgObj->dbName, TSDB_DB_FNAME_LEN); sdbRelease(pSdb, pVgObj); char dbname[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 3779bea564..50bba755c9 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -457,7 +457,7 @@ int32_t mndProcessConfigClusterReq(SRpcMsg *pReq) { _exit: tFreeSMCfgClusterReq(&cfgReq); if (code != 0) { - mError("cluster: failed to config:%s %s since %s", cfgReq.config, cfgReq.value, terrstr()); + mError("cluster: failed to config:%s %s since %s", cfgReq.config, cfgReq.value, tstrerror(code)); } else { mInfo("cluster: success to config:%s %s", cfgReq.config, cfgReq.value); } diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index 106680da7f..402c4321a1 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -244,7 +244,7 @@ int32_t mndCompactGetDbName(SMnode *pMnode, int32_t compactId, char *dbname, int TAOS_RETURN(code); } - (void)strncpy(dbname, pCompact->dbname, len); + tstrncpy(dbname, pCompact->dbname, len); mndReleaseCompact(pMnode, pCompact); TAOS_RETURN(code); } @@ -321,7 +321,7 @@ int32_t mndRetrieveCompact(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, TAOS_CHECK_GOTO(tNameFromString(&name, pCompact->dbname, T_NAME_ACCT | T_NAME_DB), &lino, _OVER); (void)tNameGetDbName(&name, varDataVal(tmpBuf)); } else { - (void)strncpy(varDataVal(tmpBuf), pCompact->dbname, TSDB_SHOW_SQL_LEN); + tstrncpy(varDataVal(tmpBuf), pCompact->dbname, TSDB_SHOW_SQL_LEN); } varDataSetLen(tmpBuf, strlen(varDataVal(tmpBuf))); RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)tmpBuf, false), pCompact, &lino, _OVER); @@ -516,11 +516,14 @@ int32_t mndProcessKillCompactReq(SRpcMsg *pReq) { code = TSDB_CODE_ACTION_IN_PROGRESS; - char obj[TSDB_INT32_ID_LEN] = {0}; - (void)sprintf(obj, "%d", pCompact->compactId); - - auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen); - + char obj[TSDB_INT32_ID_LEN] = {0}; + int32_t nBytes = snprintf(obj, sizeof(obj), "%d", pCompact->compactId); + if ((uint32_t)nBytes < sizeof(obj)) { + auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, + killCompactReq.sqlLen); + } else { + mError("compact:%" PRId32 " failed to audit since %s", pCompact->compactId, tstrerror(TSDB_CODE_OUT_OF_RANGE)); + } _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("failed to kill compact %" PRId32 " since %s", killCompactReq.compactId, terrstr()); @@ -546,6 +549,8 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c if (pDetail->compactId == compactId && pDetail->vgId == rsp->vgId && pDetail->dnodeId == rsp->dnodeId) { pDetail->newNumberFileset = rsp->numberFileset; pDetail->newFinished = rsp->finished; + pDetail->progress = rsp->progress; + pDetail->remainingTime = rsp->remainingTime; sdbCancelFetch(pMnode->pSdb, pIter); sdbRelease(pMnode->pSdb, pDetail); @@ -641,7 +646,7 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) { char detail[1024] = {0}; int32_t len = tsnprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d", - TMSG_INFO(TDMT_VND_QUERY_COMPACT_PROGRESS), epSet.numOfEps, epSet.inUse); + TMSG_INFO(TDMT_VND_QUERY_COMPACT_PROGRESS), epSet.numOfEps, epSet.inUse); for (int32_t i = 0; i < epSet.numOfEps; ++i) { len += tsnprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); } @@ -856,7 +861,7 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { return 0; } -void mndCompactPullup(SMnode *pMnode) { +static void mndCompactPullup(SMnode *pMnode) { int32_t code = 0; SSdb *pSdb = pMnode->pSdb; SArray *pArray = taosArrayInit(sdbGetSize(pSdb, SDB_COMPACT), sizeof(int32_t)); @@ -888,9 +893,115 @@ void mndCompactPullup(SMnode *pMnode) { } taosArrayDestroy(pArray); } +#ifdef TD_ENTERPRISE +static int32_t mndCompactDispatchAudit(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, STimeWindow *tw) { + if (!tsEnableAudit || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) { + return 0; + } + + SName name = {0}; + int32_t sqlLen = 0; + char sql[256] = {0}; + char skeyStr[40] = {0}; + char ekeyStr[40] = {0}; + char *pDbName = pDb->name; + + if (tNameFromString(&name, pDb->name, T_NAME_ACCT | T_NAME_DB) == 0) { + pDbName = name.dbname; + } + + if (taosFormatUtcTime(skeyStr, sizeof(skeyStr), tw->skey, pDb->cfg.precision) == 0 && + taosFormatUtcTime(ekeyStr, sizeof(ekeyStr), tw->ekey, pDb->cfg.precision) == 0) { + sqlLen = tsnprintf(sql, sizeof(sql), "compact db %s start with '%s' end with '%s'", pDbName, skeyStr, ekeyStr); + } else { + sqlLen = tsnprintf(sql, sizeof(sql), "compact db %s start with %" PRIi64 " end with %" PRIi64, pDbName, tw->skey, + tw->ekey); + } + auditRecord(NULL, pMnode->clusterId, "autoCompactDB", name.dbname, "", sql, sqlLen); -static int32_t mndProcessCompactTimer(SRpcMsg *pReq) { - mTrace("start to process compact timer"); - mndCompactPullup(pReq->info.node); + return 0; +} + +extern int32_t mndCompactDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, STimeWindow tw, SArray *vgroupIds); +static int32_t mndCompactDispatch(SRpcMsg *pReq) { + int32_t code = 0; + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + int64_t curMs = taosGetTimestampMs(); + int64_t curMin = curMs / 60000LL; + + void *pIter = NULL; + SDbObj *pDb = NULL; + while ((pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb))) { + if (pDb->cfg.compactInterval <= 0) { + mDebug("db:%p,%s, compact interval is %dm, skip", pDb, pDb->name, pDb->cfg.compactInterval); + sdbRelease(pSdb, pDb); + continue; + } + + // daysToKeep2 would be altered + if (pDb->cfg.compactEndTime && (pDb->cfg.compactEndTime <= -pDb->cfg.daysToKeep2)) { + mWarn("db:%p,%s, compact end time:%dm <= -keep2:%dm , skip", pDb, pDb->name, pDb->cfg.compactEndTime, + -pDb->cfg.daysToKeep2); + sdbRelease(pSdb, pDb); + continue; + } + + int64_t compactStartTime = pDb->cfg.compactStartTime ? pDb->cfg.compactStartTime : -pDb->cfg.daysToKeep2; + int64_t compactEndTime = pDb->cfg.compactEndTime ? pDb->cfg.compactEndTime : -pDb->cfg.daysPerFile; + + if (compactStartTime >= compactEndTime) { + mDebug("db:%p,%s, compact start time:%" PRIi64 "m >= end time:%" PRIi64 "m, skip", pDb, pDb->name, + compactStartTime, compactEndTime); + sdbRelease(pSdb, pDb); + continue; + } + + int64_t remainder = ((curMin - (int64_t)pDb->cfg.compactTimeOffset * 60LL) % pDb->cfg.compactInterval); + if (remainder != 0) { + mDebug("db:%p,%s, current time:%" PRIi64 "m is not divisible by compact interval:%dm, offset:%" PRIi8 + "h, remainder:%" PRIi64 "m, skip", + pDb, pDb->name, curMin, pDb->cfg.compactInterval, pDb->cfg.compactTimeOffset, remainder); + sdbRelease(pSdb, pDb); + continue; + } + + if ((pDb->compactStartTime / 60000LL) == curMin) { + mDebug("db:%p:%s, compact has already been dispatched at %" PRIi64 "m(%" PRIi64 "ms), skip", pDb, pDb->name, + curMin, pDb->compactStartTime); + sdbRelease(pSdb, pDb); + continue; + } + + STimeWindow tw = { + .skey = convertTimePrecision(curMs + compactStartTime * 60000LL, TSDB_TIME_PRECISION_MILLI, pDb->cfg.precision), + .ekey = convertTimePrecision(curMs + compactEndTime * 60000LL, TSDB_TIME_PRECISION_MILLI, pDb->cfg.precision)}; + + if ((code = mndCompactDb(pMnode, NULL, pDb, tw, NULL)) == 0) { + mInfo("db:%p,%s, succeed to dispatch compact with range:[%" PRIi64 ",%" PRIi64 "], interval:%dm, start:%" PRIi64 + "m, end:%" PRIi64 "m, offset:%" PRIi8 "h", + pDb, pDb->name, tw.skey, tw.ekey, pDb->cfg.compactInterval, compactStartTime, compactEndTime, + pDb->cfg.compactTimeOffset); + } else { + mWarn("db:%p,%s, failed to dispatch compact with range:[%" PRIi64 ",%" PRIi64 "], interval:%dm, start:%" PRIi64 + "m, end:%" PRIi64 "m, offset:%" PRIi8 "h, since %s", + pDb, pDb->name, tw.skey, tw.ekey, pDb->cfg.compactInterval, compactStartTime, compactEndTime, + pDb->cfg.compactTimeOffset, tstrerror(code)); + } + + TAOS_UNUSED(mndCompactDispatchAudit(pMnode, pReq, pDb, &tw)); + + sdbRelease(pSdb, pDb); + } + return 0; +} +#endif + +static int32_t mndProcessCompactTimer(SRpcMsg *pReq) { +#ifdef TD_ENTERPRISE + mTrace("start to process compact timer"); + mndCompactPullup(pReq->info.node); + TAOS_UNUSED(mndCompactDispatch(pReq)); +#endif return 0; } diff --git a/source/dnode/mnode/impl/src/mndCompactDetail.c b/source/dnode/mnode/impl/src/mndCompactDetail.c index cbd0df7e68..9a053066b2 100644 --- a/source/dnode/mnode/impl/src/mndCompactDetail.c +++ b/source/dnode/mnode/impl/src/mndCompactDetail.c @@ -90,6 +90,14 @@ int32_t mndRetrieveCompactDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB TAOS_CHECK_RETURN_WITH_RELEASE(colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->startTime, false), pSdb, pCompactDetail); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + TAOS_CHECK_RETURN_WITH_RELEASE(colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->progress, false), + pSdb, pCompactDetail); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + TAOS_CHECK_RETURN_WITH_RELEASE( + colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->remainingTime, false), pSdb, pCompactDetail); + numOfRows++; sdbRelease(pSdb, pCompactDetail); } @@ -101,7 +109,7 @@ int32_t mndRetrieveCompactDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB void tFreeCompactDetailObj(SCompactDetailObj *pCompact) {} -int32_t tSerializeSCompactDetailObj(void *buf, int32_t bufLen, const SCompactDetailObj *pObj) { +static int32_t tSerializeSCompactDetailObj(void *buf, int32_t bufLen, const SCompactDetailObj *pObj) { SEncoder encoder = {0}; int32_t code = 0; int32_t lino; @@ -118,6 +126,9 @@ int32_t tSerializeSCompactDetailObj(void *buf, int32_t bufLen, const SCompactDet TAOS_CHECK_EXIT(tEncodeI64(&encoder, pObj->startTime)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pObj->newNumberFileset)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pObj->newFinished)); + // 1. add progress and remaining time + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pObj->progress)); + TAOS_CHECK_EXIT(tEncodeI64v(&encoder, pObj->remainingTime)); tEndEncode(&encoder); @@ -131,7 +142,7 @@ _exit: return tlen; } -int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailObj *pObj) { +static int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailObj *pObj) { int32_t code = 0; int32_t lino; SDecoder decoder = {0}; @@ -147,6 +158,14 @@ int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailO TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pObj->startTime)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pObj->newNumberFileset)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pObj->newFinished)); + // 1. add progress and remaining time decode + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pObj->progress)); + TAOS_CHECK_EXIT(tDecodeI64v(&decoder, &pObj->remainingTime)); + } else { + pObj->progress = 0; + pObj->remainingTime = 0; + } tEndDecode(&decoder); diff --git a/source/dnode/mnode/impl/src/mndConfig.c b/source/dnode/mnode/impl/src/mndConfig.c new file mode 100644 index 0000000000..06555d47a6 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndConfig.c @@ -0,0 +1,893 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "audit.h" +#include "mndConfig.h" +#include "mndDnode.h" +#include "mndPrivilege.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "tutil.h" + +#define CFG_VER_NUMBER 1 +#define CFG_RESERVE_SIZE 63 +#define CFG_ALTER_TIMEOUT 3 * 1000 + +static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue); +static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq); +static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq); +static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); +static int32_t mndProcessConfigReq(SRpcMsg *pReq); +static int32_t mndInitWriteCfg(SMnode *pMnode); +static int32_t mndTryRebuildCfg(SMnode *pMnode); +static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array); +static void cfgArrayCleanUp(SArray *array); +static void cfgObjArrayCleanUp(SArray *array); + +static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue, ECfgDataType dtype, + int32_t tsmmConfigVersion); + +int32_t mndSetCreateConfigCommitLogs(STrans *pTrans, SConfigObj *obj); + +int32_t mndInitConfig(SMnode *pMnode) { + int32_t code = 0; + SSdbTable table = {.sdbType = SDB_CFG, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mnCfgActionEncode, + .decodeFp = (SdbDecodeFp)mndCfgActionDecode, + .insertFp = (SdbInsertFp)mndCfgActionInsert, + .updateFp = (SdbUpdateFp)mndCfgActionUpdate, + .deleteFp = (SdbDeleteFp)mndCfgActionDelete, + .deployFp = (SdbDeployFp)mndCfgActionDeploy, + .prepareFp = (SdbPrepareFp)mndCfgActionPrepare}; + + mndSetMsgHandle(pMnode, TDMT_MND_CONFIG, mndProcessConfigReq); + mndSetMsgHandle(pMnode, TDMT_MND_CONFIG_DNODE, mndProcessConfigDnodeReq); + mndSetMsgHandle(pMnode, TDMT_DND_CONFIG_DNODE_RSP, mndProcessConfigDnodeRsp); + mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq); + + return sdbSetTable(pMnode->pSdb, table); +} + +SSdbRaw *mnCfgActionEncode(SConfigObj *obj) { + int32_t code = 0; + int32_t lino = 0; + void *buf = NULL; + SSdbRaw *pRaw = NULL; + + SEncoder encoder; + tEncoderInit(&encoder, NULL, 0); + if ((code = tEncodeSConfigObj(&encoder, obj)) < 0) { + tEncoderClear(&encoder); + TSDB_CHECK_CODE(code, lino, _over); + } + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + + int32_t size = sizeof(int32_t) + tlen; + pRaw = sdbAllocRaw(SDB_CFG, CFG_VER_NUMBER, size); + TSDB_CHECK_NULL(pRaw, code, lino, _over, terrno); + + buf = taosMemoryMalloc(tlen); + TSDB_CHECK_NULL(buf, code, lino, _over, terrno); + + tEncoderInit(&encoder, buf, tlen); + if ((code = tEncodeSConfigObj(&encoder, obj)) < 0) { + tEncoderClear(&encoder); + TSDB_CHECK_CODE(code, lino, _over); + } + + tEncoderClear(&encoder); + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, tlen, _over); + SDB_SET_BINARY(pRaw, dataPos, buf, tlen, _over); + SDB_SET_DATALEN(pRaw, dataPos, _over); + +_over: + taosMemoryFreeClear(buf); + if (code != TSDB_CODE_SUCCESS) { + mError("cfg:%s, failed to encode to raw:%p at line:%d since %s", obj->name, pRaw, lino, tstrerror(code)); + sdbFreeRaw(pRaw); + terrno = code; + return NULL; + } + + terrno = 0; + mTrace("cfg:%s, encode to raw:%p, row:%p", obj->name, pRaw, obj); + return pRaw; +} + +SSdbRow *mndCfgActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int32_t lino = 0; + SSdbRow *pRow = NULL; + SConfigObj *pObj = NULL; + void *buf = NULL; + int8_t sver = 0; + int32_t tlen; + int32_t dataPos = 0; + + code = sdbGetRawSoftVer(pRaw, &sver); + TSDB_CHECK_CODE(code, lino, _over); + + if (sver != CFG_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + goto _over; + } + + pRow = sdbAllocRow(sizeof(SConfigObj)); + TSDB_CHECK_NULL(pRow, code, lino, _over, terrno); + + pObj = sdbGetRowObj(pRow); + TSDB_CHECK_NULL(pObj, code, lino, _over, terrno); + + SDB_GET_INT32(pRaw, dataPos, &tlen, _over); + + buf = taosMemoryMalloc(tlen + 1); + TSDB_CHECK_NULL(buf, code, lino, _over, terrno); + + SDB_GET_BINARY(pRaw, dataPos, buf, tlen, _over); + + SDecoder decoder; + tDecoderInit(&decoder, buf, tlen + 1); + code = tDecodeSConfigObj(&decoder, pObj); + tDecoderClear(&decoder); + + if (code < 0) { + tFreeSConfigObj(pObj); + } + +_over: + taosMemoryFreeClear(buf); + + if (code != TSDB_CODE_SUCCESS) { + mError("cfg:%s, failed to decode from raw:%p since %s at:%d", pObj->name, pRaw, tstrerror(code), lino); + taosMemoryFreeClear(pRow); + terrno = code; + return NULL; + } else { + mTrace("config:%s, decode from raw:%p, row:%p", pObj->name, pRaw, pObj); + terrno = 0; + return pRow; + } +} + +static int32_t mndCfgActionInsert(SSdb *pSdb, SConfigObj *obj) { + mTrace("cfg:%s, perform insert action, row:%p", obj->name, obj); + return 0; +} + +static int32_t mndCfgActionDelete(SSdb *pSdb, SConfigObj *obj) { + mTrace("cfg:%s, perform delete action, row:%p", obj->name, obj); + tFreeSConfigObj(obj); + return 0; +} + +static int32_t mndCfgActionUpdate(SSdb *pSdb, SConfigObj *pOld, SConfigObj *pNew) { + mTrace("cfg:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); + switch (pNew->dtype) { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + pOld->bval = pNew->bval; + break; + case CFG_DTYPE_INT32: + pOld->i32 = pNew->i32; + break; + case CFG_DTYPE_INT64: + pOld->i64 = pNew->i64; + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + pOld->fval = pNew->fval; + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + taosMemoryFree(pOld->str); + pOld->str = taosStrdup(pNew->str); + if (pOld->str == NULL) { + return terrno; + } + break; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t mndCfgActionDeploy(SMnode *pMnode) { return mndInitWriteCfg(pMnode); } + +static int32_t mndCfgActionPrepare(SMnode *pMnode) { return mndTryRebuildCfg(pMnode); } + +static int32_t mndProcessConfigReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SConfigReq configReq = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SArray *array = NULL; + + code = tDeserializeSConfigReq(pReq->pCont, pReq->contLen, &configReq); + if (code != 0) { + mError("failed to deserialize config req, since %s", terrstr()); + goto _OVER; + } + + SConfigObj *vObj = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); + if (vObj == NULL) { + mInfo("failed to acquire mnd config version, since %s", terrstr()); + goto _OVER; + } + + array = taosArrayInit(16, sizeof(SConfigItem)); + if (array == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + SConfigRsp configRsp = {0}; + configRsp.forceReadConfig = configReq.forceReadConfig; + + configRsp.cver = vObj->i32; + if (configRsp.forceReadConfig) { + // compare config array from configReq with current config array + if (compareSConfigItemArrays(taosGetGlobalCfg(tsCfg), configReq.array, array)) { + configRsp.array = array; + } else { + configRsp.isConifgVerified = 1; + } + } else { + if (configReq.cver == vObj->i32) { + configRsp.isVersionVerified = 1; + } else { + code = initConfigArrayFromSdb(pMnode, array); + if (code != 0) { + mError("failed to init config array from sdb, since %s", terrstr()); + goto _OVER; + } + configRsp.array = array; + } + } + + int32_t contLen = tSerializeSConfigRsp(NULL, 0, &configRsp); + if (contLen < 0) { + code = contLen; + goto _OVER; + } + void *pHead = rpcMallocCont(contLen); + if (pHead == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + contLen = tSerializeSConfigRsp(pHead, contLen, &configRsp); + if (contLen < 0) { + rpcFreeCont(pHead); + code = contLen; + goto _OVER; + } + pReq->info.rspLen = contLen; + pReq->info.rsp = pHead; + +_OVER: + if (code != 0) { + mError("failed to process config req, since %s", tstrerror(code)); + } + sdbRelease(pMnode->pSdb, vObj); + cfgArrayCleanUp(array); + return TSDB_CODE_SUCCESS; +} + +int32_t mndInitWriteCfg(SMnode *pMnode) { + int code = 0; + size_t sz = 0; + + mInfo("init write cfg to sdb"); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "init-write-config"); + if (pTrans == NULL) { + mError("failed to init write cfg in create trans, since %s", terrstr()); + goto _OVER; + } + + // encode mnd config version + SConfigObj *versionObj = mndInitConfigVersion(); + if ((code = mndSetCreateConfigCommitLogs(pTrans, versionObj)) != 0) { + mError("failed to init mnd config version, since %s", tstrerror(code)); + tFreeSConfigObj(versionObj); + taosMemoryFree(versionObj); + goto _OVER; + } + tFreeSConfigObj(versionObj); + taosMemoryFree(versionObj); + sz = taosArrayGetSize(taosGetGlobalCfg(tsCfg)); + + for (int i = 0; i < sz; ++i) { + SConfigItem *item = taosArrayGet(taosGetGlobalCfg(tsCfg), i); + SConfigObj *obj = mndInitConfigObj(item); + if (obj == NULL) { + code = terrno; + goto _OVER; + } + if ((code = mndSetCreateConfigCommitLogs(pTrans, obj)) != 0) { + mError("failed to init mnd config:%s, since %s", item->name, tstrerror(code)); + tFreeSConfigObj(obj); + taosMemoryFree(obj); + goto _OVER; + } + tFreeSConfigObj(obj); + taosMemoryFree(obj); + } + if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; + +_OVER: + if (code != 0) { + mError("failed to init write cfg, since %s", tstrerror(code)); + } + mndTransDrop(pTrans); + return code; +} + +int32_t mndTryRebuildCfg(SMnode *pMnode) { + int32_t code = 0; + int32_t sz = -1; + STrans *pTrans = NULL; + SAcctObj *vObj = NULL, *obj = NULL; + SArray *addArray = NULL; + vObj = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); + if (vObj == NULL) { + if ((code = mndInitWriteCfg(pMnode)) < 0) goto _exit; + mInfo("failed to acquire mnd config version, try to rebuild config in sdb."); + } else { + sz = taosArrayGetSize(taosGetGlobalCfg(tsCfg)); + addArray = taosArrayInit(4, sizeof(SConfigObj)); + for (int i = 0; i < sz; ++i) { + SConfigItem *item = taosArrayGet(taosGetGlobalCfg(tsCfg), i); + obj = sdbAcquire(pMnode->pSdb, SDB_CFG, item->name); + if (obj == NULL) { + SConfigObj *newObj = mndInitConfigObj(item); + if (newObj == NULL) { + code = terrno; + goto _exit; + } + if (NULL == taosArrayPush(addArray, newObj)) { + code = terrno; + goto _exit; + } + } else { + sdbRelease(pMnode->pSdb, obj); + } + } + int32_t addSize = taosArrayGetSize(addArray); + if (addSize > 0) { + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "add-config"); + if (pTrans == NULL) { + code = terrno; + goto _exit; + } + for (int i = 0; i < addSize; ++i) { + SConfigObj *AddObj = taosArrayGet(addArray, i); + if ((code = mndSetCreateConfigCommitLogs(pTrans, AddObj)) != 0) goto _exit; + } + if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _exit; + mInfo("add new config to sdb, nums:%d", addSize); + } + } +_exit: + if (code != 0) { + mError("failed to try rebuild config in sdb, since %s", tstrerror(code)); + } + sdbRelease(pMnode->pSdb, vObj); + sdbRelease(pMnode->pSdb, obj); + cfgObjArrayCleanUp(addArray); + mndTransDrop(pTrans); + TAOS_RETURN(code); +} + +int32_t mndSetCreateConfigCommitLogs(STrans *pTrans, SConfigObj *item) { + int32_t code = 0; + SSdbRaw *pCommitRaw = mnCfgActionEncode(item); + if (pCommitRaw == NULL) { + code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw) != 0)) TAOS_RETURN(code); + if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)) != 0) TAOS_RETURN(code); + return TSDB_CODE_SUCCESS; +} + +static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { + int32_t code = 0; + char *p = pMCfgReq->config; + while (*p) { + if (*p == ' ') { + break; + } + p++; + } + + size_t optLen = p - pMCfgReq->config; + strncpy(pDCfgReq->config, pMCfgReq->config, optLen); + pDCfgReq->config[optLen] = 0; + + if (' ' == pMCfgReq->config[optLen]) { + // 'key value' + if (strlen(pMCfgReq->value) != 0) goto _err; + (void)strcpy(pDCfgReq->value, p + 1); + } else { + // 'key' 'value' + if (strlen(pMCfgReq->value) == 0) goto _err; + (void)strcpy(pDCfgReq->value, pMCfgReq->value); + } + + TAOS_RETURN(code); + +_err: + mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); + code = TSDB_CODE_INVALID_CFG; + TAOS_RETURN(code); +} + +static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) { + int32_t code = -1; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + + int64_t curMs = taosGetTimestampMs(); + + while (1) { + SDnodeObj *pDnode = NULL; + pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); + if (pIter == NULL) break; + + if (pDnode->id == dnodeId || dnodeId == -1 || dnodeId == 0) { + bool online = mndIsDnodeOnline(pDnode, curMs); + if (!online) { + mWarn("dnode:%d, is offline, skip to send config req", pDnode->id); + continue; + } + SEpSet epSet = mndGetDnodeEpset(pDnode); + int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, pDcfgReq); + void *pBuf = rpcMallocCont(bufLen); + + if (pBuf == NULL) { + sdbCancelFetch(pMnode->pSdb, pIter); + sdbRelease(pMnode->pSdb, pDnode); + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + + if ((bufLen = tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq)) <= 0) { + sdbCancelFetch(pMnode->pSdb, pIter); + sdbRelease(pMnode->pSdb, pDnode); + code = bufLen; + rpcFreeCont(pBuf); + return code; + } + + mInfo("dnode:%d, send config req to dnode, config:%s value:%s", pDnode->id, pDcfgReq->config, pDcfgReq->value); + SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; + SRpcMsg rpcRsp = {0}; + + code = rpcSendRecvWithTimeout(pMnode->msgCb.statusRpc, &epSet, &rpcMsg, &rpcRsp, NULL, CFG_ALTER_TIMEOUT); + if (code != 0) { + mError("failed to send config req to dnode:%d, since %s", pDnode->id, tstrerror(code)); + sdbCancelFetch(pMnode->pSdb, pIter); + sdbRelease(pMnode->pSdb, pDnode); + return code; + } + + code = rpcRsp.code; + if (code != 0) { + mError("failed to alter config %s,on dnode:%d, since %s", pDcfgReq->config, pDnode->id, tstrerror(code)); + sdbCancelFetch(pMnode->pSdb, pIter); + sdbRelease(pMnode->pSdb, pDnode); + return code; + } + rpcFreeCont(rpcRsp.pCont); + } + sdbRelease(pSdb, pDnode); + } + + if (code == -1) { + code = TSDB_CODE_MND_DNODE_NOT_EXIST; + } + TAOS_RETURN(code); +} + +static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { + int32_t code = 0; + int32_t lino = -1; + SMnode *pMnode = pReq->info.node; + SMCfgDnodeReq cfgReq = {0}; + SConfigObj *vObj = sdbAcquire(pMnode->pSdb, SDB_CFG, "tsmmConfigVersion"); + if (vObj == NULL) { + goto _err_out; + } + + TAOS_CHECK_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq)); + int8_t updateIpWhiteList = 0; + mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); + if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE)) != 0) { + goto _err_out; + } + + SDCfgDnodeReq dcfgReq = {0}; + if (strcasecmp(cfgReq.config, "resetlog") == 0) { + tstrncpy(dcfgReq.config, "resetlog", 9); + goto _send_req; +#ifdef TD_ENTERPRISE + } else if (strncasecmp(cfgReq.config, "s3blocksize", 12) == 0) { + int32_t optLen = strlen("s3blocksize"); + int32_t flag = -1; + int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); + if (code < 0) { + goto _err_out; + } + + if (flag > 1024 * 1024 || (flag > -1 && flag < 1024) || flag < -1) { + mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]", + cfgReq.dnodeId, flag); + code = TSDB_CODE_INVALID_CFG; + goto _err_out; + } + + tstrncpy(dcfgReq.config, "s3blocksize", 12); + snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); +#endif + } else { + TAOS_CHECK_GOTO(mndMCfg2DCfg(&cfgReq, &dcfgReq), &lino, _err_out); + if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) { + mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId); + code = TSDB_CODE_INVALID_CFG; + goto _err_out; + } + if (strncasecmp(dcfgReq.config, "enableWhiteList", strlen("enableWhiteList")) == 0) { + updateIpWhiteList = 1; + } + + CfgAlterType alterType = (cfgReq.dnodeId == 0 || cfgReq.dnodeId == -1) ? CFG_ALTER_ALL_DNODES : CFG_ALTER_DNODE; + TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true, alterType), &lino, + _err_out); + } + SConfigItem *pItem = cfgGetItem(taosGetCfg(), dcfgReq.config); + // Update config in sdb. + if (pItem == NULL) { + mError("failed to find config:%s while process config dnode req", cfgReq.config); + code = TSDB_CODE_CFG_NOT_FOUND; + goto _err_out; + } + if (pItem->category == CFG_CATEGORY_GLOBAL) { + TAOS_CHECK_GOTO(mndConfigUpdateTrans(pMnode, dcfgReq.config, dcfgReq.value, pItem->dtype, ++vObj->i32), &lino, + _err_out); + } +_send_req : + +{ // audit + char obj[50] = {0}; + (void)sprintf(obj, "%d", cfgReq.dnodeId); + + auditRecord(pReq, pMnode->clusterId, "alterDnode", obj, "", cfgReq.sql, cfgReq.sqlLen); +} + dcfgReq.version = vObj->i32; + code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq); + if (code != 0) { + mError("failed to send config req to dnode:%d, since %s", cfgReq.dnodeId, tstrerror(code)); + goto _err_out; + } + // dont care suss or succ; + if (updateIpWhiteList) mndRefreshUserIpWhiteList(pMnode); + tFreeSMCfgDnodeReq(&cfgReq); + sdbRelease(pMnode->pSdb, vObj); + TAOS_RETURN(code); + +_err_out: + mError("failed to process config dnode req, since %s", tstrerror(code)); + tFreeSMCfgDnodeReq(&cfgReq); + sdbRelease(pMnode->pSdb, vObj); + TAOS_RETURN(code); +} + +static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp) { + mInfo("config rsp from dnode"); + return 0; +} + +// get int32_t value from 'SMCfgDnodeReq' +static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32_t *pOutValue) { + int32_t code = 0; + if (' ' != pMCfgReq->config[optLen] && 0 != pMCfgReq->config[optLen]) { + goto _err; + } + + if (' ' == pMCfgReq->config[optLen]) { + // 'key value' + if (strlen(pMCfgReq->value) != 0) goto _err; + *pOutValue = taosStr2Int32(pMCfgReq->config + optLen + 1, NULL, 10); + } else { + // 'key' 'value' + if (strlen(pMCfgReq->value) == 0) goto _err; + *pOutValue = taosStr2Int32(pMCfgReq->value, NULL, 10); + } + + TAOS_RETURN(code); + +_err: + mError(" failed to set config since:%s", tstrerror(code)); + TAOS_RETURN(code); +} + +static int32_t mndConfigUpdateTrans(SMnode *pMnode, const char *name, char *pValue, ECfgDataType dtype, + int32_t tsmmConfigVersion) { + int32_t code = -1; + int32_t lino = -1; + SConfigObj *pVersion = taosMemoryMalloc(sizeof(SConfigObj)), *pObj = taosMemoryMalloc(sizeof(SConfigObj)); + if (pVersion == NULL || pObj == NULL) { + code = terrno; + goto _OVER; + } + tstrncpy(pVersion->name, "tsmmConfigVersion", CFG_NAME_MAX_LEN); + pVersion->dtype = CFG_DTYPE_INT32; + pVersion->i32 = tsmmConfigVersion; + + pObj->dtype = dtype; + tstrncpy(pObj->name, name, CFG_NAME_MAX_LEN); + + TAOS_CHECK_GOTO(mndUpdateObj(pObj, name, pValue), &lino, _OVER); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-config"); + if (pTrans == NULL) { + code = terrno; + goto _OVER; + } + mInfo("trans:%d, used to update config:%s to value:%s", pTrans->id, name, pValue); + TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, pVersion), &lino, _OVER); + TAOS_CHECK_GOTO(mndSetCreateConfigCommitLogs(pTrans, pObj), &lino, _OVER); + if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; + code = 0; +_OVER: + if (code != 0) { + mError("failed to update config:%s to value:%s, since %s", name, pValue, tstrerror(code)); + } + mndTransDrop(pTrans); + tFreeSConfigObj(pVersion); + taosMemoryFree(pVersion); + tFreeSConfigObj(pObj); + taosMemoryFree(pObj); + return code; +} + +static int32_t initConfigArrayFromSdb(SMnode *pMnode, SArray *array) { + int32_t code = 0; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SConfigObj *obj = NULL; + + while (1) { + pIter = sdbFetch(pSdb, SDB_CFG, pIter, (void **)&obj); + if (pIter == NULL) break; + if (obj == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + if (strcasecmp(obj->name, "tsmmConfigVersion") == 0) { + sdbRelease(pSdb, obj); + continue; + } + SConfigItem item = {0}; + item.dtype = obj->dtype; + item.name = taosStrdup(obj->name); + if (item.name == NULL) { + code = terrno; + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, obj); + goto _exit; + } + switch (obj->dtype) { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + item.bval = obj->bval; + break; + case CFG_DTYPE_INT32: + item.i32 = obj->i32; + break; + case CFG_DTYPE_INT64: + item.i64 = obj->i64; + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + item.fval = obj->fval; + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + item.str = taosStrdup(obj->str); + if (item.str == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, obj); + code = terrno; + goto _exit; + } + break; + } + if (taosArrayPush(array, &item) == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, obj); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + break; + } + sdbRelease(pSdb, obj); + } +_exit: + if (code != 0) { + mError("failed to init config array from sdb, since %s", tstrerror(code)); + } + return code; +} + +static void cfgArrayCleanUp(SArray *array) { + if (array == NULL) { + return; + } + + int32_t sz = taosArrayGetSize(array); + for (int32_t i = 0; i < sz; ++i) { + SConfigItem *item = taosArrayGet(array, i); + if (item->dtype == CFG_DTYPE_STRING || item->dtype == CFG_DTYPE_DIR || item->dtype == CFG_DTYPE_LOCALE || + item->dtype == CFG_DTYPE_CHARSET || item->dtype == CFG_DTYPE_TIMEZONE) { + taosMemoryFreeClear(item->str); + } + taosMemoryFreeClear(item->name); + } + + taosArrayDestroy(array); +} + +static void cfgObjArrayCleanUp(SArray *array) { + if (array == NULL) { + return; + } + int32_t sz = taosArrayGetSize(array); + for (int32_t i = 0; i < sz; ++i) { + SConfigObj *obj = taosArrayGet(array, i); + tFreeSConfigObj(obj); + taosMemoryFree(obj); + } + taosArrayDestroy(array); +} + +SArray *initVariablesFromItems(SArray *pItems) { + if (pItems == NULL) { + return NULL; + } + + int32_t sz = taosArrayGetSize(pItems); + + SArray *pInfos = taosArrayInit(sz, sizeof(SVariablesInfo)); + if (pInfos == NULL) { + mError("failed to init array while init variables from items, since %s", tstrerror(terrno)); + return NULL; + } + for (int32_t i = 0; i < sz; ++i) { + SConfigItem *pItem = taosArrayGet(pItems, i); + SVariablesInfo info = {0}; + strcpy(info.name, pItem->name); + + // init info value + switch (pItem->dtype) { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + sprintf(info.value, "%d", pItem->bval); + break; + case CFG_DTYPE_INT32: + sprintf(info.value, "%d", pItem->i32); + break; + case CFG_DTYPE_INT64: + sprintf(info.value, "%" PRId64, pItem->i64); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + sprintf(info.value, "%f", pItem->fval); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + sprintf(info.value, "%s", pItem->str); + break; + } + + // init info scope + switch (pItem->scope) { + case CFG_SCOPE_SERVER: + strcpy(info.scope, "server"); + break; + case CFG_SCOPE_CLIENT: + strcpy(info.scope, "client"); + break; + case CFG_SCOPE_BOTH: + strcpy(info.scope, "both"); + break; + default: + strcpy(info.scope, "unknown"); + break; + } + // init info category + switch (pItem->category) { + case CFG_CATEGORY_GLOBAL: + strcpy(info.category, "global"); + break; + case CFG_CATEGORY_LOCAL: + strcpy(info.category, "local"); + break; + default: + strcpy(info.category, "unknown"); + break; + } + if (NULL == taosArrayPush(pInfos, &info)) { + mError("failed to push info to array while init variables from items,since %s", tstrerror(terrno)); + taosArrayDestroy(pInfos); + return NULL; + } + } + + return pInfos; +} + +static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { + SShowVariablesRsp rsp = {0}; + int32_t code = -1; + + if ((code = mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_SHOW_VARIABLES)) != 0) { + goto _OVER; + } + + SVariablesInfo info = {0}; + + rsp.variables = initVariablesFromItems(taosGetGlobalCfg(tsCfg)); + if (rsp.variables == NULL) { + code = terrno; + goto _OVER; + } + int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp); + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + code = terrno; + goto _OVER; + } + + if ((rspLen = tSerializeSShowVariablesRsp(pRsp, rspLen, &rsp)) <= 0) { + rpcFreeCont(pRsp); + code = rspLen; + goto _OVER; + } + + pReq->info.rspLen = rspLen; + pReq->info.rsp = pRsp; + code = 0; + +_OVER: + + if (code != 0) { + mError("failed to get show variables info since %s", tstrerror(code)); + } + + tFreeSShowVariablesRsp(&rsp); + TAOS_RETURN(code); +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 0538a5fe83..fbce8f544a 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -614,7 +614,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { END: mndTransDrop(pTrans); tDeleteSMqConsumerObj(pConsumerNew); - taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree); + taosArrayDestroyP(subscribe.topicNames, NULL); return (code == TSDB_CODE_TMQ_NO_NEED_REBALANCE || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) ? 0 : code; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index f3baa04f9b..9d02fdf115 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -14,10 +14,12 @@ */ #define _DEFAULT_SOURCE -#include "mndDb.h" #include "audit.h" +#include "command.h" #include "mndArbGroup.h" #include "mndCluster.h" +#include "mndConfig.h" +#include "mndDb.h" #include "mndDnode.h" #include "mndIndex.h" #include "mndPrivilege.h" @@ -34,10 +36,9 @@ #include "systable.h" #include "thttp.h" #include "tjson.h" -#include "command.h" #define DB_VER_NUMBER 1 -#define DB_RESERVE_SIZE 27 +#define DB_RESERVE_SIZE 14 static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); @@ -151,6 +152,10 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT8(pRaw, dataPos, pDb->cfg.withArbitrator, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.encryptAlgorithm, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->tsmaVersion, _OVER); + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.compactTimeOffset, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.compactStartTime, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.compactEndTime, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.compactInterval, _OVER) SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -250,6 +255,10 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.withArbitrator, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.encryptAlgorithm, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->tsmaVersion, _OVER); + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.compactTimeOffset, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.compactStartTime, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.compactEndTime, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.compactInterval, _OVER) SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) taosInitRWLatch(&pDb->lock); @@ -365,6 +374,10 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { pOld->cfg.s3KeepLocal = pNew->cfg.s3KeepLocal; pOld->cfg.s3Compact = pNew->cfg.s3Compact; pOld->cfg.withArbitrator = pNew->cfg.withArbitrator; + pOld->cfg.compactInterval = pNew->cfg.compactInterval; + pOld->cfg.compactStartTime = pNew->cfg.compactStartTime; + pOld->cfg.compactEndTime = pNew->cfg.compactEndTime; + pOld->cfg.compactTimeOffset = pNew->cfg.compactTimeOffset; pOld->compactStartTime = pNew->compactStartTime; pOld->tsmaVersion = pNew->tsmaVersion; taosWUnLockLatch(&pOld->lock); @@ -416,7 +429,12 @@ static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) { return TSDB_CODE_MND_INVALID_DB; } - int32_t acctId = atoi(dbName); + int32_t acctId; + int32_t code = taosStr2int32(dbName, &acctId); + if (code != 0) { + return code; + } + if (acctId != pUser->acctId) { return TSDB_CODE_MND_INVALID_DB_ACCT; } @@ -795,6 +813,10 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, .tsdbPageSize = pCreate->tsdbPageSize, .withArbitrator = pCreate->withArbitrator, .encryptAlgorithm = pCreate->encryptAlgorithm, + .compactInterval = pCreate->compactInterval, + .compactStartTime = pCreate->compactStartTime, + .compactEndTime = pCreate->compactEndTime, + .compactTimeOffset = pCreate->compactTimeOffset, }; dbObj.cfg.numOfRetensions = pCreate->numOfRetensions; @@ -1138,6 +1160,34 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { code = 0; } + if (pAlter->compactInterval >= TSDB_DEFAULT_COMPACT_INTERVAL && pAlter->compactInterval != pDb->cfg.compactInterval) { + pDb->cfg.compactInterval = pAlter->compactInterval; + pDb->vgVersion++; + code = 0; + } + + if (pAlter->compactStartTime != pDb->cfg.compactStartTime && + (pAlter->compactStartTime == TSDB_DEFAULT_COMPACT_START_TIME || + pAlter->compactStartTime <= -pDb->cfg.daysPerFile)) { + pDb->cfg.compactStartTime = pAlter->compactStartTime; + pDb->vgVersion++; + code = 0; + } + + if (pAlter->compactEndTime != pDb->cfg.compactEndTime && + (pAlter->compactEndTime == TSDB_DEFAULT_COMPACT_END_TIME || pAlter->compactEndTime <= -pDb->cfg.daysPerFile)) { + pDb->cfg.compactEndTime = pAlter->compactEndTime; + pDb->vgVersion++; + code = 0; + } + + if (pAlter->compactTimeOffset >= TSDB_MIN_COMPACT_TIME_OFFSET && + pAlter->compactTimeOffset != pDb->cfg.compactTimeOffset) { + pDb->cfg.compactTimeOffset = pAlter->compactTimeOffset; + pDb->vgVersion++; + code = 0; + } + TAOS_RETURN(code); } @@ -1371,6 +1421,18 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) { cfgRsp->s3Compact = pDb->cfg.s3Compact; cfgRsp->withArbitrator = pDb->cfg.withArbitrator; cfgRsp->encryptAlgorithm = pDb->cfg.encryptAlgorithm; + cfgRsp->compactInterval = pDb->cfg.compactInterval; + cfgRsp->compactStartTime = pDb->cfg.compactStartTime; + cfgRsp->compactEndTime = pDb->cfg.compactEndTime; + if (cfgRsp->compactInterval > 0) { + if (cfgRsp->compactStartTime == 0) { + cfgRsp->compactStartTime = -cfgRsp->daysToKeep2; + } + if (cfgRsp->compactEndTime == 0) { + cfgRsp->compactEndTime = -cfgRsp->daysPerFile; + } + } + cfgRsp->compactTimeOffset = pDb->cfg.compactTimeOffset; } static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { @@ -1561,8 +1623,8 @@ static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj * static int32_t mndSetDropDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { int32_t code = 0; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; while (1) { SVgObj *pVgroup = NULL; @@ -1942,9 +2004,9 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, continue; } else { mTrace("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64 - " numOfTables:%d, changed to vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d", - pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs, - pDbCacheInfo->numOfTable, pDb->vgVersion, pDb->cfgVersion, pDb->stateTs, numOfTable); + " numOfTables:%d, changed to vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d", + pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs, + pDbCacheInfo->numOfTable, pDb->vgVersion, pDb->cfgVersion, pDb->stateTs, numOfTable); } if (pDbCacheInfo->cfgVersion < pDb->cfgVersion) { @@ -1956,7 +2018,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, rsp.pTsmaRsp = taosMemoryCalloc(1, sizeof(STableTSMAInfoRsp)); if (rsp.pTsmaRsp) rsp.pTsmaRsp->pTsmas = taosArrayInit(4, POINTER_BYTES); if (rsp.pTsmaRsp && rsp.pTsmaRsp->pTsmas) { - bool exist = false; + bool exist = false; int32_t code = mndGetDbTsmas(pMnode, 0, pDb->uid, rsp.pTsmaRsp, &exist); if (TSDB_CODE_SUCCESS != code) { mndReleaseDb(pMnode, pDb); @@ -2387,7 +2449,8 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)strictVstr, false), &lino, _OVER); char durationVstr[128] = {0}; - int32_t len = formatDurationOrKeep(&durationVstr[VARSTR_HEADER_SIZE], sizeof(durationVstr) - VARSTR_HEADER_SIZE, pDb->cfg.daysPerFile); + int32_t len = formatDurationOrKeep(&durationVstr[VARSTR_HEADER_SIZE], sizeof(durationVstr) - VARSTR_HEADER_SIZE, + pDb->cfg.daysPerFile); varDataSetLen(durationVstr, len); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -2403,9 +2466,9 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, int32_t lenKeep2 = formatDurationOrKeep(keep2Str, sizeof(keep2Str), pDb->cfg.daysToKeep2); if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { - len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep1Str, keep2Str, keep0Str); + len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep1Str, keep2Str, keep0Str); } else { - len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep0Str, keep1Str, keep2Str); + len = sprintf(&keepVstr[VARSTR_HEADER_SIZE], "%s,%s,%s", keep0Str, keep1Str, keep2Str); } varDataSetLen(keepVstr, len); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 61606b0364..7ea0ba67bd 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -13,9 +13,10 @@ * along with this program. If not, see . */ #define _DEFAULT_SOURCE -#include "mndDef.h" #include "mndConsumer.h" +#include "mndDef.h" #include "taoserror.h" +#include "tunit.h" static void *freeStreamTasks(SArray *pTaskLevel); @@ -72,7 +73,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { TAOS_CHECK_RETURN(tEncodeI32(pEncoder, innerSz)); for (int32_t j = 0; j < innerSz; j++) { SStreamTask *pTask = taosArrayGetP(pArray, j); - if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){ + if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER) { pTask->ver = SSTREAM_TASK_VER; } TAOS_CHECK_RETURN(tEncodeStreamTask(pEncoder, pTask)); @@ -273,9 +274,9 @@ void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp, int8_t sver) { static void *topicNameDup(void *p) { return taosStrdup((char *)p); } -int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, - char *topic, SCMSubscribeReq *subscribe, SMqConsumerObj** ppConsumer) { - int32_t code = 0; +int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, char *topic, SCMSubscribeReq *subscribe, + SMqConsumerObj **ppConsumer) { + int32_t code = 0; SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); if (pConsumer == NULL) { code = terrno; @@ -294,30 +295,30 @@ int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, pConsumer->createTime = taosGetTimestampMs(); pConsumer->updateType = updateType; - if (updateType == CONSUMER_ADD_REB){ + if (updateType == CONSUMER_ADD_REB) { pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *)); - if(pConsumer->rebNewTopics == NULL){ + if (pConsumer->rebNewTopics == NULL) { code = terrno; goto END; } - char* topicTmp = taosStrdup(topic); + char *topicTmp = taosStrdup(topic); if (taosArrayPush(pConsumer->rebNewTopics, &topicTmp) == NULL) { code = terrno; goto END; } - }else if (updateType == CONSUMER_REMOVE_REB) { + } else if (updateType == CONSUMER_REMOVE_REB) { pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *)); - if(pConsumer->rebRemovedTopics == NULL){ + if (pConsumer->rebRemovedTopics == NULL) { code = terrno; goto END; } - char* topicTmp = taosStrdup(topic); + char *topicTmp = taosStrdup(topic); if (taosArrayPush(pConsumer->rebRemovedTopics, &topicTmp) == NULL) { code = terrno; goto END; } - }else if (updateType == CONSUMER_INSERT_SUB){ + } else if (updateType == CONSUMER_INSERT_SUB) { tstrncpy(pConsumer->clientId, subscribe->clientId, tListLen(pConsumer->clientId)); pConsumer->withTbName = subscribe->withTbName; pConsumer->autoCommit = subscribe->autoCommit; @@ -329,13 +330,13 @@ int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, tstrncpy(pConsumer->fqdn, subscribe->fqdn, TSDB_FQDN_LEN); pConsumer->rebNewTopics = taosArrayDup(subscribe->topicNames, topicNameDup); - if (pConsumer->rebNewTopics == NULL){ + if (pConsumer->rebNewTopics == NULL) { code = terrno; goto END; } pConsumer->assignedTopics = subscribe->topicNames; subscribe->topicNames = NULL; - }else if (updateType == CONSUMER_UPDATE_SUB){ + } else if (updateType == CONSUMER_UPDATE_SUB) { pConsumer->assignedTopics = subscribe->topicNames; subscribe->topicNames = NULL; } @@ -350,10 +351,10 @@ END: void tClearSMqConsumerObj(SMqConsumerObj *pConsumer) { if (pConsumer == NULL) return; - taosArrayDestroyP(pConsumer->currentTopics, (FDelete)taosMemoryFree); - taosArrayDestroyP(pConsumer->rebNewTopics, (FDelete)taosMemoryFree); - taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree); - taosArrayDestroyP(pConsumer->assignedTopics, (FDelete)taosMemoryFree); + taosArrayDestroyP(pConsumer->currentTopics, NULL); + taosArrayDestroyP(pConsumer->rebNewTopics, NULL); + taosArrayDestroyP(pConsumer->rebRemovedTopics, NULL); + taosArrayDestroyP(pConsumer->assignedTopics, NULL); } void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer) { @@ -504,12 +505,12 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer, int8_t s buf = taosDecodeFixedI32(buf, &pConsumer->autoCommitInterval); buf = taosDecodeFixedI32(buf, &pConsumer->resetOffsetCfg); } - if (sver > 2){ + if (sver > 2) { buf = taosDecodeFixedI32(buf, &pConsumer->maxPollIntervalMs); buf = taosDecodeFixedI32(buf, &pConsumer->sessionTimeoutMs); buf = taosDecodeStringTo(buf, pConsumer->user); buf = taosDecodeStringTo(buf, pConsumer->fqdn); - } else{ + } else { pConsumer->maxPollIntervalMs = DEFAULT_MAX_POLL_INTERVAL; pConsumer->sessionTimeoutMs = DEFAULT_SESSION_TIMEOUT; } @@ -517,7 +518,7 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer, int8_t s return (void *)buf; } -int32_t tEncodeOffRows(void **buf, SArray *offsetRows){ +int32_t tEncodeOffRows(void **buf, SArray *offsetRows) { int32_t tlen = 0; int32_t szVgs = taosArrayGetSize(offsetRows); tlen += taosEncodeFixedI32(buf, szVgs); @@ -545,11 +546,10 @@ int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) { tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp); - return tlen + tEncodeOffRows(buf, pConsumerEp->offsetRows); } -void *tDecodeOffRows(const void *buf, SArray **offsetRows, int8_t sver){ +void *tDecodeOffRows(const void *buf, SArray **offsetRows, int8_t sver) { int32_t szVgs = 0; buf = taosDecodeFixedI32(buf, &szVgs); if (szVgs > 0) { @@ -568,7 +568,7 @@ void *tDecodeOffRows(const void *buf, SArray **offsetRows, int8_t sver){ } else { // do nothing } - if(sver > 2){ + if (sver > 2) { buf = taosDecodeFixedI64(buf, &offRows->ever); } } @@ -587,7 +587,7 @@ void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp, int8_t s } int32_t tNewSubscribeObj(const char *key, SMqSubscribeObj **ppSub) { - int32_t code = 0; + int32_t code = 0; SMqSubscribeObj *pSubObj = taosMemoryCalloc(1, sizeof(SMqSubscribeObj)); MND_TMQ_NULL_CHECK(pSubObj); @@ -598,7 +598,7 @@ int32_t tNewSubscribeObj(const char *key, SMqSubscribeObj **ppSub) { MND_TMQ_NULL_CHECK(pSubObj->consumerHash); pSubObj->unassignedVgs = taosArrayInit(0, POINTER_BYTES); MND_TMQ_NULL_CHECK(pSubObj->unassignedVgs); - if (ppSub){ + if (ppSub) { *ppSub = pSubObj; } return code; @@ -609,7 +609,7 @@ END: } int32_t tCloneSubscribeObj(const SMqSubscribeObj *pSub, SMqSubscribeObj **ppSub) { - int32_t code = 0; + int32_t code = 0; SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj)); if (pSubNew == NULL) { code = terrno; @@ -730,6 +730,188 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub, int8_t sver) { return (void *)buf; } +SConfigObj *mndInitConfigObj(SConfigItem *pItem) { + SConfigObj *pObj = taosMemoryCalloc(1, sizeof(SConfigObj)); + if (pObj == NULL) { + return NULL; + } + strncpy(pObj->name, pItem->name, CFG_NAME_MAX_LEN); + pObj->dtype = pItem->dtype; + switch (pItem->dtype) { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + pObj->bval = pItem->bval; + break; + case CFG_DTYPE_INT32: + pObj->i32 = pItem->i32; + break; + case CFG_DTYPE_INT64: + pObj->i64 = pItem->i64; + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + pObj->fval = pItem->fval; + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + pObj->str = taosStrdup(pItem->str); + if (pObj->str == NULL) { + taosMemoryFree(pObj); + return NULL; + } + break; + } + return pObj; +} + +int32_t mndUpdateObj(SConfigObj *pObjNew, const char *name, char *value) { + int32_t code = 0; + switch (pObjNew->dtype) { + case CFG_DTYPE_BOOL: { + bool tmp = false; + if (strcasecmp(value, "true") == 0) { + tmp = true; + } + if (atoi(value) > 0) { + tmp = true; + } + pObjNew->bval = tmp; + break; + } + case CFG_DTYPE_INT32: { + int32_t ival; + TAOS_CHECK_RETURN(taosStrHumanToInt32(value, &ival)); + pObjNew->i32 = ival; + break; + } + case CFG_DTYPE_INT64: { + int64_t ival; + TAOS_CHECK_RETURN(taosStrHumanToInt64(value, &ival)); + pObjNew->i64 = ival; + break; + } + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: { + float dval = 0; + TAOS_CHECK_RETURN(parseCfgReal(value, &dval)); + pObjNew->fval = dval; + break; + } + case CFG_DTYPE_DIR: + case CFG_DTYPE_TIMEZONE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_STRING: { + pObjNew->str = taosStrdup(value); + if (pObjNew->str == NULL) { + code = terrno; + return code; + } + break; + } + case CFG_DTYPE_NONE: + break; + default: + code = TSDB_CODE_INVALID_CFG; + break; + } + return code; +} + +SConfigObj *mndInitConfigVersion() { + SConfigObj *pObj = taosMemoryCalloc(1, sizeof(SConfigObj)); + if (pObj == NULL) { + return NULL; + } + tstrncpy(pObj->name, "tsmmConfigVersion", CFG_NAME_MAX_LEN); + pObj->dtype = CFG_DTYPE_INT32; + pObj->i32 = 0; + return pObj; +} + +int32_t tEncodeSConfigObj(SEncoder *pEncoder, const SConfigObj *pObj) { + TAOS_CHECK_RETURN(tStartEncode(pEncoder)); + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->name)); + + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->dtype)); + switch (pObj->dtype) { + case CFG_DTYPE_BOOL: + TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->bval)); + break; + case CFG_DTYPE_INT32: + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->i32)); + break; + case CFG_DTYPE_INT64: + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->i64)); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + TAOS_CHECK_RETURN(tEncodeFloat(pEncoder, pObj->fval)); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + if (pObj->str != NULL) { + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->str)); + } else { + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, "")); + } + break; + default: + break; + } + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSConfigObj(SDecoder *pDecoder, SConfigObj *pObj) { + TAOS_CHECK_RETURN(tStartDecode(pDecoder)); + TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->name)); + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, (int32_t *)&pObj->dtype)); + switch (pObj->dtype) { + case CFG_DTYPE_NONE: + break; + case CFG_DTYPE_BOOL: + TAOS_CHECK_RETURN(tDecodeBool(pDecoder, &pObj->bval)); + break; + case CFG_DTYPE_INT32: + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pObj->i32)); + break; + case CFG_DTYPE_INT64: + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->i64)); + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + TAOS_CHECK_RETURN(tDecodeFloat(pDecoder, &pObj->fval)); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->str)); + break; + } + tEndDecode(pDecoder); + TAOS_RETURN(TSDB_CODE_SUCCESS); +} + +void tFreeSConfigObj(SConfigObj *obj) { + if (obj == NULL) { + return; + } + if (obj->dtype == CFG_DTYPE_STRING || obj->dtype == CFG_DTYPE_DIR || obj->dtype == CFG_DTYPE_LOCALE || + obj->dtype == CFG_DTYPE_CHARSET || obj->dtype == CFG_DTYPE_TIMEZONE) { + taosMemoryFree(obj->str); + } +} + // SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { // SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry)); // if (pEntryNew == NULL) return NULL; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 0011c11b0a..409aa4c9c9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -28,6 +28,7 @@ #include "mndUser.h" #include "mndVgroup.h" #include "taos_monitor.h" +#include "tconfig.h" #include "tjson.h" #include "tmisce.h" #include "tunit.h" @@ -76,12 +77,9 @@ static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOld, SDnodeObj *pNew); static int32_t mndProcessDnodeListReq(SRpcMsg *pReq); -static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq); static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq); static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq); -static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq); -static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessStatusReq(SRpcMsg *pReq); static int32_t mndProcessNotifyReq(SRpcMsg *pReq); static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq); @@ -96,8 +94,6 @@ static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter); -static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue); - #ifdef _GRANT int32_t mndUpdClusterInfo(SRpcMsg *pReq); #else @@ -118,12 +114,9 @@ int32_t mndInitDnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CREATE_DNODE, mndProcessCreateDnodeReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_DNODE, mndProcessDropDnodeReq); - mndSetMsgHandle(pMnode, TDMT_MND_CONFIG_DNODE, mndProcessConfigDnodeReq); - mndSetMsgHandle(pMnode, TDMT_DND_CONFIG_DNODE_RSP, mndProcessConfigDnodeRsp); mndSetMsgHandle(pMnode, TDMT_MND_STATUS, mndProcessStatusReq); mndSetMsgHandle(pMnode, TDMT_MND_NOTIFY, mndProcessNotifyReq); mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq); - mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq); mndSetMsgHandle(pMnode, TDMT_MND_RESTORE_DNODE, mndProcessRestoreDnodeReq); mndSetMsgHandle(pMnode, TDMT_MND_STATIS, mndProcessStatisReq); mndSetMsgHandle(pMnode, TDMT_MND_AUDIT, mndProcessAuditReq); @@ -462,7 +455,7 @@ int32_t mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) { dInfo.isMnode = 0; } - if(taosArrayPush(pDnodeInfo, &dInfo) == NULL){ + if (taosArrayPush(pDnodeInfo, &dInfo) == NULL) { code = terrno; sdbCancelFetch(pSdb, pIter); break; @@ -471,12 +464,12 @@ int32_t mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) { TAOS_RETURN(code); } -#define CHECK_MONITOR_PARA(para,err) \ -if (pCfg->monitorParas.para != para) { \ - mError("dnode:%d, para:%d inconsistent with cluster:%d", pDnode->id, pCfg->monitorParas.para, para); \ - terrno = err; \ - return err;\ -} +#define CHECK_MONITOR_PARA(para, err) \ + if (pCfg->monitorParas.para != para) { \ + mError("dnode:%d, para:%d inconsistent with cluster:%d", pDnode->id, pCfg->monitorParas.para, para); \ + terrno = err; \ + return err; \ + } static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const SClusterCfg *pCfg) { CHECK_MONITOR_PARA(tsEnableMonitor, DND_REASON_STATUS_MONITOR_SWITCH_NOT_MATCH); @@ -486,7 +479,8 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const S CHECK_MONITOR_PARA(tsSlowLogScope, DND_REASON_STATUS_MONITOR_SLOW_LOG_SCOPE_NOT_MATCH); if (0 != strcasecmp(pCfg->monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb)) { - mError("dnode:%d, tsSlowLogExceptDb:%s inconsistent with cluster:%s", pDnode->id, pCfg->monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb); + mError("dnode:%d, tsSlowLogExceptDb:%s inconsistent with cluster:%s", pDnode->id, + pCfg->monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb); terrno = TSDB_CODE_DNODE_INVALID_MONITOR_PARAS; return DND_REASON_STATUS_MONITOR_NOT_MATCH; } @@ -582,8 +576,8 @@ static bool mndUpdateMnodeState(SMnodeObj *pObj, SMnodeLoad *pMload) { return stateChanged; } -extern char* tsMonFwUri; -extern char* tsMonSlowLogUri; +extern char *tsMonFwUri; +extern char *tsMonSlowLogUri; static int32_t mndProcessStatisReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStatisReq statisReq = {0}; @@ -595,9 +589,9 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { mInfo("process statis req,\n %s", statisReq.pCont); } - if (statisReq.type == MONITOR_TYPE_COUNTER){ + if (statisReq.type == MONITOR_TYPE_COUNTER) { monSendContent(statisReq.pCont, tsMonFwUri); - }else if(statisReq.type == MONITOR_TYPE_SLOW_LOG){ + } else if (statisReq.type == MONITOR_TYPE_SLOW_LOG) { monSendContent(statisReq.pCont, tsMonSlowLogUri); } @@ -1057,150 +1051,32 @@ _OVER: TAOS_RETURN(code); } -static void getSlowLogScopeString(int32_t scope, char* result){ - if(scope == SLOW_LOG_TYPE_NULL) { +static void getSlowLogScopeString(int32_t scope, char *result) { + if (scope == SLOW_LOG_TYPE_NULL) { (void)strcat(result, "NONE"); return; } - while(scope > 0){ - if(scope & SLOW_LOG_TYPE_QUERY) { + while (scope > 0) { + if (scope & SLOW_LOG_TYPE_QUERY) { (void)strcat(result, "QUERY"); scope &= ~SLOW_LOG_TYPE_QUERY; - } else if(scope & SLOW_LOG_TYPE_INSERT) { + } else if (scope & SLOW_LOG_TYPE_INSERT) { (void)strcat(result, "INSERT"); scope &= ~SLOW_LOG_TYPE_INSERT; - } else if(scope & SLOW_LOG_TYPE_OTHERS) { + } else if (scope & SLOW_LOG_TYPE_OTHERS) { (void)strcat(result, "OTHERS"); scope &= ~SLOW_LOG_TYPE_OTHERS; - } else{ + } else { (void)printf("invalid slow log scope:%d", scope); return; } - if(scope > 0) { + if (scope > 0) { (void)strcat(result, "|"); } } } -static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { - SShowVariablesRsp rsp = {0}; - int32_t code = -1; - - if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_SHOW_VARIABLES) != 0) { - goto _OVER; - } - - rsp.variables = taosArrayInit(16, sizeof(SVariablesInfo)); - if (NULL == rsp.variables) { - mError("failed to alloc SVariablesInfo array while process show variables req"); - code = terrno; - goto _OVER; - } - - SVariablesInfo info = {0}; - - (void)strcpy(info.name, "statusInterval"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsStatusInterval); - (void)strcpy(info.scope, "server"); - // fill info.info - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - (void)strcpy(info.name, "timezone"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsTimezoneStr); - (void)strcpy(info.scope, "both"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - (void)strcpy(info.name, "locale"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsLocale); - (void)strcpy(info.scope, "both"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - (void)strcpy(info.name, "charset"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsCharset); - (void)strcpy(info.scope, "both"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - (void)strcpy(info.name, "monitor"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsEnableMonitor); - (void)strcpy(info.scope, "server"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - (void)strcpy(info.name, "monitorInterval"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsMonitorInterval); - (void)strcpy(info.scope, "server"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - (void)strcpy(info.name, "slowLogThreshold"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsSlowLogThreshold); - (void)strcpy(info.scope, "server"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - (void)strcpy(info.name, "slowLogMaxLen"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsSlowLogMaxLen); - (void)strcpy(info.scope, "server"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - char scopeStr[64] = {0}; - getSlowLogScopeString(tsSlowLogScope, scopeStr); - (void)strcpy(info.name, "slowLogScope"); - (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", scopeStr); - (void)strcpy(info.scope, "server"); - if (taosArrayPush(rsp.variables, &info) == NULL) { - code = terrno; - goto _OVER; - } - - int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp); - void *pRsp = rpcMallocCont(rspLen); - if (pRsp == NULL) { - code = terrno; - goto _OVER; - } - - if ((rspLen = tSerializeSShowVariablesRsp(pRsp, rspLen, &rsp)) <= 0) { - code = rspLen; - goto _OVER; - } - - pReq->info.rspLen = rspLen; - pReq->info.rsp = pRsp; - code = 0; - -_OVER: - - if (code != 0) { - mError("failed to get show variables info since %s", tstrerror(code)); - } - - tFreeSShowVariablesRsp(&rsp); - TAOS_RETURN(code); -} - static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -1437,144 +1313,6 @@ _OVER: TAOS_RETURN(code); } -static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { - int32_t code = 0; - char *p = pMCfgReq->config; - while (*p) { - if (*p == ' ') { - break; - } - p++; - } - - size_t optLen = p - pMCfgReq->config; - (void)strncpy(pDCfgReq->config, pMCfgReq->config, optLen); - pDCfgReq->config[optLen] = 0; - - if (' ' == pMCfgReq->config[optLen]) { - // 'key value' - if (strlen(pMCfgReq->value) != 0) goto _err; - (void)strcpy(pDCfgReq->value, p + 1); - } else { - // 'key' 'value' - if (strlen(pMCfgReq->value) == 0) goto _err; - (void)strcpy(pDCfgReq->value, pMCfgReq->value); - } - - TAOS_RETURN(code); - -_err: - mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); - code = TSDB_CODE_INVALID_CFG; - TAOS_RETURN(code); -} - -static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) { - int32_t code = -1; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - while (1) { - SDnodeObj *pDnode = NULL; - pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); - if (pIter == NULL) break; - - if (pDnode->id == dnodeId || dnodeId == -1 || dnodeId == 0) { - SEpSet epSet = mndGetDnodeEpset(pDnode); - int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, pDcfgReq); - void *pBuf = rpcMallocCont(bufLen); - - if (pBuf != NULL) { - if ((bufLen = tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq)) <= 0) { - code = bufLen; - return code; - } - mInfo("dnode:%d, send config req to dnode, config:%s value:%s", dnodeId, pDcfgReq->config, pDcfgReq->value); - SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; - code = tmsgSendReq(&epSet, &rpcMsg); - } - } - - sdbRelease(pSdb, pDnode); - } - - if (code == -1) { - code = TSDB_CODE_MND_DNODE_NOT_EXIST; - } - TAOS_RETURN(code); -} - -static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { - int32_t code = 0; - SMnode *pMnode = pReq->info.node; - SMCfgDnodeReq cfgReq = {0}; - TAOS_CHECK_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq)); - int8_t updateIpWhiteList = 0; - mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); - if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE)) != 0) { - tFreeSMCfgDnodeReq(&cfgReq); - TAOS_RETURN(code); - } - - SDCfgDnodeReq dcfgReq = {0}; - if (strcasecmp(cfgReq.config, "resetlog") == 0) { - (void)strcpy(dcfgReq.config, "resetlog"); -#ifdef TD_ENTERPRISE - } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) { - int32_t optLen = strlen("s3blocksize"); - int32_t flag = -1; - int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); - if (code < 0) return code; - - if (flag > 1024 * 1024 || (flag > -1 && flag < 1024) || flag < -1) { - mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]", - cfgReq.dnodeId, flag); - code = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - TAOS_RETURN(code); - } - - strcpy(dcfgReq.config, "s3blocksize"); - snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); -#endif - } else { - TAOS_CHECK_GOTO (mndMCfg2DCfg(&cfgReq, &dcfgReq), NULL, _err_out); - if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) { - mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId); - code = TSDB_CODE_INVALID_CFG; - goto _err_out; - } - if (strncasecmp(dcfgReq.config, "enableWhiteList", strlen("enableWhiteList")) == 0) { - updateIpWhiteList = 1; - } - - TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true), NULL, _err_out); - } - - { // audit - char obj[50] = {0}; - (void)sprintf(obj, "%d", cfgReq.dnodeId); - - auditRecord(pReq, pMnode->clusterId, "alterDnode", obj, "", cfgReq.sql, cfgReq.sqlLen); - } - - tFreeSMCfgDnodeReq(&cfgReq); - - code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq); - - // dont care suss or succ; - if (updateIpWhiteList) mndRefreshUserIpWhiteList(pMnode); - TAOS_RETURN(code); - -_err_out: - tFreeSMCfgDnodeReq(&cfgReq); - TAOS_RETURN(code); -} - -static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp) { - mInfo("config rsp from dnode"); - return 0; -} - static int32_t mndProcessCreateEncryptKeyReqImpl(SRpcMsg *pReq, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) { int32_t code = 0; SMnode *pMnode = pReq->info.node; @@ -1870,31 +1608,6 @@ static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter) { sdbCancelFetchByType(pSdb, pIter, SDB_DNODE); } -// get int32_t value from 'SMCfgDnodeReq' -static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32_t *pOutValue) { - int32_t code = 0; - if (' ' != pMCfgReq->config[optLen] && 0 != pMCfgReq->config[optLen]) { - goto _err; - } - - if (' ' == pMCfgReq->config[optLen]) { - // 'key value' - if (strlen(pMCfgReq->value) != 0) goto _err; - *pOutValue = atoi(pMCfgReq->config + optLen + 1); - } else { - // 'key' 'value' - if (strlen(pMCfgReq->value) == 0) goto _err; - *pOutValue = atoi(pMCfgReq->value); - } - - TAOS_RETURN(code); - -_err: - mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); - code = TSDB_CODE_INVALID_CFG; - TAOS_RETURN(code); -} - SArray *mndGetAllDnodeFqdns(SMnode *pMnode) { SDnodeObj *pObj = NULL; void *pIter = NULL; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index fbd8bde5f3..2621f3761f 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -15,11 +15,12 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" -#include "mndArbGroup.h" #include "mndAnode.h" +#include "mndArbGroup.h" #include "mndCluster.h" #include "mndCompact.h" #include "mndCompactDetail.h" +#include "mndConfig.h" #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" @@ -254,7 +255,7 @@ static void mndIncreaseUpTime(SMnode *pMnode) { .pCont = pReq, .contLen = contLen, .info.notFreeAhandle = 1, - .info.ahandle = (void *)0x9527}; + .info.ahandle = 0}; // TODO check return value if (tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg) < 0) { mError("failed to put into write-queue since %s, line:%d", terrstr(), __LINE__); @@ -530,7 +531,7 @@ static int32_t mndInitWal(SMnode *pMnode) { code = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; TAOS_RETURN(code); } else { - (void)strncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN + 1); } } #endif @@ -611,6 +612,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-snode", mndInitSnode, mndCleanupSnode)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-anode", mndInitAnode, mndCleanupAnode)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-arbgroup", mndInitArbGroup, mndCleanupArbGroup)); + TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-config", mndInitConfig, NULL)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant)); @@ -637,7 +639,6 @@ static int32_t mndInitSteps(SMnode *pMnode) { TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-query", mndInitQuery, mndCleanupQuery)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync)); TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-telem", mndInitTelem, mndCleanupTelem)); - return 0; } @@ -796,6 +797,12 @@ int32_t mndStart(SMnode *pMnode) { return -1; } mndSetRestored(pMnode, true); + } else { + if (sdbPrepare(pMnode->pSdb) != 0) { + mError("failed to prepare sdb while start mnode"); + return -1; + } + mndSetRestored(pMnode, true); } grantReset(pMnode, TSDB_GRANT_ALL, 0); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 6b1c97b399..4df5727c66 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -14,10 +14,10 @@ */ #define _DEFAULT_SOURCE +#include "mndMnode.h" #include "audit.h" #include "mndCluster.h" #include "mndDnode.h" -#include "mndMnode.h" #include "mndPrivilege.h" #include "mndShow.h" #include "mndSync.h" @@ -248,6 +248,10 @@ bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) { } void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { + if (pMnode == NULL || pEpSet == NULL) { + return; + } + SSdb *pSdb = pMnode->pSdb; int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE); if (totalMnodes == 0) { @@ -722,10 +726,13 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { code = mndCreateMnode(pMnode, pReq, pDnode, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; - char obj[40] = {0}; - sprintf(obj, "%d", createReq.dnodeId); - - auditRecord(pReq, pMnode->clusterId, "createMnode", "", obj, createReq.sql, createReq.sqlLen); + char obj[40] = {0}; + int32_t bytes = snprintf(obj, sizeof(obj), "%d", createReq.dnodeId); + if ((uint32_t)bytes < sizeof(obj)) { + auditRecord(pReq, pMnode->clusterId, "createMnode", "", obj, createReq.sql, createReq.sqlLen); + } else { + mError("mnode:%d, failed to audit create req since %s", createReq.dnodeId, tstrerror(TSDB_CODE_OUT_OF_RANGE)); + } _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { @@ -917,7 +924,9 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->id, false); if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, terrstr()); + mError("mnode:%d, failed to set col data val since %s", pObj->id, tstrerror(code)); + sdbCancelFetch(pSdb, pShow->pIter); + sdbRelease(pSdb, pObj); goto _out; } @@ -927,7 +936,9 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, b1, false); if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, terrstr()); + mError("mnode:%d, failed to set col data val since %s", pObj->id, tstrerror(code)); + sdbCancelFetch(pSdb, pShow->pIter); + sdbRelease(pSdb, pObj); goto _out; } @@ -947,10 +958,8 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB STR_WITH_MAXSIZE_TO_VARSTR(b2, role, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)b2, false); - if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, terrstr()); - goto _out; - } + if (code != 0) goto _err; + const char *status = "ready"; if (objStatus == SDB_STATUS_CREATING) status = "creating"; if (objStatus == SDB_STATUS_DROPPING) status = "dropping"; @@ -959,25 +968,16 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB STR_WITH_MAXSIZE_TO_VARSTR(b3, status, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)b3, false); - if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, terrstr()); - goto _out; - } + if (code != 0) goto _err; pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->createdTime, false); - if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, terrstr()); - goto _out; - } + if (code != 0) goto _err; int64_t roleTimeMs = (isDnodeOnline) ? pObj->roleTimeMs : 0; pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)&roleTimeMs, false); - if (code != 0) { - mError("mnode:%d, failed to set col data val since %s", pObj->id, terrstr()); - goto _out; - } + if (code != 0) goto _err; numOfRows++; sdbRelease(pSdb, pObj); @@ -988,6 +988,13 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB _out: sdbRelease(pSdb, pSelfObj); return numOfRows; + +_err: + mError("mnode:%d, failed to set col data val since %s", pObj->id, tstrerror(code)); + sdbCancelFetch(pSdb, pShow->pIter); + sdbRelease(pSdb, pObj); + sdbRelease(pSdb, pSelfObj); + return numOfRows; } static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter) { diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 9eefaf6a20..e691ecb904 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -14,12 +14,12 @@ */ #define _DEFAULT_SOURCE +#include "mndProfile.h" #include "audit.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" #include "mndPrivilege.h" -#include "mndProfile.h" #include "mndQnode.h" #include "mndShow.h" #include "mndSma.h" @@ -344,10 +344,13 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { code = 0; - char detail[1000] = {0}; - (void)sprintf(detail, "app:%s", connReq.app); - - auditRecord(pReq, pMnode->clusterId, "login", "", "", detail, strlen(detail)); + char detail[1000] = {0}; + int32_t nBytes = snprintf(detail, sizeof(detail), "app:%s", connReq.app); + if ((uint32_t)nBytes < sizeof(detail)) { + auditRecord(pReq, pMnode->clusterId, "login", "", "", detail, strlen(detail)); + } else { + mError("failed to audit logic since %s", tstrerror(TSDB_CODE_OUT_OF_RANGE)); + } _OVER: diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 1906835122..49dc62d471 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -144,6 +144,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { type = TSDB_MGMT_TABLE_TSMAS; } else if (strncasecmp(name, TSDB_INS_DISK_USAGE, len) == 0) { type = TSDB_MGMT_TABLE_USAGE; + } else if (strncasecmp(name, TSDB_INS_TABLE_FILESETS, len) == 0) { + type = TSDB_MGMT_TABLE_FILESETS; } else { mError("invalid show name:%s len:%d", name, len); } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 64a11fbe5b..9db7ae0156 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndSma.h" +#include "functionMgt.h" #include "mndDb.h" #include "mndDnode.h" #include "mndIndex.h" @@ -31,7 +32,6 @@ #include "mndVgroup.h" #include "parser.h" #include "tname.h" -#include "functionMgt.h" #define TSDB_SMA_VER_NUMBER 1 #define TSDB_SMA_RESERVE_SIZE 64 @@ -48,8 +48,8 @@ static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq); static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndDestroySmaObj(SSmaObj *pSmaObj); -static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq); -static int32_t mndProcessDropTSMAReq(SRpcMsg* pReq); +static int32_t mndProcessCreateTSMAReq(SRpcMsg *pReq); +static int32_t mndProcessDropTSMAReq(SRpcMsg *pReq); // sma and tag index comm func static int32_t mndProcessDropIdxReq(SRpcMsg *pReq); @@ -61,11 +61,11 @@ static void mndCancelRetrieveTSMA(SMnode *pMnode, void *pIter); static int32_t mndProcessGetTbTSMAReq(SRpcMsg *pReq); typedef struct SCreateTSMACxt { - SMnode * pMnode; + SMnode *pMnode; const SRpcMsg *pRpcReq; union { const SMCreateSmaReq *pCreateSmaReq; - const SMDropSmaReq * pDropSmaReq; + const SMDropSmaReq *pDropSmaReq; }; SDbObj *pDb; SStbObj *pSrcStb; @@ -298,16 +298,13 @@ void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma) { sdbRelease(pSdb, pSma); } -SDbObj *mndAcquireDbBySma(SMnode *pMnode, const char *db) { - - return mndAcquireDb(pMnode, db); -} +SDbObj *mndAcquireDbBySma(SMnode *pMnode, const char *db) { return mndAcquireDb(pMnode, db); } static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) { SEncoder encoder = {0}; int32_t contLen = 0; SName name = {0}; - int32_t code = tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + int32_t code = tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); if (TSDB_CODE_SUCCESS != code) { return NULL; } @@ -368,7 +365,7 @@ static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, SEncoder encoder = {0}; int32_t contLen; SName name = {0}; - int32_t code = tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + int32_t code = tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); if (TSDB_CODE_SUCCESS != code) { terrno = code; return NULL; @@ -424,9 +421,9 @@ static int32_t mndSetCreateSmaRedoLogs(SMnode *pMnode, STrans *pTrans, SSmaObj * TAOS_RETURN(code); } -static int32_t mndSetCreateSmaUndoLogs(SMnode* pMnode, STrans* pTrans, SSmaObj* pSma) { - int32_t code = 0; - SSdbRaw * pUndoRaw = mndSmaActionEncode(pSma); +static int32_t mndSetCreateSmaUndoLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + int32_t code = 0; + SSdbRaw *pUndoRaw = mndSmaActionEncode(pSma); if (!pUndoRaw) { code = TSDB_CODE_MND_RETURN_VALUE_NULL; if (terrno != 0) code = terrno; @@ -670,7 +667,8 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea // check the maxDelay if (streamObj.conf.triggerParam < TSDB_MIN_ROLLUP_MAX_DELAY) { int64_t msInterval = -1; - int32_t code = convertTimeFromPrecisionToUnit(pCreate->interval, pDb->cfg.precision, TIME_UNIT_MILLISECOND, &msInterval); + int32_t code = + convertTimeFromPrecisionToUnit(pCreate->interval, pDb->cfg.precision, TIME_UNIT_MILLISECOND, &msInterval); if (TSDB_CODE_SUCCESS != code) { mError("sma:%s, failed to create since convert time failed: %s", smaObj.name, tstrerror(code)); return code; @@ -792,7 +790,7 @@ static int32_t mndCheckCreateSmaReq(SMCreateSmaReq *pCreate) { } static int32_t mndGetStreamNameFromSmaName(char *streamName, char *smaName) { - SName n; + SName n; int32_t code = tNameFromString(&n, smaName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); if (TSDB_CODE_SUCCESS != code) { return code; @@ -984,10 +982,10 @@ static int32_t mndSetDropSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SD } static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *pSma) { - int32_t code = -1; - SVgObj *pVgroup = NULL; - SStbObj *pStb = NULL; - STrans *pTrans = NULL; + int32_t code = -1; + SVgObj *pVgroup = NULL; + SStbObj *pStb = NULL; + STrans *pTrans = NULL; SStreamObj *pStream = NULL; pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId); @@ -1023,7 +1021,6 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p goto _OVER; } - code = mndAcquireStream(pMnode, streamName, &pStream); if (pStream == NULL || pStream->smaId != pSma->uid || code != 0) { sdbRelease(pMnode->pSdb, pStream); @@ -1124,8 +1121,8 @@ _OVER: int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { int32_t code = 0; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; while (1) { SSmaObj *pSma = NULL; @@ -1232,7 +1229,7 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp FOREACH(node, pList) { SFunctionNode *pFunc = (SFunctionNode *)node; extOffset += tsnprintf(rsp->indexExts + extOffset, sizeof(rsp->indexExts) - extOffset - 1, "%s%s", - (extOffset ? "," : ""), pFunc->functionName); + (extOffset ? "," : ""), pFunc->functionName); } *exist = true; @@ -1439,8 +1436,8 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc SName smaName = {0}; SName stbName = {0}; - char n2[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - char n3[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char n2[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char n3[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; code = tNameFromString(&smaName, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; if (TSDB_CODE_SUCCESS == code) { @@ -1448,7 +1445,7 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc STR_TO_VARSTR(n2, (char *)mndGetDbStr(pSma->db)); code = tNameFromString(&stbName, pSma->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); } - SColumnInfoData* pColInfo = NULL; + SColumnInfoData *pColInfo = NULL; if (TSDB_CODE_SUCCESS == code) { STR_TO_VARSTR(n3, (char *)tNameGetTableName(&stbName)); @@ -1540,7 +1537,7 @@ static void mndCancelRetrieveIdx(SMnode *pMnode, void *pIter) { taosMemoryFree(p); } -static void initSMAObj(SCreateTSMACxt* pCxt) { +static void initSMAObj(SCreateTSMACxt *pCxt) { memcpy(pCxt->pSma->name, pCxt->pCreateSmaReq->name, TSDB_TABLE_FNAME_LEN); memcpy(pCxt->pSma->stb, pCxt->pCreateSmaReq->stb, TSDB_TABLE_FNAME_LEN); memcpy(pCxt->pSma->db, pCxt->pDb->name, TSDB_DB_FNAME_LEN); @@ -1549,7 +1546,7 @@ static void initSMAObj(SCreateTSMACxt* pCxt) { pCxt->pSma->uid = mndGenerateUid(pCxt->pCreateSmaReq->name, TSDB_TABLE_FNAME_LEN); memcpy(pCxt->pSma->dstTbName, pCxt->targetStbFullName, TSDB_TABLE_FNAME_LEN); - pCxt->pSma->dstTbUid = 0; // not used + pCxt->pSma->dstTbUid = 0; // not used pCxt->pSma->stbUid = pCxt->pSrcStb ? pCxt->pSrcStb->uid : pCxt->pCreateSmaReq->normSourceTbUid; pCxt->pSma->dbUid = pCxt->pDb->uid; pCxt->pSma->interval = pCxt->pCreateSmaReq->interval; @@ -1598,7 +1595,7 @@ static int32_t mndCreateTSMABuildCreateStreamReq(SCreateTSMACxt *pCxt) { if (!pCxt->pCreateStreamReq->pTags) { return terrno; } - SField f = {0}; + SField f = {0}; int32_t code = 0; if (pCxt->pSrcStb) { for (int32_t idx = 0; idx < pCxt->pCreateStreamReq->numOfTags - 1; ++idx) { @@ -1626,9 +1623,9 @@ static int32_t mndCreateTSMABuildCreateStreamReq(SCreateTSMACxt *pCxt) { if (TSDB_CODE_SUCCESS == code) { // construct output cols - SNode* pNode; + SNode *pNode; FOREACH(pNode, pCxt->pProjects) { - SExprNode* pExprNode = (SExprNode*)pNode; + SExprNode *pExprNode = (SExprNode *)pNode; f.bytes = pExprNode->resType.bytes; f.type = pExprNode->resType.type; f.flags = COL_SMA_ON; @@ -1642,7 +1639,7 @@ static int32_t mndCreateTSMABuildCreateStreamReq(SCreateTSMACxt *pCxt) { return code; } -static int32_t mndCreateTSMABuildDropStreamReq(SCreateTSMACxt* pCxt) { +static int32_t mndCreateTSMABuildDropStreamReq(SCreateTSMACxt *pCxt) { tstrncpy(pCxt->pDropStreamReq->name, pCxt->streamName, TSDB_STREAM_FNAME_LEN); pCxt->pDropStreamReq->igNotExists = false; pCxt->pDropStreamReq->sql = taosStrdup(pCxt->pDropSmaReq->name); @@ -1685,14 +1682,13 @@ static int32_t mndSetUpdateDbTsmaVersionCommitLogs(SMnode *pMnode, STrans *pTran TAOS_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); } -static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { +static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt *pCxt) { int32_t code = -1; STransAction createStreamRedoAction = {0}; STransAction createStreamUndoAction = {0}; STransAction dropStbUndoAction = {0}; SMDropStbReq dropStbReq = {0}; - STrans *pTrans = - mndTransCreate(pCxt->pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TSMA, pCxt->pRpcReq, "create-tsma"); + STrans *pTrans = mndTransCreate(pCxt->pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TSMA, pCxt->pRpcReq, "create-tsma"); if (!pTrans) { code = terrno; goto _OVER; @@ -1713,7 +1709,9 @@ static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { code = terrno; goto _OVER; } - if (createStreamRedoAction.contLen != tSerializeSCMCreateStreamReq(createStreamRedoAction.pCont, createStreamRedoAction.contLen, pCxt->pCreateStreamReq)) { + if (createStreamRedoAction.contLen != tSerializeSCMCreateStreamReq(createStreamRedoAction.pCont, + createStreamRedoAction.contLen, + pCxt->pCreateStreamReq)) { mError("sma: %s, failed to create due to create stream req encode failure", pCxt->pCreateSmaReq->name); code = TSDB_CODE_INVALID_MSG; goto _OVER; @@ -1728,7 +1726,8 @@ static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { code = terrno; goto _OVER; } - if (createStreamUndoAction.contLen != tSerializeSMDropStreamReq(createStreamUndoAction.pCont, createStreamUndoAction.contLen, pCxt->pDropStreamReq)) { + if (createStreamUndoAction.contLen != + tSerializeSMDropStreamReq(createStreamUndoAction.pCont, createStreamUndoAction.contLen, pCxt->pDropStreamReq)) { mError("sma: %s, failed to create due to drop stream req encode failure", pCxt->pCreateSmaReq->name); code = TSDB_CODE_INVALID_MSG; goto _OVER; @@ -1746,7 +1745,8 @@ static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { code = terrno; goto _OVER; } - if (dropStbUndoAction.contLen != tSerializeSMDropStbReq(dropStbUndoAction.pCont, dropStbUndoAction.contLen, &dropStbReq)) { + if (dropStbUndoAction.contLen != + tSerializeSMDropStbReq(dropStbUndoAction.pCont, dropStbUndoAction.contLen, &dropStbReq)) { mError("sma: %s, failed to create due to drop stb req encode failure", pCxt->pCreateSmaReq->name); code = TSDB_CODE_INVALID_MSG; goto _OVER; @@ -1781,7 +1781,7 @@ static int32_t mndCreateTSMA(SCreateTSMACxt *pCxt) { pCxt->pSma = &sma; initSMAObj(pCxt); - SNodeList* pProjects = NULL; + SNodeList *pProjects = NULL; code = nodesStringToList(pCxt->pCreateSmaReq->expr, &pProjects); if (TSDB_CODE_SUCCESS != code) { goto _OVER; @@ -1830,8 +1830,8 @@ _OVER: TAOS_RETURN(code); } -static int32_t mndTSMAGenerateOutputName(const char* tsmaName, char* streamName, char* targetStbName) { - SName smaName; +static int32_t mndTSMAGenerateOutputName(const char *tsmaName, char *streamName, char *targetStbName) { + SName smaName; int32_t code = tNameFromString(&smaName, tsmaName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1841,17 +1841,17 @@ static int32_t mndTSMAGenerateOutputName(const char* tsmaName, char* streamName, return TSDB_CODE_SUCCESS; } -static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq) { +static int32_t mndProcessCreateTSMAReq(SRpcMsg *pReq) { #ifdef WINDOWS TAOS_RETURN(TSDB_CODE_MND_INVALID_PLATFORM); #endif - SMnode * pMnode = pReq->info.node; + SMnode *pMnode = pReq->info.node; int32_t code = -1; - SDbObj * pDb = NULL; - SStbObj * pStb = NULL; - SSmaObj * pSma = NULL; - SSmaObj * pBaseTsma = NULL; - SStreamObj * pStream = NULL; + SDbObj *pDb = NULL; + SStbObj *pStb = NULL; + SSmaObj *pSma = NULL; + SSmaObj *pBaseTsma = NULL; + SStreamObj *pStream = NULL; int64_t mTraceId = TRACE_GET_ROOTID(&pReq->info.traceId); SMCreateSmaReq createReq = {0}; @@ -1941,16 +1941,16 @@ static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq) { } SCreateTSMACxt cxt = { - .pMnode = pMnode, - .pCreateSmaReq = &createReq, - .pCreateStreamReq = NULL, - .streamName = streamName, - .targetStbFullName = streamTargetStbFullName, - .pDb = pDb, - .pRpcReq = pReq, - .pSma = NULL, - .pBaseSma = pBaseTsma, - .pSrcStb = pStb, + .pMnode = pMnode, + .pCreateSmaReq = &createReq, + .pCreateStreamReq = NULL, + .streamName = streamName, + .targetStbFullName = streamTargetStbFullName, + .pDb = pDb, + .pRpcReq = pReq, + .pSma = NULL, + .pBaseSma = pBaseTsma, + .pSrcStb = pStb, }; code = mndCreateTSMA(&cxt); @@ -1971,10 +1971,10 @@ _OVER: TAOS_RETURN(code); } -static int32_t mndDropTSMA(SCreateTSMACxt* pCxt) { - int32_t code = -1; +static int32_t mndDropTSMA(SCreateTSMACxt *pCxt) { + int32_t code = -1; STransAction dropStreamRedoAction = {0}; - STrans *pTrans = mndTransCreate(pCxt->pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TSMA, pCxt->pRpcReq, "drop-tsma"); + STrans *pTrans = mndTransCreate(pCxt->pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TSMA, pCxt->pRpcReq, "drop-tsma"); if (!pTrans) { code = terrno; goto _OVER; @@ -2021,7 +2021,8 @@ static int32_t mndDropTSMA(SCreateTSMACxt* pCxt) { code = terrno; goto _OVER; } - if (dropStbRedoAction.contLen != tSerializeSMDropStbReq(dropStbRedoAction.pCont, dropStbRedoAction.contLen, &dropStbReq)) { + if (dropStbRedoAction.contLen != + tSerializeSMDropStbReq(dropStbRedoAction.pCont, dropStbRedoAction.contLen, &dropStbReq)) { mError("tsma: %s, failedto drop due to drop stb req encode failure", pCxt->pDropSmaReq->name); code = TSDB_CODE_INVALID_MSG; goto _OVER; @@ -2044,9 +2045,9 @@ _OVER: TAOS_RETURN(code); } -static bool hasRecursiveTsmasBasedOnMe(SMnode* pMnode, const SSmaObj* pSma) { +static bool hasRecursiveTsmasBasedOnMe(SMnode *pMnode, const SSmaObj *pSma) { SSmaObj *pSmaObj = NULL; - void * pIter = NULL; + void *pIter = NULL; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void **)&pSmaObj); if (pIter == NULL) break; @@ -2060,25 +2061,25 @@ static bool hasRecursiveTsmasBasedOnMe(SMnode* pMnode, const SSmaObj* pSma) { return false; } -static int32_t mndProcessDropTSMAReq(SRpcMsg* pReq) { +static int32_t mndProcessDropTSMAReq(SRpcMsg *pReq) { int32_t code = -1; SMDropSmaReq dropReq = {0}; - SSmaObj * pSma = NULL; - SDbObj * pDb = NULL; - SMnode * pMnode = pReq->info.node; + SSmaObj *pSma = NULL; + SDbObj *pDb = NULL; + SMnode *pMnode = pReq->info.node; if (tDeserializeSMDropSmaReq(pReq->pCont, pReq->contLen, &dropReq) != TSDB_CODE_SUCCESS) { code = TSDB_CODE_INVALID_MSG; goto _OVER; } - char streamName[TSDB_TABLE_FNAME_LEN] = {0}; - char streamTargetStbFullName[TSDB_TABLE_FNAME_LEN] = {0}; + char streamName[TSDB_TABLE_FNAME_LEN] = {0}; + char streamTargetStbFullName[TSDB_TABLE_FNAME_LEN] = {0}; code = mndTSMAGenerateOutputName(dropReq.name, streamName, streamTargetStbFullName); if (TSDB_CODE_SUCCESS != code) { goto _OVER; } - SStbObj* pStb = mndAcquireStb(pMnode, streamTargetStbFullName); + SStbObj *pStb = mndAcquireStb(pMnode, streamTargetStbFullName); pSma = mndAcquireSma(pMnode, dropReq.name); if (!pSma && dropReq.igNotExists) { @@ -2113,13 +2114,13 @@ static int32_t mndProcessDropTSMAReq(SRpcMsg* pReq) { } SCreateTSMACxt cxt = { - .pDb = pDb, - .pMnode = pMnode, - .pRpcReq = pReq, - .pSma = pSma, - .streamName = streamName, - .targetStbFullName = streamTargetStbFullName, - .pDropSmaReq = &dropReq, + .pDb = pDb, + .pMnode = pMnode, + .pRpcReq = pReq, + .pSma = pSma, + .streamName = streamName, + .targetStbFullName = streamTargetStbFullName, + .pDropSmaReq = &dropReq, }; code = mndDropTSMA(&cxt); @@ -2134,10 +2135,10 @@ _OVER: } static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { - SDbObj * pDb = NULL; + SDbObj *pDb = NULL; int32_t numOfRows = 0; - SSmaObj * pSma = NULL; - SMnode * pMnode = pReq->info.node; + SSmaObj *pSma = NULL; + SMnode *pMnode = pReq->info.node; int32_t code = 0; SColumnInfoData *pColInfo; if (pShow->pIter == NULL) { @@ -2153,7 +2154,7 @@ static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo while (numOfRows < rows) { pIter->pSmaIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter->pSmaIter, (void **)&pSma); if (pIter->pSmaIter == NULL) break; - SDbObj* pSrcDb = mndAcquireDb(pMnode, pSma->db); + SDbObj *pSrcDb = mndAcquireDb(pMnode, pSma->db); if ((pDb && pSma->dbUid != pDb->uid) || !pSrcDb) { sdbRelease(pMnode->pSdb, pSma); @@ -2176,7 +2177,7 @@ static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo if (TSDB_CODE_SUCCESS == code) { STR_TO_VARSTR(db, (char *)mndGetDbStr(pSma->db)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - code = colDataSetVal(pColInfo, numOfRows, (const char*)db, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)db, false); } if (TSDB_CODE_SUCCESS == code) { @@ -2186,12 +2187,12 @@ static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo if (TSDB_CODE_SUCCESS == code) { STR_TO_VARSTR(srcTb, (char *)tNameGetTableName(&n)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - code = colDataSetVal(pColInfo, numOfRows, (const char*)srcTb, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)srcTb, false); } if (TSDB_CODE_SUCCESS == code) { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - code = colDataSetVal(pColInfo, numOfRows, (const char*)db, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)db, false); } if (TSDB_CODE_SUCCESS == code) { @@ -2200,29 +2201,29 @@ static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo if (TSDB_CODE_SUCCESS == code) { char targetTb[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(targetTb, (char*)tNameGetTableName(&n)); + STR_TO_VARSTR(targetTb, (char *)tNameGetTableName(&n)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - code = colDataSetVal(pColInfo, numOfRows, (const char*)targetTb, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)targetTb, false); } if (TSDB_CODE_SUCCESS == code) { // stream name pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - code = colDataSetVal(pColInfo, numOfRows, (const char*)smaName, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)smaName, false); } if (TSDB_CODE_SUCCESS == code) { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - code = colDataSetVal(pColInfo, numOfRows, (const char*)(&pSma->createdTime), false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)(&pSma->createdTime), false); } // interval - char interval[64 + VARSTR_HEADER_SIZE] = {0}; + char interval[64 + VARSTR_HEADER_SIZE] = {0}; int32_t len = 0; if (TSDB_CODE_SUCCESS == code) { if (!IS_CALENDAR_TIME_DURATION(pSma->intervalUnit)) { len = tsnprintf(interval + VARSTR_HEADER_SIZE, 64, "%" PRId64 "%c", pSma->interval, - getPrecisionUnit(pSrcDb->cfg.precision)); + getPrecisionUnit(pSrcDb->cfg.precision)); } else { len = tsnprintf(interval + VARSTR_HEADER_SIZE, 64, "%" PRId64 "%c", pSma->interval, pSma->intervalUnit); } @@ -2243,17 +2244,17 @@ static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo // func list len = 0; SNode *pNode = NULL, *pFunc = NULL; - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code) { code = nodesStringToNode(pSma->ast, &pNode); } if (TSDB_CODE_SUCCESS == code) { - char * start = buf + VARSTR_HEADER_SIZE; + char *start = buf + VARSTR_HEADER_SIZE; FOREACH(pFunc, ((SSelectStmt *)pNode)->pProjectionList) { if (nodeType(pFunc) == QUERY_NODE_FUNCTION) { SFunctionNode *pFuncNode = (SFunctionNode *)pFunc; if (!fmIsTSMASupportedFunc(pFuncNode->funcId)) continue; len += tsnprintf(start, TSDB_MAX_SAVED_SQL_LEN - len, "%s%s", start != buf + VARSTR_HEADER_SIZE ? "," : "", - ((SExprNode *)pFunc)->userAlias); + ((SExprNode *)pFunc)->userAlias); if (len >= TSDB_MAX_SAVED_SQL_LEN) { len = TSDB_MAX_SAVED_SQL_LEN; break; @@ -2297,7 +2298,8 @@ static void mndCancelRetrieveTSMA(SMnode *pMnode, void *pIter) { taosMemoryFree(p); } -int32_t dumpTSMAInfoFromSmaObj(const SSmaObj* pSma, const SStbObj* pDestStb, STableTSMAInfo* pInfo, const SSmaObj* pBaseTsma) { +int32_t dumpTSMAInfoFromSmaObj(const SSmaObj *pSma, const SStbObj *pDestStb, STableTSMAInfo *pInfo, + const SSmaObj *pBaseTsma) { int32_t code = 0; pInfo->interval = pSma->interval; pInfo->unit = pSma->intervalUnit; @@ -2336,7 +2338,7 @@ int32_t dumpTSMAInfoFromSmaObj(const SSmaObj* pSma, const SStbObj* pDestStb, STa SSelectStmt *pSelect = (SSelectStmt *)pNode; FOREACH(pFunc, pSelect->pProjectionList) { STableTSMAFuncInfo funcInfo = {0}; - SFunctionNode * pFuncNode = (SFunctionNode *)pFunc; + SFunctionNode *pFuncNode = (SFunctionNode *)pFunc; if (!fmIsTSMASupportedFunc(pFuncNode->funcId)) continue; funcInfo.funcId = pFuncNode->funcId; funcInfo.colId = ((SColumnNode *)pFuncNode->pParameterList->pHead->pNode)->colId; @@ -2383,9 +2385,9 @@ int32_t dumpTSMAInfoFromSmaObj(const SSmaObj* pSma, const SStbObj* pDestStb, STa } // @note remember to mndReleaseSma(*ppOut) -static int32_t mndGetDeepestBaseForTsma(SMnode* pMnode, SSmaObj* pSma, SSmaObj** ppOut) { - int32_t code = 0; - SSmaObj* pRecursiveTsma = NULL; +static int32_t mndGetDeepestBaseForTsma(SMnode *pMnode, SSmaObj *pSma, SSmaObj **ppOut) { + int32_t code = 0; + SSmaObj *pRecursiveTsma = NULL; if (pSma->baseSmaName[0]) { pRecursiveTsma = mndAcquireSma(pMnode, pSma->baseSmaName); if (!pRecursiveTsma) { @@ -2393,7 +2395,7 @@ static int32_t mndGetDeepestBaseForTsma(SMnode* pMnode, SSmaObj* pSma, SSmaObj** return TSDB_CODE_MND_SMA_NOT_EXIST; } while (pRecursiveTsma->baseSmaName[0]) { - SSmaObj* pTmpSma = pRecursiveTsma; + SSmaObj *pTmpSma = pRecursiveTsma; pRecursiveTsma = mndAcquireSma(pMnode, pTmpSma->baseSmaName); if (!pRecursiveTsma) { mError("base tsma: %s for tsma: %s not found", pTmpSma->baseSmaName, pTmpSma->name); @@ -2407,7 +2409,6 @@ static int32_t mndGetDeepestBaseForTsma(SMnode* pMnode, SSmaObj* pSma, SSmaObj** return code; } - static int32_t mndGetTSMA(SMnode *pMnode, char *tsmaFName, STableTSMAInfoRsp *rsp, bool *exist) { int32_t code = -1; SSmaObj *pSma = NULL; @@ -2450,17 +2451,17 @@ static int32_t mndGetTSMA(SMnode *pMnode, char *tsmaFName, STableTSMAInfoRsp *rs TAOS_RETURN(code); } -typedef bool (*tsmaFilter)(const SSmaObj* pSma, void* param); +typedef bool (*tsmaFilter)(const SSmaObj *pSma, void *param); -static int32_t mndGetSomeTsmas(SMnode* pMnode, STableTSMAInfoRsp* pRsp, tsmaFilter filtered, void* param, bool* exist) { - int32_t code = 0; - SSmaObj * pSma = NULL; - SSmaObj * pBaseTsma = NULL; - SSdb * pSdb = pMnode->pSdb; - void * pIter = NULL; - SStreamObj * pStream = NULL; - SStbObj * pStb = NULL; - bool shouldRetry = false; +static int32_t mndGetSomeTsmas(SMnode *pMnode, STableTSMAInfoRsp *pRsp, tsmaFilter filtered, void *param, bool *exist) { + int32_t code = 0; + SSmaObj *pSma = NULL; + SSmaObj *pBaseTsma = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SStreamObj *pStream = NULL; + SStbObj *pStb = NULL; + bool shouldRetry = false; while (1) { pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); @@ -2479,7 +2480,7 @@ static int32_t mndGetSomeTsmas(SMnode* pMnode, STableTSMAInfoRsp* pRsp, tsmaFilt } SName smaName; - char streamName[TSDB_TABLE_FNAME_LEN] = {0}; + char streamName[TSDB_TABLE_FNAME_LEN] = {0}; code = tNameFromString(&smaName, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); if (TSDB_CODE_SUCCESS != code) { sdbRelease(pSdb, pSma); @@ -2541,8 +2542,8 @@ static int32_t mndGetSomeTsmas(SMnode* pMnode, STableTSMAInfoRsp* pRsp, tsmaFilt return TSDB_CODE_SUCCESS; } -static bool tsmaTbFilter(const SSmaObj* pSma, void* param) { - const char* tbFName = param; +static bool tsmaTbFilter(const SSmaObj *pSma, void *param) { + const char *tbFName = param; return pSma->stb[0] != tbFName[0] || strcmp(pSma->stb, tbFName) != 0; } @@ -2550,7 +2551,7 @@ static int32_t mndGetTableTSMA(SMnode *pMnode, char *tbFName, STableTSMAInfoRsp return mndGetSomeTsmas(pMnode, pRsp, tsmaTbFilter, tbFName, exist); } -static bool tsmaDbFilter(const SSmaObj* pSma, void* param) { +static bool tsmaDbFilter(const SSmaObj *pSma, void *param) { uint64_t *dbUid = param; return pSma->dbUid != *dbUid; } @@ -2564,7 +2565,7 @@ static int32_t mndProcessGetTbTSMAReq(SRpcMsg *pReq) { int32_t code = -1; STableTSMAInfoReq tsmaReq = {0}; bool exist = false; - SMnode * pMnode = pReq->info.node; + SMnode *pMnode = pReq->info.node; TAOS_CHECK_GOTO(tDeserializeTableTSMAInfoReq(pReq->pCont, pReq->contLen, &tsmaReq), NULL, _OVER); @@ -2635,12 +2636,12 @@ static int32_t mkNonExistTSMAInfo(const STSMAVersion *pTsmaVer, STableTSMAInfo * int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t numOfTsmas, void **ppRsp, int32_t *pRspLen) { - int32_t code = -1; - STSMAHbRsp hbRsp = {0}; - int32_t rspLen = 0; - void * pRsp = NULL; - char tsmaFName[TSDB_TABLE_FNAME_LEN] = {0}; - STableTSMAInfo * pTsmaInfo = NULL; + int32_t code = -1; + STSMAHbRsp hbRsp = {0}; + int32_t rspLen = 0; + void *pRsp = NULL; + char tsmaFName[TSDB_TABLE_FNAME_LEN] = {0}; + STableTSMAInfo *pTsmaInfo = NULL; hbRsp.pTsmas = taosArrayInit(numOfTsmas, POINTER_BYTES); if (!hbRsp.pTsmas) { @@ -2649,13 +2650,13 @@ int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t } for (int32_t i = 0; i < numOfTsmas; ++i) { - STSMAVersion* pTsmaVer = &pTsmaVersions[i]; + STSMAVersion *pTsmaVer = &pTsmaVersions[i]; pTsmaVer->dbId = be64toh(pTsmaVer->dbId); pTsmaVer->tsmaId = be64toh(pTsmaVer->tsmaId); pTsmaVer->version = ntohl(pTsmaVer->version); snprintf(tsmaFName, sizeof(tsmaFName), "%s.%s", pTsmaVer->dbFName, pTsmaVer->name); - SSmaObj* pSma = mndAcquireSma(pMnode, tsmaFName); + SSmaObj *pSma = mndAcquireSma(pMnode, tsmaFName); if (!pSma) { code = mkNonExistTSMAInfo(pTsmaVer, &pTsmaInfo); if (code) goto _OVER; @@ -2683,7 +2684,7 @@ int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t continue; } - SStbObj* pDestStb = mndAcquireStb(pMnode, pSma->dstTbName); + SStbObj *pDestStb = mndAcquireStb(pMnode, pSma->dstTbName); if (!pDestStb) { mInfo("tsma: %s.%" PRIx64 " dest stb: %s not found, maybe dropped", tsmaFName, pTsmaVer->tsmaId, pSma->dstTbName); code = mkNonExistTSMAInfo(pTsmaVer, &pTsmaInfo); @@ -2698,7 +2699,7 @@ int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t } // dump smaObj into rsp - STableTSMAInfo * pInfo = NULL; + STableTSMAInfo *pInfo = NULL; pInfo = taosMemoryCalloc(1, sizeof(STableTSMAInfo)); if (!pInfo) { code = terrno; @@ -2707,7 +2708,7 @@ int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t goto _OVER; } - SSmaObj* pBaseSma = NULL; + SSmaObj *pBaseSma = NULL; code = mndGetDeepestBaseForTsma(pMnode, pSma, &pBaseSma); if (code == 0) code = dumpTSMAInfoFromSmaObj(pSma, pDestStb, pInfo, pBaseSma); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index eb6c326d1e..ae1afdb0db 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -627,9 +627,12 @@ static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead)); - terrno = tEncodeSVDropStbReq(&encoder, &req); + int32_t code = tEncodeSVDropStbReq(&encoder, &req); tEncoderClear(&encoder); - if (terrno != 0) return NULL; + if (code != 0) { + terrno = code; + return NULL; + } *pContLen = contLen; return pHead; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 0a107518df..cab61648e1 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2343,7 +2343,7 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { } int32_t total = taosArrayGetSize(*pReqTaskList); - if (total == numOfTasks) { // all tasks has send the reqs + if (total == numOfTasks) { // all tasks have sent the reqs int64_t checkpointId = mndStreamGenChkptId(pMnode, false); mInfo("stream:0x%" PRIx64 " all tasks req checkpoint, start checkpointId:%" PRId64, req.streamId, checkpointId); diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 4b3db28aa1..b7b2764442 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -15,6 +15,8 @@ #include "mndStream.h" #include "mndTrans.h" +#include "mndMnode.h" +#include "tmisce.h" typedef struct SFailedCheckpointInfo { int64_t streamUid; @@ -31,7 +33,7 @@ static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList); static int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info); static bool validateHbMsg(const SArray *pNodeList, int32_t vgId); static void cleanupAfterProcessHbMsg(SStreamHbMsg *pReq, SArray *pFailedChkptList, SArray *pOrphanTasks); -static void doSendHbMsgRsp(int32_t code, SRpcHandleInfo *pRpcInfo, int32_t vgId, int32_t msgId); +static void doSendHbMsgRsp(int32_t code, SRpcHandleInfo *pRpcInfo, SEpSet* pEpset, int32_t vgId, int32_t msgId); static void checkforOrphanTask(SMnode* pMnode, STaskStatusEntry* p, SArray* pOrphanTasks); void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage) { @@ -329,6 +331,8 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SArray *pFailedChkpt = NULL; SArray *pOrphanTasks = NULL; int32_t code = 0; + SDecoder decoder = {0}; + SEpSet mnodeEpset = {0}; if ((code = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { if (suspendAllStreams(pMnode, &pReq->info) < 0) { @@ -336,7 +340,6 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { } } - SDecoder decoder = {0}; tDecoderInit(&decoder, pReq->pCont, pReq->contLen); if (tDecodeStreamHbMsg(&decoder, &req) < 0) { @@ -357,13 +360,15 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { TAOS_RETURN(terrno); } + mndGetMnodeEpSet(pMnode, &mnodeEpset); + streamMutexLock(&execInfo.lock); mndInitStreamExecInfo(pMnode, &execInfo); if (!validateHbMsg(execInfo.pNodeList, req.vgId)) { mError("vgId:%d not exists in nodeList buf, discarded", req.vgId); - doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); + doSendHbMsgRsp(terrno, &pReq->info, &mnodeEpset, req.vgId, req.msgId); streamMutexUnlock(&execInfo.lock); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); @@ -383,9 +388,9 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if ((pEntry->lastHbMsgId == req.msgId) && (pEntry->lastHbMsgTs == req.ts)) { mError("vgId:%d HbMsgId:%d already handled, bh msg discard, and send HbRsp", pEntry->nodeId, req.msgId); - // return directly and after the vnode to continue to send the next HbMsg. + // return directly and allow the vnode to continue to send the next HbMsg. terrno = TSDB_CODE_SUCCESS; - doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); + doSendHbMsgRsp(terrno, &pReq->info, &mnodeEpset, req.vgId, req.msgId); streamMutexUnlock(&execInfo.lock); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); @@ -529,7 +534,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { streamMutexUnlock(&execInfo.lock); - doSendHbMsgRsp(TSDB_CODE_SUCCESS, &pReq->info, req.vgId, req.msgId); + doSendHbMsgRsp(TSDB_CODE_SUCCESS, &pReq->info, &mnodeEpset, req.vgId, req.msgId); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); return code; @@ -552,12 +557,13 @@ void cleanupAfterProcessHbMsg(SStreamHbMsg *pReq, SArray *pFailedChkptList, SArr taosArrayDestroy(pOrphanTasks); } -void doSendHbMsgRsp(int32_t code, SRpcHandleInfo *pRpcInfo, int32_t vgId, int32_t msgId) { +void doSendHbMsgRsp(int32_t code, SRpcHandleInfo *pRpcInfo, SEpSet* pMndEpset, int32_t vgId, int32_t msgId) { int32_t ret = 0; int32_t tlen = 0; void *buf = NULL; - const SMStreamHbRspMsg msg = {.msgId = msgId}; + SMStreamHbRspMsg msg = {.msgId = msgId};//, .mndEpset = *pMndEpset}; + epsetAssign(&msg.mndEpset, pMndEpset); tEncodeSize(tEncodeStreamHbRsp, &msg, tlen, ret); if (ret < 0) { diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 9720dc6b8a..5de8487ced 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -108,30 +108,94 @@ static bool checkStatusForEachReplica(SVgObj *pVgroup) { return true; } -int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray **pList) { +static int32_t mndAddSnodeInfo(SMnode *pMnode, SArray *pVgroupList) { + SSnodeObj *pObj = NULL; + void *pIter = NULL; + int32_t code = 0; + + while (1) { + pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj); + if (pIter == NULL) { + break; + } + + SNodeEntry entry = {.nodeId = SNODE_HANDLE}; + code = addEpIntoEpSet(&entry.epset, pObj->pDnode->fqdn, pObj->pDnode->port); + if (code) { + sdbRelease(pMnode->pSdb, pObj); + sdbCancelFetch(pMnode->pSdb, pIter); + mError("failed to extract epset for fqdn:%s during task vgroup snapshot", pObj->pDnode->fqdn); + return code; + } + + char buf[256] = {0}; + code = epsetToStr(&entry.epset, buf, tListLen(buf)); + if (code != 0) { // print error and continue + mError("failed to convert epset to str, code:%s", tstrerror(code)); + } + + void *p = taosArrayPush(pVgroupList, &entry); + if (p == NULL) { + code = terrno; + sdbRelease(pMnode->pSdb, pObj); + sdbCancelFetch(pMnode->pSdb, pIter); + mError("failed to put entry in vgroup list, nodeId:%d code:%s", entry.nodeId, tstrerror(code)); + return code; + } else { + mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf); + } + + sdbRelease(pMnode->pSdb, pObj); + } + + return code; +} + +static int32_t mndCheckMnodeStatus(SMnode* pMnode) { + int32_t code = 0; + ESdbStatus objStatus; + void *pIter = NULL; + SMnodeObj *pObj = NULL; + + while (1) { + pIter = sdbFetchAll(pMnode->pSdb, SDB_MNODE, pIter, (void **)&pObj, &objStatus, true); + if (pIter == NULL) { + break; + } + + if (pObj->syncState != TAOS_SYNC_STATE_LEADER && pObj->syncState != TAOS_SYNC_STATE_FOLLOWER) { + mDebug("mnode sync state:%d not leader/follower", pObj->syncState); + sdbRelease(pMnode->pSdb, pObj); + sdbCancelFetch(pMnode->pSdb, pIter); + return TSDB_CODE_FAILED; + } + + if (objStatus != SDB_STATUS_READY) { + mWarn("mnode status:%d not ready", objStatus); + sdbRelease(pMnode->pSdb, pObj); + sdbCancelFetch(pMnode->pSdb, pIter); + return TSDB_CODE_FAILED; + } + + sdbRelease(pMnode->pSdb, pObj); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t mndCheckAndAddVgroupsInfo(SMnode *pMnode, SArray *pVgroupList, bool* allReady) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; SVgObj *pVgroup = NULL; int32_t code = 0; - SArray *pVgroupList = NULL; SHashObj *pHash = NULL; - pVgroupList = taosArrayInit(4, sizeof(SNodeEntry)); - if (pVgroupList == NULL) { - mError("failed to prepare arraylist during take vgroup snapshot, code:%s", tstrerror(terrno)); - code = terrno; - goto _err; - } - pHash = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (pHash == NULL) { mError("failed to prepare hashmap during take vgroup snapshot, code:%s", tstrerror(terrno)); - code = terrno; - goto _err; + return terrno; } - *allReady = true; - while (1) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) { @@ -148,7 +212,7 @@ int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray **pList) { mError("failed to put info into hashmap during task vgroup snapshot, code:%s", tstrerror(code)); sdbRelease(pSdb, pVgroup); sdbCancelFetch(pSdb, pIter); - goto _err; // take snapshot failed, and not all ready + goto _end; // take snapshot failed, and not all ready } } else { if (*pReplica != pVgroup->replica) { @@ -158,7 +222,7 @@ int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray **pList) { } } - // if not all ready till now, no need to check the remaining vgroups. + // if not all ready till now, no need to check the remaining vgroups, // but still we need to put the info of the existed vgroups into the snapshot list if (*allReady) { *allReady = checkStatusForEachReplica(pVgroup); @@ -176,7 +240,7 @@ int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray **pList) { code = terrno; sdbRelease(pSdb, pVgroup); sdbCancelFetch(pSdb, pIter); - goto _err; + goto _end; } else { mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf); } @@ -184,51 +248,49 @@ int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray **pList) { sdbRelease(pSdb, pVgroup); } - SSnodeObj *pObj = NULL; - while (1) { - pIter = sdbFetch(pSdb, SDB_SNODE, pIter, (void **)&pObj); - if (pIter == NULL) { - break; - } +_end: + taosHashCleanup(pHash); + return code; +} - SNodeEntry entry = {.nodeId = SNODE_HANDLE}; - code = addEpIntoEpSet(&entry.epset, pObj->pDnode->fqdn, pObj->pDnode->port); - if (code) { - sdbRelease(pSdb, pObj); - sdbCancelFetch(pSdb, pIter); - mError("failed to extract epset for fqdn:%s during task vgroup snapshot", pObj->pDnode->fqdn); - goto _err; - } +int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray **pList) { + int32_t code = 0; + SArray *pVgroupList = NULL; - char buf[256] = {0}; - code = epsetToStr(&entry.epset, buf, tListLen(buf)); - if (code != 0) { // print error and continue - mError("failed to convert epset to str, code:%s", tstrerror(code)); - } + *pList = NULL; + *allReady = true; - void *p = taosArrayPush(pVgroupList, &entry); - if (p == NULL) { - code = terrno; - sdbRelease(pSdb, pObj); - sdbCancelFetch(pSdb, pIter); - mError("failed to put entry in vgroup list, nodeId:%d code:%s", entry.nodeId, tstrerror(code)); - goto _err; - } else { - mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf); - } + pVgroupList = taosArrayInit(4, sizeof(SNodeEntry)); + if (pVgroupList == NULL) { + mError("failed to prepare arraylist during take vgroup snapshot, code:%s", tstrerror(terrno)); + code = terrno; + goto _err; + } - sdbRelease(pSdb, pObj); + // 1. check for all vnodes status + code = mndCheckAndAddVgroupsInfo(pMnode, pVgroupList, allReady); + if (code) { + goto _err; + } + + // 2. add snode info + code = mndAddSnodeInfo(pMnode, pVgroupList); + if (code) { + goto _err; + } + + // 3. check for mnode status + code = mndCheckMnodeStatus(pMnode); + if (code != TSDB_CODE_SUCCESS) { + *allReady = false; } *pList = pVgroupList; - taosHashCleanup(pHash); return code; _err: *allReady = false; taosArrayDestroy(pVgroupList); - taosHashCleanup(pHash); - return code; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 63390d4772..b281ddba86 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -213,7 +213,7 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) { _OVER: (void)taosThreadRwlockUnlock(&ipWhiteMgt.rw); - taosArrayDestroyP(fqdns, (FDelete)taosMemoryFree); + taosArrayDestroyP(fqdns, NULL); if (code < 0) { mError("failed to update ip white list for user: %s at line %d since %s", user, lino, tstrerror(code)); } @@ -640,8 +640,8 @@ int32_t mndFetchAllIpWhite(SMnode *pMnode, SHashObj **ppIpWhiteTab) { } _OVER: - taosArrayDestroyP(fqdns, taosMemoryFree); - taosArrayDestroyP(pUserNames, taosMemoryFree); + taosArrayDestroyP(fqdns, NULL); + taosArrayDestroyP(pUserNames, NULL); if (code < 0) { mError("failed to fetch all ip white list at line %d since %s", lino, tstrerror(code)); @@ -1706,8 +1706,8 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate if (pCreate->isImport != 1) { taosEncryptPass_c((uint8_t *)pCreate->pass, strlen(pCreate->pass), userObj.pass); } else { - // mInfo("pCreate->pass:%s", pCreate->pass) - strncpy(userObj.pass, pCreate->pass, TSDB_PASSWORD_LEN); + // mInfo("pCreate->pass:%s", pCreate->eass) + memcpy(userObj.pass, pCreate->pass, TSDB_PASSWORD_LEN); } tstrncpy(userObj.user, pCreate->user, TSDB_USER_LEN); tstrncpy(userObj.acct, acct, TSDB_USER_LEN); @@ -1803,6 +1803,43 @@ _OVER: TAOS_RETURN(code); } +static int32_t mndCheckPasswordFmt(const char *pwd) { + int32_t len = strlen(pwd); + if (len < TSDB_PASSWORD_MIN_LEN || len > TSDB_PASSWORD_MAX_LEN) { + return -1; + } + + if (strcmp(pwd, "taosdata") == 0) { + return 0; + } + + bool charTypes[4] = {0}; + for (int32_t i = 0; i < len; ++i) { + if (taosIsBigChar(pwd[i])) { + charTypes[0] = true; + } else if (taosIsSmallChar(pwd[i])) { + charTypes[1] = true; + } else if (taosIsNumberChar(pwd[i])) { + charTypes[2] = true; + } else if (taosIsSpecialChar(pwd[i])) { + charTypes[3] = true; + } else { + return -1; + } + } + + int32_t numOfTypes = 0; + for (int32_t i = 0; i < 4; ++i) { + numOfTypes += charTypes[i]; + } + + if (numOfTypes < 3) { + return -1; + } + + return 0; +} + static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = 0; @@ -1836,7 +1873,7 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_USER_FORMAT, &lino, _OVER); } - if (createReq.pass[0] == 0) { + if (mndCheckPasswordFmt(createReq.pass) != 0) { TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_PASS_FORMAT, &lino, _OVER); } @@ -1862,13 +1899,13 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; char detail[1000] = {0}; - (void)sprintf(detail, "enable:%d, superUser:%d, sysInfo:%d, password:xxx", createReq.enable, createReq.superUser, - createReq.sysInfo); + TAOS_UNUSED(snprintf(detail, sizeof(detail), "enable:%d, superUser:%d, sysInfo:%d, password:xxx", createReq.enable, + createReq.superUser, createReq.sysInfo)); char operation[15] = {0}; if (createReq.isImport == 1) { - (void)strcpy(operation, "importUser"); + tstrncpy(operation, "importUser", sizeof(operation)); } else { - (void)strcpy(operation, "createUser"); + tstrncpy(operation, "createUser", sizeof(operation)); } auditRecord(pReq, pMnode->clusterId, operation, "", createReq.user, detail, strlen(detail)); @@ -2325,8 +2362,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_USER_FORMAT, &lino, _OVER); } - if (TSDB_ALTER_USER_PASSWD == alterReq.alterType && - (alterReq.pass[0] == 0 || strlen(alterReq.pass) >= TSDB_PASSWORD_LEN)) { + if (TSDB_ALTER_USER_PASSWD == alterReq.alterType && mndCheckPasswordFmt(alterReq.pass) != 0) { TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_PASS_FORMAT, &lino, _OVER); } @@ -2466,9 +2502,10 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) { char detail[1000] = {0}; - (void)sprintf(detail, "alterType:%s, enable:%d, superUser:%d, sysInfo:%d, createdb:%d, tabName:%s, password:xxx", - mndUserAuditTypeStr(alterReq.alterType), alterReq.enable, alterReq.superUser, alterReq.sysInfo, - alterReq.createdb ? 1 : 0, alterReq.tabName); + (void)snprintf(detail, sizeof(detail), + "alterType:%s, enable:%d, superUser:%d, sysInfo:%d, createdb:%d, tabName:%s, password:xxx", + mndUserAuditTypeStr(alterReq.alterType), alterReq.enable, alterReq.superUser, alterReq.sysInfo, + alterReq.createdb ? 1 : 0, alterReq.tabName); auditRecord(pReq, pMnode->clusterId, "alterUser", "", alterReq.user, detail, strlen(detail)); } else if (alterReq.alterType == TSDB_ALTER_USER_SUPERUSER || alterReq.alterType == TSDB_ALTER_USER_ENABLE || alterReq.alterType == TSDB_ALTER_USER_SYSINFO || alterReq.alterType == TSDB_ALTER_USER_CREATEDB) { @@ -2841,8 +2878,8 @@ static int32_t mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, i SNode *pAst = NULL; int32_t sqlLen = 0; size_t bufSz = strlen(value) + 1; - if (bufSz < 5) bufSz = 5; - TAOS_MEMORY_REALLOC(*sql, bufSz + 1); + if (bufSz < 6) bufSz = 6; + TAOS_MEMORY_REALLOC(*sql, bufSz); if (*sql == NULL) { code = terrno; goto _exit; @@ -2856,12 +2893,12 @@ static int32_t mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, i if (nodesStringToNode(value, &pAst) == 0) { if (nodesNodeToSQL(pAst, *sql, bufSz, &sqlLen) != 0) { sqlLen = 5; - (void)sprintf(*sql, "error"); + (void)snprintf(*sql, bufSz, "error"); } nodesDestroyNode(pAst); } else { sqlLen = 5; - (void)sprintf(*sql, "error"); + (void)snprintf(*sql, bufSz, "error"); } STR_WITH_MAXSIZE_TO_VARSTR((*condition), (*sql), pShow->pMeta->pSchemas[cols].bytes); diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index bfd8909a76..15b90463b6 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -57,8 +57,6 @@ TEST_F(MndTestProfile, 01_ConnectMsg) { EXPECT_NE(connectRsp.connId, 0); EXPECT_EQ(connectRsp.superUser, 1); - EXPECT_EQ(connectRsp.epSet.inUse, 0); - EXPECT_EQ(connectRsp.epSet.numOfEps, 1); EXPECT_EQ(connectRsp.epSet.eps[0].port, 9031); EXPECT_STREQ(connectRsp.epSet.eps[0].fqdn, "localhost"); diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index f6d1587bb2..eff26bf412 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -56,10 +56,12 @@ extern "C" { } #define SDB_GET_INT64(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt64, int64_t) +#define SDB_GET_FLOAT(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawFloat, float) #define SDB_GET_INT32(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt32, int32_t) #define SDB_GET_INT16(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt16, int16_t) #define SDB_GET_INT8(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt8, int8_t) #define SDB_GET_UINT8(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawUInt8, uint8_t) +#define SDB_GET_BOOL(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawBool, bool) #define SDB_GET_RESERVE(pRaw, dataPos, valLen, pos) \ { \ @@ -81,6 +83,8 @@ extern "C" { #define SDB_SET_INT16(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt16, int16_t) #define SDB_SET_INT8(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt8, int8_t) #define SDB_SET_UINT8(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawUInt8, uint8_t) +#define SDB_SET_FLOAT(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawFloat, float) +#define SDB_SET_BOOL(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawBool, bool) #define SDB_SET_BINARY(pRaw, dataPos, val, valLen, pos) \ { \ @@ -113,6 +117,7 @@ typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj); typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj); typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc); typedef int32_t (*SdbDeployFp)(SMnode *pMnode); +typedef int32_t (*SdbPrepareFp)(SMnode *pMnode); typedef int32_t (*SdbValidateFp)(SMnode *pMnode, void *pTrans, SSdbRaw *pRaw); typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw); typedef SSdbRaw *(*SdbEncodeFp)(void *pObj); @@ -162,7 +167,8 @@ typedef enum { SDB_GRANT = 26, // grant log SDB_ARBGROUP = 27, SDB_ANODE = 28, - SDB_MAX = 29 + SDB_CFG = 29, + SDB_MAX = 30 } ESdbType; typedef struct SSdbRaw { @@ -202,6 +208,7 @@ typedef struct SSdb { SdbUpdateFp updateFps[SDB_MAX]; SdbDeleteFp deleteFps[SDB_MAX]; SdbDeployFp deployFps[SDB_MAX]; + SdbPrepareFp prepareFps[SDB_MAX]; SdbEncodeFp encodeFps[SDB_MAX]; SdbDecodeFp decodeFps[SDB_MAX]; SdbValidateFp validateFps[SDB_MAX]; @@ -218,6 +225,7 @@ typedef struct { ESdbType sdbType; EKeyType keyType; SdbDeployFp deployFp; + SdbPrepareFp prepareFp; SdbEncodeFp encodeFp; SdbDecodeFp decodeFp; SdbInsertFp insertFp; @@ -265,6 +273,14 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table); */ int32_t sdbDeploy(SSdb *pSdb); +/** + * @brief prepare the initial rows of sdb. + * + * @param pSdb The sdb object. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t sdbPrepare(SSdb *pSdb); + /** * @brief Load sdb from file. * @@ -373,7 +389,7 @@ int32_t sdbGetSize(SSdb *pSdb, ESdbType type); /** * @brief get valid number of rows, removed rows are ignored */ -int32_t sdbGetValidSize(SSdb* pSdb, ESdbType type); +int32_t sdbGetValidSize(SSdb *pSdb, ESdbType type); /** * @brief Get the max id of the table, keyType of table should be INT32 @@ -407,17 +423,21 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen); void sdbFreeRaw(SSdbRaw *pRaw); int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val); int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val); +int32_t sdbSetRawBool(SSdbRaw *pRaw, int32_t dataPos, bool val); int32_t sdbSetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t val); int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val); int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val); +int32_t sdbSetRawFloat(SSdbRaw *pRaw, int32_t dataPos, float val); int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_t valLen); int32_t sdbSetRawDataLen(SSdbRaw *pRaw, int32_t dataLen); int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status); int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val); int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val); +int32_t sdbGetRawBool(SSdbRaw *pRaw, int32_t dataPos, bool *val); int32_t sdbGetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t *val); int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val); int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val); +int32_t sdbGetRawFloat(SSdbRaw *pRaw, int32_t dataPos, float *val); int32_t sdbGetRawBinary(SSdbRaw *pRaw, int32_t dataPos, char *pVal, int32_t valLen); int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver); int32_t sdbGetRawTotalSize(SSdbRaw *pRaw); diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 8d0898e8ac..6a273f9ab3 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -127,6 +127,7 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) { pSdb->deployFps[sdbType] = table.deployFp; pSdb->encodeFps[sdbType] = table.encodeFp; pSdb->decodeFps[sdbType] = table.decodeFp; + pSdb->prepareFps[sdbType] = table.prepareFp; pSdb->validateFps[sdbType] = table.validateFp; int32_t hashType = 0; diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 6122b08a0b..2d752a2aff 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -48,6 +48,26 @@ static int32_t sdbDeployData(SSdb *pSdb) { return 0; } +static int32_t sdbPrepareData(SSdb *pSdb) { + int32_t code = 0; + mInfo("start to prepare sdb"); + + for (int32_t i = SDB_MAX - 1; i >= 0; --i) { + SdbPrepareFp fp = pSdb->prepareFps[i]; + if (fp == NULL) continue; + + mInfo("start to prepare sdb:%s", sdbTableName(i)); + code = (*fp)(pSdb->pMnode); + if (code != 0) { + mError("failed to prepare sdb:%s since %s", sdbTableName(i), tstrerror(code)); + return -1; + } + } + + mInfo("sdb prepare success"); + return 0; +} + static void sdbResetData(SSdb *pSdb) { mInfo("start to reset sdb"); @@ -370,7 +390,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { opts.source = pRaw->pData; opts.result = plantContent; opts.unitLen = 16; - strncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN + 1); count = CBC_Decrypt(&opts); @@ -395,7 +415,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { code = sdbWriteWithoutFree(pSdb, pRaw); if (code != 0) { - mError("failed to read sdb file:%s since %s", file, terrstr()); + mError("failed to exec sdbWrite while read sdb file:%s since %s", file, terrstr()); goto _OVER; } } @@ -515,7 +535,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb, int32_t skip_type) { opts.source = pRaw->pData; opts.result = newData; opts.unitLen = 16; - strncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN + 1); int32_t count = CBC_Encrypt(&opts); @@ -646,6 +666,15 @@ int32_t sdbDeploy(SSdb *pSdb) { return 0; } +int32_t sdbPrepare(SSdb *pSdb) { + int32_t code = 0; + code = sdbPrepareData(pSdb); + if (code != 0) { + TAOS_RETURN(code); + } + return 0; +} + static SSdbIter *sdbCreateIter(SSdb *pSdb) { SSdbIter *pIter = taosMemoryCalloc(1, sizeof(SSdbIter)); if (pIter == NULL) { diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 3f85ccb087..17804804dc 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -76,6 +76,8 @@ const char *sdbTableName(ESdbType type) { return "arb_group"; case SDB_ANODE: return "anode"; + case SDB_CFG: + return "config"; default: return "undefine"; } @@ -259,6 +261,9 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * } int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw) { + if (pRaw->type == SDB_CFG) { + mTrace("sdb write cfg"); + } SHashObj *hash = sdbGetHash(pSdb, pRaw->type); if (hash == NULL) return terrno; @@ -444,7 +449,7 @@ void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStat void sdbCancelFetch(SSdb *pSdb, void *pIter) { if (pIter == NULL) return; - SSdbRow *pRow = *(SSdbRow **)pIter; + SSdbRow *pRow = *(SSdbRow **)pIter; mTrace("cancel fetch row:%p", pRow); SHashObj *hash = sdbGetHash(pSdb, pRow->type); if (hash == NULL) return; @@ -532,12 +537,12 @@ int64_t sdbGetTableVer(SSdb *pSdb, ESdbType type) { } bool countValid(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { - int32_t* pInt = p1; + int32_t *pInt = p1; (*pInt) += 1; return true; } -int32_t sdbGetValidSize(SSdb* pSdb, ESdbType type) { +int32_t sdbGetValidSize(SSdb *pSdb, ESdbType type) { int32_t num = 0; sdbTraverse(pSdb, type, countValid, &num, 0, 0); return num; diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index 24ac2ac462..37131300c6 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -84,6 +84,14 @@ int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val) { return 0; } +int32_t sdbSetRawBool(SSdbRaw *pRaw, int32_t dataPos, bool val) { + if (val) { + return sdbSetRawUInt8(pRaw, dataPos, 1); + } else { + return sdbSetRawUInt8(pRaw, dataPos, 0); + } +} + int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val) { int32_t code = 0; if (pRaw == NULL) { @@ -132,6 +140,22 @@ int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val) { return 0; } +int32_t sdbSetRawFloat(SSdbRaw *pRaw, int32_t dataPos, float val) { + int32_t code = 0; + if (pRaw == NULL) { + code = TSDB_CODE_INVALID_PTR; + TAOS_RETURN(code); + } + + if (dataPos + sizeof(float) > pRaw->dataLen) { + code = TSDB_CODE_SDB_INVALID_DATA_LEN; + TAOS_RETURN(code); + } + + *(int64_t *)(pRaw->pData + dataPos) = val; + return 0; +} + int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_t valLen) { int32_t code = 0; if (pRaw == NULL) { @@ -214,6 +238,21 @@ int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val) { return 0; } +int32_t sdbGetRawBool(SSdbRaw *pRaw, int32_t dataPos, bool *val) { + int32_t code = 0; + uint8_t v = 0; + code = sdbGetRawUInt8(pRaw, dataPos, &v); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (v) { + *val = true; + } else { + *val = false; + } + return TSDB_CODE_SUCCESS; +} + int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val) { int32_t code = 0; if (pRaw == NULL) { @@ -262,6 +301,22 @@ int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val) { return 0; } +int32_t sdbGetRawFloat(SSdbRaw *pRaw, int32_t dataPos, float *val) { + int32_t code = 0; + if (pRaw == NULL) { + code = TSDB_CODE_INVALID_PTR; + TAOS_RETURN(code); + } + + if (dataPos + sizeof(float) > pRaw->dataLen) { + code = TSDB_CODE_SDB_INVALID_DATA_LEN; + TAOS_RETURN(code); + } + + *val = *(int64_t *)(pRaw->pData + dataPos); + return 0; +} + int32_t sdbGetRawBinary(SSdbRaw *pRaw, int32_t dataPos, char *pVal, int32_t valLen) { int32_t code = 0; if (pRaw == NULL) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 68a7766370..11b3cdf3c6 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -341,6 +341,12 @@ struct SVnodeCfg { #define TABLE_IS_COL_COMPRESSED(FLG) (((FLG) & (TABLE_COL_COMPRESSED)) != 0) #define TABLE_SET_COL_COMPRESSED(FLG) ((FLG) |= TABLE_COL_COMPRESSED) +struct SFileSetReader; +int32_t tsdbFileSetReaderOpen(void *pVnode, struct SFileSetReader **ppReader); +int32_t tsdbFileSetReaderNext(struct SFileSetReader *pReader); +int32_t tsdbFileSetGetEntryField(struct SFileSetReader *pReader, const char *field, void *value); +void tsdbFileSetReaderClose(struct SFileSetReader **ppReader); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0080e3f7c2..29248e360a 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -250,6 +250,7 @@ void *tsdbTbDataIterDestroy(STbDataIter *pIter); void tsdbTbDataIterOpen(STbData *pTbData, STsdbRowKey *pFrom, int8_t backward, STbDataIter *pIter); bool tsdbTbDataIterNext(STbDataIter *pIter); void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj *pTableMap, int64_t *rowsNum); +int32_t tsdbMemTableSaveToCache(SMemTable *pMemTable, void *func); // STbData int32_t tsdbGetNRowsInTbData(STbData *pTbData); @@ -335,7 +336,6 @@ struct STsdbFS { typedef struct { rocksdb_t *db; rocksdb_comparator_t *my_comparator; - rocksdb_cache_t *blockcache; rocksdb_block_based_table_options_t *tableoptions; rocksdb_options_t *options; rocksdb_flushoptions_t *flushoptions; @@ -347,6 +347,7 @@ typedef struct { tb_uid_t suid; tb_uid_t uid; STSchema *pTSchema; + SArray *ctxArray; } SRocksCache; typedef struct { @@ -379,7 +380,7 @@ struct STsdb { struct { SVHashTable *ht; SArray *arr; - } * commitInfo; + } *commitInfo; }; struct TSDBKEY { @@ -1082,6 +1083,9 @@ void tsdbRemoveFile(const char *path); } \ } while (0) +int32_t tsdbInit(); +void tsdbCleanUp(); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 9a5bea33e3..cc440fd73b 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -138,6 +138,7 @@ static int32_t metaOpenImpl(SVnode *pVnode, SMeta **ppMeta, const char *metaDir, int32_t code = 0; int32_t lino; int32_t offset; + int32_t pathLen = 0; char path[TSDB_FILENAME_LEN] = {0}; char indexFullPath[128] = {0}; @@ -150,14 +151,15 @@ static int32_t metaOpenImpl(SVnode *pVnode, SMeta **ppMeta, const char *metaDir, taosRemoveDir(path); } - if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + strlen(path) + 1)) == NULL) { + pathLen = strlen(path) + 1; + if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + pathLen)) == NULL) { TSDB_CHECK_CODE(code = terrno, lino, _exit); } metaInitLock(pMeta); pMeta->path = (char *)&pMeta[1]; - strcpy(pMeta->path, path); + tstrncpy(pMeta->path, path, pathLen); int32_t ret = taosRealPath(pMeta->path, NULL, strlen(path) + 1); pMeta->pVnode = pVnode; @@ -195,7 +197,7 @@ static int32_t metaOpenImpl(SVnode *pVnode, SMeta **ppMeta, const char *metaDir, code = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0); TSDB_CHECK_CODE(code, lino, _exit); - sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); + (void)tsnprintf(indexFullPath, sizeof(indexFullPath), "%s/%s", pMeta->path, "invert"); ret = taosMkDir(indexFullPath); SIndexOpts opts = {.cacheSize = 8 * 1024 * 1024}; @@ -207,7 +209,7 @@ static int32_t metaOpenImpl(SVnode *pVnode, SMeta **ppMeta, const char *metaDir, // open pTtlMgr ("ttlv1.idx") char logPrefix[128] = {0}; - sprintf(logPrefix, "vgId:%d", TD_VID(pVnode)); + (void)tsnprintf(logPrefix, sizeof(logPrefix), "vgId:%d", TD_VID(pVnode)); code = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 87beb8842b..c19a2e3ce2 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -164,7 +164,7 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) { metaReaderClear(&mr); return code; } - strncpy(tbName, mr.me.name, TSDB_TABLE_NAME_LEN); + tstrncpy(tbName, mr.me.name, TSDB_TABLE_NAME_LEN); metaReaderClear(&mr); return 0; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index bce8bb3e29..c32d4b30b0 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -1189,7 +1189,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs (*pMetaRsp)->tableType = TSDB_CHILD_TABLE; (*pMetaRsp)->tuid = pReq->uid; (*pMetaRsp)->suid = pReq->ctb.suid; - strcpy((*pMetaRsp)->tbName, pReq->name); + tstrncpy((*pMetaRsp)->tbName, pReq->name, strlen(pReq->name) + 1); } else { ret = metaUpdateMetaRsp(pReq->uid, pReq->name, &pReq->ntb.schemaRow, *pMetaRsp); if (ret < 0) { @@ -1834,7 +1834,8 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type = pAlterTbReq->type; pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].flags = pAlterTbReq->flags; pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId = entry.ntbEntry.ncid++; - strcpy(pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].name, pAlterTbReq->colName); + tstrncpy(pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].name, pAlterTbReq->colName, + strlen(pAlterTbReq->colName) + 1); ++pMeta->pVnode->config.vndStats.numOfNTimeSeries; metaTimeSeriesNotifyCheck(pMeta); @@ -1943,7 +1944,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl goto _err; } pSchema->version++; - strcpy(pColumn->name, pAlterTbReq->colNewName); + tstrncpy(pColumn->name, pAlterTbReq->colNewName, strlen(pAlterTbReq->colNewName) + 1); break; } diff --git a/source/dnode/vnode/src/meta/metaTtl.c b/source/dnode/vnode/src/meta/metaTtl.c index 4077e9fa5d..2b67c27234 100644 --- a/source/dnode/vnode/src/meta/metaTtl.c +++ b/source/dnode/vnode/src/meta/metaTtl.c @@ -49,18 +49,20 @@ const char *ttlV1Tbname = "ttlv1.idx"; int32_t ttlMgrOpen(STtlManger **ppTtlMgr, TDB *pEnv, int8_t rollback, const char *logPrefix, int32_t flushThreshold) { int32_t code = TSDB_CODE_SUCCESS; int64_t startNs = taosGetTimestampNs(); + int32_t pathLen = 0; *ppTtlMgr = NULL; STtlManger *pTtlMgr = (STtlManger *)tdbOsCalloc(1, sizeof(*pTtlMgr)); if (pTtlMgr == NULL) TAOS_RETURN(terrno); - char *logBuffer = (char *)tdbOsCalloc(1, strlen(logPrefix) + 1); + pathLen = strlen(logPrefix) + 1; + char *logBuffer = (char *)tdbOsCalloc(1, pathLen); if (logBuffer == NULL) { tdbOsFree(pTtlMgr); TAOS_RETURN(terrno); } - (void)strcpy(logBuffer, logPrefix); + tstrncpy(logBuffer, logPrefix, pathLen); pTtlMgr->logPrefix = logBuffer; pTtlMgr->flushThreshold = flushThreshold; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 5a515b1440..7d83dbcf84 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -308,7 +308,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat if (!pStreamTask->exec.qmsg) { TAOS_RETURN(terrno); } - (void)sprintf(pStreamTask->exec.qmsg, "%s", RSMA_EXEC_TASK_FLAG); + TAOS_UNUSED(snprintf(pStreamTask->exec.qmsg, strlen(RSMA_EXEC_TASK_FLAG) + 1, "%s", RSMA_EXEC_TASK_FLAG)); pStreamTask->chkInfo.checkpointId = streamMetaGetLatestCheckpointId(pStreamTask->pMeta); tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 2cef541cdb..e6dbf5e822 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -172,9 +172,6 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } - rocksdb_cache_t *cache = rocksdb_cache_create_lru(5 * 1024 * 1024); - pTsdb->rCache.blockcache = cache; - rocksdb_block_based_table_options_t *tableoptions = rocksdb_block_based_options_create(); pTsdb->rCache.tableoptions = tableoptions; @@ -185,7 +182,6 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { rocksdb_options_set_create_if_missing(options, 1); rocksdb_options_set_comparator(options, cmp); - rocksdb_block_based_options_set_block_cache(tableoptions, cache); rocksdb_options_set_block_based_table_factory(options, tableoptions); rocksdb_options_set_info_log_level(options, 2); // WARN_LEVEL // rocksdb_options_set_inplace_update_support(options, 1); @@ -234,9 +230,15 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { pTsdb->rCache.suid = -1; pTsdb->rCache.uid = -1; pTsdb->rCache.pTSchema = NULL; + pTsdb->rCache.ctxArray = taosArrayInit(16, sizeof(SLastUpdateCtx)); + if (!pTsdb->rCache.ctxArray) { + TAOS_CHECK_GOTO(terrno, &lino, _err7); + } TAOS_RETURN(code); +_err7: + (void)taosThreadMutexDestroy(&pTsdb->rCache.writeBatchMutex); _err6: rocksdb_writebatch_destroy(writebatch); _err5: @@ -248,7 +250,6 @@ _err3: _err2: rocksdb_options_destroy(options); rocksdb_block_based_options_destroy(tableoptions); - rocksdb_cache_destroy(cache); _err: rocksdb_comparator_destroy(cmp); @@ -264,9 +265,9 @@ static void tsdbCloseRocksCache(STsdb *pTsdb) { rocksdb_writeoptions_destroy(pTsdb->rCache.writeoptions); rocksdb_options_destroy(pTsdb->rCache.options); rocksdb_block_based_options_destroy(pTsdb->rCache.tableoptions); - rocksdb_cache_destroy(pTsdb->rCache.blockcache); rocksdb_comparator_destroy(pTsdb->rCache.my_comparator); taosMemoryFree(pTsdb->rCache.pTSchema); + taosArrayDestroy(pTsdb->rCache.ctxArray); } static void rocksMayWrite(STsdb *pTsdb, bool force) { @@ -491,10 +492,283 @@ int tsdbCacheFlushDirty(const void *key, size_t klen, void *value, void *ud) { return 0; } +static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { + bool deleted = false; + while (*iSkyline > 0) { + TSDBKEY *pItemBack = (TSDBKEY *)taosArrayGet(pSkyline, *iSkyline); + TSDBKEY *pItemFront = (TSDBKEY *)taosArrayGet(pSkyline, *iSkyline - 1); + + if (key->ts > pItemBack->ts) { + return false; + } else if (key->ts >= pItemFront->ts && key->ts <= pItemBack->ts) { + if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { + // if (key->version <= pItemFront->version || key->version <= pItemBack->version) { + return true; + } else { + if (*iSkyline > 1) { + --*iSkyline; + } else { + return false; + } + } + } else { + if (*iSkyline > 1) { + --*iSkyline; + } else { + return false; + } + } + } + + return deleted; +} + +// Get next non-deleted row from imem +static TSDBROW *tsdbImemGetNextRow(STbDataIter *pTbIter, SArray *pSkyline, int64_t *piSkyline) { + int32_t code = 0; + + if (tsdbTbDataIterNext(pTbIter)) { + TSDBROW *pMemRow = tsdbTbDataIterGet(pTbIter); + TSDBKEY rowKey = TSDBROW_KEY(pMemRow); + bool deleted = tsdbKeyDeleted(&rowKey, pSkyline, piSkyline); + if (!deleted) { + return pMemRow; + } + } + + return NULL; +} + +// Get first non-deleted row from imem +static TSDBROW *tsdbImemGetFirstRow(SMemTable *imem, STbData *pIMem, STbDataIter *pTbIter, SArray *pSkyline, + int64_t *piSkyline) { + int32_t code = 0; + + tsdbTbDataIterOpen(pIMem, NULL, 1, pTbIter); + TSDBROW *pMemRow = tsdbTbDataIterGet(pTbIter); + if (pMemRow) { + // if non deleted, return the found row. + TSDBKEY rowKey = TSDBROW_KEY(pMemRow); + bool deleted = tsdbKeyDeleted(&rowKey, pSkyline, piSkyline); + if (!deleted) { + return pMemRow; + } + } else { + return NULL; + } + + // continue to find the non-deleted first row from imem, using get next row + return tsdbImemGetNextRow(pTbIter, pSkyline, piSkyline); +} + +void tsdbCacheInvalidateSchema(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int32_t sver) { + SRocksCache *pRCache = &pTsdb->rCache; + if (!pRCache->pTSchema || sver <= pTsdb->rCache.sver) return; + + if (suid > 0 && suid == pRCache->suid) { + pRCache->sver = -1; + pRCache->suid = -1; + } + if (suid == 0 && uid == pRCache->uid) { + pRCache->sver = -1; + pRCache->uid = -1; + } +} + +static int32_t tsdbUpdateSkm(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int32_t sver) { + SRocksCache *pRCache = &pTsdb->rCache; + if (pRCache->pTSchema && sver == pRCache->sver) { + if (suid > 0 && suid == pRCache->suid) { + return 0; + } + if (suid == 0 && uid == pRCache->uid) { + return 0; + } + } + + pRCache->suid = suid; + pRCache->uid = uid; + pRCache->sver = sver; + tDestroyTSchema(pRCache->pTSchema); + return metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pRCache->pTSchema); +} + +static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray *updCtxArray); + +int32_t tsdbLoadFromImem(SMemTable *imem, int64_t suid, int64_t uid) { + int32_t code = 0; + int32_t lino = 0; + STsdb *pTsdb = imem->pTsdb; + SArray *pMemDelData = NULL; + SArray *pSkyline = NULL; + int64_t iSkyline = 0; + STbDataIter tbIter = {0}; + TSDBROW *pMemRow = NULL; + STSchema *pTSchema = NULL; + SSHashObj *iColHash = NULL; + int32_t sver; + int32_t nCol; + SArray *ctxArray = pTsdb->rCache.ctxArray; + STsdbRowKey tsdbRowKey = {0}; + + STbData *pIMem = tsdbGetTbDataFromMemTable(imem, suid, uid); + + // load imem tomb data and build skyline + TAOS_CHECK_GOTO(loadMemTombData(&pMemDelData, NULL, pIMem, INT64_MAX), &lino, _exit); + + // tsdbBuildDeleteSkyline + size_t delSize = TARRAY_SIZE(pMemDelData); + if (delSize > 0) { + pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); + if (!pSkyline) { + TAOS_CHECK_EXIT(terrno); + } + + TAOS_CHECK_EXIT(tsdbBuildDeleteSkyline(pMemDelData, 0, (int32_t)(delSize - 1), pSkyline)); + iSkyline = taosArrayGetSize(pSkyline) - 1; + } + + pMemRow = tsdbImemGetFirstRow(imem, pIMem, &tbIter, pSkyline, &iSkyline); + if (!pMemRow) { + goto _exit; + } + + // iter first row to last_row/last col values to ctxArray, and mark last null col ids + sver = TSDBROW_SVERSION(pMemRow); + TAOS_CHECK_GOTO(tsdbUpdateSkm(pTsdb, suid, uid, sver), &lino, _exit); + pTSchema = pTsdb->rCache.pTSchema; + nCol = pTSchema->numOfCols; + + tsdbRowGetKey(pMemRow, &tsdbRowKey); + + STSDBRowIter iter = {0}; + TAOS_CHECK_EXIT(tsdbRowIterOpen(&iter, pMemRow, pTSchema)); + + int32_t iCol = 0; + for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal && iCol < nCol; pColVal = tsdbRowIterNext(&iter), iCol++) { + SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal}; + if (!taosArrayPush(ctxArray, &updateCtx)) { + TAOS_CHECK_EXIT(terrno); + } + + if (COL_VAL_IS_VALUE(pColVal)) { + updateCtx.lflag = LFLAG_LAST; + if (!taosArrayPush(ctxArray, &updateCtx)) { + TAOS_CHECK_EXIT(terrno); + } + } else { + if (!iColHash) { + iColHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT)); + if (iColHash == NULL) { + TAOS_CHECK_EXIT(terrno); + } + } + + if (tSimpleHashPut(iColHash, &pColVal->cid, sizeof(pColVal->cid), &pColVal->cid, sizeof(pColVal->cid))) { + TAOS_CHECK_EXIT(terrno); + } + } + } + tsdbRowClose(&iter); + + // continue to get next row to fill null last col values + pMemRow = tsdbImemGetNextRow(&tbIter, pSkyline, &iSkyline); + while (pMemRow) { + if (tSimpleHashGetSize(iColHash) == 0) { + break; + } + + sver = TSDBROW_SVERSION(pMemRow); + TAOS_CHECK_EXIT(tsdbUpdateSkm(pTsdb, suid, uid, sver)); + pTSchema = pTsdb->rCache.pTSchema; + + STsdbRowKey tsdbRowKey = {0}; + tsdbRowGetKey(pMemRow, &tsdbRowKey); + + STSDBRowIter iter = {0}; + TAOS_CHECK_EXIT(tsdbRowIterOpen(&iter, pMemRow, pTSchema)); + + int32_t iCol = 0; + for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal && iCol < nCol; pColVal = tsdbRowIterNext(&iter), iCol++) { + if (tSimpleHashGet(iColHash, &pColVal->cid, sizeof(pColVal->cid)) && COL_VAL_IS_VALUE(pColVal)) { + SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal}; + if (!taosArrayPush(ctxArray, &updateCtx)) { + TAOS_CHECK_EXIT(terrno); + } + + TAOS_CHECK_EXIT(tSimpleHashRemove(iColHash, &pColVal->cid, sizeof(pColVal->cid))); + } + } + tsdbRowClose(&iter); + + pMemRow = tsdbImemGetNextRow(&tbIter, pSkyline, &iSkyline); + } + + TAOS_CHECK_GOTO(tsdbCacheUpdate(pTsdb, suid, uid, ctxArray), &lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); + + tsdbRowClose(&iter); + } + + taosArrayClear(ctxArray); + // destroy any allocated resource + tSimpleHashCleanup(iColHash); + if (pMemDelData) { + taosArrayDestroy(pMemDelData); + } + if (pSkyline) { + taosArrayDestroy(pSkyline); + } + + TAOS_RETURN(code); +} + +static int32_t tsdbCacheUpdateFromIMem(STsdb *pTsdb) { + if (!pTsdb) return 0; + if (!pTsdb->imem) return 0; + + int32_t code = 0; + int32_t lino = 0; + SMemTable *imem = pTsdb->imem; + int32_t nTbData = imem->nTbData; + int64_t nRow = imem->nRow; + int64_t nDel = imem->nDel; + + if (nRow == 0 || nTbData == 0) return 0; + + TAOS_CHECK_EXIT(tsdbMemTableSaveToCache(imem, tsdbLoadFromImem)); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); + } else { + tsdbInfo("vgId:%d %s done, nRow:%" PRId64 " nDel:%" PRId64, TD_VID(pTsdb->pVnode), __func__, nRow, nDel); + } + + TAOS_RETURN(code); +} + int32_t tsdbCacheCommit(STsdb *pTsdb) { int32_t code = 0; - char *err = NULL; + // 0, tsdbCacheUpdateFromIMem if updateCacheBatch + // flush dirty data of lru into rocks + // 4, and update when writing if !updateCacheBatch + // 5, merge cache & mem if updateCacheBatch + + if (tsUpdateCacheBatch) { + code = tsdbCacheUpdateFromIMem(pTsdb); + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + + TAOS_RETURN(code); + } + } + + char *err = NULL; SLRUCache *pCache = pTsdb->lruCache; rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; @@ -1300,97 +1574,55 @@ _exit: TAOS_RETURN(code); } -void tsdbCacheInvalidateSchema(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int32_t sver) { - SRocksCache *pRCache = &pTsdb->rCache; - if (!pRCache->pTSchema || sver <= pTsdb->rCache.sver) return; - - if (suid > 0 && suid == pRCache->suid) { - pRCache->sver = -1; - pRCache->suid = -1; - } - if (suid == 0 && uid == pRCache->uid) { - pRCache->sver = -1; - pRCache->uid = -1; - } -} - -static int32_t tsdbUpdateSkm(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int32_t sver) { - SRocksCache *pRCache = &pTsdb->rCache; - if (pRCache->pTSchema && sver == pRCache->sver) { - if (suid > 0 && suid == pRCache->suid) { - return 0; - } - if (suid == 0 && uid == pRCache->uid) { - return 0; - } - } - - pRCache->suid = suid; - pRCache->uid = uid; - pRCache->sver = sver; - tDestroyTSchema(pRCache->pTSchema); - return metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pRCache->pTSchema); -} - int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int64_t version, int32_t nRow, SRow **aRow) { int32_t code = 0, lino = 0; // 1. prepare last - TSDBROW lRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[nRow - 1], .version = version}; - STSchema *pTSchema = NULL; - int32_t sver = TSDBROW_SVERSION(&lRow); - SArray *ctxArray = NULL; - SSHashObj *iColHash = NULL; + TSDBROW lRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[nRow - 1], .version = version}; + STSchema *pTSchema = NULL; + int32_t sver = TSDBROW_SVERSION(&lRow); + SSHashObj *iColHash = NULL; + STSDBRowIter iter = {0}; TAOS_CHECK_GOTO(tsdbUpdateSkm(pTsdb, suid, uid, sver), &lino, _exit); pTSchema = pTsdb->rCache.pTSchema; TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version}; int32_t nCol = pTSchema->numOfCols; - - ctxArray = taosArrayInit(nCol * 2, sizeof(SLastUpdateCtx)); - if (ctxArray == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); - } + SArray *ctxArray = pTsdb->rCache.ctxArray; // 1. prepare by lrow STsdbRowKey tsdbRowKey = {0}; tsdbRowGetKey(&lRow, &tsdbRowKey); - STSDBRowIter iter = {0}; TAOS_CHECK_GOTO(tsdbRowIterOpen(&iter, &lRow, pTSchema), &lino, _exit); int32_t iCol = 0; for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal && iCol < nCol; pColVal = tsdbRowIterNext(&iter), iCol++) { SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal}; if (!taosArrayPush(ctxArray, &updateCtx)) { - tsdbRowClose(&iter); TAOS_CHECK_GOTO(terrno, &lino, _exit); } if (COL_VAL_IS_VALUE(pColVal)) { updateCtx.lflag = LFLAG_LAST; if (!taosArrayPush(ctxArray, &updateCtx)) { - tsdbRowClose(&iter); TAOS_CHECK_GOTO(terrno, &lino, _exit); } } else { if (!iColHash) { iColHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)); if (iColHash == NULL) { - tsdbRowClose(&iter); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } } if (tSimpleHashPut(iColHash, &iCol, sizeof(iCol), NULL, 0)) { - tsdbRowClose(&iter); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } } } - tsdbRowClose(&iter); // 2. prepare by the other rows for (int32_t iRow = nRow - 2; iRow >= 0; --iRow) { @@ -1424,32 +1656,28 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 } } - // 3. do update - code = tsdbCacheUpdate(pTsdb, suid, uid, ctxArray); - if (code < TSDB_CODE_SUCCESS) { - tsdbTrace("vgId:%d, %s tsdbCacheUpdate failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); - } + TAOS_CHECK_GOTO(tsdbCacheUpdate(pTsdb, suid, uid, ctxArray), &lino, _exit); _exit: if (code) { tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); } - taosArrayDestroy(ctxArray); + tsdbRowClose(&iter); tSimpleHashCleanup(iColHash); + taosArrayClear(ctxArray); TAOS_RETURN(code); } int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlockData *pBlockData) { - int32_t code = 0, lino = 0; + int32_t code = 0, lino = 0; + STSDBRowIter iter = {0}; + STSchema *pTSchema = NULL; + SArray *ctxArray = NULL; TSDBROW lRow = tsdbRowFromBlockData(pBlockData, pBlockData->nRow - 1); - - STSchema *pTSchema = NULL; - int32_t sver = TSDBROW_SVERSION(&lRow); - SArray *ctxArray = NULL; + int32_t sver = TSDBROW_SVERSION(&lRow); TAOS_CHECK_RETURN(metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema)); @@ -1500,29 +1728,18 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo } // 2. prepare last row - STSDBRowIter iter = {0}; - code = tsdbRowIterOpen(&iter, &lRow, pTSchema); - if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s tsdbRowIterOpen failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); - TAOS_CHECK_GOTO(code, &lino, _exit); - } + TAOS_CHECK_GOTO(tsdbRowIterOpen(&iter, &lRow, pTSchema), &lino, _exit); for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal; pColVal = tsdbRowIterNext(&iter)) { SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal}; if (!taosArrayPush(ctxArray, &updateCtx)) { TAOS_CHECK_GOTO(terrno, &lino, _exit); } } - tsdbRowClose(&iter); - // 3. do update - code = tsdbCacheUpdate(pTsdb, suid, uid, ctxArray); - if (code != TSDB_CODE_SUCCESS) { - tsdbTrace("vgId:%d, %s tsdbCacheUpdate failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); - } + TAOS_CHECK_GOTO(tsdbCacheUpdate(pTsdb, suid, uid, ctxArray), &lino, _exit); _exit: + tsdbRowClose(&iter); taosMemoryFreeClear(pTSchema); taosArrayDestroy(ctxArray); @@ -1789,8 +2006,9 @@ _exit: TAOS_RETURN(code); } -int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) { - int32_t code = 0; +static int32_t tsdbCacheGetBatchFromLru(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, + int8_t ltype, SArray *keyArray) { + int32_t code = 0, lino = 0; SArray *remainCols = NULL; SArray *ignoreFromRocks = NULL; SLRUCache *pCache = pTsdb->lruCache; @@ -1811,6 +2029,10 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache key.lflag = (tempType & CACHESCAN_RETRIEVE_LAST) >> 3; } + if (!taosArrayPush(keyArray, &key)) { + TAOS_CHECK_EXIT(terrno); + } + LRUHandle *h = taosLRUCacheLookup(pCache, &key, ROCKS_KEY_LEN); SLastCol *pLastCol = h ? (SLastCol *)taosLRUCacheValue(pCache, h) : NULL; if (h && pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { @@ -1914,6 +2136,492 @@ _exit: TAOS_RETURN(code); } +typedef enum SMEMNEXTROWSTATES { + SMEMNEXTROW_ENTER, + SMEMNEXTROW_NEXT, +} SMEMNEXTROWSTATES; + +typedef struct SMemNextRowIter { + SMEMNEXTROWSTATES state; + STbData *pMem; // [input] + STbDataIter iter; // mem buffer skip list iterator + int64_t lastTs; +} SMemNextRowIter; + +static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, + int nCols) { + SMemNextRowIter *state = (SMemNextRowIter *)iter; + int32_t code = 0; + *pIgnoreEarlierTs = false; + switch (state->state) { + case SMEMNEXTROW_ENTER: { + if (state->pMem != NULL) { + /* + if (state->pMem->maxKey <= state->lastTs) { + *ppRow = NULL; + *pIgnoreEarlierTs = true; + + TAOS_RETURN(code); + } + */ + tsdbTbDataIterOpen(state->pMem, NULL, 1, &state->iter); + + TSDBROW *pMemRow = tsdbTbDataIterGet(&state->iter); + if (pMemRow) { + *ppRow = pMemRow; + state->state = SMEMNEXTROW_NEXT; + + TAOS_RETURN(code); + } + } + + *ppRow = NULL; + + TAOS_RETURN(code); + } + case SMEMNEXTROW_NEXT: + if (tsdbTbDataIterNext(&state->iter)) { + *ppRow = tsdbTbDataIterGet(&state->iter); + + TAOS_RETURN(code); + } else { + *ppRow = NULL; + + TAOS_RETURN(code); + } + default: + break; + } + +_err: + *ppRow = NULL; + + TAOS_RETURN(code); +} + +typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, + int nCols); +typedef int32_t (*_next_row_clear_fn_t)(void *iter); + +typedef struct { + TSDBROW *pRow; + bool stop; + bool next; + bool ignoreEarlierTs; + void *iter; + _next_row_fn_t nextRowFn; + _next_row_clear_fn_t nextRowClearFn; +} TsdbNextRowState; + +typedef struct { + SArray *pMemDelData; + SArray *pSkyline; + int64_t iSkyline; + SBlockIdx idx; + SMemNextRowIter memState; + SMemNextRowIter imemState; + TSDBROW memRow, imemRow; + TsdbNextRowState input[2]; + SCacheRowsReader *pr; + STsdb *pTsdb; +} MemNextRowIter; + +static int32_t memRowIterOpen(MemNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, + STsdbReadSnap *pReadSnap, SCacheRowsReader *pr) { + int32_t code = 0, lino = 0; + + STbData *pMem = NULL; + if (pReadSnap->pMem) { + pMem = tsdbGetTbDataFromMemTable(pReadSnap->pMem, suid, uid); + } + + STbData *pIMem = NULL; + if (pReadSnap->pIMem) { + pIMem = tsdbGetTbDataFromMemTable(pReadSnap->pIMem, suid, uid); + } + + pIter->pTsdb = pTsdb; + + pIter->pMemDelData = NULL; + + TAOS_CHECK_GOTO(loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer), &lino, _exit); + + pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; + + pIter->input[0] = (TsdbNextRowState){&pIter->memRow, true, false, false, &pIter->memState, getNextRowFromMem, NULL}; + pIter->input[1] = (TsdbNextRowState){&pIter->imemRow, true, false, false, &pIter->imemState, getNextRowFromMem, NULL}; + + if (pMem) { + pIter->memState.pMem = pMem; + pIter->memState.state = SMEMNEXTROW_ENTER; + pIter->input[0].stop = false; + pIter->input[0].next = true; + } + + if (pIMem) { + pIter->imemState.pMem = pIMem; + pIter->imemState.state = SMEMNEXTROW_ENTER; + pIter->input[1].stop = false; + pIter->input[1].next = true; + } + + pIter->pr = pr; + +_exit: + if (code) { + tsdbError("tsdb/cache: %s failed at line %d since %s.", __func__, lino, tstrerror(code)); + } + + TAOS_RETURN(code); +} + +static void memRowIterClose(MemNextRowIter *pIter) { + for (int i = 0; i < 2; ++i) { + if (pIter->input[i].nextRowClearFn) { + (void)pIter->input[i].nextRowClearFn(pIter->input[i].iter); + } + } + + if (pIter->pSkyline) { + taosArrayDestroy(pIter->pSkyline); + } + + if (pIter->pMemDelData) { + taosArrayDestroy(pIter->pMemDelData); + } +} + +static void freeTableInfoFunc(void *param) { + void **p = (void **)param; + taosMemoryFreeClear(*p); +} + +static STableLoadInfo *getTableLoadInfo(SCacheRowsReader *pReader, uint64_t uid) { + if (!pReader->pTableMap) { + pReader->pTableMap = tSimpleHashInit(pReader->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); + if (!pReader->pTableMap) { + return NULL; + } + + tSimpleHashSetFreeFp(pReader->pTableMap, freeTableInfoFunc); + } + + STableLoadInfo *pInfo = NULL; + STableLoadInfo **ppInfo = tSimpleHashGet(pReader->pTableMap, &uid, sizeof(uid)); + if (!ppInfo) { + pInfo = taosMemoryCalloc(1, sizeof(STableLoadInfo)); + if (pInfo) { + if (tSimpleHashPut(pReader->pTableMap, &uid, sizeof(uint64_t), &pInfo, POINTER_BYTES)) { + return NULL; + } + } + + return pInfo; + } + + return *ppInfo; +} + +static TSDBROW *memRowIterGet(MemNextRowIter *pIter, bool isLast, int16_t *aCols, int nCols) { + int32_t code = 0, lino = 0; + + for (;;) { + for (int i = 0; i < 2; ++i) { + if (pIter->input[i].next && !pIter->input[i].stop) { + TAOS_CHECK_GOTO(pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, + &pIter->input[i].ignoreEarlierTs, isLast, aCols, nCols), + &lino, _exit); + + if (pIter->input[i].pRow == NULL) { + pIter->input[i].stop = true; + pIter->input[i].next = false; + } + } + } + + if (pIter->input[0].stop && pIter->input[1].stop) { + return NULL; + } + + TSDBROW *max[2] = {0}; + int iMax[2] = {-1, -1}; + int nMax = 0; + SRowKey maxKey = {.ts = TSKEY_MIN}; + + for (int i = 0; i < 2; ++i) { + if (!pIter->input[i].stop && pIter->input[i].pRow != NULL) { + STsdbRowKey tsdbRowKey = {0}; + tsdbRowGetKey(pIter->input[i].pRow, &tsdbRowKey); + + // merging & deduplicating on client side + int c = tRowKeyCompare(&maxKey, &tsdbRowKey.key); + if (c <= 0) { + if (c < 0) { + nMax = 0; + maxKey = tsdbRowKey.key; + } + + iMax[nMax] = i; + max[nMax++] = pIter->input[i].pRow; + } + pIter->input[i].next = false; + } + } + + TSDBROW *merge[2] = {0}; + int iMerge[2] = {-1, -1}; + int nMerge = 0; + for (int i = 0; i < nMax; ++i) { + TSDBKEY maxKey1 = TSDBROW_KEY(max[i]); + + if (!pIter->pSkyline) { + pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); + TSDB_CHECK_NULL(pIter->pSkyline, code, lino, _exit, terrno); + + uint64_t uid = pIter->idx.uid; + STableLoadInfo *pInfo = getTableLoadInfo(pIter->pr, uid); + TSDB_CHECK_NULL(pInfo, code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); + + if (pInfo->pTombData == NULL) { + pInfo->pTombData = taosArrayInit(4, sizeof(SDelData)); + TSDB_CHECK_NULL(pInfo->pTombData, code, lino, _exit, terrno); + } + + if (!taosArrayAddAll(pInfo->pTombData, pIter->pMemDelData)) { + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } + + size_t delSize = TARRAY_SIZE(pInfo->pTombData); + if (delSize > 0) { + code = tsdbBuildDeleteSkyline(pInfo->pTombData, 0, (int32_t)(delSize - 1), pIter->pSkyline); + TAOS_CHECK_GOTO(code, &lino, _exit); + } + pIter->iSkyline = taosArrayGetSize(pIter->pSkyline) - 1; + } + + bool deleted = tsdbKeyDeleted(&maxKey1, pIter->pSkyline, &pIter->iSkyline); + if (!deleted) { + iMerge[nMerge] = iMax[i]; + merge[nMerge++] = max[i]; + } + + pIter->input[iMax[i]].next = deleted; + } + + if (nMerge > 0) { + pIter->input[iMerge[0]].next = true; + + return merge[0]; + } + } + +_exit: + if (code) { + tsdbError("tsdb/cache: %s failed at line %d since %s.", __func__, lino, tstrerror(code)); + } + + return NULL; +} + +static int32_t cloneTSchema(STSchema *pSrc, STSchema **ppDst) { + int32_t len = sizeof(STSchema) + sizeof(STColumn) * pSrc->numOfCols; + *ppDst = taosMemoryMalloc(len); + if (NULL == *ppDst) { + TAOS_RETURN(terrno); + } + memcpy(*ppDst, pSrc, len); + + TAOS_RETURN(TSDB_CODE_SUCCESS); +} + +static int32_t updateTSchema(int32_t sversion, SCacheRowsReader *pReader, uint64_t uid) { + if (NULL == pReader->pCurrSchema && sversion == pReader->pSchema->version) { + TAOS_RETURN(cloneTSchema(pReader->pSchema, &pReader->pCurrSchema)); + } + + if (NULL != pReader->pCurrSchema && sversion == pReader->pCurrSchema->version) { + TAOS_RETURN(TSDB_CODE_SUCCESS); + } + + taosMemoryFreeClear(pReader->pCurrSchema); + TAOS_RETURN( + metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, sversion, &pReader->pCurrSchema)); +} + +static int32_t tsdbCacheGetBatchFromMem(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, + SArray *keyArray) { + int32_t code = 0; + int32_t lino = 0; + STSchema *pTSchema = pr->pSchema; + SLRUCache *pCache = pTsdb->lruCache; + SArray *pCidList = pr->pCidList; + int numKeys = TARRAY_SIZE(pCidList); + MemNextRowIter iter = {0}; + SSHashObj *iColHash = NULL; + + // 1, get from mem, imem filtered with delete info + TAOS_CHECK_EXIT(memRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->info.suid, pr->pReadSnap, pr)); + + TSDBROW *pRow = memRowIterGet(&iter, false, NULL, 0); + if (!pRow) { + goto _exit; + } + + int32_t sversion = TSDBROW_SVERSION(pRow); + if (sversion != -1) { + TAOS_CHECK_EXIT(updateTSchema(sversion, pr, uid)); + + pTSchema = pr->pCurrSchema; + } + int32_t nCol = pTSchema->numOfCols; + + STsdbRowKey rowKey = {0}; + tsdbRowGetKey(pRow, &rowKey); + + STSDBRowIter rowIter = {0}; + TAOS_CHECK_EXIT(tsdbRowIterOpen(&rowIter, pRow, pTSchema)); + + int32_t iCol = 0, jCol = 0, jnCol = TARRAY_SIZE(pLastArray); + for (SColVal *pColVal = tsdbRowIterNext(&rowIter); pColVal && iCol < nCol && jCol < jnCol;) { + SLastCol *pTargetCol = &((SLastCol *)TARRAY_DATA(pLastArray))[jCol]; + if (pColVal->cid < pTargetCol->colVal.cid) { + pColVal = tsdbRowIterNext(&rowIter), ++iCol; + + continue; + } + if (pColVal->cid > pTargetCol->colVal.cid) { + break; + } + + int32_t cmp_res = tRowKeyCompare(&pTargetCol->rowKey, &rowKey.key); + if (!IS_LAST_KEY(((SLastKey *)TARRAY_DATA(keyArray))[jCol])) { + if (cmp_res < 0 || (cmp_res == 0 && !COL_VAL_IS_NONE(pColVal))) { + SLastCol lastCol = {.rowKey = rowKey.key, .colVal = *pColVal, .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_VALID}; + TAOS_CHECK_EXIT(tsdbCacheReallocSLastCol(&lastCol, NULL)); + + tsdbCacheFreeSLastColItem(pTargetCol); + taosArraySet(pLastArray, jCol, &lastCol); + } + } else { + if (COL_VAL_IS_VALUE(pColVal)) { + if (cmp_res <= 0) { + SLastCol lastCol = { + .rowKey = rowKey.key, .colVal = *pColVal, .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_VALID}; + TAOS_CHECK_EXIT(tsdbCacheReallocSLastCol(&lastCol, NULL)); + + tsdbCacheFreeSLastColItem(pTargetCol); + taosArraySet(pLastArray, jCol, &lastCol); + } + } else { + if (!iColHash) { + iColHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT)); + if (iColHash == NULL) { + TAOS_CHECK_EXIT(terrno); + } + } + + if (tSimpleHashPut(iColHash, &pColVal->cid, sizeof(pColVal->cid), &jCol, sizeof(jCol))) { + TAOS_CHECK_EXIT(terrno); + } + } + } + + ++jCol; + + if (jCol < jnCol && ((SLastCol *)TARRAY_DATA(pLastArray))[jCol].colVal.cid > pColVal->cid) { + pColVal = tsdbRowIterNext(&rowIter), ++iCol; + } + } + tsdbRowClose(&rowIter); + + if (iColHash && tSimpleHashGetSize(iColHash) > 0) { + pRow = memRowIterGet(&iter, false, NULL, 0); + while (pRow) { + if (tSimpleHashGetSize(iColHash) == 0) { + break; + } + + sversion = TSDBROW_SVERSION(pRow); + if (sversion != -1) { + TAOS_CHECK_EXIT(updateTSchema(sversion, pr, uid)); + + pTSchema = pr->pCurrSchema; + } + nCol = pTSchema->numOfCols; + + STsdbRowKey tsdbRowKey = {0}; + tsdbRowGetKey(pRow, &tsdbRowKey); + + STSDBRowIter rowIter = {0}; + TAOS_CHECK_EXIT(tsdbRowIterOpen(&rowIter, pRow, pTSchema)); + + iCol = 0; + for (SColVal *pColVal = tsdbRowIterNext(&rowIter); pColVal && iCol < nCol; + pColVal = tsdbRowIterNext(&rowIter), iCol++) { + int32_t *pjCol = tSimpleHashGet(iColHash, &pColVal->cid, sizeof(pColVal->cid)); + if (pjCol && COL_VAL_IS_VALUE(pColVal)) { + SLastCol *pTargetCol = &((SLastCol *)TARRAY_DATA(pLastArray))[*pjCol]; + + int32_t cmp_res = tRowKeyCompare(&pTargetCol->rowKey, &tsdbRowKey.key); + if (cmp_res <= 0) { + SLastCol lastCol = { + .rowKey = tsdbRowKey.key, .colVal = *pColVal, .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_VALID}; + TAOS_CHECK_EXIT(tsdbCacheReallocSLastCol(&lastCol, NULL)); + + tsdbCacheFreeSLastColItem(pTargetCol); + taosArraySet(pLastArray, *pjCol, &lastCol); + } + + TAOS_CHECK_EXIT(tSimpleHashRemove(iColHash, &pColVal->cid, sizeof(pColVal->cid))); + } + } + tsdbRowClose(&rowIter); + + pRow = memRowIterGet(&iter, false, NULL, 0); + } + } + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); + + tsdbRowClose(&rowIter); + } + + tSimpleHashCleanup(iColHash); + + memRowIterClose(&iter); + + TAOS_RETURN(code); +} + +int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) { + int32_t code = 0; + int32_t lino = 0; + + SArray *keyArray = taosArrayInit(16, sizeof(SLastKey)); + if (!keyArray) { + TAOS_CHECK_EXIT(terrno); + } + + TAOS_CHECK_EXIT(tsdbCacheGetBatchFromLru(pTsdb, uid, pLastArray, pr, ltype, keyArray)); + + if (tsUpdateCacheBatch) { + TAOS_CHECK_EXIT(tsdbCacheGetBatchFromMem(pTsdb, uid, pLastArray, pr, keyArray)); + } + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); + } + + if (keyArray) { + taosArrayDestroy(keyArray); + } + + TAOS_RETURN(code); +} + int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) { int32_t code = 0, lino = 0; // fetch schema @@ -2159,37 +2867,6 @@ static int32_t getTableDelDataFromTbData(STbData *pTbData, SArray *aDelData) { TAOS_RETURN(code); } -static void freeTableInfoFunc(void *param) { - void **p = (void **)param; - taosMemoryFreeClear(*p); -} - -static STableLoadInfo *getTableLoadInfo(SCacheRowsReader *pReader, uint64_t uid) { - if (!pReader->pTableMap) { - pReader->pTableMap = tSimpleHashInit(pReader->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); - if (!pReader->pTableMap) { - return NULL; - } - - tSimpleHashSetFreeFp(pReader->pTableMap, freeTableInfoFunc); - } - - STableLoadInfo *pInfo = NULL; - STableLoadInfo **ppInfo = tSimpleHashGet(pReader->pTableMap, &uid, sizeof(uid)); - if (!ppInfo) { - pInfo = taosMemoryCalloc(1, sizeof(STableLoadInfo)); - if (pInfo) { - if (tSimpleHashPut(pReader->pTableMap, &uid, sizeof(uint64_t), &pInfo, POINTER_BYTES)) { - return NULL; - } - } - - return pInfo; - } - - return *ppInfo; -} - static uint64_t *getUidList(SCacheRowsReader *pReader) { if (!pReader->uidList) { int32_t numOfTables = pReader->numOfTables; @@ -2774,114 +3451,6 @@ _err: TAOS_RETURN(code); } -typedef enum SMEMNEXTROWSTATES { - SMEMNEXTROW_ENTER, - SMEMNEXTROW_NEXT, -} SMEMNEXTROWSTATES; - -typedef struct SMemNextRowIter { - SMEMNEXTROWSTATES state; - STbData *pMem; // [input] - STbDataIter iter; // mem buffer skip list iterator - int64_t lastTs; -} SMemNextRowIter; - -static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, - int nCols) { - SMemNextRowIter *state = (SMemNextRowIter *)iter; - int32_t code = 0; - *pIgnoreEarlierTs = false; - switch (state->state) { - case SMEMNEXTROW_ENTER: { - if (state->pMem != NULL) { - /* - if (state->pMem->maxKey <= state->lastTs) { - *ppRow = NULL; - *pIgnoreEarlierTs = true; - - TAOS_RETURN(code); - } - */ - tsdbTbDataIterOpen(state->pMem, NULL, 1, &state->iter); - - TSDBROW *pMemRow = tsdbTbDataIterGet(&state->iter); - if (pMemRow) { - *ppRow = pMemRow; - state->state = SMEMNEXTROW_NEXT; - - TAOS_RETURN(code); - } - } - - *ppRow = NULL; - - TAOS_RETURN(code); - } - case SMEMNEXTROW_NEXT: - if (tsdbTbDataIterNext(&state->iter)) { - *ppRow = tsdbTbDataIterGet(&state->iter); - - TAOS_RETURN(code); - } else { - *ppRow = NULL; - - TAOS_RETURN(code); - } - default: - break; - } - -_err: - *ppRow = NULL; - - TAOS_RETURN(code); -} - -static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { - bool deleted = false; - while (*iSkyline > 0) { - TSDBKEY *pItemBack = (TSDBKEY *)taosArrayGet(pSkyline, *iSkyline); - TSDBKEY *pItemFront = (TSDBKEY *)taosArrayGet(pSkyline, *iSkyline - 1); - - if (key->ts > pItemBack->ts) { - return false; - } else if (key->ts >= pItemFront->ts && key->ts <= pItemBack->ts) { - if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { - // if (key->version <= pItemFront->version || key->version <= pItemBack->version) { - return true; - } else { - if (*iSkyline > 1) { - --*iSkyline; - } else { - return false; - } - } - } else { - if (*iSkyline > 1) { - --*iSkyline; - } else { - return false; - } - } - } - - return deleted; -} - -typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, - int nCols); -typedef int32_t (*_next_row_clear_fn_t)(void *iter); - -typedef struct { - TSDBROW *pRow; - bool stop; - bool next; - bool ignoreEarlierTs; - void *iter; - _next_row_fn_t nextRowFn; - _next_row_clear_fn_t nextRowClearFn; -} TsdbNextRowState; - typedef struct CacheNextRowIter { SArray *pMemDelData; SArray *pSkyline; @@ -3160,7 +3729,6 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI } _err: - if (code) { tsdbError("tsdb/cache: %s failed at line %d since %s.", __func__, lino, tstrerror(code)); } @@ -3187,31 +3755,6 @@ static int32_t initLastColArrayPartial(STSchema *pTSchema, SArray **ppColArray, TAOS_RETURN(TSDB_CODE_SUCCESS); } -static int32_t cloneTSchema(STSchema *pSrc, STSchema **ppDst) { - int32_t len = sizeof(STSchema) + sizeof(STColumn) * pSrc->numOfCols; - *ppDst = taosMemoryMalloc(len); - if (NULL == *ppDst) { - TAOS_RETURN(terrno); - } - memcpy(*ppDst, pSrc, len); - - TAOS_RETURN(TSDB_CODE_SUCCESS); -} - -static int32_t updateTSchema(int32_t sversion, SCacheRowsReader *pReader, uint64_t uid) { - if (NULL == pReader->pCurrSchema && sversion == pReader->pSchema->version) { - TAOS_RETURN(cloneTSchema(pReader->pSchema, &pReader->pCurrSchema)); - } - - if (NULL != pReader->pCurrSchema && sversion == pReader->pCurrSchema->version) { - TAOS_RETURN(TSDB_CODE_SUCCESS); - } - - taosMemoryFreeClear(pReader->pCurrSchema); - TAOS_RETURN( - metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, sversion, &pReader->pCurrSchema)); -} - static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols, int nCols, int16_t *slotIds) { int32_t code = 0, lino = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 0c9d9e56cf..9a7cdca8f7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -396,7 +396,7 @@ static int32_t tsdbFSAddEntryToFileObjHash(STFileHash *hash, const char *fname) STFileHashEntry *entry = taosMemoryMalloc(sizeof(*entry)); if (entry == NULL) return terrno; - strncpy(entry->fname, fname, TSDB_FILENAME_LEN); + tstrncpy(entry->fname, fname, TSDB_FILENAME_LEN); uint32_t idx = MurmurHash3_32(fname, strlen(fname)) % hash->numBucket; @@ -946,7 +946,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { arg->tsdb = fs->tsdb; arg->fid = fset->fid; - code = vnodeAsync(&fset->channel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg, NULL); + code = vnodeAsync(&fset->channel, EVA_PRIORITY_HIGH, tsdbMerge, taosAutoMemoryFree, arg, NULL); TSDB_CHECK_CODE(code, lino, _exit); fset->mergeScheduled = true; } @@ -1239,3 +1239,146 @@ void tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid) { } } } + +struct SFileSetReader { + STsdb *pTsdb; + STFileSet *pFileSet; + int32_t fid; + int64_t startTime; + int64_t endTime; + int64_t lastCompactTime; + int64_t totalSize; +}; + +int32_t tsdbFileSetReaderOpen(void *pVnode, struct SFileSetReader **ppReader) { + if (pVnode == NULL || ppReader == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + STsdb *pTsdb = ((SVnode *)pVnode)->pTsdb; + + (*ppReader) = taosMemoryCalloc(1, sizeof(struct SFileSetReader)); + if (*ppReader == NULL) { + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, __LINE__, + tstrerror(terrno)); + return terrno; + } + + (*ppReader)->pTsdb = pTsdb; + (*ppReader)->fid = INT32_MIN; + (*ppReader)->pFileSet = NULL; + + return TSDB_CODE_SUCCESS; +} + +static int32_t tsdbFileSetReaderNextNoLock(struct SFileSetReader *pReader) { + STsdb *pTsdb = pReader->pTsdb; + int32_t code = TSDB_CODE_SUCCESS; + + tsdbTFileSetClear(&pReader->pFileSet); + + STFileSet *fset = &(STFileSet){ + .fid = pReader->fid, + }; + + STFileSet **fsetPtr = TARRAY2_SEARCH(pReader->pTsdb->pFS->fSetArr, &fset, tsdbTFileSetCmprFn, TD_GT); + if (fsetPtr == NULL) { + pReader->fid = INT32_MAX; + return TSDB_CODE_NOT_FOUND; + } + + // ref file set + code = tsdbTFileSetInitRef(pReader->pTsdb, *fsetPtr, &pReader->pFileSet); + if (code) return code; + + // get file set details + pReader->fid = pReader->pFileSet->fid; + tsdbFidKeyRange(pReader->fid, pTsdb->keepCfg.days, pTsdb->keepCfg.precision, &pReader->startTime, &pReader->endTime); + pReader->lastCompactTime = 0; // TODO + pReader->totalSize = 0; + for (int32_t i = 0; i < TSDB_FTYPE_MAX; i++) { + STFileObj *fobj = pReader->pFileSet->farr[i]; + if (fobj) { + pReader->totalSize += fobj->f->size; + } + } + SSttLvl *lvl; + TARRAY2_FOREACH(pReader->pFileSet->lvlArr, lvl) { + STFileObj *fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { pReader->totalSize += fobj->f->size; } + } + + return code; +} + +int32_t tsdbFileSetReaderNext(struct SFileSetReader *pReader) { + int32_t code = TSDB_CODE_SUCCESS; + (void)taosThreadMutexLock(&pReader->pTsdb->mutex); + code = tsdbFileSetReaderNextNoLock(pReader); + (void)taosThreadMutexUnlock(&pReader->pTsdb->mutex); + return code; +} + +int32_t tsdbFileSetGetEntryField(struct SFileSetReader *pReader, const char *field, void *value) { + const char *fieldName; + + if (pReader->fid == INT32_MIN || pReader->fid == INT32_MAX) { + return TSDB_CODE_INVALID_PARA; + } + + fieldName = "fileset_id"; + if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { + *(int32_t *)value = pReader->fid; + return TSDB_CODE_SUCCESS; + } + + fieldName = "start_time"; + if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { + *(int64_t *)value = pReader->startTime; + return TSDB_CODE_SUCCESS; + } + + fieldName = "end_time"; + if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { + *(int64_t *)value = pReader->endTime; + return TSDB_CODE_SUCCESS; + } + + fieldName = "total_size"; + if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { + *(int64_t *)value = pReader->totalSize; + return TSDB_CODE_SUCCESS; + } + + fieldName = "last_compact_time"; + if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { + *(int64_t *)value = pReader->lastCompactTime; + return TSDB_CODE_SUCCESS; + } + + fieldName = "should_compact"; + if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { + *(char *)value = 0; // TODO + return TSDB_CODE_SUCCESS; + } + + fieldName = "details"; + if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { + // TODO + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_INVALID_PARA; +} + +void tsdbFileSetReaderClose(struct SFileSetReader **ppReader) { + if (ppReader == NULL || *ppReader == NULL) { + return; + } + + tsdbTFileSetClear(&(*ppReader)->pFileSet); + taosMemoryFree(*ppReader); + + *ppReader = NULL; + return; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 5b26d17519..aaf48924e5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -325,6 +325,27 @@ void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj *pTableMap, int64_t * taosRUnLockLatch(&pMemTable->latch); } +typedef int32_t (*__tsdb_cache_update)(SMemTable *imem, int64_t suid, int64_t uid); + +int32_t tsdbMemTableSaveToCache(SMemTable *pMemTable, void *func) { + int32_t code = 0; + __tsdb_cache_update cb = (__tsdb_cache_update)func; + + for (int32_t i = 0; i < pMemTable->nBucket; ++i) { + STbData *pTbData = pMemTable->aBucket[i]; + while (pTbData) { + code = (*cb)(pMemTable, pTbData->suid, pTbData->uid); + if (code) { + TAOS_RETURN(code); + } + + pTbData = pTbData->next; + } + } + + return code; +} + static int32_t tsdbMemTableRehash(SMemTable *pMemTable) { int32_t code = 0; @@ -659,7 +680,7 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, pTbData->maxKey = key.key.ts; } - if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) { + if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config) && !tsUpdateCacheBatch) { if (tsdbCacheColFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, pBlockData) != 0) { tsdbError("vgId:%d, failed to update cache data from table suid:%" PRId64 " uid:%" PRId64 " at version %" PRId64, TD_VID(pMemTable->pTsdb->pVnode), pTbData->suid, pTbData->uid, version); @@ -721,7 +742,7 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, if (key.key.ts >= pTbData->maxKey) { pTbData->maxKey = key.key.ts; } - if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) { + if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config) && !tsUpdateCacheBatch) { TAOS_UNUSED(tsdbCacheRowFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, version, nRow, aRow)); } diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index c1f8f45d7e..b2e4621878 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -18,6 +18,22 @@ extern int32_t tsdbOpenCompMonitor(STsdb *tsdb); extern void tsdbCloseCompMonitor(STsdb *tsdb); +extern int32_t tsdbInitCompact(); +extern void tsdbCleanupCompact(); + +int32_t tsdbInit() { +#ifdef TD_ENTERPRISE + return tsdbInitCompact(); +#endif + return 0; +} + +void tsdbCleanUp() { +#ifdef TD_ENTERPRISE + tsdbCleanupCompact(); +#endif + return; +} void tsdbSetKeepCfg(STsdb *pTsdb, STsdbCfg *pCfg) { STsdbKeepCfg *pKeepCfg = &pTsdb->keepCfg; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 5f77d03efc..df2279d3fe 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -150,7 +150,7 @@ _end: void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) { if (pBuf == NULL) return; if (pBuf->pData != NULL) { - taosArrayDestroyP(pBuf->pData, (FDelete)taosMemoryFree); + taosArrayDestroyP(pBuf->pData, NULL); pBuf->pData = NULL; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 53e1c57f14..bf79b2482d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -104,7 +104,7 @@ int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppF } pFD->path = (char *)&pFD[1]; - strcpy(pFD->path, path); + tstrncpy(pFD->path, path, strlen(path) + 1); pFD->szPage = szPage; pFD->flag = flag; pFD->szPage = szPage; @@ -174,8 +174,7 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e opts.source = pFD->pBuf + count; opts.result = PacketData; opts.unitLen = 128; - // strncpy(opts.key, tsEncryptKey, 16); - strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN + 1); NewLen = CBC_Encrypt(&opts); @@ -248,8 +247,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor opts.source = pFD->pBuf + count; opts.result = PacketData; opts.unitLen = 128; - // strncpy(opts.key, tsEncryptKey, 16); - strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN + 1); NewLen = CBC_Decrypt(&opts); diff --git a/source/dnode/vnode/src/vnd/vnodeAsync.c b/source/dnode/vnode/src/vnd/vnodeAsync.c index 9e4fbd84a9..424ed0f325 100644 --- a/source/dnode/vnode/src/vnd/vnodeAsync.c +++ b/source/dnode/vnode/src/vnd/vnodeAsync.c @@ -330,7 +330,7 @@ static int32_t vnodeAsyncInit(SVAsync **async, const char *label) { return terrno; } - strcpy((char *)((*async) + 1), label); + tstrncpy((char *)((*async) + 1), label, strlen(label) + 1); (*async)->label = (const char *)((*async) + 1); (void)taosThreadMutexInit(&(*async)->mutex, NULL); diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 7c789e84ae..9c153bc8a1 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -265,7 +265,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tsEncryptKey[0] == 0) { return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; } else { - strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN + 1); } } #endif @@ -292,7 +292,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tsEncryptKey[0] == 0) { return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; } else { - strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN + 1); } } #endif @@ -303,7 +303,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tsEncryptKey[0] == 0) { return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; } else { - strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); } } #endif diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 3ebcf50858..28d27b8893 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -103,9 +103,11 @@ static int32_t vnodeGetBufPoolToUse(SVnode *pVnode) { } code = taosThreadCondTimedWait(&pVnode->poolNotEmpty, &pVnode->mutex, &ts); - if (code && code != TSDB_CODE_TIMEOUT_ERROR) { - TSDB_CHECK_CODE(code, lino, _exit); + // ignore timeout error and retry + if (code == TSDB_CODE_TIMEOUT_ERROR) { + code = TSDB_CODE_SUCCESS; } + TSDB_CHECK_CODE(code, lino, _exit); } } } diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index e2bacb3dc1..df6fb17730 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -63,6 +63,12 @@ void initTsdbReaderAPI(TsdReader* pReader) { pReader->tsdSetFilesetDelimited = (void (*)(void*))tsdbSetFilesetDelimited; pReader->tsdSetSetNotifyCb = (void (*)(void*, TsdReaderNotifyCbFn, void*))tsdbReaderSetNotifyCb; + + // file set iterate + pReader->fileSetReaderOpen = tsdbFileSetReaderOpen; + pReader->fileSetReadNext = tsdbFileSetReaderNext; + pReader->fileSetGetEntryField = tsdbFileSetGetEntryField; + pReader->fileSetReaderClose = tsdbFileSetReaderClose; } void initMetadataAPI(SStoreMeta* pMeta) { diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 781736edba..9d326defdd 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -15,6 +15,7 @@ #include "cos.h" #include "monitor.h" +#include "tsdb.h" #include "vnd.h" static volatile int32_t VINIT = 0; @@ -26,6 +27,7 @@ int vnodeInit(int nthreads, StopDnodeFp stopDnodeFp) { TAOS_CHECK_RETURN(vnodeAsyncOpen(nthreads)); TAOS_CHECK_RETURN(walInit(stopDnodeFp)); + TAOS_CHECK_RETURN(tsdbInit()); monInitVnode(); @@ -34,6 +36,7 @@ int vnodeInit(int nthreads, StopDnodeFp stopDnodeFp) { void vnodeCleanup() { if (atomic_val_compare_exchange_32(&VINIT, 1, 0) == 0) return; + tsdbCleanUp(); vnodeAsyncClose(); walCleanUp(); smaCleanUp(); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 2d2446415e..b9e686932e 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -67,8 +67,17 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs SVnodeInfo oldInfo = {0}; oldInfo.config = vnodeCfgDefault; if (vnodeLoadInfo(dir, &oldInfo) == 0) { - vWarn("vgId:%d, vnode config info already exists at %s.", oldInfo.config.vgId, dir); - return (oldInfo.config.dbId == info.config.dbId) ? 0 : -1; + code = (oldInfo.config.dbId == info.config.dbId) ? 0 : TSDB_CODE_VND_ALREADY_EXIST_BUT_NOT_MATCH; + if (code == 0) { + vWarn("vgId:%d, vnode config info already exists at %s.", oldInfo.config.vgId, dir); + } else { + vError("vgId:%d, vnode config info already exists at %s. oldDbId:%" PRId64 "(%s) at cluster:%" PRId64 + ", newDbId:%" PRId64 "(%s) at cluser:%" PRId64 ", code:%s", + oldInfo.config.vgId, dir, oldInfo.config.dbId, oldInfo.config.dbname, + oldInfo.config.syncCfg.nodeInfo[oldInfo.config.syncCfg.myIndex].clusterId, info.config.dbId, + info.config.dbname, info.config.syncCfg.nodeInfo[info.config.syncCfg.myIndex].clusterId, tstrerror(code)); + } + return code; } vInfo("vgId:%d, save config while create", info.config.vgId); @@ -159,7 +168,7 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t static int32_t vnodeVgroupIdLen(int32_t vgId) { char tmp[TSDB_FILENAME_LEN]; - sprintf(tmp, "%d", vgId); + (void)tsnprintf(tmp, TSDB_FILENAME_LEN, "%d", vgId); return strlen(tmp); } @@ -190,7 +199,14 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr char *tsdbFilePrefixPos = strstr(oldRname, tsdbFilePrefix); if (tsdbFilePrefixPos == NULL) continue; - int32_t tsdbFileVgId = atoi(tsdbFilePrefixPos + prefixLen); + int32_t tsdbFileVgId = 0; + ret = taosStr2int32(tsdbFilePrefixPos + prefixLen, &tsdbFileVgId); + if (ret != 0) { + vError("vgId:%d, failed to get tsdb file vgid since %s", dstVgId, tstrerror(ret)); + tfsClosedir(tsdbDir); + return ret; + } + if (tsdbFileVgId == srcVgId) { char *tsdbFileSurfixPos = tsdbFilePrefixPos + prefixLen + vnodeVgroupIdLen(srcVgId); @@ -401,7 +417,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } pVnode->path = (char *)&pVnode[1]; - strcpy(pVnode->path, path); + tstrncpy(pVnode->path, path, strlen(path) + 1); pVnode->config = info.config; pVnode->state.committed = info.state.committed; pVnode->state.commitTerm = info.state.commitTerm; @@ -456,7 +472,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } // open wal - sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); + (void)tsnprintf(tdir, sizeof(tdir), "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); ret = taosRealPath(tdir, NULL, sizeof(tdir)); TAOS_UNUSED(ret); @@ -468,7 +484,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } // open tq - sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); + (void)tsnprintf(tdir, sizeof(tdir), "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); ret = taosRealPath(tdir, NULL, sizeof(tdir)); TAOS_UNUSED(ret); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 6ca5803c19..f52a0c3aba 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -71,11 +71,11 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { if (infoReq.option == REQ_OPT_TBUID) reqTbUid = true; metaRsp.dbId = pVnode->config.dbId; - (void)strcpy(metaRsp.tbName, infoReq.tbName); + tstrncpy(metaRsp.tbName, infoReq.tbName, TSDB_TABLE_NAME_LEN); (void)memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); if (!reqTbUid) { - TAOS_UNUSED(sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName)); + (void)tsnprintf(tableFName, TSDB_TABLE_FNAME_LEN, "%s.%s", infoReq.dbFName, infoReq.tbName); code = vnodeValidateTableHash(pVnode, tableFName); if (code) { goto _exit4; @@ -105,7 +105,7 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaRsp.tuid = mer1.me.uid; if (mer1.me.type == TSDB_SUPER_TABLE) { - (void)strcpy(metaRsp.stbName, mer1.me.name); + tstrncpy(metaRsp.stbName, mer1.me.name, TSDB_TABLE_NAME_LEN); schema = mer1.me.stbEntry.schemaRow; schemaTag = mer1.me.stbEntry.schemaTag; metaRsp.suid = mer1.me.uid; @@ -113,7 +113,7 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit2; - (void)strcpy(metaRsp.stbName, mer2.me.name); + tstrncpy(metaRsp.stbName, mer2.me.name, TSDB_TABLE_NAME_LEN); metaRsp.suid = mer2.me.uid; schema = mer2.me.stbEntry.schemaRow; schemaTag = mer2.me.stbEntry.schemaTag; @@ -220,10 +220,10 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { goto _exit; } - (void)strcpy(cfgRsp.tbName, cfgReq.tbName); + tstrncpy(cfgRsp.tbName, cfgReq.tbName, TSDB_TABLE_NAME_LEN); (void)memcpy(cfgRsp.dbFName, cfgReq.dbFName, sizeof(cfgRsp.dbFName)); - (void)sprintf(tableFName, "%s.%s", cfgReq.dbFName, cfgReq.tbName); + (void)tsnprintf(tableFName, TSDB_TABLE_FNAME_LEN, "%s.%s", cfgReq.dbFName, cfgReq.tbName); code = vnodeValidateTableHash(pVnode, tableFName); if (code) { goto _exit; @@ -246,7 +246,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_LOCK); if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; - (void)strcpy(cfgRsp.stbName, mer2.me.name); + tstrncpy(cfgRsp.stbName, mer2.me.name, TSDB_TABLE_NAME_LEN); schema = mer2.me.stbEntry.schemaRow; schemaTag = mer2.me.stbEntry.schemaTag; cfgRsp.ttl = mer1.me.ctbEntry.ttlDays; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d15bc68832..d9b41869c7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -968,7 +968,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { return; } - strcpy(pMetaRsp->dbFName, pVnode->config.dbname); + tstrncpy(pMetaRsp->dbFName, pVnode->config.dbname, TSDB_DB_FNAME_LEN); pMetaRsp->dbId = pVnode->config.dbId; pMetaRsp->vgId = TD_VID(pVnode); pMetaRsp->precision = pVnode->config.tsdbCfg.precision; @@ -1093,7 +1093,7 @@ static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode *pVnode, int64_t ver, void expiredTb.suid = *uid; terrno = metaReaderGetTableEntryByUid(&mr, *uid); if (terrno < 0) goto _end; - strncpy(buf, mr.me.name, TSDB_TABLE_NAME_LEN); + tstrncpy(buf, mr.me.name, TSDB_TABLE_NAME_LEN); void *p = taosArrayPush(pNames, buf); if (p == NULL) { goto _end; @@ -1216,7 +1216,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, rcode = -1; goto _exit; } - strcpy(str, pCreateReq->name); + tstrncpy(str, pCreateReq->name, TSDB_TABLE_FNAME_LEN); if (taosArrayPush(tbNames, &str) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; rcode = -1; @@ -1225,7 +1225,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, } // validate hash - sprintf(tbName, "%s.%s", pVnode->config.dbname, pCreateReq->name); + (void)tsnprintf(tbName, TSDB_TABLE_FNAME_LEN, "%s.%s", pVnode->config.dbname, pCreateReq->name); if (vnodeValidateTableHash(pVnode, tbName) < 0) { cRsp.code = TSDB_CODE_VND_HASH_MISMATCH; if (taosArrayPush(rsp.pArray, &cRsp) == NULL) { @@ -1321,7 +1321,7 @@ _exit: taosArrayDestroy(tbUids); tDecoderClear(&decoder); tEncoderClear(&encoder); - taosArrayDestroyP(tbNames, taosMemoryFree); + taosArrayDestroyP(tbNames, NULL); return rcode; } @@ -1518,7 +1518,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in pRsp->code = terrno; goto _exit; } - strcpy(str, pDropTbReq->name); + tstrncpy(str, pDropTbReq->name, TSDB_TABLE_FNAME_LEN); if (taosArrayPush(tbNames, &str) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; pRsp->code = terrno; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index c194b51b07..199bac5246 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -519,7 +519,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTabl vgList = taosArrayInit(1, sizeof(SVgroupInfo)); if (NULL == vgList) { ctgError("taosArrayInit %d failed", (int32_t)sizeof(SVgroupInfo)); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_JRET(terrno); } if (NULL == taosArrayPush(vgList, &vgroupInfo)) { @@ -877,13 +877,13 @@ int32_t catalogInit(SCatalogCfg* cfg) { gCtgMgmt.timer = taosTmrInit(0, 0, 0, "catalog"); if (NULL == gCtgMgmt.timer) { qError("init timer failed, error:%s", tstrerror(terrno)); - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(terrno); } gCtgMgmt.cacheTimer = taosTmrStart(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer); if (NULL == gCtgMgmt.cacheTimer) { qError("start cache timer failed"); - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(terrno); } CTG_ERR_RET(ctgStartUpdateThread()); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index e3f7ad4811..e21c1a6486 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1173,12 +1173,16 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableIndex) { SArray* pRes = taosArrayInit(pJob->tbIndexNum, sizeof(SMetaRes)); + if (NULL == pRes) { + CTG_ERR_RET(terrno); + } + if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableIndex, NULL, pRes)) { taosArrayDestroy(pRes); } if (NULL == pJob->jobRes.pTableIndex) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } } @@ -1198,12 +1202,16 @@ int32_t ctgDumpTbCfgRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableCfg) { SArray* pRes = taosArrayInit(pJob->tbCfgNum, sizeof(SMetaRes)); + if (NULL == pRes) { + CTG_ERR_RET(terrno); + } + if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableCfg, NULL, pRes)) { taosArrayDestroy(pRes); } if (NULL == pJob->jobRes.pTableCfg) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } } @@ -1223,12 +1231,16 @@ int32_t ctgDumpTbTagRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableTag) { SArray* pRes = taosArrayInit(pJob->tbTagNum, sizeof(SMetaRes)); + if (NULL == pRes) { + CTG_ERR_RET(terrno); + } + if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableTag, NULL, pRes)) { taosArrayDestroy(pRes); } if (NULL == pJob->jobRes.pTableTag) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 9db3913375..ef5da66bfe 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -3729,7 +3729,7 @@ int32_t ctgGetTbNamesFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbNa for (int32_t i = 0; i < tbNum; ++i) { CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); if (NULL == taosArrayPush(ctx->pResList, &(SMetaData){0})) { - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_JRET(terrno); } } @@ -3863,7 +3863,7 @@ int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsC taosMemoryFree(pViewMeta->querySql); taosMemoryFree(pViewMeta->user); taosMemoryFree(pViewMeta); - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(terrno); } pViewMeta->pSchema = taosMemoryMalloc(pViewMeta->numOfCols * sizeof(SSchema)); if (pViewMeta->pSchema == NULL) { diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 46a615aeed..ec93b7dee2 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -34,9 +34,10 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu SBatchRspMsg* pRsp = NULL; if (TSDB_CODE_SUCCESS == rspCode && pMsg->pData && (pMsg->len > 0)) { - if (tDeserializeSBatchRsp(pMsg->pData, pMsg->len, &batchRsp) < 0) { + code = tDeserializeSBatchRsp(pMsg->pData, pMsg->len, &batchRsp); + if (code < 0) { ctgError("tDeserializeSBatchRsp failed, msgLen:%d", pMsg->len); - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(code); } msgNum = taosArrayGetSize(batchRsp.pRsps); @@ -673,7 +674,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT newBatch.batchId = atomic_add_fetch_32(&pJob->batchId, 1); if (0 != taosHashPut(pBatchs, &vgId, sizeof(vgId), &newBatch, sizeof(newBatch))) { - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_JRET(terrno); } ctgDebug("task %d %s req added to batch %d, target vgId %d", pTask->taskId, TMSG_INFO(msgType), newBatch.batchId, @@ -778,7 +779,7 @@ int32_t ctgBuildBatchReqMsg(SCtgBatch* pBatch, int32_t vgId, void** msg, int32_t int32_t msgSize = tSerializeSBatchReq(NULL, 0, &batchReq); if (msgSize < 0) { qError("tSerializeSBatchReq failed"); - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(msgSize); } *msg = taosMemoryCalloc(1, msgSize); @@ -786,9 +787,10 @@ int32_t ctgBuildBatchReqMsg(SCtgBatch* pBatch, int32_t vgId, void** msg, int32_t qError("calloc batchReq msg failed, size:%d", msgSize); CTG_ERR_RET(terrno); } - if (tSerializeSBatchReq(*msg, msgSize, &batchReq) < 0) { + msgSize = tSerializeSBatchReq(*msg, msgSize, &batchReq); + if (msgSize < 0) { qError("tSerializeSBatchReq failed"); - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(msgSize); } *pSize = msgSize; @@ -835,7 +837,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_QNODE_LIST; - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; ctgDebug("try to get qnode list from mnode, mgmtEpInUse:%d", pConn->mgmtEps.inUse); @@ -892,7 +894,7 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_DNODE_LIST; - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; ctgDebug("try to get dnode list from mnode, mgmtEpInUse:%d", pConn->mgmtEps.inUse); @@ -946,7 +948,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU int32_t msgLen = 0; int32_t reqType = TDMT_MND_USE_DB; SCtgTask* pTask = tReq ? tReq->pTask : NULL; - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); @@ -1001,7 +1003,7 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_DB_CFG; - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; ctgDebug("try to get db cfg from mnode, dbFName:%s", dbFName); @@ -1059,7 +1061,7 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_INDEX; - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; ctgDebug("try to get index from mnode, indexName:%s", indexName); @@ -1117,7 +1119,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_TABLE_INDEX; - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; char tbFName[TSDB_TABLE_FNAME_LEN]; ctgDebug("try to get tb index from mnode, tbFName:%s", tbFName); @@ -1183,7 +1185,7 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_RETRIEVE_FUNC; - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; ctgDebug("try to get udf info from mnode, funcName:%s", funcName); @@ -1241,7 +1243,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_USER_AUTH; - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; ctgDebug("try to get user auth from mnode, user:%s", user); @@ -1304,7 +1306,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const int32_t reqType = TDMT_MND_TABLE_META; char tbFName[TSDB_TABLE_FNAME_LEN]; (void)snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; ctgDebug("try to get table meta from mnode, tbFName:%s", tbFName); @@ -1370,7 +1372,7 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa int32_t reqType = (pTask && pTask->type == CTG_TASK_GET_TB_NAME ? TDMT_VND_TABLE_NAME : TDMT_VND_TABLE_META); char tbFName[TSDB_TABLE_FNAME_LEN]; (void)snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, pTableName->tname); - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse]; ctgDebug("try to get table meta from vnode, vgId:%d, ep num:%d, ep %s:%d, tbFName:%s", vgroupInfo->vgId, @@ -1443,8 +1445,7 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t msgLen = 0; int32_t reqType = TDMT_VND_TABLE_CFG; char tbFName[TSDB_TABLE_FNAME_LEN]; - - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; char dbFName[TSDB_DB_FNAME_LEN]; (void)tNameGetFullDbName(pTableName, dbFName); SBuildTableInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; @@ -1518,7 +1519,7 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t msgLen = 0; int32_t reqType = TDMT_MND_TABLE_CFG; char tbFName[TSDB_TABLE_FNAME_LEN]; - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; char dbFName[TSDB_DB_FNAME_LEN]; (void)tNameGetFullDbName(pTableName, dbFName); SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; @@ -1580,7 +1581,7 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_SERVER_VERSION; - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; qDebug("try to get svr ver from mnode"); @@ -1634,7 +1635,7 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* int32_t msgLen = 0; int32_t reqType = TDMT_MND_VIEW_META; SCtgTask* pTask = tReq ? tReq->pTask : NULL; - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pName, fullName); if (code) { @@ -1696,7 +1697,7 @@ int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa char* msg = NULL; int32_t msgLen = 0; SCtgTask* pTask = tReq ? tReq->pTask : NULL; - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(name, tbFName); if (code) { @@ -1768,7 +1769,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c } SCtgTask* pTask = tReq ? tReq->pTask : NULL; - void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemMalloc : (MallocType)rpcMallocCont; SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse]; ctgDebug("try to get stream progress from vnode, vgId:%d, ep num:%d, ep %s:%d, target:%s", vgroupInfo->vgId, diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index f8591b2121..5ab19c4419 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1620,7 +1620,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) { taosHashCancelIterate(src->vgHash, pIter); taosHashCleanup((*dst)->vgHash); taosMemoryFreeClear(*dst); - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(terrno); } pIter = taosHashIterate(src->vgHash, pIter); @@ -1787,7 +1787,7 @@ static int32_t ctgCloneDbVgroup(void* pSrc, void** ppDst) { } *ppDst = taosArrayDup((const SArray*)pSrc, NULL); - return (*ppDst) ? TSDB_CODE_SUCCESS : TSDB_CODE_OUT_OF_MEMORY; + return (*ppDst) ? TSDB_CODE_SUCCESS : terrno; #else return TSDB_CODE_CTG_INTERNAL_ERROR; #endif @@ -1824,7 +1824,7 @@ static int32_t ctgCloneDbInfo(void* pSrc, void** ppDst) { #if 0 SDbInfo* pDst = taosMemoryMalloc(sizeof(SDbInfo)); if (NULL == pDst) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } TAOS_MEMCPY(pDst, pSrc, sizeof(SDbInfo)); @@ -2263,13 +2263,13 @@ static int32_t ctgCloneMetaDataArray(SArray* pSrc, __array_item_dup_fn_t copyFun int32_t size = taosArrayGetSize(pSrc); *pDst = taosArrayInit(size, sizeof(SMetaRes)); if (NULL == *pDst) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } for (int32_t i = 0; i < size; ++i) { SMetaRes* pRes = taosArrayGet(pSrc, i); SMetaRes res = {.code = pRes->code, .pRes = copyFunc(pRes->pRes)}; if (NULL == res.pRes) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } taosArrayPush(*pDst, &res); } @@ -2630,11 +2630,11 @@ int32_t ctgBuildViewNullRes(SCtgTask* pTask, SCtgViewsCtx* pCtx) { int32_t dupViewMetaFromRsp(SViewMetaRsp* pRsp, SViewMeta* pViewMeta) { pViewMeta->querySql = tstrdup(pRsp->querySql); if (NULL == pViewMeta->querySql) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(terrno); } pViewMeta->user = tstrdup(pRsp->user); if (NULL == pViewMeta->user) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(terrno); } pViewMeta->version = pRsp->version; pViewMeta->viewId = pRsp->viewId; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 88c3d7307b..3966315c30 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "command.h" #include "catalog.h" +#include "command.h" #include "commandInt.h" #include "scheduler.h" #include "systable.h" @@ -54,7 +54,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfCols = htonl(numOfCols); int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeBufSize, numOfCols); - if(len < 0) { + if (len < 0) { taosMemoryFree(*pRsp); *pRsp = NULL; return terrno; @@ -223,7 +223,7 @@ static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** if (NULL == pDesc || NULL == pDesc->pMeta) { return TSDB_CODE_INVALID_PARA; } - int32_t numOfRows = TABLE_TOTAL_COL_NUM(pDesc->pMeta); + int32_t numOfRows = TABLE_TOTAL_COL_NUM(pDesc->pMeta); SSDataBlock* pBlock = NULL; int32_t code = buildDescResultDataBlock(&pBlock); @@ -411,11 +411,17 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char keep0Str[128] = {0}; char keep1Str[128] = {0}; char keep2Str[128] = {0}; + char compactIntervalStr[13] = {0}; + char compactStartTimeStr[13] = {0}; + char compactEndTimeStr[13] = {0}; int32_t lenDuration = formatDurationOrKeep(durationStr, sizeof(durationStr), pCfg->daysPerFile); int32_t lenKeep0 = formatDurationOrKeep(keep0Str, sizeof(keep0Str), pCfg->daysToKeep0); int32_t lenKeep1 = formatDurationOrKeep(keep1Str, sizeof(keep1Str), pCfg->daysToKeep1); int32_t lenKeep2 = formatDurationOrKeep(keep2Str, sizeof(keep2Str), pCfg->daysToKeep2); + UNUSED(formatDurationOrKeep(compactIntervalStr, sizeof(compactIntervalStr), pCfg->compactInterval)); + UNUSED(formatDurationOrKeep(compactStartTimeStr, sizeof(compactStartTimeStr), pCfg->compactStartTime)); + UNUSED(formatDurationOrKeep(compactEndTimeStr, sizeof(compactEndTimeStr), pCfg->compactEndTime)); if (IS_SYS_DBNAME(dbName)) { len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, @@ -428,13 +434,15 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, "PRECISION '%s' REPLICA %d " "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d " "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 - " KEEP_TIME_OFFSET %d ENCRYPT_ALGORITHM '%s' S3_CHUNKPAGES %d S3_KEEPLOCAL %dm S3_COMPACT %d", + " KEEP_TIME_OFFSET %dh ENCRYPT_ALGORITHM '%s' S3_CHUNKPAGES %d S3_KEEPLOCAL %dm S3_COMPACT %d " + "COMPACT_INTERVAL %s COMPACT_TIME_RANGE %s,%s COMPACT_TIME_OFFSET %"PRIi8 "h", dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, durationStr, pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, keep0Str, keep1Str, keep2Str, pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups, 1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize, pCfg->keepTimeOffset, encryptAlgorithmStr(pCfg->encryptAlgorithm), - pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact); + pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact, compactIntervalStr, compactStartTimeStr, + compactEndTimeStr, pCfg->compactTimeOffset); if (pRetentions) { len += tsnprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, @@ -726,7 +734,8 @@ static void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STab } } -static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* pDbCfg, char* tbName, STableCfg* pCfg, void* charsetCxt) { +static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* pDbCfg, char* tbName, STableCfg* pCfg, + void* charsetCxt) { int32_t code = TSDB_CODE_SUCCESS; QRY_ERR_RET(blockDataEnsureCapacity(pBlock, 1)); pBlock->info.rows = 1; @@ -905,7 +914,7 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { goto _return; } - if (cfgCheckRangeForDynUpdate(tsCfg, pStmt->config, pStmt->value, false)) { + if (cfgCheckRangeForDynUpdate(tsCfg, pStmt->config, pStmt->value, false, CFG_ALTER_LOCAL)) { return terrno; } @@ -960,6 +969,12 @@ static int32_t buildLocalVariablesResultDataBlock(SSDataBlock** pOutput) { goto _exit; } + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD5_LEN; + if (taosArrayPush(pBlock->pDataBlock, &infoData) == NULL) { + goto _exit; + } + *pOutput = pBlock; _exit: @@ -1058,7 +1073,8 @@ static int32_t execShowCreateView(SShowCreateViewStmt* pStmt, SRetrieveTableRsp* return code; } -int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp, int8_t biMode, void* charsetCxt) { +int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp, int8_t biMode, + void* charsetCxt) { switch (nodeType(pStmt)) { case QUERY_NODE_DESCRIBE_STMT: return execDescribe(sysInfoUser, pStmt, pRsp, biMode); diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 60944d8a02..0778d5d5f8 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -2056,7 +2056,7 @@ static int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { qError("taosHashPut to explainGroupHash failed, taskIdx:%d", n); - QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + QRY_ERR_JRET(terrno); } } diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 01bc762a1d..7df3439fc1 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -39,6 +39,7 @@ typedef void (*FGetDataLength)(struct SDataSinkHandle* pHandle, int64_t* pLen, i typedef int32_t (*FGetDataBlock)(struct SDataSinkHandle* pHandle, SOutputData* pOutput); typedef int32_t (*FDestroyDataSinker)(struct SDataSinkHandle* pHandle); typedef int32_t (*FGetCacheSize)(struct SDataSinkHandle* pHandle, uint64_t* size); +typedef int32_t (*FGetSinkFlags)(struct SDataSinkHandle* pHandle, uint64_t* flags); typedef struct SDataSinkHandle { FPutDataBlock fPut; @@ -48,12 +49,13 @@ typedef struct SDataSinkHandle { FGetDataBlock fGetData; FDestroyDataSinker fDestroy; FGetCacheSize fGetCacheSize; + FGetSinkFlags fGetFlags; } SDataSinkHandle; -int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle); -int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, +int32_t createDataDispatcher(SDataSinkManager* pManager, SDataSinkNode** ppDataSink, DataSinkHandle* pHandle); +int32_t createDataDeleter(SDataSinkManager* pManager, SDataSinkNode** ppDataSink, DataSinkHandle* pHandle, void* pParam); -int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, +int32_t createDataInserter(SDataSinkManager* pManager, SDataSinkNode** ppDataSink, DataSinkHandle* pHandle, void* pParam); #ifdef __cplusplus diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index aebf5d97b6..48afa78251 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -942,7 +942,7 @@ void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int3 struct SOperatorInfo* pOperator); STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); -int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); +int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, int64_t* defaultBufsz); extern void doDestroyExchangeOperatorInfo(void* param); diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index 94bf791ef8..5713726501 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -581,7 +581,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n } uint32_t defaultPgsz = 0; - uint32_t defaultBufsz = 0; + int64_t defaultBufsz = 0; code = getBufferPgSize(pAggSup->resultRowSize, &defaultPgsz, &defaultBufsz); if (code) { qError("failed to get buff page size, rowSize:%d", pAggSup->resultRowSize); diff --git a/source/libs/executor/src/anomalywindowoperator.c b/source/libs/executor/src/anomalywindowoperator.c index 3bc9c806b0..dd1a52022e 100644 --- a/source/libs/executor/src/anomalywindowoperator.c +++ b/source/libs/executor/src/anomalywindowoperator.c @@ -131,11 +131,18 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p int32_t itemSize = sizeof(int32_t) + pInfo->aggSup.resultRowSize + pInfo->anomalyKey.bytes; pInfo->anomalySup.pResultRow = taosMemoryCalloc(1, itemSize); + if (pInfo->anomalySup.pResultRow == NULL) { + code = terrno; + goto _error; + } pInfo->anomalySup.blocks = taosArrayInit(16, sizeof(SSDataBlock*)); + if (pInfo->anomalySup.blocks == NULL) { + code = terrno; + goto _error; + } pInfo->anomalySup.windows = taosArrayInit(16, sizeof(STimeWindow)); - - if (pInfo->anomalySup.windows == NULL || pInfo->anomalySup.blocks == NULL || pInfo->anomalySup.pResultRow == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + if (pInfo->anomalySup.windows == NULL) { + code = terrno; goto _error; } @@ -269,8 +276,8 @@ static int32_t anomalyCacheBlock(SAnomalyWindowOperatorInfo* pInfo, SSDataBlock* int32_t code = createOneDataBlock(pSrc, true, &pDst); if (code != 0) return code; - if (pDst == NULL) return TSDB_CODE_OUT_OF_MEMORY; - if (taosArrayPush(pInfo->anomalySup.blocks, &pDst) == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (pDst == NULL) return code; + if (taosArrayPush(pInfo->anomalySup.blocks, &pDst) == NULL) return terrno; return 0; } @@ -664,4 +671,4 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p } void destroyForecastInfo(void* param) {} -#endif \ No newline at end of file +#endif diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index c284e9a8a9..1ef8317232 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -41,6 +41,7 @@ typedef struct SDataCacheEntry { typedef struct SDataDeleterHandle { SDataSinkHandle sink; SDataSinkManager* pManager; + SDataSinkNode* pSinkNode; SDataBlockDescNode* pSchema; SDataDeleterNode* pDeleter; SDeleterParam* pParam; @@ -50,6 +51,7 @@ typedef struct SDataDeleterHandle { bool queryEnd; uint64_t useconds; uint64_t cachedSize; + uint64_t flags; TdThreadMutex mutex; } SDataDeleterHandle; @@ -111,7 +113,7 @@ static int32_t allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, if (taosQueueItemSize(pDeleter->pDataBlocks) > capacity) { qError("SinkNode queue is full, no capacity, max:%d, current:%d, no capacity", capacity, taosQueueItemSize(pDeleter->pDataBlocks)); - return TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } pBuf->allocSize = sizeof(SDataCacheEntry) + sizeof(SDeleterRes); @@ -257,6 +259,8 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { } taosCloseQueue(pDeleter->pDataBlocks); (void)taosThreadMutexDestroy(&pDeleter->mutex); + nodesDestroyNode((SNode*)pDeleter->pSinkNode); + pDeleter->pSinkNode = NULL; taosMemoryFree(pDeleter->pManager); @@ -270,8 +274,18 @@ static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) { return TSDB_CODE_SUCCESS; } -int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, + +static int32_t getSinkFlags(struct SDataSinkHandle* pHandle, uint64_t* pFlags) { + SDataDeleterHandle* pDispatcher = (SDataDeleterHandle*)pHandle; + + *pFlags = atomic_load_64(&pDispatcher->flags); + return TSDB_CODE_SUCCESS; +} + + +int32_t createDataDeleter(SDataSinkManager* pManager, SDataSinkNode** ppDataSink, DataSinkHandle* pHandle, void* pParam) { + SDataSinkNode* pDataSink = *ppDataSink; int32_t code = TSDB_CODE_SUCCESS; if (pParam == NULL) { code = TSDB_CODE_QRY_INVALID_INPUT; @@ -296,9 +310,12 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData deleter->sink.fGetData = getDataBlock; deleter->sink.fDestroy = destroyDataSinker; deleter->sink.fGetCacheSize = getCacheSize; + deleter->sink.fGetFlags = getSinkFlags; deleter->pManager = pManager; deleter->pDeleter = pDeleterNode; deleter->pSchema = pDataSink->pInputDataBlockDesc; + deleter->pSinkNode = pDataSink; + *ppDataSink = NULL; deleter->pParam = pParam; deleter->status = DS_BUF_EMPTY; @@ -307,6 +324,7 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData if (code) { goto _end; } + deleter->flags = DS_FLAG_USE_MEMPOOL; code = taosThreadMutexInit(&deleter->mutex, NULL); if (code) { goto _end; @@ -323,6 +341,9 @@ _end: } else { taosMemoryFree(pManager); } + + nodesDestroyNode((SNode *)*ppDataSink); + *ppDataSink = NULL; return code; } diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index f255d0b95c..e28658cd2a 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -43,6 +43,7 @@ typedef struct SDataDispatchHandle { SDataSinkHandle sink; SDataSinkManager* pManager; SDataBlockDescNode* pSchema; + SDataSinkNode* pSinkNode; STaosQueue* pDataBlocks; SDataDispatchBuf nextOutput; int32_t outPutColCounts; @@ -50,6 +51,7 @@ typedef struct SDataDispatchHandle { bool queryEnd; uint64_t useconds; uint64_t cachedSize; + uint64_t flags; void* pCompressBuf; int32_t bufSize; TdThreadMutex mutex; @@ -355,6 +357,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; (void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDispatcher->cachedSize); taosMemoryFreeClear(pDispatcher->nextOutput.pData); + nodesDestroyNode((SNode*)pDispatcher->pSinkNode); while (!taosQueueEmpty(pDispatcher->pDataBlocks)) { SDataDispatchBuf* pBuf = NULL; @@ -381,6 +384,13 @@ static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) { return TSDB_CODE_SUCCESS; } +static int32_t getSinkFlags(struct SDataSinkHandle* pHandle, uint64_t* pFlags) { + SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; + + *pFlags = atomic_load_64(&pDispatcher->flags); + return TSDB_CODE_SUCCESS; +} + static int32_t blockDescNodeCheck(SDataBlockDescNode* pInputDataBlockDesc) { if(tsSafetyCheckLevel == TSDB_SAFETY_CHECK_LEVELL_NEVER) { return TSDB_CODE_SUCCESS; @@ -428,12 +438,13 @@ int32_t getOutputColCounts(SDataBlockDescNode* pInputDataBlockDesc) { return numOfCols; } -int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) { +int32_t createDataDispatcher(SDataSinkManager* pManager, SDataSinkNode** ppDataSink, DataSinkHandle* pHandle) { int32_t code; + SDataSinkNode* pDataSink = *ppDataSink; code = blockDescNodeCheck(pDataSink->pInputDataBlockDesc); if (code) { qError("failed to check input data block desc, code:%d", code); - return code; + goto _return; } SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle)); @@ -448,10 +459,12 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD dispatcher->sink.fGetData = getDataBlock; dispatcher->sink.fDestroy = destroyDataSinker; dispatcher->sink.fGetCacheSize = getCacheSize; - + dispatcher->sink.fGetFlags = getSinkFlags; dispatcher->pManager = pManager; pManager = NULL; dispatcher->pSchema = pDataSink->pInputDataBlockDesc; + dispatcher->pSinkNode = pDataSink; + *ppDataSink = NULL; dispatcher->outPutColCounts = getOutputColCounts(dispatcher->pSchema); dispatcher->status = DS_BUF_EMPTY; dispatcher->queryEnd = false; @@ -466,6 +479,8 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD goto _return; } + dispatcher->flags = DS_FLAG_USE_MEMPOOL; + *pHandle = dispatcher; return TSDB_CODE_SUCCESS; @@ -476,5 +491,9 @@ _return: if (dispatcher) { dsDestroyDataSinker(dispatcher); } + + nodesDestroyNode((SNode *)*ppDataSink); + *ppDataSink = NULL; + return terrno; } diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index b29bef1f1e..ec041cba70 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -45,6 +45,7 @@ typedef struct SDataInserterHandle { bool fullOrderColList; uint64_t useconds; uint64_t cachedSize; + uint64_t flags; TdThreadMutex mutex; tsem_t ready; bool explain; @@ -138,7 +139,7 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, void* pMsg, int pParam->pInserter = pInserter; pMsgSendInfo->param = pParam; - pMsgSendInfo->paramFreeFp = taosMemoryFree; + pMsgSendInfo->paramFreeFp = taosAutoMemoryFree; pMsgSendInfo->msgInfo.pData = pMsg; pMsgSendInfo->msgInfo.len = msgLen; pMsgSendInfo->msgType = TDMT_VND_SUBMIT; @@ -434,6 +435,9 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { taosMemoryFree(pInserter->pSchema); taosMemoryFree(pInserter->pParam); taosHashCleanup(pInserter->pCols); + nodesDestroyNode((SNode *)pInserter->pNode); + pInserter->pNode = NULL; + (void)taosThreadMutexDestroy(&pInserter->mutex); taosMemoryFree(pInserter->pManager); @@ -447,8 +451,16 @@ static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) { return TSDB_CODE_SUCCESS; } -int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, +static int32_t getSinkFlags(struct SDataSinkHandle* pHandle, uint64_t* pFlags) { + SDataInserterHandle* pDispatcher = (SDataInserterHandle*)pHandle; + + *pFlags = atomic_load_64(&pDispatcher->flags); + return TSDB_CODE_SUCCESS; +} + +int32_t createDataInserter(SDataSinkManager* pManager, SDataSinkNode** ppDataSink, DataSinkHandle* pHandle, void* pParam) { + SDataSinkNode* pDataSink = *ppDataSink; SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle)); if (NULL == inserter) { taosMemoryFree(pParam); @@ -462,12 +474,14 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat inserter->sink.fGetData = NULL; inserter->sink.fDestroy = destroyDataSinker; inserter->sink.fGetCacheSize = getCacheSize; + inserter->sink.fGetFlags = getSinkFlags; inserter->pManager = pManager; inserter->pNode = pInserterNode; inserter->pParam = pParam; inserter->status = DS_BUF_EMPTY; inserter->queryEnd = false; inserter->explain = pInserterNode->explain; + *ppDataSink = NULL; int64_t suid = 0; int32_t code = pManager->pAPI->metaFn.getTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, @@ -521,5 +535,8 @@ _return: taosMemoryFree(pManager); } + nodesDestroyNode((SNode *)*ppDataSink); + *ppDataSink = NULL; + return terrno; } diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 55fc520477..255dc901d6 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -39,23 +39,23 @@ int32_t dsDataSinkGetCacheSize(SDataSinkStat* pStat) { return TSDB_CODE_SUCCESS; } -int32_t dsCreateDataSinker(void* pSinkManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void* pParam, const char* id) { +int32_t dsCreateDataSinker(void* pSinkManager, SDataSinkNode** ppDataSink, DataSinkHandle* pHandle, void* pParam, const char* id) { SDataSinkManager* pManager = pSinkManager; - switch ((int)nodeType(pDataSink)) { + switch ((int)nodeType(*ppDataSink)) { case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: - return createDataDispatcher(pManager, pDataSink, pHandle); + return createDataDispatcher(pManager, ppDataSink, pHandle); case QUERY_NODE_PHYSICAL_PLAN_DELETE: { - return createDataDeleter(pManager, pDataSink, pHandle, pParam); + return createDataDeleter(pManager, ppDataSink, pHandle, pParam); } case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: { - return createDataInserter(pManager, pDataSink, pHandle, pParam); + return createDataInserter(pManager, ppDataSink, pHandle, pParam); } default: break; } taosMemoryFree(pSinkManager); - qError("invalid input node type:%d, %s", nodeType(pDataSink), id); + qError("invalid input node type:%d, %s", nodeType(*ppDataSink), id); return TSDB_CODE_QRY_INVALID_INPUT; } @@ -101,3 +101,9 @@ void dsDestroyDataSinker(DataSinkHandle handle) { (void)pHandleImpl->fDestroy(pHandleImpl); taosMemoryFree(pHandleImpl); } + +int32_t dsGetSinkFlags(DataSinkHandle handle, uint64_t* pFlags) { + SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; + return pHandleImpl->fGetFlags(pHandleImpl, pFlags); +} + diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 7222f2d297..7c4a4adaca 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -354,15 +354,15 @@ static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); if (pInfo->pSources == NULL) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); return terrno; } if (pExNode->node.dynamicOp) { pInfo->pHashSources = tSimpleHashInit(numOfSources * 2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)); if (NULL == pInfo->pHashSources) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return TSDB_CODE_OUT_OF_MEMORY; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; } } @@ -370,7 +370,7 @@ static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)nodesListGetNode((SNodeList*)pExNode->pSrcEndPoints, i); if (!pNode) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } void* tmp = taosArrayPush(pInfo->pSources, pNode); if (!tmp) { @@ -640,16 +640,16 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas if (pSource->localExec) { SDataBuf pBuf = {0}; - int32_t code = (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId, + int32_t code = (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->sId, pTaskInfo->id.queryId, pSource->clientId, pSource->taskId, 0, pSource->execId, &pBuf.pData, pTaskInfo->localFetch.explainRes); code = loadRemoteDataCallback(pWrapper, &pBuf, code); - QUERY_CHECK_CODE(code, lino, _end); taosMemoryFree(pWrapper); + QUERY_CHECK_CODE(code, lino, _end); } else { SResFetchReq req = {0}; req.header.vgId = pSource->addr.nodeId; - req.sId = pSource->schedId; + req.sId = pSource->sId; req.clientId = pSource->clientId; req.taskId = pSource->taskId; req.queryId = pTaskInfo->id.queryId; @@ -668,7 +668,7 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas int32_t msgSize = tSerializeSResFetchReq(NULL, 0, &req); if (msgSize < 0) { - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + pTaskInfo->code = msgSize; taosMemoryFree(pWrapper); freeOperatorParam(req.pOpParam, OP_GET_PARAM); return pTaskInfo->code; @@ -676,14 +676,15 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas void* msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + pTaskInfo->code = terrno; taosMemoryFree(pWrapper); freeOperatorParam(req.pOpParam, OP_GET_PARAM); return pTaskInfo->code; } - if (tSerializeSResFetchReq(msg, msgSize, &req) < 0) { - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + msgSize = tSerializeSResFetchReq(msg, msgSize, &req); + if (msgSize < 0) { + pTaskInfo->code = msgSize; taosMemoryFree(pWrapper); taosMemoryFree(msg); freeOperatorParam(req.pOpParam, OP_GET_PARAM); @@ -703,18 +704,19 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas taosMemoryFreeClear(msg); taosMemoryFree(pWrapper); qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + pTaskInfo->code = terrno; return pTaskInfo->code; } pMsgSendInfo->param = pWrapper; - pMsgSendInfo->paramFreeFp = taosMemoryFree; + pMsgSendInfo->paramFreeFp = taosAutoMemoryFree; pMsgSendInfo->msgInfo.pData = msg; pMsgSendInfo->msgInfo.len = msgSize; pMsgSendInfo->msgType = pSource->fetchMsgType; pMsgSendInfo->fp = loadRemoteDataCallback; int64_t transporterId = 0; + void* poolHandle = NULL; code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo); QUERY_CHECK_CODE(code, lino, _end); int64_t* pRpcHandle = taosArrayGet(pExchangeInfo->pFetchRpcHandles, sourceIndex); @@ -898,7 +900,7 @@ int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDa blockDataCleanup(pb); } else { code = createOneDataBlock(pExchangeInfo->pDummyBlock, false, &pb); - QUERY_CHECK_NULL(pb, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY); + QUERY_CHECK_NULL(pb, code, lino, _end, code); } int32_t compLen = *(int32_t*)pStart; @@ -1055,7 +1057,7 @@ int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasic void* tmp = taosArrayPush(pExchangeInfo->pSourceDataInfo, &dataInfo); if (!tmp) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); return terrno; } pIdx->inUseIdx = taosArrayGetSize(pExchangeInfo->pSourceDataInfo) - 1; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index ccb7005dbd..cce754a8c8 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -332,7 +332,6 @@ SArray* createSortInfo(SNodeList* pNodeList) { SArray* pList = taosArrayInit(numOfCols, sizeof(SBlockOrderInfo)); if (pList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return pList; } @@ -342,7 +341,7 @@ SArray* createSortInfo(SNodeList* pNodeList) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); taosArrayDestroy(pList); pList = NULL; - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; break; } SBlockOrderInfo bi = {0}; @@ -355,7 +354,6 @@ SArray* createSortInfo(SNodeList* pNodeList) { if (!tmp) { taosArrayDestroy(pList); pList = NULL; - terrno = TSDB_CODE_OUT_OF_MEMORY; break; } } @@ -1129,7 +1127,7 @@ static int32_t optimizeTbnameInCondImpl(void* pVnode, SArray* pExistedUidList, S if (numOfExisted > 0) { uHash = taosHashInit(numOfExisted / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (!uHash) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); return terrno; } @@ -1330,7 +1328,7 @@ static int32_t copyExistedUids(SArray* pUidTagList, const SArray* pUidList) { STUidTagInfo info = {.uid = *uid}; void* tmp = taosArrayPush(pUidTagList, &info); if (!tmp) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); return code; } } @@ -1356,15 +1354,13 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN tagFilterAssist ctx = {0}; ctx.colHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); if (ctx.colHash == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, end); } ctx.cInfoList = taosArrayInit(4, sizeof(SColumnInfo)); if (ctx.cInfoList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, end); } @@ -1705,6 +1701,7 @@ int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, nodesDestroyList(groupNew); pAPI->metaReaderFn.clearReader(&mr); + return TSDB_CODE_SUCCESS; } @@ -1716,7 +1713,6 @@ SArray* makeColumnArrayFromList(SNodeList* pNodeList) { size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn)); if (pList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -1724,7 +1720,7 @@ SArray* makeColumnArrayFromList(SNodeList* pNodeList) { SColumnNode* pColNode = (SColumnNode*)nodesListGetNode(pNodeList, i); if (!pColNode) { taosArrayDestroy(pList); - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR)); return NULL; } @@ -1740,7 +1736,7 @@ SArray* makeColumnArrayFromList(SNodeList* pNodeList) { void* tmp = taosArrayPush(pList, &c); if (!tmp) { taosArrayDestroy(pList); - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); return NULL; } } @@ -1835,7 +1831,6 @@ static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, i static SColumn* createColumn(int32_t blockId, int32_t slotId, int32_t colId, SDataType* pType, EColumnType colType) { SColumn* pCol = taosMemoryCalloc(1, sizeof(SColumn)); if (pCol == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -2174,7 +2169,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, } bool tmp = pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); if (!tmp) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _end); } } else { @@ -2187,7 +2182,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, if (pCtx->sfp.getEnv != NULL) { bool tmp = pCtx->sfp.getEnv(pExpr->pExpr->_function.pFunctNode, &env); if (!tmp) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; QUERY_CHECK_CODE(code, lino, _end); } } @@ -2236,6 +2231,8 @@ _end: } taosMemoryFreeClear(*rowEntryInfoOffset); taosMemoryFreeClear(pFuncCtx); + + terrno = code; return NULL; } return pFuncCtx; @@ -3073,6 +3070,7 @@ SNodeList* makeColsNodeArrFromSortKeys(SNodeList* pSortKeys) { int32_t code = nodesListMakeAppend(&ret, pSortKey->pExpr); if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + terrno = code; return NULL; } } @@ -3098,3 +3096,4 @@ _end: } return code; } + diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a0e31a8a3e..dffab1b163 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -441,6 +441,7 @@ static int32_t filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S } _end: + pAPI->metaReaderFn.clearReader(&mr); (*ppArrayRes) = qa; @@ -599,6 +600,11 @@ void qUpdateOperatorParam(qTaskInfo_t tinfo, void* pParam) { ((SExecTaskInfo*)tinfo)->paramSet = false; } +int32_t qExecutorInit(void) { + taosThreadOnce(&initPoolOnce, initRefPool); + return TSDB_CODE_SUCCESS; +} + int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, int8_t compressResult, char* sql, EOPTR_EXEC_MODEL model) { @@ -630,8 +636,18 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, goto _error; } + SDataSinkNode* pSink = NULL; + if (readHandle->localExec) { + code = nodesCloneNode((SNode *)pSubplan->pDataSink, (SNode **)&pSink); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to nodesCloneNode, srcType:%d, code:%s, %s", nodeType(pSubplan->pDataSink), tstrerror(code), (*pTask)->id.str); + taosMemoryFree(pSinkManager); + goto _error; + } + } + // pSinkParam has been freed during create sinker. - code = dsCreateDataSinker(pSinkManager, pSubplan->pDataSink, handle, pSinkParam, (*pTask)->id.str); + code = dsCreateDataSinker(pSinkManager, readHandle->localExec ? &pSink : &pSubplan->pDataSink, handle, pSinkParam, (*pTask)->id.str); if (code) { qError("s-task:%s failed to create data sinker, code:%s", (*pTask)->id.str, tstrerror(code)); } @@ -726,7 +742,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo QUERY_CHECK_CODE(code, lino, _end); void* tmp = taosArrayPush(pTaskInfo->pResultBlockList, &p1); - QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY); + QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); p = p1; } else { void* tmp = taosArrayGet(pTaskInfo->pResultBlockList, blockIndex); @@ -881,7 +897,7 @@ int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo taosWUnLockLatch(&pTaskInfo->stopInfo.lock); if (!tmp) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); return terrno; } return TSDB_CODE_SUCCESS; @@ -1570,8 +1586,7 @@ void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { if (pMsg->contLen > 0) { buf.pData = taosMemoryCalloc(1, pMsg->contLen); if (buf.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - pMsg->code = TSDB_CODE_OUT_OF_MEMORY; + pMsg->code = terrno; } else { memcpy(buf.pData, pMsg->pCont, pMsg->contLen); } @@ -1590,7 +1605,6 @@ SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo) { code = getTableListInfo(pTaskInfo, &plist); if (code || plist == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; return NULL; } diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index b8d15aad31..51e7cd896a 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -438,9 +438,6 @@ static int32_t doCreateConstantValColumnSMAInfo(SInputColumnInfoData* pInput, SF return terrno; } pInput->pColumnDataAgg[paramIndex] = da; - if (da == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } } else { da = pInput->pColumnDataAgg[paramIndex]; } @@ -974,7 +971,7 @@ void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) { } } -int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz) { +int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, int64_t* defaultBufsz) { *defaultPgsz = 4096; uint32_t last = *defaultPgsz; while (*defaultPgsz < rowSize * 4) { @@ -1056,7 +1053,7 @@ int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr, S if (pSup->pExprInfo != NULL) { pSup->pCtx = createSqlFunctionCtx(pExprInfo, numOfExpr, &pSup->rowEntryInfoOffset, pStore); if (pSup->pCtx == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -1145,7 +1142,7 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* if (!pTable) { taosArrayDestroy(pDeleterParam->pUidList); taosMemoryFree(pDeleterParam); - return TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } void* tmp = taosArrayPush(pDeleterParam->pUidList, &pTable->uid); if (!tmp) { diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index bcbd3f7563..5dbd490223 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -387,10 +387,10 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t return terrno; } - if (pInfo->pFillInfo == NULL || pInfo->p == NULL) { + if (pInfo->pFillInfo == NULL) { taosMemoryFree(pInfo->pFillInfo); taosMemoryFree(pInfo->p); - return TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } else { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index d47ab366b6..bdcdeb2c32 100644 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -384,7 +384,7 @@ void freeGcBlkBufInfo(void* ptr) { static int32_t addBlkToDirtyBufList(SGroupCacheOperatorInfo* pGCache, SGcDownstreamCtx* pCtx, SGcBlkCacheInfo* pCache, SGcBlkBufInfo* pBufInfo) { if (0 != taosHashPut(pCache->pDirtyBlk, &pBufInfo->basic.blkId, sizeof(pBufInfo->basic.blkId), pBufInfo, sizeof(*pBufInfo))) { freeGcBlkBufInfo(pBufInfo); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pBufInfo = taosHashGet(pCache->pDirtyBlk, &pBufInfo->basic.blkId, sizeof(pBufInfo->basic.blkId)); if (NULL == pBufInfo) { @@ -906,7 +906,7 @@ static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam* *ppGrp = taosHashGet(pGrpHash, &uid, sizeof(uid)); if (NULL == *ppGrp) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } QRY_ERR_RET(initNewGroupData(pCtx, *ppGrp, pParam->downstreamIdx, vgId, pGCache->batchFetch, pGcParam->needCache)); @@ -1147,12 +1147,12 @@ static int32_t groupCacheSessionWait(struct SOperatorInfo* pOperator, SGcDownstr if (NULL == pGroup->waitQueue) { pGroup->waitQueue = taosArrayInit(1, POINTER_BYTES); if (NULL == pGroup->waitQueue) { - QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + QRY_ERR_JRET(terrno); } } if (NULL == taosArrayPush(pGroup->waitQueue, &pSession)) { - QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + QRY_ERR_JRET(terrno); } if (!pSession->semInit) { @@ -1413,7 +1413,7 @@ static int32_t initGroupCacheDownstreamCtx(SOperatorInfo* pOperator) { pCtx->lastBlkUid = 0; pCtx->pVgTbHash = tSimpleHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)); if (NULL == pCtx->pVgTbHash) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } tSimpleHashSetFreeFp(pCtx->pVgTbHash, freeSGcVgroupCtx); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index d6e3d26267..08b4ce240e 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -614,7 +614,6 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc if (pDataBlock->pBlockAgg) { pDstBlock->pBlockAgg = taosMemoryCalloc(numOfCols, sizeof(SColumnDataAgg)); if (pDstBlock->pBlockAgg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; blockDataDestroy(pDstBlock); return NULL; } @@ -1161,7 +1160,7 @@ int32_t createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNo } uint32_t defaultPgsz = 0; - uint32_t defaultBufsz = 0; + int64_t defaultBufsz = 0; pInfo->binfo.pRes = createDataBlockFromDescNode(pPartNode->node.pOutputDataBlockDesc); QUERY_CHECK_NULL(pInfo->binfo.pRes, code, lino, _error, terrno); diff --git a/source/libs/executor/src/hashjoinoperator.c b/source/libs/executor/src/hashjoinoperator.c index 27efd59c12..64ce62cb66 100644 --- a/source/libs/executor/src/hashjoinoperator.c +++ b/source/libs/executor/src/hashjoinoperator.c @@ -804,7 +804,7 @@ static int32_t hJoinAddRowToHashImpl(SHJoinOperatorInfo* pJoin, SGroupData* pGro pRow->next = NULL; if (tSimpleHashPut(pJoin->pKeyHash, pTable->keyData, keyLen, &group, sizeof(group))) { taosMemoryFree(pRow); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } else { pRow->next = pGroup->rows; @@ -1201,7 +1201,7 @@ int32_t createHashJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDow size_t hashCap = pInfo->pBuild->inputStat.inputRowNum > 0 ? (pInfo->pBuild->inputStat.inputRowNum * 1.5) : 1024; pInfo->pKeyHash = tSimpleHashInit(hashCap, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY)); if (pInfo->pKeyHash == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return; } diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c index 0dfe89e10e..81e43560e3 100644 --- a/source/libs/executor/src/mergeoperator.c +++ b/source/libs/executor/src/mergeoperator.c @@ -331,7 +331,7 @@ int32_t openNonSortMergeOperator(SOperatorInfo* pOperator) { pNonSortMergeInfo->pSourceStatus = taosMemoryCalloc(pOperator->numOfDownstream, sizeof(*pNonSortMergeInfo->pSourceStatus)); if (NULL == pNonSortMergeInfo->pSourceStatus) { - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + pTaskInfo->code = terrno; return pTaskInfo->code; } diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 67603c965b..5d25a81f6f 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -722,7 +722,7 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInf code = getOperatorExplainExecInfo(operatorInfo->pDownstream[i], pExecInfoList); if (code != TSDB_CODE_SUCCESS) { // taosMemoryFreeClear(*pRes); - return TSDB_CODE_OUT_OF_MEMORY; + return code; } } diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index 37461382dd..c6a1900b41 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -102,7 +102,6 @@ int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHand int32_t code = doCreateTask(pPlan->id.queryId, taskId, vgId, model, &pHandle->api, pTaskInfo); if (*pTaskInfo == NULL || code != 0) { nodesDestroyNode((SNode*)pPlan); - taosMemoryFree(sql); return code; } @@ -112,7 +111,16 @@ int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHand } } - TSWAP((*pTaskInfo)->sql, sql); + if (NULL != sql) { + (*pTaskInfo)->sql = taosStrdup(sql); + if (NULL == (*pTaskInfo)->sql) { + code = terrno; + nodesDestroyNode((SNode*)pPlan); + doDestroyTask(*pTaskInfo); + (*pTaskInfo) = NULL; + return code; + } + } (*pTaskInfo)->pSubplan = pPlan; (*pTaskInfo)->pWorkerCb = pHandle->pWorkerCb; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index cc4258b84b..1060cbcffe 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1167,7 +1167,7 @@ static int32_t createTableListInfoFromParam(SOperatorInfo* pOperator) { if (TSDB_CODE_DUP_KEY == terrno) { continue; } - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } info.uid = *pUid; @@ -4572,11 +4572,10 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* int32_t numOfTags; SExprInfo* pTagExpr = createExpr(pTableScanNode->pTags, &numOfTags); if (pTagExpr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; goto _error; } - if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags, &pTaskInfo->storageAPI.functionStore) != 0) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags, &pTaskInfo->storageAPI.functionStore); + if (code != 0) { goto _error; } } @@ -5173,7 +5172,7 @@ static int32_t doTagScanFromMetaEntryNext(SOperatorInfo* pOperator, SSDataBlock* while (pInfo->curPos < size && pRes->info.rows < pOperator->resultInfo.capacity) { code = doTagScanOneTable(pOperator, pRes, &mr, &pTaskInfo->storageAPI); - if (code != TSDB_CODE_OUT_OF_MEMORY) { + if (code != TSDB_CODE_OUT_OF_MEMORY && code != TSDB_CODE_QRY_REACH_QMEM_THRESHOLD && code != TSDB_CODE_QRY_QUERY_MEM_EXHAUSTED) { // ignore other error code = TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index a6ca20c5ee..ed073d21a0 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -112,10 +112,10 @@ int32_t createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortN goto _error; } SNodeList* pSortColsNodeArr = makeColsNodeArrFromSortKeys(pSortNode->pSortKeys); - if (!pSortColsNodeArr) code = TSDB_CODE_OUT_OF_MEMORY; + if (!pSortColsNodeArr) code = terrno; if (TSDB_CODE_SUCCESS == code) { pGroupIdCalc->pSortColsArr = makeColumnArrayFromList(pSortColsNodeArr); - if (!pGroupIdCalc->pSortColsArr) code = TSDB_CODE_OUT_OF_MEMORY; + if (!pGroupIdCalc->pSortColsArr) code = terrno; nodesClearList(pSortColsNodeArr); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 29b3f473ba..fa6008eba7 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -549,7 +549,7 @@ void doStreamEventSaveCheckpoint(SOperatorInfo* pOperator) { int32_t len = doStreamEventEncodeOpState(NULL, 0, pOperator); void* buf = taosMemoryCalloc(1, len); if (!buf) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); return; } void* pBuf = buf; diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index e576e0d920..91af218a7e 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -1466,7 +1466,7 @@ void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) { int32_t len = doStreamIntervalEncodeOpState(NULL, 0, pOperator); void* buf = taosMemoryCalloc(1, len); if (!buf) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); return; } void* pBuf = buf; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 6d21f43889..13ae220116 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -76,6 +76,9 @@ typedef struct SSysTableScanInfo { STableListInfo* pTableListInfo; SReadHandle* pHandle; SStorageAPI* pAPI; + + // file set iterate + struct SFileSetReader* pFileSetReader; } SSysTableScanInfo; typedef struct { @@ -609,7 +612,6 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { } if (!pInfo->pCur || !pInfo->pSchema) { - terrno = TSDB_CODE_OUT_OF_MEMORY; qError("sysTableScanUserCols failed since %s", terrstr()); blockDataDestroy(pDataBlock); pInfo->loadInfo.totalRows = 0; @@ -1014,7 +1016,7 @@ static int32_t sysTableGetGeomText(char* iGeom, int32_t nGeom, char** output, in char* outputWKT = NULL; if (nGeom == 0) { - if (!(*output = taosStrdup(""))) code = TSDB_CODE_OUT_OF_MEMORY; + if (!(*output = taosStrdup(""))) code = terrno; *nOutput = 0; return code; } @@ -1112,8 +1114,9 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, code = sysTableGetGeomText(tagVal.pData, tagVal.nData, &tagData, &tagLen); QUERY_CHECK_CODE(code, lino, _end); } else if (tagType == TSDB_DATA_TYPE_VARBINARY) { - if (taosAscii2Hex(tagVal.pData, tagVal.nData, (void**)&tagData, &tagLen) < 0) { - qError("varbinary for systable failed since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + code = taosAscii2Hex(tagVal.pData, tagVal.nData, (void**)&tagData, &tagLen); + if (code < 0) { + qError("varbinary for systable failed since %s", tstrerror(code)); } } else if (IS_VAR_DATA_TYPE(tagType)) { tagData = (char*)tagVal.pData; @@ -1438,7 +1441,7 @@ static int32_t doSetUserTableMetaInfo(SStoreMetaReader* pMetaReaderFn, SStoreMet SMetaReader mr1 = {0}; pMetaReaderFn->initReader(&mr1, pVnode, META_READER_NOLOCK, pMetaFn); - + int64_t suid = pMReader->me.ctbEntry.suid; code = pMetaReaderFn->getTableEntryByUid(&mr1, suid); if (code != TSDB_CODE_SUCCESS) { @@ -1601,6 +1604,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { SMetaReader mr = {0}; pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); + code = doSetUserTableMetaInfo(&pAPI->metaReaderFn, &pAPI->metaFn, pInfo->readHandle.vnode, &mr, *uid, dbname, vgId, p, numOfRows, GET_TASKID(pTaskInfo)); @@ -1748,7 +1752,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { SMetaReader mr = {0}; pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn); - + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; code = pAPI->metaReaderFn.getTableEntryByUid(&mr, suid); if (code != TSDB_CODE_SUCCESS) { @@ -2209,6 +2213,258 @@ static SSDataBlock* sysTableScanUserSTables(SOperatorInfo* pOperator) { return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } +static int32_t doSetQueryFileSetRow() { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + // TODO + +_exit: + return code; +} + +static SSDataBlock* sysTableBuildUserFileSets(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSysTableScanInfo* pInfo = pOperator->info; + SSDataBlock* p = NULL; + + // open cursor if not opened + if (pInfo->pFileSetReader == NULL) { + code = pAPI->tsdReader.fileSetReaderOpen(pInfo->readHandle.vnode, &pInfo->pFileSetReader); + QUERY_CHECK_CODE(code, lino, _end); + } + + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + const char* db = NULL; + int32_t vgId = 0; + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + code = tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + QUERY_CHECK_CODE(code, lino, _end); + + code = tNameGetDbName(&sn, varDataVal(dbname)); + QUERY_CHECK_CODE(code, lino, _end); + + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_FILESETS); + QUERY_CHECK_NULL(p, code, lino, _end, terrno); + + code = blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); + QUERY_CHECK_CODE(code, lino, _end); + + char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t ret = 0; + + // loop to query each entry + for (;;) { + int32_t ret = pAPI->tsdReader.fileSetReadNext(pInfo->pFileSetReader); + if (ret) { + if (ret == TSDB_CODE_NOT_FOUND) { + // no more scan entry + break; + } else { + code = ret; + QUERY_CHECK_CODE(code, lino, _end); + } + } + + // fill the data block + { + SColumnInfoData* pColInfoData; + int32_t index = 0; + + // db_name + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, db, false); + QUERY_CHECK_CODE(code, lino, _end); + + // vgroup_id + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, (char*)&vgId, false); + QUERY_CHECK_CODE(code, lino, _end); + + // fileset_id + int32_t filesetId = 0; + code = pAPI->tsdReader.fileSetGetEntryField(pInfo->pFileSetReader, "fileset_id", &filesetId); + QUERY_CHECK_CODE(code, lino, _end); + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, (char*)&filesetId, false); + QUERY_CHECK_CODE(code, lino, _end); + + // start_time + int64_t startTime = 0; + code = pAPI->tsdReader.fileSetGetEntryField(pInfo->pFileSetReader, "start_time", &startTime); + QUERY_CHECK_CODE(code, lino, _end); + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, (char*)&startTime, false); + QUERY_CHECK_CODE(code, lino, _end); + + // end_time + int64_t endTime = 0; + code = pAPI->tsdReader.fileSetGetEntryField(pInfo->pFileSetReader, "end_time", &endTime); + QUERY_CHECK_CODE(code, lino, _end); + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, (char*)&endTime, false); + QUERY_CHECK_CODE(code, lino, _end); + + // total_size + int64_t totalSize = 0; + code = pAPI->tsdReader.fileSetGetEntryField(pInfo->pFileSetReader, "total_size", &totalSize); + QUERY_CHECK_CODE(code, lino, _end); + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, (char*)&totalSize, false); + QUERY_CHECK_CODE(code, lino, _end); + + // last_compact + int64_t lastCompacat = 0; + code = pAPI->tsdReader.fileSetGetEntryField(pInfo->pFileSetReader, "last_compact_time", &lastCompacat); + QUERY_CHECK_CODE(code, lino, _end); + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, (char*)&lastCompacat, false); + QUERY_CHECK_CODE(code, lino, _end); + + // shold_compact + bool shouldCompact = false; + code = pAPI->tsdReader.fileSetGetEntryField(pInfo->pFileSetReader, "should_compact", &shouldCompact); + QUERY_CHECK_CODE(code, lino, _end); + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, (char*)&shouldCompact, false); + QUERY_CHECK_CODE(code, lino, _end); + + // // details + // const char* details = NULL; + // code = pAPI->tsdReader.fileSetGetEntryField(pInfo->pFileSetReader, "details", &details); + // QUERY_CHECK_CODE(code, lino, _end); + // pColInfoData = taosArrayGet(p->pDataBlock, index++); + // QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + // code = colDataSetVal(pColInfoData, numOfRows, (char*)&vgId, false); + // QUERY_CHECK_CODE(code, lino, _end); + } + + // check capacity + if (++numOfRows >= pOperator->resultInfo.capacity) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + code = relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + QUERY_CHECK_CODE(code, lino, _end); + + code = doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); + QUERY_CHECK_CODE(code, lino, _end); + + blockDataCleanup(p); + numOfRows = 0; + + if (pInfo->pRes->info.rows > 0) { + break; + } + } + } + + if (numOfRows > 0) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + code = relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + QUERY_CHECK_CODE(code, lino, _end); + + code = doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); + QUERY_CHECK_CODE(code, lino, _end); + + blockDataCleanup(p); + numOfRows = 0; + } + + blockDataDestroy(p); + p = NULL; + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + blockDataDestroy(p); + pTaskInfo->code = code; + pAPI->tsdReader.fileSetReaderClose(&pInfo->pFileSetReader); + T_LONG_JMP(pTaskInfo->env, code); + } + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +static SSDataBlock* sysTableScanUserFileSets(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SSysTableScanInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SNode* pCondition = pInfo->pCondition; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + if (pInfo->readHandle.mnd != NULL) { + // do nothing on mnode + qTrace("This operator do nothing on mnode, task id:%s", GET_TASKID(pTaskInfo)); + return NULL; + } else { +#if 0 + if (pInfo->showRewrite == false) { + if (pCondition != NULL && pInfo->pIdx == NULL) { + SSTabFltArg arg = { + .pMeta = pInfo->readHandle.vnode, .pVnode = pInfo->readHandle.vnode, .pAPI = &pTaskInfo->storageAPI}; + + SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); + QUERY_CHECK_NULL(idx, code, lino, _end, terrno); + idx->init = 0; + idx->uids = taosArrayInit(128, sizeof(int64_t)); + QUERY_CHECK_NULL(idx->uids, code, lino, _end, terrno); + idx->lastIdx = 0; + + pInfo->pIdx = idx; // set idx arg + + int flt = optSysTabFilte(&arg, pCondition, idx->uids); + if (flt == 0) { + pInfo->pIdx->init = 1; + SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); + return blk; + } else if ((flt == -1) || (flt == -2)) { + qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo)); + } + } else if (pCondition != NULL && (pInfo->pIdx != NULL && pInfo->pIdx->init == 1)) { + SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); + return blk; + } + } +#endif + + return sysTableBuildUserFileSets(pOperator); + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } + return NULL; +} + static int32_t getSysTableDbNameColId(const char* pTable) { // if (0 == strcmp(TSDB_INS_TABLE_INDEXES, pTable)) { // return 1; @@ -2308,6 +2564,8 @@ static int32_t doSysTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) pBlock = sysTableScanUserSTables(pOperator); } else if (strncasecmp(name, TSDB_INS_DISK_USAGE, TSDB_TABLE_FNAME_LEN) == 0) { pBlock = sysTableScanUsage(pOperator); + } else if (strncasecmp(name, TSDB_INS_TABLE_FILESETS, TSDB_TABLE_FNAME_LEN) == 0) { + pBlock = sysTableScanUserFileSets(pOperator); } else { // load the meta from mnode of the given epset pBlock = sysTableScanFromMNode(pOperator, pInfo, name, pTaskInfo); } @@ -2396,7 +2654,7 @@ static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableSca SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + pTaskInfo->code = terrno; taosMemoryFree(buf1); return NULL; } @@ -2528,7 +2786,8 @@ int32_t createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNo QUERY_CHECK_CODE(code, lino, _error); if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || - strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { + strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_FILESETS, TSDB_TABLE_FNAME_LEN) == 0) { pInfo->readHandle = *(SReadHandle*)readHandle; } else { if (tsem_init(&pInfo->ready, 0, 0) != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index c2aa07dacc..52b5e0eb19 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -630,7 +630,7 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, ret); } if(isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) { @@ -711,7 +711,7 @@ static bool filterWindowWithLimit(SIntervalAggOperatorInfo* pOperatorInfo, STime if (pOperatorInfo->limit == 0) return true; if (pOperatorInfo->pBQ == NULL) { - pOperatorInfo->pBQ = createBoundedQueue(pOperatorInfo->limit - 1, tsKeyCompFn, taosMemoryFree, pOperatorInfo); + pOperatorInfo->pBQ = createBoundedQueue(pOperatorInfo->limit - 1, tsKeyCompFn, taosAutoMemoryFree, pOperatorInfo); QUERY_CHECK_NULL(pOperatorInfo->pBQ, code, lino, _end, terrno); } @@ -780,7 +780,7 @@ static bool hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, ret); } TSKEY ekey = ascScan ? win.ekey : win.skey; @@ -827,7 +827,7 @@ static bool hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, code); } ekey = ascScan ? nextWin.ekey : nextWin.skey; @@ -2271,7 +2271,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, ret); } TSKEY ekey = ascScan ? win.ekey : win.skey; @@ -2312,7 +2312,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* int32_t code = outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &win); if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, code); } STimeWindow nextWin = win; @@ -2329,7 +2329,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, code); } ekey = ascScan ? nextWin.ekey : nextWin.skey; @@ -2354,7 +2354,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* code = outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &nextWin); if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, code); } } diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 5d397be623..8126e53bd6 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -1647,7 +1647,7 @@ SNode* qptMakeDownstreamSrcNode(SNode** ppNode) { pDs->addr.nodeId = qptCtx.param.vnode.vgId; memcpy(&pDs->addr.epSet, &qptCtx.param.vnode.epSet, sizeof(pDs->addr.epSet)); pDs->taskId = (QPT_CORRECT_HIGH_PROB() && qptCtx.buildCtx.pCurrTask) ? qptCtx.buildCtx.pCurrTask->id.taskId : taosRand(); - pDs->schedId = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.schedulerId : taosRand(); + pDs->sId = QPT_CORRECT_HIGH_PROB() ? 0 : taosRand(); pDs->execId = taosRand(); pDs->fetchMsgType = QPT_CORRECT_HIGH_PROB() ? (QPT_RAND_BOOL_V ? TDMT_SCH_FETCH : TDMT_SCH_MERGE_FETCH) : taosRand(); pDs->localExec = QPT_RAND_BOOL_V; @@ -3133,7 +3133,7 @@ void qptExecPlan(SReadHandle* pReadHandle, SNode* pNode, SExecTaskInfo* pTaskInf case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: case QUERY_NODE_PHYSICAL_PLAN_DELETE: { DataSinkHandle handle = NULL; - qptCtx.result.code = dsCreateDataSinker(NULL, (SDataSinkNode*)pNode, &handle, NULL, NULL); + qptCtx.result.code = dsCreateDataSinker(NULL, (SDataSinkNode**)&pNode, &handle, NULL, NULL); dsDestroyDataSinker(handle); break; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 554d01929d..1c77d12fb2 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2450,13 +2450,21 @@ static int32_t doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (IS_VAR_DATA_TYPE(type)) { - pInfo->bytes = varDataTLen(pData); + if (type == TSDB_DATA_TYPE_JSON) { + pInfo->bytes = getJsonValueLen(pData); + } else { + pInfo->bytes = varDataTLen(pData); + } } (void)memcpy(pInfo->buf, pData, pInfo->bytes); if (pkData != NULL) { if (IS_VAR_DATA_TYPE(pInfo->pkType)) { - pInfo->pkBytes = varDataTLen(pkData); + if (pInfo->pkType == TSDB_DATA_TYPE_JSON) { + pInfo->pkBytes = getJsonValueLen(pkData); + } else { + pInfo->pkBytes = varDataTLen(pkData); + } } (void)memcpy(pInfo->buf + pInfo->bytes, pkData, pInfo->pkBytes); pInfo->pkData = pInfo->buf + pInfo->bytes; @@ -2985,7 +2993,11 @@ static int32_t doSaveLastrow(SqlFunctionCtx* pCtx, char* pData, int32_t rowIndex pInfo->isNull = false; if (IS_VAR_DATA_TYPE(pInputCol->info.type)) { - pInfo->bytes = varDataTLen(pData); + if (pInputCol->info.type == TSDB_DATA_TYPE_JSON) { + pInfo->bytes = getJsonValueLen(pData); + } else { + pInfo->bytes = varDataTLen(pData); + } } (void)memcpy(pInfo->buf, pData, pInfo->bytes); @@ -2994,7 +3006,11 @@ static int32_t doSaveLastrow(SqlFunctionCtx* pCtx, char* pData, int32_t rowIndex if (pCtx->hasPrimaryKey && !colDataIsNull_s(pkCol, rowIndex)) { char* pkData = colDataGetData(pkCol, rowIndex); if (IS_VAR_DATA_TYPE(pInfo->pkType)) { - pInfo->pkBytes = varDataTLen(pkData); + if (pInfo->pkType == TSDB_DATA_TYPE_JSON) { + pInfo->pkBytes = getJsonValueLen(pkData); + } else { + pInfo->pkBytes = varDataTLen(pkData); + } } (void)memcpy(pInfo->buf + pInfo->bytes, pkData, pInfo->pkBytes); pInfo->pkData = pInfo->buf + pInfo->bytes; @@ -5874,7 +5890,11 @@ void modeFunctionCleanupExt(SqlFunctionCtx* pCtx) { static int32_t saveModeTupleData(SqlFunctionCtx* pCtx, char* data, SModeInfo* pInfo, STuplePos* pPos) { if (IS_VAR_DATA_TYPE(pInfo->colType)) { - (void)memcpy(pInfo->buf, data, varDataTLen(data)); + if (pInfo->colType == TSDB_DATA_TYPE_JSON) { + (void)memcpy(pInfo->buf, data, getJsonValueLen(data)); + } else { + (void)memcpy(pInfo->buf, data, varDataTLen(data)); + } } else { (void)memcpy(pInfo->buf, data, pInfo->colBytes); } @@ -5884,7 +5904,16 @@ static int32_t saveModeTupleData(SqlFunctionCtx* pCtx, char* data, SModeInfo* pI static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, char* data) { int32_t code = TSDB_CODE_SUCCESS; - int32_t hashKeyBytes = IS_STR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; + int32_t hashKeyBytes; + if (IS_VAR_DATA_TYPE(pInfo->colType)) { + if (pInfo->colType == TSDB_DATA_TYPE_JSON) { + hashKeyBytes = getJsonValueLen(data); + } else { + hashKeyBytes = varDataTLen(data); + } + } else { + hashKeyBytes = pInfo->colBytes; + } SModeItem* pHashItem = (SModeItem*)taosHashGet(pInfo->pHash, data, hashKeyBytes); if (pHashItem == NULL) { @@ -6769,14 +6798,32 @@ static void doSaveRateInfo(SRateInfo* pRateInfo, bool isFirst, int64_t ts, char* pRateInfo->firstValue = v; pRateInfo->firstKey = ts; if (pRateInfo->firstPk) { - int32_t pkBytes = IS_VAR_DATA_TYPE(pRateInfo->pkType) ? varDataTLen(pk) : pRateInfo->pkBytes; + int32_t pkBytes; + if (IS_VAR_DATA_TYPE(pRateInfo->pkType)) { + if (pRateInfo->pkType == TSDB_DATA_TYPE_JSON) { + pkBytes = getJsonValueLen(pk); + } else { + pkBytes = varDataTLen(pk); + } + } else { + pkBytes = pRateInfo->pkBytes; + } (void)memcpy(pRateInfo->firstPk, pk, pkBytes); } } else { pRateInfo->lastValue = v; pRateInfo->lastKey = ts; if (pRateInfo->lastPk) { - int32_t pkBytes = IS_VAR_DATA_TYPE(pRateInfo->pkType) ? varDataTLen(pk) : pRateInfo->pkBytes; + int32_t pkBytes; + if (IS_VAR_DATA_TYPE(pRateInfo->pkType)) { + if (pRateInfo->pkType == TSDB_DATA_TYPE_JSON) { + pkBytes = getJsonValueLen(pk); + } else { + pkBytes = varDataTLen(pk); + } + } else { + pkBytes = pRateInfo->pkBytes; + } (void)memcpy(pRateInfo->lastPk, pk, pkBytes); } } diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 73f400c93e..405049fe6f 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -44,7 +44,7 @@ static int32_t loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx, SFi pIdList = *(SArray **)p; } else { taosMemoryFree(*buffer); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } int32_t offset = 0; diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c index 8789762a85..9fed5c2a6e 100644 --- a/source/libs/geometry/src/geosWrapper.c +++ b/source/libs/geometry/src/geosWrapper.c @@ -99,8 +99,8 @@ static int32_t initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData return terrno; } - (void)sprintf( - wktPatternWithSpace, + (void)tsnprintf( + wktPatternWithSpace, 4 * 1024, "^( *)point( *)z?m?( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( " "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *)\\)))|linestring( *)z?m?( " "*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( " diff --git a/source/libs/monitorfw/src/taos_monitor_util.c b/source/libs/monitorfw/src/taos_monitor_util.c index b0eff27507..1240526d48 100644 --- a/source/libs/monitorfw/src/taos_monitor_util.c +++ b/source/libs/monitorfw/src/taos_monitor_util.c @@ -41,10 +41,11 @@ void taos_monitor_split_str_metric(char** arr, taos_metric_t* metric, const char memset(name, 0, size + 1); memcpy(name, metric->name, size); - char* s = strtok(name, del); + char* saveptr; + char* s = strtok_r(name, del, &saveptr); while (s != NULL) { *arr++ = s; - s = strtok(NULL, del); + s = strtok_r(NULL, del, &saveptr); } *buf = name; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 27c642470d..22f6dc7418 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -836,6 +836,44 @@ static int32_t physiProjectCopy(const SProjectPhysiNode* pSrc, SProjectPhysiNode return TSDB_CODE_SUCCESS; } +static int32_t dataSinkNodeCopy(const SDataSinkNode* pSrc, SDataSinkNode* pDst) { + CLONE_NODE_FIELD_EX(pInputDataBlockDesc, SDataBlockDescNode*); + return TSDB_CODE_SUCCESS; +} + +static int32_t physiDispatchCopy(const SDataDispatcherNode* pSrc, SDataDispatcherNode* pDst) { + COPY_BASE_OBJECT_FIELD(sink, dataSinkNodeCopy); + return TSDB_CODE_SUCCESS; +} + +static int32_t physiInserterCopy(const SDataInserterNode* pSrc, SDataInserterNode* pDst) { + COPY_BASE_OBJECT_FIELD(sink, dataSinkNodeCopy); + return TSDB_CODE_SUCCESS; +} + +static int32_t physiQueryInserterCopy(const SQueryInserterNode* pSrc, SQueryInserterNode* pDst) { + COPY_BASE_OBJECT_FIELD(sink, dataSinkNodeCopy); + CLONE_NODE_LIST_FIELD(pCols); + COPY_SCALAR_FIELD(tableId); + COPY_SCALAR_FIELD(stableId); + COPY_SCALAR_FIELD(tableType); + COPY_CHAR_ARRAY_FIELD(tableName); + COPY_SCALAR_FIELD(vgId); + COPY_OBJECT_FIELD(epSet, sizeof(SEpSet)); + COPY_SCALAR_FIELD(explain); + return TSDB_CODE_SUCCESS; +} + +static int32_t physiDeleterCopy(const SDataDeleterNode* pSrc, SDataDeleterNode* pDst) { + COPY_BASE_OBJECT_FIELD(sink, dataSinkNodeCopy); + COPY_SCALAR_FIELD(tableId); + COPY_SCALAR_FIELD(tableType); + COPY_CHAR_ARRAY_FIELD(tableFName); + COPY_CHAR_ARRAY_FIELD(tsColName); + COPY_OBJECT_FIELD(deleteTimeRange, sizeof(STimeWindow)); + return TSDB_CODE_SUCCESS; +} + static int32_t dataBlockDescCopy(const SDataBlockDescNode* pSrc, SDataBlockDescNode* pDst) { COPY_SCALAR_FIELD(dataBlockId); CLONE_NODE_LIST_FIELD(pSlots); @@ -858,7 +896,7 @@ static int32_t downstreamSourceCopy(const SDownstreamSourceNode* pSrc, SDownstre COPY_OBJECT_FIELD(addr, sizeof(SQueryNodeAddr)); COPY_SCALAR_FIELD(clientId); COPY_SCALAR_FIELD(taskId); - COPY_SCALAR_FIELD(schedId); + COPY_SCALAR_FIELD(sId); COPY_SCALAR_FIELD(execId); COPY_SCALAR_FIELD(fetchMsgType); COPY_SCALAR_FIELD(localExec); @@ -1091,6 +1129,18 @@ int32_t nodesCloneNode(const SNode* pNode, SNode** ppNode) { case QUERY_NODE_PHYSICAL_PLAN_PROJECT: code = physiProjectCopy((const SProjectPhysiNode*)pNode, (SProjectPhysiNode*)pDst); break; + case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: + code = physiDispatchCopy((const SDataDispatcherNode*)pNode, (SDataDispatcherNode*)pDst); + break; + //case QUERY_NODE_PHYSICAL_PLAN_INSERT: + // code = physiInserterCopy((const SDataInserterNode*)pNode, (SDataInserterNode*)pDst); + // break; + case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: + code = physiQueryInserterCopy((const SQueryInserterNode*)pNode, (SQueryInserterNode*)pDst); + break; + case QUERY_NODE_PHYSICAL_PLAN_DELETE: + code = physiDeleterCopy((const SDataDeleterNode*)pNode, (SDataDeleterNode*)pDst); + break; default: break; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 88f920b47f..6d4d89607f 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -183,6 +183,8 @@ const char* nodesNodeName(ENodeType type) { return "DescribeStmt"; case QUERY_NODE_COMPACT_DATABASE_STMT: return "CompactDatabaseStmt"; + case QUERY_NODE_COMPACT_VGROUPS_STMT: + return "CompactVgroupsStmt"; case QUERY_NODE_CREATE_STREAM_STMT: return "CreateStreamStmt"; case QUERY_NODE_DROP_STREAM_STMT: @@ -1262,7 +1264,7 @@ static const char* jkStreamOption_destHasPrimaryKey = "StreamOptionDestHasPrimar static int32_t streamNodeOptionToJson(const void* pObj, SJson* pJson) { const SStreamNodeOption* pNode = (const SStreamNodeOption*)pObj; - int32_t code = tjsonAddIntegerToObject(pJson, jkStreamOption_triggerType, pNode->triggerType); + int32_t code = tjsonAddIntegerToObject(pJson, jkStreamOption_triggerType, pNode->triggerType); if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkStreamOption_watermark, pNode->watermark); @@ -1284,7 +1286,7 @@ static int32_t streamNodeOptionToJson(const void* pObj, SJson* pJson) { static int32_t jsonToStreamNodeOption(const SJson* pJson, void* pObj) { SStreamNodeOption* pNode = (SStreamNodeOption*)pObj; - int32_t code = tjsonGetTinyIntValue(pJson, jkStreamOption_triggerType, &pNode->triggerType); + int32_t code = tjsonGetTinyIntValue(pJson, jkStreamOption_triggerType, &pNode->triggerType); if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkStreamOption_watermark, &pNode->watermark); @@ -1349,7 +1351,8 @@ static int32_t logicInterpFuncNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkInterpFuncLogicPlanTimeSeries, nodeToJson, pNode->pTimeSeries); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkInterpFuncLogicPlanStreamNodeOption, streamNodeOptionToJson, &pNode->streamNodeOption); + code = + tjsonAddObject(pJson, jkInterpFuncLogicPlanStreamNodeOption, streamNodeOptionToJson, &pNode->streamNodeOption); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkInterpFuncLogicPlanRangeInterval, pNode->rangeInterval); @@ -1393,7 +1396,8 @@ static int32_t jsonToLogicInterpFuncNode(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkInterpFuncLogicPlanTimeSeries, &pNode->pTimeSeries); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonToObject(pJson, jkInterpFuncLogicPlanStreamNodeOption, jsonToStreamNodeOption, &pNode->streamNodeOption); + code = + tjsonToObject(pJson, jkInterpFuncLogicPlanStreamNodeOption, jsonToStreamNodeOption, &pNode->streamNodeOption); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkInterpFuncLogicPlanRangeInterval, &pNode->rangeInterval); @@ -3394,7 +3398,8 @@ static int32_t physiInterpFuncNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkInterpFuncPhysiPlanTimeSeries, nodeToJson, pNode->pTimeSeries); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkInterpFuncPhysiPlanStreamNodeOption, streamNodeOptionToJson, &pNode->streamNodeOption); + code = + tjsonAddObject(pJson, jkInterpFuncPhysiPlanStreamNodeOption, streamNodeOptionToJson, &pNode->streamNodeOption); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkInterpFuncPhysiPlanRangeInterval, pNode->rangeInterval); @@ -3441,7 +3446,8 @@ static int32_t jsonToPhysiInterpFuncNode(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkInterpFuncPhysiPlanTimeSeries, &pNode->pTimeSeries); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonToObject(pJson, jkInterpFuncPhysiPlanStreamNodeOption, jsonToStreamNodeOption, &pNode->streamNodeOption); + code = + tjsonToObject(pJson, jkInterpFuncPhysiPlanStreamNodeOption, jsonToStreamNodeOption, &pNode->streamNodeOption); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkInterpFuncPhysiPlanRangeInterval, &pNode->rangeInterval); @@ -4240,7 +4246,7 @@ static int32_t datumToJson(const void* pObj, SJson* pJson) { case TSDB_DATA_TYPE_NCHAR: { // cJSON only support utf-8 encoding. Convert memory content to hex string. int32_t bufSize = varDataLen(pNode->datum.p) * 2 + 1; - char* buf = taosMemoryCalloc(bufSize, sizeof(char)); + char* buf = taosMemoryCalloc(bufSize, sizeof(char)); if (!buf) return terrno; code = taosHexEncode(varDataVal(pNode->datum.p), buf, varDataLen(pNode->datum.p), bufSize); if (code != TSDB_CODE_SUCCESS) { @@ -5333,7 +5339,7 @@ static int32_t jsonToColumnDefNode(const SJson* pJson, void* pObj) { static const char* jkDownstreamSourceAddr = "Addr"; static const char* jkDownstreamSourceClientId = "ClientId"; static const char* jkDownstreamSourceTaskId = "TaskId"; -static const char* jkDownstreamSourceSchedId = "SchedId"; +static const char* jkDownstreamSourceSeriousId = "SeriousId"; static const char* jkDownstreamSourceExecId = "ExecId"; static const char* jkDownstreamSourceFetchMsgType = "FetchMsgType"; @@ -5348,7 +5354,7 @@ static int32_t downstreamSourceNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceTaskId, pNode->taskId); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceSchedId, pNode->schedId); + code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceSeriousId, pNode->sId); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceExecId, pNode->execId); @@ -5371,7 +5377,7 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) { code = tjsonGetUBigIntValue(pJson, jkDownstreamSourceTaskId, &pNode->taskId); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetUBigIntValue(pJson, jkDownstreamSourceSchedId, &pNode->schedId); + code = tjsonGetUBigIntValue(pJson, jkDownstreamSourceSeriousId, &pNode->sId); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkDownstreamSourceExecId, &pNode->execId); @@ -5386,13 +5392,13 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) { static const char* jkWindowOffsetStartOffset = "StartOffset"; static const char* jkWindowOffsetEndOffset = "EndOffset"; static int32_t windowOffsetNodeToJson(const void* pObj, SJson* pJson) { - const SWindowOffsetNode* pNode = (const SWindowOffsetNode*)pObj; + const SWindowOffsetNode* pNode = (const SWindowOffsetNode*)pObj; - int32_t code = tjsonAddObject(pJson, jkWindowOffsetStartOffset, nodeToJson, pNode->pStartOffset); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkWindowOffsetEndOffset, nodeToJson, pNode->pEndOffset); + int32_t code = tjsonAddObject(pJson, jkWindowOffsetStartOffset, nodeToJson, pNode->pStartOffset); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkWindowOffsetEndOffset, nodeToJson, pNode->pEndOffset); } - return code; + return code; } static int32_t jsonToWindowOffsetNode(const SJson* pJson, void* pObj) { @@ -5428,6 +5434,9 @@ static const char* jkDatabaseOptionsS3ChunkSize = "S3ChunkSize"; static const char* jkDatabaseOptionsS3KeepLocalNode = "S3KeepLocalNode"; static const char* jkDatabaseOptionsS3KeepLocal = "S3KeepLocal"; static const char* jkDatabaseOptionsS3Compact = "S3Compact"; +static const char* jkDatabaseOptionsCompactIntervalNode = "compactIntervalNode"; +static const char* jkDatabaseOptionsCompactTimeRange = "compactTimeRange"; +static const char* jkDatabaseOptionsCompactTimeOffsetNode = "compactTimeOffsetNode"; static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) { const SDatabaseOptions* pNode = (const SDatabaseOptions*)pObj; @@ -5499,6 +5508,15 @@ static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsS3Compact, pNode->s3Compact); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDatabaseOptionsCompactIntervalNode, nodeToJson, pNode->pCompactIntervalNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkDatabaseOptionsCompactTimeRange, pNode->pCompactTimeRangeList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDatabaseOptionsCompactTimeOffsetNode, nodeToJson, pNode->pCompactTimeOffsetNode); + } return code; } @@ -5573,6 +5591,15 @@ static int32_t jsonToDatabaseOptions(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsS3Compact, &pNode->s3Compact); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDatabaseOptionsCompactIntervalNode, (SNode**)&pNode->pCompactIntervalNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkDatabaseOptionsCompactTimeRange, &pNode->pCompactTimeRangeList); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDatabaseOptionsCompactTimeOffsetNode, (SNode**)&pNode->pCompactTimeOffsetNode); + } return code; } @@ -5641,11 +5668,11 @@ static const char* jkColumnOptionsEncode = "encode"; static const char* jkColumnOptionsCompress = "compress"; static const char* jkColumnOptionsLevel = "level"; static int32_t columnOptionsToJson(const void* pObj, SJson* pJson) { - const SColumnOptions* pNode = (const SColumnOptions*)pObj; - int32_t code = tjsonAddStringToObject(pJson, jkColumnOptionsEncode, pNode->encode); - code = tjsonAddStringToObject(pJson, jkColumnOptionsCompress, pNode->compress); - code = tjsonAddStringToObject(pJson, jkColumnOptionsLevel, pNode->compressLevel); - return code; + const SColumnOptions* pNode = (const SColumnOptions*)pObj; + int32_t code = tjsonAddStringToObject(pJson, jkColumnOptionsEncode, pNode->encode); + code = tjsonAddStringToObject(pJson, jkColumnOptionsCompress, pNode->compress); + code = tjsonAddStringToObject(pJson, jkColumnOptionsLevel, pNode->compressLevel); + return code; } static int32_t jsonToColumnOptions(const SJson* pJson, void* pObj) { @@ -6816,7 +6843,7 @@ static int32_t dropQnodeStmtToJson(const void* pObj, SJson* pJson) { return drop static int32_t jsonToDropQnodeStmt(const SJson* pJson, void* pObj) { return jsonToDropComponentNodeStmt(pJson, pObj); } static const char* jkCreateAnodeStmtUrl = "Url"; -static const char* jkUpdateDropANodeStmtId = "AnodeId"; +static const char* jkUpdateDropANodeStmtId = "AnodeId"; static int32_t createAnodeStmtToJson(const void* pObj, SJson* pJson) { const SCreateAnodeStmt* pNode = (const SCreateAnodeStmt*)pObj; @@ -7158,6 +7185,16 @@ static int32_t jsonToCompactDatabaseStmt(const SJson* pJson, void* pObj) { return tjsonGetStringValue(pJson, jkCompactDatabaseStmtDbName, pNode->dbName); } +static int32_t compactVgroupsStmtToJson(const void* pObj, SJson* pJson) { + const SCompactVgroupsStmt* pNode = (const SCompactVgroupsStmt*)pObj; + return 0; +} + +static int32_t jsonToCompactVgroupsStmt(const SJson* pJson, void* pObj) { + SCompactVgroupsStmt* pNode = (SCompactVgroupsStmt*)pObj; + return 0; +} + static const char* jkCreateStreamStmtStreamName = "StreamName"; static const char* jkCreateStreamStmtTargetDbName = "TargetDbName"; static const char* jkCreateStreamStmtTargetTabName = "TargetTabName"; @@ -7490,7 +7527,7 @@ static int32_t jsonToShowClusterMachinesStmt(const SJson* pJson, void* pObj) { r static int32_t jsonToShowEncryptionsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } static int32_t showUsageStmtStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } -static int32_t jsonToShowUsageStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } +static int32_t jsonToShowUsageStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } static const char* jkShowDnodeVariablesStmtDnodeId = "DnodeId"; static const char* jkShowDnodeVariablesStmtLikePattern = "LikePattern"; @@ -8042,6 +8079,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return describeStmtToJson(pObj, pJson); case QUERY_NODE_COMPACT_DATABASE_STMT: return compactDatabaseStmtToJson(pObj, pJson); + case QUERY_NODE_COMPACT_VGROUPS_STMT: + return compactVgroupsStmtToJson(pObj, pJson); case QUERY_NODE_CREATE_STREAM_STMT: return createStreamStmtToJson(pObj, pJson); case QUERY_NODE_DROP_STREAM_STMT: @@ -8407,6 +8446,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToDescribeStmt(pJson, pObj); case QUERY_NODE_COMPACT_DATABASE_STMT: return jsonToCompactDatabaseStmt(pJson, pObj); + case QUERY_NODE_COMPACT_VGROUPS_STMT: + return jsonToCompactVgroupsStmt(pJson, pObj); case QUERY_NODE_CREATE_STREAM_STMT: return jsonToCreateStreamStmt(pJson, pObj); case QUERY_NODE_DROP_STREAM_STMT: diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 4992c2a06b..930a88aea0 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -1761,7 +1761,7 @@ static int32_t downstreamSourceNodeInlineToMsg(const void* pObj, STlvEncoder* pE code = tlvEncodeValueU64(pEncoder, pNode->taskId); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeValueU64(pEncoder, pNode->schedId); + code = tlvEncodeValueU64(pEncoder, pNode->sId); } if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueI32(pEncoder, pNode->execId); @@ -1788,7 +1788,7 @@ static int32_t msgToDownstreamSourceNodeInlineToMsg(STlvDecoder* pDecoder, void* code = tlvDecodeValueU64(pDecoder, &pNode->taskId); } if (TSDB_CODE_SUCCESS == code) { - code = tlvDecodeValueU64(pDecoder, &pNode->schedId); + code = tlvDecodeValueU64(pDecoder, &pNode->sId); } if (TSDB_CODE_SUCCESS == code) { code = tlvDecodeValueI32(pDecoder, &pNode->execId); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 847b921ddf..7beaeaa46c 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -471,7 +471,8 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) { code = makeNode(type, sizeof(SSetOperator), &pNode); break; case QUERY_NODE_RANGE_AROUND: - code = makeNode(type, sizeof(SRangeAroundNode), &pNode); break; + code = makeNode(type, sizeof(SRangeAroundNode), &pNode); + break; case QUERY_NODE_SELECT_STMT: code = makeNode(type, sizeof(SSelectStmt), &pNode); break; @@ -593,6 +594,9 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) { case QUERY_NODE_COMPACT_DATABASE_STMT: code = makeNode(type, sizeof(SCompactDatabaseStmt), &pNode); break; + case QUERY_NODE_COMPACT_VGROUPS_STMT: + code = makeNode(type, sizeof(SCompactVgroupsStmt), &pNode); + break; case QUERY_NODE_CREATE_FUNCTION_STMT: code = makeNode(type, sizeof(SCreateFunctionStmt), &pNode); break; @@ -1079,7 +1083,7 @@ void nodesDestroyNode(SNode* pNode) { taosMemoryFreeClear(pReal->pMeta); taosMemoryFreeClear(pReal->pVgroupList); taosArrayDestroyEx(pReal->pSmaIndexes, destroySmaIndex); - taosArrayDestroyP(pReal->tsmaTargetTbVgInfo, taosMemoryFree); + taosArrayDestroyP(pReal->tsmaTargetTbVgInfo, NULL); taosArrayDestroy(pReal->tsmaTargetTbInfo); break; } @@ -1156,6 +1160,9 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode((SNode*)pOptions->s3KeepLocalStr); nodesDestroyList(pOptions->pKeep); nodesDestroyList(pOptions->pRetentions); + nodesDestroyNode((SNode*)pOptions->pCompactIntervalNode); + nodesDestroyList(pOptions->pCompactTimeRangeList); + nodesDestroyNode((SNode*)pOptions->pCompactTimeOffsetNode); break; } case QUERY_NODE_TABLE_OPTIONS: { @@ -1320,14 +1327,20 @@ void nodesDestroyNode(SNode* pNode) { } break; } - case QUERY_NODE_CREATE_DATABASE_STMT: - nodesDestroyNode((SNode*)((SCreateDatabaseStmt*)pNode)->pOptions); + case QUERY_NODE_CREATE_DATABASE_STMT: { + SDatabaseOptions* pOptions = ((SCreateDatabaseStmt*)pNode)->pOptions; + taosMemoryFreeClear(pOptions->pDbCfg); + nodesDestroyNode((SNode*)pOptions); break; + } case QUERY_NODE_DROP_DATABASE_STMT: // no pointer field break; - case QUERY_NODE_ALTER_DATABASE_STMT: - nodesDestroyNode((SNode*)((SAlterDatabaseStmt*)pNode)->pOptions); + case QUERY_NODE_ALTER_DATABASE_STMT: { + SDatabaseOptions* pOptions = ((SAlterDatabaseStmt*)pNode)->pOptions; + taosMemoryFreeClear(pOptions->pDbCfg); + nodesDestroyNode((SNode*)pOptions); break; + } case QUERY_NODE_FLUSH_DATABASE_STMT: // no pointer field case QUERY_NODE_TRIM_DATABASE_STMT: // no pointer field break; @@ -1443,6 +1456,14 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pStmt->pEnd); break; } + case QUERY_NODE_COMPACT_VGROUPS_STMT: { + SCompactVgroupsStmt* pStmt = (SCompactVgroupsStmt*)pNode; + nodesDestroyNode(pStmt->pDbName); + nodesDestroyList(pStmt->vgidList); + nodesDestroyNode(pStmt->pStart); + nodesDestroyNode(pStmt->pEnd); + break; + } case QUERY_NODE_CREATE_FUNCTION_STMT: // no pointer field case QUERY_NODE_DROP_FUNCTION_STMT: // no pointer field break; @@ -1627,7 +1648,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyList(pLogicNode->pTags); nodesDestroyNode(pLogicNode->pSubtable); taosArrayDestroyEx(pLogicNode->pFuncTypes, destroyFuncParam); - taosArrayDestroyP(pLogicNode->pTsmaTargetTbVgInfo, taosMemoryFree); + taosArrayDestroyP(pLogicNode->pTsmaTargetTbVgInfo, NULL); taosArrayDestroy(pLogicNode->pTsmaTargetTbInfo); break; } @@ -2012,7 +2033,7 @@ int32_t nodesMakeList(SNodeList** ppListOut) { int32_t nodesListAppend(SNodeList* pList, SNode* pNode) { if (NULL == pList || NULL == pNode) { - return TSDB_CODE_FAILED; + return TSDB_CODE_INVALID_PARA; } SListCell* p = NULL; int32_t code = nodesCalloc(1, sizeof(SListCell), (void**)&p); diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index a7453a17b1..e69a3da4a9 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -70,6 +70,9 @@ typedef enum EDatabaseOptionType { DB_OPTION_KEEP_TIME_OFFSET, DB_OPTION_ENCRYPT_ALGORITHM, DB_OPTION_DNODES, + DB_OPTION_COMPACT_INTERVAL, + DB_OPTION_COMPACT_TIME_RANGE, + DB_OPTION_COMPACT_TIME_OFFSET, } EDatabaseOptionType; typedef enum ETableOptionType { @@ -200,6 +203,8 @@ SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t maxSpeed); SNode* createS3MigrateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart, SNode* pEnd); +SNode* createCompactVgroupsStmt(SAstCreateContext* pCxt, SNode* pDbName, SNodeList* vgidList, SNode* pStart, + SNode* pEnd); SNode* createDefaultTableOptions(SAstCreateContext* pCxt); SNode* createAlterTableOptions(SAstCreateContext* pCxt); SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, void* pVal); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 9e45c5f75e..63eb09d509 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -229,6 +229,7 @@ cmd ::= FLUSH DATABASE db_name(A). cmd ::= TRIM DATABASE db_name(A) speed_opt(B). { pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &A, B); } cmd ::= S3MIGRATE DATABASE db_name(A). { pCxt->pRootNode = createS3MigrateDatabaseStmt(pCxt, &A); } cmd ::= COMPACT DATABASE db_name(A) start_opt(B) end_opt(C). { pCxt->pRootNode = createCompactStmt(pCxt, &A, B, C); } +cmd ::= COMPACT db_name_cond_opt(A) VGROUPS IN NK_LP integer_list(B) NK_RP start_opt(C) end_opt(D). { pCxt->pRootNode = createCompactVgroupsStmt(pCxt, A, B, C, D); } %type not_exists_opt { bool } %destructor not_exists_opt { } @@ -287,6 +288,12 @@ db_options(A) ::= db_options(B) S3_COMPACT NK_INTEGER(C). db_options(A) ::= db_options(B) KEEP_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP_TIME_OFFSET, &C); } db_options(A) ::= db_options(B) ENCRYPT_ALGORITHM NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_ENCRYPT_ALGORITHM, &C); } db_options(A) ::= db_options(B) DNODES NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DNODES, &C); } +db_options(A) ::= db_options(B) COMPACT_INTERVAL NK_INTEGER (C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_INTERVAL, &C); } +db_options(A) ::= db_options(B) COMPACT_INTERVAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_INTERVAL, &C); } +db_options(A) ::= db_options(B) COMPACT_TIME_RANGE signed_integer_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); } +db_options(A) ::= db_options(B) COMPACT_TIME_RANGE signed_variable_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); } +db_options(A) ::= db_options(B) COMPACT_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_OFFSET, &C); } +db_options(A) ::= db_options(B) COMPACT_TIME_OFFSET NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_OFFSET, &C); } alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); } alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setAlterDatabaseOption(pCxt, B, &C); } @@ -322,17 +329,33 @@ alter_db_option(A) ::= S3_KEEPLOCAL NK_VARIABLE(B). alter_db_option(A) ::= S3_COMPACT NK_INTEGER(B). { A.type = DB_OPTION_S3_COMPACT, A.val = B; } alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_KEEP_TIME_OFFSET; A.val = B; } alter_db_option(A) ::= ENCRYPT_ALGORITHM NK_STRING(B). { A.type = DB_OPTION_ENCRYPT_ALGORITHM; A.val = B; } +alter_db_option(A) ::= COMPACT_INTERVAL NK_INTEGER(B). { A.type = DB_OPTION_COMPACT_INTERVAL; A.val = B; } +alter_db_option(A) ::= COMPACT_INTERVAL NK_VARIABLE(B). { A.type = DB_OPTION_COMPACT_INTERVAL; A.val = B; } +alter_db_option(A) ::= COMPACT_TIME_RANGE signed_integer_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } +alter_db_option(A) ::= COMPACT_TIME_RANGE signed_variable_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } +alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_COMPACT_TIME_OFFSET; A.val = B; } +alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_VARIABLE(B). { A.type = DB_OPTION_COMPACT_TIME_OFFSET; A.val = B; } %type integer_list { SNodeList* } %destructor integer_list { nodesDestroyList($$); } integer_list(A) ::= NK_INTEGER(B). { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); } integer_list(A) ::= integer_list(B) NK_COMMA NK_INTEGER(C). { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); } +%type signed_integer_list { SNodeList* } +%destructor signed_integer_list { nodesDestroyList($$); } +signed_integer_list(A) ::= signed_integer(B). { A = createNodeList(pCxt, B); } +signed_integer_list(A) ::= signed_integer_list(B) NK_COMMA signed_integer(C). { A = addNodeToList(pCxt, B, C); } + %type variable_list { SNodeList* } %destructor variable_list { nodesDestroyList($$); } variable_list(A) ::= NK_VARIABLE(B). { A = createNodeList(pCxt, createDurationValueNode(pCxt, &B)); } variable_list(A) ::= variable_list(B) NK_COMMA NK_VARIABLE(C). { A = addNodeToList(pCxt, B, createDurationValueNode(pCxt, &C)); } +%type signed_variable_list { SNodeList* } +%destructor signed_variable_list { nodesDestroyList($$); } +signed_variable_list(A) ::= signed_variable(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } +signed_variable_list(A) ::= signed_variable_list(B) NK_COMMA signed_variable(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } + %type retention_list { SNodeList* } %destructor retention_list { nodesDestroyList($$); } retention_list(A) ::= retention(B). { A = createNodeList(pCxt, B); } @@ -1044,21 +1067,33 @@ literal(A) ::= NK_QUESTION(B). duration_literal(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } -signed(A) ::= NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); } -signed(A) ::= NK_PLUS NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); } -signed(A) ::= NK_MINUS(B) NK_INTEGER(C). { +signed_variable(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } +signed_variable(A) ::= NK_PLUS NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } +signed_variable(A) ::= NK_MINUS(B) NK_VARIABLE(C). { + SToken t = B; + t.n = (C.z + C.n) - B.z; + A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &t)); + } + +signed_integer(A) ::= NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); } +signed_integer(A) ::= NK_PLUS NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); } +signed_integer(A) ::= NK_MINUS(B) NK_INTEGER(C). { SToken t = B; t.n = (C.z + C.n) - B.z; A = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } -signed(A) ::= NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } -signed(A) ::= NK_PLUS NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } -signed(A) ::= NK_MINUS(B) NK_FLOAT(C). { + +signed_float(A) ::= NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } +signed_float(A) ::= NK_PLUS NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } +signed_float(A) ::= NK_MINUS(B) NK_FLOAT(C). { SToken t = B; t.n = (C.z + C.n) - B.z; A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } +signed(A) ::= signed_integer(B). { A = B; } +signed(A) ::= signed_float(B). { A = B; } + signed_literal(A) ::= signed(B). { A = B; } signed_literal(A) ::= NK_STRING(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } signed_literal(A) ::= NK_BOOL(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 5ecef23de7..a13472620b 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1838,6 +1838,10 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { pOptions->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; pOptions->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; pOptions->dnodeListStr[0] = 0; + pOptions->compactInterval = TSDB_DEFAULT_COMPACT_INTERVAL; + pOptions->compactStartTime = TSDB_DEFAULT_COMPACT_START_TIME; + pOptions->compactEndTime = TSDB_DEFAULT_COMPACT_END_TIME; + pOptions->compactTimeOffset = TSDB_DEFAULT_COMPACT_TIME_OFFSET; return (SNode*)pOptions; _err: return NULL; @@ -1882,6 +1886,10 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { pOptions->withArbitrator = -1; pOptions->encryptAlgorithm = -1; pOptions->dnodeListStr[0] = 0; + pOptions->compactInterval = -1; + pOptions->compactStartTime = -1; + pOptions->compactEndTime = -1; + pOptions->compactTimeOffset = -1; return (SNode*)pOptions; _err: return NULL; @@ -2029,6 +2037,24 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED } else { COPY_STRING_FORM_STR_TOKEN(pDbOptions->dnodeListStr, (SToken*)pVal); } + break; + case DB_OPTION_COMPACT_INTERVAL: + if (TK_NK_INTEGER == ((SToken*)pVal)->type) { + pDbOptions->compactInterval = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + } else { + pDbOptions->pCompactIntervalNode = (SValueNode*)createDurationValueNode(pCxt, (SToken*)pVal); + } + break; + case DB_OPTION_COMPACT_TIME_RANGE: + pDbOptions->pCompactTimeRangeList = pVal; + break; + case DB_OPTION_COMPACT_TIME_OFFSET: + if (TK_NK_INTEGER == ((SToken*)pVal)->type) { + pDbOptions->compactTimeOffset = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + } else { + pDbOptions->pCompactTimeOffsetNode = (SValueNode*)createDurationValueNode(pCxt, (SToken*)pVal); + } + break; default: break; } @@ -2038,6 +2064,8 @@ _err: return NULL; } + + SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal) { return setDatabaseOptionImpl(pCxt, pOptions, type, pVal, false); } @@ -2047,6 +2075,7 @@ SNode* setAlterDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOp switch (pAlterOption->type) { case DB_OPTION_KEEP: case DB_OPTION_RETENTIONS: + case DB_OPTION_COMPACT_TIME_RANGE: return setDatabaseOptionImpl(pCxt, pOptions, pAlterOption->type, pAlterOption->pList, true); default: break; @@ -2153,6 +2182,30 @@ _err: return NULL; } +SNode* createCompactVgroupsStmt(SAstCreateContext* pCxt, SNode* pDbName, SNodeList* vgidList, SNode* pStart, + SNode* pEnd) { + CHECK_PARSER_STATUS(pCxt); + if (NULL == pDbName) { + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "database not specified"); + pCxt->errCode = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + CHECK_PARSER_STATUS(pCxt); + } + SCompactVgroupsStmt* pStmt = NULL; + pCxt->errCode = nodesMakeNode(QUERY_NODE_COMPACT_VGROUPS_STMT, (SNode**)&pStmt); + CHECK_MAKE_NODE(pStmt); + pStmt->pDbName = pDbName; + pStmt->vgidList = vgidList; + pStmt->pStart = pStart; + pStmt->pEnd = pEnd; + return (SNode*)pStmt; +_err: + nodesDestroyNode(pDbName); + nodesDestroyList(vgidList); + nodesDestroyNode(pStart); + nodesDestroyNode(pEnd); + return NULL; +} + SNode* createDefaultTableOptions(SAstCreateContext* pCxt) { CHECK_PARSER_STATUS(pCxt); STableOptions* pOptions = NULL; @@ -2887,13 +2940,16 @@ static int32_t getIpV4RangeFromWhitelistItem(char* ipRange, SIpV4Range* pIpRange *slash = '\0'; struct in_addr addr; if (uv_inet_pton(AF_INET, ipCopy, &addr) == 0) { - int prefix = atoi(slash + 1); - if (prefix < 0 || prefix > 32) { - code = TSDB_CODE_PAR_INVALID_IP_RANGE; - } else { - pIpRange->ip = addr.s_addr; - pIpRange->mask = prefix; - code = TSDB_CODE_SUCCESS; + int32_t prefix = 0; + code = taosStr2int32(slash + 1, &prefix); + if (code == 0) { + if (prefix < 0 || prefix > 32) { + code = TSDB_CODE_PAR_INVALID_IP_RANGE; + } else { + pIpRange->ip = addr.s_addr; + pIpRange->mask = prefix; + code = TSDB_CODE_SUCCESS; + } } } else { code = TSDB_CODE_PAR_INVALID_IP_RANGE; diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 3aa60c283e..1687916cb0 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -46,7 +46,7 @@ int32_t buildQueryAfterParse(SQuery** pQuery, SNode* pRootNode, int16_t placehol int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) { SAstCreateContext cxt; initAstCreateContext(pParseCxt, &cxt); - void* pParser = ParseAlloc((FMalloc)taosMemoryMalloc); + void* pParser = ParseAlloc((FMalloc)taosMemMalloc); if (!pParser) return terrno; int32_t i = 0; while (1) { @@ -88,7 +88,7 @@ int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) { } abort_parse: - ParseFree(pParser, (FFree)taosMemoryFree); + ParseFree(pParser, (FFree)taosAutoMemoryFree); if (TSDB_CODE_SUCCESS == cxt.errCode) { int32_t code = buildQueryAfterParse(pQuery, cxt.pRootNode, cxt.placeholderNo, &cxt.pPlaceholderValues); if (TSDB_CODE_SUCCESS != code) { @@ -184,7 +184,8 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c } if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS) || 0 == strcmp(pTable, TSDB_INS_TABLE_TABLES) || - 0 == strcmp(pTable, TSDB_INS_TABLE_COLS) || 0 == strcmp(pTable, TSDB_INS_DISK_USAGE)) && + 0 == strcmp(pTable, TSDB_INS_TABLE_COLS) || 0 == strcmp(pTable, TSDB_INS_DISK_USAGE) || + 0 == strcmp(pTable, TSDB_INS_TABLE_FILESETS)) && QUERY_NODE_SELECT_STMT == nodeType(pCxt->pStmt)) { code = collectMetaKeyFromInsTags(pCxt); } @@ -641,6 +642,20 @@ static int32_t collectMetaKeyFromShowTables(SCollectMetaKeyCxt* pCxt, SShowStmt* return code; } +static int32_t collectMetaKeyFromShowFilesets(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { + int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_FILESETS, + pCxt->pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache); + } + if (TSDB_CODE_SUCCESS == code) { + code = + reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, ((SValueNode*)pStmt->pDbName)->literal, + NULL, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache); + } + return code; +} + static int32_t collectMetaKeyFromShowTags(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TAGS, pCxt->pMetaCache); @@ -859,6 +874,10 @@ static int32_t collectMetaKeyFromCompactDatabase(SCollectMetaKeyCxt* pCxt, SComp return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); } +static int32_t collectMetaKeyFromCompactVgroups(SCollectMetaKeyCxt* pCxt, SCompactVgroupsStmt* pStmt) { + return reserveDbCfgInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache); +} + static int32_t collectMetaKeyFromGrant(SCollectMetaKeyCxt* pCxt, SGrantStmt* pStmt) { if ('\0' == pStmt->tabName[0]) { return TSDB_CODE_SUCCESS; @@ -999,6 +1018,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromDescribe(pCxt, (SDescribeStmt*)pStmt); case QUERY_NODE_COMPACT_DATABASE_STMT: return collectMetaKeyFromCompactDatabase(pCxt, (SCompactDatabaseStmt*)pStmt); + case QUERY_NODE_COMPACT_VGROUPS_STMT: + return collectMetaKeyFromCompactVgroups(pCxt, (SCompactVgroupsStmt*)pStmt); case QUERY_NODE_CREATE_STREAM_STMT: return collectMetaKeyFromCreateStream(pCxt, (SCreateStreamStmt*)pStmt); case QUERY_NODE_GRANT_STMT: @@ -1037,6 +1058,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromShowStreams(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_TABLES_STMT: return collectMetaKeyFromShowTables(pCxt, (SShowStmt*)pStmt); + case QUERY_NODE_SHOW_FILESETS_STMT: + return collectMetaKeyFromShowFilesets(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_TAGS_STMT: return collectMetaKeyFromShowTags(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_TABLE_TAGS_STMT: diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index c5e9ad4ae4..e2069deefc 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1849,10 +1849,9 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* tstrncpy(pStbRowsCxt->ctbName.tname, tbName, sizeof(pStbRowsCxt->ctbName.tname)); tstrncpy(pStmt->usingTableName.tname, pStmt->targetTableName.tname, sizeof(pStmt->usingTableName.tname)); tstrncpy(pStmt->targetTableName.tname, tbName, sizeof(pStmt->targetTableName.tname)); - tstrncpy(pStmt->usingTableName.dbname, pStmt->targetTableName.dbname, sizeof(pStmt->usingTableName.dbname)); pStmt->usingTableName.type = 1; - + pStmt->pTableMeta->tableType = TSDB_CHILD_TABLE; // set the table type to child table for parse cache *bFoundTbName = true; } } else if (pCols->pColIndex[i] < numOfCols) { @@ -2060,9 +2059,7 @@ static int32_t parseStbBoundInfo(SVnodeModifyOpStmt* pStmt, SStbRowsDataContext* insDestroyBoundColInfo(&((*ppTableDataCxt)->boundColsInfo)); (*ppTableDataCxt)->boundColsInfo = pStbRowsCxt->boundColsInfo; - (*ppTableDataCxt)->boundColsInfo.numOfCols = pStbRowsCxt->boundColsInfo.numOfBound; - (*ppTableDataCxt)->boundColsInfo.numOfBound = pStbRowsCxt->boundColsInfo.numOfBound; - (*ppTableDataCxt)->boundColsInfo.hasBoundCols = pStbRowsCxt->boundColsInfo.hasBoundCols; + (*ppTableDataCxt)->boundColsInfo.pColIndex = taosMemoryCalloc(pStbRowsCxt->boundColsInfo.numOfBound, sizeof(int16_t)); if (NULL == (*ppTableDataCxt)->boundColsInfo.pColIndex) { return terrno; @@ -3175,9 +3172,8 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal .isStmtBind = pCxt->isStmtBind}; int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery); - SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)((*pQuery)->pRoot); if (TSDB_CODE_SUCCESS == code) { - code = parseInsertSqlImpl(&context, pStmt); + code = parseInsertSqlImpl(&context, (SVnodeModifyOpStmt*)((*pQuery)->pRoot)); } if (TSDB_CODE_SUCCESS == code) { code = setNextStageInfo(&context, *pQuery, pCatalogReq); diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 9e6abe1517..c69abea0ca 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -606,6 +606,13 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c code = terrno; goto end; } + } else { + SVCreateTbReq* tmp = pDataBlock->pData->pCreateTbReq; + taosMemoryFreeClear(tmp->name); + taosMemoryFreeClear(tmp->ctb.pTag); + taosMemoryFreeClear(tmp->ctb.stbName); + taosArrayDestroy(tmp->ctb.tagName); + tmp->ctb.tagName = NULL; } code = insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName, @@ -1017,11 +1024,11 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA if (NULL == tags) { return TSDB_CODE_APP_ERROR; } - /* + if (pDataBlock->pMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pMeta->tableType != TSDB_CHILD_TABLE) { return TSDB_CODE_TSC_STMT_API_ERROR; } - */ + SSchema* pSchema = getTableTagSchema(pDataBlock->pMeta); if (tags->numOfBound <= 0) { *fieldNum = 0; diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index b38e2f05f0..ea2e9d712f 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -72,6 +72,9 @@ static SKeyword keywordTable[] = { {"COMP", TK_COMP}, {"COMPACT", TK_COMPACT}, {"COMPACTS", TK_COMPACTS}, + {"COMPACT_INTERVAL", TK_COMPACT_INTERVAL}, + {"COMPACT_TIME_OFFSET", TK_COMPACT_TIME_OFFSET}, + {"COMPACT_TIME_RANGE", TK_COMPACT_TIME_RANGE}, {"CONNECTION", TK_CONNECTION}, {"CONNECTIONS", TK_CONNECTIONS}, {"CONNS", TK_CONNS}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a36d44db7a..f191080512 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "parTranslater.h" #include "parInt.h" +#include "parTranslater.h" #include "tdatablock.h" #include "catalog.h" @@ -370,6 +370,13 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = { .numOfShowCols = 1, .pShowCols = {"*"} }, + { + .showType = QUERY_NODE_SHOW_FILESETS_STMT, + .pDbName = TSDB_INFORMATION_SCHEMA_DB, + .pTableName = TSDB_INS_TABLE_FILESETS, + .numOfShowCols = 1, + .pShowCols = {"*"} + }, }; // clang-format on @@ -3809,7 +3816,8 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { bool partionByTbname = hasTbnameFunction(pSelect->pPartitionByList); FOREACH(pPartKey, pSelect->pPartitionByList) { if (nodesEqualNode(pPartKey, *pNode)) { - return (pSelect->hasAggFuncs || pSelect->pWindow) ? rewriteExprToGroupKeyFunc(pCxt, pNode) : DEAL_RES_IGNORE_CHILD; + return (pSelect->hasAggFuncs || pSelect->pWindow) ? rewriteExprToGroupKeyFunc(pCxt, pNode) + : DEAL_RES_IGNORE_CHILD; } if ((partionByTbname) && QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) { @@ -4122,7 +4130,8 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) { static bool sysTableFromVnode(const char* pTable) { return ((0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) || (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)) || - (0 == strcmp(pTable, TSDB_INS_TABLE_COLS)) || 0 == strcmp(pTable, TSDB_INS_DISK_USAGE)); + (0 == strcmp(pTable, TSDB_INS_TABLE_COLS)) || 0 == strcmp(pTable, TSDB_INS_DISK_USAGE) || + (0 == strcmp(pTable, TSDB_INS_TABLE_FILESETS))); } static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); } @@ -4195,7 +4204,8 @@ static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, if (TSDB_CODE_SUCCESS == code && ((0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && !hasUserDbCond) || 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_COLS) || - (0 == strcmp(pRealTable->table.tableName, TSDB_INS_DISK_USAGE) && !hasUserDbCond))) { + (0 == strcmp(pRealTable->table.tableName, TSDB_INS_DISK_USAGE) && !hasUserDbCond) || + 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_FILESETS))) { code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &pVgs); } @@ -4295,7 +4305,8 @@ static bool isSingleTable(SRealTableNode* pRealTable) { return 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS) && 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_COLS) && - 0 != strcmp(pRealTable->table.tableName, TSDB_INS_DISK_USAGE); + 0 != strcmp(pRealTable->table.tableName, TSDB_INS_DISK_USAGE) && + 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_FILESETS); } return (TSDB_CHILD_TABLE == tableType || TSDB_NORMAL_TABLE == tableType); } @@ -4322,7 +4333,7 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo if (TSDB_CODE_SUCCESS == code && pRealTable->pTsmas && (pRealTable->pMeta->tableType == TSDB_CHILD_TABLE || pRealTable->pMeta->tableType == TSDB_NORMAL_TABLE)) { if (pRealTable->tsmaTargetTbVgInfo) { - taosArrayDestroyP(pRealTable->tsmaTargetTbVgInfo, taosMemoryFree); + taosArrayDestroyP(pRealTable->tsmaTargetTbVgInfo, NULL); pRealTable->tsmaTargetTbVgInfo = NULL; } char buf[TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN + 1]; @@ -5439,7 +5450,8 @@ static int32_t doCheckFillValues(STranslateContext* pCxt, SFillNode* pFill, SNod int32_t fillNo = 0; SNodeListNode* pFillValues = (SNodeListNode*)pFill->pValues; SNode* pProject = NULL; - if (!pFillValues) return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch"); + if (!pFillValues) + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch"); FOREACH(pProject, pProjectionList) { if (needFill(pProject)) { if (fillNo >= LIST_LENGTH(pFillValues->pNodeList)) { @@ -6292,7 +6304,7 @@ static int32_t translateInterpEvery(STranslateContext* pCxt, SNode** pEvery) { } static EDealRes hasRowTsOriginFuncWalkNode(SNode* pNode, void* ctx) { - bool *hasRowTsOriginFunc = ctx; + bool* hasRowTsOriginFunc = ctx; if (nodeType(pNode) == QUERY_NODE_FUNCTION) { SFunctionNode* pFunc = (SFunctionNode*)pNode; if (fmIsRowTsOriginFunc(pFunc->funcId)) { @@ -6307,10 +6319,12 @@ static int32_t checkInterpForStream(STranslateContext* pCxt, SSelectStmt* pSelec if (pCxt->createStream) { SFillNode* pFill = (SFillNode*)pSelect->pFill; if (pFill->mode == FILL_MODE_NEAR) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "FILL NEAR is not supported by stream"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "FILL NEAR is not supported by stream"); } if (pSelect->pRangeAround) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "RANGE with around is not supported by stream"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "RANGE with around is not supported by stream"); } } return TSDB_CODE_SUCCESS; @@ -6335,24 +6349,25 @@ static int32_t translateInterpFill(STranslateContext* pCxt, SSelectStmt* pSelect nodesWalkExprs(pSelect->pProjectionList, hasRowTsOriginFuncWalkNode, &hasRowTsOriginFunc); if (hasRowTsOriginFunc && pCxt->createStream) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "_irowts_origin is not supported by stream"); + "_irowts_origin is not supported by stream"); } if (TSDB_CODE_SUCCESS == code) { SFillNode* pFill = (SFillNode*)pSelect->pFill; if (pSelect->pRangeAround) { if (pFill->mode != FILL_MODE_PREV && pFill->mode != FILL_MODE_NEXT && pFill->mode != FILL_MODE_NEAR) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, - "Range with interval can only used with fill PREV/NEXT/NEAR"); + "Range with interval can only used with fill PREV/NEXT/NEAR"); } - if (TSDB_CODE_SUCCESS == code) - code = doCheckFillValues(pCxt, pFill, pSelect->pProjectionList); + if (TSDB_CODE_SUCCESS == code) code = doCheckFillValues(pCxt, pFill, pSelect->pProjectionList); } else { if (FILL_MODE_PREV == pFill->mode || FILL_MODE_NEXT == pFill->mode || FILL_MODE_NEAR == pFill->mode) { if (pFill->pValues) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Can't specify fill values"); } } else { - if (hasRowTsOriginFunc) return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC, "_irowts_origin can only be used with FILL PREV/NEXT/NEAR"); + if (hasRowTsOriginFunc) + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC, + "_irowts_origin can only be used with FILL PREV/NEXT/NEAR"); } code = checkFillValues(pCxt, pFill, pSelect->pProjectionList); } @@ -6370,7 +6385,8 @@ static int32_t translateInterpAround(STranslateContext* pCxt, SSelectStmt* pSele if (nodeType(pAround->pInterval) == QUERY_NODE_VALUE) { SValueNode* pVal = (SValueNode*)pAround->pInterval; if (pVal->datum.i == 0) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, "Range interval cannot be 0"); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, + "Range interval cannot be 0"); } int8_t unit = pVal->unit; if (unit == TIME_UNIT_YEAR || unit == TIME_UNIT_MONTH) { @@ -6423,7 +6439,7 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { // single point interp every can be omitted } else { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, - "Missing RANGE clause, EVERY clause or FILL clause"); + "Missing RANGE clause, EVERY clause or FILL clause"); } } } else { @@ -6432,8 +6448,7 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { "Range clause with around interval can't be used with EVERY clause"); } if (!pSelect->pFill) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, - "Missing FILL clause"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, "Missing FILL clause"); } } } @@ -6926,7 +6941,7 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* taosMemoryFreeClear(vgsInfo); } } - taosArrayDestroyP(pTbNames, taosMemoryFree); + taosArrayDestroyP(pTbNames, NULL); if (code) break; } if (TSDB_CODE_SUCCESS != code) { @@ -7755,6 +7770,12 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS pReq->encryptAlgorithm = pStmt->pOptions->encryptAlgorithm; tstrncpy(pReq->dnodeListStr, pStmt->pOptions->dnodeListStr, TSDB_DNODE_LIST_LEN); + // auto-compact options + pReq->compactInterval = pStmt->pOptions->compactInterval; + pReq->compactStartTime = pStmt->pOptions->compactStartTime; + pReq->compactEndTime = pStmt->pOptions->compactEndTime; + pReq->compactTimeOffset = pStmt->pOptions->compactTimeOffset; + return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq); } @@ -7768,8 +7789,8 @@ static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const cha return TSDB_CODE_SUCCESS; } -static int32_t checkDbRangeOption(STranslateContext* pCxt, const char* pName, int32_t val, int32_t minVal, - int32_t maxVal) { +static int32_t checkDbRangeOption(STranslateContext* pCxt, const char* pName, int64_t val, int64_t minVal, + int64_t maxVal) { return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_DB_OPTION, pName, val, minVal, maxVal, true); } @@ -8081,6 +8102,16 @@ static int32_t checkDbTbPrefixSuffixOptions(STranslateContext* pCxt, int32_t tbP return TSDB_CODE_SUCCESS; } +static FORCE_INLINE int32_t translateGetDbCfg(STranslateContext* pCxt, const char* pDbName, SDbCfgInfo** ppDbCfg) { + if (*ppDbCfg) { + return TSDB_CODE_SUCCESS; + } + if (!(*ppDbCfg = taosMemoryCalloc(1, sizeof(SDbCfgInfo)))) { + return terrno; + } + return getDBCfg(pCxt, pDbName, *ppDbCfg); +} + static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { int32_t daysPerFile = pOptions->daysPerFile; int32_t s3KeepLocal = pOptions->s3KeepLocal; @@ -8088,13 +8119,9 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa if (-1 == daysPerFile && -1 == daysToKeep0) { return TSDB_CODE_SUCCESS; } else if (-1 == daysPerFile || -1 == daysToKeep0) { - SDbCfgInfo dbCfg = {0}; - int32_t code = getDBCfg(pCxt, pDbName, &dbCfg); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - daysPerFile = (-1 == daysPerFile ? dbCfg.daysPerFile : daysPerFile); - daysToKeep0 = (-1 == daysToKeep0 ? dbCfg.daysToKeep0 : daysToKeep0); + TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pDbName, &pOptions->pDbCfg)); + daysPerFile = (-1 == daysPerFile ? pOptions->pDbCfg->daysPerFile : daysPerFile); + daysToKeep0 = (-1 == daysToKeep0 ? pOptions->pDbCfg->daysToKeep0 : daysToKeep0); } if (daysPerFile > daysToKeep0 / 3) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, @@ -8118,6 +8145,126 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa return TSDB_CODE_SUCCESS; } +static int32_t checkDbCompactIntervalOption(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { + int32_t code = 0; + int64_t interval = 0; + int32_t keep2 = pOptions->keep[2]; + + if (NULL != pOptions->pCompactIntervalNode) { + if (DEAL_RES_ERROR == translateValue(pCxt, pOptions->pCompactIntervalNode)) { + return pCxt->errCode; + } + if (TIME_UNIT_MINUTE != pOptions->pCompactIntervalNode->unit && + TIME_UNIT_HOUR != pOptions->pCompactIntervalNode->unit && + TIME_UNIT_DAY != pOptions->pCompactIntervalNode->unit) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_interval unit: %c, only %c, %c, %c allowed", + pOptions->pCompactIntervalNode->unit, TIME_UNIT_MINUTE, TIME_UNIT_HOUR, + TIME_UNIT_DAY); + } + interval = getBigintFromValueNode(pOptions->pCompactIntervalNode); + if (interval != 0) { + if (keep2 == -1) { // alter db + TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pDbName, &pOptions->pDbCfg)); + keep2 = pOptions->pDbCfg->daysToKeep2; + } + code = checkDbRangeOption(pCxt, "compact_interval", interval, TSDB_MIN_COMPACT_INTERVAL, keep2); + } + } else if (pOptions->compactInterval > 0) { + interval = pOptions->compactInterval * 1440; // convert to minutes + if (keep2 == -1) { // alter db + TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pDbName, &pOptions->pDbCfg)); + keep2 = pOptions->pDbCfg->daysToKeep2; + } + code = checkDbRangeOption(pCxt, "compact_interval", interval, TSDB_MIN_COMPACT_INTERVAL, keep2); + } + if (code == 0) pOptions->compactInterval = interval; + return code; +} + +static int32_t checkDbCompactTimeRangeOption(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { + if (NULL == pOptions->pCompactTimeRangeList) { + return TSDB_CODE_SUCCESS; + } + + if (LIST_LENGTH(pOptions->pCompactTimeRangeList) != 2) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_time_range, should have 2 values"); + } + + SValueNode* pStart = (SValueNode*)nodesListGetNode(pOptions->pCompactTimeRangeList, 0); + SValueNode* pEnd = (SValueNode*)nodesListGetNode(pOptions->pCompactTimeRangeList, 1); + if (DEAL_RES_ERROR == translateValue(pCxt, pStart)) { + return pCxt->errCode; + } + if (DEAL_RES_ERROR == translateValue(pCxt, pEnd)) { + return pCxt->errCode; + } + if (IS_DURATION_VAL(pStart->flag)) { + if (TIME_UNIT_MINUTE != pStart->unit && TIME_UNIT_HOUR != pStart->unit && TIME_UNIT_DAY != pStart->unit) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_time_range start unit: %c, only %c, %c, %c allowed", + pStart->unit, TIME_UNIT_MINUTE, TIME_UNIT_HOUR, TIME_UNIT_DAY); + } + } else { + pStart->datum.i *= 1440; + } + if (IS_DURATION_VAL(pEnd->flag)) { + if (TIME_UNIT_MINUTE != pEnd->unit && TIME_UNIT_HOUR != pEnd->unit && TIME_UNIT_DAY != pEnd->unit) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_time_range end unit: %c, only %c, %c, %c allowed", + pEnd->unit, TIME_UNIT_MINUTE, TIME_UNIT_HOUR, TIME_UNIT_DAY); + } + } else { + pEnd->datum.i *= 1440; + } + pOptions->compactStartTime = getBigintFromValueNode(pStart); + pOptions->compactEndTime = getBigintFromValueNode(pEnd); + + if (pOptions->compactStartTime >= pOptions->compactEndTime) { + return generateSyntaxErrMsgExt( + &pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_time_range: %dm,%dm, start time should be less than end time", + pOptions->compactStartTime, pOptions->compactEndTime); + } + + int32_t keep2 = pOptions->keep[2]; + int32_t days = pOptions->daysPerFile; + if (keep2 == -1 || days == -1) { // alter db + TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pDbName, &pOptions->pDbCfg)); + keep2 = pOptions->pDbCfg->daysToKeep2; + days = pOptions->pDbCfg->daysPerFile; + } + if (pOptions->compactStartTime < -keep2 || pOptions->compactStartTime > -days) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_time_range: %dm, start_time should be in range: [%dm, %dm]", + pOptions->compactStartTime, -keep2, -days); + } + if (pOptions->compactEndTime < -keep2 || pOptions->compactEndTime > -days) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_time_range: %dm, end time should be in range: [%dm, %dm]", + pOptions->compactEndTime, -keep2, -days); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t checkDbCompactTimeOffsetOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { + if (pOptions->pCompactTimeOffsetNode) { + if (DEAL_RES_ERROR == translateValue(pCxt, pOptions->pCompactTimeOffsetNode)) { + return pCxt->errCode; + } + if (TIME_UNIT_HOUR != pOptions->pCompactTimeOffsetNode->unit) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_time_offset unit: %c, only %c allowed", + pOptions->pCompactTimeOffsetNode->unit, TIME_UNIT_HOUR); + } + pOptions->compactTimeOffset = getBigintFromValueNode(pOptions->pCompactTimeOffsetNode) / 60; + } + return checkDbRangeOption(pCxt, "compact_time_offset", pOptions->compactTimeOffset, TSDB_MIN_COMPACT_TIME_OFFSET, + TSDB_MAX_COMPACT_TIME_OFFSET); +} + static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { int32_t code = checkDbRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, TSDB_MAX_BUFFER_PER_VNODE); @@ -8234,6 +8381,15 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "s3_compact", pOptions->s3Compact, TSDB_MIN_S3_COMPACT, TSDB_MAX_S3_COMPACT); } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbCompactIntervalOption(pCxt, pDbName, pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbCompactTimeRangeOption(pCxt, pDbName, pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbCompactTimeOffsetOption(pCxt, pOptions); + } return code; } @@ -8470,14 +8626,17 @@ static int32_t buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStm pReq->s3KeepLocal = pStmt->pOptions->s3KeepLocal; pReq->s3Compact = pStmt->pOptions->s3Compact; pReq->withArbitrator = pStmt->pOptions->withArbitrator; + pReq->compactInterval = pStmt->pOptions->compactInterval; + pReq->compactStartTime = pStmt->pOptions->compactStartTime; + pReq->compactEndTime = pStmt->pOptions->compactEndTime; + pReq->compactTimeOffset = pStmt->pOptions->compactTimeOffset; return code; } static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt) { if (pStmt->pOptions->walLevel == 0) { - SDbCfgInfo dbCfg = {0}; - int32_t code = getDBCfg(pCxt, pStmt->dbName, &dbCfg); - if (TSDB_CODE_SUCCESS == code && dbCfg.replications > 1) { + TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pStmt->dbName, &pStmt->pOptions->pDbCfg)); + if (pStmt->pOptions->pDbCfg->replications > 1) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, "Invalid option, wal_level 0 should be used with replica 1"); } @@ -9913,6 +10072,9 @@ static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt return code; } +#define MIN_MAX_COMPACT_TASKS 1 +#define MAX_MAX_COMPACT_TASKS 100 + static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pStmt) { SMCfgDnodeReq cfgReq = {0}; cfgReq.dnodeId = pStmt->dnodeId; @@ -9920,7 +10082,12 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt tstrncpy(cfgReq.value, pStmt->value, TSDB_DNODE_VALUE_LEN); int32_t code = 0; - if (0 == strncasecmp(cfgReq.config, "encrypt_key", 12)) { + + const char* validConfigs[] = { + "encrypt_key", + tsAlterCompactTaskKeywords, + }; + if (0 == strncasecmp(cfgReq.config, validConfigs[0], strlen(validConfigs[0]) + 1)) { int32_t klen = strlen(cfgReq.value); if (klen > ENCRYPT_KEY_LEN || klen < ENCRYPT_KEY_LEN_MIN) { tFreeSMCfgDnodeReq(&cfgReq); @@ -9929,6 +10096,28 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt ENCRYPT_KEY_LEN_MIN, ENCRYPT_KEY_LEN); } code = buildCmdMsg(pCxt, TDMT_MND_CREATE_ENCRYPT_KEY, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq); + } else if (0 == strncasecmp(cfgReq.config, validConfigs[1], strlen(validConfigs[1]) + 1)) { + char* endptr = NULL; + int32_t maxCompactTasks = taosStr2Int32(cfgReq.value, &endptr, 10); + int32_t minMaxCompactTasks = MIN_MAX_COMPACT_TASKS; + int32_t maxMaxCompactTasks = MAX_MAX_COMPACT_TASKS; + + // check format + if (endptr == cfgReq.value || endptr[0] != '\0') { + tFreeSMCfgDnodeReq(&cfgReq); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_DNODE_INVALID_COMPACT_TASKS, + "Invalid max compact tasks: %s", cfgReq.value); + } + + // check range + if (maxCompactTasks < minMaxCompactTasks || maxCompactTasks > maxMaxCompactTasks) { + tFreeSMCfgDnodeReq(&cfgReq); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_DNODE_INVALID_COMPACT_TASKS, + "Invalid max compact tasks: %d, valid range [%d,%d]", maxCompactTasks, + minMaxCompactTasks, maxMaxCompactTasks); + } + + code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_DNODE, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq); } else { code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_DNODE, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq); } @@ -10620,27 +10809,28 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) return code; } -static int32_t translateCompactRange(STranslateContext* pCxt, SCompactDatabaseStmt* pStmt, SCompactDbReq* pReq) { +static int32_t translateCompactRange(STranslateContext* pCxt, const char* dbName, SNode* pStart, SNode* pEnd, + STimeWindow* timeRange) { SDbCfgInfo dbCfg = {0}; - int32_t code = getDBCfg(pCxt, pStmt->dbName, &dbCfg); - if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pStart) { - ((SValueNode*)pStmt->pStart)->node.resType.precision = dbCfg.precision; - ((SValueNode*)pStmt->pStart)->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; - code = doTranslateValue(pCxt, (SValueNode*)pStmt->pStart); + int32_t code = getDBCfg(pCxt, dbName, &dbCfg); + if (TSDB_CODE_SUCCESS == code && NULL != pStart) { + ((SValueNode*)pStart)->node.resType.precision = dbCfg.precision; + ((SValueNode*)pStart)->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; + code = doTranslateValue(pCxt, (SValueNode*)pStart); } - if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pEnd) { - ((SValueNode*)pStmt->pEnd)->node.resType.precision = dbCfg.precision; - ((SValueNode*)pStmt->pEnd)->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; - code = doTranslateValue(pCxt, (SValueNode*)pStmt->pEnd); + if (TSDB_CODE_SUCCESS == code && NULL != pEnd) { + ((SValueNode*)pEnd)->node.resType.precision = dbCfg.precision; + ((SValueNode*)pEnd)->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; + code = doTranslateValue(pCxt, (SValueNode*)pEnd); } if (TSDB_CODE_SUCCESS == code) { - pReq->timeRange.skey = NULL != pStmt->pStart ? ((SValueNode*)pStmt->pStart)->datum.i : INT64_MIN; - pReq->timeRange.ekey = NULL != pStmt->pEnd ? ((SValueNode*)pStmt->pEnd)->datum.i : INT64_MAX; + timeRange->skey = NULL != pStart ? ((SValueNode*)pStart)->datum.i : INT64_MIN; + timeRange->ekey = NULL != pEnd ? ((SValueNode*)pEnd)->datum.i : INT64_MAX; } return code; } -static int32_t translateCompact(STranslateContext* pCxt, SCompactDatabaseStmt* pStmt) { +static int32_t translateCompactDb(STranslateContext* pCxt, SCompactDatabaseStmt* pStmt) { SCompactDbReq compactReq = {0}; SName name; int32_t code = TSDB_CODE_SUCCESS; @@ -10648,7 +10838,7 @@ static int32_t translateCompact(STranslateContext* pCxt, SCompactDatabaseStmt* p if (TSDB_CODE_SUCCESS != code) return code; (void)tNameGetFullDbName(&name, compactReq.db); - code = translateCompactRange(pCxt, pStmt, &compactReq); + code = translateCompactRange(pCxt, pStmt->dbName, pStmt->pStart, pStmt->pEnd, &compactReq.timeRange); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_COMPACT_DB, (FSerializeFunc)tSerializeSCompactDbReq, &compactReq); } @@ -10656,6 +10846,85 @@ static int32_t translateCompact(STranslateContext* pCxt, SCompactDatabaseStmt* p return code; } +static int32_t translateVgroupList(STranslateContext* pCxt, SNodeList* vgroupList, SArray** ppVgroups) { + int32_t code = TSDB_CODE_SUCCESS; + SHashObj* pHash = NULL; + int32_t numOfVgroups = LIST_LENGTH(vgroupList); + + (*ppVgroups) = taosArrayInit(numOfVgroups, sizeof(int64_t)); + if (NULL == *ppVgroups) { + return terrno; + } + + pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (NULL == pHash) { + code = terrno; + } + + if (TSDB_CODE_SUCCESS == code) { + SNode* pNode = NULL; + FOREACH(pNode, vgroupList) { + SValueNode* pVal = (SValueNode*)pNode; + if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { + code = TSDB_CODE_VND_INVALID_VGROUP_ID; + break; + } + + int64_t vgroupId = getBigintFromValueNode(pVal); + + if (NULL != taosHashGet(pHash, &vgroupId, sizeof(vgroupId))) { + code = TSDB_CODE_PAR_INVALID_VGID_LIST; + break; + } + + code = taosHashPut(pHash, &vgroupId, sizeof(vgroupId), NULL, 0); + if (code) { + break; + } + + if (NULL == taosArrayPush(*ppVgroups, &vgroupId)) { + code = terrno; + break; + } + } + } + + taosHashCleanup(pHash); + if (code) { + taosArrayDestroy(*ppVgroups); + *ppVgroups = NULL; + } + return code; +} + +static int32_t translateCompactVgroups(STranslateContext* pCxt, SCompactVgroupsStmt* pStmt) { + int32_t code = TSDB_CODE_SUCCESS; + SName name; + SCompactDbReq req = {0}; + + code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, + strlen(((SValueNode*)pStmt->pDbName)->literal)); + if (TSDB_CODE_SUCCESS == code) { + (void)tNameGetFullDbName(&name, req.db); + } + + if (TSDB_CODE_SUCCESS == code) { + code = + translateCompactRange(pCxt, ((SValueNode*)pStmt->pDbName)->literal, pStmt->pStart, pStmt->pEnd, &req.timeRange); + } + + if (TSDB_CODE_SUCCESS == code) { + code = translateVgroupList(pCxt, pStmt->vgidList, &req.vgroupIds); + } + + if (TSDB_CODE_SUCCESS == code) { + code = buildCmdMsg(pCxt, TDMT_MND_COMPACT_DB, (FSerializeFunc)tSerializeSCompactDbReq, &req); + } + + tFreeSCompactDbReq(&req); + return code; +} + static int32_t translateKillConnection(STranslateContext* pCxt, SKillStmt* pStmt) { SKillConnReq killReq = {0}; killReq.connId = pStmt->targetId; @@ -13079,7 +13348,10 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { code = translateDescribe(pCxt, (SDescribeStmt*)pNode); break; case QUERY_NODE_COMPACT_DATABASE_STMT: - code = translateCompact(pCxt, (SCompactDatabaseStmt*)pNode); + code = translateCompactDb(pCxt, (SCompactDatabaseStmt*)pNode); + break; + case QUERY_NODE_COMPACT_VGROUPS_STMT: + code = translateCompactVgroups(pCxt, (SCompactVgroupsStmt*)pNode); break; case QUERY_NODE_ALTER_CLUSTER_STMT: code = translateAlterCluster(pCxt, (SAlterClusterStmt*)pNode); @@ -13348,8 +13620,12 @@ static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pS tstrncpy((*pSchema)[2].name, "scope", TSDB_COL_NAME_LEN); (*pSchema)[3].type = TSDB_DATA_TYPE_BINARY; - (*pSchema)[3].bytes = TSDB_CONFIG_INFO_LEN; - tstrncpy((*pSchema)[3].name, "info", TSDB_COL_NAME_LEN); + (*pSchema)[3].bytes = TSDB_CONFIG_CATEGORY_LEN; + strcpy((*pSchema)[3].name, "category"); + + (*pSchema)[4].type = TSDB_DATA_TYPE_BINARY; + (*pSchema)[4].bytes = TSDB_CONFIG_INFO_LEN; + strcpy((*pSchema)[4].name, "info"); return TSDB_CODE_SUCCESS; } @@ -13402,6 +13678,7 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS case QUERY_NODE_SHOW_VARIABLES_STMT: return extractShowVariablesResultSchema(numOfCols, pSchema); case QUERY_NODE_COMPACT_DATABASE_STMT: + case QUERY_NODE_COMPACT_VGROUPS_STMT: return extractCompactDbResultSchema(numOfCols, pSchema); default: break; @@ -16796,6 +17073,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { break; case QUERY_NODE_SHOW_VARIABLES_STMT: case QUERY_NODE_COMPACT_DATABASE_STMT: + case QUERY_NODE_COMPACT_VGROUPS_STMT: pQuery->haveResultSet = true; pQuery->execMode = QUERY_EXEC_MODE_RPC; if (NULL != pCxt->pCmdMsg) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 8c9fd89db6..9706644324 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -613,10 +613,10 @@ static int32_t getIntegerFromAuthStr(const char* pStart, char** pNext) { return taosStr2Int32(buf, NULL, 10); } -static void getStringFromAuthStr(const char* pStart, char* pStr, char** pNext) { +static void getStringFromAuthStr(const char* pStart, char* pStr, uint32_t dstLen, char** pNext) { char* p = strchr(pStart, '*'); if (NULL == p) { - tstrncpy(pStr, pStart, strlen(pStart) + 1); + tstrncpy(pStr, pStart, dstLen); *pNext = NULL; } else { strncpy(pStr, pStart, p - pStart); @@ -629,10 +629,10 @@ static void getStringFromAuthStr(const char* pStart, char* pStr, char** pNext) { static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUserAuth) { char* p = NULL; - getStringFromAuthStr(pStr, pUserAuth->user, &p); + getStringFromAuthStr(pStr, pUserAuth->user, TSDB_USER_LEN, &p); pUserAuth->tbName.acctId = getIntegerFromAuthStr(p, &p); - getStringFromAuthStr(p, pUserAuth->tbName.dbname, &p); - getStringFromAuthStr(p, pUserAuth->tbName.tname, &p); + getStringFromAuthStr(p, pUserAuth->tbName.dbname, TSDB_DB_NAME_LEN, &p); + getStringFromAuthStr(p, pUserAuth->tbName.tname, TSDB_TABLE_NAME_LEN, &p); if (pUserAuth->tbName.tname[0]) { pUserAuth->tbName.type = TSDB_TABLE_NAME_T; } else { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 00d897cf8b..11cf926081 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -3524,9 +3524,10 @@ static void eliminateProjPushdownProjIdx(SNodeList* pParentProjects, SNodeList* } } -static int32_t eliminateProjOptFindProjPrefixWithOrderCheck(SProjectLogicNode* pProj, SProjectLogicNode* pChild, SNodeList** pNewChildTargets, bool *orderMatch) { +static int32_t eliminateProjOptFindProjPrefixWithOrderCheck(SProjectLogicNode* pProj, SProjectLogicNode* pChild, + SNodeList** pNewChildTargets, bool* orderMatch) { int32_t code = 0; - SNode* pProjection = NULL, *pChildTarget = NULL; + SNode * pProjection = NULL, *pChildTarget = NULL; *orderMatch = true; FORBOTH(pProjection, pProj->pProjections, pChildTarget, pChild->node.pTargets) { if (!pProjection) break; @@ -3557,7 +3558,7 @@ static int32_t eliminateProjOptPushTargetsToSetOpChildren(SProjectLogicNode* pSe FOREACH(pChildProj, pSetOp->node.pChildren) { if (QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pChildProj)) { SProjectLogicNode* pChildLogic = (SProjectLogicNode*)pChildProj; - SNodeList* pNewChildTargetsForChild = NULL; + SNodeList* pNewChildTargetsForChild = NULL; code = eliminateProjOptFindProjPrefixWithOrderCheck(pSetOp, pChildLogic, &pNewChildTargetsForChild, &orderMatch); if (TSDB_CODE_SUCCESS != code) break; nodesDestroyList(pChildLogic->node.pTargets); @@ -3584,8 +3585,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* if (NULL == pProjectNode->node.pParent) { SNodeList* pNewChildTargets = NULL; SNode * pProjection = NULL, *pChildTarget = NULL; - isSetOpProj = - QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pChild) && ((SProjectLogicNode*)pChild)->isSetOpProj; + isSetOpProj = QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pChild) && ((SProjectLogicNode*)pChild)->isSetOpProj; if (isSetOpProj) { // For sql: select ... from (select ... union all select ...); // When eliminating the outer proj (the outer select), we have to make sure that the outer proj projections and @@ -3649,8 +3649,10 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* // if pChild is a project logic node, remove its projection which is not reference by its target. if (needReplaceTargets) { alignProjectionWithTarget(pChild); - // Since we have eliminated the outer proj, we need to push down the new targets to the children of the set operation. - if (isSetOpProj && orderMatch && !sizeMatch) code = eliminateProjOptPushTargetsToSetOpChildren((SProjectLogicNode*)pChild); + // Since we have eliminated the outer proj, we need to push down the new targets to the children of the set + // operation. + if (isSetOpProj && orderMatch && !sizeMatch) + code = eliminateProjOptPushTargetsToSetOpChildren((SProjectLogicNode*)pChild); } } pCxt->optimized = true; @@ -4326,7 +4328,8 @@ static EDealRes lastRowScanOptSetColDataType(SNode* pNode, void* pContext) { return lastRowScanOptGetColAndSetDataType(pNode, pContext, true); } -static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCols, SNodeList* pLastRowCols, bool erase, int32_t pkBytes) { +static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCols, SNodeList* pLastRowCols, bool erase, + int32_t pkBytes) { SNode* pTarget = NULL; WHERE_EACH(pTarget, pTargets) { bool found = false; @@ -5442,7 +5445,8 @@ static bool tbCntScanOptIsEligibleConds(STbCntScanOptInfo* pInfo, SNode* pCondit static bool tbCntScanOptIsEligibleScan(STbCntScanOptInfo* pInfo) { if (0 != strcmp(pInfo->pScan->tableName.dbname, TSDB_INFORMATION_SCHEMA_DB) || 0 != strcmp(pInfo->pScan->tableName.tname, TSDB_INS_TABLE_TABLES) || NULL != pInfo->pScan->pGroupTags || - 0 != strcmp(pInfo->pScan->tableName.tname, TSDB_INS_DISK_USAGE)) { + 0 != strcmp(pInfo->pScan->tableName.tname, TSDB_INS_DISK_USAGE) || + 0 != strcmp(pInfo->pScan->tableName.tname, TSDB_INS_TABLE_FILESETS)) { return false; } if (1 == pInfo->pScan->pVgroupList->numOfVgroups && MNODE_HANDLE == pInfo->pScan->pVgroupList->vgroups[0].vgId) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index ca3187e04f..c60024b323 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -90,11 +90,12 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); return code; } - *ppKey = taosMemoryCalloc(1, strlen(pVal->literal) + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); + int32_t literalLen = strlen(pVal->literal); + *ppKey = taosMemoryCalloc(1, literalLen + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } - TAOS_STRNCAT(*ppKey, pVal->literal, strlen(pVal->literal)); + TAOS_STRNCAT(*ppKey, pVal->literal, literalLen); TAOS_STRNCAT(*ppKey, ".", 2); TAOS_STRNCAT(*ppKey, ((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN); *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); @@ -810,7 +811,8 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TABLES) || 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TAGS) || 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_COLS) || - 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_DISK_USAGE)) { + 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_DISK_USAGE) || + 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_FILESETS)) { if (pScanLogicNode->pVgroupList) { vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 22141c33ee..690d38aac0 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -625,7 +625,7 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { qError("taosHashPut failed, vgId:%d", vgInfo->vgId); taosHashCancelIterate(pSrc->vgHash, pIter); freeVgInfo(*pDst); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pIter = taosHashIterate(pSrc->vgHash, pIter); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 9d9c169c05..0ee61726e3 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -64,7 +64,7 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { qDebug("the %dth vgroup, id %d, epNum %d, current %s port %d", i, pVgInfo->vgId, pVgInfo->epSet.numOfEps, pVgInfo->epSet.eps[pVgInfo->epSet.inUse].fqdn, pVgInfo->epSet.eps[pVgInfo->epSet.inUse].port); if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pVgInfo->vgId, sizeof(int32_t), pVgInfo, sizeof(SVgroupInfo))) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 6d81baf91a..53e1c3d288 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -28,6 +28,7 @@ extern "C" { #include "tref.h" #include "trpc.h" #include "ttimer.h" +#include "theap.h" #define QW_DEFAULT_SCHEDULER_NUMBER 100 #define QW_DEFAULT_TASK_NUMBER 10000 @@ -37,6 +38,24 @@ extern "C" { #define QW_SCH_TIMEOUT_MSEC 180000 #define QW_MIN_RES_ROWS 16384 +#define QW_THREAD_MAX_SCHED_TASK_NUM 10 + +#define QW_QUERY_MEM_POOL_NAME "Query" +#define QW_MAX_RETIRE_JOB_NUM 10000 + +#define QW_DEFAULT_THREAD_TASK_NUM 3 + +#define QW_RETIRE_JOB_BATCH_NUM 5 + +#define QW_DEFAULT_TIMEOUT_INTERVAL_SECS 600 + +enum { + QW_CONC_TASK_LEVEL_LOW = 1, + QW_CONC_TASK_LEVEL_MIDDLE, + QW_CONC_TASK_LEVEL_HIGH, + QW_CONC_TASK_LEVEL_FULL +}; + enum { QW_PHASE_PRE_QUERY = 1, QW_PHASE_POST_QUERY, @@ -112,6 +131,27 @@ typedef struct SQWTaskStatus { int8_t status; } SQWTaskStatus; +typedef struct SQWSessionInfo { + void *mgmt; + uint64_t sId; + uint64_t qId; + uint64_t cId; + uint64_t tId; + int64_t rId; + int32_t eId; + void *sessionMp; +} SQWSessionInfo; + +typedef struct SQWJobInfo { + int8_t retired; + int32_t errCode; + SMemPoolJob* memInfo; + + SRWLatch lock; + int8_t destroyed; + SHashObj* pSessions; +} SQWJobInfo; + typedef struct SQWTaskCtx { SRWLatch lock; int8_t phase; @@ -134,6 +174,7 @@ typedef struct SQWTaskCtx { bool queryExecDone; bool queryInQueue; bool explainRsped; + bool sinkWithMemPool; int32_t rspCode; int64_t affectedRows; // for insert ...select stmt @@ -142,10 +183,15 @@ typedef struct SQWTaskCtx { int8_t events[QW_EVENT_MAX]; - SArray *explainRes; - void *taskHandle; - void *sinkHandle; - SArray *tbInfo; // STbVerInfo + SArray *explainRes; + void *taskHandle; + void *sinkHandle; + SArray *tbInfo; // STbVerInfo + + int64_t lastAckTs; + + void *memPoolSession; + SQWJobInfo *pJobInfo; } SQWTaskCtx; typedef struct SQWSchStatus { @@ -201,7 +247,8 @@ typedef struct SQWorker { SQWStat stat; int32_t *destroyed; - int8_t nodeStopped; + int8_t nodeStopped; + int32_t lastChkTs; } SQWorker; typedef struct SQWorkerMgmt { @@ -212,13 +259,43 @@ typedef struct SQWorkerMgmt { int32_t paramIdx; } SQWorkerMgmt; -#define QW_CTX_NOT_EXISTS_ERR_CODE(mgmt) \ - (atomic_load_8(&(mgmt)->nodeStopped) ? TSDB_CODE_VND_STOPPED : TSDB_CODE_QRY_TASK_CTX_NOT_EXIST) + +typedef struct SQWRetireCtx { + BoundedQueue* pJobQueue; +} SQWRetireCtx; + +typedef struct SQueryExecStat { + int64_t taskInitNum; + int64_t taskRunNum; + int64_t taskExecDestroyNum; + int64_t taskSinkDestroyNum; + int64_t taskDestroyNum; +} SQueryExecStat; + +typedef struct SQueryMgmt { + SRWLatch taskMgmtLock; + int32_t concTaskLevel; + SHashObj* pJobInfo; + SQueryExecStat stat; +} SQueryMgmt; + +#define QW_CTX_NOT_EXISTS_ERR_CODE(mgmt) (atomic_load_8(&(mgmt)->nodeStopped) ? TSDB_CODE_VND_STOPPED : TSDB_CODE_QRY_TASK_CTX_NOT_EXIST) #define QW_FPARAMS_DEF SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId, int32_t eId #define QW_IDS() sId, qId, cId, tId, rId, eId #define QW_FPARAMS() mgmt, QW_IDS() +extern SQueryMgmt gQueryMgmt; + +#define QW_SINK_ENABLE_MEMPOOL(_ctx) \ + do { \ + if ((_ctx)->sinkWithMemPool) { \ + taosEnableMemPoolUsage((_ctx)->memPoolSession); \ + } \ + } while (0) + +#define QW_SINK_DISABLE_MEMPOOL() taosDisableMemPoolUsage() + #define QW_STAT_INC(_item, _n) (void)atomic_add_fetch_64(&(_item), _n) #define QW_STAT_DEC(_item, _n) (void)atomic_sub_fetch_64(&(_item), _n) #define QW_STAT_GET(_item) atomic_load_64(&(_item)) @@ -257,6 +334,19 @@ typedef struct SQWorkerMgmt { #define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch) #define QW_QUERY_NOT_STARTED(ctx) (QW_GET_PHASE(ctx) == -1) +#define QW_SET_QCID(id, qId, cId) \ + do { \ + *(uint64_t *)(id) = (qId); \ + *(uint64_t *)((char *)(id) + sizeof(qId)) = (cId); \ + } while (0) + +#define QW_GET_QCID(id, qId, cId) \ + do { \ + (qId) = *(uint64_t *)(id); \ + (cId) = *(uint64_t *)((char *)(id) + sizeof(qId)); \ + } while (0) + + #define QW_SET_QTID(id, qId, cId, tId, eId) \ do { \ *(uint64_t *)(id) = (qId); \ @@ -273,6 +363,18 @@ typedef struct SQWorkerMgmt { (eId) = *(int32_t *)((char *)(id) + sizeof(qId) + sizeof(cId) + sizeof(tId)); \ } while (0) +#define QW_SET_TEID(id, tId, eId) \ + do { \ + *(uint64_t *)(id) = (tId); \ + *(uint32_t *)((char *)(id) + sizeof(tId)) = (eId); \ + } while (0) + +#define QW_GET_TEID(id, tId, eId) \ + do { \ + (tId) = *(uint64_t *)(id); \ + (eId) = *(uint32_t *)((char *)(id) + sizeof(tId)); \ + } while (0) + #define QW_ERR_RET(c) \ do { \ int32_t _code = (c); \ @@ -309,33 +411,33 @@ typedef struct SQWorkerMgmt { } \ } while (0) -#define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%" PRIx64 " " param, mgmt, sId, __VA_ARGS__) -#define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%" PRIx64 " " param, mgmt, sId, __VA_ARGS__) +#define QW_SCH_ELOG(param, ...) qError("QW:%p clientId:%" PRIx64 " " param, mgmt, clientId, __VA_ARGS__) +#define QW_SCH_DLOG(param, ...) qDebug("QW:%p clientId:%" PRIx64 " " param, mgmt, clientId, __VA_ARGS__) #define QW_TASK_ELOG(param, ...) \ - qError("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__) + qError("QID:0x%" PRIx64 ",SID:%" PRId64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, sId, cId, tId, eId, __VA_ARGS__) #define QW_TASK_WLOG(param, ...) \ - qWarn("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__) + qWarn("QID:0x%" PRIx64 ",SID:%" PRId64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, sId, cId, tId, eId, __VA_ARGS__) #define QW_TASK_DLOG(param, ...) \ - qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__) + qDebug("QID:0x%" PRIx64 ",SID:%" PRId64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, sId, cId, tId, eId, __VA_ARGS__) #define QW_TASK_DLOGL(param, ...) \ - qDebugL("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__) + qDebugL("QID:0x%" PRIx64 ",SID:%" PRId64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, sId, cId, tId, eId, __VA_ARGS__) #define QW_TASK_ELOG_E(param) \ - qError("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId) + qError("QID:0x%" PRIx64 ",SID:%" PRId64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, sId, cId, tId, eId) #define QW_TASK_WLOG_E(param) \ - qWarn("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId) + qWarn("QID:0x%" PRIx64 ",SID:%" PRId64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, sId, cId, tId, eId) #define QW_TASK_DLOG_E(param) \ - qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId) + qDebug("QID:0x%" PRIx64 ",SID:%" PRId64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, sId, cId, tId, eId) #define QW_SCH_TASK_ELOG(param, ...) \ - qError("QW:%p SID:0x%" PRIx64 ",QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, \ + qError("QW:%p SID:%" PRId64 ",QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, \ qId, cId, tId, eId, __VA_ARGS__) #define QW_SCH_TASK_WLOG(param, ...) \ - qWarn("QW:%p SID:0x%" PRIx64 ",QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, \ + qWarn("QW:%p SID:%" PRId64 ",QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, \ cId, tId, eId, __VA_ARGS__) #define QW_SCH_TASK_DLOG(param, ...) \ - qDebug("QW:%p SID:0x%" PRIx64 ",QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, \ + qDebug("QW:%p SID:%" PRId64 ",QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, \ qId, cId, tId, eId, __VA_ARGS__) #define QW_LOCK_DEBUG(...) \ @@ -414,7 +516,7 @@ static FORCE_INLINE int32_t qwRelease(int64_t refId) { return taosReleaseRef(gQw char *qwPhaseStr(int32_t phase); char *qwBufStatusStr(int32_t bufStatus); -int32_t qwAcquireAddScheduler(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch); +int32_t qwAcquireAddScheduler(SQWorker *mgmt, uint64_t clientId, int32_t rwType, SQWSchStatus **sch); void qwReleaseScheduler(int32_t rwType, SQWorker *mgmt); int32_t qwAddTaskStatus(QW_FPARAMS_DEF, int32_t status); int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx); @@ -430,10 +532,11 @@ void qwSetHbParam(int64_t refId, SQWHbParam **pParam); int32_t qwUpdateTimeInQueue(SQWorker *mgmt, int64_t ts, EQueueType type); int64_t qwGetTimeInQueue(SQWorker *mgmt, EQueueType type); void qwClearExpiredSch(SQWorker *mgmt, SArray *pExpiredSch); -int32_t qwAcquireScheduler(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch); -void qwFreeTaskCtx(SQWTaskCtx *ctx); +int32_t qwAcquireScheduler(SQWorker *mgmt, uint64_t clientId, int32_t rwType, SQWSchStatus **sch); +void qwFreeTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx *ctx); int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx); +void qwDbgDumpJobsInfo(void); void qwDbgDumpMgmtInfo(SQWorker *mgmt); int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore, bool dynamicTask); int32_t qwDbgBuildAndSendRedirectRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, SEpSet *pEpSet); @@ -442,6 +545,16 @@ void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped); void qwDbgSimulateSleep(void); void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped); int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx); +int32_t qwInitQueryPool(void); +void qwDestroyJobInfo(void* job); +bool qwStopTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool forceStop, int32_t errCode); +void qwStopAllTasks(SQWorker *mgmt); +void qwChkDropTimeoutQuery(SQWorker *mgmt, int32_t currTs); +bool qwRetireJob(SQWJobInfo* pJob); +void qwDestroySession(QW_FPARAMS_DEF, SQWJobInfo *pJobInfo, void* session); +int32_t qwInitSession(QW_FPARAMS_DEF, SQWTaskCtx *ctx, void** ppSession); +void qwFreeTaskHandle(SQWTaskCtx *ctx); +void qwFreeSinkHandle(SQWTaskCtx *ctx); #ifdef __cplusplus } diff --git a/source/libs/qworker/inc/qwMsg.h b/source/libs/qworker/inc/qwMsg.h index 6d57a3df46..366b9d09f1 100644 --- a/source/libs/qworker/inc/qwMsg.h +++ b/source/libs/qworker/inc/qwMsg.h @@ -36,7 +36,7 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes); int32_t qwBuildAndSendDropRsp(SRpcHandleInfo *pConn, int32_t code); int32_t qwBuildAndSendCancelRsp(SRpcHandleInfo *pConn, int32_t code); -int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, +int32_t qwBuildAndSendFetchRsp(SQWTaskCtx *ctx, int32_t rspType, SRpcHandleInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code); void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, int32_t rawDataLen, bool qComplete); int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn); @@ -47,7 +47,7 @@ void qwFreeFetchRsp(void *msg); int32_t qwMallocFetchRsp(int8_t rpcMalloc, int32_t length, SRetrieveTableRsp **rsp); int32_t qwBuildAndSendHbRsp(SRpcHandleInfo *pConn, SSchedulerHbRsp *rsp, int32_t code); int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn); -int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t sId, SRpcHandleInfo *pConn); +int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t clientId, SRpcHandleInfo *pConn); int32_t qwBuildAndSendDropMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn); #ifdef __cplusplus diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index 5cef253cae..f4015ea28d 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -100,7 +100,7 @@ void qwDbgDumpSchInfo(SQWorker *mgmt, SQWSchStatus *sch, int32_t i) { int32_t taskNum = taosHashGetSize(sch->tasksHash); QW_DLOG("***The %dth scheduler status, hbBrokenTs:%" PRId64 ",taskNum:%d", i, sch->hbBrokenTs, taskNum); - uint64_t qId, cId, tId; + uint64_t qId, cId, tId, sId = 0; int32_t eId; SQWTaskStatus *pTask = NULL; void *pIter = taosHashIterate(sch->tasksHash, NULL); @@ -122,19 +122,20 @@ void qwDbgDumpTasksInfo(SQWorker *mgmt) { int32_t i = 0; SQWTaskCtx *ctx = NULL; - uint64_t qId, cId, tId; + uint64_t qId, cId, tId, sId; int32_t eId; void *pIter = taosHashIterate(mgmt->ctxHash, NULL); while (pIter) { ctx = (SQWTaskCtx *)pIter; void *key = taosHashGetKey(pIter, NULL); QW_GET_QTID(key, qId, cId, tId, eId); + sId = ctx->sId; QW_TASK_DLOG("%p lock:%x, phase:%d, type:%d, explain:%d, needFetch:%d, localExec:%d, queryMsgType:%d, " - "sId:%" PRId64 ", level:%d, queryGotData:%d, queryRsped:%d, queryEnd:%d, queryContinue:%d, queryInQueue:%d, " + "level:%d, queryGotData:%d, queryRsped:%d, queryEnd:%d, queryContinue:%d, queryInQueue:%d, " "rspCode:%x, affectedRows:%" PRId64 ", taskHandle:%p, sinkHandle:%p, tbNum:%d, events:%d,%d,%d,%d,%d", ctx, ctx->lock, ctx->phase, ctx->taskType, ctx->explain, ctx->needFetch, ctx->localExec, ctx->queryMsgType, - ctx->sId, ctx->level, ctx->queryGotData, ctx->queryRsped, ctx->queryEnd, ctx->queryContinue, + ctx->level, ctx->queryGotData, ctx->queryRsped, ctx->queryEnd, ctx->queryContinue, ctx->queryInQueue, ctx->rspCode, ctx->affectedRows, ctx->taskHandle, ctx->sinkHandle, (int32_t)taosArrayGetSize(ctx->tbInfo), ctx->events[QW_EVENT_CANCEL], ctx->events[QW_EVENT_READY], ctx->events[QW_EVENT_FETCH], ctx->events[QW_EVENT_DROP], ctx->events[QW_EVENT_CQUERY]); @@ -171,6 +172,37 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt) { qwDbgDumpTasksInfo(mgmt); } +void qwDbgDumpJobsInfo(void) { + if (!gQWDebug.dumpEnable) { + return; + } + + qDebug("total remain job num %d, task initNum:%" PRId64 " - %" PRId64 ", task destroyNum:%" PRId64 " - %" PRId64 " - %" PRId64, + taosHashGetSize(gQueryMgmt.pJobInfo), atomic_load_64(&gQueryMgmt.stat.taskInitNum), atomic_load_64(&gQueryMgmt.stat.taskRunNum), + atomic_load_64(&gQueryMgmt.stat.taskExecDestroyNum), atomic_load_64(&gQueryMgmt.stat.taskSinkDestroyNum), + atomic_load_64(&gQueryMgmt.stat.taskDestroyNum)); + + size_t keyLen = 0; + char* id = NULL; + int32_t jobIdx = 0; + SQWJobInfo* pJob = (SQWJobInfo*)taosHashIterate(gQueryMgmt.pJobInfo, NULL); + while (NULL != pJob) { + qDebug("QID:0x%" PRIx64 " CID:0x%" PRIx64 " the %dth remain job", pJob->memInfo->jobId, pJob->memInfo->clientId, jobIdx++); + + int32_t sessionIdx = 0; + SQWSessionInfo* pSession = (SQWSessionInfo*)taosHashIterate(pJob->pSessions, NULL); + while (NULL != pSession) { + qDebug("QID:0x%" PRIx64 ",SID:%" PRId64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d the %dth remain session", + pSession->qId, pSession->sId, pSession->cId, pSession->tId, pSession->eId, sessionIdx++); + + pSession = (SQWSessionInfo*)taosHashIterate(pJob->pSessions, pSession); + } + + pJob = (SQWJobInfo *)taosHashIterate(gQueryMgmt.pJobInfo, pJob); + } +} + + int32_t qwDbgBuildAndSendRedirectRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, SEpSet *pEpSet) { int32_t contLen = 0; char *rsp = NULL; diff --git a/source/libs/qworker/src/qwMem.c b/source/libs/qworker/src/qwMem.c new file mode 100644 index 0000000000..89475bca13 --- /dev/null +++ b/source/libs/qworker/src/qwMem.c @@ -0,0 +1,213 @@ +#include "qwInt.h" +#include "qworker.h" + +void qwSetConcurrentTaskNumCb(int32_t taskNum) { + int32_t finTaskNum = TMIN(taskNum, tsNumOfQueryThreads * QW_DEFAULT_THREAD_TASK_NUM); + + if (tsQueryMaxConcurrentTaskNum > 0) { + finTaskNum = TMIN(taskNum, tsQueryMaxConcurrentTaskNum); + } + finTaskNum = TMAX(finTaskNum, tsQueryMinConcurrentTaskNum); + + atomic_store_32(&tsQueryConcurrentTaskNum, finTaskNum); + + atomic_store_32(&gQueryMgmt.concTaskLevel, QW_CONC_TASK_LEVEL_FULL); +} + +void qwDecConcurrentTaskNumCb(void) { + int32_t concTaskLevel = atomic_load_32(&gQueryMgmt.concTaskLevel); + if (concTaskLevel <= QW_CONC_TASK_LEVEL_LOW) { + qError("Unable to decrease concurrent task num, current task level:%d", concTaskLevel); + return; + } + + //TODO +} + +void qwIncConcurrentTaskNumCb(void) { + int32_t concTaskLevel = atomic_load_32(&gQueryMgmt.concTaskLevel); + if (concTaskLevel >= QW_CONC_TASK_LEVEL_FULL) { + qError("Unable to increase concurrent task num, current task level:%d", concTaskLevel); + return; + } + + //TODO +} + +int32_t qwInitJobInfo(QW_FPARAMS_DEF, SQWJobInfo* pJob) { + pJob->pSessions= taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == pJob->pSessions) { + QW_TASK_ELOG("fail to init session hash, code: 0x%x", terrno); + return terrno; + } + + int32_t code = taosMemPoolCallocJob(qId, cId, (void**)&pJob->memInfo); + if (TSDB_CODE_SUCCESS != code) { + taosHashCleanup(pJob->pSessions); + pJob->pSessions = NULL; + return code; + } + + return code; +} + + +int32_t qwInitJobHash(void) { + int32_t code = TSDB_CODE_SUCCESS; + SHashObj* pHash = NULL; + + if (NULL == atomic_load_ptr(&gQueryMgmt.pJobInfo)) { + pHash = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == pHash) { + qError("init job hash failed, error:0x%x", terrno); + return terrno; + } + + if (NULL != atomic_val_compare_exchange_ptr(&gQueryMgmt.pJobInfo, NULL, pHash)) { + taosHashCleanup(pHash); + return code; + } + + taosHashSetFreeFp(gQueryMgmt.pJobInfo, qwDestroyJobInfo); + } + + return code; +} + + +void qwDestroySession(QW_FPARAMS_DEF, SQWJobInfo *pJobInfo, void* session) { + char id[sizeof(tId) + sizeof(eId) + 1] = {0}; + QW_SET_TEID(id, tId, eId); + + TAOS_UNUSED(taosHashRemove(pJobInfo->pSessions, id, sizeof(id))); + + taosMemPoolDestroySession(gMemPoolHandle, session); + + int32_t remainSessions = atomic_sub_fetch_32(&pJobInfo->memInfo->remainSession, 1); + + QW_TASK_DLOG("task session destoryed, remainSessions:%d", remainSessions); + + if (0 == remainSessions) { + QW_LOCK(QW_WRITE, &pJobInfo->lock); + if (/*0 == taosHashGetSize(pJobInfo->pSessions) && */0 == atomic_load_32(&pJobInfo->memInfo->remainSession)) { + atomic_store_8(&pJobInfo->destroyed, 1); + QW_UNLOCK(QW_WRITE, &pJobInfo->lock); + + char id2[sizeof(qId) + sizeof(cId) + 1] = {0}; + QW_SET_QCID(id2, qId, cId); + TAOS_UNUSED(taosHashRemove(gQueryMgmt.pJobInfo, id2, sizeof(id2))); + + QW_TASK_DLOG_E("the whole query job removed"); + } else { + QW_TASK_DLOG("job not removed, remainSessions:%d, %d", taosHashGetSize(pJobInfo->pSessions), pJobInfo->memInfo->remainSession); + QW_UNLOCK(QW_WRITE, &pJobInfo->lock); + } + } +} + +int32_t qwRetrieveJobInfo(QW_FPARAMS_DEF, SQWJobInfo** ppJob) { + int32_t code = TSDB_CODE_SUCCESS; + SQWJobInfo* pJob = NULL; + char id[sizeof(qId) + sizeof(cId) + 1] = {0}; + + if (NULL == gQueryMgmt.pJobInfo) { + QW_ERR_RET(qwInitJobHash()); + } + + QW_SET_QCID(id, qId, cId); + + while (true) { + pJob = (SQWJobInfo*)taosHashAcquire(gQueryMgmt.pJobInfo, id, sizeof(id)); + if (NULL == pJob) { + SQWJobInfo jobInfo = {0}; + code = qwInitJobInfo(QW_FPARAMS(), &jobInfo); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + code = taosHashPut(gQueryMgmt.pJobInfo, id, sizeof(id), &jobInfo, sizeof(jobInfo)); + if (TSDB_CODE_SUCCESS != code) { + qwDestroyJobInfo(&jobInfo); + if (TSDB_CODE_DUP_KEY == code) { + code = TSDB_CODE_SUCCESS; + continue; + } + + QW_TASK_ELOG("fail to put job to job hash, error: %s", tstrerror(code)); + return code; + } + + QW_TASK_DLOG_E("job added to hash"); + + pJob = (SQWJobInfo*)taosHashAcquire(gQueryMgmt.pJobInfo, id, sizeof(id)); + if (NULL == pJob) { + QW_TASK_WLOG_E("job not in job hash, may be dropped"); + continue; + } + } + + QW_LOCK(QW_READ, &pJob->lock); + if (atomic_load_8(&pJob->destroyed)) { + QW_UNLOCK(QW_READ, &pJob->lock); + continue; + } + + (void)atomic_add_fetch_32(&pJob->memInfo->remainSession, 1); + QW_UNLOCK(QW_READ, &pJob->lock); + + break; + } + + *ppJob = pJob; + + return code; +} + +int32_t qwInitSession(QW_FPARAMS_DEF, SQWTaskCtx *ctx, void** ppSession) { + int32_t code = TSDB_CODE_SUCCESS; + SQWJobInfo* pJob = NULL; + SQWSessionInfo session = {.mgmt = mgmt, + .sId = sId, + .qId = qId, + .cId = cId, + .tId = tId, + .rId = rId, + .eId = eId + }; + + do { + QW_ERR_JRET(qwRetrieveJobInfo(QW_FPARAMS(), &pJob)); + + ctx->pJobInfo = pJob; + + char id[sizeof(tId) + sizeof(eId) + 1] = {0}; + QW_SET_TEID(id, tId, eId); + + QW_ERR_JRET(taosMemPoolInitSession(gMemPoolHandle, ppSession, pJob->memInfo, id)); + session.sessionMp = *ppSession; + + code = taosHashPut(pJob->pSessions, id, sizeof(id), &session, sizeof(session)); + if (TSDB_CODE_SUCCESS != code) { + QW_TASK_ELOG("fail to put session into query session hash, code: 0x%x", code); + QW_ERR_JRET(code); + } + + break; + } while (true); + + QW_TASK_DLOG_E("session initialized"); + +_return: + + if (NULL != pJob) { + if (TSDB_CODE_SUCCESS != code) { + qwDestroySession(QW_FPARAMS(), pJob, *ppSession); + } + + taosHashRelease(gQueryMgmt.pJobInfo, pJob); + } + + return code; +} + + diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 7dbad90cc0..0452a05fe4 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -73,7 +73,7 @@ int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t c int32_t msgSize = tSerializeSQueryTableRsp(NULL, 0, &rsp); if (msgSize < 0) { qError("tSerializeSQueryTableRsp failed"); - QW_RET(TSDB_CODE_OUT_OF_MEMORY); + QW_RET(msgSize); } void *pRsp = rpcMallocCont(msgSize); @@ -82,9 +82,10 @@ int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t c QW_RET(terrno); } - if (tSerializeSQueryTableRsp(pRsp, msgSize, &rsp) < 0) { + msgSize = tSerializeSQueryTableRsp(pRsp, msgSize, &rsp); + if (msgSize < 0) { qError("tSerializeSQueryTableRsp %d failed", msgSize); - QW_RET(TSDB_CODE_OUT_OF_MEMORY); + QW_RET(msgSize); } SRpcMsg rpcRsp = { @@ -162,7 +163,7 @@ int32_t qwBuildAndSendHbRsp(SRpcHandleInfo *pConn, SSchedulerHbRsp *pStatus, int return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, +int32_t qwBuildAndSendFetchRsp(SQWTaskCtx *ctx, int32_t rspType, SRpcHandleInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code) { if (NULL == pRsp) { pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); @@ -184,6 +185,10 @@ int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieve rpcRsp.info.compressed = pRsp->compressed; tmsgSendRsp(&rpcRsp); + if (NULL != ctx) { + ctx->lastAckTs = taosGetTimestampSec(); + } + return TSDB_CODE_SUCCESS; } @@ -241,7 +246,7 @@ int32_t qwBuildAndSendDropMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { int32_t msgSize = tSerializeSTaskDropReq(NULL, 0, &qMsg); if (msgSize < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq get size, msgSize:%d", msgSize); - QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + QW_ERR_RET(msgSize); } void *msg = rpcMallocCont(msgSize); @@ -249,11 +254,12 @@ int32_t qwBuildAndSendDropMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", msgSize); QW_ERR_RET(terrno); } - - if (tSerializeSTaskDropReq(msg, msgSize, &qMsg) < 0) { + + msgSize = tSerializeSTaskDropReq(msg, msgSize, &qMsg); + if (msgSize < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq failed, msgSize:%d", msgSize); rpcFreeCont(msg); - QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + QW_ERR_RET(msgSize); } SRpcMsg pNewMsg = { @@ -279,7 +285,7 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { SQueryContinueReq *req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq)); if (NULL == req) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(SQueryContinueReq)); - QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + QW_ERR_RET(terrno); } req->header.vgId = mgmt->nodeId; @@ -322,7 +328,7 @@ int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { int32_t msgSize = tSerializeSTaskDropReq(NULL, 0, &qMsg); if (msgSize < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq get size, msgSize:%d", msgSize); - QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + QW_ERR_RET(msgSize); } void *msg = rpcMallocCont(msgSize); @@ -330,11 +336,12 @@ int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", msgSize); QW_ERR_RET(terrno); } - - if (tSerializeSTaskDropReq(msg, msgSize, &qMsg) < 0) { + + msgSize = tSerializeSTaskDropReq(msg, msgSize, &qMsg); + if (msgSize < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq failed, msgSize:%d", msgSize); rpcFreeCont(msg); - QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + QW_ERR_RET(msgSize); } SRpcMsg brokenMsg = { @@ -350,25 +357,27 @@ int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { return TSDB_CODE_SUCCESS; } -int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t sId, SRpcHandleInfo *pConn) { +int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t clientId, SRpcHandleInfo *pConn) { SSchedulerHbReq req = {0}; req.header.vgId = mgmt->nodeId; - req.sId = sId; + req.clientId = clientId; int32_t msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { QW_SCH_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + QW_ERR_RET(msgSize); } void *msg = rpcMallocCont(msgSize); if (NULL == msg) { QW_SCH_ELOG("calloc %d failed", msgSize); QW_ERR_RET(terrno); } - if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { + + msgSize = tSerializeSSchedulerHbReq(msg, msgSize, &req); + if (msgSize < 0) { QW_SCH_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); rpcFreeCont(msg); - QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + QW_ERR_RET(msgSize); } SRpcMsg brokenMsg = { @@ -720,6 +729,7 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_ int32_t code = 0; SSchedulerHbReq req = {0}; SQWorker *mgmt = (SQWorker *)qWorkerMgmt; + uint64_t clientId = 0; QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE)); QW_STAT_INC(mgmt->stat.msgStat.hbProcessed, 1); @@ -735,7 +745,8 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_ QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - uint64_t sId = req.sId; + clientId = req.clientId; + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .code = pMsg->code, .connInfo = pMsg->info}; if (TSDB_CODE_RPC_BROKEN_LINK == pMsg->code) { QW_SCH_DLOG("receive Hb msg due to network broken, error:%s", tstrerror(pMsg->code)); diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 917579deb0..2809f05590 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -70,7 +70,7 @@ int32_t qwSetTaskStatus(QW_FPARAMS_DEF, SQWTaskStatus *task, int8_t status, bool return TSDB_CODE_SUCCESS; } -int32_t qwAddSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType) { +int32_t qwAddSchedulerImpl(SQWorker *mgmt, uint64_t clientId, int32_t rwType) { SQWSchStatus newSch = {0}; newSch.tasksHash = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); @@ -82,14 +82,14 @@ int32_t qwAddSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType) { } QW_LOCK(QW_WRITE, &mgmt->schLock); - int32_t code = taosHashPut(mgmt->schHash, &sId, sizeof(sId), &newSch, sizeof(newSch)); + int32_t code = taosHashPut(mgmt->schHash, &clientId, sizeof(clientId), &newSch, sizeof(newSch)); if (0 != code) { if (!HASH_NODE_EXIST(code)) { QW_UNLOCK(QW_WRITE, &mgmt->schLock); QW_SCH_ELOG("taosHashPut new sch to scheduleHash failed, errno:%d", errno); taosHashCleanup(newSch.tasksHash); - QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + QW_ERR_RET(code); } taosHashCleanup(newSch.tasksHash); @@ -99,15 +99,15 @@ int32_t qwAddSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType) { return TSDB_CODE_SUCCESS; } -int32_t qwAcquireSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch, int32_t nOpt) { +int32_t qwAcquireSchedulerImpl(SQWorker *mgmt, uint64_t clientId, int32_t rwType, SQWSchStatus **sch, int32_t nOpt) { while (true) { QW_LOCK(rwType, &mgmt->schLock); - *sch = taosHashGet(mgmt->schHash, &sId, sizeof(sId)); + *sch = taosHashGet(mgmt->schHash, &clientId, sizeof(clientId)); if (NULL == (*sch)) { QW_UNLOCK(rwType, &mgmt->schLock); if (QW_NOT_EXIST_ADD == nOpt) { - QW_ERR_RET(qwAddSchedulerImpl(mgmt, sId, rwType)); + QW_ERR_RET(qwAddSchedulerImpl(mgmt, clientId, rwType)); nOpt = QW_NOT_EXIST_RET_ERR; @@ -126,12 +126,12 @@ int32_t qwAcquireSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQW return TSDB_CODE_SUCCESS; } -int32_t qwAcquireAddScheduler(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch) { - return qwAcquireSchedulerImpl(mgmt, sId, rwType, sch, QW_NOT_EXIST_ADD); +int32_t qwAcquireAddScheduler(SQWorker *mgmt, uint64_t clientId, int32_t rwType, SQWSchStatus **sch) { + return qwAcquireSchedulerImpl(mgmt, clientId, rwType, sch, QW_NOT_EXIST_ADD); } -int32_t qwAcquireScheduler(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch) { - return qwAcquireSchedulerImpl(mgmt, sId, rwType, sch, QW_NOT_EXIST_RET_ERR); +int32_t qwAcquireScheduler(SQWorker *mgmt, uint64_t clientId, int32_t rwType, SQWSchStatus **sch) { + return qwAcquireSchedulerImpl(mgmt, clientId, rwType, sch, QW_NOT_EXIST_RET_ERR); } void qwReleaseScheduler(int32_t rwType, SQWorker *mgmt) { QW_UNLOCK(rwType, &mgmt->schLock); } @@ -144,6 +144,7 @@ int32_t qwAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, S *task = taosHashGet(sch->tasksHash, id, sizeof(id)); if (NULL == (*task)) { QW_UNLOCK(rwType, &sch->tasksLock); + QW_TASK_ELOG_E("task status not exists"); QW_ERR_RET(TSDB_CODE_QRY_TASK_NOT_EXIST); } @@ -173,7 +174,7 @@ int32_t qwAddTaskStatusImpl(QW_FPARAMS_DEF, SQWSchStatus *sch, int32_t rwType, i } } else { QW_TASK_ELOG("taosHashPut to tasksHash failed, error:%x - %s", code, tstrerror(code)); - QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + QW_ERR_RET(code); } } QW_UNLOCK(QW_WRITE, &sch->tasksLock); @@ -190,7 +191,7 @@ int32_t qwAddTaskStatusImpl(QW_FPARAMS_DEF, SQWSchStatus *sch, int32_t rwType, i int32_t qwAddTaskStatus(QW_FPARAMS_DEF, int32_t status) { SQWSchStatus *tsch = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireAddScheduler(mgmt, sId, QW_READ, &tsch)); + QW_ERR_RET(qwAcquireAddScheduler(mgmt, cId, QW_READ, &tsch)); QW_ERR_JRET(qwAddTaskStatusImpl(QW_FPARAMS(), tsch, 0, status, NULL)); @@ -253,10 +254,12 @@ int32_t qwAddTaskCtxImpl(QW_FPARAMS_DEF, bool acquire, SQWTaskCtx **ctx) { } } else { QW_TASK_ELOG("taosHashPut to ctxHash failed, error:%x", code); - QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + QW_ERR_RET(code); } } + atomic_add_fetch_64(&gQueryMgmt.stat.taskInitNum, 1); + if (acquire && ctx) { QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), ctx)); } else if (ctx) { @@ -272,17 +275,34 @@ int32_t qwAddAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { return qwAddTask void qwReleaseTaskCtx(SQWorker *mgmt, void *ctx) { taosHashRelease(mgmt->ctxHash, ctx); } -void qwFreeTaskHandle(qTaskInfo_t *taskHandle) { +void qwFreeTaskHandle(SQWTaskCtx *ctx) { // Note: free/kill may in RC - qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); - if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { - tsEnableRandErr = true; + qTaskInfo_t otaskHandle = atomic_load_ptr(&ctx->taskHandle); + if (otaskHandle && otaskHandle == atomic_val_compare_exchange_ptr(&ctx->taskHandle, otaskHandle, NULL)) { + taosEnableMemPoolUsage(ctx->memPoolSession); qDestroyTask(otaskHandle); - tsEnableRandErr = false; + taosDisableMemPoolUsage(); + + atomic_add_fetch_64(&gQueryMgmt.stat.taskExecDestroyNum, 1); + qDebug("task handle destroyed"); } } +void qwFreeSinkHandle(SQWTaskCtx *ctx) { + // Note: free/kill may in RC + void *osinkHandle = atomic_load_ptr(&ctx->sinkHandle); + if (osinkHandle && osinkHandle == atomic_val_compare_exchange_ptr(&ctx->sinkHandle, osinkHandle, NULL)) { + QW_SINK_ENABLE_MEMPOOL(ctx); + dsDestroyDataSinker(osinkHandle); + QW_SINK_DISABLE_MEMPOOL(); + + atomic_add_fetch_64(&gQueryMgmt.stat.taskSinkDestroyNum, 1); + + qDebug("sink handle destroyed"); + } +} + int32_t qwKillTaskHandle(SQWTaskCtx *ctx, int32_t rspCode) { int32_t code = 0; @@ -297,9 +317,9 @@ int32_t qwKillTaskHandle(SQWTaskCtx *ctx, int32_t rspCode) { QW_RET(code); } -void qwFreeTaskCtx(SQWTaskCtx *ctx) { +void qwFreeTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { if (ctx->ctrlConnInfo.handle) { - tmsgReleaseHandle(&ctx->ctrlConnInfo, TAOS_CONN_SERVER); + tmsgReleaseHandle(&ctx->ctrlConnInfo, TAOS_CONN_SERVER, ctx->rspCode); } ctx->ctrlConnInfo.handle = NULL; @@ -307,15 +327,15 @@ void qwFreeTaskCtx(SQWTaskCtx *ctx) { // NO need to release dataConnInfo - qwFreeTaskHandle(&ctx->taskHandle); + qwFreeTaskHandle(ctx); - if (ctx->sinkHandle) { - dsDestroyDataSinker(ctx->sinkHandle); - ctx->sinkHandle = NULL; - qDebug("sink handle destroyed"); - } + qwFreeSinkHandle(ctx); taosArrayDestroy(ctx->tbInfo); + + if (gMemPoolHandle && ctx->memPoolSession) { + qwDestroySession(QW_FPARAMS(), ctx->pJobInfo, ctx->memPoolSession); + } } static void freeExplainExecItem(void *param) { @@ -323,20 +343,19 @@ static void freeExplainExecItem(void *param) { taosMemoryFree(pInfo->verboseInfo); } - int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; qTaskInfo_t taskHandle = ctx->taskHandle; ctx->explainRsped = true; - + SArray *execInfoList = taosArrayInit(4, sizeof(SExplainExecInfo)); if (NULL == execInfoList) { QW_ERR_JRET(terrno); } - + QW_ERR_JRET(qGetExplainExecInfo(taskHandle, execInfoList)); - + if (ctx->localExec) { SExplainLocalRsp localRsp = {0}; localRsp.rsp.numOfPlans = taosArrayGetSize(execInfoList); @@ -354,7 +373,7 @@ int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { if (NULL == taosArrayPush(ctx->explainRes, &localRsp)) { QW_ERR_JRET(terrno); } - + taosArrayDestroy(execInfoList); execInfoList = NULL; } else { @@ -374,12 +393,11 @@ _return: return code; } - - int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; QW_SET_QTID(id, qId, cId, tId, eId); SQWTaskCtx octx; + int32_t code = TSDB_CODE_SUCCESS; SQWTaskCtx *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); if (NULL == ctx) { @@ -389,22 +407,30 @@ int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { octx = *ctx; + if (ctx->pJobInfo && TSDB_CODE_SUCCESS != ctx->pJobInfo->errCode) { + QW_UPDATE_RSP_CODE(ctx, ctx->pJobInfo->errCode); + } + atomic_store_ptr(&ctx->taskHandle, NULL); atomic_store_ptr(&ctx->sinkHandle, NULL); + atomic_store_ptr(&ctx->pJobInfo, NULL); + atomic_store_ptr(&ctx->memPoolSession, NULL); QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_DROP); if (taosHashRemove(mgmt->ctxHash, id, sizeof(id))) { QW_TASK_ELOG_E("taosHashRemove from ctx hash failed"); - QW_ERR_RET(QW_CTX_NOT_EXISTS_ERR_CODE(mgmt)); + code = QW_CTX_NOT_EXISTS_ERR_CODE(mgmt); } - qwFreeTaskCtx(&octx); + qwFreeTaskCtx(QW_FPARAMS(), &octx); ctx->tbInfo = NULL; QW_TASK_DLOG_E("task ctx dropped"); + + atomic_add_fetch_64(&gQueryMgmt.stat.taskDestroyNum, 1); - return TSDB_CODE_SUCCESS; + return code; } int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { @@ -415,7 +441,7 @@ int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; QW_SET_QTID(id, qId, cId, tId, eId); - if (qwAcquireScheduler(mgmt, sId, QW_WRITE, &sch)) { + if (qwAcquireScheduler(mgmt, cId, QW_WRITE, &sch)) { QW_TASK_WLOG_E("scheduler does not exist"); return TSDB_CODE_SUCCESS; } @@ -449,7 +475,7 @@ int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status, bool dynamicTask) { SQWTaskStatus *task = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireScheduler(mgmt, sId, QW_READ, &sch)); + QW_ERR_RET(qwAcquireScheduler(mgmt, cId, QW_READ, &sch)); QW_ERR_JRET(qwAcquireTaskStatus(QW_FPARAMS(), QW_READ, sch, &task)); QW_ERR_JRET(qwSetTaskStatus(QW_FPARAMS(), task, status, dynamicTask)); @@ -464,7 +490,6 @@ _return: QW_RET(code); } - int32_t qwHandleDynamicTaskEnd(QW_FPARAMS_DEF) { char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; QW_SET_QTID(id, qId, cId, tId, eId); @@ -525,16 +550,17 @@ void qwSetHbParam(int64_t refId, SQWHbParam **pParam) { } int32_t qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) { - char dbFName[TSDB_DB_FNAME_LEN]; - char tbName[TSDB_TABLE_NAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; STbVerInfo tbInfo; - int32_t i = 0; - int32_t code = TSDB_CODE_SUCCESS; - bool tbGet = false; + int32_t i = 0; + int32_t code = TSDB_CODE_SUCCESS; + bool tbGet = false; while (true) { tbGet = false; - code = qGetQueryTableSchemaVersion(pTaskInfo, dbFName, TSDB_DB_FNAME_LEN, tbName, TSDB_TABLE_NAME_LEN, &tbInfo.sversion, &tbInfo.tversion, i, &tbGet); + code = qGetQueryTableSchemaVersion(pTaskInfo, dbFName, TSDB_DB_FNAME_LEN, tbName, TSDB_TABLE_NAME_LEN, + &tbInfo.sversion, &tbInfo.tversion, i, &tbGet); if (TSDB_CODE_SUCCESS != code || !tbGet) { break; } @@ -551,11 +577,11 @@ int32_t qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) { QW_ERR_RET(terrno); } } - + if (NULL == taosArrayPush(ctx->tbInfo, &tbInfo)) { QW_ERR_RET(terrno); } - + i++; } @@ -567,6 +593,9 @@ void qwCloseRef(void) { if (atomic_load_32(&gQwMgmt.qwNum) <= 0 && gQwMgmt.qwRef >= 0) { taosCloseRef(gQwMgmt.qwRef); // ignore error gQwMgmt.qwRef = -1; + + taosHashCleanup(gQueryMgmt.pJobInfo); + gQueryMgmt.pJobInfo = NULL; } taosWUnLockLatch(&gQwMgmt.lock); } @@ -585,20 +614,22 @@ void qwDestroyImpl(void *pMgmt) { if (taosTmrStop(mgmt->hbTimer)) { qTrace("stop qworker hb timer may failed"); } - + mgmt->hbTimer = NULL; taosTmrCleanUp(mgmt->timer); - uint64_t qId, cId, tId; + uint64_t qId, cId, tId, sId; int32_t eId; + int64_t rId = 0; void *pIter = taosHashIterate(mgmt->ctxHash, NULL); while (pIter) { SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; void *key = taosHashGetKey(pIter, NULL); QW_GET_QTID(key, qId, cId, tId, eId); + sId = ctx->sId; - qwFreeTaskCtx(ctx); + qwFreeTaskCtx(QW_FPARAMS(), ctx); QW_TASK_DLOG_E("task ctx freed"); pIter = taosHashIterate(mgmt->ctxHash, pIter); taskCount++; @@ -634,7 +665,7 @@ int32_t qwOpenRef(void) { if (gQwMgmt.qwRef < 0) { taosWUnLockLatch(&gQwMgmt.lock); qError("init qworker ref failed"); - QW_RET(TSDB_CODE_OUT_OF_MEMORY); + QW_RET(gQwMgmt.qwRef); } } taosWUnLockLatch(&gQwMgmt.lock); @@ -686,25 +717,172 @@ void qwClearExpiredSch(SQWorker *mgmt, SArray *pExpiredSch) { int32_t code = TSDB_CODE_SUCCESS; int32_t num = taosArrayGetSize(pExpiredSch); for (int32_t i = 0; i < num; ++i) { - uint64_t *sId = taosArrayGet(pExpiredSch, i); + uint64_t *clientId = taosArrayGet(pExpiredSch, i); SQWSchStatus *pSch = NULL; - if (NULL == sId) { - qError("get the %dth sch failed, code:%x", i, terrno); + if (NULL == clientId) { + qError("get the %dth client failed, code:%x", i, terrno); break; } - code = qwAcquireScheduler(mgmt, *sId, QW_WRITE, &pSch); + code = qwAcquireScheduler(mgmt, *clientId, QW_WRITE, &pSch); if (TSDB_CODE_SUCCESS != code) { - qError("acquire sch %" PRIx64 " failed, code:%x", *sId, code); + qError("acquire client %" PRIx64 " failed, code:%x", *clientId, code); continue; } if (taosHashGetSize(pSch->tasksHash) <= 0) { qwDestroySchStatus(pSch); - code = taosHashRemove(mgmt->schHash, sId, sizeof(*sId)); - qDebug("sch %" PRIx64 " destroy result code:%x", *sId, code); + code = taosHashRemove(mgmt->schHash, clientId, sizeof(*clientId)); + qDebug("client %" PRIx64 " destroy result code:%x", *clientId, code); } qwReleaseScheduler(QW_WRITE, mgmt); } } + +void qwDestroyJobInfo(void *job) { + if (NULL == job) { + return; + } + + SQWJobInfo *pJob = (SQWJobInfo *)job; + + taosMemoryFreeClear(pJob->memInfo); + taosHashCleanup(pJob->pSessions); + pJob->pSessions = NULL; +} + +bool qwStopTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool forceStop, int32_t errCode) { + int32_t code = TSDB_CODE_SUCCESS; + bool resFreed = false; + + QW_LOCK(QW_WRITE, &ctx->lock); + + QW_TASK_DLOG("start to stop task, forceStop:%d, error:%s", forceStop, tstrerror(errCode)); + + if ((!forceStop) && (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP) || QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP))) { + QW_TASK_WLOG_E("task already dropping"); + QW_UNLOCK(QW_WRITE, &ctx->lock); + + return resFreed; + } + + if (QW_QUERY_RUNNING(ctx)) { + code = qwKillTaskHandle(ctx, errCode); + if (TSDB_CODE_SUCCESS != code) { + QW_TASK_ELOG("task running, async kill failed, error: %x", code); + } else { + QW_TASK_DLOG_E("task running, async killed"); + } + } else if (QW_FETCH_RUNNING(ctx)) { + QW_TASK_DLOG_E("task fetching"); + QW_UPDATE_RSP_CODE(ctx, errCode); + if (forceStop) { + QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); + QW_TASK_DLOG_E("update drop received"); + } + } else if (forceStop) { + QW_UPDATE_RSP_CODE(ctx, errCode); + code = qwDropTask(QW_FPARAMS()); + if (TSDB_CODE_SUCCESS != code) { + QW_TASK_ELOG("task drop failed, error: %x", code); + } else { + QW_TASK_DLOG_E("task dropped"); + resFreed = true; + } + } else { + QW_UPDATE_RSP_CODE(ctx, errCode); + + qwFreeTaskHandle(ctx); + qwFreeSinkHandle(ctx); + + resFreed = true; + + QW_TASK_DLOG_E("task resources freed"); + } + + QW_UNLOCK(QW_WRITE, &ctx->lock); + + return resFreed; +} + +bool qwRetireTask(QW_FPARAMS_DEF, int32_t errCode) { + SQWTaskCtx *ctx = NULL; + + int32_t code = qwAcquireTaskCtx(QW_FPARAMS(), &ctx); + if (TSDB_CODE_SUCCESS != code) { + return false; + } + + bool retired = qwStopTask(QW_FPARAMS(), ctx, false, errCode); + + qwReleaseTaskCtx(mgmt, ctx); + + return retired; +} + +bool qwRetireJob(SQWJobInfo *pJob) { + if (NULL == pJob) { + return false; + } + + bool retired = true; + void *pIter = taosHashIterate(pJob->pSessions, NULL); + while (pIter) { + SQWSessionInfo *pSession = (SQWSessionInfo *)pIter; + + if (!qwRetireTask((SQWorker *)pSession->mgmt, pSession->sId, pSession->qId, pSession->cId, pSession->tId, pSession->rId, pSession->eId, pJob->errCode)) { + retired = false; + } + + pIter = taosHashIterate(pJob->pSessions, pIter); + } + + return retired; +} + + +void qwStopAllTasks(SQWorker *mgmt) { + uint64_t qId, cId, tId, sId; + int32_t eId; + int64_t rId = 0; + + void *pIter = taosHashIterate(mgmt->ctxHash, NULL); + while (pIter) { + SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; + void *key = taosHashGetKey(pIter, NULL); + QW_GET_QTID(key, qId, cId, tId, eId); + + sId = ctx->sId; + + (void)qwStopTask(QW_FPARAMS(), ctx, true, TSDB_CODE_VND_STOPPED); + + pIter = taosHashIterate(mgmt->ctxHash, pIter); + } +} + + +void qwChkDropTimeoutQuery(SQWorker *mgmt, int32_t currTs) { + uint64_t qId, cId, tId, sId; + int32_t eId; + int64_t rId = 0; + + void *pIter = taosHashIterate(mgmt->ctxHash, NULL); + while (pIter) { + SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; + if (((ctx->lastAckTs <= 0) || (currTs - ctx->lastAckTs) < tsQueryNoFetchTimeoutSec) && (!QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP))) { + pIter = taosHashIterate(mgmt->ctxHash, pIter); + continue; + } + + void *key = taosHashGetKey(pIter, NULL); + QW_GET_QTID(key, qId, cId, tId, eId); + + sId = ctx->sId; + + (void)qwStopTask(QW_FPARAMS(), ctx, true, (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) ? ctx->rspCode : TSDB_CODE_QRY_NO_FETCH_TIMEOUT); + + pIter = taosHashIterate(mgmt->ctxHash, pIter); + } +} + diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index aae31480e2..50af01cd6f 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -18,71 +18,23 @@ SQWorkerMgmt gQwMgmt = { .qwNum = 0, }; -void qwStopAllTasks(SQWorker *mgmt) { - uint64_t qId, cId, tId, sId; - int32_t eId; - int64_t rId = 0; - int32_t code = TSDB_CODE_SUCCESS; +TdThreadOnce gQueryPoolInit = PTHREAD_ONCE_INIT; +SQueryMgmt gQueryMgmt = {0}; - void *pIter = taosHashIterate(mgmt->ctxHash, NULL); - while (pIter) { - SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; - void *key = taosHashGetKey(pIter, NULL); - QW_GET_QTID(key, qId, cId, tId, eId); - - QW_LOCK(QW_WRITE, &ctx->lock); - - sId = ctx->sId; - - QW_TASK_DLOG_E("start to force stop task"); - - if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP) || QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG_E("task already dropping"); - QW_UNLOCK(QW_WRITE, &ctx->lock); - - pIter = taosHashIterate(mgmt->ctxHash, pIter); - continue; - } - - if (QW_QUERY_RUNNING(ctx)) { - code = qwKillTaskHandle(ctx, TSDB_CODE_VND_STOPPED); - if (TSDB_CODE_SUCCESS != code) { - QW_TASK_ELOG("task running, async kill failed, error: %x", code); - } else { - QW_TASK_DLOG_E("task running, async killed"); - } - } else if (QW_FETCH_RUNNING(ctx)) { - QW_UPDATE_RSP_CODE(ctx, TSDB_CODE_VND_STOPPED); - QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); - QW_TASK_DLOG_E("task fetching, update drop received"); - } else { - code = qwDropTask(QW_FPARAMS()); - if (TSDB_CODE_SUCCESS != code) { - QW_TASK_ELOG("task drop failed, error: %x", code); - } else { - QW_TASK_DLOG_E("task dropped"); - } - } - - QW_UNLOCK(QW_WRITE, &ctx->lock); - - pIter = taosHashIterate(mgmt->ctxHash, pIter); - } -} int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; SQWSchStatus *sch = NULL; - QW_ERR_RET(qwAcquireScheduler(mgmt, req->sId, QW_READ, &sch)); + QW_ERR_RET(qwAcquireScheduler(mgmt, req->clientId, QW_READ, &sch)); QW_LOCK(QW_WRITE, &sch->hbConnLock); sch->hbBrokenTs = taosGetTimestampMs(); if (qwMsg->connInfo.handle == sch->hbConnInfo.handle) { - tmsgReleaseHandle(&sch->hbConnInfo, TAOS_CONN_SERVER); + tmsgReleaseHandle(&sch->hbConnInfo, TAOS_CONN_SERVER, 0); sch->hbConnInfo.handle = NULL; sch->hbConnInfo.ahandle = NULL; @@ -100,20 +52,29 @@ int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *re int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { qTaskInfo_t taskHandle = ctx->taskHandle; + int32_t code = TSDB_CODE_SUCCESS; ctx->queryExecDone = true; if (TASK_TYPE_TEMP == ctx->taskType && taskHandle) { if (ctx->explain && !ctx->explainRsped) { - QW_ERR_RET(qwSendExplainResponse(QW_FPARAMS(), ctx)); + QW_ERR_JRET(qwSendExplainResponse(QW_FPARAMS(), ctx)); } if (!ctx->needFetch) { + QW_SINK_ENABLE_MEMPOOL(ctx); dsGetDataLength(ctx->sinkHandle, &ctx->affectedRows, NULL, NULL); + QW_SINK_DISABLE_MEMPOOL(); } } - return TSDB_CODE_SUCCESS; +_return: + + if ((!ctx->dynamicTask) && (!ctx->explain || ctx->explainRsped)) { + qwFreeTaskHandle(ctx); + } + + return code; } int32_t qwSendQueryRsp(QW_FPARAMS_DEF, int32_t msgType, SQWTaskCtx *ctx, int32_t rspCode, bool quickRsp) { @@ -123,6 +84,7 @@ int32_t qwSendQueryRsp(QW_FPARAMS_DEF, int32_t msgType, SQWTaskCtx *ctx, int32_t QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, rspCode, tstrerror(rspCode)); } + ctx->lastAckTs = taosGetTimestampSec(); ctx->queryRsped = true; } @@ -151,7 +113,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { if (NULL == pResList) { QW_ERR_RET(terrno); } - + while (true) { QW_TASK_DLOG("start to execTask, loopIdx:%d", i++); @@ -161,9 +123,10 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { if (taskHandle) { qwDbgSimulateSleep(); - tsEnableRandErr = true; + taosEnableMemPoolUsage(ctx->memPoolSession); code = qExecTaskOpt(taskHandle, pResList, &useconds, &hasMore, &localFetch); - tsEnableRandErr = false; + taosDisableMemPoolUsage(); + if (code) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) { QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -184,7 +147,10 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { } SInputData inputData = {.pData = pRes}; + QW_SINK_ENABLE_MEMPOOL(ctx); code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); + QW_SINK_DISABLE_MEMPOOL(); + if (code) { QW_TASK_ELOG("dsPutDataBlock failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); @@ -212,7 +178,9 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { ctx->queryExecDone = true; } + QW_SINK_ENABLE_MEMPOOL(ctx); dsEndPut(sinkHandle, useconds); + QW_SINK_DISABLE_MEMPOOL(); if (queryStop) { *queryStop = true; @@ -245,7 +213,11 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { _return: taosArrayDestroy(pResList); - + + if (TSDB_CODE_SUCCESS != code) { + qwFreeTaskHandle(ctx); + } + QW_RET(code); } @@ -261,7 +233,7 @@ bool qwTaskNotInExec(SQWTaskCtx *ctx) { int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) { int32_t taskNum = 0; int32_t code = TSDB_CODE_SUCCESS; - + hbInfo->connInfo = sch->hbConnInfo; hbInfo->rsp.epId = sch->hbEpId; @@ -325,7 +297,9 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, *pRawDataLen = 0; while (true) { + QW_SINK_ENABLE_MEMPOOL(ctx); dsGetDataLength(ctx->sinkHandle, &len, &rawLen, &queryEnd); + QW_SINK_DISABLE_MEMPOOL(); if (len < 0) { QW_TASK_ELOG("invalid length from dsGetDataLength, length:%" PRId64 "", len); @@ -334,7 +308,10 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, if (len == 0) { if (queryEnd) { + QW_SINK_ENABLE_MEMPOOL(ctx); code = dsGetDataBlock(ctx->sinkHandle, &output); + QW_SINK_DISABLE_MEMPOOL(); + if (code) { QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); @@ -377,7 +354,10 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, ((int32_t *)output.pData)[1] = rawLen; output.pData += sizeof(int32_t) * 2; + QW_SINK_ENABLE_MEMPOOL(ctx); code = dsGetDataBlock(ctx->sinkHandle, &output); + QW_SINK_DISABLE_MEMPOOL(); + if (code) { QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); @@ -419,7 +399,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, _return: *rspMsg = pRsp; - + return code; } @@ -430,7 +410,9 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes int32_t code = 0; SOutputData output = {0}; + QW_SINK_ENABLE_MEMPOOL(ctx); dsGetDataLength(ctx->sinkHandle, &len, &rawLen, &queryEnd); + QW_SINK_DISABLE_MEMPOOL(); if (len <= 0 || len != sizeof(SDeleterRes)) { QW_TASK_ELOG("invalid length from dsGetDataLength, length:%" PRId64, len); @@ -442,7 +424,10 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes QW_ERR_RET(terrno); } + QW_SINK_ENABLE_MEMPOOL(ctx); code = dsGetDataBlock(ctx->sinkHandle, &output); + QW_SINK_DISABLE_MEMPOOL(); + if (code) { QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); taosMemoryFree(output.pData); @@ -491,13 +476,16 @@ int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg, int32 qwBuildFetchRsp(rsp, &sOutput, dataLen, rawLen, qComplete); if (qComplete) { atomic_store_8((int8_t *)&ctx->queryEnd, true); + if (!ctx->dynamicTask) { + qwFreeSinkHandle(ctx); + } } } qwMsg->connInfo = ctx->dataConnInfo; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - QW_ERR_RET(qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code)); + QW_ERR_RET(qwBuildAndSendFetchRsp(ctx, ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code)); rsp = NULL; QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), @@ -523,7 +511,9 @@ int32_t qwStartDynamicTaskNewExec(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg ctx->queryEnd = false; #endif + QW_SINK_ENABLE_MEMPOOL(ctx); dsReset(ctx->sinkHandle); + QW_SINK_DISABLE_MEMPOOL(); qUpdateOperatorParam(ctx->taskHandle, qwMsg->msg); @@ -551,17 +541,20 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_LOCK(QW_WRITE, &ctx->lock); - QW_SET_PHASE(ctx, phase); + if (ctx->pJobInfo && (atomic_load_8(&ctx->pJobInfo->retired) || atomic_load_32(&ctx->pJobInfo->errCode))) { + QW_TASK_ELOG("job already failed, error:%s", tstrerror(ctx->pJobInfo->errCode)); + QW_ERR_JRET(ctx->pJobInfo->errCode); + } if (atomic_load_8((int8_t *)&ctx->queryEnd) && !ctx->dynamicTask) { - QW_TASK_ELOG_E("query already end"); + QW_TASK_ELOG("query already end, phase:%d", phase); QW_ERR_JRET(TSDB_CODE_QW_MSG_ERROR); } switch (phase) { case QW_PHASE_PRE_QUERY: { if (QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { - QW_TASK_ELOG("task already dropped at wrong phase %s", qwPhaseStr(phase)); + QW_TASK_ELOG("task already dropped at phase %s", qwPhaseStr(phase)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_STATUS_ERROR); } @@ -631,6 +624,8 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_ERR_JRET(ctx->rspCode); } + QW_SET_PHASE(ctx, phase); + _return: if (ctx) { @@ -640,7 +635,7 @@ _return: qwReleaseTaskCtx(mgmt, ctx); } - if (code != TSDB_CODE_SUCCESS) { + if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_QRY_TASK_CTX_NOT_EXIST) { QW_TASK_ELOG("end to handle event at phase %s, code:%s", qwPhaseStr(phase), tstrerror(code)); } else { QW_TASK_DLOG("end to handle event at phase %s, code:%s", qwPhaseStr(phase), tstrerror(code)); @@ -712,7 +707,8 @@ _return: } if (code) { - (void)qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask); // already in error, ignore new error + (void)qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, + ctx->dynamicTask); // already in error, ignore new error } QW_UNLOCK(QW_WRITE, &ctx->lock); @@ -744,6 +740,10 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { ctx->sId = sId; ctx->phase = -1; + if (NULL != gMemPoolHandle) { + QW_ERR_JRET(qwInitSession(QW_FPARAMS(), ctx, &ctx->memPoolSession)); + } + QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT)); QW_ERR_JRET(qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true)); @@ -752,9 +752,15 @@ _return: if (ctx) { QW_UPDATE_RSP_CODE(ctx, code); + if (code) { + (void)qwDropTask(QW_FPARAMS()); + } + qwReleaseTaskCtx(mgmt, ctx); } + QW_TASK_DLOG("task preprocess %s, code:%s", code ? "failed": "succeed", tstrerror(code)); + return code; } @@ -776,19 +782,20 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { ctx->queryMsgType = qwMsg->msgType; ctx->localExec = false; + taosEnableMemPoolUsage(ctx->memPoolSession); code = qMsgToSubplan(qwMsg->msg, qwMsg->msgLen, &plan); + taosDisableMemPoolUsage(); + if (TSDB_CODE_SUCCESS != code) { - code = TSDB_CODE_INVALID_MSG; QW_TASK_ELOG("task physical plan to subplan failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); } - tsEnableRandErr = true; + taosEnableMemPoolUsage(ctx->memPoolSession); code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, qwMsg->msgInfo.compressMsg, sql, OPTR_EXEC_MODEL_BATCH); - tsEnableRandErr = false; - - sql = NULL; + taosDisableMemPoolUsage(); + if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); qDestroyTask(pTaskInfo); @@ -801,8 +808,14 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { QW_ERR_JRET(TSDB_CODE_APP_ERROR); } + atomic_add_fetch_64(&gQueryMgmt.stat.taskRunNum, 1); + + uint64_t flags = 0; + dsGetSinkFlags(sinkHandle, &flags); + ctx->level = plan->level; ctx->dynamicTask = qIsDynamicExecTask(pTaskInfo); + ctx->sinkWithMemPool = flags & DS_FLAG_USE_MEMPOOL; atomic_store_ptr(&ctx->taskHandle, pTaskInfo); atomic_store_ptr(&ctx->sinkHandle, sinkHandle); @@ -869,12 +882,15 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { if (qComplete) { atomic_store_8((int8_t *)&ctx->queryEnd, true); atomic_store_8((int8_t *)&ctx->queryContinue, 0); + if (!ctx->dynamicTask) { + qwFreeSinkHandle(ctx); + } } qwMsg->connInfo = ctx->dataConnInfo; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - QW_ERR_JRET(qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code)); + QW_ERR_JRET(qwBuildAndSendFetchRsp(ctx, ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code)); rsp = NULL; QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, @@ -897,13 +913,13 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); qwMsg->connInfo = ctx->dataConnInfo; - code = qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, NULL, 0, code); + code = qwBuildAndSendFetchRsp(ctx, ctx->fetchMsgType + 1, &qwMsg->connInfo, NULL, 0, code); if (TSDB_CODE_SUCCESS != code) { - QW_TASK_ELOG("fetch rsp send fail, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), - 0); + QW_TASK_ELOG("fetch rsp send fail, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, + tstrerror(code), 0); } else { - QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), - 0); + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, + tstrerror(code), 0); } } @@ -958,6 +974,9 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { qwBuildFetchRsp(rsp, &sOutput, dataLen, rawDataLen, qComplete); if (qComplete) { atomic_store_8((int8_t *)&ctx->queryEnd, true); + if (!ctx->dynamicTask) { + qwFreeSinkHandle(ctx); + } } } @@ -1003,10 +1022,10 @@ _return: } if (!rsped) { - code = qwBuildAndSendFetchRsp(qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); + code = qwBuildAndSendFetchRsp(ctx, qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); if (TSDB_CODE_SUCCESS != code) { - QW_TASK_ELOG("fetch rsp send fail, msgType:%s, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1), - qwMsg->connInfo.handle, code, tstrerror(code), dataLen); + QW_TASK_ELOG("fetch rsp send fail, msgType:%s, handle:%p, code:%x - %s, dataLen:%d", + TMSG_INFO(qwMsg->msgType + 1), qwMsg->connInfo.handle, code, tstrerror(code), dataLen); } else { QW_TASK_DLOG("fetch rsp send, msgType:%s, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1), qwMsg->connInfo.handle, code, tstrerror(code), dataLen); @@ -1057,9 +1076,10 @@ _return: if (code) { if (ctx) { QW_UPDATE_RSP_CODE(ctx, code); - (void)qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask); // task already failed, no more error handling + (void)qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, + ctx->dynamicTask); // task already failed, no more error handling } else { - tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER); + tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER, code); } } @@ -1069,7 +1089,7 @@ _return: if (ctx) { if (qwMsg->connInfo.handle != ctx->ctrlConnInfo.handle) { - tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER); + tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER, 0); } qwReleaseTaskCtx(mgmt, ctx); @@ -1111,7 +1131,8 @@ _return: if (code) { if (ctx) { QW_UPDATE_RSP_CODE(ctx, code); - (void)qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask); // task already failed, no more error handling + (void)qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, + ctx->dynamicTask); // task already failed, no more error handling } } @@ -1135,15 +1156,15 @@ int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { QW_RET(qwProcessHbLinkBroken(mgmt, qwMsg, req)); } - QW_ERR_JRET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch)); - QW_ERR_JRET(qwRegisterHbBrokenLinkArg(mgmt, req->sId, &qwMsg->connInfo)); + QW_ERR_JRET(qwAcquireAddScheduler(mgmt, req->clientId, QW_READ, &sch)); + QW_ERR_JRET(qwRegisterHbBrokenLinkArg(mgmt, req->clientId, &qwMsg->connInfo)); sch->hbBrokenTs = 0; QW_LOCK(QW_WRITE, &sch->hbConnLock); if (sch->hbConnInfo.handle) { - tmsgReleaseHandle(&sch->hbConnInfo, TAOS_CONN_SERVER); + tmsgReleaseHandle(&sch->hbConnInfo, TAOS_CONN_SERVER, 0); sch->hbConnInfo.handle = NULL; } @@ -1152,8 +1173,9 @@ int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { QW_UNLOCK(QW_WRITE, &sch->hbConnLock); - QW_DLOG("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, handle:%p, ahandle:%p", req->sId, - req->epId.nodeId, req->epId.ep.fqdn, req->epId.ep.port, qwMsg->connInfo.handle, qwMsg->connInfo.ahandle); + QW_DLOG("hb connection updated, clientId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, handle:%p, ahandle:%p", + req->clientId, req->epId.nodeId, req->epId.ep.fqdn, req->epId.ep.port, qwMsg->connInfo.handle, + qwMsg->connInfo.ahandle); qwReleaseScheduler(QW_READ, mgmt); @@ -1163,7 +1185,7 @@ _return: code = qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code); if (code) { - tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER); + tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER, 0); qwMsg->connInfo.handle = NULL; } @@ -1189,13 +1211,20 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { SQWHbInfo *rspList = NULL; SArray *pExpiredSch = NULL; int32_t code = 0; + int32_t currTs = taosGetTimestampSec(); qwDbgDumpMgmtInfo(mgmt); + qwDbgDumpJobsInfo(); if (gQWDebug.forceStop) { qwStopAllTasks(mgmt); } + if (mgmt->lastChkTs > 0 && (currTs - mgmt->lastChkTs) >= QW_DEFAULT_TIMEOUT_INTERVAL_SECS) { + qwChkDropTimeoutQuery(mgmt, currTs); + mgmt->lastChkTs = currTs; + } + QW_LOCK(QW_READ, &mgmt->schLock); int32_t schNum = taosHashGetSize(mgmt->schHash); @@ -1204,7 +1233,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { if (taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer)) { qError("reset qworker hb timer error, timer stoppped"); } - (void)qwRelease(refId); // ignore error + (void)qwRelease(refId); // ignore error return; } @@ -1218,7 +1247,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { if (taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer)) { qError("reset qworker hb timer error, timer stoppped"); } - (void)qwRelease(refId); // ignore error + (void)qwRelease(refId); // ignore error return; } @@ -1229,13 +1258,13 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { while (pIter) { SQWSchStatus *sch1 = (SQWSchStatus *)pIter; if (NULL == sch1->hbConnInfo.handle) { - uint64_t *sId = taosHashGetKey(pIter, NULL); - QW_TLOG("cancel send hb to sch %" PRIx64 " cause of no connection handle", *sId); + uint64_t *clientId = taosHashGetKey(pIter, NULL); + QW_TLOG("cancel send hb to client %" PRIx64 " cause of no connection handle", *clientId); if (sch1->hbBrokenTs > 0 && ((currentMs - sch1->hbBrokenTs) > QW_SCH_TIMEOUT_MSEC) && taosHashGetSize(sch1->tasksHash) <= 0) { - if (NULL == taosArrayPush(pExpiredSch, sId)) { - QW_ELOG("add sId 0x%" PRIx64 " to expiredSch failed, code:%x", *sId, terrno); + if (NULL == taosArrayPush(pExpiredSch, clientId)) { + QW_ELOG("add clientId 0x%" PRIx64 " to expiredSch failed, code:%x", *clientId, terrno); taosHashCancelIterate(mgmt->schHash, pIter); break; } @@ -1260,7 +1289,7 @@ _return: QW_UNLOCK(QW_READ, &mgmt->schLock); for (int32_t j = 0; j < i; ++j) { - (void)qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code); // ignore error + (void)qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code); // ignore error /*QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code),*/ /*(rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0));*/ tFreeSSchedulerHbRsp(&rspList[j].rsp); @@ -1277,7 +1306,7 @@ _return: qError("reset qworker hb timer error, timer stoppped"); } - (void)qwRelease(refId); // ignore error + (void)qwRelease(refId); // ignore error } int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) { @@ -1288,15 +1317,14 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) { SQWTaskCtx ctx = {0}; code = qMsgToSubplan(qwMsg->msg, qwMsg->msgLen, &plan); + if (TSDB_CODE_SUCCESS != code) { code = TSDB_CODE_INVALID_MSG; QW_TASK_ELOG("task physical plan to subplan failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); } - tsEnableRandErr = true; code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, 0, NULL, OPTR_EXEC_MODEL_BATCH); - tsEnableRandErr = false; if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -1313,18 +1341,24 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) { ctx.taskHandle = pTaskInfo; ctx.sinkHandle = sinkHandle; + uint64_t flags = 0; + dsGetSinkFlags(sinkHandle, &flags); + + ctx.sinkWithMemPool = flags & DS_FLAG_USE_MEMPOOL; + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), &ctx, NULL)); QW_ERR_JRET(qwGetDeleteResFromSink(QW_FPARAMS(), &ctx, pRes)); _return: - qwFreeTaskCtx(&ctx); + qwFreeTaskCtx(QW_FPARAMS(), &ctx); QW_RET(TSDB_CODE_SUCCESS); } int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const SMsgCb *pMsgCb) { + int32_t code = TSDB_CODE_SUCCESS; if (NULL == qWorkerMgmt || (pMsgCb && pMsgCb->mgmt == NULL)) { qError("invalid param to init qworker"); QW_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -1335,7 +1369,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S TAOS_MEMSET(gQwMgmt.param, 0, sizeof(gQwMgmt.param)); } - int32_t code = qwOpenRef(); + code = qwOpenRef(); if (code) { (void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); QW_RET(code); @@ -1370,7 +1404,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S mgmt->timer = taosTmrInit(0, 0, 0, "qworker"); if (NULL == mgmt->timer) { qError("init timer failed, error:%s", tstrerror(terrno)); - QW_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + QW_ERR_JRET(terrno); } mgmt->nodeType = nodeType; @@ -1393,9 +1427,13 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, (void *)param, mgmt->timer); if (NULL == mgmt->hbTimer) { qError("start hb timer failed"); - QW_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + QW_ERR_JRET(terrno); } + QW_ERR_JRET(qExecutorInit()); + + mgmt->lastChkTs = taosGetTimestampSec(); + *qWorkerMgmt = mgmt; qDebug("qworker initialized, type:%d, id:%d, handle:%p", mgmt->nodeType, mgmt->nodeId, mgmt); @@ -1405,7 +1443,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S _return: if (mgmt->refId >= 0) { - (void)qwRelease(mgmt->refId); // ignore error + (void)qwRelease(mgmt->refId); // ignore error } else { taosHashCleanup(mgmt->schHash); taosHashCleanup(mgmt->ctxHash); @@ -1506,8 +1544,9 @@ int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64 if (NULL == rHandle.pMsgCb) { QW_ERR_JRET(terrno); } - + rHandle.pMsgCb->clientRpc = qwMsg->connInfo.handle; + rHandle.localExec = true; code = qCreateExecTask(&rHandle, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, 0, NULL, OPTR_EXEC_MODEL_BATCH); if (code) { @@ -1575,6 +1614,9 @@ int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64 qwBuildFetchRsp(rsp, &sOutput, dataLen, rawLen, qComplete); if (qComplete) { atomic_store_8((int8_t *)&ctx->queryEnd, true); + if (!ctx->dynamicTask) { + qwFreeSinkHandle(ctx); + } } break; @@ -1590,3 +1632,70 @@ _return: QW_RET(code); } + +void qWorkerRetireJob(uint64_t jobId, uint64_t clientId, int32_t errCode) { + char id[sizeof(jobId) + sizeof(clientId) + 1] = {0}; + QW_SET_QCID(id, jobId, clientId); + + SQWJobInfo *pJob = (SQWJobInfo *)taosHashGet(gQueryMgmt.pJobInfo, id, sizeof(id)); + if (NULL == pJob) { + qError("QID:0x%" PRIx64 " CID:0x%" PRIx64 " fail to get job from job hash", jobId, clientId); + return; + } + + if (0 == atomic_val_compare_exchange_32(&pJob->errCode, 0, errCode) && + 0 == atomic_val_compare_exchange_8(&pJob->retired, 0, 1)) { + qDebug("QID:0x%" PRIx64 " CID:0x%" PRIx64 " mark retired, errCode: 0x%x, allocSize:%" PRId64, jobId, clientId, + errCode, atomic_load_64(&pJob->memInfo->allocMemSize)); + + (void)qwRetireJob(pJob); + } else { + qDebug("QID:0x%" PRIx64 " already retired, retired: %d, errCode: 0x%x, allocSize:%" PRId64, jobId, + atomic_load_8(&pJob->retired), atomic_load_32(&pJob->errCode), atomic_load_64(&pJob->memInfo->allocMemSize)); + } +} + +void qWorkerRetireJobs(int64_t retireSize, int32_t errCode) { + qDebug("need to retire jobs in batch, targetRetireSize:%" PRId64 ", remainJobNum:%d, task initNum:%" PRId64 " - %" PRId64 + ", task destroyNum:%" PRId64 " - %" PRId64 " - %" PRId64, + retireSize, taosHashGetSize(gQueryMgmt.pJobInfo), atomic_load_64(&gQueryMgmt.stat.taskInitNum), atomic_load_64(&gQueryMgmt.stat.taskRunNum), + atomic_load_64(&gQueryMgmt.stat.taskExecDestroyNum), atomic_load_64(&gQueryMgmt.stat.taskSinkDestroyNum), + atomic_load_64(&gQueryMgmt.stat.taskDestroyNum)); + + SQWJobInfo* pJob = (SQWJobInfo*)taosHashIterate(gQueryMgmt.pJobInfo, NULL); + int32_t jobNum = 0; + int32_t alreadyJobNum = 0; + int64_t retiredSize = 0; + while (retiredSize < retireSize && NULL != pJob && jobNum < QW_RETIRE_JOB_BATCH_NUM) { + if (atomic_load_8(&pJob->retired)) { + pJob = (SQWJobInfo*)taosHashIterate(gQueryMgmt.pJobInfo, pJob); + alreadyJobNum++; + continue; + } + + if (0 == atomic_val_compare_exchange_32(&pJob->errCode, 0, errCode) && 0 == atomic_val_compare_exchange_8(&pJob->retired, 0, 1)) { + int64_t aSize = atomic_load_64(&pJob->memInfo->allocMemSize); + bool retired = qwRetireJob(pJob); + + retiredSize += aSize; + + jobNum++; + + qDebug("QID:0x%" PRIx64 " CID:0x%" PRIx64 " job mark retired in batch, retired:%d, usedSize:%" PRId64 ", retireSize:%" PRId64, + pJob->memInfo->jobId, pJob->memInfo->clientId, retired, aSize, retireSize); + } else { + qDebug("QID:0x%" PRIx64 " CID:0x%" PRIx64 " job may already failed, errCode:%s", pJob->memInfo->jobId, pJob->memInfo->clientId, tstrerror(pJob->errCode)); + } + + pJob = (SQWJobInfo *)taosHashIterate(gQueryMgmt.pJobInfo, pJob); + } + + taosHashCancelIterate(gQueryMgmt.pJobInfo, pJob); + + qDebug("job retire in batch done, [prev:%d, curr:%d, total:%d] jobs, direct retiredSize:%" PRId64 " targetRetireSize:%" PRId64 + ", task initNum:%" PRId64 " - %" PRId64 ", task destroyNum:%" PRId64 " - %" PRId64 " - %" PRId64, + alreadyJobNum, jobNum, taosHashGetSize(gQueryMgmt.pJobInfo), retiredSize, retireSize, + atomic_load_64(&gQueryMgmt.stat.taskInitNum), atomic_load_64(&gQueryMgmt.stat.taskRunNum), + atomic_load_64(&gQueryMgmt.stat.taskExecDestroyNum), atomic_load_64(&gQueryMgmt.stat.taskSinkDestroyNum), + atomic_load_64(&gQueryMgmt.stat.taskDestroyNum)); +} diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 8c6db7b8ce..9a369cd4c4 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -51,7 +51,6 @@ int32_t sclCreateColumnInfoData(SDataType *pType, int32_t numOfRows, SScalarPara int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows, true); if (code != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_OUT_OF_MEMORY; colDataDestroy(pColumnData); taosMemoryFree(pColumnData); return terrno; @@ -1235,7 +1234,7 @@ EDealRes sclRewriteFunction(SNode **pNode, SScalarCtx *ctx) { sclError("calloc %d failed", len); sclFreeParam(&output); nodesDestroyNode((SNode *)res); - ctx->code = TSDB_CODE_OUT_OF_MEMORY; + ctx->code = terrno; return DEAL_RES_ERROR; } (void)memcpy(res->datum.p, output.columnData->pData, len); @@ -1246,7 +1245,7 @@ EDealRes sclRewriteFunction(SNode **pNode, SScalarCtx *ctx) { sclError("calloc %d failed", (int)(varDataTLen(output.columnData->pData) + 1)); sclFreeParam(&output); nodesDestroyNode((SNode *)res); - ctx->code = TSDB_CODE_OUT_OF_MEMORY; + ctx->code = terrno; return DEAL_RES_ERROR; } (void)memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData)); @@ -1440,7 +1439,7 @@ EDealRes sclRewriteCaseWhen(SNode **pNode, SScalarCtx *ctx) { sclError("calloc %d failed", (int)(varDataTLen(output.columnData->pData) + 1)); sclFreeParam(&output); nodesDestroyNode((SNode *)res); - ctx->code = TSDB_CODE_OUT_OF_MEMORY; + ctx->code = terrno; return DEAL_RES_ERROR; } (void)memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData)); @@ -1494,7 +1493,7 @@ EDealRes sclWalkFunction(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_OUT_OF_MEMORY; + ctx->code = terrno; sclFreeParam(&output); return DEAL_RES_ERROR; } @@ -1513,7 +1512,7 @@ EDealRes sclWalkLogic(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_OUT_OF_MEMORY; + ctx->code = terrno; sclFreeParam(&output); return DEAL_RES_ERROR; } @@ -1532,7 +1531,7 @@ EDealRes sclWalkOperator(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_OUT_OF_MEMORY; + ctx->code = terrno; sclFreeParam(&output); return DEAL_RES_ERROR; } @@ -1613,7 +1612,7 @@ EDealRes sclWalkCaseWhen(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_OUT_OF_MEMORY; + ctx->code = terrno; return DEAL_RES_ERROR; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 393bb947cf..fd1bd927b0 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1078,7 +1078,7 @@ int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_ int32_t code = sclCreateColumnInfoData(&t, input->numOfRows, output); if (code != TSDB_CODE_SUCCESS) { - return TSDB_CODE_OUT_OF_MEMORY; + return code; } code = vectorConvertSingleColImpl(input, output, NULL, startIndex, numOfRows); diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 8bbadd0e22..98eded7363 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -74,7 +74,7 @@ int32_t flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) { if (IS_VAR_DATA_TYPE(dataType)) { vnode->datum.p = (char *)taosMemoryMalloc(varDataTLen(value)); if (NULL == vnode->datum.p) { - FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + FLT_ERR_RET(terrno); } varDataCopy(vnode->datum.p, value); vnode->node.resType.bytes = varDataLen(value); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 6a5188f208..3eae06d9bb 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -94,7 +94,7 @@ int32_t scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t SSDataBlock *res = NULL; int32_t code = createDataBlock(&res); if (code != 0 || NULL == res->pDataBlock) { - SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCL_ERR_RET(code); } SColumnInfoData idata = {0}; @@ -121,7 +121,7 @@ int32_t scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t } if (NULL == taosArrayPush(pBlockList, &res)) { - SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCL_ERR_RET(terrno); } *dataBlockId = taosArrayGetSize(pBlockList) - 1; res->info.id.blockId = *dataBlockId; @@ -161,7 +161,7 @@ int32_t scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { if (IS_VAR_DATA_TYPE(dataType)) { vnode->datum.p = (char *)taosMemoryMalloc(varDataTLen(value)); if (NULL == vnode->datum.p) { - SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCL_ERR_RET(terrno); } varDataCopy(vnode->datum.p, value); vnode->node.resType.bytes = varDataTLen(value); @@ -1385,7 +1385,7 @@ int32_t makeCalculate(void *json, void *key, int32_t rightType, void *rightData, SCL_ERR_RET(makeJsonArrow(&src, &opNode, json, (char *)key)); if (NULL == taosArrayPush(blockList, &src)) { - SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCL_ERR_RET(terrno); } SCL_ERR_RET(makeOperator(&opNode, blockList, opType, rightType, rightData, isReverse)); @@ -2483,7 +2483,7 @@ TEST(columnTest, greater_and_lower) { int32_t scltMakeDataBlock(SScalarParam **pInput, int32_t type, void *pVal, int32_t num, bool setVal) { SScalarParam *input = (SScalarParam *)taosMemoryCalloc(1, sizeof(SScalarParam)); if (NULL == input) { - SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCL_ERR_RET(terrno); } int32_t bytes; switch (type) { @@ -2515,7 +2515,7 @@ int32_t scltMakeDataBlock(SScalarParam **pInput, int32_t type, void *pVal, int32 input->columnData = (SColumnInfoData *)taosMemoryCalloc(1, sizeof(SColumnInfoData)); if (NULL == input->columnData) { - SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCL_ERR_RET(terrno); } input->numOfRows = num; diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index ef643852ea..607f43a06f 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -144,7 +144,7 @@ typedef struct SSchedulerCfg { typedef struct SSchedulerMgmt { uint64_t clientId; // unique clientId uint64_t taskId; // sequential taksId - uint64_t sId; // schedulerId + uint64_t seriousId; // sequential seriousId SSchedulerCfg cfg; bool exit; int32_t jobRef; @@ -163,6 +163,7 @@ typedef struct SSchCallbackParamHeader { typedef struct SSchTaskCallbackParam { SSchCallbackParamHeader head; uint64_t queryId; + uint64_t seriousId; int64_t refId; uint64_t clientId; uint64_t taskId; @@ -218,6 +219,7 @@ typedef struct SSchRedirectCtx { } SSchRedirectCtx; typedef struct SSchTimerParam { + int8_t exit; int64_t rId; uint64_t queryId; uint64_t taskId; @@ -226,6 +228,8 @@ typedef struct SSchTimerParam { typedef struct SSchTask { uint64_t clientId; // current client id uint64_t taskId; // task id + uint64_t seriousId; + uint64_t failedSeriousId; SRWLatch lock; // task reentrant lock int32_t maxExecTimes; // task max exec times int32_t maxRetryTimes; // task max retry times @@ -254,6 +258,7 @@ typedef struct SSchTask { SArray *parents; // the data destination tasks, get data from current task, element is SQueryTask* void *handle; // task send handle bool registerdHb; // registered in hb + SSchTimerParam delayLaunchPar; } SSchTask; typedef struct SSchJobAttr { @@ -274,6 +279,7 @@ typedef struct { typedef struct SSchJob { int64_t refId; uint64_t queryId; + uint64_t seriousId; SSchJobAttr attr; int32_t levelNum; int32_t taskNum; @@ -406,8 +412,13 @@ extern SSchedulerMgmt schMgmt; (SCH_NETWORK_ERR(_code) && ((_task)->level->level > (_job)->levelIdx)) #define SCH_TASK_RETRY_NETWORK_ERR(_task, _code) (SCH_NETWORK_ERR(_code) && (_task)->redirectCtx.inRedirect) +#if 0 #define SCH_JOB_NEED_RETRY(_job, _task, _msgType, _code) \ (SCH_REDIRECT_MSGTYPE(_msgType) && SCH_TOP_LEVEL_NETWORK_ERR(_job, _task, _code)) +#else +#define SCH_JOB_NEED_RETRY(_job, _task, _msgType, _code) \ + (SCH_REDIRECT_MSGTYPE(_msgType) && (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_NETWORK_ERR(_code))) +#endif #define SCH_TASKSET_NEED_RETRY(_job, _task, _msgType, _code) \ (SCH_REDIRECT_MSGTYPE(_msgType) && \ (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_LOW_LEVEL_NETWORK_ERR((_job), (_task), (_code)) || \ @@ -451,23 +462,23 @@ extern SSchedulerMgmt schMgmt; (_task)->profile.endTs = us; \ } while (0) -#define SCH_JOB_ELOG(param, ...) qError("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) -#define SCH_JOB_DLOG(param, ...) qDebug("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) +#define SCH_JOB_ELOG(param, ...) qError("QID:0x%" PRIx64 ",SID:%" PRId64 " " param, pJob->queryId, pJob->seriousId, __VA_ARGS__) +#define SCH_JOB_DLOG(param, ...) qDebug("QID:0x%" PRIx64 ",SID:%" PRId64 " " param, pJob->queryId, pJob->seriousId, __VA_ARGS__) #define SCH_TASK_ELOG(param, ...) \ - qError("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \ + qError("QID:0x%" PRIx64 ",SID:%" PRId64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, pJob->seriousId, SCH_CLIENT_ID(pTask), \ SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__) #define SCH_TASK_DLOG(param, ...) \ - qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \ + qDebug("QID:0x%" PRIx64 ",SID:%" PRId64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, pJob->seriousId, SCH_CLIENT_ID(pTask), \ SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__) #define SCH_TASK_TLOG(param, ...) \ - qTrace("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \ + qTrace("QID:0x%" PRIx64 ",SID:%" PRId64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, pJob->seriousId, SCH_CLIENT_ID(pTask), \ SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__) #define SCH_TASK_DLOGL(param, ...) \ - qDebugL("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \ + qDebugL("QID:0x%" PRIx64 ",SID:%" PRId64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, pJob->seriousId, SCH_CLIENT_ID(pTask), \ SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__) #define SCH_TASK_WLOG(param, ...) \ - qWarn("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \ + qWarn("QID:0x%" PRIx64 ",SID:%" PRId64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, pJob->seriousId, SCH_CLIENT_ID(pTask), \ SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__) #define SCH_SET_ERRNO(_err) \ @@ -618,7 +629,7 @@ void schCloseJobRef(void); int32_t schAsyncExecJob(SSchedulerReq *pReq, int64_t *pJob); int32_t schJobFetchRows(SSchJob *pJob); int32_t schJobFetchRowsA(SSchJob *pJob); -int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId); +int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, uint64_t seriousId, int32_t execId); int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList); int32_t schDumpEpSet(SEpSet *pEpSet, char **ppRes); char *schGetOpStr(SCH_OP_TYPE type); @@ -649,6 +660,7 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list); int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type, SSchTask *pTask); int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level); void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); +void schStopTaskDelayTimer(SSchJob *pJob, SSchTask* pTask, bool syncOp); int32_t schValidateSubplan(SSchJob *pJob, SSubplan *pSubplan, int32_t level, int32_t idx, int32_t taskNum); int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel); int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 375a316185..25052d2c15 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -345,6 +345,9 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { pJob->levelNum = levelNum; SCH_RESET_JOB_LEVEL_IDX(pJob); + + atomic_add_fetch_64(&pJob->seriousId, 1); + SCH_JOB_DLOG("job seriousId set to 0x%" PRIx64, pJob->seriousId); SSchLevel level = {0}; SNodeListNode *plans = NULL; @@ -674,7 +677,7 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { continue; } - SCH_ERR_RET(schLaunchTask(pJob, pTask)); + SCH_ERR_RET(schDelayLaunchTask(pJob, pTask)); } } @@ -994,17 +997,34 @@ int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode) { return TSDB_CODE_SUCCESS; } -int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) { - int8_t origInRetry = atomic_val_compare_exchange_8(&pJob->inRetry, 0, 1); - if (0 != origInRetry) { - SCH_JOB_DLOG("job already in retry, origInRetry: %d", pJob->inRetry); - return TSDB_CODE_SCH_IGNORE_ERROR; +int32_t schResetJobForRetry(SSchJob *pJob, SSchTask *pTask, int32_t rspCode, bool *inRetry) { + while (true) { + if (pTask->seriousId < atomic_load_64(&pJob->seriousId)) { + SCH_TASK_DLOG("task sId %" PRId64 " is smaller than current job sId %" PRId64, pTask->seriousId, pJob->seriousId); + return TSDB_CODE_SCH_IGNORE_ERROR; + } + + int8_t origInRetry = atomic_val_compare_exchange_8(&pJob->inRetry, 0, 1); + if (0 != origInRetry) { + SCH_JOB_DLOG("job already in retry, origInRetry: %d", pJob->inRetry); + taosUsleep(1); + continue; + } + + if (pTask->seriousId < atomic_load_64(&pJob->seriousId)) { + SCH_TASK_DLOG("task sId %" PRId64 " is smaller than current job sId %" PRId64, pTask->seriousId, pJob->seriousId); + return TSDB_CODE_SCH_IGNORE_ERROR; + } + + break; } *inRetry = true; SCH_ERR_RET(schChkResetJobRetry(pJob, rspCode)); + atomic_add_fetch_64(&pJob->seriousId, 1); + int32_t code = 0; int32_t numOfLevels = taosArrayGetSize(pJob->levels); for (int32_t i = 0; i < numOfLevels; ++i) { @@ -1031,13 +1051,19 @@ int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) { SCH_UNLOCK_TASK(pTask); SCH_RET(code); } - qClearSubplanExecutionNode(pTask->plan); schResetTaskForRetry(pJob, pTask); + + SCH_LOCK(SCH_WRITE, &pTask->planLock); + qClearSubplanExecutionNode(pTask->plan); + SCH_UNLOCK(SCH_WRITE, &pTask->planLock); + SCH_UNLOCK_TASK(pTask); } } SCH_RESET_JOB_LEVEL_IDX(pJob); + + SCH_JOB_DLOG("update job sId to %" PRId64, pJob->seriousId); return TSDB_CODE_SUCCESS; } @@ -1053,7 +1079,7 @@ int32_t schHandleJobRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pMsg, int32_ SCH_TASK_DLOG("start to redirect all job tasks cause of error: %s", tstrerror(rspCode)); - SCH_ERR_JRET(schResetJobForRetry(pJob, rspCode, &inRetry)); + SCH_ERR_JRET(schResetJobForRetry(pJob, pTask, rspCode, &inRetry)); SCH_ERR_JRET(schLaunchJob(pJob)); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 3321fdb4b5..fd255f53cf 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -142,7 +142,7 @@ int32_t schProcessExplainRsp(SSchJob *pJob, SSchTask *pTask, SExplainRsp *rsp) { return TSDB_CODE_SUCCESS; } -int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode) { +int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; int32_t msgSize = pMsg->len; int32_t msgType = pMsg->msgType; @@ -444,27 +444,38 @@ _return: // Note: no more task error processing, handled in function internal -int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode) { +int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, uint64_t seriousId, int32_t execId, SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; int32_t msgType = pMsg->msgType; bool dropExecNode = (msgType == TDMT_SCH_LINK_BROKEN || SCH_NETWORK_ERR(rspCode)); if (SCH_IS_QUERY_JOB(pJob)) { - SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, dropExecNode, pMsg->handle, execId)); + SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, dropExecNode, pMsg->handle, seriousId, execId)); } SCH_ERR_JRET(schValidateRspMsgType(pJob, pTask, msgType)); + if (pTask->seriousId < atomic_load_64(&pJob->seriousId)) { + SCH_TASK_DLOG("task sId %" PRId64 " is smaller than current job sId %" PRId64, pTask->seriousId, pJob->seriousId); + SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); + } + int32_t reqType = IsReq(pMsg) ? pMsg->msgType : (pMsg->msgType - 1); +#if 0 if (SCH_JOB_NEED_RETRY(pJob, pTask, reqType, rspCode)) { SCH_RET(schHandleJobRetry(pJob, pTask, (SDataBuf *)pMsg, rspCode)); } else if (SCH_TASKSET_NEED_RETRY(pJob, pTask, reqType, rspCode)) { SCH_RET(schHandleTaskSetRetry(pJob, pTask, (SDataBuf *)pMsg, rspCode)); } +#else + if (SCH_JOB_NEED_RETRY(pJob, pTask, reqType, rspCode)) { + SCH_RET(schHandleJobRetry(pJob, pTask, (SDataBuf *)pMsg, rspCode)); + } +#endif pTask->redirectCtx.inRedirect = false; - SCH_RET(schProcessResponseMsg(pJob, pTask, execId, pMsg, rspCode)); + SCH_RET(schProcessResponseMsg(pJob, pTask, pMsg, rspCode)); _return: @@ -482,7 +493,7 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { tstrerror(rspCode)); SCH_ERR_JRET(schProcessOnCbBegin(&pJob, &pTask, pParam->queryId, pParam->refId, pParam->taskId)); - code = schHandleResponseMsg(pJob, pTask, pParam->execId, pMsg, rspCode); + code = schHandleResponseMsg(pJob, pTask, pParam->seriousId, pParam->execId, pMsg, rspCode); pMsg->pData = NULL; schProcessOnCbEnd(pJob, pTask, code); @@ -500,10 +511,10 @@ _return: int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, - pParam->clientId, pParam->taskId, code); + qDebug("QID:0x%" PRIx64 ",SID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", + pParam->queryId, pParam->seriousId, pParam->clientId, pParam->taskId, code); // called if drop task rsp received code - (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error + (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT, 0); // ignore error if (pMsg->handle == NULL) { qError("sch handle is NULL, may be already released and mem lea"); @@ -517,8 +528,8 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, - pParam->clientId, pParam->taskId, code); + qDebug("QID:0x%" PRIx64 ",SID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", + pParam->queryId, pParam->seriousId, pParam->clientId, pParam->taskId, code); if (pMsg) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -529,7 +540,7 @@ int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) { int32_t schHandleLinkBrokenCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchCallbackParamHeader *head = (SSchCallbackParamHeader *)param; - (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error + (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT, 0); // ignore error qDebug("handle %p is broken", pMsg->handle); @@ -559,7 +570,7 @@ int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code) { if (code) { qError("hb rsp error:%s", tstrerror(code)); - (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error + (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT, 0); // ignore error SCH_ERR_JRET(code); } @@ -594,6 +605,7 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo } param->queryId = pJob->queryId; + param->seriousId = pTask->seriousId; param->refId = pJob->refId; param->clientId = SCH_CLIENT_ID(pTask); param->taskId = SCH_TASK_ID(pTask); @@ -651,7 +663,7 @@ int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, void *msg, uint3 SCH_ERR_JRET(terrno); } - msgSendInfo->paramFreeFp = taosMemoryFree; + msgSendInfo->paramFreeFp = taosAutoMemoryFree; SCH_ERR_JRET(schMakeCallbackParam(pJob, pTask, msgType, isHb, trans, &msgSendInfo->param)); SCH_ERR_JRET(schGetCallbackFp(msgType, &msgSendInfo->fp)); @@ -822,7 +834,7 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { param->pTrans = pJob->conn.pTrans; pMsgSendInfo->param = param; - pMsgSendInfo->paramFreeFp = taosMemoryFree; + pMsgSendInfo->paramFreeFp = taosAutoMemoryFree; pMsgSendInfo->fp = fp; SRpcCtxVal ctxVal = {.val = pMsgSendInfo, .clone = schCloneSMsgSendInfo}; @@ -942,7 +954,7 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) { pDst->param = NULL; SCH_ERR_JRET(schCloneCallbackParam(pSrc->param, (SSchCallbackParamHeader **)&pDst->param)); - pDst->paramFreeFp = taosMemoryFree; + pDst->paramFreeFp = taosAutoMemoryFree; *dst = pDst; @@ -1042,7 +1054,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { int32_t msgType = TDMT_SCH_QUERY_HEARTBEAT; req.header.vgId = nodeEpId->nodeId; - req.sId = schMgmt.sId; + req.clientId = schMgmt.clientId; TAOS_MEMCPY(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); SCH_LOCK(SCH_READ, &schMgmt.hbLock); @@ -1137,7 +1149,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, case TDMT_VND_DELETE: { SVDeleteReq req = {0}; req.header.vgId = addr->nodeId; - req.sId = schMgmt.sId; + req.sId = pTask->seriousId; req.queryId = pJob->queryId; req.clientId = pTask->clientId; req.taskId = pTask->taskId; @@ -1171,7 +1183,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SSubQueryMsg qMsg; qMsg.header.vgId = addr->nodeId; qMsg.header.contLen = 0; - qMsg.sId = schMgmt.sId; + qMsg.sId = pTask->seriousId; qMsg.queryId = pJob->queryId; qMsg.clientId = pTask->clientId; qMsg.taskId = pTask->taskId; @@ -1227,7 +1239,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, case TDMT_SCH_MERGE_FETCH: { SResFetchReq req = {0}; req.header.vgId = addr->nodeId; - req.sId = schMgmt.sId; + req.sId = pTask->seriousId; req.queryId = pJob->queryId; req.clientId = pTask->clientId; req.taskId = pTask->taskId; @@ -1255,7 +1267,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, STaskDropReq qMsg; qMsg.header.vgId = addr->nodeId; qMsg.header.contLen = 0; - qMsg.sId = schMgmt.sId; + qMsg.sId = pTask->seriousId; qMsg.queryId = pJob->queryId; qMsg.clientId = pTask->clientId; qMsg.taskId = pTask->taskId; @@ -1285,7 +1297,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_ERR_RET(schMakeHbRpcCtx(pJob, pTask, &rpcCtx)); SSchedulerHbReq req = {0}; - req.sId = schMgmt.sId; + req.clientId = schMgmt.clientId; req.header.vgId = addr->nodeId; req.epId.nodeId = addr->nodeId; TAOS_MEMCPY(&req.epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); @@ -1313,7 +1325,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, STaskNotifyReq qMsg; qMsg.header.vgId = addr->nodeId; qMsg.header.contLen = 0; - qMsg.sId = schMgmt.sId; + qMsg.sId = pTask->seriousId; qMsg.queryId = pJob->queryId; qMsg.clientId = pTask->clientId; qMsg.taskId = pTask->taskId; diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 37249b5418..cb8a68fe4f 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -63,8 +63,10 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * pTask->plan = pPlan; pTask->level = pLevel; + pTask->seriousId = pJob->seriousId; pTask->execId = -1; pTask->failedExecId = -2; + pTask->failedSeriousId = 0; pTask->timeoutUsec = SCH_DEFAULT_TASK_TIMEOUT_USEC; pTask->clientId = getClientId(); pTask->taskId = schGenTaskId(); @@ -161,16 +163,18 @@ int32_t schUpdateTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int3 return TSDB_CODE_SUCCESS; } -int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId) { +int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, uint64_t seriousId, int32_t execId) { if (dropExecNode) { SCH_RET(schDropTaskExecNode(pJob, pTask, handle, execId)); } SCH_ERR_RET(schUpdateTaskExecNode(pJob, pTask, handle, execId)); - if ((execId != pTask->execId || execId <= pTask->failedExecId) || pTask->waitRetry) { // ignore it - SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, - pTask->execId, pTask->waitRetry); + if ((seriousId != pTask->seriousId || seriousId <= pTask->failedSeriousId) || + (execId != pTask->execId || execId <= pTask->failedExecId) || pTask->waitRetry) { // ignore it + SCH_TASK_DLOG("handle not updated since seriousId:0x%" PRIx64 " or execId:%d is not lastest," + "current seriousId:0x%" PRIx64 " execId %d, failedSeriousId:0x%" PRIx64 " failedExecId:%d, waitRetry %d", + seriousId, execId, pTask->seriousId, pTask->execId, pTask->failedSeriousId, pTask->failedExecId, pTask->waitRetry); SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR); } @@ -185,6 +189,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) } pTask->failedExecId = pTask->execId; + pTask->failedSeriousId = pTask->seriousId; int8_t jobStatus = 0; if (schJobNeedToStop(pJob, &jobStatus)) { @@ -308,7 +313,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { .type = QUERY_NODE_DOWNSTREAM_SOURCE, .clientId = pTask->clientId, .taskId = pTask->taskId, - .schedId = schMgmt.sId, + .sId = pTask->seriousId, .execId = pTask->execId, .addr = pTask->succeedAddr, .fetchMsgType = SCH_FETCH_TYPE(pTask), @@ -326,7 +331,13 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { if (SCH_TASK_READY_FOR_LAUNCH(readyNum, parent)) { SCH_TASK_DLOG("all %d children task done, start to launch parent task 0x%" PRIx64, readyNum, parent->taskId); - SCH_ERR_RET(schLaunchTask(pJob, parent)); + + parent->seriousId = pJob->seriousId; + TSWAP(pTask, parent); + SCH_TASK_DLOG("task seriousId set to 0x%" PRIx64, pTask->seriousId); + TSWAP(pTask, parent); + + SCH_ERR_RET(schDelayLaunchTask(pJob, parent)); } } @@ -403,9 +414,11 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet, pCtx->periodMs = tsRedirectMaxPeriod; } - int64_t leftTime = tsMaxRetryWaitTime - lastTime; - pTask->delayExecMs = leftTime < pCtx->periodMs ? leftTime : pCtx->periodMs; - + if (SCH_IS_DATA_BIND_TASK(pTask)) { + int64_t leftTime = tsMaxRetryWaitTime - lastTime; + pTask->delayExecMs = leftTime < pCtx->periodMs ? leftTime : pCtx->periodMs; + } + pCtx->roundTimes = 0; goto _return; @@ -424,12 +437,11 @@ _return: void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask) { pTask->waitRetry = true; - schDropTaskOnExecNode(pJob, pTask); if (pTask->delayTimer) { - if (!taosTmrStopA(&pTask->delayTimer)) { - SCH_TASK_WLOG("stop task delayTimer failed, may stopped, status:%d", pTask->status); - } + taosTmrStop(pTask->delayTimer); } + + schDropTaskOnExecNode(pJob, pTask); taosHashClear(pTask->execNodes); (void)schRemoveTaskFromExecList(pJob, pTask); // ignore error schDeregisterTaskHb(pJob, pTask); @@ -565,6 +577,7 @@ int32_t schHandleTaskSetRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, i SCH_ERR_JRET(schResetTaskSetLevelInfo(pJob, pTask)); SCH_RESET_JOB_LEVEL_IDX(pJob); + atomic_add_fetch_64(&pJob->seriousId, 1); code = schDoTaskRedirect(pJob, pTask, pData, rspCode); @@ -585,11 +598,11 @@ int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) { int32_t code = taosHashPut(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES); if (0 != code) { if (HASH_NODE_EXIST(code)) { - SCH_TASK_ELOG("task already in execTask list, code:%x", code); - SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + SCH_TASK_DLOG("task already in execTask list, code:%x", code); + return TSDB_CODE_SUCCESS; } - SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); + SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:0x%x", errno); SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -745,6 +758,10 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { (void)atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); + if (pTask->delayTimer) { + taosTmrStop(pTask->delayTimer); + } + (void)schRemoveTaskFromExecList(pJob, pTask); // ignore error SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); @@ -767,7 +784,7 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(schSwitchTaskCandidateAddr(pJob, pTask)); } - SCH_ERR_RET(schLaunchTask(pJob, pTask)); + SCH_ERR_RET(schDelayLaunchTask(pJob, pTask)); return TSDB_CODE_SUCCESS; } @@ -1103,7 +1120,10 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { int32_t code = 0; if (NULL == pTask->msg) { // TODO add more detailed reason for failure + SCH_LOCK(SCH_WRITE, &pTask->planLock); code = qSubPlanToMsg(plan, &pTask->msg, &pTask->msgLen); + SCH_UNLOCK(SCH_WRITE, &pTask->planLock); + if (TSDB_CODE_SUCCESS != code) { SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg, pTask->msgLen); @@ -1154,7 +1174,7 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { } } - SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->clientId, pTask->taskId, + SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, pJob->seriousId, pJob->queryId, pTask->clientId, pTask->taskId, pJob->refId, pTask->execId, &qwMsg, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { @@ -1188,14 +1208,21 @@ int32_t schLaunchTaskImpl(void *param) { SCH_LOCK_TASK(pTask); } - int8_t status = 0; - int32_t code = 0; - - (void)atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); pTask->execId++; pTask->retryTimes++; pTask->waitRetry = false; + int8_t status = 0; + int32_t code = 0; + + if (atomic_load_64(&pTask->seriousId) < atomic_load_64(&pJob->seriousId)) { + SCH_TASK_DLOG("task seriousId:0x%" PRIx64 " is smaller than job seriousId:0x%" PRIx64 ", skip launch", + pTask->seriousId, pJob->seriousId); + goto _return; + } + + (void)atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); + SCH_TASK_DLOG("start to launch %s task, execId %d, retry %d", SCH_IS_LOCAL_EXEC_TASK(pJob, pTask) ? "LOCAL" : "REMOTE", pTask->execId, pTask->retryTimes); @@ -1294,43 +1321,47 @@ void schHandleTimerEvent(void *param, void *tmrId) { SSchJob *pJob = NULL; int32_t code = 0; + qDebug("delayTimer %" PRIuPTR " is launched", (uintptr_t)tmrId); + int64_t rId = pTimerParam->rId; uint64_t queryId = pTimerParam->queryId; uint64_t taskId = pTimerParam->taskId; - taosMemoryFree(pTimerParam); if (schProcessOnCbBegin(&pJob, &pTask, queryId, rId, taskId)) { return; } - code = schLaunchTask(pJob, pTask); + if (0 == atomic_load_8(&pTask->delayLaunchPar.exit)) { + code = schLaunchTask(pJob, pTask); + } else { + SCH_TASK_DLOG("task will not be launched since query job exiting, status: %d", pTask->status); + } schProcessOnCbEnd(pJob, pTask, code); } int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { if (pTask->delayExecMs > 0) { - SSchTimerParam *param = taosMemoryMalloc(sizeof(SSchTimerParam)); - if (NULL == param) { - SCH_TASK_ELOG("taosMemoryMalloc %d failed", (int)sizeof(SSchTimerParam)); - SCH_ERR_RET(terrno); - } + pTask->delayLaunchPar.rId = pJob->refId; + pTask->delayLaunchPar.queryId = pJob->queryId; + pTask->delayLaunchPar.taskId = pTask->taskId; - param->rId = pJob->refId; - param->queryId = pJob->queryId; - param->taskId = pTask->taskId; + SCH_ERR_RET(schPushTaskToExecList(pJob, pTask)); + SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXEC); if (NULL == pTask->delayTimer) { - pTask->delayTimer = taosTmrStart(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer); + pTask->delayTimer = taosTmrStart(schHandleTimerEvent, pTask->delayExecMs, (void *)&pTask->delayLaunchPar, schMgmt.timer); if (NULL == pTask->delayTimer) { SCH_TASK_ELOG("start delay timer failed, handle:%p", schMgmt.timer); SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + SCH_TASK_DLOG("task delayTimer %" PRIuPTR " is started", (uintptr_t)pTask->delayTimer); + return TSDB_CODE_SUCCESS; } - if (taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer, &pTask->delayTimer)) { + if (taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)&pTask->delayLaunchPar, schMgmt.timer, &pTask->delayTimer)) { SCH_TASK_ELOG("taosTmrReset delayExec timer failed, handle:%p", schMgmt.timer); } @@ -1345,8 +1376,11 @@ int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level) { for (int32_t i = 0; i < level->taskNum; ++i) { SSchTask *pTask = taosArrayGet(level->subTasks, i); + pTask->seriousId = pJob->seriousId; + + SCH_TASK_DLOG("task seriousId set to 0x%" PRIx64, pTask->seriousId); - SCH_ERR_RET(schLaunchTask(pJob, pTask)); + SCH_ERR_RET(schDelayLaunchTask(pJob, pTask)); } return TSDB_CODE_SUCCESS; @@ -1361,12 +1395,11 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { while (pIter) { SSchTask *pTask = *(SSchTask **)pIter; - SCH_LOCK_TASK(pTask); if (pTask->delayTimer) { - if (!taosTmrStopA(&pTask->delayTimer)) { - SCH_TASK_WLOG("stop delayTimer failed, status:%d", pTask->status); - } + schStopTaskDelayTimer(pJob, pTask, true); } + + SCH_LOCK_TASK(pTask); schDropTaskOnExecNode(pJob, pTask); SCH_UNLOCK_TASK(pTask); @@ -1414,7 +1447,7 @@ int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { } } - SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->clientId, pTask->taskId, + SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, pJob->seriousId, pJob->queryId, pTask->clientId, pTask->taskId, pJob->refId, pTask->execId, &pRsp, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index ac34099417..1eb7dd5281 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -22,7 +22,7 @@ #include "tref.h" #include "trpc.h" -FORCE_INLINE int32_t schAcquireJob(int64_t refId, SSchJob** ppJob) { +FORCE_INLINE int32_t schAcquireJob(int64_t refId, SSchJob **ppJob) { qDebug("sch acquire jobId:0x%" PRIx64, refId); *ppJob = (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); if (NULL == *ppJob) { @@ -41,7 +41,7 @@ FORCE_INLINE int32_t schReleaseJob(int64_t refId) { return taosReleaseRef(schMgmt.jobRef, refId); } -FORCE_INLINE int32_t schReleaseJobEx(int64_t refId, int32_t* released) { +FORCE_INLINE int32_t schReleaseJobEx(int64_t refId, int32_t *released) { if (0 == refId) { return TSDB_CODE_SUCCESS; } @@ -50,7 +50,7 @@ FORCE_INLINE int32_t schReleaseJobEx(int64_t refId, int32_t* released) { return taosReleaseRefEx(schMgmt.jobRef, refId, released); } -int32_t schDumpEpSet(SEpSet *pEpSet, char** ppRes) { +int32_t schDumpEpSet(SEpSet *pEpSet, char **ppRes) { *ppRes = NULL; if (NULL == pEpSet) { return TSDB_CODE_SUCCESS; @@ -89,7 +89,7 @@ char *schGetOpStr(SCH_OP_TYPE type) { } void schFreeHbTrans(SSchHbTrans *pTrans) { - (void)rpcReleaseHandle((void *)pTrans->trans.pHandleId, TAOS_CONN_CLIENT); + (void)rpcReleaseHandle((void *)pTrans->trans.pHandleId, TAOS_CONN_CLIENT, 0); schFreeRpcCtx(&pTrans->rpcCtx); } @@ -202,11 +202,12 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); if (NULL == addr) { - SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, + (int32_t)taosArrayGetSize(pTask->candidateAddrs)); return; } - SQueryNodeEpId epId = {0}; + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; @@ -240,11 +241,12 @@ int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); if (NULL == addr) { - SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, + (int32_t)taosArrayGetSize(pTask->candidateAddrs)); return TSDB_CODE_SCH_INTERNAL_ERROR; } - SQueryNodeEpId epId = {0}; + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; @@ -276,8 +278,8 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { SCH_UNLOCK(SCH_WRITE, &hb->lock); SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); - qDebug("hb connection updated, sId:0x%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, pTrans:%p, pHandle:%p", schMgmt.sId, - epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->pTrans, trans->pHandle); + qDebug("hb connection updated, nodeId:%d, fqdn:%s, port:%d, pTrans:%p, pHandle:%p", epId->nodeId, epId->ep.fqdn, + epId->ep.port, trans->pTrans, trans->pHandle); return TSDB_CODE_SUCCESS; } @@ -323,7 +325,8 @@ uint64_t schGenUUID(void) { uint64_t pid = taosGetPId(); int32_t val = atomic_add_fetch_32(&requestSerialId, 1); - uint64_t id = ((uint64_t)((hashId & 0x0FFF)) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); + uint64_t id = + ((uint64_t)((hashId & 0x0FFF)) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); return id; } #endif @@ -373,17 +376,17 @@ void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) *pTask = *task; } -int32_t schValidateSubplan(SSchJob *pJob, SSubplan* pSubplan, int32_t level, int32_t idx, int32_t taskNum) { +int32_t schValidateSubplan(SSchJob *pJob, SSubplan *pSubplan, int32_t level, int32_t idx, int32_t taskNum) { if (NULL == pSubplan) { SCH_JOB_ELOG("fail to get the %dth subplan, taskNum: %d, level: %d", idx, taskNum, level); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - + if (QUERY_NODE_PHYSICAL_SUBPLAN != nodeType(pSubplan)) { SCH_JOB_ELOG("invalid subplan type, level:%d, subplanNodeType:%d", level, nodeType(pSubplan)); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - + if (pSubplan->subplanType < SUBPLAN_TYPE_MERGE || pSubplan->subplanType > SUBPLAN_TYPE_COMPUTE) { SCH_JOB_ELOG("invalid subplanType %d, level:%d, subplan idx:%d", pSubplan->subplanType, level, idx); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -396,15 +399,17 @@ int32_t schValidateSubplan(SSchJob *pJob, SSubplan* pSubplan, int32_t level, int if (SCH_IS_DATA_BIND_PLAN(pSubplan)) { if (pSubplan->execNode.epSet.numOfEps <= 0) { - SCH_JOB_ELOG("no execNode specifed for data src plan %d, numOfEps:%d", pSubplan->subplanType, pSubplan->execNode.epSet.numOfEps); + SCH_JOB_ELOG("no execNode specifed for data src plan %d, numOfEps:%d", pSubplan->subplanType, + pSubplan->execNode.epSet.numOfEps); SCH_ERR_RET(TSDB_CODE_SCH_DATA_SRC_EP_MISS); } if (pSubplan->execNode.epSet.inUse >= pSubplan->execNode.epSet.numOfEps) { - SCH_JOB_ELOG("invalid epset inUse %d for data src plan %d, numOfEps:%d", pSubplan->execNode.epSet.inUse, pSubplan->subplanType, pSubplan->execNode.epSet.numOfEps); + SCH_JOB_ELOG("invalid epset inUse %d for data src plan %d, numOfEps:%d", pSubplan->execNode.epSet.inUse, + pSubplan->subplanType, pSubplan->execNode.epSet.numOfEps); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } } - + if (NULL == pSubplan->pNode && pSubplan->subplanType != SUBPLAN_TYPE_MODIFY) { SCH_JOB_ELOG("empty plan root node, level:%d, subplan idx:%d, subplanType:%d", level, idx, pSubplan->subplanType); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -418,4 +423,23 @@ int32_t schValidateSubplan(SSchJob *pJob, SSubplan* pSubplan, int32_t level, int return TSDB_CODE_SUCCESS; } +void schStopTaskDelayTimer(SSchJob *pJob, SSchTask *pTask, bool syncOp) { + SCH_TASK_DLOG("try to stop task delayTimer %" PRIuPTR, (uintptr_t)pTask->delayTimer); + tmr_h delayTimer = pTask->delayTimer; + atomic_store_8(&pTask->delayLaunchPar.exit, 1); + + if (!taosTmrStopA(&pTask->delayTimer)) { + SCH_TASK_DLOG("task delayTimer %" PRIuPTR " not stopped", (uintptr_t)delayTimer); + + if (syncOp) { + while (!taosTmrIsStopped(&delayTimer)) { + taosMsleep(1); + } + + SCH_TASK_DLOG("task delayTimer %" PRIuPTR " is stopped", (uintptr_t)delayTimer); + } else { + SCH_TASK_WLOG("stop task delayTimer %" PRIuPTR " failed, may stopped, status:%d", (uintptr_t)delayTimer, pTask->status); + } + } +} diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index db9ecd6025..21659d9cd5 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -56,12 +56,7 @@ int32_t schedulerInit() { SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - if (taosGetSystemUUIDU64(&schMgmt.sId)) { - qError("generate schedulerId failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_SYS_ERROR); - } - - qInfo("scheduler 0x%" PRIx64 " initialized, maxJob:%u", schMgmt.sId, schMgmt.cfg.maxJobNum); + qInfo("scheduler 0x%" PRIx64 " initialized, maxJob:%u", getClientId(), schMgmt.cfg.maxJobNum); return TSDB_CODE_SUCCESS; } @@ -172,7 +167,7 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) { SSchJob *pJob = NULL; (void)schAcquireJob(*jobId, &pJob); if (NULL == pJob) { - qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); + qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); return; } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 6e13e37e88..a9878ec9a9 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -54,7 +54,7 @@ namespace { -extern "C" int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, +extern "C" int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, uint64_t sId, int32_t execId, SDataBuf *pMsg, int32_t rspCode); extern "C" int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t rspCode); @@ -591,7 +591,7 @@ void *schtSendRsp(void *param) { msg.msgType = TDMT_VND_SUBMIT_RSP; msg.pData = rmsg; - (void)schHandleResponseMsg(pJob, task, task->execId, &msg, 0); + (void)schHandleResponseMsg(pJob, task, task->seriousId, task->execId, &msg, 0); pIter = taosHashIterate(pJob->execTasks, pIter); } @@ -621,7 +621,7 @@ void *schtCreateFetchRspThread(void *param) { msg.msgType = TDMT_SCH_MERGE_FETCH_RSP; msg.pData = rmsg; - code = schHandleResponseMsg(pJob, pJob->fetchTask, pJob->fetchTask->execId, &msg, 0); + code = schHandleResponseMsg(pJob, pJob->fetchTask, pJob->fetchTask->seriousId, pJob->fetchTask->execId, &msg, 0); (void)schReleaseJob(job); @@ -925,7 +925,7 @@ TEST(queryTest, normalCase) { msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; - code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0); + code = schHandleResponseMsg(pJob, task, task->seriousId, task->execId, &msg, 0); ASSERT_EQ(code, 0); pIter = taosHashIterate(pJob->execTasks, pIter); @@ -941,7 +941,7 @@ TEST(queryTest, normalCase) { msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; - code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0); + code = schHandleResponseMsg(pJob, task, task->seriousId, task->execId, &msg, 0); ASSERT_EQ(code, 0); } @@ -1040,7 +1040,7 @@ TEST(queryTest, readyFirstCase) { msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; - code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0); + code = schHandleResponseMsg(pJob, task, task->seriousId, task->execId, &msg, 0); ASSERT_EQ(code, 0); pIter = taosHashIterate(pJob->execTasks, pIter); @@ -1057,7 +1057,7 @@ TEST(queryTest, readyFirstCase) { msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; - code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0); + code = schHandleResponseMsg(pJob, task, task->seriousId, task->execId, &msg, 0); ASSERT_EQ(code, 0); } @@ -1163,7 +1163,7 @@ TEST(queryTest, flowCtrlCase) { msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; - code = schHandleResponseMsg(pJob, task, task->execId, &msg, 0); + code = schHandleResponseMsg(pJob, task, task->seriousId, task->execId, &msg, 0); ASSERT_EQ(code, 0); } diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 6a10b21c53..2b5312c0a0 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -166,10 +166,12 @@ int32_t streamStateDel_rocksdb(SStreamState* pState, const SWinKey* key); int32_t streamStateClear_rocksdb(SStreamState* pState); void streamStateCurNext_rocksdb(SStreamStateCur* pCur); int32_t streamStateGetFirst_rocksdb(SStreamState* pState, SWinKey* key); -int32_t streamStateGetGroupKVByCur_rocksdb(SStreamState* pState, SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); +int32_t streamStateGetGroupKVByCur_rocksdb(SStreamState* pState, SStreamStateCur* pCur, SWinKey* pKey, + const void** pVal, int32_t* pVLen); int32_t streamStateAddIfNotExist_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); void streamStateCurPrev_rocksdb(SStreamStateCur* pCur); -int32_t streamStateGetKVByCur_rocksdb(SStreamState* pState, SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); +int32_t streamStateGetKVByCur_rocksdb(SStreamState* pState, SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, + int32_t* pVLen); SStreamStateCur* streamStateGetAndCheckCur_rocksdb(SStreamState* pState, SWinKey* key); SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey* key); @@ -217,16 +219,15 @@ int32_t streamStateFillGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* p // partag cf int32_t streamStatePutParTag_rocksdb(SStreamState* pState, int64_t groupId, const void* tag, int32_t tagLen); int32_t streamStateGetParTag_rocksdb(SStreamState* pState, int64_t groupId, void** tagVal, int32_t* tagLen); -void streamStateParTagSeekKeyNext_rocksdb(SStreamState* pState, const int64_t groupId, SStreamStateCur* pCur); -int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGroupId, const void** pVal, int32_t* pVLen); +void streamStateParTagSeekKeyNext_rocksdb(SStreamState* pState, const int64_t groupId, SStreamStateCur* pCur); +int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGroupId, const void** pVal, + int32_t* pVLen); // parname cf int32_t streamStatePutParName_rocksdb(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]); int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, void** pVal); int32_t streamStateDeleteParName_rocksdb(SStreamState* pState, int64_t groupId); -void streamStateDestroy_rocksdb(SStreamState* pState, bool remove); - // default cf int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen); int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** pVal, int32_t* pVLen); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 381b852e4e..37fb081a87 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -153,7 +153,6 @@ void taskDbUnRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp); int32_t chkpAddExtraInfo(char* pChkpIdDir, int64_t chkpId, int64_t processId); int32_t chkpLoadExtraInfo(char* pChkpIdDir, int64_t* chkpId, int64_t* processId); -#define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX)); int32_t copyFiles(const char* src, const char* dst); uint32_t nextPow2(uint32_t x); @@ -1156,13 +1155,17 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { taosArrayDestroy(pBackend->chkpSaved); pBackend->chkpSaved = chkpDup; + chkpDup = NULL; TAOS_UNUSED(taosThreadRwlockUnlock(&pBackend->chkpDirLock)); for (int i = 0; i < taosArrayGetSize(chkpDel); i++) { int64_t id = *(int64_t*)taosArrayGet(chkpDel, i); char tbuf[256] = {0}; - sprintf(tbuf, "%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, id); + if (snprintf(tbuf, sizeof(tbuf), "%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, id) >= sizeof(tbuf)) { + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_CHECK_GOTO(code, NULL, _exception); + } stInfo("backend remove obsolete checkpoint: %s", tbuf); if (taosIsDir(tbuf)) { @@ -1187,12 +1190,17 @@ int chkpIdComp(const void* a, const void* b) { } int32_t taskDbLoadChkpInfo(STaskDbWrapper* pBackend) { int32_t code = 0; - char* pChkpDir = taosMemoryCalloc(1, 256); + int32_t nBytes = 0; + int32_t cap = 256; + char* pChkpDir = taosMemoryCalloc(1, cap); if (pChkpDir == NULL) { return terrno; } - sprintf(pChkpDir, "%s%s%s", pBackend->path, TD_DIRSEP, "checkpoints"); + nBytes = snprintf(pChkpDir, cap, "%s%s%s", pBackend->path, TD_DIRSEP, "checkpoints"); + if (nBytes >= cap) { + return TSDB_CODE_OUT_OF_RANGE; + } if (!taosIsDir(pChkpDir)) { taosMemoryFree(pChkpDir); return 0; @@ -1413,12 +1421,18 @@ int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) { if (pSnapInfo == NULL) return 0; SStreamMeta* pMeta = arg; int32_t code = 0; + int32_t cap = 256; + int32_t nBytes = 0; streamMutexLock(&pMeta->backendMutex); - char buf[128] = {0}; + char buf[256] = {0}; for (int i = 0; i < taosArrayGetSize(pSnapInfo); i++) { SStreamTaskSnap* pSnap = taosArrayGet(pSnapInfo, i); - sprintf(buf, "0x%" PRIx64 "-0x%x", pSnap->streamId, (int32_t)pSnap->taskId); + nBytes = snprintf(buf, cap, "0x%" PRIx64 "-0x%x", pSnap->streamId, (int32_t)pSnap->taskId); + if (nBytes <= 0 || nBytes >= cap) { + code = TSDB_CODE_OUT_OF_RANGE; + break; + } STaskDbWrapper** pTaskDb = taosHashGet(pMeta->pTaskDbUnique, buf, strlen(buf)); if (pTaskDb == NULL || *pTaskDb == NULL) { stWarn("stream backend:%p failed to find task db, streamId:% " PRId64 "", pMeta, pSnap->streamId); @@ -1430,7 +1444,7 @@ int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) { taskDbUnRefChkp(*pTaskDb, pSnap->chkpId); } streamMutexUnlock(&pMeta->backendMutex); - return 0; + return code; } #ifdef BUILD_NO_CALL int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId) { @@ -1685,9 +1699,6 @@ void streamBackendDelCompare(void* backend, void* arg) { taosMemoryFree(node); } } -#ifdef BUILD_NO_CALL -void streamStateDestroy_rocksdb(SStreamState* pState, bool remove) { streamStateCloseBackend(pState, remove); } -#endif void destroyRocksdbCfInst(RocksdbCfInst* inst) { int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); if (inst->pHandle) { @@ -1712,7 +1723,7 @@ void destroyRocksdbCfInst(RocksdbCfInst* inst) { } // |key|-----value------| -// |key|ttl|len|userData| +// |key|ttl|len|userData int defaultKeyComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen) { int len = aLen < bLen ? aLen : bLen; @@ -2447,43 +2458,50 @@ void taskDbDestroyChkpOpt(STaskDbWrapper* pTaskDb) { int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** stateFullPath) { int32_t code = 0; - char* statePath = taosMemoryCalloc(1, strlen(path) + 128); + int32_t cap = strlen(path) + 128, nBytes = 0; + char* statePath = NULL; + char* dbPath = NULL; + + statePath = taosMemoryCalloc(1, cap); if (statePath == NULL) { - return terrno; + TAOS_CHECK_GOTO(terrno, NULL, _err); + } + + nBytes = snprintf(statePath, cap, "%s%s%s", path, TD_DIRSEP, key); + if (nBytes < 0 || nBytes >= cap) { + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_CHECK_GOTO(code, NULL, _err); } - sprintf(statePath, "%s%s%s", path, TD_DIRSEP, key); if (!taosDirExist(statePath)) { code = taosMulMkDir(statePath); - if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); - stError("failed to create dir: %s, reason:%s", statePath, tstrerror(code)); - taosMemoryFree(statePath); - return code; - } + TAOS_CHECK_GOTO(code, NULL, _err); } - char* dbPath = taosMemoryCalloc(1, strlen(statePath) + 128); + dbPath = taosMemoryCalloc(1, cap); if (dbPath == NULL) { - taosMemoryFree(statePath); - return terrno; + TAOS_CHECK_GOTO(terrno, NULL, _err); + } + nBytes = snprintf(dbPath, cap, "%s%s%s", statePath, TD_DIRSEP, "state"); + if (nBytes < 0 || nBytes >= cap) { + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_CHECK_GOTO(code, NULL, _err); } - sprintf(dbPath, "%s%s%s", statePath, TD_DIRSEP, "state"); if (!taosDirExist(dbPath)) { code = taosMulMkDir(dbPath); - if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); - stError("failed to create dir: %s, reason:%s", dbPath, tstrerror(code)); - taosMemoryFree(statePath); - taosMemoryFree(dbPath); - return code; - } + TAOS_CHECK_GOTO(code, NULL, _err); } *dbFullPath = dbPath; *stateFullPath = statePath; return 0; +_err: + stError("failed to create dir: %s, reason:%s", dbPath, tstrerror(code)); + + taosMemoryFree(statePath); + taosMemoryFree(dbPath); + return code; } void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) { @@ -2864,6 +2882,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t int64_t streamId; int32_t taskId, dummy = 0; char suffix[64] = {0}; + int32_t code = 0; rocksdb_options_t** cfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); RocksdbCfParam* params = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam)); @@ -2873,6 +2892,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t for (int i = 0; i < nCf; i++) { char* cf = cfs[i]; char funcname[64] = {0}; + cfOpts[i] = rocksdb_options_create_copy(handle->dbOpt); if (i == 0) continue; if (3 == sscanf(cf, "0x%" PRIx64 "-%d_%s", &streamId, &taskId, funcname)) { @@ -2909,7 +2929,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t taosMemoryFree(params); taosMemoryFree(cfOpts); // fix other leak - return -1; + return TSDB_CODE_THIRDPARTY_ERROR; } else { stDebug("succ to open rocksdb cf"); } @@ -2929,8 +2949,13 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t char funcname[64] = {0}; if (3 == sscanf(cf, "0x%" PRIx64 "-%d_%s", &streamId, &taskId, funcname)) { - char idstr[128] = {0}; - sprintf(idstr, "0x%" PRIx64 "-%d", streamId, taskId); + char idstr[128] = {0}; + int32_t nBytes = snprintf(idstr, sizeof(idstr), "0x%" PRIx64 "-%d", streamId, taskId); + if (nBytes <= 0 || nBytes >= sizeof(idstr)) { + code = TSDB_CODE_OUT_OF_RANGE; + stError("failed to open cf since %s", tstrerror(code)); + return code; + } int idx = streamStateGetCfIdx(NULL, funcname); @@ -2997,117 +3022,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t taosMemoryFree(cfOpts); return 0; } -#ifdef BUILD_NO_CALL -int streamStateOpenBackend(void* backend, SStreamState* pState) { - taosAcquireRef(streamBackendId, pState->streamBackendRid); - SBackendWrapper* handle = backend; - SBackendCfWrapper* pBackendCfWrapper = taosMemoryCalloc(1, sizeof(SBackendCfWrapper)); - streamMutexLock(&handle->cfMutex); - RocksdbCfInst** ppInst = taosHashGet(handle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); - if (ppInst != NULL && *ppInst != NULL) { - RocksdbCfInst* inst = *ppInst; - pBackendCfWrapper->rocksdb = inst->db; - pBackendCfWrapper->pHandle = (void**)inst->pHandle; - pBackendCfWrapper->writeOpts = inst->wOpt; - pBackendCfWrapper->readOpts = inst->rOpt; - pBackendCfWrapper->cfOpts = (void**)(inst->cfOpt); - pBackendCfWrapper->dbOpt = handle->dbOpt; - pBackendCfWrapper->param = inst->param; - pBackendCfWrapper->pBackend = handle; - pBackendCfWrapper->pComparNode = inst->pCompareNode; - streamMutexUnlock(&handle->cfMutex); - pBackendCfWrapper->backendId = pState->streamBackendRid; - memcpy(pBackendCfWrapper->idstr, pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr)); - - int64_t id = taosAddRef(streamBackendCfWrapperId, pBackendCfWrapper); - pState->pTdbState->backendCfWrapperId = id; - pState->pTdbState->pBackendCfWrapper = pBackendCfWrapper; - stInfo("succ to open state %p on backendWrapper, %p, %s", pState, pBackendCfWrapper, pBackendCfWrapper->idstr); - - inst->pHandle = NULL; - inst->cfOpt = NULL; - inst->param = NULL; - - inst->wOpt = NULL; - inst->rOpt = NULL; - return 0; - } - streamMutexUnlock(&handle->cfMutex); - - char* err = NULL; - int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); - - RocksdbCfParam* param = taosMemoryCalloc(cfLen, sizeof(RocksdbCfParam)); - const rocksdb_options_t** cfOpt = taosMemoryCalloc(cfLen, sizeof(rocksdb_options_t*)); - for (int i = 0; i < cfLen; i++) { - cfOpt[i] = rocksdb_options_create_copy(handle->dbOpt); - // refactor later - rocksdb_block_based_table_options_t* tableOpt = rocksdb_block_based_options_create(); - rocksdb_block_based_options_set_block_cache(tableOpt, handle->cache); - rocksdb_block_based_options_set_partition_filters(tableOpt, 1); - - rocksdb_filterpolicy_t* filter = rocksdb_filterpolicy_create_bloom(15); - rocksdb_block_based_options_set_filter_policy(tableOpt, filter); - - rocksdb_options_set_block_based_table_factory((rocksdb_options_t*)cfOpt[i], tableOpt); - - param[i].tableOpt = tableOpt; - }; - - rocksdb_comparator_t** pCompare = taosMemoryCalloc(cfLen, sizeof(rocksdb_comparator_t*)); - for (int i = 0; i < cfLen; i++) { - SCfInit* cf = &ginitDict[i]; - - rocksdb_comparator_t* compare = rocksdb_comparator_create(NULL, cf->destroyCmp, cf->cmpKey, cf->cmpName); - rocksdb_options_set_comparator((rocksdb_options_t*)cfOpt[i], compare); - pCompare[i] = compare; - } - rocksdb_column_family_handle_t** cfHandle = taosMemoryCalloc(cfLen, sizeof(rocksdb_column_family_handle_t*)); - pBackendCfWrapper->rocksdb = handle->db; - pBackendCfWrapper->pHandle = (void**)cfHandle; - pBackendCfWrapper->writeOpts = rocksdb_writeoptions_create(); - pBackendCfWrapper->readOpts = rocksdb_readoptions_create(); - pBackendCfWrapper->cfOpts = (void**)cfOpt; - pBackendCfWrapper->dbOpt = handle->dbOpt; - pBackendCfWrapper->param = param; - pBackendCfWrapper->pBackend = handle; - pBackendCfWrapper->backendId = pState->streamBackendRid; - taosThreadRwlockInit(&pBackendCfWrapper->rwLock, NULL); - SCfComparator compare = {.comp = pCompare, .numOfComp = cfLen}; - pBackendCfWrapper->pComparNode = streamBackendAddCompare(handle, &compare); - rocksdb_writeoptions_disable_WAL(pBackendCfWrapper->writeOpts, 1); - memcpy(pBackendCfWrapper->idstr, pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr)); - - int64_t id = taosAddRef(streamBackendCfWrapperId, pBackendCfWrapper); - pState->pTdbState->backendCfWrapperId = id; - pState->pTdbState->pBackendCfWrapper = pBackendCfWrapper; - stInfo("succ to open state %p on backendWrapper %p %s", pState, pBackendCfWrapper, pBackendCfWrapper->idstr); - return 0; -} - -void streamStateCloseBackend(SStreamState* pState, bool remove) { - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SBackendWrapper* pHandle = wrapper->pBackend; - - stInfo("start to close state on backend: %p", pHandle); - - streamMutexLock(&pHandle->cfMutex); - RocksdbCfInst** ppInst = taosHashGet(pHandle->cfInst, wrapper->idstr, strlen(pState->pTdbState->idstr) + 1); - if (ppInst != NULL && *ppInst != NULL) { - RocksdbCfInst* inst = *ppInst; - taosMemoryFree(inst); - taosHashRemove(pHandle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); - } - streamMutexUnlock(&pHandle->cfMutex); - - char* status[] = {"close", "drop"}; - stInfo("start to %s state %p on backendWrapper %p %s", status[remove == false ? 0 : 1], pState, wrapper, - wrapper->idstr); - wrapper->remove |= remove; // update by other pState - taosReleaseRef(streamBackendCfWrapperId, pState->pTdbState->backendCfWrapperId); -} -#endif void streamStateDestroyCompar(void* arg) { SCfComparator* comp = (SCfComparator*)arg; for (int i = 0; i < comp->numOfComp; i++) { @@ -3386,7 +3301,8 @@ int32_t streamStateGetFirst_rocksdb(SStreamState* pState, SWinKey* key) { return streamStateDel_rocksdb(pState, &tmp); } -int32_t streamStateFillGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { +int32_t streamStateFillGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, + int32_t* pVLen) { if (!pCur) { return -1; } @@ -4343,7 +4259,7 @@ int32_t streamStatePutParTag_rocksdb(SStreamState* pState, int64_t groupId, cons void streamStateParTagSeekKeyNext_rocksdb(SStreamState* pState, const int64_t groupId, SStreamStateCur* pCur) { if (pCur == NULL) { - return ; + return; } STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; pCur->number = pState->number; @@ -4353,13 +4269,13 @@ void streamStateParTagSeekKeyNext_rocksdb(SStreamState* pState, const int64_t gr int i = streamStateGetCfIdx(pState, "partag"); if (i < 0) { stError("streamState failed to put to cf name:%s", "partag"); - return ; + return; } char buf[128] = {0}; int32_t klen = ginitDict[i].enFunc((void*)&groupId, buf); if (!streamStateIterSeekAndValid(pCur->iter, buf, klen)) { - return ; + return; } // skip ttl expired data while (rocksdb_iter_valid(pCur->iter) && iterValueIsStale(pCur->iter)) { @@ -4371,13 +4287,15 @@ void streamStateParTagSeekKeyNext_rocksdb(SStreamState* pState, const int64_t gr size_t kLen = 0; char* keyStr = (char*)rocksdb_iter_key(pCur->iter, &kLen); TAOS_UNUSED(parKeyDecode((void*)&curGroupId, keyStr)); - if (curGroupId > groupId) return ; + if (curGroupId > groupId) return; rocksdb_iter_next(pCur->iter); } } -int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGroupId, const void** pVal, int32_t* pVLen) { +int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGroupId, const void** pVal, + int32_t* pVLen) { + stDebug("streamStateFillGetKVByCur_rocksdb"); if (!pCur) { return -1; } @@ -4715,11 +4633,12 @@ int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) { while (pIter) { char* name = taosHashGetKey(pIter, &len); if (!isBkdDataMeta(name, len) && !taosHashGet(p1, name, len)) { - char* fname = taosMemoryCalloc(1, len + 1); + int32_t cap = len + 1; + char* fname = taosMemoryCalloc(1, cap); if (fname == NULL) { return terrno; } - TAOS_UNUSED(strncpy(fname, name, len)); + tstrncpy(fname, name, cap); if (taosArrayPush(diff, &fname) == NULL) { taosMemoryFree(fname); return terrno; @@ -4743,17 +4662,32 @@ int32_t compareHashTable(SHashObj* p1, SHashObj* p2, SArray* add, SArray* del) { void hashTableToDebug(SHashObj* pTbl, char** buf) { size_t sz = taosHashGetSize(pTbl); int32_t total = 0; - char* p = taosMemoryCalloc(1, sz * 16 + 4); - void* pIter = taosHashIterate(pTbl, NULL); + int32_t cap = sz * 16 + 4; + + char* p = taosMemoryCalloc(1, cap); + if (p == NULL) { + stError("failed to alloc memory for stream snapshot debug info"); + return; + } + + void* pIter = taosHashIterate(pTbl, NULL); while (pIter) { size_t len = 0; char* name = taosHashGetKey(pIter, &len); - char* tname = taosMemoryCalloc(1, len + 1); - memcpy(tname, name, len); - total += sprintf(p + total, "%s,", tname); + if (name == NULL || len <= 0) { + pIter = taosHashIterate(pTbl, pIter); + continue; + } + int32_t left = cap - strlen(p); + int32_t nBytes = snprintf(p + total, left, "%s,", name); + if (nBytes <= 0 || nBytes >= left) { + stError("failed to debug snapshot info since %s", tstrerror(TSDB_CODE_OUT_OF_RANGE)); + taosMemoryFree(p); + return; + } pIter = taosHashIterate(pTbl, pIter); - taosMemoryFree(tname); + total += nBytes; } if (total > 0) { p[total - 1] = 0; @@ -4764,13 +4698,30 @@ void strArrayDebugInfo(SArray* pArr, char** buf) { int32_t sz = taosArrayGetSize(pArr); if (sz <= 0) return; - char* p = (char*)taosMemoryCalloc(1, 64 + sz * 64); - int32_t total = 0; + int32_t code = 0; + int32_t total = 0, nBytes = 0; + int32_t cap = 64 + sz * 64; + + char* p = (char*)taosMemoryCalloc(1, cap); + if (p == NULL) { + stError("failed to alloc memory for stream snapshot debug info"); + return; + } for (int i = 0; i < sz; i++) { - char* name = taosArrayGetP(pArr, i); - total += sprintf(p + total, "%s,", name); + char* name = taosArrayGetP(pArr, i); + int32_t left = cap - strlen(p); + nBytes = snprintf(p + total, left, "%s,", name); + if (nBytes <= 0 || nBytes >= left) { + code = TSDB_CODE_OUT_OF_RANGE; + stError("failed to debug snapshot info since %s", tstrerror(code)); + taosMemoryFree(p); + return; + } + + total += nBytes; } + p[total - 1] = 0; *buf = p; @@ -4780,16 +4731,16 @@ void dbChkpDebugInfo(SDbChkp* pDb) { char* p[4] = {NULL}; hashTableToDebug(pDb->pSstTbl[pDb->idx], &p[0]); - stTrace("chkp previous file: [%s]", p[0]); + if (p[0]) stTrace("chkp previous file: [%s]", p[0]); hashTableToDebug(pDb->pSstTbl[1 - pDb->idx], &p[1]); - stTrace("chkp curr file: [%s]", p[1]); + if (p[1]) stTrace("chkp curr file: [%s]", p[1]); strArrayDebugInfo(pDb->pAdd, &p[2]); - stTrace("chkp newly addded file: [%s]", p[2]); + if (p[2]) stTrace("chkp newly addded file: [%s]", p[2]); strArrayDebugInfo(pDb->pDel, &p[3]); - stTrace("chkp newly deleted file: [%s]", p[3]); + if (p[3]) stTrace("chkp newly deleted file: [%s]", p[3]); for (int i = 0; i < 4; i++) { taosMemoryFree(p[i]); @@ -4821,8 +4772,8 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { return TSDB_CODE_OUT_OF_RANGE; } - taosArrayClearP(p->pAdd, taosMemoryFree); - taosArrayClearP(p->pDel, taosMemoryFree); + taosArrayClearP(p->pAdd, NULL); + taosArrayClearP(p->pDel, NULL); taosHashClear(p->pSstTbl[1 - p->idx]); TdDirPtr pDir = taosOpenDir(p->buf); @@ -4875,13 +4826,14 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { size_t len = 0; char* name = taosHashGetKey(pIter, &len); if (name != NULL && !isBkdDataMeta(name, len)) { - char* fname = taosMemoryCalloc(1, len + 1); + int32_t cap = len + 1; + char* fname = taosMemoryCalloc(1, cap); if (fname == NULL) { TAOS_UNUSED(taosThreadRwlockUnlock(&p->rwLock)); return terrno; } - TAOS_UNUSED(strncpy(fname, name, len)); + tstrncpy(fname, name, cap); if (taosArrayPush(p->pAdd, &fname) == NULL) { taosMemoryFree(fname); TAOS_UNUSED(taosThreadRwlockUnlock(&p->rwLock)); @@ -4899,8 +4851,8 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { int32_t code = compareHashTable(p->pSstTbl[p->idx], p->pSstTbl[1 - p->idx], p->pAdd, p->pDel); if (code != 0) { // dead code - taosArrayClearP(p->pAdd, taosMemoryFree); - taosArrayClearP(p->pDel, taosMemoryFree); + taosArrayClearP(p->pAdd, NULL); + taosArrayClearP(p->pDel, NULL); taosHashClear(p->pSstTbl[1 - p->idx]); p->update = 0; return code; @@ -4996,9 +4948,9 @@ void dbChkpDestroy(SDbChkp* pChkp) { taosMemoryFree(pChkp->buf); taosMemoryFree(pChkp->path); - taosArrayDestroyP(pChkp->pSST, taosMemoryFree); - taosArrayDestroyP(pChkp->pAdd, taosMemoryFree); - taosArrayDestroyP(pChkp->pDel, taosMemoryFree); + taosArrayDestroyP(pChkp->pSST, NULL); + taosArrayDestroyP(pChkp->pAdd, NULL); + taosArrayDestroyP(pChkp->pDel, NULL); taosHashCleanup(pChkp->pSstTbl[0]); taosHashCleanup(pChkp->pSstTbl[1]); @@ -5174,8 +5126,8 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { TAOS_UNUSED(taosCloseFile(&pFile)); // clear delta data buf - taosArrayClearP(p->pAdd, taosMemoryFree); - taosArrayClearP(p->pDel, taosMemoryFree); + taosArrayClearP(p->pAdd, NULL); + taosArrayClearP(p->pDel, NULL); code = 0; _ERROR: @@ -5356,7 +5308,8 @@ SStreamStateCur* streamStateSeekKeyPrev_rocksdb(SStreamState* pState, const SWin return NULL; } -int32_t streamStateGetGroupKVByCur_rocksdb(SStreamState* pState, SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { +int32_t streamStateGetGroupKVByCur_rocksdb(SStreamState* pState, SStreamStateCur* pCur, SWinKey* pKey, + const void** pVal, int32_t* pVLen) { if (!pCur) { return -1; } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 01cd201a1b..876cd1b472 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -816,7 +816,7 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d stDebug("s-task:%s remove redundant files in uploading checkpointId:%" PRId64 " data", idStr, checkpointId); } - taosArrayDestroyP(toDelFiles, taosMemoryFree); + taosArrayDestroyP(toDelFiles, NULL); double el = (taosGetTimestampMs() - now) / 1000.0; if (code == TSDB_CODE_SUCCESS) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index b4fcf1edc9..ad9be63674 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -71,7 +71,7 @@ static int32_t tInitStreamDispatchReq(SStreamDispatchReq* pReq, const SStreamTas pReq->data = taosArrayInit(numOfBlocks, POINTER_BYTES); pReq->dataLen = taosArrayInit(numOfBlocks, sizeof(int32_t)); if (pReq->data == NULL || pReq->dataLen == NULL) { - taosArrayDestroyP(pReq->data, taosMemoryFree); + taosArrayDestroyP(pReq->data, NULL); taosArrayDestroy(pReq->dataLen); return terrno; } @@ -234,7 +234,7 @@ int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups) { for (int32_t i = 0; i < numOfVgroups; i++) { - taosArrayDestroyP(pReq[i].data, taosMemoryFree); + taosArrayDestroyP(pReq[i].data, NULL); taosArrayDestroy(pReq[i].dataLen); } diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index 3a4e3eef89..53b6a38b35 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -127,6 +127,67 @@ static int32_t doSendHbMsgInfo(SStreamHbMsg* pMsg, SStreamMeta* pMeta, SEpSet* p return tmsgSendReq(pEpset, &msg); } +static int32_t streamTaskGetMndEpset(SStreamMeta* pMeta, SEpSet* pEpSet) { + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); + STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; + SStreamTask* pTask = NULL; + + int32_t code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask); + if (code != 0) { + continue; + } + + if (pTask->info.fillHistory == 1) { + streamMetaReleaseTask(pMeta, pTask); + continue; + } + + epsetAssign(pEpSet, &pTask->info.mnodeEpset); + streamMetaReleaseTask(pMeta, pTask); + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_FAILED; +} + +static void streamTaskUpdateMndEpset(SStreamMeta* pMeta, SEpSet* pEpSet) { + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); + STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; + SStreamTask* pTask = NULL; + + int32_t code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask); + if (code != 0) { + stError("vgId:%d s-task:0x%x failed to acquire it for updating mnode epset, code:%s", pMeta->vgId, pId->taskId, + tstrerror(code)); + continue; + } + + // ignore this error since it is only for log file + char buf[256] = {0}; + int32_t ret = epsetToStr(&pTask->info.mnodeEpset, buf, tListLen(buf)); + if (ret != 0) { // print error and continue + stError("failed to convert epset to str, code:%s", tstrerror(ret)); + } + + char newBuf[256] = {0}; + ret = epsetToStr(pEpSet, newBuf, tListLen(newBuf)); + if (ret != 0) { + stError("failed to convert epset to str, code:%s", tstrerror(ret)); + } + + epsetAssign(&pTask->info.mnodeEpset, pEpSet); + stInfo("s-task:0x%x update mnd epset, from %s to %s", pId->taskId, buf, newBuf); + streamMetaReleaseTask(pMeta, pTask); + } + + stDebug("vgId:%d update mnd epset for %d tasks completed", pMeta->vgId, numOfTasks); +} + // NOTE: this task should be executed within the SStreamMeta lock region. int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { SEpSet epset = {0}; @@ -140,24 +201,11 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { stDebug("vgId:%d hbMsg rsp not recv, send current hbMsg, msgId:%d, total:%d again", pMeta->vgId, pInfo->hbMsg.msgId, pInfo->hbCount); - for(int32_t i = 0; i < numOfTasks; ++i) { - SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); - STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; - SStreamTask* pTask = NULL; - - code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask); - if (code != 0) { - continue; - } - - if (pTask->info.fillHistory == 1) { - streamMetaReleaseTask(pMeta, pTask); - continue; - } - - epsetAssign(&epset, &pTask->info.mnodeEpset); - streamMetaReleaseTask(pMeta, pTask); - break; + code = streamTaskGetMndEpset(pMeta, &epset); + if (code != 0) { + stError("vgId:%d failed to get the mnode epset, not retrying sending hbMsg, msgId:%d", pMeta->vgId, + pInfo->hbMsg.msgId); + return code; } pInfo->msgSendTs = taosGetTimestampMs(); @@ -384,9 +432,11 @@ void streamMetaGetHbSendInfo(SMetaHbInfo* pInfo, int64_t* pStartTs, int32_t* pSe } int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp) { - stDebug("vgId:%d process hbMsg rsp, msgId:%d rsp confirmed", pMeta->vgId, pRsp->msgId); SMetaHbInfo* pInfo = pMeta->pHbInfo; + SEpSet epset = {0}; + int32_t code = 0; + stDebug("vgId:%d process hbMsg rsp, msgId:%d rsp confirmed", pMeta->vgId, pRsp->msgId); streamMetaWLock(pMeta); // current waiting rsp recved @@ -396,6 +446,13 @@ int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp) { pInfo->hbCount += 1; pInfo->msgSendTs = -1; + + code = streamTaskGetMndEpset(pMeta, &epset); + if (!isEpsetEqual(&pRsp->mndEpset, &epset) && (code == 0)) { + // we need to update the mnode epset for each tasks + stInfo("vgId:%d mnode epset updated, update mnode epset for all tasks", pMeta->vgId); + streamTaskUpdateMndEpset(pMeta, &pRsp->mndEpset); + } } else { stWarn("vgId:%d recv expired hb rsp, msgId:%d, discarded", pMeta->vgId, pRsp->msgId); } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 23a98ef3ae..8e8e300ee6 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -99,7 +99,7 @@ int32_t metaRefMgtInit() { void metaRefMgtCleanup() { void* pIter = taosHashIterate(gMetaRefMgt.pTable, NULL); while (pIter) { - int64_t* p = *(int64_t**) pIter; + int64_t* p = *(int64_t**)pIter; taosMemoryFree(p); pIter = taosHashIterate(gMetaRefMgt.pTable, pIter); } @@ -118,14 +118,14 @@ int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) { if (p == NULL) { code = taosHashPut(gMetaRefMgt.pTable, &rid, sizeof(rid), &rid, sizeof(void*)); if (code) { - stError("vgId:%d failed to put into refId mgt, refId:%" PRId64" %p, code:%s", (int32_t)vgId, *rid, rid, + stError("vgId:%d failed to put into refId mgt, refId:%" PRId64 " %p, code:%s", (int32_t)vgId, *rid, rid, tstrerror(code)); return code; - } else { // not -// stInfo("add refId:%"PRId64" vgId:%d, %p", *rid, (int32_t)vgId, rid); + } else { // not + // stInfo("add refId:%"PRId64" vgId:%d, %p", *rid, (int32_t)vgId, rid); } } else { - stFatal("try to add refId:%"PRId64" vgId:%d, %p that already added into mgt", *rid, (int32_t) vgId, rid); + stFatal("try to add refId:%" PRId64 " vgId:%d, %p that already added into mgt", *rid, (int32_t)vgId, rid); } streamMutexUnlock(&gMetaRefMgt.mutex); @@ -292,6 +292,7 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key) void* p = taskDbAddRef(*ppBackend); if (p == NULL) { stError("s-task:0x%x failed to ref backend", pTask->id.taskId); + streamMutexUnlock(&pMeta->backendMutex); return TSDB_CODE_FAILED; } @@ -669,7 +670,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { int64_t refId = pTask->id.refId; int32_t ret = taosRemoveRef(streamTaskRefPool, pTask->id.refId); if (ret != 0) { - stError("s-task:0x%x failed to remove ref, refId:%"PRId64, (int32_t) id[1], refId); + stError("s-task:0x%x failed to remove ref, refId:%" PRId64, (int32_t)id[1], refId); } } else { stDebug("s-task:%s vgId:%d refId:%" PRId64 " task meta save to disk", pTask->id.idStr, vgId, pTask->id.refId); @@ -727,14 +728,14 @@ int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTa int32_t ret = taosRemoveRef(streamTaskRefPool, refId); if (ret != 0) { - stError("s-task:0x%x failed to remove ref, refId:%"PRId64, (int32_t) id.taskId, refId); + stError("s-task:0x%x failed to remove ref, refId:%" PRId64, (int32_t)id.taskId, refId); } return code; } if ((code = streamMetaSaveTask(pMeta, pTask)) != 0) { int32_t unused = taosHashRemove(pMeta->pTasksMap, &id, sizeof(id)); - void* pUnused = taosArrayPop(pMeta->pTaskList); + void* pUnused = taosArrayPop(pMeta->pTaskList); int32_t ret = taosRemoveRef(streamTaskRefPool, refId); if (ret) { @@ -745,7 +746,7 @@ int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTa if ((code = streamMetaCommit(pMeta)) != 0) { int32_t unused = taosHashRemove(pMeta->pTasksMap, &id, sizeof(id)); - void* pUnused = taosArrayPop(pMeta->pTaskList); + void* pUnused = taosArrayPop(pMeta->pTaskList); int32_t ret = taosRemoveRef(streamTaskRefPool, refId); if (ret) { @@ -783,7 +784,7 @@ int32_t streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_ SStreamTask* p = taosAcquireRef(streamTaskRefPool, *pTaskRefId); if (p == NULL) { - stDebug("s-task:%x failed to acquire task refId:%"PRId64", may have been destoried", taskId, *pTaskRefId); + stDebug("s-task:%x failed to acquire task refId:%" PRId64 ", may have been destoried", taskId, *pTaskRefId); return TSDB_CODE_STREAM_TASK_NOT_EXIST; } @@ -946,11 +947,10 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t pTask->info.delaySchedParam = 0; } - int64_t refId = pTask->id.refId; int32_t ret = taosRemoveRef(streamTaskRefPool, refId); if (ret != 0) { - stError("s-task:0x%x failed to remove ref, refId:%"PRId64, (int32_t) id.taskId, refId); + stError("s-task:0x%x failed to remove ref, refId:%" PRId64, (int32_t)id.taskId, refId); } streamMetaReleaseTask(pMeta, pTask); @@ -1115,7 +1115,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { STaskId id = streamTaskGetTaskId(pTask); tFreeStreamTask(pTask); - void* px = taosArrayPush(pRecycleList, &id); + void* px = taosArrayPush(pRecycleList, &id); if (px == NULL) { stError("s-task:0x%x failed record the task into recycle list due to out of memory", taskId); } @@ -1165,7 +1165,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { continue; } - stInfo("s-task:0x%x vgId:%d set refId:%"PRId64, (int32_t) id.taskId, vgId, pTask->id.refId); + stInfo("s-task:0x%x vgId:%d set refId:%" PRId64, (int32_t)id.taskId, vgId, pTask->id.refId); if (pTask->info.fillHistory == 0) { int32_t val = atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1); } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 5461b5899b..89f0ea9e1f 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -120,7 +120,8 @@ SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, i SStreamTask* pStreamTask = pTask; pState->streamId = streamId; pState->taskId = taskId; - TAOS_UNUSED(tsnprintf(pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr), "0x%" PRIx64 "-0x%x", pState->streamId, pState->taskId)); + TAOS_UNUSED(tsnprintf(pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr), "0x%" PRIx64 "-0x%x", + pState->streamId, pState->taskId)); code = streamTaskSetDb(pStreamTask->pMeta, pTask, pState->pTdbState->idstr); QUERY_CHECK_CODE(code, lino, _end); @@ -539,7 +540,6 @@ int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId) { void streamStateDestroy(SStreamState* pState, bool remove) { streamFileStateDestroy(pState->pFileState); - // streamStateDestroy_rocksdb(pState, remove); tSimpleHashCleanup(pState->parNameMap); // do nothong taosMemoryFreeClear(pState->pTdbState); @@ -584,7 +584,8 @@ int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey return getCountWinResultBuff(pState->pFileState, pKey, winCount, ppVal, pVLen, pWinCode); } -int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen) { +int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, + int32_t* pVLen) { return createCountWinResultBuff(pState->pFileState, pKey, winCount, pVal, pVLen); } @@ -605,9 +606,7 @@ SStreamStateCur* streamStateGroupGetCur(SStreamState* pState) { return pCur; } -void streamStateGroupCurNext(SStreamStateCur* pCur) { - streamFileStateGroupCurNext(pCur); -} +void streamStateGroupCurNext(SStreamStateCur* pCur) { streamFileStateGroupCurNext(pCur); } int32_t streamStateGroupGetKVByCur(SStreamStateCur* pCur, int64_t* pKey, void** pVal, int32_t* pVLen) { if (pVal != NULL) { @@ -616,13 +615,9 @@ int32_t streamStateGroupGetKVByCur(SStreamStateCur* pCur, int64_t* pKey, void** return streamFileStateGroupGetKVByCur(pCur, pKey, pVal, pVLen); } -void streamStateClearExpiredState(SStreamState* pState) { - clearExpiredState(pState->pFileState); -} +void streamStateClearExpiredState(SStreamState* pState) { clearExpiredState(pState->pFileState); } -void streamStateSetFillInfo(SStreamState* pState) { - setFillInfo(pState->pFileState); -} +void streamStateSetFillInfo(SStreamState* pState) { setFillInfo(pState->pFileState); } int32_t streamStateGetPrev(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal, int32_t* pVLen, int32_t* pWinCode) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index ed9eb4b224..fbde104f4e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -692,7 +692,7 @@ int32_t syncGetArbToken(int64_t rid, char* outToken) { memset(outToken, 0, TSDB_ARB_TOKEN_SIZE); (void)taosThreadMutexLock(&pSyncNode->arbTokenMutex); - strncpy(outToken, pSyncNode->arbToken, TSDB_ARB_TOKEN_SIZE); + tstrncpy(outToken, pSyncNode->arbToken, TSDB_ARB_TOKEN_SIZE); (void)taosThreadMutexUnlock(&pSyncNode->arbTokenMutex); syncNodeRelease(pSyncNode); @@ -2933,7 +2933,7 @@ void syncNodeLogConfigInfo(SSyncNode* ths, SSyncCfg* cfg, char* str) { n += tsnprintf(buf + n, len - n, "%s", "{"); for (int i = 0; i < ths->peersEpset->numOfEps; i++) { n += tsnprintf(buf + n, len - n, "%s:%d%s", ths->peersEpset->eps[i].fqdn, ths->peersEpset->eps[i].port, - (i + 1 < ths->peersEpset->numOfEps ? ", " : "")); + (i + 1 < ths->peersEpset->numOfEps ? ", " : "")); } n += tsnprintf(buf + n, len - n, "%s", "}"); diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 02ab997f69..aeec8f93f0 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -52,7 +52,7 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i pDb->encryptAlgorithm = encryptAlgorithm; if (encryptKey != NULL) { - strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN + 1); } ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache)); diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 2753fe30d6..ca39bd1a23 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -262,7 +262,7 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { */ // Open the journal char jTxnFileName[TDB_FILENAME_LEN]; - sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + (void)tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); pTxn->jfd = tdbOsOpen(jTxnFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); if (TDB_FD_INVALID(pTxn->jfd)) { tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); @@ -365,7 +365,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) { char jTxnFileName[TDB_FILENAME_LEN]; - sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + (void)tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); // remove the journal file if (tdbOsClose(pTxn->jfd) < 0) { @@ -459,7 +459,7 @@ static char *tdbEncryptPage(SPager *pPager, char *pPageData, int32_t pageSize, c opts.source = pPageData + count; opts.result = packetData; opts.unitLen = 128; - strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN + 1); int32_t newLen = CBC_Encrypt(&opts); @@ -595,7 +595,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { } char jTxnFileName[TDB_FILENAME_LEN]; - sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + (void)tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); @@ -927,7 +927,7 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage opts.source = pPage->pData + count; opts.result = packetData; opts.unitLen = 128; - strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN + 1); int newLen = CBC_Decrypt(&opts); @@ -1179,7 +1179,7 @@ int tdbPagerRestoreJournals(SPager *pPager) { int dirLen = strlen(pPager->pEnv->dbName); memcpy(jname, pPager->pEnv->dbName, dirLen); jname[dirLen] = '/'; - sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); + (void)tsnprintf(jname + dirLen + 1, TD_PATH_MAX - dirLen - 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); code = tdbPagerRestore(pPager, jname); if (code) { taosArrayDestroy(pTxnList); diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 51fda85f0a..4ac72e8918 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -200,10 +200,8 @@ bool tfsIsSameFile(const STfsFile *pFile1, const STfsFile *pFile2) { if (pFile1->did.level != pFile2->did.level) return false; if (pFile1->did.id != pFile2->did.id) return false; char nameBuf1[TMPNAME_LEN], nameBuf2[TMPNAME_LEN]; - (void)strncpy(nameBuf1, pFile1->rname, TMPNAME_LEN); - (void)strncpy(nameBuf2, pFile2->rname, TMPNAME_LEN); - nameBuf1[TMPNAME_LEN - 1] = 0; - nameBuf2[TMPNAME_LEN - 1] = 0; + tstrncpy(nameBuf1, pFile1->rname, TMPNAME_LEN); + tstrncpy(nameBuf2, pFile2->rname, TMPNAME_LEN); TAOS_UNUSED(taosRealPath(nameBuf1, NULL, TMPNAME_LEN)); TAOS_UNUSED(taosRealPath(nameBuf2, NULL, TMPNAME_LEN)); if (strncmp(nameBuf1, nameBuf2, TMPNAME_LEN) != 0) return false; @@ -573,7 +571,7 @@ static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { TAOS_RETURN(TSDB_CODE_FS_INVLD_CFG); } - strncpy(pCfg->dir, dirName, TSDB_FILENAME_LEN); + tstrncpy(pCfg->dir, dirName, TSDB_FILENAME_LEN); TAOS_RETURN(0); } @@ -592,7 +590,16 @@ static int32_t tfsFormatDir(char *idir, char *odir) { wordfree(&wep); TAOS_RETURN(code); } - strcpy(odir, tmp); + + int32_t dirLen = strlen(tmp); + if (dirLen < 0 || dirLen >= TSDB_FILENAME_LEN) { + wordfree(&wep); + code = TSDB_CODE_OUT_OF_RANGE; + fError("failed to mount %s to FS since %s, real path:%s, len:%d", idir, tstrerror(code), tmp, dirLen); + TAOS_RETURN(code); + } + + tstrncpy(odir, tmp, TSDB_FILENAME_LEN); wordfree(&wep); TAOS_RETURN(0); diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index 6ad130017a..631f7029f5 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -24,7 +24,3 @@ if(${BUILD_WITH_UV_TRANS}) add_definitions(-DUSE_UV) endif(${BUILD_WITH_UV}) endif(${BUILD_WITH_UV_TRANS}) - -if(${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 6ec5f74f62..44559f2db2 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -311,8 +311,8 @@ void transRefCliHandle(void* handle); int32_t transUnrefCliHandle(void* handle); int32_t transGetRefCount(void* handle); -int32_t transReleaseCliHandle(void* handle); -int32_t transReleaseSrvHandle(void* handle); +int32_t transReleaseCliHandle(void* handle, int32_t status); +int32_t transReleaseSrvHandle(void* handle, int32_t status); int32_t transSendRequest(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* pCtx); int32_t transSendRecv(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp); diff --git a/source/libs/transport/src/tmsgcb.c b/source/libs/transport/src/tmsgcb.c index 4c969003a9..e87011f097 100644 --- a/source/libs/transport/src/tmsgcb.c +++ b/source/libs/transport/src/tmsgcb.c @@ -48,6 +48,7 @@ int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { } return code; } + int32_t tmsgSendSyncReq(const SEpSet* epSet, SRpcMsg* pMsg) { int32_t code = (*defaultMsgCb.sendSyncReqFp)(epSet, pMsg); if (code != 0) { @@ -69,7 +70,9 @@ void tmsgSendRsp(SRpcMsg* pMsg) { void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg) { (*defaultMsgCb.registerBrokenLinkArgFp)(pMsg); } -void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) { (*defaultMsgCb.releaseHandleFp)(pHandle, type); } +void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type, int32_t status) { + (*defaultMsgCb.releaseHandleFp)(pHandle, type, status); +} void tmsgReportStartup(const char* name, const char* desc) { (*defaultMsgCb.reportStartupFp)(name, desc); } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index de129773a0..d7576005d7 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -23,7 +23,7 @@ void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient}; void (*taosRefHandle[])(void* handle) = {transRefSrvHandle, transRefCliHandle}; void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, NULL}; -int (*transReleaseHandle[])(void* handle) = {transReleaseSrvHandle, transReleaseCliHandle}; +int (*transReleaseHandle[])(void* handle, int32_t status) = {transReleaseSrvHandle, transReleaseCliHandle}; static int32_t transValidLocalFqdn(const char* localFqdn, uint32_t* ip) { int32_t code = taosGetIpv4FromFqdn(localFqdn, ip); @@ -158,7 +158,6 @@ void* rpcMallocCont(int64_t contLen) { char* start = taosMemoryCalloc(1, size); if (start == NULL) { tError("failed to malloc msg, size:%" PRId64, size); - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } else { tTrace("malloc mem:%p size:%" PRId64, start, size); @@ -217,7 +216,9 @@ void rpcRefHandle(void* handle, int8_t type) { (*taosRefHandle[type])(handle); } void rpcUnrefHandle(void* handle, int8_t type) { (*taosUnRefHandle[type])(handle); } int32_t rpcRegisterBrokenLinkArg(SRpcMsg* msg) { return transRegisterMsg(msg); } -int32_t rpcReleaseHandle(void* handle, int8_t type) { return (*transReleaseHandle[type])(handle); } +int32_t rpcReleaseHandle(void* handle, int8_t type, int32_t status) { + return (*transReleaseHandle[type])(handle, status); +} // client only int32_t rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index ed84404d05..468d9d9b50 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -561,9 +561,39 @@ int32_t cliBuildRespFromCont(SCliReq* pReq, STransMsg* pResp, STransMsgHead* pHe pResp->info.handle = (void*)qid; return 0; } + +int8_t cliMayNotifyUserOnRecvReleaseExcept(SCliConn* conn, STransMsgHead* pHead, SCliReq* pReq) { + int32_t code = 0; + if (pHead->code == 0 || pHead->msgType != TDMT_SCH_TASK_RELEASE) { + return 0; + } + // no ahandle, no need to notify user + if (pReq == NULL || pReq->ctx == NULL || pReq->ctx->ahandle == NULL) { + return 0; + } + + SCliThrd* pThrd = conn->hostThrd; + STransMsg resp = {.code = pHead->code}; + int64_t qId = taosHton64(pHead->qid); + STraceId* trace = &pHead->traceId; + code = cliBuildExceptResp(pThrd, pReq, &resp); + if (code != 0) { + tGWarn("%s conn %p failed to build except resp for req:%" PRId64 " since %s", CONN_GET_INST_LABEL(conn), conn, qId, + tstrerror(code)); + } + code = cliNotifyCb(conn, NULL, &resp); + if (code != 0) { + tGWarn("%s conn %p failed to notify user for req:%" PRId64 " since %s", CONN_GET_INST_LABEL(conn), conn, qId, + tstrerror(code)); + } + + destroyReq(pReq); + return 1; +} int32_t cliHandleState_mayHandleReleaseResp(SCliConn* conn, STransMsgHead* pHead) { int32_t code = 0; SCliThrd* pThrd = conn->hostThrd; + int8_t notifyUser = 0; if (pHead->msgType == TDMT_SCH_TASK_RELEASE || pHead->msgType == TDMT_SCH_TASK_RELEASE + 1) { int64_t qId = taosHton64(pHead->qid); STraceId* trace = &pHead->traceId; @@ -603,7 +633,11 @@ int32_t cliHandleState_mayHandleReleaseResp(SCliConn* conn, STransMsgHead* pHead removeReqFromSendQ(pReq); STraceId* trace = &pReq->msg.info.traceId; tGDebug("start to free msg %p", pReq); - destroyReqWrapper(pReq, pThrd); + + if (cliMayNotifyUserOnRecvReleaseExcept(conn, pHead, pReq)) { + } else { + destroyReqWrapper(pReq, pThrd); + } } taosMemoryFree(pHead); return 1; @@ -707,7 +741,6 @@ void cliHandleResp(SCliConn* conn) { tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, seq:%" PRId64 ", sid:%" PRId64 ", code:%s", CONN_GET_INST_LABEL(conn), conn, TMSG_INFO(resp.msgType), conn->dst, conn->src, pHead->msgLen, seq, qId, tstrerror(pHead->code)); - code = cliNotifyCb(conn, pReq, &resp); if (code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { tGWarn("%s msg need retry", CONN_GET_INST_LABEL(conn)); @@ -1266,6 +1299,8 @@ static void cliHandleException(SCliConn* conn) { if (conn->registered) { int8_t ref = transGetRefCount(conn); if (ref == 0 && !uv_is_closing((uv_handle_t*)conn->stream)) { +// tTrace("%s conn %p fd %d,%d,%d,%p uv_closed", CONN_GET_INST_LABEL(conn), conn, conn->stream->u.fd, +// conn->stream->io_watcher.fd, conn->stream->accepted_fd, conn->stream->queued_fds); uv_close((uv_handle_t*)conn->stream, cliDestroy); } } @@ -1594,6 +1629,8 @@ static int32_t cliDoConn(SCliThrd* pThrd, SCliConn* conn) { TAOS_CHECK_GOTO(terrno, &lino, _exception1); } + tTrace("%s conn %p fd %d openend", pInst->label, conn, fd); + int ret = uv_tcp_open((uv_tcp_t*)conn->stream, fd); if (ret != 0) { tError("%s conn %p failed to set stream since %s", transLabel(pInst), conn, uv_err_name(ret)); @@ -1809,9 +1846,7 @@ FORCE_INLINE int32_t cliBuildExceptResp(SCliThrd* pThrd, SCliReq* pReq, STransMs STrans* pInst = pThrd->pInst; - SReqCtx* pCtx = pReq ? pReq->ctx : NULL; - STransMsg resp = {0}; - // resp.code = (conn->connnected ? TSDB_CODE_RPC_BROKEN_LINK : TSDB_CODE_RPC_NETWORK_UNAVAIL); + SReqCtx* pCtx = pReq ? pReq->ctx : NULL; pResp->msgType = pReq ? pReq->msg.msgType + 1 : 0; pResp->info.cliVer = pInst->compatibilityVer; pResp->info.ahandle = pCtx ? pCtx->ahandle : 0; @@ -3129,15 +3164,18 @@ SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) { SCliThrd* pThrd = transGetWorkThrdFromHandle(trans, handle); return pThrd; } -int32_t transReleaseCliHandle(void* handle) { +int32_t transReleaseCliHandle(void* handle, int32_t status) { int32_t code = 0; SCliThrd* pThrd = transGetWorkThrdFromHandle(NULL, (int64_t)handle); if (pThrd == NULL) { return TSDB_CODE_RPC_BROKEN_LINK; } - STransMsg tmsg = { - .msgType = TDMT_SCH_TASK_RELEASE, .info.handle = handle, .info.ahandle = (void*)0, .info.qId = (int64_t)handle}; + STransMsg tmsg = {.msgType = TDMT_SCH_TASK_RELEASE, + .info.handle = handle, + .info.ahandle = (void*)0, + .info.qId = (int64_t)handle, + code = status}; TRACE_SET_MSGID(&tmsg.info.traceId, tGenIdPI64()); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index ecbdd0db84..941444953d 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1862,7 +1862,7 @@ void transUnrefSrvHandle(void* handle) { } } -int32_t transReleaseSrvHandle(void* handle) { +int32_t transReleaseSrvHandle(void* handle, int32_t status) { int32_t code = 0; SRpcHandleInfo* info = handle; SExHandle* exh = info->handle; @@ -1875,7 +1875,7 @@ int32_t transReleaseSrvHandle(void* handle) { ASYNC_ERR_JRET(pThrd); STransMsg tmsg = {.msgType = TDMT_SCH_TASK_RELEASE, - .code = 0, + .code = status, .info.handle = exh, .info.ahandle = NULL, .info.refId = refId, diff --git a/source/libs/transport/test/transUT.cpp b/source/libs/transport/test/transUT.cpp index 8e396d59d7..a50ff47253 100644 --- a/source/libs/transport/test/transUT.cpp +++ b/source/libs/transport/test/transUT.cpp @@ -96,7 +96,7 @@ class Client { } void SendAndRecvNoHandle(SRpcMsg *req, SRpcMsg *resp) { if (req->info.handle != NULL) { - rpcReleaseHandle(req->info.handle, TAOS_CONN_CLIENT); + rpcReleaseHandle(req->info.handle, TAOS_CONN_CLIENT, 0); req->info.handle = NULL; } SendAndRecv(req, resp); @@ -191,7 +191,7 @@ static void processReleaseHandleCb(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) rpcMsg.code = 0; rpcSendResponse(&rpcMsg); - rpcReleaseHandle(&pMsg->info, TAOS_CONN_SERVER); + rpcReleaseHandle(&pMsg->info, TAOS_CONN_SERVER, 0); } static void processRegisterFailure(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { { @@ -366,7 +366,7 @@ TEST_F(TransEnv, cliPersistHandle) { //} handle = resp.info.handle; } - rpcReleaseHandle(handle, TAOS_CONN_CLIENT); + rpcReleaseHandle(handle, TAOS_CONN_CLIENT, 0); for (int i = 0; i < 10; i++) { SRpcMsg req = {0}; req.msgType = 1; diff --git a/source/libs/transport/test/transUT2.cpp b/source/libs/transport/test/transUT2.cpp index 54d23b1f64..d6b336b014 100644 --- a/source/libs/transport/test/transUT2.cpp +++ b/source/libs/transport/test/transUT2.cpp @@ -106,7 +106,7 @@ class Client { } void SendAndRecvNoHandle(SRpcMsg *req, SRpcMsg *resp) { if (req->info.handle != NULL) { - rpcReleaseHandle(req->info.handle, TAOS_CONN_CLIENT); + rpcReleaseHandle(req->info.handle, TAOS_CONN_CLIENT, 0); req->info.handle = NULL; } SendAndRecv(req, resp); @@ -201,7 +201,7 @@ static void processReleaseHandleCb(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) rpcMsg.code = 0; rpcSendResponse(&rpcMsg); - rpcReleaseHandle(&pMsg->info, TAOS_CONN_SERVER); + rpcReleaseHandle(&pMsg->info, TAOS_CONN_SERVER, 0); } static void processRegisterFailure(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { // { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index da5e1f47e9..43a2ff6a23 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -539,7 +539,7 @@ int32_t decryptBody(SWalCfg *cfg, SWalCkHead *pHead, int32_t plainBodyLen, const opts.source = pHead->head.body; opts.result = newBody; opts.unitLen = 16; - TAOS_UNUSED(strncpy((char *)opts.key, cfg->encryptKey, 16)); + tstrncpy((char *)opts.key, cfg->encryptKey, sizeof(opts.key)); int32_t count = CBC_Decrypt(&opts); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 69e70a0f47..998668beac 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -332,7 +332,7 @@ _exit: wError("vgId:%d, %s failed at line %d since %s", pWal->cfg.vgId, __func__, lino, tstrerror(code)); } - TAOS_RETURN(TSDB_CODE_SUCCESS); + TAOS_RETURN(code); } static FORCE_INLINE int32_t walCheckAndRoll(SWal *pWal) { @@ -621,7 +621,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy opts.source = newBody; opts.result = newBodyEncrypted; opts.unitLen = 16; - TAOS_UNUSED(strncpy((char *)opts.key, pWal->cfg.encryptKey, ENCRYPT_KEY_LEN)); + tstrncpy((char *)opts.key, pWal->cfg.encryptKey, ENCRYPT_KEY_LEN + 1); int32_t count = CBC_Encrypt(&opts); @@ -689,8 +689,13 @@ _exit: static int32_t walInitWriteFile(SWal *pWal) { TdFilePtr pIdxTFile, pLogTFile; + int64_t fileFirstVer = -1; + int32_t code = 0; SWalFileInfo *pRet = taosArrayGetLast(pWal->fileInfoSet); - int64_t fileFirstVer = pRet->firstVer; + if (pRet == NULL) { + fileFirstVer = pWal->vers.lastVer + 1; + } + fileFirstVer = pRet->firstVer; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, fileFirstVer, fnameStr); @@ -707,6 +712,13 @@ static int32_t walInitWriteFile(SWal *pWal) { // switch file pWal->pIdxFile = pIdxTFile; pWal->pLogFile = pLogTFile; + if (taosArrayGetSize(pWal->fileInfoSet) == 0) { + code = walRollFileInfo(pWal); + if (code < 0) { + wError("vgId:%d, failed to roll file info while init write file since %s", pWal->cfg.vgId, terrstr()); + TAOS_RETURN(code); + } + } pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; TAOS_RETURN(TSDB_CODE_SUCCESS); diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index 96deaa4196..6b33e68377 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -14,6 +14,7 @@ target_include_directories( PUBLIC "${TD_SOURCE_DIR}/contrib/pthread" PUBLIC "${TD_SOURCE_DIR}/contrib/iconv" PUBLIC "${TD_SOURCE_DIR}/contrib/msvcregex" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) # iconv if(TD_WINDOWS) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 5030eb1f67..8a2606c4c2 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -198,7 +198,7 @@ _err: } TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions) { - if(path == NULL) { + if (path == NULL) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } @@ -301,6 +301,28 @@ int32_t taosStatFile(const char *path, int64_t *size, int64_t *mtime, int64_t *a return 0; } + +int32_t taosGetFileDiskID(const char *path, int64_t *diskid) { + OS_PARAM_CHECK(path); +#ifdef WINDOWS + struct _stati64 fileStat; + int32_t code = _stati64(path, &fileStat); +#else + struct stat fileStat; + int32_t code = stat(path, &fileStat); +#endif + if (-1 == code) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } + + if (diskid != NULL) { + *diskid = fileStat.st_dev; + } + + return 0; +} + int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { #ifdef WINDOWS if (pFile == NULL || pFile->hFile == NULL) { @@ -429,7 +451,7 @@ HANDLE taosOpenFileNotStream(const char *path, int32_t tdFileOptions) { } int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { - if (pFile == NULL || buf == NULL) { + if (pFile == NULL || buf == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -1572,7 +1594,7 @@ FILE *taosOpenCFile(const char *filename, const char *mode) { } int taosSeekCFile(FILE *file, int64_t offset, int whence) { - if(NULL == file) { + if (NULL == file) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index cb0a2646fd..484436fafe 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -87,8 +87,7 @@ int32_t taosSetSystemLocale(const char *inLocale) { return terrno; } - (void)memcpy(tsLocale, inLocale, strlen(inLocale) + 1); - + tstrncpy(tsLocale, locale, TD_LOCALE_LEN); return 0; } diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index f4aa966d2b..fdbf4853ad 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -260,7 +260,7 @@ int32_t taosMemoryDbgInitRestore() { #endif } -void *taosMemoryMalloc(int64_t size) { +void *taosMemMalloc(int64_t size) { #ifdef USE_TD_MEMORY void *tmp = malloc(size + sizeof(TdMemoryInfo)); if (tmp == NULL) return NULL; @@ -290,7 +290,7 @@ void *taosMemoryMalloc(int64_t size) { #endif } -void *taosMemoryCalloc(int64_t num, int64_t size) { +void *taosMemCalloc(int64_t num, int64_t size) { #ifdef USE_TD_MEMORY int32_t memorySize = num * size; char *tmp = calloc(memorySize + sizeof(TdMemoryInfo), 1); @@ -322,7 +322,7 @@ void *taosMemoryCalloc(int64_t num, int64_t size) { #endif } -void *taosMemoryRealloc(void *ptr, int64_t size) { +void *taosMemRealloc(void *ptr, int64_t size) { #ifdef USE_TD_MEMORY if (ptr == NULL) return taosMemoryMalloc(size); @@ -360,7 +360,7 @@ void *taosMemoryRealloc(void *ptr, int64_t size) { #endif } -char *taosStrdup(const char *ptr) { +char *taosStrdupi(const char *ptr) { #ifdef USE_TD_MEMORY if (ptr == NULL) return NULL; @@ -390,7 +390,8 @@ char *taosStrdup(const char *ptr) { #endif } -void taosMemoryFree(void *ptr) { + +void taosMemFree(void *ptr) { if (NULL == ptr) return; #ifdef USE_TD_MEMORY TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); @@ -406,7 +407,7 @@ void taosMemoryFree(void *ptr) { #endif } -int64_t taosMemorySize(void *ptr) { +int64_t taosMemSize(void *ptr) { if (ptr == NULL) return 0; #ifdef USE_TD_MEMORY @@ -427,16 +428,22 @@ int64_t taosMemorySize(void *ptr) { #endif } -void taosMemoryTrim(int32_t size) { +int32_t taosMemTrim(int32_t size, bool* trimed) { #if defined(WINDOWS) || defined(DARWIN) || defined(_ALPINE) // do nothing - return; + return TSDB_CODE_SUCCESS; #else - (void)malloc_trim(size); + if (trimed) { + *trimed = malloc_trim(size); + } else { + malloc_trim(size); + } + + return TSDB_CODE_SUCCESS; #endif } -void *taosMemoryMallocAlign(uint32_t alignment, int64_t size) { +void *taosMemMallocAlign(uint32_t alignment, int64_t size) { #ifdef USE_TD_MEMORY return NULL; #else @@ -461,7 +468,7 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size) { } return p; #else - return taosMemoryMalloc(size); + return taosMemMalloc(size); #endif #endif } diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 1ae8ac2a28..380e9f84d3 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -75,7 +75,7 @@ char *strsep(char **stringp, const char *delim) { /* NOTREACHED */ } /* Duplicate a string, up to at most size characters */ -char *taosStrndup(const char *s, int size) { +char *taosStrndupi(const char *s, int64_t size) { if (s == NULL) return NULL; size_t l; char *s2; @@ -104,7 +104,7 @@ char *stpncpy(char *dest, const char *src, int n) { return memset(dest, '\0', n - size); } #else -char *taosStrndup(const char *s, int size) { +char *taosStrndupi(const char *s, int64_t size) { if (s == NULL) { return NULL; } @@ -116,37 +116,38 @@ char *taosStrndup(const char *s, int size) { } #endif +char *tstrndup(const char *str, int64_t size) { +#ifdef WINDOWS + return taosStrndupi(str, size); +#else + char* p = strndup(str, size); + if (str != NULL && NULL == p) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return p; + +#endif +} + + int32_t taosStr2int64(const char *str, int64_t *val) { if (str == NULL || val == NULL) { return TSDB_CODE_INVALID_PARA; } + errno = 0; char *endptr = NULL; int64_t ret = strtoll(str, &endptr, 10); - if (errno == ERANGE && (ret == LLONG_MAX || ret == LLONG_MIN)) { + if (errno != 0) { return TAOS_SYSTEM_ERROR(errno); - } else if (errno == EINVAL && ret == 0) { - return TSDB_CODE_INVALID_PARA; } else { + if (endptr == str) { + return TSDB_CODE_INVALID_PARA; + } *val = ret; return 0; } } -int32_t taosStr2int16(const char *str, int16_t *val) { - OS_PARAM_CHECK(str); - OS_PARAM_CHECK(val); - int64_t tmp = 0; - int32_t code = taosStr2int64(str, &tmp); - if (code) { - return code; - } else if (tmp > INT16_MAX || tmp < INT16_MIN) { - return TAOS_SYSTEM_ERROR(ERANGE); - } else { - *val = (int16_t)tmp; - return 0; - } -} - int32_t taosStr2int32(const char *str, int32_t *val) { OS_PARAM_CHECK(str); OS_PARAM_CHECK(val); @@ -161,6 +162,20 @@ int32_t taosStr2int32(const char *str, int32_t *val) { return 0; } } +int32_t taosStr2int16(const char *str, int16_t *val) { + OS_PARAM_CHECK(str); + OS_PARAM_CHECK(val); + int64_t tmp = 0; + int32_t code = taosStr2int64(str, &tmp); + if (code) { + return code; + } else if (tmp > INT16_MAX || tmp < INT16_MIN) { + return TAOS_SYSTEM_ERROR(ERANGE); + } else { + *val = (int16_t)tmp; + return 0; + } +} int32_t taosStr2int8(const char *str, int8_t *val) { OS_PARAM_CHECK(str); @@ -177,6 +192,70 @@ int32_t taosStr2int8(const char *str, int8_t *val) { } } +int32_t taosStr2Uint64(const char *str, uint64_t *val) { + if (str == NULL || val == NULL) { + return TSDB_CODE_INVALID_PARA; + } + char *endptr = NULL; + errno = 0; + uint64_t ret = strtoull(str, &endptr, 10); + + if (errno != 0) { + return TAOS_SYSTEM_ERROR(errno); + } else { + if (endptr == str) { + return TSDB_CODE_INVALID_PARA; + } + *val = ret; + return 0; + } +} + +int32_t taosStr2Uint32(const char *str, uint32_t *val) { + OS_PARAM_CHECK(str); + OS_PARAM_CHECK(val); + uint64_t tmp = 0; + int32_t code = taosStr2Uint64(str, &tmp); + if (code) { + return code; + } else if (tmp > UINT32_MAX) { + return TAOS_SYSTEM_ERROR(ERANGE); + } else { + *val = (int32_t)tmp; + return 0; + } +} + +int32_t taosStr2Uint16(const char *str, uint16_t *val) { + OS_PARAM_CHECK(str); + OS_PARAM_CHECK(val); + uint64_t tmp = 0; + int32_t code = taosStr2Uint64(str, &tmp); + if (code) { + return code; + } else if (tmp > UINT16_MAX) { + return TAOS_SYSTEM_ERROR(ERANGE); + } else { + *val = (int16_t)tmp; + return 0; + } +} + +int32_t taosStr2Uint8(const char *str, uint8_t *val) { + OS_PARAM_CHECK(str); + OS_PARAM_CHECK(val); + uint64_t tmp = 0; + int32_t code = taosStr2Uint64(str, &tmp); + if (code) { + return code; + } else if (tmp > UINT8_MAX) { + return TAOS_SYSTEM_ERROR(ERANGE); + } else { + *val = (int8_t)tmp; + return 0; + } +} + int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { if ((f1_ucs4 == NULL || f2_ucs4 == NULL)) { return TSDB_CODE_INVALID_PARA; @@ -223,7 +302,7 @@ int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) terrno = TSDB_CODE_INVALID_PARA; return terrno; } - + (void)memcpy(target_ucs4, source_ucs4, len_ucs4 * sizeof(TdUcs4)); return TSDB_CODE_SUCCESS; @@ -306,7 +385,7 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4 if (ucs4_max_len == 0) { return true; } - if(ucs4_max_len < 0 || mbs == NULL || ucs4 == NULL) { + if (ucs4_max_len < 0 || mbs == NULL || ucs4 == NULL) { terrno = TSDB_CODE_INVALID_PARA; return false; } @@ -323,8 +402,8 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4 return false; } - size_t ucs4_input_len = mbsLength; - size_t outLeft = ucs4_max_len; + size_t ucs4_input_len = mbsLength; + size_t outLeft = ucs4_max_len; if (iconv(conv, (char **)&mbs, &ucs4_input_len, (char **)&ucs4, &outLeft) == -1) { char buf[512] = {0}; snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s %d %d", strerror(terrno), errno, EILSEQ); @@ -353,7 +432,7 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* chars if (ucs4_max_len == 0) { return 0; } - if(ucs4_max_len < 0 || ucs4 == NULL || mbs == NULL) { + if (ucs4_max_len < 0 || ucs4 == NULL || mbs == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -370,8 +449,8 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* chars return terrno; } - size_t ucs4_input_len = ucs4_max_len; - size_t outLen = ucs4_max_len; + size_t ucs4_input_len = ucs4_max_len; + size_t outLen = ucs4_max_len; if (iconv(conv, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) { code = TAOS_SYSTEM_ERROR(errno); taosReleaseConv(idx, conv, C2M, charsetCxt); @@ -391,7 +470,7 @@ int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t c if (ucs4_max_len == 0) { return 0; } - if(ucs4_max_len < 0 || ucs4 == NULL || mbs == NULL) { + if (ucs4_max_len < 0 || ucs4 == NULL || mbs == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -407,13 +486,13 @@ int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t c terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - + return (int32_t)(ucs4_max_len - outLen); #endif } bool taosValidateEncodec(const char *encodec) { - if(encodec == NULL) { + if (encodec == NULL) { terrno = TSDB_CODE_INVALID_PARA; return false; } @@ -465,7 +544,7 @@ int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len, int32_t } int32_t taosHexDecode(const char *src, char *dst, int32_t len) { - if(!src || !dst || len <= 0) { + if (!src || !dst || len <= 0) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -508,9 +587,10 @@ int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size) { return mbstowcs(pWchars, pStrs, size); } -int32_t taosWcharToMb(char *pStr, TdWchar wchar) { +int32_t taosWcharToMb(char *pStr, TdWchar wchar) { OS_PARAM_CHECK(pStr); - return wctomb(pStr, wchar); } + return wctomb(pStr, wchar); +} char *taosStrCaseStr(const char *str, const char *pattern) { if (str == NULL) { @@ -532,7 +612,7 @@ char *taosStrCaseStr(const char *str, const char *pattern) { } int64_t taosStr2Int64(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -544,7 +624,7 @@ int64_t taosStr2Int64(const char *str, char **pEnd, int32_t radix) { } uint64_t taosStr2UInt64(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -556,7 +636,7 @@ uint64_t taosStr2UInt64(const char *str, char **pEnd, int32_t radix) { } int32_t taosStr2Int32(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -568,7 +648,7 @@ int32_t taosStr2Int32(const char *str, char **pEnd, int32_t radix) { } uint32_t taosStr2UInt32(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -580,7 +660,7 @@ uint32_t taosStr2UInt32(const char *str, char **pEnd, int32_t radix) { } int16_t taosStr2Int16(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -592,7 +672,7 @@ int16_t taosStr2Int16(const char *str, char **pEnd, int32_t radix) { } uint16_t taosStr2UInt16(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -604,7 +684,7 @@ uint16_t taosStr2UInt16(const char *str, char **pEnd, int32_t radix) { } int8_t taosStr2Int8(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -613,7 +693,7 @@ int8_t taosStr2Int8(const char *str, char **pEnd, int32_t radix) { } uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -625,7 +705,7 @@ uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix) { } double taosStr2Double(const char *str, char **pEnd) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -634,7 +714,7 @@ double taosStr2Double(const char *str, char **pEnd) { } float taosStr2Float(const char *str, char **pEnd) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -650,7 +730,7 @@ bool isHex(const char *z, uint32_t n) { } bool isValidateHex(const char *z, uint32_t n) { - if(!z) { + if (!z) { terrno = TSDB_CODE_INVALID_PARA; return false; } @@ -676,12 +756,12 @@ int32_t taosHex2Ascii(const char *z, uint32_t n, void **data, uint32_t *size) { } return 0; } - + uint8_t *tmp = (uint8_t *)taosMemoryCalloc(*size, 1); if (tmp == NULL) { return terrno; } - + int8_t num = 0; uint8_t *byte = tmp + *size - 1; @@ -701,7 +781,7 @@ int32_t taosHex2Ascii(const char *z, uint32_t n, void **data, uint32_t *size) { } } *data = tmp; - + return 0; } @@ -777,7 +857,7 @@ int32_t taosAscii2Hex(const char *z, uint32_t n, void **data, uint32_t *size) { if (tmp == NULL) { return terrno; } - + *data = tmp; *(tmp++) = '\\'; *(tmp++) = 'x'; @@ -786,7 +866,7 @@ int32_t taosAscii2Hex(const char *z, uint32_t n, void **data, uint32_t *size) { tmp[i * 2] = valueOf(val >> 4); tmp[i * 2 + 1] = valueOf(val & 0x0F); } - + return 0; } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index dc3258bf9c..526f1a33e4 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -779,6 +779,59 @@ int32_t taosGetProcMemory(int64_t *usedKB) { #endif } +int32_t taosGetSysAvailMemory(int64_t *availSize) { +#ifdef WINDOWS + MEMORYSTATUSEX memsStat; + memsStat.dwLength = sizeof(memsStat); + if (!GlobalMemoryStatusEx(&memsStat)) { + return -1; + } + + int64_t nMemFree = memsStat.ullAvailPhys; + int64_t nMemTotal = memsStat.ullTotalPhys; + + *availSize = nMemTotal - nMemFree; + return 0; +#elif defined(_TD_DARWIN_64) + *availSize = 0; + return 0; +#else + TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) { + return terrno; + } + + ssize_t bytes = 0; + char line[128] = {0}; + int32_t expectedSize = 13; //"MemAvailable:" + while (!taosEOFFile(pFile)) { + bytes = taosGetsFile(pFile, sizeof(line), line); + if (bytes < 0) { + break; + } + if (line[0] != 'M' && line[3] != 'A') { + line[0] = 0; + continue; + } + if (0 == strncmp(line, "MemAvailable:", expectedSize)) { + break; + } + } + + if (0 == line[0]) { + return TSDB_CODE_UNSUPPORT_OS; + } + + char tmp[32]; + (void)sscanf(line, "%s %" PRId64, tmp, availSize); + + *availSize *= 1024; + + (void)taosCloseFile(&pFile); + return 0; +#endif +} + int32_t taosGetSysMemory(int64_t *usedKB) { OS_PARAM_CHECK(usedKB); #ifdef WINDOWS diff --git a/source/os/test/osEnvTests.cpp b/source/os/test/osEnvTests.cpp index d5b3cac8a6..6af6f84b68 100644 --- a/source/os/test/osEnvTests.cpp +++ b/source/os/test/osEnvTests.cpp @@ -84,9 +84,8 @@ TEST(osEnvTests, osSufficient) { ret = osTempSpaceSufficient(); EXPECT_EQ(ret, true); - osSetSystemLocale(NULL, NULL); - osSetSystemLocale(NULL, NULL); - osSetSystemLocale("1", "2"); + taosSetSystemLocale(NULL); + taosSetSystemLocale("1"); osSetProcPath(1, NULL); osSetProcPath(0, (char **)&ret); diff --git a/source/os/test/osStringTests.cpp b/source/os/test/osStringTests.cpp index de07d21959..36f5207346 100644 --- a/source/os/test/osStringTests.cpp +++ b/source/os/test/osStringTests.cpp @@ -154,3 +154,483 @@ TEST(osStringTests, ostsnprintfTests) { EXPECT_EQ(ret, 11); EXPECT_STREQ(buffer, "Float: 3.14"); } +TEST(osStringTests, osStr2Int64) { + int64_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2int64(NULL, &val); + assert(result == TSDB_CODE_INVALID_PARA); + + result = taosStr2int64("123", NULL); + ASSERT_NE(result, 0); + + // 测试无效输入 + result = taosStr2int64("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2int64("", &val); + ASSERT_NE(result, 0); + + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%lld", LLONG_MAX); + result = taosStr2int64(large_num, &val); + assert(result == 0); + assert(val == LLONG_MAX); + + snprintf(large_num, sizeof(large_num), "%lld", LLONG_MIN); + result = taosStr2int64(large_num, &val); + assert(result == 0); + assert(val == LLONG_MIN); + + result = taosStr2int64("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2int64("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2int64("12345", &val); + assert(result == 0); + assert(val == 12345); + + result = taosStr2int64("-12345", &val); + assert(result == 0); + assert(val == -12345); + + result = taosStr2int64("0", &val); + assert(result == 0); + assert(val == 0); + + // 测试带空格的字符串 + result = taosStr2int64(" 12345", &val); + assert(result == 0); + assert(val == 12345); + + result = taosStr2int64("12345 ", &val); + assert(result == 0); + assert(val == 12345); +} +TEST(osStringTests, osStr2int32) { + int32_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2int32(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + result = taosStr2int32("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + // 测试无效输入 + result = taosStr2int32("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2int32("", &val); + ASSERT_NE(result, 0); + + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%d", INT_MAX); + result = taosStr2int32(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT_MAX); + + snprintf(large_num, sizeof(large_num), "%d", INT_MIN); + result = taosStr2int32(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT_MIN); + + // 测试大于 INT32 范围的值 + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT_MAX + 1); + result = taosStr2int32(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT_MIN - 1); + result = taosStr2int32(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + + result = taosStr2int32("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2int32("abc123", &val); + ASSERT_NE(result, 0); + + // 测试有效的整数字符串 + result = taosStr2int32("12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2int32("-12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, -12345); + + result = taosStr2int32("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); + + // 测试带空格的字符串 + result = taosStr2int32(" 12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2int32("12345 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); +} + +TEST(osStringTests, taosStr2int16) { + int16_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2int16(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + result = taosStr2int16("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + // 测试无效输入 + result = taosStr2int16("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2int16("", &val); + ASSERT_NE(result, 0); + + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%d", INT16_MAX); + result = taosStr2int16(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT16_MAX); + + snprintf(large_num, sizeof(large_num), "%d", INT16_MIN); + result = taosStr2int16(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT16_MIN); + + // 测试大于 INT16 范围的值 + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT16_MAX + 1); + result = taosStr2int16(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT16_MIN - 1); + result = taosStr2int16(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + result = taosStr2int16("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2int16("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2int16("12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2int16("-12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, -12345); + + result = taosStr2int16("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); + + // 测试带空格的字符串 + result = taosStr2int16(" 12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2int16("12345 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); +} + + +TEST(osStringTests, taosStr2int8) { + int8_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2int8(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + result = taosStr2int8("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + // 测试无效输入 + result = taosStr2int8("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2int8("", &val); + ASSERT_NE(result, 0); + + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%d", INT8_MAX); + result = taosStr2int8(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT8_MAX); + + snprintf(large_num, sizeof(large_num), "%d", INT8_MIN); + result = taosStr2int8(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT8_MIN); + + // 测试大于 INT8 范围的值 + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT8_MAX + 1); + result = taosStr2int8(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT8_MIN - 1); + result = taosStr2int8(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + result = taosStr2int8("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2int8("abc123", &val); + ASSERT_NE(result, 0); + + // 测试有效的整数字符串 + result = taosStr2int8("123", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2int8("-123", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, -123); + + result = taosStr2int8("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); + + // 测试带空格的字符串 + result = taosStr2int8(" 123", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2int8("123 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); +} + +TEST(osStringTests, osStr2Uint64) { + uint64_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2Uint64(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + result = taosStr2Uint64("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + // 测试无效输入 + result = taosStr2Uint64("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2Uint64("", &val); + ASSERT_NE(result, 0); + + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%llu", ULLONG_MAX); + result = taosStr2Uint64(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, ULLONG_MAX); + + result = taosStr2Uint64("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2Uint64("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2Uint64("12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2Uint64("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); + + // 测试带空格的字符串 + result = taosStr2Uint64(" 12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2Uint64("12345 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + +} + +TEST(osStringTests, taosStr2Uint32) { + uint32_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2Uint32(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + result = taosStr2Uint32("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + // 测试无效输入 + result = taosStr2Uint32("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2Uint32("", &val); + ASSERT_NE(result, 0); + + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%u", UINT32_MAX); + result = taosStr2Uint32(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, UINT32_MAX); + + // 测试大于 UINT32 范围的值 + snprintf(large_num, sizeof(large_num), "%llu", (unsigned long long)UINT32_MAX + 1); + result = taosStr2Uint32(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + result = taosStr2Uint32("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2Uint32("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2Uint32("12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2Uint32("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); + + // 测试带空格的字符串 + result = taosStr2Uint32(" 12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2Uint32("12345 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); +} + +TEST(osStringTests, taosStr2Uint16) { + uint16_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2Uint16(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + result = taosStr2Uint16("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + // 测试无效输入 + result = taosStr2Uint16("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2Uint16("", &val); + ASSERT_NE(result, 0); + + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%u", UINT16_MAX); + result = taosStr2Uint16(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, UINT16_MAX); + + // 测试大于 UINT16 范围的值 + snprintf(large_num, sizeof(large_num), "%llu", (unsigned long long)UINT16_MAX + 1); + result = taosStr2Uint16(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + result = taosStr2Uint16("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2Uint16("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2Uint16("12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2Uint16("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); + + // 测试带空格的字符串 + result = taosStr2Uint16(" 12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2Uint16("12345 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); +} + +TEST(osStringTests, taosStr2Uint8) { + uint8_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2Uint8(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + result = taosStr2Uint8("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + // 测试无效输入 + result = taosStr2Uint8("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2Uint8("", &val); + ASSERT_NE(result, 0); + + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%u", UINT8_MAX); + result = taosStr2Uint8(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, UINT8_MAX); + + // 测试大于 UINT8 范围的值 + snprintf(large_num, sizeof(large_num), "%llu", (unsigned long long)UINT8_MAX + 1); + result = taosStr2Uint8(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + result = taosStr2Uint8("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2Uint8("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2Uint8("123", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2Uint8("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); + + // 测试带空格的字符串 + result = taosStr2Uint8(" 123", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2Uint8("123 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); +} + diff --git a/source/util/inc/tmempoolInt.h b/source/util/inc/tmempoolInt.h new file mode 100755 index 0000000000..8d4cb57ddc --- /dev/null +++ b/source/util/inc/tmempoolInt.h @@ -0,0 +1,529 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MEMPOOL_INT_H_ +#define _TD_MEMPOOL_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" +#include "tglobal.h" +#include "thash.h" +#include "tlockfree.h" + +#define MP_CHUNK_CACHE_ALLOC_BATCH_SIZE 1000 +#define MP_NSCHUNK_CACHE_ALLOC_BATCH_SIZE 500 +#define MP_SESSION_CACHE_ALLOC_BATCH_SIZE 100 + +#define MP_MAX_KEEP_FREE_CHUNK_NUM 1000 +#define MP_MAX_MALLOC_MEM_SIZE 0xFFFFFFFFFF + +#define MP_DEFAULT_MEM_CHK_INTERVAL_MS 10 +#define MP_MIN_MEM_CHK_INTERVAL_MS 1 +#define MP_MEMORY_TRIM_INTERVAL_TIMES 500 + +#define MP_RETIRE_HIGH_THRESHOLD_PERCENT (0.95) +#define MP_RETIRE_MID_THRESHOLD_PERCENT (0.9) +#define MP_RETIRE_LOW_THRESHOLD_PERCENT (0.85) +#define MP_RETIRE_UNIT_PERCENT (0.1) +#define MP_RETIRE_UNIT_MIN_SIZE (50 * 1048576L) +#define MP_CFG_UPDATE_MIN_RESERVE_SIZE (50 * 1024 * 1048576L) + +#define MP_DEFAULT_RESERVE_MEM_PERCENT 20 +#define MP_MIN_FREE_SIZE_AFTER_RESERVE (4 * 1024 * 1048576L) +#define MP_MIN_MEM_POOL_SIZE (5 * 1024 * 1048576L) + +// FLAGS AREA +#define MP_CHUNK_FLAG_IN_USE (1 << 0) +#define MP_CHUNK_FLAG_NS_CHUNK (1 << 1) + +// STAT FLAGS +#define MP_LOG_FLAG_ALL_MEM (1 << 0) +#define MP_LOG_FLAG_ALL_CHUNK (1 << 1) +#define MP_LOG_FLAG_ALL_POS (1 << 2) +#define MP_LOG_FLAG_ALL_SESSION (1 << 3) +#define MP_LOG_FLAG_ALL_NODE (1 << 4) +#define MP_LOG_FLAG_ALL_POOL (1 << 5) + +#define MP_LOG_FLAG_SOME_POS (1 << 6) +#define MP_LOG_FLAG_SOME_SESSION (1 << 7) +#define MP_LOG_FLAG_SOME_NODE (1 << 8) +#define MP_LOG_FLAG_SOME_POOL (1 << 9) + +#define MP_STAT_FLAG_LOG_ALL (0xFFFFFFFFFFFFFFFF) + +// STAT PROCESURE FLAGS +#define MP_STAT_PROC_FLAG_EXEC (1 << 0) +#define MP_STAT_PROC_FLAG_INPUT_ERR (1 << 1) +#define MP_STAT_PROC_FLAG_RES_SUCC (1 << 2) +#define MP_STAT_PROC_FLAG_RES_FAIL (1 << 3) + +// CTRL FUNC FLAGS +#define MP_CTRL_FLAG_PRINT_STAT (1 << 0) +#define MP_CTRL_FLAG_CHECK_STAT (1 << 1) +#define MP_CTRL_FLAG_LOCK_DBG (1 << 2) +#define MP_CTRL_FLAG_LOG_MAXSIZE (1 << 3) + +typedef enum EMPStatLogItem { + E_MP_STAT_LOG_MEM_MALLOC = 1, + E_MP_STAT_LOG_MEM_CALLOC, + E_MP_STAT_LOG_MEM_REALLOC, + E_MP_STAT_LOG_MEM_FREE, + E_MP_STAT_LOG_MEM_STRDUP, + E_MP_STAT_LOG_MEM_STRNDUP, + E_MP_STAT_LOG_MEM_TRIM, + + E_MP_STAT_LOG_CHUNK_MALLOC, + E_MP_STAT_LOG_CHUNK_RECYCLE, + E_MP_STAT_LOG_CHUNK_REUSE, + E_MP_STAT_LOG_CHUNK_FREE, +} EMPStatLogItem; + +// MEM HEADER FLAGS +#define MP_MEM_HEADER_FLAG_NS_CHUNK (1 << 0) + +typedef struct SMPMemHeader { + uint64_t flags : 24; + uint64_t size : 40; +} SMPMemHeader; + +typedef struct SMPMemTailer { + uint8_t tail; +} SMPMemTailer; + +typedef struct SMPListNode { + void* pNext; +} SMPListNode; + +typedef struct SMPChunk { + SMPListNode list; + char* pMemStart; + int32_t flags; + /* KEEP ABOVE SAME WITH SMPNSChunk */ + + uint32_t offset; +} SMPChunk; + +typedef struct SMPNSChunk { + SMPListNode list; + char* pMemStart; + int32_t flags; + /* KEEP ABOVE SAME WITH SMPChunk */ + + uint64_t offset; + uint64_t memBytes; +} SMPNSChunk; + +typedef struct SMPCacheGroup { + int32_t nodesNum; + int32_t idleOffset; + void* pNodes; + void* pNext; +} SMPCacheGroup; + +typedef struct SMPStatInput { + char* file; + int64_t size; + int64_t origSize; + int32_t procFlags; + int32_t line; + void* pMem; + void* pOrigMem; +} SMPStatInput; + +typedef struct SMPCtrlInfo { + int64_t statFlags; + int64_t funcFlags; +} SMPCtrlInfo; + +typedef struct SMPStatSession { + int64_t initSucc; + int64_t initFail; + int64_t destroyNum; +} SMPStatSession; + +typedef struct SMPAllocStat { + int64_t allocTimes; + int64_t allocBytes; + // int64_t freeIDs[]; // TODO +} SMPAllocStat; + +typedef struct SMPFreeStat { + int64_t freeTimes; + int64_t freeBytes; +} SMPFreeStat; + +typedef struct SMPFileLineId { + uint32_t fileId; + int32_t line; +} SMPFileLineId; + +typedef struct SMPFileLine { + SMPFileLineId fl; + int64_t size; +} SMPFileLine; + +typedef struct SMPStatPos { + int64_t logErrTimes; + SHashObj* fileHash; // fileId => fileName + SHashObj* remainHash; // pointer => SMPFileLine + SHashObj* allocHash; // alloc fl => SMPAllocStat + SHashObj* freeHash; // free fl => SMPFreeStat +} SMPStatPos; + +typedef struct SMPStatInfo { + SMPStatDetail statDetail; + SMPStatSession statSession; + SHashObj* sessStat; + SHashObj* nodeStat; + SMPStatPos posStat; +} SMPStatInfo; + +typedef struct SMPJob { + SMemPoolJob job; // KEEP IT FIRST + SMPStatInfo stat; +} SMPJob; + +typedef struct SMPSessionChunk { + int64_t allocChunkNum; + int64_t allocChunkMemSize; + int64_t reUseChunkNum; + + int32_t srcChunkNum; + SMPChunk* srcChunkHead; + SMPChunk* srcChunkTail; + + int32_t inUseChunkNum; + SMPChunk* inUseChunkHead; + SMPChunk* inUseChunkTail; + + SMPNSChunk* inUseNSChunkHead; + SMPNSChunk* inUseNSChunkTail; + + SMPChunk* reUseChunkHead; + SMPChunk* reUseChunkTail; + + SMPNSChunk* reUseNSChunkHead; + SMPNSChunk* reUseNSChunkTail; +} SMPSessionChunk; + +typedef struct SMPSession { + // SMPListNode list; + + char* sessionId; + SMPJob* pJob; + SMPCtrlInfo ctrl; + int64_t allocMemSize; + int64_t maxAllocMemSize; + + // SMPSessionChunk chunk; + + SMPStatInfo stat; +} SMPSession; + +typedef struct SMPCacheGroupInfo { + int16_t nodeSize; + int64_t allocNum; + int32_t groupNum; + SMPCacheGroup* pGrpHead; + void* pIdleList; +} SMPCacheGroupInfo; + +typedef struct SMPChunkMgmt { + int32_t maxChunkNum; + int16_t maxDiscardSize; + double threadChunkReserveNum; + int64_t allocChunkNum; + int64_t allocChunkSize; + int64_t allocNSChunkNum; + int64_t allocNSChunkSize; + + SMPCacheGroupInfo chunkCache; + SMPCacheGroupInfo NSChunkCache; + + int32_t readyChunkNum; + int32_t readyChunkReserveNum; + int32_t readyChunkLowNum; + int32_t readyChunkGotNum; + SRWLatch readyChunkLock; + SMPChunk* readyChunkHead; + SMPChunk* readyChunkTail; + + int64_t readyNSChunkNum; + SMPChunk* readyNSChunkHead; + SMPChunk* readyNSChunkTail; +} SMPChunkMgmt; + +typedef struct SMemPool { + char* name; + int16_t slotId; + SRWLatch cfgLock; + SMemPoolCfg cfg; + // int64_t retireThreshold[3]; + int64_t retireUnit; + + int64_t maxAllocMemSize; + int64_t allocMemSize; + + SMPCacheGroupInfo sessionCache; + + SMPChunkMgmt chunk; + + SMPStatInfo stat; +} SMemPool; + +typedef enum EMPMemStrategy { + E_MP_STRATEGY_DIRECT = 1, + E_MP_STRATEGY_CHUNK, +} EMPMemStrategy; + +typedef struct SMPMsgQueue { + SMemPool* pPool; + int8_t lowLevelRetire; + int8_t midLevelRetire; +} SMPMsgQueue; + +typedef struct SMemPoolMgmt { + EMPMemStrategy strategy; + SMPCtrlInfo ctrl; + SArray* poolList; + SRWLatch poolLock; + TdThread poolMgmtThread; + SMPMsgQueue msgQueue; + tsem2_t threadSem; + int8_t modExit; + int64_t waitMs; + int32_t code; + int8_t needTrim; +} SMemPoolMgmt; + +extern SMemPoolMgmt gMPMgmt; + +typedef int32_t (*mpAllocFunc)(SMemPool*, SMPSession*, int64_t*, uint32_t, void**); +typedef void (*mpFreeFunc)(SMemPool*, SMPSession*, void*, int64_t*); +typedef int64_t (*mpGetSizeFunc)(SMemPool*, SMPSession*, void*); +typedef int32_t (*mpReallocFunc)(SMemPool*, SMPSession*, void**, int64_t*, int64_t*); +typedef int32_t (*mpInitSessionFunc)(SMemPool*, SMPSession*); +typedef int32_t (*mpInitFunc)(SMemPool*, char*, SMemPoolCfg*); +typedef int32_t (*mpUpdateCfgFunc)(SMemPool*); +typedef int32_t (*mpTrimFunc)(SMemPool*, SMPSession*, int32_t, bool*); + +typedef struct SMPStrategyFp { + mpInitFunc initFp; + mpAllocFunc allocFp; + mpFreeFunc freeFp; + mpGetSizeFunc getSizeFp; + mpReallocFunc reallocFp; + mpInitSessionFunc initSessionFp; + mpUpdateCfgFunc updateCfgFp; + mpTrimFunc trimFp; +} SMPStrategyFp; + +#define MP_GET_FLAG(st, f) ((st) & (f)) +#define MP_SET_FLAG(st, f) (st) |= (f) +#define MP_CLR_FLAG(st, f) (st) &= (~f) + +enum { + MP_READ = 1, + MP_WRITE, +}; + +#define MP_STAT_FORMAT "%-8s => inErr:%10" PRId64 ", exec:%12" PRId64 ", succ:%12" PRId64 ", fail:%12" PRId64 +#define MP_STAT_ORIG_FORMAT \ + "%-8s => inErr:%10" PRId64 ", exec:%12" PRId64 ", succ:%12" PRId64 ", fail:%12" PRId64 ", oExec:%12" PRId64 \ + ", oSucc:%12" PRId64 ", oFail:%12" PRId64 + +#define MP_STAT_VALUE(_name, _item) _name, (_item).inErr, (_item).exec, (_item).succ, (_item).fail +#define MP_STAT_ORIG_VALUE(_name, _item) \ + _name, (_item).inErr, (_item).exec, (_item).succ, (_item).fail, (_item).origExec, (_item).origSucc, (_item).origFail + +#define MP_INIT_MEM_HEADER(_header, _size, _nsChunk) \ + do { \ + (_header)->size = _size; \ + if (_nsChunk) { \ + MP_SET_FLAG((_header)->flags, MP_MEM_HEADER_FLAG_NS_CHUNK); \ + } \ + } while (0) + +#define MP_ADD_TO_CHUNK_LIST(_chunkHead, _chunkTail, _chunkNum, _chunk) \ + do { \ + if (NULL == _chunkHead) { \ + _chunkHead = _chunk; \ + _chunkTail = _chunk; \ + } else { \ + (_chunkTail)->list.pNext = _chunk; \ + (_chunkTail) = _chunk; \ + } \ + (_chunkNum)++; \ + } while (0) + +#define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 + +#define MP_TRY_LOCK(type, _lock, _res) \ + do { \ + if (MP_READ == (type)) { \ + if (MP_GET_FLAG(gMPMgmt.ctrl.funcFlags, MP_CTRL_FLAG_LOCK_DBG)) { \ + uDebug("MP TRY RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + } \ + (_res) = taosRTryLockLatch(_lock); \ + if (MP_GET_FLAG(gMPMgmt.ctrl.funcFlags, MP_CTRL_FLAG_LOCK_DBG)) { \ + uDebug("MP TRY RLOCK%p:%d %s, %s:%d E", (_lock), atomic_load_32(_lock), (_res) ? "failed" : "succeed", \ + __FILE__, __LINE__); \ + } \ + } else { \ + if (MP_GET_FLAG(gMPMgmt.ctrl.funcFlags, MP_CTRL_FLAG_LOCK_DBG)) { \ + uDebug("MP TRY WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + } \ + (_res) = taosWTryLockLatch(_lock); \ + if (MP_GET_FLAG(gMPMgmt.ctrl.funcFlags, MP_CTRL_FLAG_LOCK_DBG)) { \ + uDebug("MP TRY WLOCK%p:%d %s, %s:%d E", (_lock), atomic_load_32(_lock), (_res) ? "failed" : "succeed", \ + __FILE__, __LINE__); \ + } \ + } \ + } while (0) + +#define MP_LOCK(type, _lock) \ + do { \ + if (MP_READ == (type)) { \ + if (MP_GET_FLAG(gMPMgmt.ctrl.funcFlags, MP_CTRL_FLAG_LOCK_DBG)) { \ + uDebug("MP RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + } \ + taosRLockLatch(_lock); \ + if (MP_GET_FLAG(gMPMgmt.ctrl.funcFlags, MP_CTRL_FLAG_LOCK_DBG)) { \ + uDebug("MP RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + } \ + } else { \ + if (MP_GET_FLAG(gMPMgmt.ctrl.funcFlags, MP_CTRL_FLAG_LOCK_DBG)) { \ + uDebug("MP WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + } \ + taosWLockLatch(_lock); \ + if (MP_GET_FLAG(gMPMgmt.ctrl.funcFlags, MP_CTRL_FLAG_LOCK_DBG)) { \ + uDebug("MP WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + } \ + } \ + } while (0) + +#define MP_UNLOCK(type, _lock) \ + do { \ + if (MP_READ == (type)) { \ + if (MP_GET_FLAG(gMPMgmt.ctrl.funcFlags, MP_CTRL_FLAG_LOCK_DBG)) { \ + uDebug("MP RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + } \ + taosRUnLockLatch(_lock); \ + if (MP_GET_FLAG(gMPMgmt.ctrl.funcFlags, MP_CTRL_FLAG_LOCK_DBG)) { \ + uDebug("MP RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + } \ + } else { \ + if (MP_GET_FLAG(gMPMgmt.ctrl.funcFlags, MP_CTRL_FLAG_LOCK_DBG)) { \ + uDebug("MP WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + } \ + taosWUnLockLatch(_lock); \ + if (MP_GET_FLAG(gMPMgmt.ctrl.funcFlags, MP_CTRL_FLAG_LOCK_DBG)) { \ + uDebug("MP WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + } \ + } \ + } while (0) + +#define MP_ERR_RET(c) \ + do { \ + int32_t _code = c; \ + if (_code != TSDB_CODE_SUCCESS) { \ + terrno = _code; \ + return _code; \ + } \ + } while (0) + +#define MP_RET(c) \ + do { \ + int32_t _code = c; \ + if (_code != TSDB_CODE_SUCCESS) { \ + terrno = _code; \ + } \ + return _code; \ + } while (0) + +#define MP_ERR_JRET(c) \ + do { \ + code = c; \ + if (code != TSDB_CODE_SUCCESS) { \ + terrno = code; \ + goto _return; \ + } \ + } while (0) + +#define MP_CHECK_QUOTA(_pool, _job, _size) \ + do { \ + if (*(_pool)->cfg.jobQuota > 0) { \ + int64_t cAllocSize = atomic_add_fetch_64(&(_job)->job.allocMemSize, (_size)); \ + if (cAllocSize > (*(_pool)->cfg.jobQuota * 1048576L)) { \ + uWarn("job 0x%" PRIx64 " remainSession:%d allocSize %" PRId64 " is over than quota %dMB", (_job)->job.jobId, \ + (_job)->job.remainSession, cAllocSize, *(_pool)->cfg.jobQuota); \ + (_pool)->cfg.cb.reachFp(pJob->job.jobId, (_job)->job.clientId, TSDB_CODE_QRY_REACH_QMEM_THRESHOLD); \ + mpSchedTrim(NULL); \ + terrno = TSDB_CODE_QRY_REACH_QMEM_THRESHOLD; \ + return NULL; \ + } else { \ + uDebug("job 0x%" PRIx64 " remainSession:%d allocSize %" PRId64 " is lower than quota %dMB", (_job)->job.jobId, \ + (_job)->job.remainSession, cAllocSize, *(_pool)->cfg.jobQuota); \ + } \ + } \ + if (atomic_load_64(&tsCurrentAvailMemorySize) <= ((_pool)->cfg.reserveSize + (_size))) { \ + uWarn("%s pool sysAvailMemSize %" PRId64 " can't alloc %" PRId64 " while keeping reserveSize %" PRId64 " bytes", \ + (_pool)->name, atomic_load_64(&tsCurrentAvailMemorySize), (_size), (_pool)->cfg.reserveSize); \ + (_pool)->cfg.cb.reachFp((_job)->job.jobId, (_job)->job.clientId, TSDB_CODE_QRY_QUERY_MEM_EXHAUSTED); \ + mpSchedTrim(NULL); \ + terrno = TSDB_CODE_QRY_QUERY_MEM_EXHAUSTED; \ + return NULL; \ + } \ + } while (0) + +// direct +void* mpDirectAlloc(SMemPool* pPool, SMPJob* pJob, int64_t size); +void* mpDirectAlignAlloc(SMemPool* pPool, SMPJob* pJob, uint32_t alignment, int64_t size); +void* mpDirectCalloc(SMemPool* pPool, SMPJob* pJob, int64_t num, int64_t size); +void mpDirectFree(SMemPool* pPool, SMPJob* pJob, void* ptr); +void* mpDirectRealloc(SMemPool* pPool, SMPJob* pJob, void* ptr, int64_t size); +void* mpDirectStrdup(SMemPool* pPool, SMPJob* pJob, const void* ptr); +void* mpDirectStrndup(SMemPool* pPool, SMPJob* pJob, const void* ptr, int64_t size); + +int32_t mpDirectFullAlloc(SMemPool* pPool, SMPSession* pSession, int64_t* size, uint32_t alignment, void** ppRes); +int64_t mpDirectGetMemSize(SMemPool* pPool, SMPSession* pSession, void* ptr); +void mpDirectFullFree(SMemPool* pPool, SMPSession* pSession, void* ptr, int64_t* origSize); +int32_t mpDirectFullRealloc(SMemPool* pPool, SMPSession* pSession, void** pPtr, int64_t* size, int64_t* origSize); +int32_t mpDirectTrim(SMemPool* pPool, SMPSession* pSession, int32_t size, bool* trimed); + +// chunk +int32_t mpChunkInit(SMemPool* pPool, char* poolName, SMemPoolCfg* cfg); +int64_t mpChunkGetMemSize(SMemPool* pPool, SMPSession* pSession, void* ptr); +int32_t mpChunkAlloc(SMemPool* pPool, SMPSession* pSession, int64_t* size, uint32_t alignment, void** ppRes); +void mpChunkFree(SMemPool* pPool, SMPSession* pSession, void* ptr, int64_t* origSize); +int32_t mpChunkRealloc(SMemPool* pPool, SMPSession* pSession, void** pPtr, int64_t* size, int64_t* origSize); +int32_t mpChunkInitSession(SMemPool* pPool, SMPSession* pSession); +int32_t mpChunkUpdateCfg(SMemPool* pPool); + +int32_t mpPopIdleNode(SMemPool* pPool, SMPCacheGroupInfo* pInfo, void** ppRes); +int32_t mpChkFullQuota(SMemPool* pPool, SMPSession* pSession, int64_t size); +void mpUpdateAllocSize(SMemPool* pPool, SMPSession* pSession, int64_t size, int64_t addSize); +int32_t mpAddCacheGroup(SMemPool* pPool, SMPCacheGroupInfo* pInfo, SMPCacheGroup* pHead); +int32_t mpMalloc(SMemPool* pPool, SMPSession* pSession, int64_t* size, uint32_t alignment, void** ppRes); +void mpSchedTrim(int64_t* loopTimes); + +#ifdef __cplusplus +} +#endif + +#endif /* _TD_MEMPOOL_INT_H_ */ diff --git a/source/util/src/mpChunk.c b/source/util/src/mpChunk.c new file mode 100755 index 0000000000..2c1c415c04 --- /dev/null +++ b/source/util/src/mpChunk.c @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "osMemPool.h" +#include "tmempoolInt.h" +#include "tlog.h" +#include "tutil.h" + +#if 0 +int32_t mpChunkNew(SMemPool* pPool, SMPChunk** ppChunk) { + SMPChunk* pChunk = NULL; + MP_ERR_RET(mpPopIdleNode(pPool, &pPool->chunk.chunkCache, (void**)&pChunk)); + + pChunk->pMemStart = taosMemMalloc(pPool->cfg.chunkSize); + if (NULL == pChunk->pMemStart) { + uError("add new chunk, memory malloc %d failed, code: 0x%x", pPool->cfg.chunkSize, terrno); + return terrno; + } + + pPool->chunk.allocChunkNum++; + pPool->chunk.allocChunkSize += pPool->cfg.chunkSize; + + *ppChunk = pChunk; + + return TSDB_CODE_SUCCESS; +} + + +int32_t mpChunkNewNS(SMemPool* pPool, SMPNSChunk** ppChunk, int64_t chunkSize) { + SMPNSChunk* pChunk = NULL; + MP_ERR_RET(mpPopIdleNode(pPool, &pPool->chunk.NSChunkCache, (void**)&pChunk)); + + pChunk->pMemStart = taosMemMalloc(chunkSize); + if (NULL == pChunk->pMemStart) { + uError("add new NS chunk, memory malloc %" PRId64 " failed, code: 0x%x", chunkSize, terrno); + return terrno; + } + + pChunk->memBytes = chunkSize; + MP_SET_FLAG(pChunk->flags, MP_CHUNK_FLAG_NS_CHUNK); + + pPool->chunk.allocNSChunkNum++; + pPool->chunk.allocNSChunkSize += pPool->cfg.chunkSize; + + *ppChunk = pChunk; + + return TSDB_CODE_SUCCESS; +} + + +int32_t mpChunkPrepare(SMemPool* pPool, int32_t num) { + SMPChunk* pChunk = NULL; + for (int32_t i = 0; i < num; ++i) { + MP_ERR_RET(mpChunkNew(pPool, &pChunk)); + + pPool->chunk.readyChunkTail->list.pNext = pChunk; + pPool->chunk.readyChunkTail = pChunk; + + atomic_add_fetch_32(&pPool->chunk.readyChunkNum, 1); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t mpChunkEnsureCapacity(SMemPool* pPool, SMPChunkMgmt* pChunk) { + if (E_EVICT_ALL == pPool->cfg.evicPolicy) { + return TSDB_CODE_SUCCESS; + } + + int32_t readyMissNum = pChunk->readyChunkReserveNum - atomic_load_32(&pChunk->readyChunkNum); + if (readyMissNum <= 0) { + return TSDB_CODE_SUCCESS; + } + + MP_ERR_RET(mpChunkPrepare(pPool, readyMissNum)); + + return TSDB_CODE_SUCCESS; +} + +void mpChunkNotifyLowNum(SMemPool* pPool) { + +} + +int32_t mpChunkRetrieve(SMemPool* pPool, SMPChunk** ppChunk) { + SMPCacheGroup* pCache = NULL; + SMPChunk* pChunk = NULL; + int32_t readyChunkNum = atomic_sub_fetch_32(&pPool->chunk.readyChunkNum, 1); + if (readyChunkNum >= 0) { + if (atomic_add_fetch_32(&pPool->chunk.readyChunkGotNum, 1) == pPool->chunk.readyChunkLowNum) { + mpChunkNotifyLowNum(pPool); + } + + pChunk = (SMPChunk*)atomic_load_ptr(&pPool->chunk.readyChunkHead->list.pNext); + while (atomic_val_compare_exchange_ptr(&pPool->chunk.readyChunkHead->list.pNext, pChunk, pChunk->list.pNext) != pChunk) { + pChunk = (SMPChunk*)atomic_load_ptr(&pPool->chunk.readyChunkHead->list.pNext); + } + + *ppChunk = pChunk; + + return TSDB_CODE_SUCCESS; + } else { + atomic_add_fetch_32(&pPool->chunk.readyChunkNum, 1); + } + + MP_RET(mpChunkNew(pPool, ppChunk)); +} + +int32_t mpChunkRetrieveFromSession(SMemPool* pPool, SMPSession* pSession, int64_t size, SMPChunk** ppChunk, SMPChunk** ppPreChunk) { + SMPChunk* pChunk = pSession->chunk.srcChunkHead; + while (NULL != pChunk) { + if ((pChunk->offset + size) <= pPool->cfg.chunkSize) { + *ppChunk = pChunk; + break; + } + + *ppPreChunk = pChunk; + pChunk = (SMPChunk*)pChunk->list.pNext; + } + + if (NULL == *ppChunk) { + *ppPreChunk = NULL; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t mpChunkAllocMem(SMemPool* pPool, SMPSession* pSession, int64_t size, uint32_t alignment, void** ppRes) { + int32_t code = TSDB_CODE_SUCCESS; + SMPChunk* pChunk = NULL, *preSrcChunk = NULL; + void* pRes = NULL; + int64_t totalSize = size + sizeof(SMPMemHeader) + sizeof(SMPMemTailer); + + if (pSession->chunk.srcChunkNum > 0) { + MP_ERR_JRET(mpChunkRetrieveFromSession(pPool, pSession, totalSize, &pChunk, &preSrcChunk)); + } + + if (NULL == pChunk) { + MP_ERR_JRET(mpChunkNew(pPool, &pChunk)); + + pSession->chunk.allocChunkNum++; + pSession->chunk.allocChunkMemSize += pPool->cfg.chunkSize; + mpUpdateAllocSize(pPool, pSession, totalSize, 0); + + MP_ADD_TO_CHUNK_LIST(pSession->chunk.srcChunkHead, pSession->chunk.srcChunkTail, pSession->chunk.srcChunkNum, pChunk); + MP_ADD_TO_CHUNK_LIST(pSession->chunk.inUseChunkHead, pSession->chunk.inUseChunkTail, pSession->chunk.inUseChunkNum, pChunk); + } + + SMPMemHeader* pHeader = (SMPMemHeader*)(pChunk->pMemStart + pChunk->offset); + MP_INIT_MEM_HEADER(pHeader, size, false); + + pRes = (void*)(pHeader + 1); + pChunk->offset += totalSize; + + if (pChunk->offset >= (pPool->cfg.chunkSize - pPool->chunk.maxDiscardSize)) { + if (NULL == preSrcChunk) { + pSession->chunk.srcChunkHead = NULL; + pSession->chunk.srcChunkTail = NULL; + } else { + preSrcChunk->list.pNext = pChunk->list.pNext; + } + + pSession->chunk.srcChunkNum--; + } + + +_return: + + *ppRes = pRes; + + return code; +} + +int32_t mpChunkNSAllocMem(SMemPool* pPool, SMPSession* pSession, int64_t size, uint32_t alignment, void** ppRes) { + int32_t code = TSDB_CODE_SUCCESS; + SMPNSChunk* pChunk = NULL; + void* pRes = NULL; + int64_t totalSize = size + sizeof(SMPMemHeader) + sizeof(SMPMemTailer) + alignment; + + MP_ERR_JRET(mpChunkNewNS(pPool, &pChunk, totalSize)); + SMPMemHeader* pHeader = (SMPMemHeader*)pChunk->pMemStart; + MP_INIT_MEM_HEADER(pHeader, size, false); + + pRes = (void*)(pHeader + 1); + + pSession->chunk.allocChunkNum++; + pSession->chunk.allocChunkMemSize += totalSize; + mpUpdateAllocSize(pPool, pSession, totalSize, 0); + + if (NULL == pSession->chunk.inUseNSChunkHead) { + pSession->chunk.inUseNSChunkHead = pChunk; + pSession->chunk.inUseNSChunkTail = pChunk; + } else { + pSession->chunk.inUseNSChunkTail->list.pNext = pChunk; + } + +_return: + + *ppRes = pRes; + + return code; +} + + +int32_t mpChunkInit(SMemPool* pPool, char* poolName, SMemPoolCfg* cfg) { + SMPChunkMgmt* pChunk = &pPool->chunk; + pChunk->threadChunkReserveNum = 1; + + pChunk->chunkCache.nodeSize = sizeof(SMPChunk); + pChunk->NSChunkCache.groupNum = MP_NSCHUNK_CACHE_ALLOC_BATCH_SIZE; + pChunk->NSChunkCache.nodeSize = sizeof(SMPNSChunk); + + MP_ERR_RET(mpAddCacheGroup(pPool, &pChunk->chunkCache, NULL)); + MP_ERR_RET(mpAddCacheGroup(pPool, &pChunk->NSChunkCache, NULL)); + + MP_ERR_RET(mpPopIdleNode(pPool, &pChunk->chunkCache, (void**)&pChunk->readyChunkHead)); + pChunk->readyChunkTail = pChunk->readyChunkHead; + + MP_ERR_RET(mpChunkEnsureCapacity(pPool, pChunk)); + + return TSDB_CODE_SUCCESS; +} + + +int64_t mpChunkGetMemSize(SMemPool* pPool, SMPSession* pSession, void *ptr) { + SMPMemHeader* pHeader = (SMPMemHeader*)ptr - 1; + return pHeader->size; +} + +int32_t mpChunkAlloc(SMemPool* pPool, SMPSession* pSession, int64_t* size, uint32_t alignment, void** ppRes) { + MP_RET((*size > pPool->cfg.chunkSize) ? mpChunkNSAllocMem(pPool, pSession, *size, alignment, ppRes) : mpChunkAllocMem(pPool, pSession, *size, alignment, ppRes)); +} + + +void mpChunkFree(SMemPool* pPool, SMPSession* pSession, void *ptr, int64_t* origSize) { + int64_t oSize = mpChunkGetMemSize(pPool, pSession, ptr); + if (origSize) { + *origSize = oSize; + } + + // TODO + + atomic_sub_fetch_64(&pSession->allocMemSize, oSize); + atomic_sub_fetch_64(&pPool->allocMemSize, oSize); +} + + +int32_t mpChunkRealloc(SMemPool* pPool, SMPSession* pSession, void **pPtr, int64_t* size, int64_t* origSize) { + int32_t code = TSDB_CODE_SUCCESS; + + if (*origSize >= *size) { + SMPMemHeader* pHeader = (SMPMemHeader*)((char*)*pPtr - sizeof(SMPMemHeader)); + pHeader->size = *size; + return TSDB_CODE_SUCCESS; + } + + void* res = NULL; + MP_ERR_JRET(mpMalloc(pPool, pSession, size, 0, &res)); + SMPMemHeader* pOrigHeader = (SMPMemHeader*)((char*)*pPtr - sizeof(SMPMemHeader)); + SMPMemHeader* pNewHeader = (SMPMemHeader*)((char*)res - sizeof(SMPMemHeader)); + + TAOS_MEMCPY(res, *pPtr, *origSize); + mpChunkFree(pPool, pSession, *pPtr, NULL); + *pPtr = res; + + return TSDB_CODE_SUCCESS; + +_return: + + mpChunkFree(pPool, pSession, *pPtr, NULL); + + return code; +} + +int32_t mpChunkInitSession(SMemPool* pPool, SMPSession* pSession) { + int32_t code = TSDB_CODE_SUCCESS; + SMPChunk* pChunk = NULL; + + MP_ERR_RET(mpChunkRetrieve(pPool, &pChunk)); + + pSession->chunk.allocChunkNum = 1; + pSession->chunk.allocChunkMemSize = pPool->cfg.chunkSize; + + MP_ADD_TO_CHUNK_LIST(pSession->chunk.srcChunkHead, pSession->chunk.srcChunkTail, pSession->chunk.srcChunkNum, pChunk); + MP_ADD_TO_CHUNK_LIST(pSession->chunk.inUseChunkHead, pSession->chunk.inUseChunkTail, pSession->chunk.inUseChunkNum, pChunk); + + return code; +} + +int32_t mpChunkUpdateCfg(SMemPool* pPool) { +/* + pPool->chunk.maxChunkNum = pPool->cfg.freeSize / pPool->cfg.chunkSize; + if (pPool->chunk.maxChunkNum <= 0) { + uError("invalid memory pool max chunk num, freeSize:%" PRId64 ", chunkSize:%d", pPool->cfg.freeSize, pPool->cfg.chunkSize); + return TSDB_CODE_INVALID_MEM_POOL_PARAM; + } + + pPool->chunk.readyChunkReserveNum = TMIN(pPool->cfg.threadNum * pPool->chunk.threadChunkReserveNum, pPool->chunk.maxChunkNum); + + pPool->chunk.chunkCache.groupNum = TMAX(pPool->chunk.maxChunkNum / 10, MP_CHUNK_CACHE_ALLOC_BATCH_SIZE); +*/ + return TSDB_CODE_SUCCESS; +} + +#endif + diff --git a/source/util/src/mpDirect.c b/source/util/src/mpDirect.c new file mode 100755 index 0000000000..d93f479be1 --- /dev/null +++ b/source/util/src/mpDirect.c @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "osMemPool.h" +#include "tmempoolInt.h" +#include "tlog.h" +#include "tutil.h" + +void* mpDirectAlloc(SMemPool* pPool, SMPJob* pJob, int64_t size) { + MP_CHECK_QUOTA(pPool, pJob, size); + + return taosMemMalloc(size); +} + + +void* mpDirectAlignAlloc(SMemPool* pPool, SMPJob* pJob, uint32_t alignment, int64_t size) { + MP_CHECK_QUOTA(pPool, pJob, size); + + return taosMemMallocAlign(alignment, size); +} + + +void* mpDirectCalloc(SMemPool* pPool, SMPJob* pJob, int64_t num, int64_t size) { + int64_t tSize = num * size; + MP_CHECK_QUOTA(pPool, pJob, tSize); + + return taosMemCalloc(num, size); +} + +void mpDirectFree(SMemPool* pPool, SMPJob* pJob, void *ptr) { + if (*pPool->cfg.jobQuota > 0) { + (void)atomic_sub_fetch_64(&pJob->job.allocMemSize, taosMemSize(ptr)); + } + taosMemFree(ptr); +} + + +void* mpDirectRealloc(SMemPool* pPool, SMPJob* pJob, void* ptr, int64_t size) { + int32_t code = TSDB_CODE_SUCCESS; + + if (NULL == ptr) { + return mpDirectAlloc(pPool, pJob, size); + } + + if (0 == size) { + mpDirectFree(pPool, pJob, ptr); + return NULL; + } + + int64_t oSize = taosMemSize(ptr); + + MP_CHECK_QUOTA(pPool, pJob, size - oSize); + + return taosMemRealloc(ptr, size); +} + +void* mpDirectStrdup(SMemPool* pPool, SMPJob* pJob, const void* ptr) { + if (NULL == ptr) { + return NULL; + } + + int64_t oSize = strlen(ptr); + MP_CHECK_QUOTA(pPool, pJob, oSize); + + return taosStrdupi(ptr); +} + +void* mpDirectStrndup(SMemPool* pPool, SMPJob* pJob, const void* ptr, int64_t size) { + if (NULL == ptr) { + return NULL; + } + + int64_t oSize = strlen(ptr); + MP_CHECK_QUOTA(pPool, pJob, TMIN(oSize, size) + 1); + + return taosStrndupi(ptr, size); +} + + + + +int64_t mpDirectGetMemSize(SMemPool* pPool, SMPSession* pSession, void *ptr) { + return taosMemSize(ptr); +} + +void mpDirectFullFree(SMemPool* pPool, SMPSession* pSession, void *ptr, int64_t* origSize) { + int64_t oSize = taosMemSize(ptr); + if (origSize) { + *origSize = oSize; + } + + MP_LOCK(MP_READ, &pPool->cfgLock); // tmp test + + taosMemFree(ptr); + + if (NULL != pSession) { + (void)atomic_sub_fetch_64(&pSession->allocMemSize, oSize); + (void)atomic_sub_fetch_64(&pSession->pJob->job.allocMemSize, oSize); + } + + (void)atomic_sub_fetch_64(&pPool->allocMemSize, oSize); + + MP_UNLOCK(MP_READ, &pPool->cfgLock); +} + + + +int32_t mpDirectFullAlloc(SMemPool* pPool, SMPSession* pSession, int64_t* size, uint32_t alignment, void** ppRes) { + int32_t code = TSDB_CODE_SUCCESS; + void* res = NULL; + int64_t nSize = *size; + + MP_LOCK(MP_READ, &pPool->cfgLock); + + MP_ERR_JRET(mpChkFullQuota(pPool, pSession, *size)); + + res = alignment ? taosMemMallocAlign(alignment, *size) : taosMemMalloc(*size); + if (NULL != res) { + nSize = taosMemSize(res); + mpUpdateAllocSize(pPool, pSession, nSize, nSize - *size); + } else { + if (NULL != pSession) { + (void)atomic_sub_fetch_64(&pSession->pJob->job.allocMemSize, *size); + } + + (void)atomic_sub_fetch_64(&pPool->allocMemSize, *size); + + uError("malloc %" PRId64 " alignment %d failed, code: 0x%x", *size, alignment, terrno); + + code = terrno; + } + +_return: + + MP_UNLOCK(MP_READ, &pPool->cfgLock); + + *ppRes = res; + *size = nSize; + + MP_RET(code); +} + +int32_t mpDirectFullRealloc(SMemPool* pPool, SMPSession* pSession, void **pPtr, int64_t* size, int64_t* origSize) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t nSize = *size; + + MP_LOCK(MP_READ, &pPool->cfgLock); + + MP_ERR_JRET(mpChkFullQuota(pPool, pSession, *size - *origSize)); + + *pPtr = taosMemRealloc(*pPtr, *size); + if (NULL != *pPtr) { + nSize = taosMemSize(*pPtr); + mpUpdateAllocSize(pPool, pSession, nSize - *origSize, nSize - *size + *origSize); + } else { + MP_ERR_JRET(terrno); + } + +_return: + + MP_UNLOCK(MP_READ, &pPool->cfgLock); + + if (code) { + mpDirectFullFree(pPool, pSession, *pPtr, origSize); + *pPtr = NULL; + } + + *size = nSize; + + return TSDB_CODE_SUCCESS; +} + +int32_t mpDirectTrim(SMemPool* pPool, SMPSession* pSession, int32_t size, bool* trimed) { + return taosMemTrim(size, trimed); +} + + diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index b94bb512e2..b41e186e96 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -394,8 +394,14 @@ void taosArrayClearP(SArray* pArray, void (*fp)(void*)) { // fp(TARRAY_GET_ELEM(pArray, i)); // } if (pArray) { - for (int32_t i = 0; i < pArray->size; i++) { - fp(*(void**)TARRAY_GET_ELEM(pArray, i)); + if (NULL == fp) { + for (int32_t i = 0; i < pArray->size; i++) { + taosMemoryFree(*(void**)TARRAY_GET_ELEM(pArray, i)); + } + } else { + for (int32_t i = 0; i < pArray->size; i++) { + fp(*(void**)TARRAY_GET_ELEM(pArray, i)); + } } } taosArrayClear(pArray); @@ -410,8 +416,14 @@ void taosArrayDestroy(SArray* pArray) { void taosArrayDestroyP(SArray* pArray, FDelete fp) { if (pArray) { - for (int32_t i = 0; i < pArray->size; i++) { - fp(*(void**)TARRAY_GET_ELEM(pArray, i)); + if (NULL == fp) { + for (int32_t i = 0; i < pArray->size; i++) { + taosMemoryFree(*(void**)TARRAY_GET_ELEM(pArray, i)); + } + } else { + for (int32_t i = 0; i < pArray->size; i++) { + fp(*(void**)TARRAY_GET_ELEM(pArray, i)); + } } taosArrayDestroy(pArray); } @@ -566,4 +578,4 @@ void taosArraySwap(SArray* a, SArray* b) { void* data = a->pData; a->pData = b->pData; b->pData = data; -} \ No newline at end of file +} diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index f12f17853a..b91334944d 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -14,9 +14,10 @@ */ #define _DEFAULT_SOURCE -#include "tconfig.h" #include "cJSON.h" #include "taoserror.h" +#include "tconfig.h" +#include "tconv.h" #include "tenv.h" #include "tglobal.h" #include "tgrant.h" @@ -24,14 +25,14 @@ #include "tlog.h" #include "tunit.h" #include "tutil.h" -#include "tconv.h" #define CFG_NAME_PRINT_LEN 32 #define CFG_SRC_PRINT_LEN 12 struct SConfig { ECfgSrcType stype; - SArray *array; + SArray *localArray; + SArray *globalArray; TdThreadMutex lock; }; @@ -40,6 +41,7 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile); int32_t cfgLoadFromEnvVar(SConfig *pConfig); int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd); int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); +int32_t cfgSetItemVal(SConfigItem *pItem, const char *name, const char *value, ECfgSrcType stype); extern char **environ; @@ -49,8 +51,13 @@ int32_t cfgInit(SConfig **ppCfg) { TAOS_RETURN(terrno); } - pCfg->array = taosArrayInit(32, sizeof(SConfigItem)); - if (pCfg->array == NULL) { + pCfg->localArray = taosArrayInit(64, sizeof(SConfigItem)); + if (pCfg->localArray == NULL) { + taosMemoryFree(pCfg); + TAOS_RETURN(terrno); + } + pCfg->globalArray = taosArrayInit(64, sizeof(SConfigItem)); + if (pCfg->globalArray == NULL) { taosMemoryFree(pCfg); TAOS_RETURN(terrno); } @@ -89,6 +96,54 @@ int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs) { return TSDB_CODE_SUCCESS; } +int32_t cfgUpdateFromArray(SConfig *pCfg, SArray *pArgs) { + int32_t size = taosArrayGetSize(pArgs); + for (int32_t i = 0; i < size; ++i) { + SConfigItem *pItemNew = taosArrayGet(pArgs, i); + + (void)taosThreadMutexLock(&pCfg->lock); + + SConfigItem *pItemOld = cfgGetItem(pCfg, pItemNew->name); + if (pItemOld == NULL) { + (void)taosThreadMutexUnlock(&pCfg->lock); + TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); + } + switch (pItemNew->dtype) { + case CFG_DTYPE_BOOL: + pItemOld->bval = pItemNew->bval; + break; + case CFG_DTYPE_INT32: + pItemOld->i32 = pItemNew->i32; + break; + case CFG_DTYPE_INT64: + pItemOld->i64 = pItemNew->i64; + break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: + pItemOld->fval = pItemNew->fval; + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + taosMemoryFree(pItemOld->str); + pItemOld->str = taosStrdup(pItemNew->str); + if (pItemOld->str == NULL) { + (void)taosThreadMutexUnlock(&pCfg->lock); + TAOS_RETURN(terrno); + } + break; + default: + break; + } + + (void)taosThreadMutexUnlock(&pCfg->lock); + } + + return TSDB_CODE_SUCCESS; +} + void cfgItemFreeVal(SConfigItem *pItem) { if (pItem->dtype == CFG_DTYPE_STRING || pItem->dtype == CFG_DTYPE_DIR || pItem->dtype == CFG_DTYPE_LOCALE || pItem->dtype == CFG_DTYPE_CHARSET || pItem->dtype == CFG_DTYPE_TIMEZONE) { @@ -106,19 +161,29 @@ void cfgCleanup(SConfig *pCfg) { return; } - int32_t size = taosArrayGetSize(pCfg->array); + int32_t size = taosArrayGetSize(pCfg->localArray); for (int32_t i = 0; i < size; ++i) { - SConfigItem *pItem = taosArrayGet(pCfg->array, i); + SConfigItem *pItem = taosArrayGet(pCfg->localArray, i); cfgItemFreeVal(pItem); taosMemoryFreeClear(pItem->name); } - taosArrayDestroy(pCfg->array); + size = taosArrayGetSize(pCfg->globalArray); + for (int32_t i = 0; i < size; ++i) { + SConfigItem *pItem = taosArrayGet(pCfg->globalArray, i); + cfgItemFreeVal(pItem); + taosMemoryFreeClear(pItem->name); + } + + taosArrayDestroy(pCfg->localArray); + taosArrayDestroy(pCfg->globalArray); (void)taosThreadMutexDestroy(&pCfg->lock); taosMemoryFree(pCfg); } -int32_t cfgGetSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->array); } +int32_t cfgGetSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->localArray) + taosArrayGetSize(pCfg->globalArray); } +int32_t cfgGetLocalSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->localArray); } +int32_t cfgGetGlobalSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->globalArray); } static int32_t cfgCheckAndSetConf(SConfigItem *pItem, const char *conf) { cfgItemFreeVal(pItem); @@ -152,11 +217,13 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { } static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - bool tmp = false; + int32_t code = 0; + bool tmp = false; if (strcasecmp(value, "true") == 0) { tmp = true; } - if (atoi(value) > 0) { + int32_t val = 0; + if ((code = taosStr2int32(value, &val)) == 0 && val > 0) { tmp = true; } @@ -246,7 +313,7 @@ static int32_t doSetConf(SConfigItem *pItem, const char *value, ECfgSrcType styp } static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - if (stype == CFG_STYPE_ALTER_SERVER_CMD || (pItem->dynScope & CFG_DYN_CLIENT) == 0){ + if (stype == CFG_STYPE_ALTER_SERVER_CMD || (pItem->dynScope & CFG_DYN_CLIENT) == 0) { uError("failed to config timezone, not support"); TAOS_RETURN(TSDB_CODE_INVALID_CFG); } @@ -264,7 +331,7 @@ static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType } static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - if (stype == CFG_STYPE_ALTER_SERVER_CMD || stype == CFG_STYPE_ALTER_CLIENT_CMD){ + if (stype == CFG_STYPE_ALTER_SERVER_CMD || stype == CFG_STYPE_ALTER_CLIENT_CMD) { uError("failed to config charset, not support"); TAOS_RETURN(TSDB_CODE_INVALID_CFG); } @@ -290,7 +357,7 @@ static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType } static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - if (stype == CFG_STYPE_ALTER_SERVER_CMD || (pItem->dynScope & CFG_DYN_CLIENT) == 0){ + if (stype == CFG_STYPE_ALTER_SERVER_CMD || (pItem->dynScope & CFG_DYN_CLIENT) == 0) { uError("failed to config locale, not support"); TAOS_RETURN(TSDB_CODE_INVALID_CFG); } @@ -308,6 +375,7 @@ static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType s static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, const char *level, const char *primary, const char *disable, ECfgSrcType stype) { + int32_t code = 0; (void)taosThreadMutexLock(&pCfg->lock); SConfigItem *pItem = cfgGetItem(pCfg, name); @@ -328,20 +396,40 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, SDiskCfg cfg = {0}; tstrncpy(cfg.dir, pItem->str, sizeof(cfg.dir)); - cfg.level = level ? atoi(level) : 0; - cfg.primary = primary ? atoi(primary) : 1; - cfg.disable = disable ? atoi(disable) : 0; + + if (level == NULL || strlen(level) == 0) { + cfg.level = 0; + } else { + code = taosStr2int32(level, &cfg.level); + TAOS_CHECK_GOTO(code, NULL, _err); + } + + if (primary == NULL || strlen(primary) == 0) { + cfg.primary = 1; + } else { + code = taosStr2int32(primary, &cfg.primary); + TAOS_CHECK_GOTO(code, NULL, _err); + } + + if (disable == NULL || strlen(disable) == 0) { + cfg.disable = 0; + } else { + code = taosStr2int8(disable, &cfg.disable); + TAOS_CHECK_GOTO(code, NULL, _err); + } void *ret = taosArrayPush(pItem->array, &cfg); if (ret == NULL) { - (void)taosThreadMutexUnlock(&pCfg->lock); - - TAOS_RETURN(terrno); + code = terrno; + TAOS_CHECK_GOTO(code, NULL, _err); } pItem->stype = stype; (void)taosThreadMutexUnlock(&pCfg->lock); TAOS_RETURN(TSDB_CODE_SUCCESS); +_err: + (void)taosThreadMutexUnlock(&pCfg->lock); + TAOS_RETURN(code); } static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool resetArray) { @@ -365,7 +453,7 @@ static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool rese if (pDebugFlagItem == NULL) return -1; if (pDebugFlagItem->array != NULL) { SLogVar logVar = {0}; - (void)strncpy(logVar.name, name, TSDB_LOG_VAR_LEN - 1); + tstrncpy(logVar.name, name, TSDB_LOG_VAR_LEN); if (NULL == taosArrayPush(pDebugFlagItem->array, &logVar)) { TAOS_RETURN(terrno); } @@ -387,6 +475,50 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } + code = cfgSetItemVal(pItem, name, value, stype); + if (code != TSDB_CODE_SUCCESS) { + if (lock) { + (void)taosThreadMutexUnlock(&pCfg->lock); + } + TAOS_RETURN(code); + } + + if (lock) { + (void)taosThreadMutexUnlock(&pCfg->lock); + } + + TAOS_RETURN(code); +} + +int32_t cfgGetAndSetItem(SConfig *pCfg, SConfigItem **pItem, const char *name, const char *value, ECfgSrcType stype, + bool lock) { + // GRANT_CFG_SET; + int32_t code = TSDB_CODE_SUCCESS; + + if (lock) { + (void)taosThreadMutexLock(&pCfg->lock); + } + + *pItem = cfgGetItem(pCfg, name); + if (*pItem == NULL) { + (void)taosThreadMutexUnlock(&pCfg->lock); + TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); + } + TAOS_CHECK_RETURN(cfgSetItemVal(*pItem, name, value, stype)); + + if (lock) { + (void)taosThreadMutexUnlock(&pCfg->lock); + } + + TAOS_RETURN(code); +} + +int32_t cfgSetItemVal(SConfigItem *pItem, const char *name, const char *value, ECfgSrcType stype) { + int32_t code = TSDB_CODE_SUCCESS; + + if (pItem == NULL) { + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } switch (pItem->dtype) { case CFG_DTYPE_BOOL: { code = cfgSetBool(pItem, value, stype); @@ -431,18 +563,21 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy break; } - if (lock) { - (void)taosThreadMutexUnlock(&pCfg->lock); - } - TAOS_RETURN(code); } SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName) { if (pCfg == NULL) return NULL; - int32_t size = taosArrayGetSize(pCfg->array); + int32_t size = taosArrayGetSize(pCfg->localArray); for (int32_t i = 0; i < size; ++i) { - SConfigItem *pItem = taosArrayGet(pCfg->array, i); + SConfigItem *pItem = taosArrayGet(pCfg->localArray, i); + if (strcasecmp(pItem->name, pName) == 0) { + return pItem; + } + } + size = taosArrayGetSize(pCfg->globalArray); + for (int32_t i = 0; i < size; ++i) { + SConfigItem *pItem = taosArrayGet(pCfg->globalArray, i); if (strcasecmp(pItem->name, pName) == 0) { return pItem; } @@ -461,18 +596,42 @@ void cfgLock(SConfig *pCfg) { void cfgUnLock(SConfig *pCfg) { (void)taosThreadMutexUnlock(&pCfg->lock); } -int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer) { - ECfgDynType dynType = isServer ? CFG_DYN_SERVER : CFG_DYN_CLIENT; +int32_t checkItemDyn(SConfigItem *pItem, bool isServer) { + if (pItem->dynScope == CFG_DYN_NONE) { + return TSDB_CODE_SUCCESS; + } + if (isServer) { + if (pItem->dynScope == CFG_DYN_ENT_CLIENT || pItem->dynScope == CFG_DYN_ENT_CLIENT_LAZY) { + return TSDB_CODE_INVALID_CFG; + } + } else { + if (pItem->dynScope == CFG_DYN_ENT_SERVER || pItem->dynScope == CFG_DYN_ENT_SERVER_LAZY) { + return TSDB_CODE_INVALID_CFG; + } + } + return TSDB_CODE_SUCCESS; +} + +int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer, + CfgAlterType alterType) { cfgLock(pCfg); SConfigItem *pItem = cfgGetItem(pCfg, name); - if (!pItem || (pItem->dynScope & dynType) == 0) { - uError("failed to config:%s, not support update this config", name); + if (pItem == NULL) { + cfgUnLock(pCfg); + TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); + } + int32_t code = checkItemDyn(pItem, isServer); + if (code != TSDB_CODE_SUCCESS) { + cfgUnLock(pCfg); + TAOS_RETURN(code); + } + if ((pItem->category == CFG_CATEGORY_GLOBAL) && alterType == CFG_ALTER_DNODE) { + uError("failed to config:%s, not support update global config on only one dnode", name); cfgUnLock(pCfg); TAOS_RETURN(TSDB_CODE_INVALID_CFG); } - switch (pItem->dtype) { case CFG_DTYPE_STRING: { if (strcasecmp(name, "slowLogScope") == 0) { @@ -493,8 +652,9 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p } } break; case CFG_DTYPE_BOOL: { - int32_t ival = (int32_t)atoi(pVal); - if (ival != 0 && ival != 1) { + int32_t ival = 0; + code = taosStr2int32(pVal, &ival); + if (code != 0 || (ival != 0 && ival != 1)) { uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name, cfgDtypeStr(pItem->dtype), ival); cfgUnLock(pCfg); TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); @@ -552,15 +712,18 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p } static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { + SArray *array = pCfg->globalArray; + if (pItem->category == CFG_CATEGORY_LOCAL) array = pCfg->localArray; + pItem->stype = CFG_STYPE_DEFAULT; pItem->name = taosStrdup(name); if (pItem->name == NULL) { TAOS_RETURN(terrno); } - int32_t size = taosArrayGetSize(pCfg->array); + int32_t size = taosArrayGetSize(array); for (int32_t i = 0; i < size; ++i) { - SConfigItem *existItem = taosArrayGet(pCfg->array, i); + SConfigItem *existItem = taosArrayGet(array, i); if (existItem != NULL && strcmp(existItem->name, pItem->name) == 0) { taosMemoryFree(pItem->name); TAOS_RETURN(TSDB_CODE_INVALID_CFG); @@ -571,7 +734,7 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; (void)strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); - if (taosArrayPush(pCfg->array, pItem) == NULL) { + if (taosArrayPush(array, pItem) == NULL) { if (pItem->dtype == CFG_DTYPE_STRING) { taosMemoryFree(pItem->str); } @@ -583,13 +746,26 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { TAOS_RETURN(TSDB_CODE_SUCCESS); } -int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope, int8_t dynScope) { - SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .scope = scope, .dynScope = dynScope}; +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope, int8_t dynScope, int8_t category) { + SConfigItem item = { + .dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .scope = scope, .dynScope = dynScope, .category = category}; + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddInt32Ex(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, + int8_t dynScope, int8_t category) { + SConfigItem item = {.dtype = CFG_DTYPE_INT32, + .i32 = defaultVal, + .imin = minval, + .imax = maxval, + .scope = scope, + .dynScope = dynScope, + .category = category}; return cfgAddItem(pCfg, &item, name); } int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, - int8_t dynScope) { + int8_t dynScope, int8_t category) { if (defaultVal < minval || defaultVal > maxval) { TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } @@ -599,12 +775,13 @@ int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t .imin = minval, .imax = maxval, .scope = scope, - .dynScope = dynScope}; + .dynScope = dynScope, + .category = category}; return cfgAddItem(pCfg, &item, name); } int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, - int8_t dynScope) { + int8_t dynScope, int8_t category) { if (defaultVal < minval || defaultVal > maxval) { TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } @@ -614,12 +791,13 @@ int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t .imin = minval, .imax = maxval, .scope = scope, - .dynScope = dynScope}; + .dynScope = dynScope, + .category = category}; return cfgAddItem(pCfg, &item, name); } int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope, - int8_t dynScope) { + int8_t dynScope, int8_t category) { if (defaultVal < minval || defaultVal > maxval) { TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } @@ -629,12 +807,14 @@ int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float min .fmin = minval, .fmax = maxval, .scope = scope, - .dynScope = dynScope}; + .dynScope = dynScope, + .category = category}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { - SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope, .dynScope = dynScope}; +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope, + int8_t category) { + SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope, .dynScope = dynScope, .category = category}; item.str = taosStrdup(defaultVal); if (item.str == NULL) { TAOS_RETURN(terrno); @@ -642,26 +822,30 @@ int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, in return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { - SConfigItem item = {.dtype = CFG_DTYPE_DIR, .scope = scope, .dynScope = dynScope}; +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope, + int8_t category) { + SConfigItem item = {.dtype = CFG_DTYPE_DIR, .scope = scope, .dynScope = dynScope, .category = category}; TAOS_CHECK_RETURN(cfgCheckAndSetDir(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { - SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .scope = scope, .dynScope = dynScope}; +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope, + int8_t category) { + SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .scope = scope, .dynScope = dynScope, .category = category}; TAOS_CHECK_RETURN(cfgCheckAndSetConf(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { - SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .scope = scope, .dynScope = dynScope}; +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope, + int8_t category) { + SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .scope = scope, .dynScope = dynScope, .category = category}; TAOS_CHECK_RETURN(cfgCheckAndSetConf(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { - SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .scope = scope, .dynScope = dynScope}; +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope, + int8_t category) { + SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .scope = scope, .dynScope = dynScope, .category = category}; TAOS_CHECK_RETURN(cfgCheckAndSetConf(&item, defaultVal)); return cfgAddItem(pCfg, &item, name); } @@ -738,14 +922,14 @@ int32_t cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t case CFG_DTYPE_DOUBLE: len = tsnprintf(buf, bufSize, "%f", pItem->fval); break; - case CFG_DTYPE_TIMEZONE:{ -// char str1[TD_TIMEZONE_LEN] = {0}; -// time_t tx1 = taosGetTimestampSec(); -// if (taosFormatTimezoneStr(tx1, buf, NULL, str1) != 0) { -// tstrncpy(str1, "tz error", sizeof(str1)); -// } -// len = tsnprintf(buf, bufSize, "%s", str1); -// break; + case CFG_DTYPE_TIMEZONE: { + // char str1[TD_TIMEZONE_LEN] = {0}; + // time_t tx1 = taosGetTimestampSec(); + // if (taosFormatTimezoneStr(tx1, buf, NULL, str1) != 0) { + // tstrncpy(str1, "tz error", sizeof(str1)); + // } + // len = tsnprintf(buf, bufSize, "%s", str1); + // break; } case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: @@ -794,23 +978,38 @@ int32_t cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t TAOS_RETURN(TSDB_CODE_SUCCESS); } -void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump) { - if (dump) { - (void)printf(" s3 config"); - (void)printf("\n"); - (void)printf("================================================================="); - (void)printf("\n"); - } else { - uInfo(" s3 config"); - uInfo("================================================================="); +int32_t cfgDumpItemCategory(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen) { + int32_t len = 0; + switch (pItem->category) { + case CFG_CATEGORY_LOCAL: + len = tsnprintf(buf, bufSize, "local"); + break; + case CFG_CATEGORY_GLOBAL: + len = tsnprintf(buf, bufSize, "global"); + break; + default: + uError("invalid category:%d", pItem->category); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); } - char src[CFG_SRC_PRINT_LEN + 1] = {0}; - char name[CFG_NAME_PRINT_LEN + 1] = {0}; + if (len < 0) { + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + } - int32_t size = taosArrayGetSize(pCfg->array); + if (len > bufSize) { + len = bufSize; + } + + *pLen = len; + TAOS_RETURN(TSDB_CODE_SUCCESS); +} + +void cfgDumpCfgArrayS3(SArray *array, bool tsc, bool dump) { + char src[CFG_SRC_PRINT_LEN + 1] = {0}; + char name[CFG_NAME_PRINT_LEN + 1] = {0}; + int32_t size = taosArrayGetSize(array); for (int32_t i = 0; i < size; ++i) { - SConfigItem *pItem = taosArrayGet(pCfg->array, i); + SConfigItem *pItem = taosArrayGet(array, i); if (tsc && pItem->scope == CFG_SCOPE_SERVER) continue; if (dump && strcmp(pItem->name, "scriptDir") == 0) continue; if (dump && strncmp(pItem->name, "s3", 2) != 0) continue; @@ -869,7 +1068,20 @@ void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump) { break; } } +} +void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump) { + if (dump) { + (void)printf(" s3 config"); + (void)printf("\n"); + (void)printf("================================================================="); + (void)printf("\n"); + } else { + uInfo(" s3 config"); + uInfo("================================================================="); + } + cfgDumpCfgArrayS3(pCfg->localArray, tsc, dump); + cfgDumpCfgArrayS3(pCfg->globalArray, tsc, dump); if (dump) { (void)printf("=================================================================\n"); } else { @@ -891,9 +1103,9 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { char src[CFG_SRC_PRINT_LEN + 1] = {0}; char name[CFG_NAME_PRINT_LEN + 1] = {0}; - int32_t size = taosArrayGetSize(pCfg->array); + int32_t size = taosArrayGetSize(pCfg->localArray); for (int32_t i = 0; i < size; ++i) { - SConfigItem *pItem = taosArrayGet(pCfg->array, i); + SConfigItem *pItem = taosArrayGet(pCfg->localArray, i); if (tsc && pItem->scope == CFG_SCOPE_SERVER) continue; if (dump && strcmp(pItem->name, "scriptDir") == 0) continue; tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); @@ -949,9 +1161,10 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { for (size_t j = 0; j < sz; ++j) { SDiskCfg *pCfg = taosArrayGet(pItem->array, j); if (dump) { - (void)printf("%s %s %s l:%d p:%d d:%"PRIi8"\n", src, name, pCfg->dir, pCfg->level, pCfg->primary, pCfg->disable); + (void)printf("%s %s %s l:%d p:%d d:%" PRIi8 "\n", src, name, pCfg->dir, pCfg->level, pCfg->primary, + pCfg->disable); } else { - uInfo("%s %s %s l:%d p:%d d:%"PRIi8, src, name, pCfg->dir, pCfg->level, pCfg->primary, pCfg->disable); + uInfo("%s %s %s l:%d p:%d d:%" PRIi8, src, name, pCfg->dir, pCfg->level, pCfg->primary, pCfg->disable); } } break; @@ -986,7 +1199,7 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { name = value = value2 = value3 = value4 = NULL; olen = vlen = vlen2 = vlen3 = vlen4 = 0; - strncpy(line, *pEnv, sizeof(line) - 1); + tstrncpy(line, *pEnv, sizeof(line)); pEnv++; if (taosEnvToCfg(line, line) < 0) { uTrace("failed to convert env to cfg:%s", line); @@ -1031,7 +1244,7 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { int32_t index = 0; if (envCmd == NULL) TAOS_RETURN(TSDB_CODE_SUCCESS); while (envCmd[index] != NULL) { - strncpy(buf, envCmd[index], sizeof(buf) - 1); + tstrncpy(buf, envCmd[index], sizeof(buf)); buf[sizeof(buf) - 1] = 0; if (taosEnvToCfg(buf, buf) < 0) { uTrace("failed to convert env to cfg:%s", buf); @@ -1188,14 +1401,14 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { int32_t len = 0; char newValue[1024] = {0}; - strcpy(newValue, value); + tstrncpy(newValue, value, sizeof(newValue)); int32_t count = 1; while (vlen < 1024) { (void)paGetToken(value + vlen + 1 * count, &tmp, &len); if (len == 0) break; tmp[len] = 0; - strcpy(newValue + vlen, tmp); + tstrncpy(newValue + vlen, tmp, sizeof(newValue) - vlen); vlen += len; count++; } @@ -1249,71 +1462,6 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { } } -// int32_t cfgLoadFromCfgText(SConfig *pConfig, const char *configText) { -// char *line = NULL, *name, *value, *value2, *value3; -// int32_t olen, vlen, vlen2, vlen3; -// ssize_t _bytes = 0; -// int32_t code = 0; - -// TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); -// if (pFile == NULL) { -// // success when the file does not exist -// if (errno == ENOENT) { -// terrno = TAOS_SYSTEM_ERROR(errno); -// uInfo("failed to load from cfg file %s since %s, use default parameters", filepath, terrstr()); -// return 0; -// } else { -// uError("failed to load from cfg file %s since %s", filepath, terrstr()); -// return -1; -// } -// } - -// while (!taosEOFFile(pFile)) { -// name = value = value2 = value3 = NULL; -// olen = vlen = vlen2 = vlen3 = 0; - -// _bytes = taosGetLineFile(pFile, &line); -// if (_bytes <= 0) { -// break; -// } - -// if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; - -// (void)paGetToken(line, &name, &olen); -// if (olen == 0) continue; -// name[olen] = 0; - -// (void)paGetToken(name + olen + 1, &value, &vlen); -// if (vlen == 0) continue; -// value[vlen] = 0; - -// (void)paGetToken(value + vlen + 1, &value2, &vlen2); -// if (vlen2 != 0) { -// value2[vlen2] = 0; -// (void)paGetToken(value2 + vlen2 + 1, &value3, &vlen3); -// if (vlen3 != 0) value3[vlen3] = 0; -// } - -// code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); -// if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; -// if (strcasecmp(name, "dataDir") == 0) { -// code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); -// if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; -// } -// } - -// (void)taosCloseFile(&pFile); -// if (line != NULL) taosMemoryFreeClear(line); - -// if (code == 0 || (code != 0 && terrno == TSDB_CODE_CFG_NOT_FOUND)) { -// uInfo("load from cfg file %s success", filepath); -// return 0; -// } else { -// uError("failed to load from cfg file %s since %s", filepath, terrstr()); -// return -1; -// } -// } - int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { char *cfgLineBuf = NULL, *name, *value, *value2, *value3, *value4; SJson *pJson = NULL; @@ -1482,7 +1630,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl char **pEnv = environ; line[1023] = 0; while (*pEnv != NULL) { - strncpy(line, *pEnv, sizeof(line) - 1); + tstrncpy(line, *pEnv, sizeof(line)); pEnv++; if (strncmp(line, "TAOS_APOLLO_URL", 14) == 0) { char *p = strchr(line, '='); @@ -1561,10 +1709,11 @@ int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter) { } SConfigItem *cfgNextIter(SConfigIter *pIter) { - if (pIter->index < cfgGetSize(pIter->pConf)) { - return taosArrayGet(pIter->pConf->array, pIter->index++); + if (pIter->index < cfgGetGlobalSize(pIter->pConf)) { + return taosArrayGet(pIter->pConf->globalArray, pIter->index++); + } else if (pIter->index < cfgGetGlobalSize(pIter->pConf) + cfgGetLocalSize(pIter->pConf)) { + return taosArrayGet(pIter->pConf->localArray, pIter->index++ - cfgGetGlobalSize(pIter->pConf)); } - return NULL; } @@ -1575,3 +1724,8 @@ void cfgDestroyIter(SConfigIter *pIter) { taosMemoryFree(pIter); } + +SArray *taosGetLocalCfg(SConfig *pCfg) { return pCfg->localArray; } +SArray *taosGetGlobalCfg(SConfig *pCfg) { return pCfg->globalArray; } +void taosSetLocalCfg(SConfig *pCfg, SArray *pArray) { pCfg->localArray = pArray; }; +void taosSetGlobalCfg(SConfig *pCfg, SArray *pArray) { pCfg->globalArray = pArray; }; \ No newline at end of file diff --git a/source/util/src/tenv.c b/source/util/src/tenv.c index 539687878b..2108f05c5b 100644 --- a/source/util/src/tenv.c +++ b/source/util/src/tenv.c @@ -24,9 +24,6 @@ int32_t taosEnvNameToCfgName(const char *envNameStr, char *cfgNameStr, int32_t c char *p = cfgNameStr; if (envNameStr[0] != 'T' || envNameStr[1] != 'A' || envNameStr[2] != 'O' || envNameStr[3] != 'S' || envNameStr[4] != '_') { - // if(p != envNameStr) strncpy(p, envNameStr, cfgNameMaxLen - 1); - // p[cfgNameMaxLen - 1] = '\0'; - // return strlen(cfgNameStr); cfgNameStr[0] = '\0'; return TSDB_CODE_INVALID_PARA; } @@ -57,7 +54,9 @@ int32_t taosEnvToCfg(const char *envStr, char *cfgStr) { if (envStr == NULL || cfgStr == NULL) { return TSDB_CODE_INVALID_PARA; } - if (cfgStr != envStr) strcpy(cfgStr, envStr); + if (cfgStr != envStr) { + tstrncpy(cfgStr, envStr, strlen(envStr) + 1); + } char *p = strchr(cfgStr, '='); if (p != NULL) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index b00fa49349..195cb21618 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -118,6 +118,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MSG_PREPROCESSED, "Message has been proc TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_BUFFER, "Out of buffer") TAOS_DEFINE_ERROR(TSDB_CODE_INTERNAL_ERROR, "Internal error") TAOS_DEFINE_ERROR(TSDB_CODE_TIME_ERROR, "Internal error in time") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_DISK_ID, "Internal error invalid disk id") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") @@ -195,7 +196,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_DROPPING, "Object is dropping") // mnode-dnode-part1 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, "Dnode already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, "Dnode does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "Vgroup does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "Vgroup does not exist or not in db") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CANT_DROP_LEADER, "Cannot drop mnode which is leader") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, "Out of dnodes") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_CFG, "Cluster cfg inconsistent") @@ -418,7 +419,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_CHARSET, "charset not match") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_LOCALE, "locale not match") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_TTL_CHG_ON_WR, "ttlChangeOnWrite not match") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_EN_WHITELIST, "enableWhiteList not match") -TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_STOPPED, "Mnode stopped") +TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_STOPPED, "Mnode stopped") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_COMPACT_TASKS, "Invalid max compact tasks") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed") @@ -443,6 +445,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_ARB_NOT_SYNCED, "Vgroup peer is not sy TAOS_DEFINE_ERROR(TSDB_CODE_VND_WRITE_DISABLED, "Vnode write is disabled for snapshot") TAOS_DEFINE_ERROR(TSDB_CODE_VND_COLUMN_COMPRESS_ALREADY_EXIST,"Same with old param") TAOS_DEFINE_ERROR(TSDB_CODE_VND_TTL_FLUSH_INCOMPLETION, "Failed to flush all ttl modification to tdb") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_ALREADY_EXIST_BUT_NOT_MATCH, "Vnode already exist but Dbid not match") // tsdb @@ -513,6 +516,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_FILTER_NOT_SUPPORT_TYPE, "Not supported range t TAOS_DEFINE_ERROR(TSDB_CODE_QRY_FILTER_WRONG_OPTR_TYPE, "Wrong operator type") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_FILTER_RANGE_ERROR, "Wrong filter range") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_FILTER_INVALID_TYPE, "Invalid filter type") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_REACH_QMEM_THRESHOLD, "Query memory upper limit is reached") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_QUERY_MEM_EXHAUSTED, "Query memory exhausted") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NO_FETCH_TIMEOUT, "Timeout for long time no fetch") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_SUCC_TO_PARTSUSS, "Change task status from success to partial success") // grant @@ -743,6 +749,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ANOMALY_WIN_COL, "ANOMALY_WINDOW not TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ANOMALY_WIN_OPT, "ANOMALY_WINDOW option should include algo field") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_FORECAST_CLAUSE, "Invalid forecast clause") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR, "Syntax error in regular expression") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VGID_LIST, "Invalid vgid list") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error") //planner diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 88fe6618b9..562fef2027 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -318,6 +318,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo return terrno = TSDB_CODE_INVALID_PTR; } + int32_t code = TSDB_CODE_SUCCESS; uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); // need the resize process, write lock applied @@ -327,10 +328,15 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo taosHashWUnlock(pHashObj); } + SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); + if (pNewNode == NULL) { + code = terrno; + return code; + } + // disable resize taosHashRLock(pHashObj); - int32_t code = TSDB_CODE_SUCCESS; uint32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; @@ -350,33 +356,22 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo if (pNode == NULL) { // no data in hash table with the specified key, add it into hash table - SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); - if (pNewNode == NULL) { - // terrno = TSDB_CODE_OUT_OF_MEMORY; - code = terrno; - goto _exit; - } - pushfrontNodeInEntryList(pe, pNewNode); (void)atomic_add_fetch_64(&pHashObj->size, 1); } else { // not support the update operation, return error if (pHashObj->enableUpdate) { - SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); - if (pNewNode == NULL) { - // terrno = TSDB_CODE_OUT_OF_MEMORY; - code = terrno; - goto _exit; - } - doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode); } else { + taosMemoryFreeClear(pNewNode); terrno = TSDB_CODE_DUP_KEY; code = terrno; goto _exit; } } + _exit: + taosHashEntryWUnlock(pHashObj, pe); taosHashRUnlock(pHashObj); return code; diff --git a/source/util/src/theap.c b/source/util/src/theap.c index 600bc2b998..d2348ba6b5 100644 --- a/source/util/src/theap.c +++ b/source/util/src/theap.c @@ -218,7 +218,7 @@ void destroyPriorityQueue(PriorityQueue* pq) { if (pq->deleteFn) taosArrayDestroyP(pq->container, pq->deleteFn); else - taosArrayDestroy(pq->container); + taosArrayDestroyP(pq->container, NULL); taosMemoryFree(pq); } @@ -308,7 +308,11 @@ PriorityQueueNode* taosPQPush(PriorityQueue* pq, const PriorityQueueNode* node) void taosPQPop(PriorityQueue* pq) { PriorityQueueNode* top = taosPQTop(pq); - if (pq->deleteFn) pq->deleteFn(top->data); + if (pq->deleteFn) { + pq->deleteFn(top->data); + } else { + taosMemoryFree(top->data); + } pqRemove(pq, 0); } @@ -342,6 +346,13 @@ BoundedQueue* createBoundedQueue(uint32_t maxSize, pq_comp_fn fn, FDelete delete void taosBQSetFn(BoundedQueue* q, pq_comp_fn fn) { taosPQSetFn(q->queue, fn); } +void taosBQClear(BoundedQueue* q) { + if (q->queue->deleteFn) + taosArrayClearEx(q->queue->container, q->queue->deleteFn); + else + taosArrayClear(q->queue->container); +} + void destroyBoundedQueue(BoundedQueue* q) { if (!q) return; destroyPriorityQueue(q->queue); @@ -356,8 +367,10 @@ PriorityQueueNode* taosBQPush(BoundedQueue* q, PriorityQueueNode* n) { } else { void* p = top->data; top->data = n->data; - n->data = p; - if (q->queue->deleteFn) q->queue->deleteFn(n->data); + if (q->queue->deleteFn) { + n->data = p; + q->queue->deleteFn(n->data); + } } return pqHeapify(q->queue, 0, taosBQSize(q)); } else { diff --git a/source/util/src/tlockfree.c b/source/util/src/tlockfree.c index 1961b404b5..9af2e739e0 100644 --- a/source/util/src/tlockfree.c +++ b/source/util/src/tlockfree.c @@ -91,4 +91,21 @@ void taosRLockLatch(SRWLatch *pLatch) { } } +// no reentrant +int32_t taosRTryLockLatch(SRWLatch *pLatch) { + SRWLatch oLatch, nLatch; + oLatch = atomic_load_32(pLatch); + if (oLatch) { + return -1; + } + + nLatch = oLatch + 1; + if (atomic_val_compare_exchange_32(pLatch, oLatch, nLatch) == oLatch) { + return 0; + } + + return -1; +} + + void taosRUnLockLatch(SRWLatch *pLatch) { (void)atomic_fetch_sub_32(pLatch, 1); } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 7a5852f329..a9eef1bfc9 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -19,8 +19,8 @@ #include "tconfig.h" #include "tglobal.h" #include "tjson.h" -#include "tutil.h" #include "ttime.h" +#include "tutil.h" #define LOG_MAX_LINE_SIZE (10024) #define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3) @@ -166,9 +166,9 @@ static int32_t taosStartLog() { } static int32_t getDay(char *buf, int32_t bufSize) { - time_t t; + time_t t; int32_t code = taosTime(&t); - if(code != 0) { + if (code != 0) { return code; } struct tm tmInfo; @@ -204,8 +204,8 @@ int32_t taosInitSlowLog() { getFullPathName(tsLogObj.slowLogName, logFileName); - char name[PATH_MAX + TD_TIME_STR_LEN] = {0}; - char day[TD_TIME_STR_LEN] = {0}; + char name[PATH_MAX + TD_TIME_STR_LEN] = {0}; + char day[TD_TIME_STR_LEN] = {0}; int32_t code = getDay(day, sizeof(day)); if (code != 0) { (void)printf("failed to get day, reason:%s\n", tstrerror(code)); @@ -299,6 +299,8 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { return 0; } +void taosSetNoNewFile() { tsLogObj.openInProgress = 1; } + static void taosStopLog() { if (tsLogObj.logHandle) { tsLogObj.logHandle->stop = 1; @@ -398,12 +400,12 @@ typedef struct { } OldFileKeeper; static OldFileKeeper *taosOpenNewFile() { char keepName[PATH_MAX + 20]; - sprintf(keepName, "%s.%d", tsLogObj.logName, tsLogObj.flag); + TAOS_UNUSED(snprintf(keepName, sizeof(keepName), "%s.%d", tsLogObj.logName, tsLogObj.flag)); tsLogObj.flag ^= 1; tsLogObj.lines = 0; char name[PATH_MAX + 20]; - sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag); + TAOS_UNUSED(snprintf(name, sizeof(name), "%s.%d", tsLogObj.logName, tsLogObj.flag)); TAOS_UNUSED(taosUmaskFile(0)); @@ -439,9 +441,9 @@ static OldFileKeeper *taosOpenNewFile() { static void *taosThreadToCloseOldFile(void *param) { if (!param) return NULL; - taosWLockLatch(&tsLogRotateLatch); OldFileKeeper *oldFileKeeper = (OldFileKeeper *)param; taosSsleep(20); + taosWLockLatch(&tsLogRotateLatch); taosCloseLogByFd(oldFileKeeper->pOldFile); taosKeepOldLog(oldFileKeeper->keepName); taosMemoryFree(oldFileKeeper); @@ -499,7 +501,7 @@ static void taosOpenNewSlowLogFile() { taosWriteLog(tsLogObj.slowHandle); atomic_store_32(&tsLogObj.slowHandle->lock, 0); - char day[TD_TIME_STR_LEN] = {0}; + char day[TD_TIME_STR_LEN] = {0}; int32_t code = getDay(day, sizeof(day)); if (code != 0) { uError("failed to get day, reason:%s", tstrerror(code)); @@ -578,22 +580,22 @@ static void decideLogFileName(const char *fn, int32_t maxFileNum) { } if (strlen(fn) < PATH_MAX) { - strcpy(tsLogObj.logName, fn); + tstrncpy(tsLogObj.logName, fn, PATH_MAX); } } static void decideLogFileNameFlag() { - char name[PATH_MAX + 50] = "\0"; + char name[PATH_MAX] = "\0"; int64_t logstat0_mtime = 0; int64_t logstat1_mtime = 0; bool log0Exist = false; bool log1Exist = false; - if (strlen(tsLogObj.logName) < PATH_MAX + 50 - 2) { - strcpy(name, tsLogObj.logName); - strcat(name, ".0"); + int32_t logNameLen = strlen(tsLogObj.logName) + 2; + if (logNameLen < PATH_MAX) { + TAOS_UNUSED(snprintf(name, PATH_MAX, "%s%s", tsLogObj.logName, ".0")); log0Exist = taosStatFile(name, NULL, &logstat0_mtime, NULL) == 0; - name[strlen(name) - 1] = '1'; + name[logNameLen - 1] = '1'; log1Exist = taosStatFile(name, NULL, &logstat1_mtime, NULL) == 0; } @@ -627,8 +629,15 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { processLogFileName(logName, maxFileNum); - char name[PATH_MAX + 50] = "\0"; - (void)sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag); + int32_t logNameLen = strlen(tsLogObj.logName) + 2; // logName + ".0" or ".1" + + if (logNameLen < 0 || logNameLen >= PATH_MAX) { + uError("log name:%s is invalid since length:%d is out of range", logName, logNameLen); + return TSDB_CODE_INVALID_CFG; + } + + char name[PATH_MAX] = "\0"; + (void)snprintf(name, sizeof(name), "%s.%d", tsLogObj.logName, tsLogObj.flag); (void)taosThreadMutexInit(&tsLogObj.logMutex, NULL); TAOS_UNUSED(taosUmaskFile(0)); @@ -657,19 +666,19 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { return terrno; } - (void)sprintf(name, "==================================================\n"); + (void)snprintf(name, sizeof(name), "==================================================\n"); if (taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)) <= 0) { TAOS_UNUSED(printf("failed to write to log file:%s, reason:%s\n", name, tstrerror(terrno))); taosUnLockLogFile(tsLogObj.logHandle->pFile); return terrno; } - (void)sprintf(name, " new log file \n"); + (void)snprintf(name, sizeof(name), " new log file \n"); if (taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)) <= 0) { TAOS_UNUSED(printf("failed to write to log file:%s, reason:%s\n", name, tstrerror(terrno))); taosUnLockLogFile(tsLogObj.logHandle->pFile); return terrno; } - (void)sprintf(name, "==================================================\n"); + (void)snprintf(name, sizeof(name), "==================================================\n"); if (taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)) <= 0) { TAOS_UNUSED(printf("failed to write to log file:%s, reason:%s\n", name, tstrerror(terrno))); taosUnLockLogFile(tsLogObj.logHandle->pFile); @@ -706,13 +715,13 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { TAOS_UNUSED(taosGetTimeOfDay(&timeSecs)); time_t curTime = timeSecs.tv_sec; ptm = taosLocalTime(&curTime, &Tm, NULL, 0, NULL); - if (ptm == NULL){ - uError("%s failed to get local time, code:%d", __FUNCTION__ , errno); + if (ptm == NULL) { + uError("%s failed to get local time, code:%d", __FUNCTION__, errno); return 0; } - return sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " %s %s", ptm->tm_mon + 1, ptm->tm_mday, - ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId(), - LOG_EDITION_FLG, flags); + return snprintf(buffer, LOG_MAX_STACK_LINE_BUFFER_SIZE, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " %s %s", + ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, + taosGetSelfPthreadId(), LOG_EDITION_FLG, flags); } static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *buffer, int32_t len) { @@ -871,30 +880,6 @@ void taosPrintSlowLog(const char *format, ...) { taosMemoryFree(buffer); } -void taosDumpData(unsigned char *msg, int32_t len) { - if (!osLogSpaceAvailable()) return; - taosUpdateLogNums(DEBUG_DUMP); - - char temp[256] = {0}; - int32_t i, pos = 0, c = 0; - - for (i = 0; i < len; ++i) { - sprintf(temp + pos, "%02x ", msg[i]); - c++; - pos += 3; - if (c >= 16) { - temp[pos++] = '\n'; - TAOS_UNUSED((taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos) <= 0)); - c = 0; - pos = 0; - } - } - - temp[pos++] = '\n'; - - TAOS_UNUSED(taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos)); -} - static void taosCloseLogByFd(TdFilePtr pFile) { if (pFile != NULL) { taosUnLockLogFile(pFile); diff --git a/source/util/src/tmempool.c b/source/util/src/tmempool.c index 2e8fdc2b17..0a56c69505 100644 --- a/source/util/src/tmempool.c +++ b/source/util/src/tmempool.c @@ -14,114 +14,1767 @@ */ #define _DEFAULT_SOURCE -#include "tmempool.h" +#include "osMemPool.h" +#include "tmempoolInt.h" #include "tlog.h" #include "tutil.h" +#include "taos.h" +#include "tglobal.h" -typedef struct { - int32_t numOfFree; /* number of free slots */ - int32_t first; /* the first free slot */ - int32_t numOfBlock; /* the number of blocks */ - int32_t blockSize; /* block size in bytes */ - int32_t *freeList; /* the index list */ - char *pool; /* the actual mem block */ - TdThreadMutex mutex; -} pool_t; +static TdThreadOnce gMPoolInit = PTHREAD_ONCE_INIT; +void* gMemPoolHandle = NULL; +threadlocal void* threadPoolSession = NULL; +threadlocal bool threadPoolEnabled = true; -mpool_h taosMemPoolInit(int32_t numOfBlock, int32_t blockSize) { - int32_t i; - pool_t *pool_p; +SMemPoolMgmt gMPMgmt = {0}; +SMPStrategyFp gMPFps[] = { + {NULL}, + {NULL, mpDirectFullAlloc, mpDirectFullFree, mpDirectGetMemSize, mpDirectFullRealloc, NULL, NULL, mpDirectTrim}, + //{mpChunkInit, mpChunkAlloc, mpChunkFree, mpChunkGetMemSize, mpChunkRealloc, mpChunkInitSession, mpChunkUpdateCfg, NULL} +}; - if (numOfBlock <= 1 || blockSize <= 1) { - uError("invalid parameter in memPoolInit\n"); - return NULL; + +int32_t mpCheckCfg(SMemPoolCfg* cfg) { + if (cfg->chunkSize < MEMPOOL_MIN_CHUNK_SIZE || cfg->chunkSize > MEMPOOL_MAX_CHUNK_SIZE) { + uError("invalid memory pool chunkSize:%d", cfg->chunkSize); + return TSDB_CODE_INVALID_PARA; } - pool_p = (pool_t *)taosMemoryMalloc(sizeof(pool_t)); - if (pool_p == NULL) { - uError("mempool malloc failed\n"); - return NULL; + if (cfg->evicPolicy <= 0 || cfg->evicPolicy >= E_EVICT_MAX_VALUE) { + uError("invalid memory pool evicPolicy:%d", cfg->evicPolicy); + return TSDB_CODE_INVALID_PARA; + } + + if (cfg->threadNum <= 0) { + uError("invalid memory pool threadNum:%d", cfg->threadNum); + return TSDB_CODE_INVALID_PARA; + } + + return TSDB_CODE_SUCCESS; +} + + +void mpFreeCacheGroup(SMPCacheGroup* pGrp) { + if (NULL == pGrp) { + return; + } + + taosMemoryFree(pGrp->pNodes); + taosMemoryFree(pGrp); +} + + +int32_t mpAddCacheGroup(SMemPool* pPool, SMPCacheGroupInfo* pInfo, SMPCacheGroup* pHead) { + SMPCacheGroup* pGrp = NULL; + if (NULL == pInfo->pGrpHead) { + pInfo->pGrpHead = taosMemoryCalloc(1, sizeof(*pInfo->pGrpHead)); + if (NULL == pInfo->pGrpHead) { + uError("malloc pGrpHead failed, error:%s", tstrerror(terrno)); + MP_ERR_RET(terrno); + } + + pGrp = pInfo->pGrpHead; } else { - (void)memset(pool_p, 0, sizeof(pool_t)); + pGrp = (SMPCacheGroup*)taosMemoryCalloc(1, sizeof(SMPCacheGroup)); + if (NULL == pInfo->pGrpHead) { + uError("malloc SMPCacheGroup failed, error:%s", tstrerror(terrno)); + MP_ERR_RET(terrno); + } + pGrp->pNext = pHead; } - pool_p->blockSize = blockSize; - pool_p->numOfBlock = numOfBlock; - pool_p->pool = (char *)taosMemoryMalloc((size_t)(blockSize * numOfBlock)); - pool_p->freeList = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * (size_t)numOfBlock); - - if (pool_p->pool == NULL || pool_p->freeList == NULL) { - uError("failed to allocate memory\n"); - taosMemoryFreeClear(pool_p->freeList); - taosMemoryFreeClear(pool_p->pool); - taosMemoryFreeClear(pool_p); - return NULL; + pGrp->nodesNum = pInfo->groupNum; + pGrp->pNodes = taosMemoryCalloc(pGrp->nodesNum, pInfo->nodeSize); + if (NULL == pGrp->pNodes) { + uError("calloc %d %d nodes in cache group failed", pGrp->nodesNum, pInfo->nodeSize); + MP_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - (void)taosThreadMutexInit(&(pool_p->mutex), NULL); + if (pHead && atomic_val_compare_exchange_ptr(&pInfo->pGrpHead, pHead, pGrp) != pHead) { + mpFreeCacheGroup(pGrp); + return TSDB_CODE_SUCCESS; + } - (void)memset(pool_p->pool, 0, (size_t)(blockSize * numOfBlock)); - for (i = 0; i < pool_p->numOfBlock; ++i) pool_p->freeList[i] = i; + (void)atomic_add_fetch_64(&pInfo->allocNum, pGrp->nodesNum); - pool_p->first = 0; - pool_p->numOfFree = pool_p->numOfBlock; - - return (mpool_h)pool_p; + return TSDB_CODE_SUCCESS; } -char *taosMemPoolMalloc(mpool_h handle) { - char *pos = NULL; - pool_t *pool_p = (pool_t *)handle; +void mpDestroyCacheGroup(SMPCacheGroupInfo* pInfo) { + SMPCacheGroup* pGrp = pInfo->pGrpHead; + SMPCacheGroup* pNext = NULL; + while (NULL != pGrp) { + pNext = pGrp->pNext; - (void)taosThreadMutexLock(&(pool_p->mutex)); + mpFreeCacheGroup(pGrp); - if (pool_p->numOfFree > 0) { - pos = pool_p->pool + pool_p->blockSize * (pool_p->freeList[pool_p->first]); - pool_p->first++; - pool_p->first = pool_p->first % pool_p->numOfBlock; - pool_p->numOfFree--; + pGrp = pNext; } - - (void)taosThreadMutexUnlock(&(pool_p->mutex)); - - if (pos == NULL) uDebug("mempool: out of memory"); - return pos; } -void taosMemPoolFree(mpool_h handle, char *pMem) { - int32_t index; - pool_t *pool_p = (pool_t *)handle; - if (pMem == NULL) return; +int32_t mpPopIdleNode(SMemPool* pPool, SMPCacheGroupInfo* pInfo, void** ppRes) { + SMPCacheGroup* pGrp = NULL; + SMPListNode* pNode = NULL; + + while (true) { + pNode = (SMPListNode*)atomic_load_ptr(&pInfo->pIdleList); + if (NULL == pNode) { + break; + } - index = (int32_t)(pMem - pool_p->pool) % pool_p->blockSize; - if (index != 0) { - uError("invalid free address:%p\n", pMem); + if (atomic_val_compare_exchange_ptr(&pInfo->pIdleList, pNode, pNode->pNext) != pNode) { + continue; + } + + pNode->pNext = NULL; + goto _return; + } + + while (true) { + pGrp = atomic_load_ptr(&pInfo->pGrpHead); + int32_t offset = atomic_fetch_add_32(&pGrp->idleOffset, 1); + if (offset < pGrp->nodesNum) { + pNode = (SMPListNode*)((char*)pGrp->pNodes + offset * pInfo->nodeSize); + break; + } else { + (void)atomic_sub_fetch_32(&pGrp->idleOffset, 1); + } + + MP_ERR_RET(mpAddCacheGroup(pPool, pInfo, pGrp)); + } + +_return: + + *ppRes = pNode; + + return TSDB_CODE_SUCCESS; +} + +void mpPushIdleNode(SMemPool* pPool, SMPCacheGroupInfo* pInfo, SMPListNode* pNode) { + SMPCacheGroup* pGrp = NULL; + SMPListNode* pOrig = NULL; + + while (true) { + pOrig = (SMPListNode*)atomic_load_ptr(&pInfo->pIdleList); + pNode->pNext = pOrig; + + if (atomic_val_compare_exchange_ptr(&pInfo->pIdleList, pOrig, pNode) != pOrig) { + continue; + } + + break; + } +} + + +int32_t mpUpdateCfg(SMemPool* pPool) { + if (gMPFps[gMPMgmt.strategy].updateCfgFp) { + MP_ERR_RET((*gMPFps[gMPMgmt.strategy].updateCfgFp)(pPool)); + } + + uDebug("memPool %s cfg updated, reserveSize:%" PRId64 ", jobQuota:%dMB, threadNum:%d", + pPool->name, pPool->cfg.reserveSize, *pPool->cfg.jobQuota, pPool->cfg.threadNum); + + return TSDB_CODE_SUCCESS; +} + +uint32_t mpFileIdHashFp(const char* fileId, uint32_t len) { + return *(uint32_t*)fileId; +} + +void mpDestroyPosStat(SMPStatPos* pStat) { + taosHashCleanup(pStat->fileHash); + pStat->fileHash = NULL; + taosHashCleanup(pStat->remainHash); + pStat->remainHash = NULL; + taosHashCleanup(pStat->allocHash); + pStat->allocHash = NULL; + taosHashCleanup(pStat->freeHash); + pStat->freeHash = NULL; +} + +int32_t mpInitPosStat(SMPStatPos* pStat, bool sessionStat) { + pStat->remainHash = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == pStat->remainHash) { + uError("memPool init posStat remainHash failed, error:%s, sessionStat:%d", tstrerror(terrno), sessionStat); + return terrno; + } + + pStat->allocHash = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == pStat->allocHash) { + uError("memPool init posStat allocHash failed, error:%s, sessionStat:%d", tstrerror(terrno), sessionStat); + return terrno; + } + + pStat->freeHash = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == pStat->freeHash) { + uError("memPool init posStat freeHash failed, error:%s, sessionStat:%d", tstrerror(terrno), sessionStat); + return terrno; + } + + pStat->fileHash = taosHashInit(1024, mpFileIdHashFp, false, HASH_ENTRY_LOCK); + if (NULL == pStat->fileHash) { + uError("memPool init posStat fileHash failed, error:%s, sessionStat:%d", tstrerror(terrno), sessionStat); + return terrno; + } + + uDebug("memPool stat initialized, sessionStat:%d", sessionStat); + + return TSDB_CODE_SUCCESS; +} + +int32_t mpInit(SMemPool* pPool, char* poolName, SMemPoolCfg* cfg) { +// MP_ERR_RET(mpCheckCfg(cfg)); + + TAOS_MEMCPY(&pPool->cfg, cfg, sizeof(*cfg)); + + pPool->name = taosStrdup(poolName); + if (NULL == pPool->name) { + uError("calloc memory pool name %s failed", poolName); + MP_ERR_RET(terrno); + } + + MP_ERR_RET(mpUpdateCfg(pPool)); + + pPool->sessionCache.groupNum = MP_SESSION_CACHE_ALLOC_BATCH_SIZE; + pPool->sessionCache.nodeSize = sizeof(SMPSession); + + MP_ERR_RET(mpAddCacheGroup(pPool, &pPool->sessionCache, NULL)); + + if (gMPFps[gMPMgmt.strategy].initFp) { + MP_ERR_RET((*gMPFps[gMPMgmt.strategy].initFp)(pPool, poolName, cfg)); + } + + if (tsMemPoolFullFunc) { + MP_ERR_RET(mpInitPosStat(&pPool->stat.posStat, false)); + } + + return TSDB_CODE_SUCCESS; +} + +FORCE_INLINE void mpUpdateMaxAllocSize(int64_t* pMaxAllocMemSize, int64_t newSize) { + int64_t maxAllocMemSize = atomic_load_64(pMaxAllocMemSize); + while (true) { + if (newSize <= maxAllocMemSize) { + break; + } + + if (maxAllocMemSize == atomic_val_compare_exchange_64(pMaxAllocMemSize, maxAllocMemSize, newSize)) { + break; + } + + maxAllocMemSize = atomic_load_64(pMaxAllocMemSize); + } +} + +void mpUpdateAllocSize(SMemPool* pPool, SMPSession* pSession, int64_t size, int64_t addSize) { + if (addSize) { + if (NULL != pSession) { + (void)atomic_add_fetch_64(&pSession->pJob->job.allocMemSize, addSize); + } + (void)atomic_add_fetch_64(&pPool->allocMemSize, addSize); + } + + if (NULL != pSession) { + int64_t allocMemSize = atomic_add_fetch_64(&pSession->allocMemSize, size); + mpUpdateMaxAllocSize(&pSession->maxAllocMemSize, allocMemSize); + + allocMemSize = atomic_load_64(&pSession->pJob->job.allocMemSize); + mpUpdateMaxAllocSize(&pSession->pJob->job.maxAllocMemSize, allocMemSize); + } + + if (MP_GET_FLAG(gMPMgmt.ctrl.funcFlags, MP_CTRL_FLAG_LOG_MAXSIZE)) { + int64_t allocMemSize = atomic_load_64(&pPool->allocMemSize); + mpUpdateMaxAllocSize(&pPool->maxAllocMemSize, allocMemSize); + } +} + +int32_t mpPutRetireMsgToQueue(SMemPool* pPool, bool retireLowLevel) { + if (retireLowLevel) { + if (0 == atomic_val_compare_exchange_8(&gMPMgmt.msgQueue.lowLevelRetire, 0, 1)) { + atomic_store_ptr(&gMPMgmt.msgQueue.pPool, pPool); + MP_ERR_RET(tsem2_post(&gMPMgmt.threadSem)); + } + + return TSDB_CODE_SUCCESS; + } + + if (0 == atomic_val_compare_exchange_8(&gMPMgmt.msgQueue.midLevelRetire, 0, 1)) { + atomic_store_ptr(&gMPMgmt.msgQueue.pPool, pPool); + MP_ERR_RET(tsem2_post(&gMPMgmt.threadSem)); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t mpChkFullQuota(SMemPool* pPool, SMPSession* pSession, int64_t size) { + int32_t code = TSDB_CODE_SUCCESS; + if (NULL == pSession) { + (void)atomic_add_fetch_64(&pPool->allocMemSize, size); + return code; + } + + SMPJob* pJob = pSession->pJob; + int64_t cAllocSize = atomic_add_fetch_64(&pJob->job.allocMemSize, size); + int64_t quota = atomic_load_32(pPool->cfg.jobQuota); + if (quota > 0 && cAllocSize > (quota * 1048576L)) { + code = TSDB_CODE_QRY_REACH_QMEM_THRESHOLD; + uWarn("job 0x%" PRIx64 " allocSize %" PRId64 " is over than quota %" PRId64, pJob->job.jobId, cAllocSize, quota); + pPool->cfg.cb.reachFp(pJob->job.jobId, pJob->job.clientId, code); + mpSchedTrim(NULL); + (void)atomic_sub_fetch_64(&pJob->job.allocMemSize, size); + MP_RET(code); + } + + if (atomic_load_64(&tsCurrentAvailMemorySize) <= (pPool->cfg.reserveSize + size)) { + code = TSDB_CODE_QRY_QUERY_MEM_EXHAUSTED; + uWarn("%s pool sysAvailMemSize %" PRId64 " can't alloc %" PRId64" while keeping reserveSize %" PRId64 " bytes", + pPool->name, atomic_load_64(&tsCurrentAvailMemorySize), size, pPool->cfg.reserveSize); + pPool->cfg.cb.reachFp(pJob->job.jobId, pJob->job.clientId, code); + mpSchedTrim(NULL); + (void)atomic_sub_fetch_64(&pJob->job.allocMemSize, size); + MP_RET(code); + } + + (void)atomic_add_fetch_64(&pPool->allocMemSize, size); + +/* + int64_t pAllocSize = atomic_add_fetch_64(&pPool->allocMemSize, size); + if (pAllocSize >= atomic_load_32(pPool->cfg.upperLimitSize) * 1048576UL) { + code = TSDB_CODE_QRY_QUERY_MEM_EXHAUSTED; + uWarn("%s pool allocSize %" PRId64 " reaches the upperLimit %" PRId64, pPool->name, pAllocSize, atomic_load_32(pPool->cfg.upperLimitSize) * 1048576UL); + pPool->cfg.cb.retireJobFp(&pJob->job, code); + (void)atomic_sub_fetch_64(&pJob->job.allocMemSize, size); + (void)atomic_sub_fetch_64(&pPool->allocMemSize, size); + MP_RET(code); + } +*/ + + return TSDB_CODE_SUCCESS; +} + +int64_t mpGetMemorySizeImpl(SMemPool* pPool, SMPSession* pSession, void *ptr) { + return (*gMPFps[gMPMgmt.strategy].getSizeFp)(pPool, pSession, ptr); +} + +int32_t mpMalloc(SMemPool* pPool, SMPSession* pSession, int64_t* size, uint32_t alignment, void** ppRes) { + MP_RET((*gMPFps[gMPMgmt.strategy].allocFp)(pPool, pSession, size, alignment, ppRes)); +} + +int32_t mpCalloc(SMemPool* pPool, SMPSession* pSession, int64_t* size, void** ppRes) { + int32_t code = TSDB_CODE_SUCCESS; + void *res = NULL; + + MP_ERR_RET(mpMalloc(pPool, pSession, size, 0, &res)); + + if (NULL != res) { + TAOS_MEMSET(res, 0, *size); + } + +_return: + + *ppRes = res; + + return code; +} + + +void mpFree(SMemPool* pPool, SMPSession* pSession, void *ptr, int64_t* origSize) { + if (NULL == ptr) { + if (origSize) { + *origSize = 0; + } + return; } - index = (int32_t)((pMem - pool_p->pool) / pool_p->blockSize); - if (index < 0 || index >= pool_p->numOfBlock) { - uError("mempool: error, invalid address:%p", pMem); + (*gMPFps[gMPMgmt.strategy].freeFp)(pPool, pSession, ptr, origSize); +} + +int32_t mpRealloc(SMemPool* pPool, SMPSession* pSession, void **pPtr, int64_t* size, int64_t* origSize) { + int32_t code = TSDB_CODE_SUCCESS; + + if (NULL == *pPtr) { + *origSize = 0; + MP_RET(mpMalloc(pPool, pSession, size, 0, pPtr)); + } + + if (0 == *size) { + mpFree(pPool, pSession, *pPtr, origSize); + *pPtr = NULL; + return TSDB_CODE_SUCCESS; + } + + *origSize = mpGetMemorySizeImpl(pPool, pSession, *pPtr); + + MP_RET((*gMPFps[gMPMgmt.strategy].reallocFp)(pPool, pSession, pPtr, size, origSize)); +} + +int32_t mpTrim(SMemPool* pPool, SMPSession* pSession, int32_t size, bool* trimed) { + int32_t code = TSDB_CODE_SUCCESS; + + if (gMPFps[gMPMgmt.strategy].trimFp) { + MP_RET((*gMPFps[gMPMgmt.strategy].trimFp)(pPool, pSession, size, trimed)); + } + + return code; +} + + +void mpPrintStatDetail(SMPCtrlInfo* pCtrl, SMPStatDetail* pDetail, char* detailName, int64_t maxAllocSize) { + if (!MP_GET_FLAG(pCtrl->funcFlags, MP_CTRL_FLAG_PRINT_STAT)) { return; } - (void)memset(pMem, 0, (size_t)pool_p->blockSize); + uInfo("MemPool [%s] stat detail:", detailName); - (void)taosThreadMutexLock(&pool_p->mutex); - - pool_p->freeList[(pool_p->first + pool_p->numOfFree) % pool_p->numOfBlock] = index; - pool_p->numOfFree++; - - (void)taosThreadMutexUnlock(&pool_p->mutex); + uInfo("Max Used Memory Size: %" PRId64, maxAllocSize); + + uInfo("[times]:"); + switch (gMPMgmt.strategy) { + case E_MP_STRATEGY_DIRECT: + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("Malloc", pDetail->times.memMalloc)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("Calloc", pDetail->times.memCalloc)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("Realloc", pDetail->times.memRealloc)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("Strdup", pDetail->times.memStrdup)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("Strndup", pDetail->times.memStrndup)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("Free", pDetail->times.memFree)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("Trim", pDetail->times.memTrim)); + break; + case E_MP_STRATEGY_CHUNK: + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("chunkMalloc", pDetail->times.chunkMalloc)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("chunkRecycle", pDetail->times.chunkRecycle)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("chunkReUse", pDetail->times.chunkReUse)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("chunkFree", pDetail->times.chunkFree)); + break; + default: + break; + } + + uInfo("[bytes]:"); + switch (gMPMgmt.strategy) { + case E_MP_STRATEGY_DIRECT: + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("Malloc", pDetail->bytes.memMalloc)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("Calloc", pDetail->bytes.memCalloc)); + uInfo(MP_STAT_ORIG_FORMAT, MP_STAT_ORIG_VALUE("Realloc", pDetail->bytes.memRealloc)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("Strdup", pDetail->bytes.memStrdup)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("Strndup", pDetail->bytes.memStrndup)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("Free", pDetail->bytes.memFree)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("Trim", pDetail->bytes.memTrim)); + break; + case E_MP_STRATEGY_CHUNK: + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("chunkMalloc", pDetail->bytes.chunkMalloc)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("chunkRecycle", pDetail->bytes.chunkRecycle)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("chunkReUse", pDetail->bytes.chunkReUse)); + uInfo(MP_STAT_FORMAT, MP_STAT_VALUE("chunkFree", pDetail->bytes.chunkFree)); + break; + default: + break; + } } -void taosMemPoolCleanUp(mpool_h handle) { - pool_t *pool_p = (pool_t *)handle; +int32_t mpAddToRemainAllocHash(SHashObj* pHash, SMPFileLine* pFileLine) { + int32_t code = TSDB_CODE_SUCCESS; + SMPAllocStat stat = {0}, *pStat = NULL; + + while (true) { + pStat = (SMPAllocStat*)taosHashGet(pHash, &pFileLine->fl, sizeof(pFileLine->fl)); + if (NULL == pStat) { + code = taosHashPut(pHash, &pFileLine->fl, sizeof(pFileLine->fl), &stat, sizeof(stat)); + if (TSDB_CODE_SUCCESS != code) { + if (TSDB_CODE_DUP_KEY == code) { + continue; + } + + uError("taosHashPut to remain alloc hash failed, error:%s", tstrerror(code)); + return code; + } - (void)taosThreadMutexDestroy(&pool_p->mutex); - if (pool_p->pool) taosMemoryFree(pool_p->pool); - if (pool_p->freeList) taosMemoryFree(pool_p->freeList); - (void)memset(pool_p, 0, sizeof(*pool_p)); - taosMemoryFree(pool_p); + continue; + } + + (void)atomic_add_fetch_64(&pStat->allocBytes, pFileLine->size); + (void)atomic_add_fetch_64(&pStat->allocTimes, 1); + break; + } + + return TSDB_CODE_SUCCESS; } + +void mpPrintPosRemainStat(SMPStatPos* pStat) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t remainNum = taosHashGetSize(pStat->remainHash); + if (remainNum <= 0) { + uInfo("no alloc remaining memory"); + return; + } + + SHashObj* pAllocHash = taosHashInit(remainNum / 10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == pAllocHash) { + uError("taosHashInit pAllocHash failed, error:%s, remainNum:%d", tstrerror(terrno), remainNum); + return; + } + + SMPFileLine* pFileLine = NULL; + void* pIter = taosHashIterate(pStat->remainHash, NULL); + while (pIter) { + pFileLine = (SMPFileLine*)pIter; + + MP_ERR_JRET(mpAddToRemainAllocHash(pAllocHash, pFileLine)); + + pIter = taosHashIterate(pStat->remainHash, pIter); + } + + SMPAllocStat* pAlloc = NULL; + pIter = taosHashIterate(pAllocHash, NULL); + while (pIter) { + pAlloc = (SMPAllocStat*)pIter; + SMPFileLineId* pId = (SMPFileLineId*)taosHashGetKey(pIter, NULL); + SMPAllocStat* pAlloc = (SMPAllocStat*)taosHashGet(pStat->allocHash, pId, sizeof(*pId)); + char* pFileName = (char*)taosHashGet(pStat->fileHash, &pId->fileId, sizeof(pId->fileId)); + if (NULL == pAlloc || NULL == pFileName) { + uError("fail to get pId in allocHash or fileHash, pAlloc:%p, pFileName:%p", pAlloc, pFileName); + goto _return; + } + + uInfo("REMAINING: %" PRId64 " bytes alloced by %s:%d in %" PRId64 " times", pAlloc->allocBytes, pFileName, pId->line, pAlloc->allocTimes); + + pIter = taosHashIterate(pAllocHash, pIter); + } + +_return: + + taosHashCleanup(pAllocHash); +} + +void mpPrintPosAllocStat(SMPStatPos* pStat) { + +} + +void mpPrintPosFreeStat(SMPStatPos* pStat) { + +} + +void mpPrintPosStat(SMPCtrlInfo* pCtrl, SMPStatPos* pStat, char* detailName) { + if (!MP_GET_FLAG(pCtrl->funcFlags, MP_CTRL_FLAG_PRINT_STAT)) { + return; + } + + uInfo("MemPool [%s] Pos Stat:", detailName); + uInfo("error times: %" PRId64, pStat->logErrTimes); + + mpPrintPosRemainStat(pStat); + + mpPrintPosAllocStat(pStat); + + mpPrintPosFreeStat(pStat); +} + +void mpPrintNodeStat(SMPCtrlInfo* pCtrl, SHashObj* pHash, char* detailName) { + //TODO +} + +void mpPrintSessionStat(SMPCtrlInfo* pCtrl, SMPStatSession* pSessStat, char* detailName) { + if (!MP_GET_FLAG(pCtrl->funcFlags, MP_CTRL_FLAG_PRINT_STAT)) { + return; + } + + uInfo("MemPool [%s] session stat:", detailName); + uInfo("init session succeed num: %" PRId64, pSessStat->initSucc); + uInfo("init session failed num: %" PRId64, pSessStat->initFail); + uInfo("session destroyed num: %" PRId64, pSessStat->destroyNum); +} + + + +void mpLogDetailStat(SMPStatDetail* pDetail, EMPStatLogItem item, SMPStatInput* pInput) { + switch (item) { + case E_MP_STAT_LOG_MEM_MALLOC: { + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_EXEC)) { + (void)atomic_add_fetch_64(&pDetail->times.memMalloc.exec, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memMalloc.exec, pInput->size); + } + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_RES_SUCC)) { + (void)atomic_add_fetch_64(&pDetail->times.memMalloc.succ, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memMalloc.succ, pInput->size); + } + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_RES_FAIL)) { + (void)atomic_add_fetch_64(&pDetail->times.memMalloc.fail, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memMalloc.fail, pInput->size); + } + break; + } + case E_MP_STAT_LOG_MEM_CALLOC:{ + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_EXEC)) { + (void)atomic_add_fetch_64(&pDetail->times.memCalloc.exec, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memCalloc.exec, pInput->size); + } + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_RES_SUCC)) { + (void)atomic_add_fetch_64(&pDetail->times.memCalloc.succ, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memCalloc.succ, pInput->size); + } + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_RES_FAIL)) { + (void)atomic_add_fetch_64(&pDetail->times.memCalloc.fail, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memCalloc.fail, pInput->size); + } + break; + } + case E_MP_STAT_LOG_MEM_REALLOC:{ + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_EXEC)) { + (void)atomic_add_fetch_64(&pDetail->times.memRealloc.exec, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memRealloc.exec, pInput->size); + (void)atomic_add_fetch_64(&pDetail->bytes.memRealloc.origExec, pInput->origSize); + } + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_RES_SUCC)) { + (void)atomic_add_fetch_64(&pDetail->times.memRealloc.succ, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memRealloc.succ, pInput->size); + (void)atomic_add_fetch_64(&pDetail->bytes.memRealloc.origSucc, pInput->origSize); + } + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_RES_FAIL)) { + (void)atomic_add_fetch_64(&pDetail->times.memRealloc.fail, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memRealloc.fail, pInput->size); + (void)atomic_add_fetch_64(&pDetail->bytes.memRealloc.origFail, pInput->origSize); + } + break; + } + case E_MP_STAT_LOG_MEM_FREE:{ + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_EXEC)) { + (void)atomic_add_fetch_64(&pDetail->times.memFree.exec, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memFree.exec, pInput->size); + } + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_RES_SUCC)) { + (void)atomic_add_fetch_64(&pDetail->times.memFree.succ, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memFree.succ, pInput->size); + } + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_RES_FAIL)) { + (void)atomic_add_fetch_64(&pDetail->times.memFree.fail, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memFree.fail, pInput->size); + } + break; + } + case E_MP_STAT_LOG_MEM_STRDUP: { + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_EXEC)) { + (void)atomic_add_fetch_64(&pDetail->times.memStrdup.exec, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memStrdup.exec, pInput->size); + } + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_RES_SUCC)) { + (void)atomic_add_fetch_64(&pDetail->times.memStrdup.succ, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memStrdup.succ, pInput->size); + } + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_RES_FAIL)) { + (void)atomic_add_fetch_64(&pDetail->times.memStrdup.fail, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memStrdup.fail, pInput->size); + } + break; + } + case E_MP_STAT_LOG_MEM_STRNDUP: { + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_EXEC)) { + (void)atomic_add_fetch_64(&pDetail->times.memStrndup.exec, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memStrndup.exec, pInput->size); + } + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_RES_SUCC)) { + (void)atomic_add_fetch_64(&pDetail->times.memStrndup.succ, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memStrndup.succ, pInput->size); + } + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_RES_FAIL)) { + (void)atomic_add_fetch_64(&pDetail->times.memStrndup.fail, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memStrndup.fail, pInput->size); + } + break; + } + case E_MP_STAT_LOG_MEM_TRIM: { + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_EXEC)) { + (void)atomic_add_fetch_64(&pDetail->times.memTrim.exec, 1); + } + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_RES_SUCC)) { + (void)atomic_add_fetch_64(&pDetail->times.memTrim.succ, 1); + (void)atomic_add_fetch_64(&pDetail->bytes.memTrim.succ, pInput->size); + } + if (MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_RES_FAIL)) { + (void)atomic_add_fetch_64(&pDetail->times.memTrim.fail, 1); + } + break; + } + case E_MP_STAT_LOG_CHUNK_MALLOC: + case E_MP_STAT_LOG_CHUNK_RECYCLE: + case E_MP_STAT_LOG_CHUNK_REUSE: + case E_MP_STAT_LOG_CHUNK_FREE: { + + } + default: + uError("Invalid stat item: %d", item); + break; + } +} + +int32_t mpGetAllocFreeStat(SHashObj* pHash, void* pKey, int32_t keyLen, void* pNew, int32_t newSize, void** ppRes) { + void* pStat = NULL; + int32_t code = TSDB_CODE_SUCCESS; + + while (true) { + pStat = taosHashGet(pHash, pKey, keyLen); + if (NULL != pStat) { + *ppRes = pStat; + break; + } + + code = taosHashPut(pHash, pKey, keyLen, pNew, newSize); + if (code) { + if (TSDB_CODE_DUP_KEY == code) { + continue; + } + + return code; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t mpGetPosStatFileId(SMPStatPos* pStat, char* fileName, uint32_t* pId, bool sessionStat) { + uint32_t hashVal = MurmurHash3_32(fileName, strlen(fileName)); + int32_t code = taosHashPut(pStat->fileHash, &hashVal, sizeof(hashVal), fileName, strlen(fileName) + 1); + if (code && TSDB_CODE_DUP_KEY != code) { + return code; + } + + *pId = hashVal; + + return TSDB_CODE_SUCCESS; +} + +void mpLogPosStat(SMPStatPos* pStat, EMPStatLogItem item, SMPStatInput* pInput, bool sessionStat) { + if (!MP_GET_FLAG(pInput->procFlags, MP_STAT_PROC_FLAG_RES_SUCC)) { + return; + } + + int32_t code = TSDB_CODE_SUCCESS; + + switch (item) { + case E_MP_STAT_LOG_MEM_MALLOC: + case E_MP_STAT_LOG_MEM_CALLOC: + case E_MP_STAT_LOG_MEM_STRDUP: + case E_MP_STAT_LOG_MEM_STRNDUP: { + SMPAllocStat allocStat = {0}, *pAlloc = NULL; + SMPFileLine fileLine = {.fl.line = pInput->line, .size = pInput->size}; + code = mpGetPosStatFileId(pStat, pInput->file, &fileLine.fl.fileId, sessionStat); + if (TSDB_CODE_SUCCESS != code) { + uError("add pMem:%p %s:%d to fileHash failed, error:%s, sessionStat:%d", + pInput->pMem, pInput->file, pInput->line, tstrerror(code), sessionStat); + MP_ERR_JRET(code); + } + code = taosHashPut(pStat->remainHash, &pInput->pMem, POINTER_BYTES, &fileLine, sizeof(fileLine)); + if (TSDB_CODE_SUCCESS != code) { + if (TSDB_CODE_DUP_KEY == code) { + SMPFileLine* pFileLine = (SMPFileLine*)taosHashAcquire(pStat->remainHash, &pInput->pMem, POINTER_BYTES); + if (pFileLine) { + char* pFileName = (char*)taosHashGet(pStat->fileHash, &pFileLine->fl.fileId, sizeof(pFileLine->fl.fileId)); + if (NULL == pFileName) { + uError("fail to get fileId %u in fileHash", pFileLine->fl.fileId); + } else { + uError("add pMem:%p %s:%d to remainHash failed, error:%s, sessionStat:%d, origAllocAt %s:%d", + pInput->pMem, pInput->file, pInput->line, tstrerror(code), sessionStat, pFileName, pFileLine->fl.line); + MP_ERR_JRET(code); + } + } + } + + uError("add pMem:%p %s:%d to remainHash failed, error:%s, sessionStat:%d", + pInput->pMem, pInput->file, pInput->line, tstrerror(code), sessionStat); + + MP_ERR_JRET(code); + } + code = mpGetAllocFreeStat(pStat->allocHash, &fileLine.fl, sizeof(fileLine.fl), (void*)&allocStat, sizeof(allocStat), (void**)&pAlloc); + if (TSDB_CODE_SUCCESS != code) { + uError("add pMem:%p %s:%d to allocHash failed, error:%s, sessionStat:%d", + pInput->pMem, pInput->file, pInput->line, tstrerror(code), sessionStat); + MP_ERR_JRET(code); + } + + (void)atomic_add_fetch_64(&pAlloc->allocTimes, 1); + (void)atomic_add_fetch_64(&pAlloc->allocBytes, pInput->size); + break; + } + case E_MP_STAT_LOG_MEM_REALLOC: { + SMPAllocStat allocStat = {0}, *pAlloc = NULL; + SMPFreeStat freeStat = {0}, *pFree = NULL; + SMPFileLine fileLine = {.fl.line = pInput->line, .size = pInput->size}; + code = mpGetPosStatFileId(pStat, pInput->file, &fileLine.fl.fileId, sessionStat); + if (TSDB_CODE_SUCCESS != code) { + uError("realloc: add pMem:%p %s:%d to fileHash failed, error:%s, sessionStat:%d", + pInput->pMem, pInput->file, pInput->line, tstrerror(code), sessionStat); + MP_ERR_JRET(code); + } + + // ASSSRT((pInput->pOrigMem && pInput->origSize > 0) || (NULL == pInput->pOrigMem && pInput->origSize == 0)); + + if (pInput->pOrigMem && pInput->origSize > 0) { + code = taosHashRemove(pStat->remainHash, &pInput->pOrigMem, POINTER_BYTES); + if (TSDB_CODE_SUCCESS != code) { + uError("realloc: rm pOrigMem:%p %s:%d from remainHash failed, error:%s, sessionStat:%d", + pInput->pOrigMem, pInput->file, pInput->line, tstrerror(code), sessionStat); + MP_ERR_JRET(code); + } + code = mpGetAllocFreeStat(pStat->freeHash, &fileLine.fl, sizeof(fileLine.fl), (void*)&freeStat, sizeof(freeStat), (void**)&pFree); + if (TSDB_CODE_SUCCESS != code) { + uError("realloc: add pOrigMem:%p %s:%d to freeHash failed, error:%s, sessionStat:%d", + pInput->pOrigMem, pInput->file, pInput->line, tstrerror(code), sessionStat); + MP_ERR_JRET(code); + } + + (void)atomic_add_fetch_64(&pFree->freeTimes, 1); + (void)atomic_add_fetch_64(&pFree->freeBytes, pInput->origSize); + } + + code = taosHashPut(pStat->remainHash, &pInput->pMem, POINTER_BYTES, &fileLine, sizeof(fileLine)); + if (TSDB_CODE_SUCCESS != code) { + if (TSDB_CODE_DUP_KEY == code) { + SMPFileLine* pFileLine = (SMPFileLine*)taosHashAcquire(pStat->remainHash, &pInput->pMem, POINTER_BYTES); + if (pFileLine) { + char* pFileName = (char*)taosHashGet(pStat->fileHash, &pFileLine->fl.fileId, sizeof(pFileLine->fl.fileId)); + if (NULL == pFileName) { + uError("realloc: fail to get fileId %u in fileHash", pFileLine->fl.fileId); + } else { + uError("realloc: add pMem:%p %s:%d to remainHash failed, error:%s, sessionStat:%d, origAllocAt %s:%d", + pInput->pMem, pInput->file, pInput->line, tstrerror(code), sessionStat, pFileName, pFileLine->fl.line); + MP_ERR_JRET(code); + } + } + } + + uError("realloc: add pMem:%p %s:%d to remainHash failed, error:%s, sessionStat:%d", + pInput->pMem, pInput->file, pInput->line, tstrerror(code), sessionStat); + MP_ERR_JRET(code); + } + + code = mpGetAllocFreeStat(pStat->allocHash, &fileLine.fl, sizeof(fileLine.fl), (void*)&allocStat, sizeof(allocStat), (void**)&pAlloc); + if (TSDB_CODE_SUCCESS != code) { + uError("realloc: add pMem:%p %s:%d to allocHash failed, error:%s, sessionStat:%d", + pInput->pMem, pInput->file, pInput->line, tstrerror(code), sessionStat); + MP_ERR_JRET(code); + } + + (void)atomic_add_fetch_64(&pAlloc->allocTimes, 1); + (void)atomic_add_fetch_64(&pAlloc->allocBytes, pInput->size); + break; + } + case E_MP_STAT_LOG_MEM_FREE: { + SMPAllocStat allocStat = {0}, *pAlloc = NULL; + SMPFreeStat freeStat = {0}, *pFree = NULL; + SMPFileLineId fl = {.line = pInput->line}; + code = mpGetPosStatFileId(pStat, pInput->file, &fl.fileId, sessionStat); + if (TSDB_CODE_SUCCESS != code) { + uError("free: add pMem:%p %s:%d to fileHash failed, error:%s, sessionStat:%d", + pInput->pMem, pInput->file, pInput->line, tstrerror(code), sessionStat); + MP_ERR_JRET(code); + } + + code = taosHashRemove(pStat->remainHash, &pInput->pMem, POINTER_BYTES); + if (TSDB_CODE_SUCCESS != code) { + uDebug("free: rm pMem:%p %s:%d to remainHash failed, error:%s, sessionStat:%d", + pInput->pMem, pInput->file, pInput->line, tstrerror(code), sessionStat); + } + + code = mpGetAllocFreeStat(pStat->freeHash, &fl, sizeof(fl), (void*)&freeStat, sizeof(freeStat), (void**)&pFree); + if (TSDB_CODE_SUCCESS != code) { + uError("free: add pMem:%p %s:%d to freeHash failed, error:%s, sessionStat:%d", + pInput->pMem, pInput->file, pInput->line, tstrerror(code), sessionStat); + MP_ERR_JRET(code); + } + + (void)atomic_add_fetch_64(&pFree->freeTimes, 1); + (void)atomic_add_fetch_64(&pFree->freeBytes, pInput->size); + break; + } + case E_MP_STAT_LOG_MEM_TRIM: + break; + case E_MP_STAT_LOG_CHUNK_MALLOC: + case E_MP_STAT_LOG_CHUNK_RECYCLE: + case E_MP_STAT_LOG_CHUNK_REUSE: + case E_MP_STAT_LOG_CHUNK_FREE: { + break; + } + default: + uError("Invalid stat item: %d", item); + break; + } + + return; + +_return: + + (void)atomic_add_fetch_64(&pStat->logErrTimes, 1); +} + + +void mpLogStat(SMemPool* pPool, SMPSession* pSession, EMPStatLogItem item, SMPStatInput* pInput) { + bool enablePool = false, randErr = false; + + switch (item) { + case E_MP_STAT_LOG_MEM_MALLOC: + case E_MP_STAT_LOG_MEM_CALLOC: + case E_MP_STAT_LOG_MEM_REALLOC: + case E_MP_STAT_LOG_MEM_FREE: + case E_MP_STAT_LOG_MEM_STRDUP: + case E_MP_STAT_LOG_MEM_STRNDUP: + case E_MP_STAT_LOG_MEM_TRIM: { + if (pSession && MP_GET_FLAG(pSession->ctrl.statFlags, MP_LOG_FLAG_ALL_MEM)) { + mpLogDetailStat(&pSession->stat.statDetail, item, pInput); + } + if (MP_GET_FLAG(gMPMgmt.ctrl.statFlags, MP_LOG_FLAG_ALL_MEM)) { + mpLogDetailStat(&pPool->stat.statDetail, item, pInput); + } + if (pSession && MP_GET_FLAG(pSession->ctrl.statFlags, MP_LOG_FLAG_ALL_POS)) { + taosSaveDisableMemPoolUsage(enablePool, randErr); + mpLogPosStat(&pSession->stat.posStat, item, pInput, true); + taosRestoreEnableMemPoolUsage(enablePool, randErr); + } + if (MP_GET_FLAG(gMPMgmt.ctrl.statFlags, MP_LOG_FLAG_ALL_POS)) { + taosSaveDisableMemPoolUsage(enablePool, randErr); + mpLogPosStat(&pPool->stat.posStat, item, pInput, false); + taosRestoreEnableMemPoolUsage(enablePool, randErr); + } + break; + } + case E_MP_STAT_LOG_CHUNK_MALLOC: + case E_MP_STAT_LOG_CHUNK_RECYCLE: + case E_MP_STAT_LOG_CHUNK_REUSE: + case E_MP_STAT_LOG_CHUNK_FREE: { + break; + } + default: + uError("Invalid stat item: %d", item); + break; + } +} + +void mpCheckStatDetail(void* poolHandle, void* session, char* detailName) { + if (0 == tsMemPoolFullFunc) { + return; + } + + SMemPool* pPool = (SMemPool*)poolHandle; + SMPSession* pSession = (SMPSession*)session; + SMPCtrlInfo* pCtrl = NULL; + SMPStatDetail* pDetail = NULL; + + if (NULL != session) { + pCtrl = &pSession->ctrl; + pDetail = &pSession->stat.statDetail; + if (MP_GET_FLAG(pCtrl->funcFlags, MP_CTRL_FLAG_CHECK_STAT)) { + int64_t allocSize = MEMPOOL_GET_ALLOC_SIZE(pDetail); + int64_t freeSize = MEMPOOL_GET_FREE_SIZE(pDetail); + + if (allocSize != freeSize) { + uError("%s Session in JOB:0x%" PRIx64 " stat check failed, allocSize:%" PRId64 ", freeSize:%" PRId64, + detailName, pSession->pJob->job.jobId, allocSize, freeSize); + + taosMemPoolPrintStat(NULL, pSession, detailName); + } else { + uDebug("%s Session in JOB:0x%" PRIx64 " stat check succeed, allocSize:%" PRId64 ", freeSize:%" PRId64, + detailName, pSession->pJob->job.jobId, allocSize, freeSize); + } + } + } + + if (NULL != poolHandle) { + pCtrl = &gMPMgmt.ctrl; + pDetail = &pPool->stat.statDetail; + int64_t sessInit = atomic_load_64(&pPool->stat.statSession.initFail) + atomic_load_64(&pPool->stat.statSession.initSucc); + if (MP_GET_FLAG(pCtrl->funcFlags, MP_CTRL_FLAG_CHECK_STAT) && sessInit == atomic_load_64(&pPool->stat.statSession.destroyNum)) { + int64_t allocSize = pDetail->bytes.memMalloc.succ + pDetail->bytes.memCalloc.succ + pDetail->bytes.memRealloc.succ + pDetail->bytes.memStrdup.succ + pDetail->bytes.memStrndup.succ; + int64_t freeSize = pDetail->bytes.memRealloc.origSucc + pDetail->bytes.memFree.succ; + + if (allocSize != freeSize) { + uError("%s MemPool %s stat check failed, allocSize:%" PRId64 ", freeSize:%" PRId64, detailName, pPool->name, allocSize, freeSize); + + taosMemPoolPrintStat(poolHandle, NULL, detailName); + } else { + uDebug("%s MemPool %s stat check succeed, allocSize:%" PRId64 ", freeSize:%" PRId64, detailName, pPool->name, allocSize, freeSize); + } + } + } +} + + +void mpCheckUpateCfg(void) { +/* + taosRLockLatch(&gMPMgmt.poolLock); + int32_t poolNum = taosArrayGetSize(gMPMgmt.poolList); + for (int32_t i = 0; i < poolNum; ++i) { + SMemPool* pPool = (SMemPool*)taosArrayGetP(gMPMgmt.poolList, i); + if (pPool->cfg.cb.cfgUpdateFp) { + (*pPool->cfg.cb.cfgUpdateFp)((void*)pPool, &pPool->cfg); + } + } + taosRUnLockLatch(&gMPMgmt.poolLock); +*/ +} + +void mpUpdateSystemAvailableMemorySize() { + static int64_t errorTimes = 0; + int64_t sysAvailSize = 0; + + int32_t code = taosGetSysAvailMemory(&sysAvailSize); + if (TSDB_CODE_SUCCESS != code) { + errorTimes++; + if (0 == errorTimes % 1000) { + uError("get system available memory size failed, error: %s, errorTimes:%" PRId64, tstrerror(code), errorTimes); + } + + return; + } + + atomic_store_64(&tsCurrentAvailMemorySize, sysAvailSize); + + uDebug("system available memory size: %" PRId64, sysAvailSize); +} + +void mpSchedTrim(int64_t* loopTimes) { + static int64_t trimTimes = 0; + + atomic_store_8(&tsNeedTrim, 1); + if (loopTimes) { + *loopTimes = 0; + } + + uDebug("%" PRId64 "th memory trim scheduled", ++trimTimes); +} + +void* mpMgmtThreadFunc(void* param) { + int32_t timeout = 0; + int64_t retireSize = 0, loopTimes = 0; + SMemPool* pPool = (SMemPool*)atomic_load_ptr(&gMemPoolHandle); + + while (0 == atomic_load_8(&gMPMgmt.modExit)) { + mpUpdateSystemAvailableMemorySize(); + + retireSize = pPool->cfg.reserveSize - atomic_load_64(&tsCurrentAvailMemorySize); + if (retireSize > 0) { + (*pPool->cfg.cb.failFp)(retireSize, TSDB_CODE_QRY_QUERY_MEM_EXHAUSTED); + + mpSchedTrim(&loopTimes); + } + + if (0 == (++loopTimes) % 500) { + mpSchedTrim(&loopTimes); + } + + taosMsleep(MP_DEFAULT_MEM_CHK_INTERVAL_MS); +/* + timeout = tsem2_timewait(&gMPMgmt.threadSem, gMPMgmt.waitMs); + if (0 != timeout) { + mpUpdateSystemAvailableMemorySize(); + continue; + } + + if (atomic_load_8(&gMPMgmt.msgQueue.midLevelRetire)) { + (*gMPMgmt.msgQueue.pPool->cfg.cb.retireJobsFp)(gMPMgmt.msgQueue.pPool, atomic_load_64(&gMPMgmt.msgQueue.pPool->cfg.retireUnitSize), false, TSDB_CODE_QRY_QUERY_MEM_EXHAUSTED); + } else if (atomic_load_8(&gMPMgmt.msgQueue.lowLevelRetire)) { + (*gMPMgmt.msgQueue.pPool->cfg.cb.retireJobsFp)(gMPMgmt.msgQueue.pPool, atomic_load_64(&gMPMgmt.msgQueue.pPool->cfg.retireUnitSize), true, TSDB_CODE_QRY_QUERY_MEM_EXHAUSTED); + } +*/ + } + + taosMemPoolModDestroy(); + + return NULL; +} + +int32_t mpCreateMgmtThread() { + int32_t code = TSDB_CODE_SUCCESS; + TdThreadAttr thAttr; + MP_ERR_RET(taosThreadAttrInit(&thAttr)); + MP_ERR_JRET(taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE)); + code = taosThreadCreate(&gMPMgmt.poolMgmtThread, &thAttr, mpMgmtThreadFunc, NULL); + if (code != 0) { + uError("failed to create memPool mgmt thread, error: 0x%x", code); + (void)taosThreadAttrDestroy(&thAttr); + MP_ERR_JRET(code); + } + +_return: + + MP_ERR_RET(taosThreadAttrDestroy(&thAttr)); + + return code; +} + +void mpModInit(void) { + int32_t code = TSDB_CODE_SUCCESS; + + gMPMgmt.modExit = 0; + + taosInitRWLatch(&gMPMgmt.poolLock); + + gMPMgmt.poolList = taosArrayInit(10, POINTER_BYTES); + if (NULL == gMPMgmt.poolList) { + MP_ERR_JRET(terrno); + } + + gMPMgmt.strategy = E_MP_STRATEGY_DIRECT; + + gMPMgmt.ctrl.statFlags = MP_STAT_FLAG_LOG_ALL & (~MP_LOG_FLAG_ALL_POS); + gMPMgmt.ctrl.funcFlags = MP_CTRL_FLAG_PRINT_STAT | MP_CTRL_FLAG_CHECK_STAT | MP_CTRL_FLAG_LOG_MAXSIZE; + + //gMPMgmt.code = tsem2_init(&gMPMgmt.threadSem, 0, 0); + //if (TSDB_CODE_SUCCESS != gMPMgmt.code) { + // uError("failed to init sem2, error: 0x%x", gMPMgmt.code); + // return; + //} + + gMPMgmt.waitMs = MP_DEFAULT_MEM_CHK_INTERVAL_MS; + +_return: + + gMPMgmt.code = code; +} + + +void mpFreePool(void* p) { + SMemPool* pPool = *(void**)p; + taosMemoryFree(pPool); +} + + +void taosMemPoolPrintStat(void* poolHandle, void* session, char* procName) { + SMemPool* pPool = (SMemPool*)poolHandle; + SMPSession* pSession = (SMPSession*)session; + char detailName[128]; + + if (NULL != pSession && MP_GET_FLAG(pSession->ctrl.funcFlags, MP_CTRL_FLAG_PRINT_STAT)) { + snprintf(detailName, sizeof(detailName) - 1, "%s - %s", procName, "Session"); + detailName[sizeof(detailName) - 1] = 0; + mpPrintStatDetail(&pSession->ctrl, &pSession->stat.statDetail, detailName, pSession->maxAllocMemSize); + + snprintf(detailName, sizeof(detailName) - 1, "%s - %s", procName, "SessionPos"); + detailName[sizeof(detailName) - 1] = 0; + mpPrintPosStat(&pSession->ctrl, &pSession->stat.posStat, detailName); + } + + if (NULL != pPool && MP_GET_FLAG(gMPMgmt.ctrl.funcFlags, MP_CTRL_FLAG_PRINT_STAT)) { + snprintf(detailName, sizeof(detailName) - 1, "%s - %s", procName, pPool->name); + detailName[sizeof(detailName) - 1] = 0; + mpPrintSessionStat(&gMPMgmt.ctrl, &pPool->stat.statSession, detailName); + mpPrintStatDetail(&gMPMgmt.ctrl, &pPool->stat.statDetail, detailName, pPool->maxAllocMemSize); + + //snprintf(detailName, sizeof(detailName) - 1, "%s - %s", procName, "MemPoolNode"); + //detailName[sizeof(detailName) - 1] = 0; + //mpPrintNodeStat(&gMPMgmt.ctrl, pPool->stat.nodeStat, detailName); + + snprintf(detailName, sizeof(detailName) - 1, "%s - %s", procName, "MemPoolPos"); + detailName[sizeof(detailName) - 1] = 0; + mpPrintPosStat(&gMPMgmt.ctrl, &pPool->stat.posStat, detailName); + } +} + + +int32_t taosMemPoolOpen(char* poolName, SMemPoolCfg* cfg, void** poolHandle) { + int32_t code = TSDB_CODE_SUCCESS; + SMemPool* pPool = NULL; + + //MP_ERR_JRET(taosThreadOnce(&gMPoolInit, mpModInit)); + mpModInit(); + + if (TSDB_CODE_SUCCESS != gMPMgmt.code) { + uError("init memory pool failed, code: 0x%x", gMPMgmt.code); + MP_ERR_JRET(gMPMgmt.code); + } + + pPool = (SMemPool*)taosMemoryCalloc(1, sizeof(SMemPool)); + if (NULL == pPool) { + uError("calloc memory pool failed, code: 0x%x", terrno); + MP_ERR_JRET(terrno); + } + + MP_ERR_JRET(mpInit(pPool, poolName, cfg)); + + taosWLockLatch(&gMPMgmt.poolLock); + + if (NULL == taosArrayPush(gMPMgmt.poolList, &pPool)) { + taosWUnLockLatch(&gMPMgmt.poolLock); + MP_ERR_JRET(terrno); + } + + pPool->slotId = taosArrayGetSize(gMPMgmt.poolList) - 1; + + taosWUnLockLatch(&gMPMgmt.poolLock); + + uInfo("mempool %s opened", poolName); + +_return: + + if (TSDB_CODE_SUCCESS != code) { + taosMemPoolClose(pPool); + pPool = NULL; + } + + *poolHandle = pPool; + + return code; +} + +void taosMemPoolCfgUpdate(void* poolHandle, SMemPoolCfg* pCfg) { + SMemPool* pPool = (SMemPool*)poolHandle; + + (void)mpUpdateCfg(pPool); +} + +void taosMemPoolDestroySession(void* poolHandle, void* session) { + SMemPool* pPool = (SMemPool*)poolHandle; + SMPSession* pSession = (SMPSession*)session; + if (NULL == poolHandle || NULL == pSession) { + uWarn("null pointer of poolHandle %p or session %p", poolHandle, session); + return; + } + + if (tsMemPoolFullFunc) { + (void)atomic_add_fetch_64(&pPool->stat.statSession.destroyNum, 1); + mpCheckStatDetail(pPool, pSession, "DestroySession"); + mpDestroyPosStat(&pSession->stat.posStat); + } + + taosMemFreeClear(pSession->sessionId); + + TAOS_MEMSET(pSession, 0, sizeof(*pSession)); + + mpPushIdleNode(pPool, &pPool->sessionCache, (SMPListNode*)pSession); +} + +int32_t taosMemPoolInitSession(void* poolHandle, void** ppSession, void* pJob, char* sessionId) { + int32_t code = TSDB_CODE_SUCCESS; + SMemPool* pPool = (SMemPool*)poolHandle; + SMPSession* pSession = NULL; + + MP_ERR_JRET(mpPopIdleNode(pPool, &pPool->sessionCache, (void**)&pSession)); + + pSession->sessionId = taosStrdup(sessionId); + if (NULL == pSession->sessionId) { + uError("strdup sessionId failed, error:%s", tstrerror(terrno)); + MP_ERR_JRET(terrno); + } + + TAOS_MEMCPY(&pSession->ctrl, &gMPMgmt.ctrl, sizeof(pSession->ctrl)); + + if (gMPFps[gMPMgmt.strategy].initSessionFp) { + MP_ERR_JRET((*gMPFps[gMPMgmt.strategy].initSessionFp)(pPool, pSession)); + } + + if (tsMemPoolFullFunc) { + MP_ERR_JRET(mpInitPosStat(&pSession->stat.posStat, true)); + } + + pSession->pJob = (SMPJob*)pJob; + +_return: + + if (TSDB_CODE_SUCCESS != code) { + taosMemPoolDestroySession(poolHandle, pSession); + pSession = NULL; + (void)atomic_add_fetch_64(&pPool->stat.statSession.initFail, 1); + } else { + (void)atomic_add_fetch_64(&pPool->stat.statSession.initSucc, 1); + } + + *ppSession = pSession; + + return code; +} + + +void *taosMemPoolMalloc(void* poolHandle, void* session, int64_t size, char* fileName, int32_t lineNo) { + if (0 == tsMemPoolFullFunc) { + return mpDirectAlloc(poolHandle, ((SMPSession*)session)->pJob, size); + } + + int32_t code = TSDB_CODE_SUCCESS; + SMPStatInput input = {.size = size, .file = fileName, .line = lineNo, .procFlags = MP_STAT_PROC_FLAG_EXEC, .pMem = NULL}; + + if (NULL == poolHandle || NULL == fileName || size < 0) { + uError("%s invalid input param, handle:%p, session:%p, fileName:%p, size:%" PRId64, __FUNCTION__, poolHandle, session, fileName, size); + MP_ERR_JRET(TSDB_CODE_INVALID_PARA); + } + + SMemPool* pPool = (SMemPool*)poolHandle; + SMPSession* pSession = (SMPSession*)session; + + code = mpMalloc(pPool, pSession, &input.size, 0, &input.pMem); + + MP_SET_FLAG(input.procFlags, (NULL != input.pMem ? MP_STAT_PROC_FLAG_RES_SUCC : MP_STAT_PROC_FLAG_RES_FAIL)); + mpLogStat(pPool, pSession, E_MP_STAT_LOG_MEM_MALLOC, &input); + +_return: + + if (TSDB_CODE_SUCCESS != code) { + terrno = code; + } + + return input.pMem; +} + +void *taosMemPoolCalloc(void* poolHandle, void* session, int64_t num, int64_t size, char* fileName, int32_t lineNo) { + if (0 == tsMemPoolFullFunc) { + return mpDirectCalloc(poolHandle, ((SMPSession*)session)->pJob, num, size); + } + + int32_t code = TSDB_CODE_SUCCESS; + int64_t totalSize = num * size; + SMPStatInput input = {.size = totalSize, .file = fileName, .line = lineNo, .procFlags = MP_STAT_PROC_FLAG_EXEC, .pMem = NULL}; + + if (NULL == poolHandle || NULL == fileName || num < 0 || size < 0) { + uError("%s invalid input param, handle:%p, session:%p, fileName:%p, num:%" PRId64 ", size:%" PRId64, + __FUNCTION__, poolHandle, session, fileName, num, size); + MP_ERR_JRET(TSDB_CODE_INVALID_PARA); + } + + SMemPool* pPool = (SMemPool*)poolHandle; + SMPSession* pSession = (SMPSession*)session; + + code = mpCalloc(pPool, pSession, &input.size, &input.pMem); + + MP_SET_FLAG(input.procFlags, (NULL != input.pMem ? MP_STAT_PROC_FLAG_RES_SUCC : MP_STAT_PROC_FLAG_RES_FAIL)); + mpLogStat(pPool, pSession, E_MP_STAT_LOG_MEM_CALLOC, &input); + +_return: + + if (TSDB_CODE_SUCCESS != code) { + terrno = code; + } + + return input.pMem; +} + +void *taosMemPoolRealloc(void* poolHandle, void* session, void *ptr, int64_t size, char* fileName, int32_t lineNo) { + if (0 == tsMemPoolFullFunc) { + return mpDirectRealloc(poolHandle, ((SMPSession*)session)->pJob, ptr, size); + } + + int32_t code = TSDB_CODE_SUCCESS; + SMPStatInput input = {.size = size, .file = fileName, .line = lineNo, .procFlags = MP_STAT_PROC_FLAG_EXEC, .origSize = 0, .pMem = ptr, .pOrigMem = ptr}; + + if (NULL == poolHandle || NULL == fileName || size < 0) { + uError("%s invalid input param, handle:%p, session:%p, fileName:%p, size:%" PRId64, + __FUNCTION__, poolHandle, session, fileName, size); + MP_ERR_JRET(TSDB_CODE_INVALID_PARA); + } + + SMemPool* pPool = (SMemPool*)poolHandle; + SMPSession* pSession = (SMPSession*)session; + + code = mpRealloc(pPool, pSession, &input.pMem, &input.size, &input.origSize); + + if (NULL != input.pMem) { + MP_SET_FLAG(input.procFlags, MP_STAT_PROC_FLAG_RES_SUCC); + mpLogStat(pPool, pSession, E_MP_STAT_LOG_MEM_REALLOC, &input); + } else if (0 == size){ + input.pMem = input.pOrigMem; + input.pOrigMem = NULL; + input.size = input.origSize; + MP_SET_FLAG(input.procFlags, MP_STAT_PROC_FLAG_RES_SUCC); + mpLogStat(pPool, pSession, E_MP_STAT_LOG_MEM_FREE, &input); + input.pMem = NULL; + } else { + MP_SET_FLAG(input.procFlags, MP_STAT_PROC_FLAG_RES_FAIL); + mpLogStat(pPool, pSession, E_MP_STAT_LOG_MEM_REALLOC, &input); + + input.pMem = input.pOrigMem; + input.pOrigMem = NULL; + input.size = input.origSize; + input.procFlags = MP_STAT_PROC_FLAG_EXEC; + MP_SET_FLAG(input.procFlags, MP_STAT_PROC_FLAG_RES_SUCC); + mpLogStat(pPool, pSession, E_MP_STAT_LOG_MEM_FREE, &input); + input.pMem = NULL; + } + +_return: + + if (TSDB_CODE_SUCCESS != code) { + terrno = code; + } + + return input.pMem; +} + +char *taosMemPoolStrdup(void* poolHandle, void* session, const char *ptr, char* fileName, int32_t lineNo) { + if (0 == tsMemPoolFullFunc) { + return mpDirectStrdup(poolHandle, ((SMPSession*)session)->pJob, ptr); + } + + int32_t code = TSDB_CODE_SUCCESS; + int64_t size = (ptr ? strlen(ptr) : 0) + 1; + SMPStatInput input = {.size = size, .file = fileName, .line = lineNo, .procFlags = MP_STAT_PROC_FLAG_EXEC, .pMem = NULL}; + + if (NULL == poolHandle || NULL == fileName || NULL == ptr) { + uError("%s invalid input param, handle:%p, session:%p, fileName:%p, ptr:%p", + __FUNCTION__, poolHandle, session, fileName, ptr); + MP_ERR_JRET(TSDB_CODE_INVALID_PARA); + } + + SMemPool* pPool = (SMemPool*)poolHandle; + SMPSession* pSession = (SMPSession*)session; + + code = mpMalloc(pPool, pSession, &input.size, 0, &input.pMem); + if (NULL != input.pMem) { + TAOS_STRCPY(input.pMem, ptr); + *((char*)input.pMem + size - 1) = 0; + } + + MP_SET_FLAG(input.procFlags, (NULL != input.pMem ? MP_STAT_PROC_FLAG_RES_SUCC : MP_STAT_PROC_FLAG_RES_FAIL)); + mpLogStat(pPool, pSession, E_MP_STAT_LOG_MEM_STRDUP, &input); + +_return: + + if (TSDB_CODE_SUCCESS != code) { + terrno = code; + } + + return input.pMem; +} + +char *taosMemPoolStrndup(void* poolHandle, void* session, const char *ptr, int64_t size, char* fileName, int32_t lineNo) { + if (0 == tsMemPoolFullFunc) { + return mpDirectStrndup(poolHandle, ((SMPSession*)session)->pJob, ptr, size); + } + + int32_t code = TSDB_CODE_SUCCESS; + int64_t origSize = ptr ? strlen(ptr) : 0; + size = TMIN(size, origSize) + 1; + SMPStatInput input = {.size = size, .file = fileName, .line = lineNo, .procFlags = MP_STAT_PROC_FLAG_EXEC, .pMem = NULL}; + + if (NULL == poolHandle || NULL == fileName || NULL == ptr || size < 0) { + uError("%s invalid input param, handle:%p, session:%p, fileName:%p, ptr:%p, size:%" PRId64, + __FUNCTION__, poolHandle, session, fileName, ptr, size); + MP_ERR_JRET(TSDB_CODE_INVALID_PARA); + } + + SMemPool* pPool = (SMemPool*)poolHandle; + SMPSession* pSession = (SMPSession*)session; + + code = mpMalloc(pPool, pSession, &input.size, 0, &input.pMem); + if (NULL != input.pMem) { + TAOS_MEMCPY(input.pMem, ptr, size - 1); + *((char*)input.pMem + size - 1) = 0; + } + + MP_SET_FLAG(input.procFlags, (NULL != input.pMem ? MP_STAT_PROC_FLAG_RES_SUCC : MP_STAT_PROC_FLAG_RES_FAIL)); + mpLogStat(pPool, pSession, E_MP_STAT_LOG_MEM_STRNDUP, &input); + +_return: + + if (TSDB_CODE_SUCCESS != code) { + terrno = code; + } + + return input.pMem; +} + + +void taosMemPoolFree(void* poolHandle, void* session, void *ptr, char* fileName, int32_t lineNo) { + if (0 == tsMemPoolFullFunc) { + mpDirectFree(poolHandle, ((SMPSession*)session)->pJob, ptr); + return; + } + + if (NULL == ptr) { + return; + } + + int32_t code = TSDB_CODE_SUCCESS; + if (NULL == poolHandle || NULL == fileName) { + uError("%s invalid input param, handle:%p, session:%p, fileName:%p", + __FUNCTION__, poolHandle, session, fileName); + } + + SMemPool* pPool = (SMemPool*)poolHandle; + SMPSession* pSession = (SMPSession*)session; + SMPStatInput input = {.file = fileName, .line = lineNo, .procFlags = MP_STAT_PROC_FLAG_EXEC, .pMem = ptr}; + + mpFree(pPool, pSession, ptr, &input.size); + + MP_SET_FLAG(input.procFlags, MP_STAT_PROC_FLAG_RES_SUCC); + mpLogStat(pPool, pSession, E_MP_STAT_LOG_MEM_FREE, &input); +} + +int64_t taosMemPoolGetMemorySize(void* poolHandle, void* session, void *ptr, char* fileName, int32_t lineNo) { + if (0 == tsMemPoolFullFunc) { + return taosMemSize(ptr); + } + + int64_t code = 0; + if (NULL == poolHandle || NULL == fileName) { + uError("%s invalid input param, handle:%p, session:%p, fileName:%p", + __FUNCTION__, poolHandle, session, fileName); + MP_ERR_RET(TSDB_CODE_INVALID_PARA); + } + + if (NULL == ptr) { + return 0; + } + + SMemPool* pPool = (SMemPool*)poolHandle; + SMPSession* pSession = (SMPSession*)session; + + return mpGetMemorySizeImpl(pPool, pSession, ptr); +} + +void* taosMemPoolMallocAlign(void* poolHandle, void* session, uint32_t alignment, int64_t size, char* fileName, int32_t lineNo) { + if (0 == tsMemPoolFullFunc) { + return mpDirectAlignAlloc(poolHandle, ((SMPSession*)session)->pJob, alignment, size); + } + + int32_t code = TSDB_CODE_SUCCESS; + SMPStatInput input = {.size = size, .file = fileName, .line = lineNo, .procFlags = MP_STAT_PROC_FLAG_EXEC, .pMem = NULL}; + + if (NULL == poolHandle || NULL == fileName || size < 0 || alignment < POINTER_BYTES || alignment % POINTER_BYTES) { + uError("%s invalid input param, handle:%p, session:%p, fileName:%p, alignment:%u, size:%" PRId64, + __FUNCTION__, poolHandle, session, fileName, alignment, size); + MP_ERR_JRET(TSDB_CODE_INVALID_PARA); + } + + SMemPool* pPool = (SMemPool*)poolHandle; + SMPSession* pSession = (SMPSession*)session; + + code = mpMalloc(pPool, pSession, &input.size, alignment, &input.pMem); + + MP_SET_FLAG(input.procFlags, (NULL != input.pMem ? MP_STAT_PROC_FLAG_RES_SUCC : MP_STAT_PROC_FLAG_RES_FAIL)); + mpLogStat(pPool, pSession, E_MP_STAT_LOG_MEM_MALLOC, &input); + +_return: + + if (TSDB_CODE_SUCCESS != code) { + terrno = code; + } + + return input.pMem; +} + +void taosMemPoolClose(void* poolHandle) { + if (NULL == poolHandle) { + return; + } + + SMemPool* pPool = (SMemPool*)poolHandle; + + if (tsMemPoolFullFunc) { + mpCheckStatDetail(pPool, NULL, "PoolClose"); + mpDestroyPosStat(&pPool->stat.posStat); + } + + taosMemoryFree(pPool->name); + mpDestroyCacheGroup(&pPool->sessionCache); + + atomic_store_8(&gMPMgmt.modExit, 1); + + (void)taosThreadJoin(gMPMgmt.poolMgmtThread, NULL); +} + + +void taosMemPoolModDestroy(void) { + gMemPoolHandle = NULL; + + taosArrayDestroyEx(gMPMgmt.poolList, mpFreePool); + gMPMgmt.poolList = NULL; +} + + +int32_t taosMemPoolTrim(void* poolHandle, void* session, int32_t size, char* fileName, int32_t lineNo, bool* trimed) { + if (0 == tsMemPoolFullFunc) { + return taosMemTrim(size, trimed); + } + + int32_t code = TSDB_CODE_SUCCESS; + + if (NULL == poolHandle || NULL == fileName || size < 0) { + uError("%s invalid input param, handle:%p, session:%p, fileName:%p, size:%d", + __FUNCTION__, poolHandle, session, fileName, size); + MP_ERR_RET(TSDB_CODE_INVALID_PARA); + } + + SMemPool* pPool = (SMemPool*)poolHandle; + SMPSession* pSession = (SMPSession*)session; + SMPStatInput input = {.size = 0, .file = fileName, .line = lineNo, .procFlags = MP_STAT_PROC_FLAG_EXEC}; + + code = mpTrim(pPool, pSession, size, trimed); + + input.size = (trimed) ? (*trimed) : 0; + + MP_SET_FLAG(input.procFlags, ((0 == code) ? MP_STAT_PROC_FLAG_RES_SUCC : MP_STAT_PROC_FLAG_RES_FAIL)); + mpLogStat(pPool, pSession, E_MP_STAT_LOG_MEM_TRIM, &input); + + return code; +} + +int32_t taosMemPoolCallocJob(uint64_t jobId, uint64_t cId, void** ppJob) { + int32_t code = TSDB_CODE_SUCCESS; + *ppJob = taosMemoryCalloc(1, sizeof(SMPJob)); + if (NULL == *ppJob) { + uError("calloc mp job failed, code: 0x%x", terrno); + MP_ERR_RET(terrno); + } + + SMPJob* pJob = (SMPJob*)*ppJob; + pJob->job.jobId = jobId; + pJob->job.clientId = cId; + + return code; +} + +int32_t taosMemPoolTryLockPool(void* poolHandle, bool readLock) { + if (NULL == poolHandle) { + return TSDB_CODE_INVALID_PARA; + } + + int32_t code = 0; + SMemPool* pPool = (SMemPool*)poolHandle; + if (readLock) { + MP_TRY_LOCK(MP_READ, &pPool->cfgLock, code); + } else { + MP_TRY_LOCK(MP_WRITE, &pPool->cfgLock, code); + } + + return code; +} + +void taosMemPoolUnLockPool(void* poolHandle, bool readLock) { + if (NULL == poolHandle) { + return; + } + + SMemPool* pPool = (SMemPool*)poolHandle; + if (readLock) { + MP_UNLOCK(MP_READ, &pPool->cfgLock); + } else { + MP_UNLOCK(MP_WRITE, &pPool->cfgLock); + } +} + +void taosMemPoolGetUsedSizeBegin(void* poolHandle, int64_t* usedSize, bool* needEnd) { + if (NULL == poolHandle) { + *usedSize = 0; + *needEnd = false; + return; + } + + SMemPool* pPool = (SMemPool*)poolHandle; + + *needEnd = true; + *usedSize = atomic_load_64(&pPool->allocMemSize); +} + +void taosMemPoolGetUsedSizeEnd(void* poolHandle) { + SMemPool* pPool = (SMemPool*)poolHandle; + if (NULL == pPool) { + return; + } + + MP_UNLOCK(MP_WRITE, &pPool->cfgLock); +} + +int32_t taosMemPoolGetSessionStat(void* session, SMPStatDetail** ppStat, int64_t* allocSize, int64_t* maxAllocSize) { + int32_t code = TSDB_CODE_SUCCESS; + if (NULL == session || (NULL == ppStat && NULL == allocSize && NULL == maxAllocSize)) { + uError("%s invalid input param, session:%p, ppStat:%p, allocSize:%p, maxAllocSize:%p", __FUNCTION__, session, ppStat, allocSize, maxAllocSize); + MP_ERR_RET(TSDB_CODE_INVALID_PARA); + } + + SMPSession* pSession = (SMPSession*)session; + + if (ppStat) { + *ppStat = &pSession->stat.statDetail; + } + if (allocSize) { + *allocSize = atomic_load_64(&pSession->allocMemSize); + } + if (maxAllocSize) { + *maxAllocSize = atomic_load_64(&pSession->maxAllocMemSize); + } + + return code; +} + +void taosMemPoolSchedTrim(void) { + mpSchedTrim(NULL); +} + +int32_t taosMemoryPoolInit(mpReserveFailFp failFp, mpReserveReachFp reachFp) { + int32_t code = TSDB_CODE_SUCCESS; + +#ifdef LINUX + if (!tsQueryUseMemoryPool) { +#endif + uInfo("memory pool disabled cause of configured disabled"); + return code; +#ifdef LINUX + } +#endif + + code = taosGetTotalMemory(&tsTotalMemoryKB); + if (TSDB_CODE_SUCCESS != code) { + uInfo("fail to system total memory, error: %s", tstrerror(code)); + return code; + } + + if (tsTotalMemoryKB <= 0) { + uInfo("memory pool disabled since no enough system total memory, size: %" PRId64 "KB", tsTotalMemoryKB); + return code; + } + + uInfo("total memory size: %" PRId64 "KB", tsTotalMemoryKB); + + if (0 == tsMinReservedMemorySize) { + tsMinReservedMemorySize = TMAX(MIN_RESERVE_MEM_SIZE, tsTotalMemoryKB / 1024 * MP_DEFAULT_RESERVE_MEM_PERCENT / 100); + } + + SMemPoolCfg cfg = {0}; + int64_t sysAvailSize = 0; + + code = taosGetSysAvailMemory(&sysAvailSize); + if (code || sysAvailSize < MP_MIN_MEM_POOL_SIZE) { + uInfo("memory pool disabled since no enough system available memory, size: %" PRId64, sysAvailSize); + code = TSDB_CODE_SUCCESS; + return code; + } + + cfg.reserveSize = tsMinReservedMemorySize * 1048576UL; + + int64_t freeSizeAfterRes = sysAvailSize - tsMinReservedMemorySize * 1048576UL; + if (freeSizeAfterRes < MP_MIN_FREE_SIZE_AFTER_RESERVE) { + uInfo("memory pool disabled since no enough system available memory after reservied, size: %" PRId64, freeSizeAfterRes); + return code; + } + + atomic_store_64(&tsCurrentAvailMemorySize, sysAvailSize); + + cfg.evicPolicy = E_EVICT_AUTO; //TODO + cfg.chunkSize = 1048576; + cfg.jobQuota = &tsSingleQueryMaxMemorySize; + cfg.cb.failFp = failFp; + cfg.cb.reachFp = reachFp; + + code = taosMemPoolOpen("taosd", &cfg, &gMemPoolHandle); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + code = mpCreateMgmtThread(); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + uInfo("memory pool initialized, reservedSize:%dMB, freeAfterReserved:%" PRId64 "MB, jobQuota:%dMB", tsMinReservedMemorySize, freeSizeAfterRes/1048576UL, tsSingleQueryMaxMemorySize); + + return code; +} + + +void taosAutoMemoryFree(void *ptr) { + if (NULL != gMemPoolHandle && threadPoolEnabled && threadPoolSession) { + taosMemPoolFree(gMemPoolHandle, threadPoolSession, ptr, __FILE__, __LINE__); + } else { + taosMemFree(ptr); + } +} + + diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index fb8e597163..003150bde1 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -347,11 +347,13 @@ static SPageInfo* getPageInfoFromPayload(void* page) { return ppi; } -int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id, +int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int64_t inMemBufSize, const char* id, const char* dir) { + int32_t code = 0; *pBuf = NULL; SDiskbasedBuf* pPBuf = taosMemoryCalloc(1, sizeof(SDiskbasedBuf)); if (pPBuf == NULL) { + code = terrno; goto _error; } @@ -362,12 +364,14 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem pPBuf->pFile = NULL; pPBuf->id = taosStrdup(id); if (id != NULL && pPBuf->id == NULL) { + code = terrno; goto _error; } pPBuf->fileSize = 0; pPBuf->pFree = taosArrayInit(4, sizeof(SFreeListItem)); pPBuf->freePgList = tdListNew(POINTER_BYTES); if (pPBuf->pFree == NULL || pPBuf->freePgList == NULL) { + code = terrno; goto _error; } @@ -379,6 +383,7 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem pPBuf->inMemPages = inMemBufSize / pagesize; // maximum allowed pages, it is a soft limit. pPBuf->lruList = tdListNew(POINTER_BYTES); if (pPBuf->lruList == NULL) { + code = terrno; goto _error; } @@ -386,17 +391,20 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); pPBuf->pIdList = taosArrayInit(4, POINTER_BYTES); if (pPBuf->pIdList == NULL) { + code = terrno; goto _error; } pPBuf->all = tSimpleHashInit(64, fn); if (pPBuf->all == NULL) { + code = terrno; goto _error; } pPBuf->prefix = (char*)dir; pPBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); if (pPBuf->emptyDummyIdList == NULL) { + code = terrno; goto _error; } @@ -409,7 +417,7 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem _error: destroyDiskbasedBuf(pPBuf); *pBuf = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return code; } static char* doExtractPage(SDiskbasedBuf* pBuf) { diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index b11f65da8f..e8ed86b4bf 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -472,6 +472,20 @@ bool taosTmrStopA(tmr_h* timerId) { return ret; } +bool taosTmrIsStopped(tmr_h* timerId) { + uintptr_t id = (uintptr_t)*timerId; + + tmr_obj_t* timer = findTimer(id); + if (timer == NULL) { + tmrDebug("timer[id=%" PRIuPTR "] does not exist", id); + return true; + } + + uint8_t state = atomic_load_8(&timer->state); + + return (state == TIMER_STATE_CANCELED) || (state == TIMER_STATE_STOPPED); +} + bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId) { tmr_ctrl_t* ctrl = (tmr_ctrl_t*)handle; if (ctrl == NULL || ctrl->label[0] == 0) { diff --git a/source/util/src/tunit.c b/source/util/src/tunit.c index e73045cc89..357e942faa 100644 --- a/source/util/src/tunit.c +++ b/source/util/src/tunit.c @@ -95,33 +95,14 @@ static int32_t parseCfgIntWithUnit(const char* str, int64_t* res) { } int32_t taosStrHumanToInt64(const char* str, int64_t* out) { - int64_t res; + int64_t res; int32_t code = parseCfgIntWithUnit(str, &res); if (code == TSDB_CODE_SUCCESS) *out = (int64_t)res; return code; } -#ifdef BUILD_NO_CALL -void taosInt64ToHumanStr(int64_t val, char* outStr) { - if (((val >= UNIT_ONE_EXBIBYTE) || (-val >= UNIT_ONE_EXBIBYTE)) && ((val % UNIT_ONE_EXBIBYTE) == 0)) { - sprintf(outStr, "%qdE", (long long)val / UNIT_ONE_EXBIBYTE); - } else if (((val >= UNIT_ONE_PEBIBYTE) || (-val >= UNIT_ONE_PEBIBYTE)) && ((val % UNIT_ONE_PEBIBYTE) == 0)) { - sprintf(outStr, "%qdP", (long long)val / UNIT_ONE_PEBIBYTE); - } else if (((val >= UNIT_ONE_TEBIBYTE) || (-val >= UNIT_ONE_TEBIBYTE)) && ((val % UNIT_ONE_TEBIBYTE) == 0)) { - sprintf(outStr, "%qdT", (long long)val / UNIT_ONE_TEBIBYTE); - } else if (((val >= UNIT_ONE_GIBIBYTE) || (-val >= UNIT_ONE_GIBIBYTE)) && ((val % UNIT_ONE_GIBIBYTE) == 0)) { - sprintf(outStr, "%qdG", (long long)val / UNIT_ONE_GIBIBYTE); - } else if (((val >= UNIT_ONE_MEBIBYTE) || (-val >= UNIT_ONE_MEBIBYTE)) && ((val % UNIT_ONE_MEBIBYTE) == 0)) { - sprintf(outStr, "%qdM", (long long)val / UNIT_ONE_MEBIBYTE); - } else if (((val >= UNIT_ONE_KIBIBYTE) || (-val >= UNIT_ONE_KIBIBYTE)) && ((val % UNIT_ONE_KIBIBYTE) == 0)) { - sprintf(outStr, "%qdK", (long long)val / UNIT_ONE_KIBIBYTE); - } else - sprintf(outStr, "%qd", (long long)val); -} -#endif - int32_t taosStrHumanToInt32(const char* str, int32_t* out) { - int64_t res; + int64_t res; int32_t code = parseCfgIntWithUnit(str, &res); if (code == TSDB_CODE_SUCCESS) { if (res < INT32_MIN || res > INT32_MAX) { @@ -131,16 +112,3 @@ int32_t taosStrHumanToInt32(const char* str, int32_t* out) { } return code; } - -#ifdef BUILD_NO_CALL -void taosInt32ToHumanStr(int32_t val, char* outStr) { - if (((val >= UNIT_ONE_GIBIBYTE) || (-val >= UNIT_ONE_GIBIBYTE)) && ((val % UNIT_ONE_GIBIBYTE) == 0)) { - sprintf(outStr, "%qdG", (long long)val / UNIT_ONE_GIBIBYTE); - } else if (((val >= UNIT_ONE_MEBIBYTE) || (-val >= UNIT_ONE_MEBIBYTE)) && ((val % UNIT_ONE_MEBIBYTE) == 0)) { - sprintf(outStr, "%qdM", (long long)val / UNIT_ONE_MEBIBYTE); - } else if (((val >= UNIT_ONE_KIBIBYTE) || (-val >= UNIT_ONE_KIBIBYTE)) && ((val % UNIT_ONE_KIBIBYTE) == 0)) { - sprintf(outStr, "%qdK", (long long)val / UNIT_ONE_KIBIBYTE); - } else - sprintf(outStr, "%qd", (long long)val); -} -#endif diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 0862b5000c..cdcef4b807 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -525,6 +525,64 @@ bool tIsValidFileName(const char *fileName, const char *pattern) { bool tIsValidFilePath(const char *filePath, const char *pattern) { const char *filePathPattern = "^[a-zA-Z0-9:/\\_.-]+$"; - return tIsValidFileName(filePath, pattern ? pattern : filePathPattern); -} \ No newline at end of file +} + +bool taosIsBigChar(char c) { + if (c >= 'A' && c <= 'Z') { + return true; + } else { + return false; + } +} + +bool taosIsSmallChar(char c) { + if (c >= 'a' && c <= 'z') { + return true; + } else { + return false; + } +} + +bool taosIsNumberChar(char c) { + if (c >= '0' && c <= '9') { + return true; + } else { + return false; + } +} + +bool taosIsSpecialChar(char c) { + switch (c) { + case '!': + case '@': + case '#': + case '$': + case '%': + case '^': + case '&': + case '*': + case '(': + case ')': + case '-': + case '_': + case '+': + case '=': + case '[': + case ']': + case '{': + case '}': + case ':': + case ';': + case '>': + case '<': + case '?': + case '|': + case '~': + case ',': + case '.': + return true; + default: + return false; + } +} diff --git a/source/util/src/tversion.c b/source/util/src/tversion.c index d1a2ecf827..1a8d8f2d23 100644 --- a/source/util/src/tversion.c +++ b/source/util/src/tversion.c @@ -18,6 +18,7 @@ #include "taoserror.h" int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) { + int32_t code = 0; if (vstr == NULL) { return terrno = TSDB_CODE_INVALID_VERSION_STRING; } @@ -31,7 +32,10 @@ int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) { if (vstr[spos] != '.') { tmp[spos - tpos] = vstr[spos]; } else { - vnum[vpos] = atoi(tmp); + code = taosStr2int32(tmp, &vnum[vpos]); + if (code != 0) { + return code; + } memset(tmp, 0, sizeof(tmp)); vpos++; tpos = spos + 1; @@ -39,7 +43,10 @@ int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) { } if ('\0' != tmp[0] && vpos < 4) { - vnum[vpos] = atoi(tmp); + code = taosStr2int32(tmp, &vnum[vpos]); + if (code != 0) { + return code; + } } if (vnum[0] <= 0) { diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 55da8c0929..655557b180 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -159,4 +159,13 @@ if(${TD_LINUX}) add_custom_command(TARGET terrorTest POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${ERR_TBL_FILE} $ ) + + # memPoolTest + add_executable(memPoolTest "memPoolTest.cpp") + target_link_libraries(memPoolTest os util common gtest_main) + add_test( + NAME memPoolTest + COMMAND memPoolTest + ) + endif() diff --git a/source/util/test/arrayTest.cpp b/source/util/test/arrayTest.cpp index 307719b984..95468eecaa 100644 --- a/source/util/test/arrayTest.cpp +++ b/source/util/test/arrayTest.cpp @@ -182,8 +182,8 @@ TEST(arrayTest, check_duplicate_needfree) { ASSERT_STREQ(v, value); } - taosArrayClearP(pa, taosMemoryFree); - taosArrayDestroyP(pa, taosMemoryFree); + taosArrayClearP(pa, NULL); + taosArrayDestroyP(pa, NULL); } // over all diff --git a/source/util/test/cfgTest.cpp b/source/util/test/cfgTest.cpp index 33ababef10..a5812d375b 100644 --- a/source/util/test/cfgTest.cpp +++ b/source/util/test/cfgTest.cpp @@ -57,12 +57,12 @@ TEST_F(CfgTest, 02_Basic) { ASSERT_EQ(code, TSDB_CODE_SUCCESS); ASSERT_NE(pConfig, nullptr); - EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 0), 0); - EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0, 0), 0); - EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0, 0), 0); - EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0, 0), 0); - EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0, 0), 0); - EXPECT_EQ(cfgAddDir(pConfig, "test_dir", TD_TMP_DIR_PATH, 0, 0), 0); + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 0, 0), 0); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0, 0, 0), 0); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0, 0, 0), 0); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0, 0, 0), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0, 0, 0), 0); + EXPECT_EQ(cfgAddDir(pConfig, "test_dir", TD_TMP_DIR_PATH, 0, 0, 0), 0); EXPECT_EQ(cfgGetSize(pConfig), 6); diff --git a/source/util/test/memPoolTest.cpp b/source/util/test/memPoolTest.cpp new file mode 100644 index 0000000000..d0c08b1318 --- /dev/null +++ b/source/util/test/memPoolTest.cpp @@ -0,0 +1,2075 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#include + + +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif + +#include "os.h" + +#include "thash.h" +#include "theap.h" +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "stub.h" +#include "../inc/tmempoolInt.h" +#include "tglobal.h" + +namespace { + +#define MPT_PRINTF(param, ...) (void)printf("[%" PRId64 ",%" PRId64 "] " param, mptCaseLoop, mptExecLoop, __VA_ARGS__) +#define MPT_EPRINTF(param, ...) (void)printf(param, __VA_ARGS__) + +#define MPT_MAX_MEM_ACT_TIMES 300 +#define MPT_MAX_SESSION_NUM 100 +#define MPT_MAX_JOB_NUM 100 +#define MPT_MAX_THREAD_NUM 100 +#define MPT_MAX_JOB_LOOP_TIMES 100 + +#define MPT_DEFAULT_RESERVE_MEM_PERCENT 20 +#define MPT_MIN_RESERVE_MEM_SIZE (512 * 1048576UL) +#define MPT_MIN_MEM_POOL_SIZE (1048576UL) +#define MPT_MAX_RETIRE_JOB_NUM 10000 +#define MPT_DEFAULT_TASK_RUN_TIMES 10 +#define MPT_NON_POOL_ALLOC_UNIT (1048576UL) +#define MPT_NON_POOL_KEEP_ALLOC_UNIT (10485760UL * 8) +#define MPT_MAX_NON_POOL_ALLOC_TIMES 30000 + +enum { + MPT_READ = 1, + MPT_WRITE, +}; + +#define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 + + + +threadlocal void* mptThreadPoolHandle = NULL; +threadlocal void* mptThreadPoolSession = NULL; +threadlocal int32_t mptJobNum = 0; +threadlocal int32_t mptExecNum = 0; +threadlocal int32_t mptExecLoop = 0; +threadlocal int64_t mptCaseLoop = 0; + + + + + +#define MPT_SET_TEID(id, tId, eId) \ + do { \ + *(uint64_t *)(id) = (tId); \ + *(uint32_t *)((char *)(id) + sizeof(tId)) = (eId); \ + } while (0) + +#define MPT_SET_QCID(id, qId, cId) \ + do { \ + *(uint64_t *)(id) = (qId); \ + *(uint64_t *)((char *)(id) + sizeof(qId)) = (cId); \ + } while (0) + + +#define mptEnableMemoryPoolUsage(_pool, _session) do { mptThreadPoolHandle = _pool; mptThreadPoolSession = _session; } while (0) +#define mptDisableMemoryPoolUsage() (mptThreadPoolHandle = NULL, mptThreadPoolSession = NULL) +#define mptSaveDisableMemoryPoolUsage(_handle) do { (_handle) = mptThreadPoolHandle; mptThreadPoolHandle = NULL; } while (0) +#define mptRestoreEnableMemoryPoolUsage(_handle) (mptThreadPoolHandle = (_handle)) + +#define mptMemoryMalloc(_size) ((NULL != mptThreadPoolHandle) ? (taosMemPoolMalloc(mptThreadPoolHandle, mptThreadPoolSession, _size, __FILE__, __LINE__)) : (taosMemMalloc(_size))) +#define mptMemoryCalloc(_num, _size) ((NULL != mptThreadPoolHandle) ? (taosMemPoolCalloc(mptThreadPoolHandle, mptThreadPoolSession, _num, _size, __FILE__, __LINE__)) : (taosMemCalloc(_num, _size))) +#define mptMemoryRealloc(_ptr, _size) ((NULL != mptThreadPoolHandle) ? (taosMemPoolRealloc(mptThreadPoolHandle, mptThreadPoolSession, _ptr, _size, __FILE__, __LINE__)) : (taosMemRealloc(_ptr, _size))) +#define mptStrdup(_ptr) ((NULL != mptThreadPoolHandle) ? (taosMemPoolStrdup(mptThreadPoolHandle, mptThreadPoolSession, _ptr, __FILE__, __LINE__)) : (taosStrdupi(_ptr))) +#define mptStrndup(_ptr, _size) ((NULL != mptThreadPoolHandle) ? (taosMemPoolStrndup(mptThreadPoolHandle, mptThreadPoolSession, _ptr, _size, (char*)__FILE__, __LINE__)) : (taosStrndupi(_ptr, _size))) +#define mptMemoryFree(_ptr) ((NULL != mptThreadPoolHandle) ? (taosMemPoolFree(mptThreadPoolHandle, mptThreadPoolSession, _ptr, __FILE__, __LINE__)) : (taosMemFree(_ptr))) +#define mptMemorySize(_ptr) ((NULL != mptThreadPoolHandle) ? (taosMemPoolGetMemorySize(mptThreadPoolHandle, mptThreadPoolSession, _ptr, __FILE__, __LINE__)) : (taosMemSize(_ptr))) +#define mptMemoryTrim(_size, _trimed) ((NULL != mptThreadPoolHandle) ? (taosMemPoolTrim(mptThreadPoolHandle, mptThreadPoolSession, _size, __FILE__, __LINE__, _trimed)) : (taosMemTrim(_size, _trimed))) +#define mptMemoryMallocAlign(_alignment, _size) ((NULL != mptThreadPoolHandle) ? (taosMemPoolMallocAlign(mptThreadPoolHandle, mptThreadPoolSession, _alignment, _size, __FILE__, __LINE__)) : (taosMemMallocAlign(_alignment, _size))) + +enum { + MPT_SMALL_MSIZE = 0, + MPT_BIG_MSIZE, +}; + +typedef struct { + int32_t jobNum; + int32_t sessionNum; + bool memSize[2]; + bool jobQuotaRetire; + bool poolRetire; +} SMPTCaseParam; + +typedef struct SMPTJobInfo { + int8_t retired; + int32_t errCode; + SMemPoolJob* memInfo; + void* pCtx; + + SRWLatch lock; + int8_t destroyed; + SHashObj* pSessions; + int8_t initDone; +} SMPTJobInfo; + + +typedef struct { + int32_t taskActTimes; + int32_t caseLoopTimes; + int32_t jobExecTimes; + int32_t jobNum; + int32_t jobTaskNum; + int64_t maxSingleAllocSize; + bool printExecDetail; + bool printInputRow; + + bool lockDbg; +} SMPTestCtrl; + +typedef struct { + void* p; + int64_t size; +} SMPTestMemInfo; + +typedef struct { + uint64_t taskId; + SRWLatch taskExecLock; + bool destoryed; + + int64_t poolMaxUsedSize; + int64_t poolTotalUsedSize; + + SMPStatDetail stat; + + int32_t memIdx; + SMPTestMemInfo* pMemList; + + bool taskFreed; + int32_t lastAct; +} SMPTestTaskCtx; + +typedef struct { + SRWLatch jobExecLock; + + int32_t jobIdx; + int64_t jobId; + int32_t initTimes; + void* pSessions[MPT_MAX_SESSION_NUM]; + int32_t taskNum; + SMPTestTaskCtx taskCtxs[MPT_MAX_SESSION_NUM]; + + int32_t taskRunningNum; + SMPTJobInfo* pJob; + int32_t jobStatus; +} SMPTestJobCtx; + +typedef struct { + int32_t jobQuota; + bool enableMemPool; + bool reserveMode; + int64_t upperLimitSize; + int32_t reserveSize; //MB + int32_t threadNum; + int32_t randTask; +} SMPTestParam; + +typedef struct { + int64_t initNum; + int64_t retireNum; + int64_t destoryNum; +} SMPTestJobStat; + +typedef struct { + int32_t idx; + TdThread threadFp; + bool allJobs; + bool autoJob; +} SMPTestThread; + +typedef struct SMPTestCtx { + int64_t qId; + int64_t tId; + SHashObj* pJobs; + BoundedQueue* pJobQueue; + SMPTestThread threadCtxs[MPT_MAX_THREAD_NUM]; + TdThread dropThreadFp; + TdThread nPoolThreadFp; + int32_t jobNum; + int64_t totalTaskNum; + SMPTestJobCtx* jobCtxs; + SMPTestParam param; + SMPTestJobStat runStat; + + SRWLatch stringLock; + char* pSrcString; + + bool initDone; + int8_t testDone; + int64_t jobLoop; + + int32_t npIdx; + SMPTestMemInfo* npMemList; +} SMPTestCtx; + +SMPTestCtx mptCtx = {0}; +SMPTestCtrl mptCtrl = {0}; + +static int32_t MPT_TRY_LOCK(int32_t type, SRWLatch *_lock) { + int32_t code = -1; + + if (MPT_READ == (type)) { + if (mptCtrl.lockDbg) { + if (atomic_load_32((_lock)) < 0) { + uError("invalid lock value before try read lock"); + return -1; + } + uDebug("MPT TRY RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); + } + code = taosRTryLockLatch(_lock); + if (mptCtrl.lockDbg) { + uDebug("MPT TRY RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); + if (atomic_load_32((_lock)) <= 0) { + uError("invalid lock value after try read lock"); + return -1; + } + } + } else { + if (mptCtrl.lockDbg) { + if (atomic_load_32((_lock)) < 0) { + uError("invalid lock value before try write lock"); + return -1; + } + uDebug("MPT TRY WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); + } + code = taosWTryLockLatch(_lock); + if (mptCtrl.lockDbg) { + uDebug("MPT TRY WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); + if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { + uError("invalid lock value after try write lock"); + return -1; + } + } + } + + return code; +} + +#define MPT_LOCK(type, _lock) \ + do { \ + if (MPT_READ == (type)) { \ + if (mptCtrl.lockDbg) { \ + if (atomic_load_32((_lock)) < 0) { \ + uError("invalid lock value before read lock"); \ + break; \ + } \ + uDebug("MPT RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + } \ + taosRLockLatch(_lock); \ + if (mptCtrl.lockDbg) { \ + uDebug("MPT RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) <= 0) { \ + uError("invalid lock value after read lock"); \ + break; \ + } \ + } \ + } else { \ + if (mptCtrl.lockDbg) { \ + if (atomic_load_32((_lock)) < 0) { \ + uError("invalid lock value before write lock"); \ + break; \ + } \ + uDebug("MPT WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + } \ + taosWLockLatch(_lock); \ + if (mptCtrl.lockDbg) { \ + uDebug("MPT WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ + uError("invalid lock value after write lock"); \ + break; \ + } \ + } \ + } \ + } while (0) + +#define MPT_UNLOCK(type, _lock) \ + do { \ + if (MPT_READ == (type)) { \ + if (mptCtrl.lockDbg) { \ + if (atomic_load_32((_lock)) <= 0) { \ + uError("invalid lock value before read unlock"); \ + break; \ + } \ + uDebug("MPT RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + } \ + taosRUnLockLatch(_lock); \ + if (mptCtrl.lockDbg) { \ + uDebug("MPT RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) < 0) { \ + uError("invalid lock value after read unlock"); \ + break; \ + } \ + } \ + } else { \ + if (mptCtrl.lockDbg) { \ + if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ + uError("invalid lock value before write unlock"); \ + break; \ + } \ + uDebug("MPT WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + } \ + taosWUnLockLatch(_lock); \ + if (mptCtrl.lockDbg) { \ + uDebug("MPT WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) < 0) { \ + uError("invalid lock value after write unlock"); \ + break; \ + } \ + } \ + } \ + } while (0) + + + + +#if 0 +void joinTestReplaceRetrieveFp() { + static Stub stub; + stub.set(getNextBlockFromDownstreamRemain, getDummyInputBlock); + { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("getNextBlockFromDownstreamRemain", result); + for (const auto &f : result) { + stub.set(f.second, getDummyInputBlock); + } +#endif +#ifdef LINUX + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^getNextBlockFromDownstreamRemain$", result); + for (const auto &f : result) { + stub.set(f.second, getDummyInputBlock); + } +#endif + } +} +#endif + +void mptInitLogFile() { + const char *defaultLogFileNamePrefix = "mplog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + qDebugFlag = 159; + uDebugFlag = 159; + tsNumOfLogLines = INT32_MAX; + tsLogKeepDays = 10; + TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); + + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + MPT_PRINTF("failed to open log file in directory:%s\n", tsLogDir); + } + taosSetNoNewFile(); +} + +static bool mptJobMemSizeCompFn(void* l, void* r, void* param) { + SMPTJobInfo* left = (SMPTJobInfo*)l; + SMPTJobInfo* right = (SMPTJobInfo*)r; + if (atomic_load_8(&right->retired)) { + return true; + } + + return atomic_load_64(&right->memInfo->allocMemSize) < atomic_load_64(&left->memInfo->allocMemSize); +} + +void mptDeleteJobQueueData(void* pData) { + SMPTJobInfo* pJob = (SMPTJobInfo*)pData; + taosHashRelease(mptCtx.pJobs, pJob); +} + + +void mptDestroyJobInfo(void* job) { + SMPTJobInfo* pJob = (SMPTJobInfo*)job; + + taosMemFree(pJob->memInfo); + taosHashCleanup(pJob->pSessions); +} + + +void mptWriteMem(void* pStart, int64_t size) { + char* pEnd = (char*)pStart + size - 1; + char* p = (char*)pStart; + while (p <= pEnd) { + *p = 'a' + taosRand() % 26; + p += 4096; + } +} + + +void mptInit() { + osDefaultInit(); + mptInitLogFile(); + + mptCtrl.caseLoopTimes = 100000; + mptCtrl.taskActTimes = 0; + mptCtrl.maxSingleAllocSize = 104857600 * 5; + mptCtrl.jobNum = 100; + mptCtrl.jobExecTimes = 10; + mptCtrl.jobTaskNum = 0; + + mptCtx.pJobs = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + ASSERT_TRUE(NULL != mptCtx.pJobs); + + taosHashSetFreeFp(mptCtx.pJobs, mptDestroyJobInfo); + + mptCtx.pJobQueue = createBoundedQueue(10000, mptJobMemSizeCompFn, mptDeleteJobQueueData, NULL); + ASSERT_TRUE(NULL != mptCtx.pJobQueue); + mptCtx.jobCtxs = (SMPTestJobCtx*)taosMemoryCalloc(MPT_MAX_JOB_NUM, sizeof(*mptCtx.jobCtxs)); + ASSERT_TRUE(NULL != mptCtx.jobCtxs); + + mptCtx.pSrcString = (char*)taosMemoryMalloc(mptCtrl.maxSingleAllocSize); + ASSERT_TRUE(NULL != mptCtx.pSrcString); + memset(mptCtx.pSrcString, 'P', mptCtrl.maxSingleAllocSize - 1); + mptCtx.pSrcString[mptCtrl.maxSingleAllocSize - 1] = 0; + +} + +void mptDestroySession(uint64_t qId, int64_t tId, int32_t eId, int32_t taskIdx, SMPTestJobCtx* pJobCtx, void* session) { + SMPTJobInfo *pJobInfo = pJobCtx->pJob; + char id[sizeof(tId) + sizeof(eId) + 1] = {0}; + MPT_SET_TEID(id, tId, eId); + int32_t remainSessions = atomic_sub_fetch_32(&pJobInfo->memInfo->remainSession, 1); + + (void)taosHashRemove(pJobInfo->pSessions, id, sizeof(id)); + + taosMemPoolDestroySession(gMemPoolHandle, session); + + if (0 == remainSessions) { + if (0 == taosHashGetSize(pJobInfo->pSessions)) { + atomic_store_8(&pJobInfo->destroyed, 1); + + uDebug("JOB:0x%x idx:%d destroyed, code:0x%x", pJobCtx->jobId, pJobCtx->jobIdx, pJobInfo->errCode); + + atomic_add_fetch_64(&mptCtx.runStat.destoryNum, 1); + + (void)taosHashRemove(mptCtx.pJobs, &qId, sizeof(qId)); + + pJobCtx->pJob = NULL; + uInfo("the whole query job removed"); + } + } +} + + +void mptDestroyTaskCtx(SMPTestJobCtx* pJobCtx, int32_t taskIdx) { + assert(gMemPoolHandle); + + SMPTestTaskCtx* pTask = &pJobCtx->taskCtxs[taskIdx]; + + if (mptCtx.param.enableMemPool) { + mptEnableMemoryPoolUsage(gMemPoolHandle, pJobCtx->pSessions[taskIdx]); + } + + for (int32_t i = 0; i < pTask->memIdx; ++i) { + pTask->stat.times.memFree.exec++; + pTask->stat.bytes.memFree.exec+=mptMemorySize(pTask->pMemList[i].p); + pTask->stat.bytes.memFree.succ+=mptMemorySize(pTask->pMemList[i].p); + mptMemoryFree(pTask->pMemList[i].p); + pTask->pMemList[i].p = NULL; + } + + if (mptCtx.param.enableMemPool) { + mptDisableMemoryPoolUsage(); + } + + mptDestroySession(pJobCtx->jobId, pJobCtx->taskCtxs[taskIdx].taskId, 0, taskIdx, pJobCtx, pJobCtx->pSessions[taskIdx]); + pJobCtx->pSessions[taskIdx] = NULL; + + taosMemFreeClear(pTask->pMemList); + + pTask->destoryed = true; +} + + +int32_t mptInitJobInfo(uint64_t qId, SMPTJobInfo* pJob) { + pJob->pSessions= taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == pJob->pSessions) { + uError("fail to init session hash, code: 0x%x", terrno); + return terrno; + } + + int32_t code = taosMemPoolCallocJob(qId, 0, (void**)&pJob->memInfo); + if (TSDB_CODE_SUCCESS != code) { + taosHashCleanup(pJob->pSessions); + pJob->pSessions = NULL; + return code; + } + + return code; +} + + +int32_t mptInitSession(uint64_t qId, uint64_t tId, int32_t eId, SMPTestJobCtx* pJobCtx, void** ppSession) { + int32_t code = TSDB_CODE_SUCCESS; + SMPTJobInfo* pJob = NULL; + + while (true) { + pJob = (SMPTJobInfo*)taosHashAcquire(mptCtx.pJobs, &qId, sizeof(qId)); + if (NULL == pJob) { + SMPTJobInfo jobInfo = {0}; + code = mptInitJobInfo(qId, &jobInfo); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + code = taosHashPut(mptCtx.pJobs, &qId, sizeof(qId), &jobInfo, sizeof(jobInfo)); + if (TSDB_CODE_SUCCESS != code) { + mptDestroyJobInfo(&jobInfo); + if (TSDB_CODE_DUP_KEY == code) { + code = TSDB_CODE_SUCCESS; + continue; + } + + return code; + } + + pJob = (SMPTJobInfo*)taosHashAcquire(mptCtx.pJobs, &qId, sizeof(qId)); + if (NULL == pJob) { + uError("QID:0x%" PRIx64 " not in joj hash, may be dropped", qId); + return TSDB_CODE_QRY_JOB_NOT_EXIST; + } + } + + break; + } + + atomic_store_ptr(&pJobCtx->pJob, pJob); + pJob->pCtx = pJobCtx; + + char id[sizeof(tId) + sizeof(eId) + 1] = {0}; + MPT_SET_TEID(id, tId, eId); + + assert(0 == taosMemPoolInitSession(gMemPoolHandle, ppSession, pJob->memInfo, id)); + + atomic_add_fetch_32(&pJob->memInfo->remainSession, 1); + + assert(0 == taosHashPut(pJob->pSessions, id, sizeof(id), ppSession, POINTER_BYTES)); + + atomic_store_8(&pJob->initDone, 1); + +_return: + + if (NULL != pJob) { + taosHashRelease(mptCtx.pJobs, pJob); + } + + return code; +} + + + +void mptInitTask(int32_t idx, int32_t eId, SMPTestJobCtx* pJob) { + pJob->taskCtxs[idx].taskId = atomic_add_fetch_64(&mptCtx.tId, 1); + + ASSERT_TRUE(0 == mptInitSession(pJob->jobId, pJob->taskCtxs[idx].taskId, eId, pJob, &pJob->pSessions[idx])); + + pJob->taskCtxs[idx].pMemList = (SMPTestMemInfo*)taosMemoryCalloc(MPT_MAX_MEM_ACT_TIMES, sizeof(*pJob->taskCtxs[idx].pMemList)); + ASSERT_TRUE(NULL != pJob->taskCtxs[idx].pMemList); + + pJob->taskCtxs[idx].destoryed = false; + + uDebug("JOB:0x%x TASK:0x%x idx:%d initialized", pJob->jobId, pJob->taskCtxs[idx].taskId, idx); +} + +void mptInitJob(int32_t idx) { + SMPTestJobCtx* pJobCtx = &mptCtx.jobCtxs[idx]; + + pJobCtx->jobIdx = idx; + pJobCtx->jobId = atomic_add_fetch_64(&mptCtx.qId, 1); + pJobCtx->taskNum = (mptCtrl.jobTaskNum) ? mptCtrl.jobTaskNum : ((taosRand() % 10) ? (taosRand() % (MPT_MAX_SESSION_NUM/10)) : (taosRand() % MPT_MAX_SESSION_NUM)) + 1; + pJobCtx->initTimes++; + + if (!mptCtx.initDone) { + atomic_add_fetch_64(&mptCtx.totalTaskNum, pJobCtx->taskNum); + } + + for (int32_t i = 0; i < pJobCtx->taskNum; ++i) { + mptInitTask(i, 0, pJobCtx); + assert(pJobCtx->pJob); + } + + atomic_add_fetch_64(&mptCtx.runStat.initNum, 1); + + uDebug("JOB:0x%x idx:%d initialized, total times:%d, taskNum:%d", pJobCtx->jobId, idx, pJobCtx->initTimes, pJobCtx->taskNum); +} + + +void mptDestroyTask(SMPTestJobCtx* pJobCtx, int32_t taskIdx) { + if (mptCtx.param.enableMemPool && tsMemPoolFullFunc) { + SMPStatDetail* pStat = NULL; + int64_t allocSize = 0; + taosMemPoolGetSessionStat(pJobCtx->pSessions[taskIdx], &pStat, &allocSize, NULL); + int64_t usedSize = MEMPOOL_GET_USED_SIZE(pStat); + + assert(allocSize == usedSize); + assert(0 == memcmp(pStat, &pJobCtx->taskCtxs[taskIdx].stat, sizeof(*pStat))); + } + + mptDestroyTaskCtx(pJobCtx, taskIdx); +} + +int32_t mptDestroyJob(SMPTestJobCtx* pJobCtx, bool reset) { + uint64_t jobId = pJobCtx->jobId; + for (int32_t i = 0; i < pJobCtx->taskNum; ++i) { + if (!pJobCtx->taskCtxs[i].destoryed) { + mptDestroyTask(pJobCtx, i); + } + } + + //mptDestroyJobInfo(pJobCtx->pJob); + //(void)taosHashRemove(mptCtx.pJobs, &pJobCtx->jobId, sizeof(pJobCtx->jobId)); + + if (reset) { + int32_t jobIdx = pJobCtx->jobIdx; + memset((char*)pJobCtx + sizeof(pJobCtx->jobExecLock), 0, sizeof(SMPTestJobCtx) - sizeof(pJobCtx->jobExecLock)); + mptInitJob(jobIdx); + } + + MPT_PRINTF(" JOB:0x%x retired\n", jobId); + + return 0; +} + +void mptCheckCompareJobInfo(SMPTestJobCtx* pJobCtx) { + +} + +int32_t mptResetJob(SMPTestJobCtx* pJobCtx) { + if (MPT_TRY_LOCK(MPT_WRITE, &pJobCtx->jobExecLock)) { + return -1; + } + + if (NULL == atomic_load_ptr(&pJobCtx->pJob)) { + int32_t jobIdx = pJobCtx->jobIdx; + memset((char*)pJobCtx + sizeof(pJobCtx->jobExecLock), 0, sizeof(SMPTestJobCtx) - sizeof(pJobCtx->jobExecLock)); + mptInitJob(jobIdx); + + MPT_UNLOCK(MPT_WRITE, &pJobCtx->jobExecLock); + return 0; + } + + int32_t code = 0; + if (atomic_load_8(&pJobCtx->pJob->retired)) { + int32_t taskRunning = atomic_load_32(&pJobCtx->taskRunningNum); + if (0 == taskRunning) { + code = mptDestroyJob(pJobCtx, true); + } else { + uDebug("JOB:0x%x retired but will not destroy cause of task running, num:%d", pJobCtx->jobId, taskRunning); + code = -1; + } + } + + MPT_UNLOCK(MPT_WRITE, &pJobCtx->jobExecLock); + + return 0; +} + +bool mptRetireJob(SMPTJobInfo* pJob) { + SMPTestJobCtx* pCtx = (SMPTestJobCtx*)pJob->pCtx; + + if (MPT_TRY_LOCK(MPT_WRITE, &pCtx->jobExecLock)) { + return false; + } + + bool retired = false; + int32_t taskRunning = atomic_load_32(&pCtx->taskRunningNum); + if (0 == taskRunning) { + mptDestroyJob(pCtx, false); + retired = true; + } else { + uDebug("JOB:0x%x retired but will not destroy cause of task running, num:%d", pCtx->jobId, taskRunning); + } + + MPT_UNLOCK(MPT_WRITE, &pCtx->jobExecLock); + + return retired; +} + +int32_t mptGetMemPoolMaxMemSize(void* pHandle, int64_t* maxSize) { + int64_t freeSize = 0; + int64_t usedSize = 0; + bool needEnd = false; + + taosMemPoolGetUsedSizeBegin(pHandle, &usedSize, &needEnd); + int32_t code = taosGetSysAvailMemory(&freeSize); + if (needEnd) { + taosMemPoolGetUsedSizeEnd(pHandle); + } + + if (TSDB_CODE_SUCCESS != code) { + uError("get system avaiable memory size failed, error: 0x%x", code); + return code; + } + + int64_t totalSize = freeSize + usedSize; + int64_t reserveSize = TMAX(totalSize * MPT_DEFAULT_RESERVE_MEM_PERCENT / 100 / 1048576UL * 1048576UL, MPT_MIN_RESERVE_MEM_SIZE); + int64_t availSize = (totalSize - reserveSize) / 1048576UL * 1048576UL; + if (availSize < MPT_MIN_MEM_POOL_SIZE) { + uError("too little available query memory, totalAvailable: %" PRId64 ", reserveSize: %" PRId64, totalSize, reserveSize); + //return TSDB_CODE_QRY_TOO_FEW_AVAILBLE_MEM; + } + + uDebug("new pool maxSize:%" PRId64 ", usedSize:%" PRId64 ", freeSize:%" PRId64, availSize, usedSize, freeSize); + + *maxSize = availSize; + + return TSDB_CODE_SUCCESS; +} + +void mptRetireJobsCb(int64_t retireSize, int32_t errCode) { + SMPTJobInfo* pJob = (SMPTJobInfo*)taosHashIterate(mptCtx.pJobs, NULL); + int32_t jobNum = 0; + uint64_t jobId = 0; + int64_t retiredSize = 0; + while (retiredSize < retireSize && NULL != pJob) { + if (atomic_load_8(&pJob->retired) || 0 == atomic_load_8(&pJob->initDone)) { + pJob = (SMPTJobInfo*)taosHashIterate(mptCtx.pJobs, pJob); + continue; + } + + if (0 == atomic_val_compare_exchange_32(&pJob->errCode, 0, errCode) && 0 == atomic_val_compare_exchange_8(&pJob->retired, 0, 1)) { + int64_t aSize = atomic_load_64(&pJob->memInfo->allocMemSize); + jobId = pJob->memInfo->jobId; + + atomic_add_fetch_64(&mptCtx.runStat.retireNum, 1); + + bool retired = mptRetireJob(pJob); + if (retired) { + retiredSize += aSize; + } + + jobNum++; + + uDebug("QID:0x%" PRIx64 " job mark retired cause of limit reached, retired:%d, usedSize:%" PRId64 ", retireSize:%" PRId64 ", retiredSize:%" PRId64, + jobId, retired, aSize, retireSize, retiredSize); + } + + pJob = (SMPTJobInfo*)taosHashIterate(mptCtx.pJobs, pJob); + } + + taosHashCancelIterate(mptCtx.pJobs, pJob); + + uDebug("total %d jobs mark retired, retiredSize:%" PRId64 " targetRetireSize:%" PRId64, jobNum, retiredSize, retireSize); +} + + +void mptRetireJobCb(uint64_t jobId, uint64_t clientId, int32_t errCode) { + SMPTJobInfo* pJob = (SMPTJobInfo*)taosHashGet(mptCtx.pJobs, &jobId, sizeof(jobId)); + if (NULL == pJob) { + uError("QID:0x%" PRIx64 " fail to get job from job hash", jobId); + return; + } + + if (0 == atomic_val_compare_exchange_32(&pJob->errCode, 0, errCode) && 0 == atomic_val_compare_exchange_8(&pJob->retired, 0, 1)) { + uInfo("QID:0x%" PRIx64 " mark retired, errCode: 0x%x, allocSize:%" PRId64, jobId, errCode, atomic_load_64(&pJob->memInfo->allocMemSize)); + atomic_add_fetch_64(&mptCtx.runStat.retireNum, 1); + } else { + uDebug("QID:0x%" PRIx64 " already retired, retired: %d, errCode: 0x%x, allocSize:%" PRId64, jobId, atomic_load_8(&pJob->retired), atomic_load_32(&pJob->errCode), atomic_load_64(&pJob->memInfo->allocMemSize)); + } +} + +void mptInitPool(void) { + assert(0 == taosMemoryPoolInit(mptRetireJobsCb, mptRetireJobCb)); +} + + +void mptSimulateAction(SMPTestJobCtx* pJobCtx, SMPTestTaskCtx* pTask) { + int32_t actId = 0; + bool actDone = false; + int32_t size = 0; + int32_t osize = 0, nsize = 0; + + while (!actDone) { + actId = taosRand() % 10; + size = (taosRand() % 8) ? (taosRand() % (mptCtrl.maxSingleAllocSize / 100)) : (taosRand() % mptCtrl.maxSingleAllocSize); + + switch (actId) { + case 0: { // malloc + if (pTask->memIdx >= MPT_MAX_MEM_ACT_TIMES) { + break; + } + + pTask->pMemList[pTask->memIdx].p = mptMemoryMalloc(size); + if (NULL == pTask->pMemList[pTask->memIdx].p) { + pTask->stat.times.memMalloc.exec++; + pTask->stat.bytes.memMalloc.exec+=size; + pTask->stat.times.memMalloc.fail++; + pTask->stat.bytes.memMalloc.fail+=size; + uError("JOB:0x%x TASK:0x%x mpMalloc %d failed, error:%s", pJobCtx->jobId, pTask->taskId, size, tstrerror(terrno)); + return; + } + + nsize = mptMemorySize(pTask->pMemList[pTask->memIdx].p); + pTask->stat.times.memMalloc.exec++; + pTask->stat.bytes.memMalloc.exec+=nsize; + pTask->stat.bytes.memMalloc.succ+=nsize; + pTask->stat.times.memMalloc.succ++; + + mptWriteMem(pTask->pMemList[pTask->memIdx].p, size); + + pTask->memIdx++; + pTask->lastAct = actId; + actDone = true; + break; + } + case 1: { // calloc + if (pTask->memIdx >= MPT_MAX_MEM_ACT_TIMES) { + break; + } + + pTask->pMemList[pTask->memIdx].p = mptMemoryCalloc(1, size); + if (NULL == pTask->pMemList[pTask->memIdx].p) { + pTask->stat.times.memCalloc.exec++; + pTask->stat.bytes.memCalloc.exec+=size; + pTask->stat.times.memCalloc.fail++; + pTask->stat.bytes.memCalloc.fail+=size; + uError("JOB:0x%x TASK:0x%x mpCalloc %d failed, error:%s", pJobCtx->jobId, pTask->taskId, size, tstrerror(terrno)); + return; + } + + nsize = mptMemorySize(pTask->pMemList[pTask->memIdx].p); + + pTask->stat.times.memCalloc.exec++; + pTask->stat.bytes.memCalloc.exec+=nsize; + pTask->stat.times.memCalloc.succ++; + pTask->stat.bytes.memCalloc.succ+=nsize; + + mptWriteMem(pTask->pMemList[pTask->memIdx].p, size); + + pTask->memIdx++; + pTask->lastAct = actId; + actDone = true; + break; + } + case 2:{ // new realloc + break; + if (pTask->memIdx >= MPT_MAX_MEM_ACT_TIMES) { + break; + } + + pTask->pMemList[pTask->memIdx].p = mptMemoryRealloc(NULL, size); + if (NULL == pTask->pMemList[pTask->memIdx].p) { + pTask->stat.times.memRealloc.exec++; + pTask->stat.bytes.memRealloc.exec+=size; + pTask->stat.times.memRealloc.fail++; + pTask->stat.bytes.memRealloc.fail+=size; + uError("JOB:0x%x TASK:0x%x new mpRealloc %d failed, error:%s", pJobCtx->jobId, pTask->taskId, size, tstrerror(terrno)); + return; + } + + nsize = mptMemorySize(pTask->pMemList[pTask->memIdx].p); + + pTask->stat.times.memRealloc.exec++; + pTask->stat.bytes.memRealloc.exec+=nsize; + pTask->stat.bytes.memRealloc.succ+=nsize; + pTask->stat.times.memRealloc.succ++; + + mptWriteMem(pTask->pMemList[pTask->memIdx].p, size); + + pTask->memIdx++; + pTask->lastAct = actId; + actDone = true; + break; + } + case 3:{ // real realloc + break; + if (pTask->memIdx <= 0) { + break; + } + + assert(pTask->pMemList[pTask->memIdx - 1].p); + osize = mptMemorySize(pTask->pMemList[pTask->memIdx - 1].p); + size++; + pTask->pMemList[pTask->memIdx - 1].p = mptMemoryRealloc(pTask->pMemList[pTask->memIdx - 1].p, size); + if (NULL == pTask->pMemList[pTask->memIdx - 1].p) { + pTask->stat.times.memRealloc.exec++; + pTask->stat.bytes.memRealloc.exec+=size; + pTask->stat.bytes.memRealloc.origExec+=osize; + pTask->stat.times.memRealloc.fail++; + pTask->stat.bytes.memRealloc.fail+=size; + + pTask->stat.times.memFree.exec++; + pTask->stat.bytes.memFree.exec+=osize; + pTask->stat.times.memFree.succ++; + pTask->stat.bytes.memFree.succ+=osize; + uError("JOB:0x%x TASK:0x%x real mpRealloc %d failed, error:%s", pJobCtx->jobId, pTask->taskId, size, tstrerror(terrno)); + pTask->memIdx--; + return; + } + + nsize = mptMemorySize(pTask->pMemList[pTask->memIdx - 1].p); + pTask->stat.times.memRealloc.exec++; + pTask->stat.bytes.memRealloc.exec+=nsize; + pTask->stat.bytes.memRealloc.origExec+=osize; + pTask->stat.bytes.memRealloc.origSucc+=osize; + pTask->stat.times.memRealloc.succ++; + pTask->stat.bytes.memRealloc.succ+=nsize; + + mptWriteMem(pTask->pMemList[pTask->memIdx - 1].p, size); + + pTask->lastAct = actId; + actDone = true; + break; + } + case 4:{ // realloc free + if (pTask->memIdx <= 0) { + break; + } + + assert(pTask->pMemList[pTask->memIdx - 1].p); + osize = mptMemorySize(pTask->pMemList[pTask->memIdx - 1].p); + + pTask->pMemList[pTask->memIdx - 1].p = mptMemoryRealloc(pTask->pMemList[pTask->memIdx - 1].p, 0); + pTask->stat.times.memFree.exec++; + pTask->stat.bytes.memFree.exec+=osize; + assert(NULL == pTask->pMemList[pTask->memIdx - 1].p); + + pTask->stat.times.memFree.succ++; + pTask->stat.bytes.memFree.succ+=osize; + + pTask->memIdx--; + pTask->lastAct = actId; + actDone = true; + break; + } + case 5:{ // strdup + if (pTask->memIdx >= MPT_MAX_MEM_ACT_TIMES) { + break; + } + + size /= 10; + MPT_LOCK(MPT_WRITE, &mptCtx.stringLock); + mptCtx.pSrcString[size] = 0; + pTask->pMemList[pTask->memIdx].p = mptStrdup(mptCtx.pSrcString); + mptCtx.pSrcString[size] = 'W'; + MPT_UNLOCK(MPT_WRITE, &mptCtx.stringLock); + + if (NULL == pTask->pMemList[pTask->memIdx].p) { + pTask->stat.times.memStrdup.exec++; + pTask->stat.bytes.memStrdup.exec+=size + 1; + pTask->stat.times.memStrdup.fail++; + pTask->stat.bytes.memStrdup.fail+=size + 1; + uError("JOB:0x%x TASK:0x%x mpStrdup %d failed, error:%s", pJobCtx->jobId, pTask->taskId, size, tstrerror(terrno)); + return; + } + + nsize = mptMemorySize(pTask->pMemList[pTask->memIdx].p); + pTask->stat.times.memStrdup.exec++; + pTask->stat.bytes.memStrdup.exec+= nsize; + + pTask->stat.times.memStrdup.succ++; + pTask->stat.bytes.memStrdup.succ+=nsize; + + mptWriteMem(pTask->pMemList[pTask->memIdx].p, size); + + pTask->memIdx++; + pTask->lastAct = actId; + actDone = true; + break; + } + case 6:{ // strndup + if (pTask->memIdx >= MPT_MAX_MEM_ACT_TIMES) { + break; + } + + size /= 10; + + MPT_LOCK(MPT_WRITE, &mptCtx.stringLock); + assert(strlen(mptCtx.pSrcString) > size); + pTask->pMemList[pTask->memIdx].p = mptStrndup(mptCtx.pSrcString, size); + MPT_UNLOCK(MPT_WRITE, &mptCtx.stringLock); + + if (NULL == pTask->pMemList[pTask->memIdx].p) { + pTask->stat.times.memStrndup.exec++; + pTask->stat.bytes.memStrndup.exec+=size + 1; + pTask->stat.times.memStrndup.fail++; + pTask->stat.bytes.memStrndup.fail+=size + 1; + uError("JOB:0x%x TASK:0x%x mpStrndup %d failed, error:%s", pJobCtx->jobId, pTask->taskId, size, tstrerror(terrno)); + return; + } + + assert(strlen((char*)pTask->pMemList[pTask->memIdx].p) == size); + nsize = mptMemorySize(pTask->pMemList[pTask->memIdx].p); + + pTask->stat.times.memStrndup.exec++; + pTask->stat.bytes.memStrndup.exec+=nsize; + pTask->stat.times.memStrndup.succ++; + pTask->stat.bytes.memStrndup.succ+=nsize; + + mptWriteMem(pTask->pMemList[pTask->memIdx].p, size); + + pTask->memIdx++; + pTask->lastAct = actId; + actDone = true; + break; + } + case 7:{ // free + if (pTask->memIdx <= 0) { + break; + } + + assert(pTask->pMemList[pTask->memIdx - 1].p); + osize = mptMemorySize(pTask->pMemList[pTask->memIdx - 1].p); + mptMemoryFree(pTask->pMemList[pTask->memIdx - 1].p); + pTask->stat.times.memFree.exec++; + pTask->stat.times.memFree.succ++; + pTask->stat.bytes.memFree.exec+=osize; + pTask->stat.bytes.memFree.succ+=osize; + pTask->pMemList[pTask->memIdx - 1].p = NULL; + + pTask->memIdx--; + pTask->lastAct = actId; + actDone = true; + break; + } + case 8:{ // trim + bool trimed = false; + int32_t code = mptMemoryTrim(0, &trimed); + pTask->stat.times.memTrim.exec++; + if (code) { + pTask->stat.times.memTrim.fail++; + } else { + pTask->stat.times.memTrim.succ++; + if (trimed) { + pTask->stat.bytes.memTrim.succ++; + } + } + pTask->lastAct = actId; + actDone = true; + break; + } + case 9: { // malloc_align + if (pTask->memIdx >= MPT_MAX_MEM_ACT_TIMES) { + break; + } + + pTask->pMemList[pTask->memIdx].p = mptMemoryMallocAlign(8, size); + if (NULL == pTask->pMemList[pTask->memIdx].p) { + pTask->stat.times.memMalloc.exec++; + pTask->stat.bytes.memMalloc.exec+=size; + pTask->stat.times.memMalloc.fail++; + pTask->stat.bytes.memMalloc.fail+=size; + uError("JOB:0x%x TASK:0x%x mpMallocAlign %d failed, error:%s", pJobCtx->jobId, pTask->taskId, size, tstrerror(terrno)); + return; + } + + nsize = mptMemorySize(pTask->pMemList[pTask->memIdx].p); + + mptWriteMem(pTask->pMemList[pTask->memIdx].p, size); + + pTask->stat.times.memMalloc.exec++; + pTask->stat.bytes.memMalloc.exec+=nsize; + + pTask->stat.times.memMalloc.succ++; + pTask->stat.bytes.memMalloc.succ+=nsize; + pTask->memIdx++; + pTask->lastAct = actId; + actDone = true; + break; + } + default: + assert(0); + break; + } + } +} + +void mptSimulateTask(SMPTestJobCtx* pJobCtx, SMPTestTaskCtx* pTask, int32_t actTimes) { + uDebug("JOB:0x%x TASK:0x%x will start total %d actions", pJobCtx->jobId, pTask->taskId, actTimes); + + for (int32_t i = 0; i < actTimes; ++i) { + if (atomic_load_8(&pJobCtx->pJob->retired)) { + uDebug("JOB:0x%x TASK:0x%x stop running cause of job already retired", pJobCtx->jobId, pTask->taskId); + return; + } + + //MPT_PRINTF("\tTASK:0x%x will start %d:%d actions\n", pTask->taskId, i, actTimes); + + mptSimulateAction(pJobCtx, pTask); + } +} + +void mptSimulateOutTask(int64_t targetSize) { + SMPTestMemInfo* pCtx = &mptCtx.npMemList[mptCtx.npIdx]; + pCtx->size = targetSize; + pCtx->p = taosMemMalloc(pCtx->size); + if (NULL == pCtx->p) { + uError("non-pool sim malloc %" PRId64 " failed", pCtx->size); + pCtx->size = 0; + return; + } + + mptWriteMem(pCtx->p, pCtx->size); + + mptCtx.npIdx++; +} + + +void mptTaskRun(SMPTestJobCtx* pJobCtx, SMPTestTaskCtx* pCtx, int32_t idx, int32_t actTimes) { + uDebug("JOB:0x%x TASK:0x%x start running", pJobCtx->jobId, pCtx->taskId); + + if (atomic_load_8(&pJobCtx->pJob->retired)) { + uDebug("JOB:0x%x TASK:0x%x stop running cause of job already retired", pJobCtx->jobId, pCtx->taskId); + return; + } + + if (taosWTryLockLatch(&pCtx->taskExecLock)) { + uDebug("JOB:0x%x TASK:0x%x stop running cause of task already running", pJobCtx->jobId, pCtx->taskId); + return; + } + + atomic_add_fetch_32(&pJobCtx->taskRunningNum, 1); + + if (mptCtx.param.enableMemPool) { + mptEnableMemoryPoolUsage(gMemPoolHandle, pJobCtx->pSessions[idx]); + } + + mptSimulateTask(pJobCtx, pCtx, actTimes); + + if (mptCtx.param.enableMemPool) { + mptDisableMemoryPoolUsage(); + } + + //mptSimulateOutTask(pJobCtx, pCtx); + + taosWUnLockLatch(&pCtx->taskExecLock); + + atomic_sub_fetch_32(&pJobCtx->taskRunningNum, 1); + + uDebug("JOB:0x%x TASK:0x%x end running", pJobCtx->jobId, pCtx->taskId); +} + + +void mptInitJobs() { + int32_t jobNum = mptCtrl.jobNum ? mptCtrl.jobNum : MPT_MAX_JOB_NUM; + + memset(mptCtx.jobCtxs, 0, sizeof(*mptCtx.jobCtxs) * jobNum); + mptCtx.totalTaskNum = 0; + + for (int32_t i = 0; i < jobNum; ++i) { + mptInitJob(i); + } +} + +void mptCheckPoolUsedSize(int32_t jobNum) { + int64_t usedSize = 0; + bool needEnd = false; + int64_t poolUsedSize = 0; + int32_t sleepTimes = 0; + + while (true) { + if (taosMemPoolTryLockPool(gMemPoolHandle, false)) { + taosUsleep(1); + continue; + } + + taosMemPoolGetUsedSizeBegin(gMemPoolHandle, &usedSize, &needEnd); + + poolUsedSize = 0; + + for (int32_t i = 0; i < jobNum; ++i) { + SMPTestJobCtx* pJobCtx = &mptCtx.jobCtxs[i]; + + sleepTimes = 0; + while (MPT_TRY_LOCK(MPT_READ, &pJobCtx->jobExecLock)) { + taosUsleep(1); + sleepTimes++; + if (sleepTimes > 100) { + break; + } + } + + if (sleepTimes > 100) { + break; + } + + if (NULL == pJobCtx->pJob) { + MPT_UNLOCK(MPT_READ, &pJobCtx->jobExecLock); + continue; + } + + int64_t jobUsedSize = 0; + for (int32_t m = 0; m < pJobCtx->taskNum; ++m) { + if (!pJobCtx->taskCtxs[m].destoryed) { + SMPStatDetail* pStat = NULL; + int64_t allocSize = 0; + taosMemPoolGetSessionStat(pJobCtx->pSessions[m], &pStat, &allocSize, NULL); + int64_t usedSize = MEMPOOL_GET_USED_SIZE(pStat); + + assert(allocSize == usedSize); + assert(0 == memcmp(pStat, &pJobCtx->taskCtxs[m].stat, sizeof(*pStat))); + + jobUsedSize += allocSize; + } + } + + assert(pJobCtx->pJob->memInfo->allocMemSize == jobUsedSize); + + MPT_UNLOCK(MPT_READ, &pJobCtx->jobExecLock); + + poolUsedSize += jobUsedSize; + } + + taosMemPoolGetUsedSizeEnd(gMemPoolHandle); + + if (sleepTimes > 100) { + continue; + } + + assert(poolUsedSize <= usedSize); + break; + } +} + +void mptLaunchSingleTask(SMPTestThread* pThread, SMPTestJobCtx* pJobCtx, int32_t taskIdx, int32_t actTimes) { + if (atomic_load_8(&pJobCtx->pJob->retired) || pJobCtx->taskCtxs[taskIdx].destoryed) { + return; + } + + MPT_PRINTF("Thread %d start to run %d:%d task\n", pThread->idx, taskIdx, pJobCtx->taskNum); + mptTaskRun(pJobCtx, &pJobCtx->taskCtxs[taskIdx], taskIdx, actTimes); + MPT_PRINTF("Thread %d end %d:%d task\n", pThread->idx, taskIdx, pJobCtx->taskNum); + +} + +void mptRunRandTasks(SMPTestThread* pThread) { + int64_t runTaskTimes = mptCtx.totalTaskNum * MPT_DEFAULT_TASK_RUN_TIMES, taskExecIdx = 0; + int32_t jobNum = mptCtrl.jobNum ? mptCtrl.jobNum : MPT_MAX_JOB_NUM; + int32_t jobIdx = 0, taskIdx = 0, code = 0; + SMPTestJobCtx* pJobCtx = NULL; + + MPT_PRINTF("Thread %d start the %d:%d exection - runTaskTimes:%" PRId64 "\n", pThread->idx, mptExecLoop, mptExecNum, runTaskTimes); + + while (runTaskTimes > 0) { + int32_t actTimes = mptCtrl.taskActTimes ? mptCtrl.taskActTimes : ((taosRand() % 10) ? (taosRand() % (MPT_MAX_MEM_ACT_TIMES/10)) : (taosRand() % MPT_MAX_MEM_ACT_TIMES)); + jobIdx = taosRand() % jobNum; + + pJobCtx = &mptCtx.jobCtxs[jobIdx]; + + if (mptResetJob(pJobCtx)) { + continue; + } + + if (MPT_TRY_LOCK(MPT_READ, &pJobCtx->jobExecLock)) { + continue; + } + + taskIdx = taosRand() % pJobCtx->taskNum; + + if (atomic_load_8(&pJobCtx->pJob->retired) || pJobCtx->taskCtxs[taskIdx].destoryed) { + MPT_UNLOCK(MPT_READ, &pJobCtx->jobExecLock); + continue; + } + + MPT_PRINTF("Thread %d start to run %d:%d task\n", pThread->idx, taskExecIdx, runTaskTimes); + mptTaskRun(pJobCtx, &pJobCtx->taskCtxs[taskIdx], taskIdx, actTimes); + MPT_PRINTF("Thread %d end %d:%d task\n", pThread->idx, taskExecIdx, runTaskTimes); + + MPT_UNLOCK(MPT_READ, &pJobCtx->jobExecLock); + + runTaskTimes--; + taskExecIdx++; + } + +} + +void mptRunLoopJobs(SMPTestThread* pThread) { + mptJobNum = (mptCtrl.jobNum) ? mptCtrl.jobNum : (taosRand() % MPT_MAX_JOB_NUM + 1); + + MPT_PRINTF("Thread %d start the %d:%d exection - jobNum:%d\n", pThread->idx, mptExecLoop, mptExecNum, mptJobNum); + + for (int32_t i = 0; i < mptJobNum; ++i) { + SMPTestJobCtx* pJobCtx = &mptCtx.jobCtxs[i]; + + if (mptResetJob(pJobCtx)) { + continue; + } + + if (MPT_TRY_LOCK(MPT_READ, &pJobCtx->jobExecLock)) { + continue; + } + + MPT_PRINTF(" Thread %d start to run %d:%d job[%d:0x%" PRIx64 "]\n", pThread->idx, i, mptJobNum, pJobCtx->jobIdx, pJobCtx->jobId); + + for (int32_t m = 0; m < pJobCtx->taskNum; ++m) { + if (atomic_load_8(&pJobCtx->pJob->retired)) { + break; + } + + int32_t actTimes = mptCtrl.taskActTimes ? mptCtrl.taskActTimes : ((taosRand() % 10) ? (taosRand() % (MPT_MAX_MEM_ACT_TIMES/10)) : (taosRand() % MPT_MAX_MEM_ACT_TIMES)); + mptLaunchSingleTask(pThread, pJobCtx, m, actTimes); + } + + MPT_UNLOCK(MPT_READ, &pJobCtx->jobExecLock); + + MPT_PRINTF(" Thread %d end %dth JOB 0x%x exec, retired:%d\n", pThread->idx, pJobCtx->jobIdx, pJobCtx->jobId, pJobCtx->pJob->retired); + } +} + +void* mptRunThreadFunc(void* param) { + SMPTestThread* pThread = (SMPTestThread*)param; + mptExecNum = (mptCtrl.jobExecTimes) ? mptCtrl.jobExecTimes : taosRand() % MPT_MAX_JOB_LOOP_TIMES + 1; + + for (int32_t n = 0; n < mptExecNum; ++n) { + mptExecLoop = n; + + if (mptCtx.param.randTask) { + mptRunRandTasks(pThread); + } else { + mptRunLoopJobs(pThread); + } + + MPT_PRINTF("Thread %d finish the %dth exection\n", pThread->idx, n); + + if (mptCtx.param.threadNum <= 1 && mptCtx.param.enableMemPool && tsMemPoolFullFunc) { + mptCheckPoolUsedSize(mptJobNum); + } + } + + return NULL; +} + +void* mptNonPoolThreadFunc(void* param) { + int64_t targetSize = MPT_NON_POOL_ALLOC_UNIT; + int64_t allocSize = 0; + + while (!atomic_load_8(&mptCtx.testDone)) { + mptSimulateOutTask(targetSize); + allocSize += targetSize; + + MPT_EPRINTF("%d:Non-pool malloc and write %" PRId64 " bytes, keep size:%" PRId64 "\n", mptCtx.npIdx - 1, targetSize, allocSize); + taosUsleep(1); + + if ((mptCtx.npIdx * targetSize) >= (tsMinReservedMemorySize * 1048576UL * 10)) { + for (int32_t i = 0; i < mptCtx.npIdx; ++i) { + taosMemFreeClear(mptCtx.npMemList[i].p); + } + + mptCtx.npIdx = 0; + targetSize += MPT_NON_POOL_ALLOC_UNIT; + allocSize = 0; + taosMsleep(100); + } + } + + return NULL; +} + + +void* mptDropThreadFunc(void* param) { + int32_t jobIdx = 0, taskIdx = 0, code = 0; + uint64_t taskId = 0; + int32_t jobNum = mptCtrl.jobNum ? mptCtrl.jobNum : MPT_MAX_JOB_NUM; + + while (!atomic_load_8(&mptCtx.testDone)) { + taosMsleep(400); + + MPT_EPRINTF("%" PRId64 " - initJobs:%" PRId64 " retireJobs:%" PRId64 " destoryJobs:%" PRId64 " remainJobs:%" PRId64 "\n", taosGetTimestampMs(), + mptCtx.runStat.initNum, mptCtx.runStat.retireNum, mptCtx.runStat.destoryNum, mptCtx.runStat.initNum - mptCtx.runStat.destoryNum); + + if (taosMemPoolTryLockPool(gMemPoolHandle, true)) { + continue; + } + + jobIdx = taosRand() % jobNum; + SMPTestJobCtx* pJobCtx = &mptCtx.jobCtxs[jobIdx]; + MPT_LOCK(MPT_WRITE, &pJobCtx->jobExecLock); + if (NULL == pJobCtx->pJob || pJobCtx->pJob->destroyed) { + MPT_UNLOCK(MPT_WRITE, &pJobCtx->jobExecLock); + taosMemPoolUnLockPool(gMemPoolHandle, true); + continue; + } + + if (taosRand() % 20) { + taskIdx = taosRand() % pJobCtx->taskNum; + if (pJobCtx->taskCtxs[taskIdx].destoryed) { + MPT_UNLOCK(MPT_WRITE, &pJobCtx->jobExecLock); + taosMemPoolUnLockPool(gMemPoolHandle, true); + continue; + } + + taskId = pJobCtx->taskCtxs[taskIdx].taskId; + mptDestroyTask(pJobCtx, taskIdx); + MPT_EPRINTF("Drop Thread destroy task %d:0x%" PRIx64 " in job %d:%" PRIx64 "\n", taskIdx, taskId, jobIdx, pJobCtx->jobId); + + MPT_UNLOCK(MPT_WRITE, &pJobCtx->jobExecLock); + } else { + code = mptDestroyJob(pJobCtx, false); + if (0 == code) { + MPT_EPRINTF("Drop Thread destroy job %d:%" PRIx64 "\n", jobIdx, pJobCtx->jobId); + } + MPT_UNLOCK(MPT_WRITE, &pJobCtx->jobExecLock); + } + + taosMemPoolUnLockPool(gMemPoolHandle, true); + } + + return NULL; +} + + +void mptStartRunThread(int32_t threadIdx) { + TdThreadAttr thattr; + ASSERT_EQ(0, taosThreadAttrInit(&thattr)); + ASSERT_EQ(0, taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE)); + mptCtx.threadCtxs[threadIdx].idx = threadIdx; + ASSERT_EQ(0, taosThreadCreate(&mptCtx.threadCtxs[threadIdx].threadFp, &thattr, mptRunThreadFunc, &mptCtx.threadCtxs[threadIdx])); + ASSERT_EQ(0, taosThreadAttrDestroy(&thattr)); +} + +void mptStartDropThread() { + TdThreadAttr thattr; + ASSERT_EQ(0, taosThreadAttrInit(&thattr)); + ASSERT_EQ(0, taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE)); + ASSERT_EQ(0, taosThreadCreate(&mptCtx.dropThreadFp, &thattr, mptDropThreadFunc, NULL)); + ASSERT_EQ(0, taosThreadAttrDestroy(&thattr)); +} + +void mptStartNonPoolThread() { + TdThreadAttr thattr; + ASSERT_EQ(0, taosThreadAttrInit(&thattr)); + ASSERT_EQ(0, taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE)); + ASSERT_EQ(0, taosThreadCreate(&mptCtx.nPoolThreadFp, &thattr, mptNonPoolThreadFunc, NULL)); + ASSERT_EQ(0, taosThreadAttrDestroy(&thattr)); +} + + + + + + +void mptDestroyJobs() { + int32_t jobNum = mptCtrl.jobNum ? mptCtrl.jobNum : MPT_MAX_JOB_NUM; + + for (int32_t i = 0; i < jobNum; ++i) { + mptDestroyJob(&mptCtx.jobCtxs[i], false); + } + + +} + +void mptDestroyNonPoolCtx() { + for (int32_t i = 0; i < mptCtx.npIdx; ++i) { + taosMemFreeClear(mptCtx.npMemList[i].p); + } + taosMemFreeClear(mptCtx.npMemList); +} + +void mptInitNonPoolCtx() { + mptCtx.npMemList = (SMPTestMemInfo*)taosMemoryCalloc(MPT_MAX_NON_POOL_ALLOC_TIMES, sizeof(*mptCtx.npMemList)); + ASSERT_TRUE(NULL != mptCtx.npMemList); +} + +void mptRunCase(SMPTestParam* param, int32_t times) { + MPT_PRINTF("\t case start the %dth running\n", times); + + mptCaseLoop = times; + memcpy(&mptCtx.param, param, sizeof(SMPTestParam)); + tsSingleQueryMaxMemorySize = param->jobQuota; + + atomic_store_8(&mptCtx.testDone, 0); + mptCtx.initDone = false; + + mptInitPool(); + + mptInitJobs(); + + mptInitNonPoolCtx(); + + mptCtx.initDone = true; + + for (int32_t i = 0; i < mptCtx.param.threadNum; ++i) { + mptStartRunThread(i); + } + + mptStartDropThread(); + mptStartNonPoolThread(); + + for (int32_t i = 0; i < mptCtx.param.threadNum; ++i) { + (void)taosThreadJoin(mptCtx.threadCtxs[i].threadFp, NULL); + } + + atomic_store_8(&mptCtx.testDone, 1); + + (void)taosThreadJoin(mptCtx.dropThreadFp, NULL); + (void)taosThreadJoin(mptCtx.nPoolThreadFp, NULL); + + mptDestroyJobs(); + mptDestroyNonPoolCtx(); + + taosMemPoolClose(gMemPoolHandle); + + while (gMemPoolHandle) { + taosMsleep(10); + } + + MPT_PRINTF("\t case end the %dth running\n", times); +} + +void mptPrintTestBeginInfo(char* caseName, SMPTestParam* param) { + MPT_PRINTF("Case [%s] begins:\n", caseName); + MPT_PRINTF("\t case loop times: %d\n", mptCtrl.caseLoopTimes); + MPT_PRINTF("\t task max act times: %d\n", mptCtrl.taskActTimes ? mptCtrl.taskActTimes : MPT_MAX_MEM_ACT_TIMES); + MPT_PRINTF("\t max single alloc size: %" PRId64 "\n", mptCtrl.maxSingleAllocSize); + MPT_PRINTF("\t job quota size: %dMB\n", param->jobQuota); + MPT_PRINTF("\t reserve mode: %d\n", param->reserveMode); + MPT_PRINTF("\t reserve size: %dMB\n", param->reserveSize); + MPT_PRINTF("\t test thread num: %d\n", param->threadNum); + MPT_PRINTF("\t random exec task: %d\n", param->randTask); +} + +void mptFreeAddrList(void** pList, int32_t num) { + for (int32_t i = 0; i < num; ++i) { + assert(pList[i]); + taosMemFree(pList[i]); + } +} + +} // namespace + +#if 1 + +#if 0 +TEST(PerfTest, GetSysAvail) { + char* caseName = "PerfTest:GetSysAvail"; + int32_t code = 0; + + int64_t msize = 1048576UL*10240; + char* p = (char*)taosMemMalloc(msize); + int64_t st = taosGetTimestampUs(); + memset(p, 0, msize); + int64_t totalUs = taosGetTimestampUs() - st; + printf("memset %" PRId64 " used time:%" PRId64 "us, speed:%dMB/ms\n", msize, totalUs, msize/1048576UL/(totalUs/1000UL)); + + int64_t freeSize = 0; + int32_t loopTimes = 1000000; + st = taosGetTimestampUs(); + int64_t lt = st; + for (int32_t i = 0; i < loopTimes; ++i) { + code = taosGetSysAvailMemory(&freeSize); + assert(0 == code); + //taosMsleep(1); + } + totalUs = taosGetTimestampUs() - st; + + printf("%d times getSysMemory total time:%" PRId64 "us, avg:%dus\n", loopTimes, totalUs, totalUs/loopTimes); +} +#endif + +#if 0 +TEST(MiscTest, monSysAvailSize) { + char* caseName = "MiscTest:monSysAvailSize"; + int32_t code = 0; + + int64_t freeSize = 0; + int32_t loopTimes = 1000000000; + for (int32_t i = 0; i < loopTimes; ++i) { + code = taosGetSysAvailMemory(&freeSize); + assert(0 == code); + printf(" %" PRId64, freeSize); + if (i && 0 == (i % 10)) { + struct tm Tm, *ptm; + struct timeval timeSecs; + + TAOS_UNUSED(taosGetTimeOfDay(&timeSecs)); + time_t curTime = timeSecs.tv_sec; + ptm = taosLocalTime(&curTime, &Tm, NULL, 0); + + printf("- %02d/%02d %02d:%02d:%02d.%06d \n", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec); + } + taosMsleep(1); + } +} +#endif + + +#if 0 +TEST(MiscTest, simNonPoolAct) { + char* caseName = "MiscTest:simNonPoolAct"; + int64_t msize = 1048576UL*1024, asize = 0; + int32_t loopTimes = 1000000; + + for (int32_t i = 0; i < loopTimes; ++i) { + asize = taosRand() % msize; + void* p = taosMemMalloc(asize); + mptWriteMem(p, asize); + + taosMsleep(100); + taosMemFree(p); + + printf("sim %dth alloc/free %" PRId64 " bytes\n", i, asize); + } +} +#endif + + +#if 0 +TEST(PerfTest, allocLatency) { + char* caseName = "PerfTest:allocLatency"; + int32_t code = 0; + + int64_t msize = 10; + void* pSession = NULL; + void* pJob = NULL; + + mptInitPool(); + + memset(mptCtx.jobCtxs, 0, sizeof(*mptCtx.jobCtxs)); + + assert(0 == taosMemPoolCallocJob(0, 0, (void**)&pJob)); + assert(0 == taosMemPoolInitSession(gMemPoolHandle, &pSession, pJob, "id")); + + int32_t loopTimes = 10000000; + int64_t st = 0; + void **addrList = (void**)taosMemCalloc(loopTimes, POINTER_BYTES); + + + // MALLOC + + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMalloc(msize); + } + int64_t totalUs3 = taosGetTimestampUs() - st; + mptFreeAddrList(addrList, loopTimes); + + + + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMalloc(msize); + } + int64_t totalUs1 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMalloc(msize); + } + int64_t totalUs2 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + + + // CALLOC + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryCalloc(1, msize); + } + int64_t totalUs11 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryCalloc(1, msize); + } + int64_t totalUs12 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryCalloc(1, msize); + } + int64_t totalUs13 = taosGetTimestampUs() - st; + //mptFreeAddrList(addrList, loopTimes); NO FREE FOR REALLOC + + // REALLOC + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryRealloc(addrList[i], msize); + } + int64_t totalUs21 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + + + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryRealloc(addrList[i], msize); + } + int64_t totalUs22 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + + + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryRealloc(addrList[i], msize); + } + int64_t totalUs23 = taosGetTimestampUs() - st; + mptFreeAddrList(addrList, loopTimes); + + + // STRDUP + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptStrdup("abc"); + } + int64_t totalUs31 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptStrdup("abc"); + } + int64_t totalUs32 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptStrdup("abc"); + } + int64_t totalUs33 = taosGetTimestampUs() - st; + mptFreeAddrList(addrList, loopTimes); + + // STRNDUP + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptStrndup("abc", 3); + } + int64_t totalUs41 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptStrndup("abc", 3); + } + int64_t totalUs42 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptStrndup("abc", 3); + } + int64_t totalUs43 = taosGetTimestampUs() - st; + mptFreeAddrList(addrList, loopTimes); + + // ALIGNALLOC + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMallocAlign(8, msize); + } + int64_t totalUs51 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMallocAlign(8, msize); + } + int64_t totalUs52 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMallocAlign(8, msize); + } + int64_t totalUs53 = taosGetTimestampUs() - st; + //mptFreeAddrList(addrList, loopTimes); NO FREE FOR GETSIZE + + + // GETSIZE + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + mptMemorySize(addrList[i]); + } + int64_t totalUs61 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + + + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + mptMemorySize(addrList[i]); + } + int64_t totalUs62 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + + + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + mptMemorySize(addrList[i]); + } + int64_t totalUs63 = taosGetTimestampUs() - st; + + // FREE + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + mptMemoryFree(addrList[i]); + } + int64_t totalUs71 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + + + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMalloc(msize); + } + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + mptMemoryFree(addrList[i]); + } + int64_t totalUs72 = taosGetTimestampUs() - st; + mptDisableMemoryPoolUsage(); + + + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMalloc(msize); + } + st = taosGetTimestampUs(); + for (int32_t i = 0; i < loopTimes; ++i) { + mptMemoryFree(addrList[i]); + } + int64_t totalUs73 = taosGetTimestampUs() - st; + + + printf("%d times each %" PRId64 " bytes, time consumed:\n" + "\tnon-fpool malloc total time:%" PRId64 "us, avg:%fus\n" + "\tfull-pool malloc total time:%" PRId64 "us, avg:%fus\n" + "\tdirect malloc total time:%" PRId64 "us, avg:%fus\n" + "\tnon-fpool calloc total time:%" PRId64 "us, avg:%fus\n" + "\tfull-pool calloc total time:%" PRId64 "us, avg:%fus\n" + "\tdirect calloc total time:%" PRId64 "us, avg:%fus\n" + "\tnon-fpool realloc total time:%" PRId64 "us, avg:%fus\n" + "\tfull-pool realloc total time:%" PRId64 "us, avg:%fus\n" + "\tdirect realloc total time:%" PRId64 "us, avg:%fus\n" + "\tnon-fpool strdup total time:%" PRId64 "us, avg:%fus\n" + "\tfull-pool strdup total time:%" PRId64 "us, avg:%fus\n" + "\tdirect strdup total time:%" PRId64 "us, avg:%fus\n" + "\tnon-fpool strndup total time:%" PRId64 "us, avg:%fus\n" + "\tfull-pool strndup total time:%" PRId64 "us, avg:%fus\n" + "\tdirect strndup total time:%" PRId64 "us, avg:%fus\n" + "\tnon-fpool alignal total time:%" PRId64 "us, avg:%fus\n" + "\tfull-pool alignal total time:%" PRId64 "us, avg:%fus\n" + "\tdirect alignal total time:%" PRId64 "us, avg:%fus\n" + "\tnon-fpool getsize total time:%" PRId64 "us, avg:%fus\n" + "\tfull-pool getsize total time:%" PRId64 "us, avg:%fus\n" + "\tdirect getsize total time:%" PRId64 "us, avg:%fus\n" + "\tnon-fpool free total time:%" PRId64 "us, avg:%fus\n" + "\tfull-pool free total time:%" PRId64 "us, avg:%fus\n" + "\tdirect free total time:%" PRId64 "us, avg:%fus\n", + loopTimes, msize, + totalUs1, ((double)totalUs1)/loopTimes, totalUs2, ((double)totalUs2)/loopTimes, totalUs3, ((double)totalUs3)/loopTimes, + totalUs11, ((double)totalUs11)/loopTimes, totalUs12, ((double)totalUs12)/loopTimes, totalUs13, ((double)totalUs13)/loopTimes, + totalUs21, ((double)totalUs21)/loopTimes, totalUs22, ((double)totalUs22)/loopTimes, totalUs23, ((double)totalUs23)/loopTimes, + totalUs31, ((double)totalUs31)/loopTimes, totalUs32, ((double)totalUs32)/loopTimes, totalUs33, ((double)totalUs33)/loopTimes, + totalUs41, ((double)totalUs41)/loopTimes, totalUs42, ((double)totalUs42)/loopTimes, totalUs43, ((double)totalUs43)/loopTimes, + totalUs51, ((double)totalUs51)/loopTimes, totalUs52, ((double)totalUs52)/loopTimes, totalUs53, ((double)totalUs53)/loopTimes, + totalUs61, ((double)totalUs61)/loopTimes, totalUs62, ((double)totalUs62)/loopTimes, totalUs63, ((double)totalUs63)/loopTimes, + totalUs71, ((double)totalUs71)/loopTimes, totalUs72, ((double)totalUs72)/loopTimes, totalUs73, ((double)totalUs73)/loopTimes); +} +#endif + + +#if 0 +TEST(poolFuncTest, SingleThreadTest) { + char* caseName = "poolFuncTest:SingleThreadTest"; + SMPTestParam param = {0}; + param.reserveMode = true; + param.threadNum = 1; + param.jobQuota = 1024; + param.enableMemPool = true; + + tsMemPoolFullFunc = 0; + + mptPrintTestBeginInfo(caseName, ¶m); + + for (int32_t i = 0; i < mptCtrl.caseLoopTimes; ++i) { + mptRunCase(¶m, i); + } + +} +#endif +#if 0 +TEST(poolFuncTest, MultiThreadTest) { + char* caseName = "poolFuncTest:MultiThreadTest"; + SMPTestParam param = {0}; + param.reserveMode = true; + param.threadNum = 6; + param.jobQuota = 1024; + param.randTask = true; + param.enableMemPool = true; + + tsMemPoolFullFunc = 0; + + mptPrintTestBeginInfo(caseName, ¶m); + + for (int32_t i = 0; i < mptCtrl.caseLoopTimes; ++i) { + mptRunCase(¶m, i); + } + +} +#endif + +#if 0 +TEST(poolFullFuncTest, SingleThreadTest) { + char* caseName = "poolFullFuncTest:SingleThreadTest"; + SMPTestParam param = {0}; + param.reserveMode = true; + param.threadNum = 1; + param.jobQuota = 1024; + param.enableMemPool = true; + + tsMemPoolFullFunc = 1; + + mptPrintTestBeginInfo(caseName, ¶m); + + for (int32_t i = 0; i < mptCtrl.caseLoopTimes; ++i) { + mptRunCase(¶m, i); + } + +} +#endif +#if 0 +TEST(poolFullFuncTest, MultiThreadTest) { + char* caseName = "poolFullFuncTest:MultiThreadTest"; + SMPTestParam param = {0}; + param.reserveMode = true; + param.threadNum = 6; + param.jobQuota = 1024; + param.randTask = true; + param.enableMemPool = true; + + tsMemPoolFullFunc = 1; + + mptPrintTestBeginInfo(caseName, ¶m); + + for (int32_t i = 0; i < mptCtrl.caseLoopTimes; ++i) { + mptRunCase(¶m, i); + } + +} +#endif + + +#if 0 +TEST(DisablePoolFuncTest, MultiThreadTest) { + char* caseName = "FuncTest:MultiThreadTest"; + SMPTestParam param = {0}; + param.reserveMode = true; + param.threadNum = 6; + param.jobQuota = 1024; + param.randTask = true; + param.enableMemPool = false; + + mptPrintTestBeginInfo(caseName, ¶m); + + for (int32_t i = 0; i < mptCtrl.caseLoopTimes; ++i) { + mptRunCase(¶m, i); + } + +} +#endif + + + + +#endif + + + + + + + + + + + +int main(int argc, char** argv) { + taosSeedRand(taosGetTimestampSec()); + mptInit(); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + + +#pragma GCC diagnosti diff --git a/tests/army/alter/alterConfig.py b/tests/army/alter/alterConfig.py index 6a22dd014f..3fd0a51576 100644 --- a/tests/army/alter/alterConfig.py +++ b/tests/army/alter/alterConfig.py @@ -173,6 +173,10 @@ class TDTestCase(TBase): self.checkRows("select * from stb0", 2, 1) time.sleep(1) + def alter_err_case(self): + tdSql.error(f"alter local 'audit 0'",expectErrInfo="Config not found") + tdSql.error(f"alter dnode 1 'audit 1'",expectErrInfo="Invalid config option") + # run def run(self): tdLog.debug(f"start to excute {__file__}") @@ -185,6 +189,8 @@ class TDTestCase(TBase): self.alterCachemodel() # TD-32907 self.alterBypassFlag() + # TS-5007 + self.alter_err_case() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/army/alter/test_alter_config.py b/tests/army/alter/test_alter_config.py new file mode 100644 index 0000000000..1b59293187 --- /dev/null +++ b/tests/army/alter/test_alter_config.py @@ -0,0 +1,652 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame.epath import * +from frame.srvCtl import * +from frame import * + +class TDTestCase: + """This test case is used to veirfy hot refresh configurations + """ + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + self.configration_dic = { + "cli": [ + { + "name": "asyncLog", + "value": 0, + "category": "local", + }, + { + "name": "enableQueryHb", + "value": 0, + "category": "local", + }, + { + "name": "keepColumnName", + "value": 1, + "category": "local", + }, + { + "name": "logKeepDays", + "value": 30, + "category": "local", + }, + { + "name": "maxInsertBatchRows", + "value": 2000000, + "category": "local", + }, + { + "name": "minSlidingTime", + "value": 10000, + "category": "local", + }, + { + "name": "minIntervalTime", + "value": 2, + "category": "local", + }, + { + "name": "numOfLogLines", + "value": 20000000, + "category": "local", + + }, + { + "name": "querySmaOptimize", + "value": 1, + "category": "local", + }, + { + "name": "queryPolicy", + "value": 3, + "category": "local", + }, + { + "name": "queryTableNotExistAsEmpty", + "value": 1, + "category": "local", + }, + { + "name": "queryPlannerTrace", + "value": 1, + "category": "local", + }, + { + "name": "queryNodeChunkSize", + "value": 16 * 1024, + "category": "local", + }, + { + "name": "queryUseNodeAllocator", + "value": 0, + "category": "local", + }, + { + "name": "smlDot2Underline", + "value": 0, + "category": "local", + }, + { + "name": "useAdapter", + "value": 1, + "category": "local", + }, + # { + # "name": "multiResultFunctionStarReturnTags", + # "value": 1, + # "category": "local", + # }, + { + "name": "maxTsmaCalcDelay", + "value": 1200, + "category": "local", + }, + { + "name": "tsmaDataDeleteMark", + "value": 1000 * 60 * 60 * 12, + "category": "local", + }, + { + "name": "bypassFlag", + "value": 4, + "category": "local", + } + ], + "svr": [ + { + "name": "audit", + "value": 0, + "category": "global" + }, + { + "name": "asyncLog", + "value": 0, + "category": "local" + }, + { + "name": "disableStream", + "value": 1, + "category": "global" + }, + { + "name": "enableWhiteList", + "value": 1, + "category": "global" + }, + { + "name": "statusInterval", + "value": 3, + "category": "global" + }, + { + "name": "telemetryReporting", + "value" : 1, + "category" : "global" + }, + { + "name" : "monitor", + "value" : 0, + "category" : "global" + }, + { + "name" : "monitorInterval", + "value" : 3, + "category" : "global" + }, + { + "name" : "monitorComp", + "value" : 1, + "category" : "global" + }, + { + "name" : "monitorForceV2", + "value" : 0, + "category" : "global" + }, + { + "name" : "monitorLogProtocol", + "value" : 1, + "category" : "global" + }, + { + "name": "monitorMaxLogs", + "value": 1000, + "category": "global" + }, + { + "name": "auditCreateTable", + "value": 0, + "category": "global" + }, + { + "name": "auditInterval", + "value": 4000, + "category": "global" + }, + { + "name": "slowLogThreshold", + "value": 20, + "category": "global" + }, + { + "name": "compressMsgSize", + "value": 0, + "category": "global" + }, + { + "name": "compressor", + "value": "GZIP_COMPRESSOR", + "category": "global" + }, + { + "name": "curRange", + "value": 200, + "category": "global" + }, + { + "name": "fPrecision", + "value": "1000.000000", + "category": "global" + }, + { + "name": "dPrecision", + "value": "1000.000000", + "category": "global" + }, + { + "name": "ifAdtFse", + "value": 1, + "category": "global" + }, + { + "name": "maxRange", + "value": 1000, + "category": "global" + }, + { + "name": "maxTsmaNum", + "value": 2, + "category": "global" + }, + { + "name": "queryRsmaTolerance", + "value": 2000, + "category": "global" + }, + { + "name": "uptimeInterval", + "value": 600, + "category": "global" + }, + { + "name": "slowLogMaxLen", + "value": 8192, + "category": "global" + }, + { + "name": "slowLogScope", + "value": "insert", + "category": "global" + }, + { + "name": "slowLogExceptDb", + "value": "db1", + "category": "global" + }, + { + "name": "mndSdbWriteDelta", + "value": 1000, + "category": "global" + }, + { + "name": "minDiskFreeSize", + "value": 100*1024*1024, + "category": "global" + }, + { + "name": "randErrorChance", + "value": 5, + "category": "global" + }, + { + "name": "randErrorDivisor", + "value": 20001, + "category": "global" + }, + { + "name": "randErrorScope", + "value": 8, + "category": "global" + }, + { + "name": "syncLogBufferMemoryAllowed", + "value": 1024 * 1024 * 20 * 10, + "category": "global" + }, + { + "name": "resolveFQDNRetryTime", + "value": 500, + "category": "global" + }, + { + "name": "syncHeartbeatInterval", + "value": 3000, + "category": "global" + }, + { + "name": "syncHeartbeatTimeout", + "value": 40000, + "category": "global" + }, + { + "name": "syncSnapReplMaxWaitN", + "value": 32, + "category": "global" + }, + { + "name": "walFsyncDataSizeLimit", + "value": 200 * 1024 * 1024, + "category": "global" + }, + { + "name": "numOfCores", + "value": "30.000000", + "category": "global" + }, + { + "name": "enableCoreFile", + "value": 0, + "category": "global" + }, + { + "name": "telemetryInterval", + "value": 6000, + "category": "global" + }, + { + "name": "cacheLazyLoadThreshold", + "value": 1000, + "category": "global" + }, + { + "name": "checkpointInterval", + "value": 120, + "category": "global" + }, + { + "name": "concurrentCheckpoint", + "value": 3, + "category": "global" + }, + { + "name": "retentionSpeedLimitMB", + "value": 24, + "category": "global" + }, + { + "name": "ttlChangeOnWrite", + "value": 1, + "category": "global" + }, + { + "name": "logKeepDays", + "value": 30, + "category": "local" + }, + { + "name": "maxStreamBackendCache", + "value": 256, + "category": "global" + }, + { + "name": "mqRebalanceInterval", + "value": 30, + "category": "global" + }, + { + "name": "numOfLogLines", + "value": 20000000, + "category": "local" + }, + { + "name": "queryRspPolicy", + "value": 1, + "category": "global" + }, + { + "name": "timeseriesThreshold", + "value": 100, + "category": "global" + }, + { + "name": "tmqMaxTopicNum", + "value": 30, + "category": "global" + }, + { + "name": "tmqRowSize", + "value": 8192, + "category": "global" + }, + { + "name": "transPullupInterval", + "value": 4, + "category": "global" + }, + { + "name": "compactPullupInterval", + "value": 20, + "category": "global" + }, + { + "name": "trimVDbIntervalSec", + "value": 7200, + "category": "global" + }, + { + "name": "ttlBatchDropNum", + "value": 20000, + "category": "global" + }, + { + "name": "ttlFlushThreshold", + "value": 200, + "category": "global" + }, + { + "name": "ttlPushInterval", + "value": 20, + "category": "global" + }, + { + "name": "ttlUnit", + "value": 86500, + "category": "global" + }, + { + "name": "udf", + "value": 0, + "category": "global" + }, + # { + # "name": "udfdLdLibPath", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "udfdResFuncs", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "s3Accesskey", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "s3BucketName", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "s3Endpoint", + # "value": 1000, + # "category": "global" + # }, + { + "name": "s3MigrateIntervalSec", + "value": 1800, + "category": "global" + }, + { + "name": "s3MigrateEnabled", + "value": 1, + "category": "global" + }, + # { + # "name": "s3BlockCacheSize", + # "value": 32, + # "category": "global" + # }, + { + "name": "s3PageCacheSize", + "value": 8192, + "category": "global" + }, + { + "name": "s3UploadDelaySec", + "value": 30, + "category": "global" + }, + { + "name": "mndLogRetention", + "value": 1000, + "category": "global" + }, + { + "name": "supportVnodes", + "value": 128, + "category": "global" + }, + { + "name": "experimental", + "value": 0, + "category": "global" + }, + { + "name": "maxTsmaNum", + "value": 2, + "category": "global" + }, + { + "name": "maxShellConns", + "value": 25000, + "category": "global" + }, + { + "name": "numOfRpcSessions", + "value": 15000, + "category": "global" + }, + { + "name": "numOfRpcThreads", + "value": 2, + "category": "global" + }, + { + "name": "rpcQueueMemoryAllowed", + "value": 1024*1024*20*10, + "category": "global" + }, + { + "name": "shellActivityTimer", + "value": 2, + "category": "global" + }, + { + "name": "timeToGetAvailableConn", + "value": 200000, + "category": "global" + }, + { + "name": "readTimeout", + "value": 800, + "category": "global" + }, + { + "name": "safetyCheckLevel", + "value": 2, + "category": "global" + }, + { + "name": "bypassFlag", + "value": 4, + "category": "global" + } + ] + } + def cli_get_param_value(self, config_name): + res = tdSql.getResult("show local variables") + for row in res: + if config_name == row[0]: + return row[1] + + def svr_get_param_value(self, config_name): + res = tdSql.getResult("show dnode 1 variables") + for row in res: + if config_name == row[1]: + return row[2] + + def configuration_alter(self): + for key in self.configration_dic: + if "svr" == key: + for item in self.configration_dic[key]: + name = item["name"] + value = item["value"] + category = item["category"] + if category == "global": + tdSql.execute(f'alter all dnodes "{name} {value}";') + else: + tdSql.execute(f'alter dnode 1 "{name} {value}";') + elif "cli" == key: + for item in self.configration_dic[key]: + name = item["name"] + value = item["value"] + category = item["category"] + if category == "global": + tdSql.execute(f'alter all dnodes "{name} {value}";') + else: + tdSql.execute(f'alter local "{name} {value}";') + else: + raise Exception(f"unknown key: {key}") + + def run(self): + self.configuration_alter() + for key in self.configration_dic: + if "cli" == key: + for item in self.configration_dic[key]: + actVal = self.cli_get_param_value(item["name"]) + assert str(actVal) == str(item["value"]), f"tem name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + elif "svr" == key: + for item in self.configration_dic[key]: + actVal = self.svr_get_param_value(item["name"]) + assert str(actVal) == str(item["value"]), f"tem name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + else: + raise Exception(f"unknown key: {key}") + + tdLog.info("success to alter all configurations") + + tdLog.info("stop and restart taosd") + + sc.dnodeStopAll() + sc.dnodeStart(1) + sc.dnodeStart(2) + sc.dnodeStart(3) + sleep(10) + + for key in self.configration_dic: + if "cli" == key: + for item in self.configration_dic[key]: + actVal = self.cli_get_param_value(item["name"]) + assert str(actVal) == str(item["value"]), f"item name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + elif "svr" == key: + for item in self.configration_dic[key]: + actVal = self.svr_get_param_value(item["name"]) + assert str(actVal) == str(item["value"]), f"item name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + else: + raise Exception(f"unknown key: {key}") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/authorith/authBasic.py b/tests/army/authorith/authBasic.py index 9a453cf687..a682b08ba0 100644 --- a/tests/army/authorith/authBasic.py +++ b/tests/army/authorith/authBasic.py @@ -40,9 +40,9 @@ class TDTestCase(TBase): def test_common_user_privileges(self): self.prepare_data() # create user - self.create_user("test", "test") + self.create_user("test", "test12@#*") # check user 'test' privileges - testconn = taos.connect(user='test', password='test') + testconn = taos.connect(user='test', password='test12@#*') cursor = testconn.cursor() testSql = TDSql() testSql.init(cursor) @@ -87,9 +87,9 @@ class TDTestCase(TBase): def test_common_user_with_createdb_privileges(self): self.prepare_data() # create user - self.create_user("test", "test") + self.create_user("test", "test12@#*") # check user 'test' privileges - testconn = taos.connect(user='test', password='test') + testconn = taos.connect(user='test', password='test12@#*') cursor = testconn.cursor() testSql = TDSql() testSql.init(cursor) @@ -133,8 +133,8 @@ class TDTestCase(TBase): testSql.checkRows(2) # create another user 'test1' - self.create_user("test1", "test1") - test1conn = taos.connect(user='test1', password='test1') + self.create_user("test1", "test12@#*^%") + test1conn = taos.connect(user='test1', password='test12@#*^%') cursor1 = test1conn.cursor() test1Sql = TDSql() test1Sql.init(cursor1) diff --git a/tests/army/cluster/test_drop_table_by_uid.py b/tests/army/cluster/test_drop_table_by_uid.py index f09006b37b..3d10863fc2 100644 --- a/tests/army/cluster/test_drop_table_by_uid.py +++ b/tests/army/cluster/test_drop_table_by_uid.py @@ -305,9 +305,9 @@ class TDTestCase(TBase): """ try: # create new user and grant create database priviledge - tdSql.execute("create user test pass 'test';") + tdSql.execute("create user test pass 'ab45*&TC';") tdSql.execute("alter user test createdb 1;") - conn = taos.connect(user="test", password="test") + conn = taos.connect(user="test", password="ab45*&TC") cursor = conn.cursor() # create database and tables with new user tdLog.info("Prepare data for test case test_abnormal_drop_table_with_non_root_user") diff --git a/tests/army/cmdline/fullopt.py b/tests/army/cmdline/fullopt.py index b80d7eac4a..7f97a6892a 100644 --- a/tests/army/cmdline/fullopt.py +++ b/tests/army/cmdline/fullopt.py @@ -57,10 +57,8 @@ class TDTestCase(TBase): # local command options = [ "DebugFlag 143", - "enableCoreFile 1", "fqdn 127.0.0.1", "firstEp 127.0.0.1", - "locale en_US.UTF-8", "metaCacheMaxSize 10000", "minimalTmpDirGB 5", "minimalLogDirGB 1", @@ -70,7 +68,6 @@ class TDTestCase(TBase): "smlTagName tagname", "smlTsDefaultName tsdef", "serverPort 6030", - "timezone tz", ] # exec for option in options: diff --git a/tests/army/output.txt b/tests/army/output.txt deleted file mode 100644 index ed3bd5da1a..0000000000 --- a/tests/army/output.txt +++ /dev/null @@ -1,91 +0,0 @@ -[10/28 19:12:21.666563] SUCC: created database (db_sub) -[10/28 19:12:21.694603] INFO: start creating 1000 table(s) with 8 thread(s) -[10/28 19:12:21.823202] SUCC: Spent 0.1290 seconds to create 1000 table(s) with 8 thread(s) speed: 7752 tables/s, already exist 0 table(s), actual 1000 table(s) pre created, 0 table(s) will be auto created -[10/28 19:12:22.127442] SUCC: thread[4] progressive mode, completed total inserted rows: 125000, 441047.79 records/second -[10/28 19:12:22.128649] SUCC: thread[7] progressive mode, completed total inserted rows: 125000, 440895.33 records/second -[10/28 19:12:22.129478] SUCC: thread[5] progressive mode, completed total inserted rows: 125000, 440151.69 records/second -[10/28 19:12:22.133756] SUCC: thread[1] progressive mode, completed total inserted rows: 125000, 433268.05 records/second -[10/28 19:12:22.135211] SUCC: thread[3] progressive mode, completed total inserted rows: 125000, 430329.63 records/second -[10/28 19:12:22.137335] SUCC: thread[0] progressive mode, completed total inserted rows: 125000, 425800.08 records/second -[10/28 19:12:22.138252] SUCC: thread[6] progressive mode, completed total inserted rows: 125000, 426330.15 records/second -[10/28 19:12:22.141351] SUCC: thread[2] progressive mode, completed total inserted rows: 125000, 422778.64 records/second -[10/28 19:12:22.141585] SUCC: Spent 0.311648 (real 0.289041) seconds to insert rows: 1000000 with 8 thread(s) into db_sub 3208748.33 (real 3459716.79) records/second -[10/28 19:12:22.141590] SUCC: insert delay, min: 0.9600ms, avg: 2.3123ms, p90: 3.1790ms, p95: 3.5080ms, p99: 4.2230ms, max: 4.9040ms -[10/28 19:28:50.798427] SUCC: created database (db_sub) -[10/28 19:28:50.828326] INFO: start creating 1000 table(s) with 8 thread(s) -[10/28 19:28:50.936429] SUCC: Spent 0.1080 seconds to create 1000 table(s) with 8 thread(s) speed: 9259 tables/s, already exist 0 table(s), actual 1000 table(s) pre created, 0 table(s) will be auto created -[10/28 19:28:51.187235] SUCC: thread[6] progressive mode, completed total inserted rows: 125000, 539204.48 records/second -[10/28 19:28:51.189941] SUCC: thread[2] progressive mode, completed total inserted rows: 125000, 532329.43 records/second -[10/28 19:28:51.191551] SUCC: thread[4] progressive mode, completed total inserted rows: 125000, 530954.66 records/second -[10/28 19:28:51.191858] SUCC: thread[1] progressive mode, completed total inserted rows: 125000, 529259.59 records/second -[10/28 19:28:51.192459] SUCC: thread[3] progressive mode, completed total inserted rows: 125000, 530229.44 records/second -[10/28 19:28:51.195372] SUCC: thread[7] progressive mode, completed total inserted rows: 125000, 522099.42 records/second -[10/28 19:28:51.197727] SUCC: thread[0] progressive mode, completed total inserted rows: 125000, 516620.72 records/second -[10/28 19:28:51.197883] SUCC: thread[5] progressive mode, completed total inserted rows: 125000, 517125.12 records/second -[10/28 19:28:51.198123] SUCC: Spent 0.255536 (real 0.237135) seconds to insert rows: 1000000 with 8 thread(s) into db_sub 3913342.93 (real 4217007.19) records/second -[10/28 19:28:51.198130] SUCC: insert delay, min: 0.9200ms, avg: 1.8971ms, p90: 2.6870ms, p95: 2.9520ms, p99: 3.5880ms, max: 4.0710ms -[10/28 19:31:44.377691] SUCC: created database (db_sub) -[10/28 19:31:44.392998] INFO: start creating 1000 table(s) with 8 thread(s) -[10/28 19:31:44.696768] SUCC: Spent 0.3040 seconds to create 1000 table(s) with 8 thread(s) speed: 3289 tables/s, already exist 0 table(s), actual 1000 table(s) pre created, 0 table(s) will be auto created -[10/28 19:31:45.126910] SUCC: thread[3] progressive mode, completed total inserted rows: 125000, 304775.47 records/second -[10/28 19:31:45.131979] SUCC: thread[0] progressive mode, completed total inserted rows: 125000, 301117.75 records/second -[10/28 19:31:45.135106] SUCC: thread[5] progressive mode, completed total inserted rows: 125000, 299854.39 records/second -[10/28 19:31:45.135675] SUCC: thread[4] progressive mode, completed total inserted rows: 125000, 298322.24 records/second -[10/28 19:31:45.137069] SUCC: thread[7] progressive mode, completed total inserted rows: 125000, 297733.89 records/second -[10/28 19:31:45.137952] SUCC: thread[1] progressive mode, completed total inserted rows: 125000, 296900.13 records/second -[10/28 19:31:45.138834] SUCC: thread[2] progressive mode, completed total inserted rows: 125000, 295170.54 records/second -[10/28 19:31:45.145048] SUCC: thread[6] progressive mode, completed total inserted rows: 125000, 291966.71 records/second -[10/28 19:31:45.145369] SUCC: Spent 0.442506 (real 0.419200) seconds to insert rows: 1000000 with 8 thread(s) into db_sub 2259856.36 (real 2385496.18) records/second -[10/28 19:31:45.145377] SUCC: insert delay, min: 1.0400ms, avg: 3.3536ms, p90: 5.3120ms, p95: 7.9660ms, p99: 13.1570ms, max: 19.1410ms -[10/28 19:44:19.873056] SUCC: created database (db_sub) -[10/28 19:44:19.904701] INFO: start creating 1000 table(s) with 8 thread(s) -[10/28 19:44:20.053846] SUCC: Spent 0.1490 seconds to create 1000 table(s) with 8 thread(s) speed: 6711 tables/s, already exist 0 table(s), actual 1000 table(s) pre created, 0 table(s) will be auto created -[10/28 19:44:20.328698] SUCC: thread[3] progressive mode, completed total inserted rows: 125000, 485742.49 records/second -[10/28 19:44:20.330777] SUCC: thread[2] progressive mode, completed total inserted rows: 125000, 481686.29 records/second -[10/28 19:44:20.331290] SUCC: thread[4] progressive mode, completed total inserted rows: 125000, 480911.65 records/second -[10/28 19:44:20.331665] SUCC: thread[1] progressive mode, completed total inserted rows: 125000, 481043.06 records/second -[10/28 19:44:20.333451] SUCC: thread[0] progressive mode, completed total inserted rows: 125000, 477172.09 records/second -[10/28 19:44:20.334745] SUCC: thread[5] progressive mode, completed total inserted rows: 125000, 475675.84 records/second -[10/28 19:44:20.335056] SUCC: thread[6] progressive mode, completed total inserted rows: 125000, 474158.37 records/second -[10/28 19:44:20.337919] SUCC: thread[7] progressive mode, completed total inserted rows: 125000, 470816.89 records/second -[10/28 19:44:20.338144] SUCC: Spent 0.277921 (real 0.261310) seconds to insert rows: 1000000 with 8 thread(s) into db_sub 3598144.80 (real 3826872.30) records/second -[10/28 19:44:20.338153] SUCC: insert delay, min: 0.9180ms, avg: 2.0905ms, p90: 2.6490ms, p95: 3.0620ms, p99: 4.1480ms, max: 4.7840ms -[10/28 19:58:27.100989] SUCC: created database (db_sub) -[10/28 19:58:27.115572] INFO: start creating 1000 table(s) with 8 thread(s) -[10/28 19:58:27.362948] SUCC: Spent 0.2470 seconds to create 1000 table(s) with 8 thread(s) speed: 4049 tables/s, already exist 0 table(s), actual 1000 table(s) pre created, 0 table(s) will be auto created -[10/28 19:58:27.807669] SUCC: thread[7] progressive mode, completed total inserted rows: 125000, 291891.03 records/second -[10/28 19:58:27.818785] SUCC: thread[1] progressive mode, completed total inserted rows: 125000, 285413.54 records/second -[10/28 19:58:27.819649] SUCC: thread[0] progressive mode, completed total inserted rows: 125000, 284193.61 records/second -[10/28 19:58:27.819844] SUCC: thread[5] progressive mode, completed total inserted rows: 125000, 284352.64 records/second -[10/28 19:58:27.820170] SUCC: thread[6] progressive mode, completed total inserted rows: 125000, 284576.63 records/second -[10/28 19:58:27.821489] SUCC: thread[4] progressive mode, completed total inserted rows: 125000, 283781.33 records/second -[10/28 19:58:27.822061] SUCC: thread[2] progressive mode, completed total inserted rows: 125000, 283112.24 records/second -[10/28 19:58:27.823513] SUCC: thread[3] progressive mode, completed total inserted rows: 125000, 282730.59 records/second -[10/28 19:58:27.823779] SUCC: Spent 0.455783 (real 0.438625) seconds to insert rows: 1000000 with 8 thread(s) into db_sub 2194026.54 (real 2279851.81) records/second -[10/28 19:58:27.823786] SUCC: insert delay, min: 0.9780ms, avg: 3.5090ms, p90: 5.5650ms, p95: 6.8600ms, p99: 10.6010ms, max: 13.4400ms -[10/28 20:00:06.417182] SUCC: created database (db_sub) -[10/28 20:00:06.448202] INFO: start creating 1000 table(s) with 8 thread(s) -[10/28 20:00:06.596961] SUCC: Spent 0.1480 seconds to create 1000 table(s) with 8 thread(s) speed: 6757 tables/s, already exist 0 table(s), actual 1000 table(s) pre created, 0 table(s) will be auto created -[10/28 20:00:06.895455] SUCC: thread[3] progressive mode, completed total inserted rows: 125000, 443978.76 records/second -[10/28 20:00:06.896986] SUCC: thread[5] progressive mode, completed total inserted rows: 125000, 442549.94 records/second -[10/28 20:00:06.897536] SUCC: thread[0] progressive mode, completed total inserted rows: 125000, 440927.99 records/second -[10/28 20:00:06.898905] SUCC: thread[2] progressive mode, completed total inserted rows: 125000, 439131.15 records/second -[10/28 20:00:06.899024] SUCC: thread[7] progressive mode, completed total inserted rows: 125000, 439628.46 records/second -[10/28 20:00:06.901861] SUCC: thread[1] progressive mode, completed total inserted rows: 125000, 435197.37 records/second -[10/28 20:00:06.902305] SUCC: thread[6] progressive mode, completed total inserted rows: 125000, 434812.86 records/second -[10/28 20:00:06.904698] SUCC: thread[4] progressive mode, completed total inserted rows: 125000, 433406.26 records/second -[10/28 20:00:06.904905] SUCC: Spent 0.301788 (real 0.284949) seconds to insert rows: 1000000 with 8 thread(s) into db_sub 3313584.37 (real 3509399.93) records/second -[10/28 20:00:06.904912] SUCC: insert delay, min: 0.8770ms, avg: 2.2796ms, p90: 3.1340ms, p95: 3.6480ms, p99: 4.8280ms, max: 6.0880ms -[10/28 20:05:34.756207] SUCC: created database (db_sub) -[10/28 20:05:34.784793] INFO: start creating 1000 table(s) with 8 thread(s) -[10/28 20:05:34.927068] SUCC: Spent 0.1430 seconds to create 1000 table(s) with 8 thread(s) speed: 6993 tables/s, already exist 0 table(s), actual 1000 table(s) pre created, 0 table(s) will be auto created -[10/28 20:05:35.213741] SUCC: thread[4] progressive mode, completed total inserted rows: 125000, 466952.82 records/second -[10/28 20:05:35.215403] SUCC: thread[3] progressive mode, completed total inserted rows: 125000, 463804.68 records/second -[10/28 20:05:35.221132] SUCC: thread[2] progressive mode, completed total inserted rows: 125000, 453322.31 records/second -[10/28 20:05:35.221224] SUCC: thread[1] progressive mode, completed total inserted rows: 125000, 453671.11 records/second -[10/28 20:05:35.222003] SUCC: thread[0] progressive mode, completed total inserted rows: 125000, 452641.07 records/second -[10/28 20:05:35.222536] SUCC: thread[5] progressive mode, completed total inserted rows: 125000, 451796.89 records/second -[10/28 20:05:35.223663] SUCC: thread[7] progressive mode, completed total inserted rows: 125000, 449643.52 records/second -[10/28 20:05:35.225246] SUCC: thread[6] progressive mode, completed total inserted rows: 125000, 447768.68 records/second -[10/28 20:05:35.225659] SUCC: Spent 0.290871 (real 0.274808) seconds to insert rows: 1000000 with 8 thread(s) into db_sub 3437950.16 (real 3638904.25) records/second -[10/28 20:05:35.225666] SUCC: insert delay, min: 0.9360ms, avg: 2.1985ms, p90: 2.9290ms, p95: 3.4580ms, p99: 4.6030ms, max: 6.2660ms diff --git a/tests/army/query/function/test_selection_function_with_json.py b/tests/army/query/function/test_selection_function_with_json.py new file mode 100644 index 0000000000..e1f8090ae3 --- /dev/null +++ b/tests/army/query/function/test_selection_function_with_json.py @@ -0,0 +1,106 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from frame import etool +from frame.etool import * +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame.common import * + +class TDTestCase(TBase): + updatecfgDict = { + "keepColumnName": "1", + "ttlChangeOnWrite": "1", + "querySmaOptimize": "1", + "slowLogScope": "none", + "queryBufferSize": 10240 + } + + def insert_data(self): + tdLog.info(f"insert data.") + tdSql.execute("drop database if exists ts_5763;") + tdSql.execute("create database ts_5763;") + tdSql.execute("use ts_5763;") + tdSql.execute("select database();") + tdSql.execute("CREATE STABLE metrics (ts TIMESTAMP, v DOUBLE) TAGS (labels JSON)") + tdSql.execute("""CREATE TABLE `metrics_0` USING `metrics` (`labels`) TAGS ('{"ident":"192.168.56.167"}');""") + tdSql.execute("""CREATE TABLE `metrics_1` USING `metrics` (`labels`) TAGS ('{"ident":"192.168.56.168"}');""") + tdSql.execute("""CREATE TABLE `metrics_2` USING `metrics` (`labels`) TAGS ('{"ident":"192.168.56.169"}');""") + tdSql.execute("""CREATE TABLE `metrics_3` USING `metrics` (`labels`) TAGS ('{"ident":"192.168.56.170"}');""") + tdSql.execute("""CREATE TABLE `metrics_5` USING `metrics` (`labels`) TAGS ('{"asset_name":"中国政务网"}');""") + tdSql.execute("""CREATE TABLE `metrics_6` USING `metrics` (`labels`) TAGS ('{"asset_name":"地大物博阿拉丁快解放啦上课交电费"}');""") + tdSql.execute("""CREATE TABLE `metrics_7` USING `metrics` (`labels`) TAGS ('{"asset_name":"no1241-上的六块腹肌阿斯利康的肌肤轮廓设计大方"}');""") + tdSql.execute("""CREATE TABLE `metrics_8` USING `metrics` (`labels`) TAGS ('{"asset_name":"no1241-上的六块腹肌阿斯利康的肌肤轮廓设计大方","ident":"192.168.0.1"}');""") + tdSql.execute("""CREATE TABLE `metrics_9` USING `metrics` (`labels`) TAGS ('{"asset_name":"no1241-上的六块腹肌阿斯利康的肌肤轮廓设计大方","ident":"192.168.0.1"}');""") + tdSql.execute("""CREATE TABLE `metrics_10` USING `metrics` (`labels`) TAGS ('{"asset_name":"上的咖啡机no1241-上的六块腹肌阿斯利康的肌肤轮廓设计大方","ident":"192.168.0.1"}');""") + + tdSql.execute("insert into metrics_0 values ('2024-12-12 16:34:39.326',1)") + tdSql.execute("insert into metrics_0 values ('2024-12-12 16:34:40.891',2)") + tdSql.execute("insert into metrics_0 values ('2024-12-12 16:34:41.986',3)") + tdSql.execute("insert into metrics_0 values ('2024-12-12 16:34:42.992',4)") + tdSql.execute("insert into metrics_0 values ('2024-12-12 16:34:46.927',5)") + tdSql.execute("insert into metrics_0 values ('2024-12-12 16:34:48.473',6)") + tdSql.execute("insert into metrics_1 select * from metrics_0") + tdSql.execute("insert into metrics_2 select * from metrics_0") + tdSql.execute("insert into metrics_3 select * from metrics_0") + tdSql.execute("insert into metrics_5 select * from metrics_0") + tdSql.execute("insert into metrics_6 select * from metrics_0") + tdSql.execute("insert into metrics_7 select * from metrics_0") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:36.459',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:37.388',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:37.622',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:37.852',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:38.081',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:38.307',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:38.535',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:38.792',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:39.035',1)") + tdSql.execute("insert into metrics_8 values ('2024-12-12 19:05:39.240',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:29.270',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:30.508',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:31.035',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:31.523',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:31.760',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:32.001',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:32.228',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:32.453',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:32.690',1)") + tdSql.execute("insert into metrics_9 values ('2024-12-12 19:05:32.906',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:14.538',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:15.114',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:15.613',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:15.853',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:16.054',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:16.295',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:16.514',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:16.731',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:16.958',1)") + tdSql.execute("insert into metrics_10 values ('2024-12-12 19:06:17.176',1)") + + for i in range(1, 10): + tdSql.query("select _wstart,first(v)-last(v), first(labels->'asset_name'),first(labels->'ident'),mode(labels->'asset_name'),mode(labels->'ident'),last(labels->'asset_name'),last(labels->'ident') from ts_5763.metrics interval(1s)") + tdSql.checkRows(18) + + def run(self): + tdLog.debug(f"start to excute {__file__}") + + self.insert_data() + + + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/whole/checkErrorCode.py b/tests/army/whole/checkErrorCode.py index b24c2ae7b5..b2aa1fce1b 100644 --- a/tests/army/whole/checkErrorCode.py +++ b/tests/army/whole/checkErrorCode.py @@ -39,7 +39,7 @@ ignoreCodes = [ '0x800003A7', '0x800003AA', '0x800003AB', '0x800003AC', '0x800003AD', '0x800003B0', '0x800003B2', '0x800003B4', '0x800003B5', '0x800003BA', '0x800003C0', '0x800003C1', '0x800003D0', '0x800003D8', '0x800003D9', '0x800003E2', '0x800003F4', '0x800003F8', '0x80000412', '0x80000413', '0x80000414', '0x80000415', '0x80000416', '0x80000417', '0x80000418', '0x80000419', '0x80000420', '0x80000421', '0x80000422', '0x80000423', - '0x80000424', '0x80000425', '0x80000426', '0x80000427', '0x80000428', '0x80000429', '0x8000042A', '0x80000430', '0x80000431', '0x80000432', + '0x80000424', '0x80000425', '0x80000426', '0x80000427', '0x80000428', '0x80000429', '0x8000042A', '0x8000042B','0x80000430', '0x80000431', '0x80000432', '0x80000433', '0x80000434', '0x80000435', '0x80000436', '0x80000437', '0x80000438', '0x80000440', '0x80000441', '0x80000442', '0x80000443', '0x80000444', '0x80000445', '0x80000446', '0x80000447', '0x80000485', '0x80000486', '0x800004A0', '0x800004A1', '0x800004B1', '0x800004B2', '0x800004B3', '0x80000504', '0x80000528', '0x80000532', '0x80000533', '0x80000534', '0x80000535', '0x80000536', '0x80000537', '0x80000538', @@ -55,7 +55,7 @@ ignoreCodes = [ '0x80002207', '0x80002406', '0x80002407', '0x80002503', '0x80002506', '0x80002507', '0x8000261B', '0x80002653', '0x80002668', '0x80002669', '0x8000266A', '0x8000266B', '0x8000266C', '0x8000266D', '0x8000266E', '0x8000266F', '0x80002670', '0x80002671', '0x80002672', '0x80002673', '0x80002674', '0x80002675', '0x80002676', '0x80002677', '0x80002678', '0x80002679', '0x8000267A', '0x8000267B', '0x8000267C', '0x8000267D', - '0x8000267E', '0x8000267F', '0x80002680', '0x80002681', '0x80002682', '0x80002683', '0x80002684', '0x80002685', '0x80002703', '0x80002806', + '0x8000267E', '0x8000267F', '0x80002680', '0x80002681', '0x80002682', '0x80002683', '0x80002684', '0x80002685', '0x80002686', '0x80002703', '0x80002806', '0x80002807', '0x80002808', '0x80002809', '0x8000280A', '0x8000280B', '0x8000280C', '0x8000280D', '0x8000280E', '0x8000280F', '0x80002810', '0x80002811', '0x80002812', '0x80002813', '0x80002814', '0x80002815', '0x8000290B', '0x80002920', '0x80003003', '0x80003006', '0x80003106', '0x80003107', '0x80003108', '0x80003109', '0x80003110', '0x80003111', '0x80003112', '0x80003250', '0x80004003', '0x80004004', '0x80004005', diff --git a/tests/develop-test/2-query/show_create_db.py b/tests/develop-test/2-query/show_create_db.py index 9589b6dc6f..e59121f631 100644 --- a/tests/develop-test/2-query/show_create_db.py +++ b/tests/develop-test/2-query/show_create_db.py @@ -25,7 +25,7 @@ class TDTestCase: def run(self): print("running {}".format(__file__)) tdSql.execute("drop database if exists scd") - tdSql.execute("create database if not exists scd") + tdSql.execute("create database if not exists scd compact_interval 0") tdSql.execute('use scd') tdSql.execute('create table stb1 (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);') @@ -35,24 +35,24 @@ class TDTestCase: tdSql.execute("create table tb3 using stb1 tags(3,'3',3.0);") - tdSql.execute('create database scd2 stt_trigger 3;') + tdSql.execute('create database scd2 stt_trigger 3 compact_interval 1;') - tdSql.execute('create database scd4 stt_trigger 13;') + tdSql.execute('create database scd4 stt_trigger 13 compact_interval 12h compact_time_range -60,-10 compact_time_offset 23h;') tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 0d COMPACT_TIME_RANGE 0d,0d COMPACT_TIME_OFFSET 0h") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 1d COMPACT_TIME_RANGE -3650d,-10d COMPACT_TIME_OFFSET 0h") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 12h COMPACT_TIME_RANGE -60d,-10d COMPACT_TIME_OFFSET 23h") self.restartTaosd(1, dbname='scd') @@ -60,16 +60,16 @@ class TDTestCase: tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 0d COMPACT_TIME_RANGE 0d,0d COMPACT_TIME_OFFSET 0h") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 1d COMPACT_TIME_RANGE -3650d,-10d COMPACT_TIME_OFFSET 0h") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 12h COMPACT_TIME_RANGE -60d,-10d COMPACT_TIME_OFFSET 23h") tdSql.execute('drop database scd') diff --git a/tests/develop-test/2-query/table_count_scan.py b/tests/develop-test/2-query/table_count_scan.py index 30672cabf2..9725c36bee 100644 --- a/tests/develop-test/2-query/table_count_scan.py +++ b/tests/develop-test/2-query/table_count_scan.py @@ -63,22 +63,25 @@ class TDTestCase: tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:06\',true,7,7,7,7,7,7,"777","7777",7,7,7,7);') - - #tdSql.query('select count(*),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') - #tdSql.checkRows(3) - #tdSql.checkData(0, 0, 35) - #tdSql.checkData(0, 1, 'information_schema') - #tdSql.checkData(0, 2, None) - #tdSql.checkData(1, 0, 3) - #tdSql.checkData(1, 1, 'tbl_count') - #tdSql.checkData(1, 2, 'stb1') - #tdSql.checkData(2, 0, 5) - #tdSql.checkData(2, 1, 'performance_schema') - #tdSql.checkData(2, 2, None) - + tdSql.query('select count(*),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') + tdSql.checkRows(3) + for i in range(0, 3): + db_name = tdSql.getData(i, 1) + if db_name == 'information_schema': + tdSql.checkData(i, 0, 36) + tdSql.checkData(i, 2, None) + elif db_name == 'performance_schema': + tdSql.checkData(i, 0, 5) + tdSql.checkData(i, 2, None) + elif db_name == 'tbl_count': + tdSql.checkData(i, 0, 3) + tdSql.checkData(i, 2, 'stb1') + else: + raise Exception("unexpected db_name: %s" % db_name) + tdSql.query('select count(1) v,db_name, stable_name from information_schema.ins_tables group by db_name, stable_name order by v desc;') tdSql.checkRows(3) - tdSql.checkData(0, 0, 35) + tdSql.checkData(0, 0, 36) tdSql.checkData(0, 1, 'information_schema') tdSql.checkData(0, 2, None) tdSql.checkData(1, 0, 5) @@ -94,7 +97,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'performance_schema') tdSql.checkData(0, 0, 3) tdSql.checkData(0, 1, 'tbl_count') - tdSql.checkData(2, 0, 35) + tdSql.checkData(2, 0, 36) tdSql.checkData(2, 1, 'information_schema') tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'") @@ -107,7 +110,7 @@ class TDTestCase: tdSql.query('select count(*) from information_schema.ins_tables') tdSql.checkRows(1) - tdSql.checkData(0, 0, 43) + tdSql.checkData(0, 0, 44) tdSql.execute('create table stba (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);') @@ -190,7 +193,7 @@ class TDTestCase: tdSql.checkData(2, 0, 5) tdSql.checkData(2, 1, 'performance_schema') tdSql.checkData(2, 2, None) - tdSql.checkData(3, 0, 35) + tdSql.checkData(3, 0, 36) tdSql.checkData(3, 1, 'information_schema') tdSql.checkData(3, 2, None) @@ -205,7 +208,7 @@ class TDTestCase: tdSql.checkData(2, 0, 5) tdSql.checkData(2, 1, 'performance_schema') tdSql.checkData(2, 2, None) - tdSql.checkData(3, 0, 35) + tdSql.checkData(3, 0, 36) tdSql.checkData(3, 1, 'information_schema') tdSql.checkData(3, 2, None) @@ -216,7 +219,7 @@ class TDTestCase: tdSql.checkData(0, 1, 'tbl_count') tdSql.checkData(1, 0, 5) tdSql.checkData(1, 1, 'performance_schema') - tdSql.checkData(2, 0, 35) + tdSql.checkData(2, 0, 36) tdSql.checkData(2, 1, 'information_schema') tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'") @@ -229,7 +232,7 @@ class TDTestCase: tdSql.query('select count(*) from information_schema.ins_tables') tdSql.checkRows(1) - tdSql.checkData(0, 0, 44) + tdSql.checkData(0, 0, 45) tdSql.execute('drop database tbl_count') diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 47a6b418f6..4aedc0991e 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -15,6 +15,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f cluster/snapshot.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_func_elapsed.py ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_function.py +,,y,army,./pytest.sh python3 ./test.py -f query/function/test_selection_function_with_json.py ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_percentile.py ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_resinfo.py ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_interp.py @@ -41,6 +42,8 @@ ,,n,army,python3 ./test.py -f cmdline/fullopt.py ,,y,army,./pytest.sh python3 ./test.py -f query/show.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f alter/alterConfig.py -N 3 +,,y,army,./pytest.sh python3 ./test.py -f alter/test_alter_config.py -N 3 +,,y,army,./pytest.sh python3 ./test.py -f alter/test_alter_config.py -N 3 -M 3 ,,y,army,./pytest.sh python3 ./test.py -f query/subquery/subqueryBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f storage/oneStageComp.py -N 3 -L 3 -D 1 ,,y,army,./pytest.sh python3 ./test.py -f storage/compressBasic.py -N 3 @@ -395,6 +398,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/subscribe_stream_privilege.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/empty_identifier.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/persisit_config.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/qmemCtrl.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact_vgroups.py + ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_create.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_insert.py diff --git a/tests/pytest/user/pass_len.py b/tests/pytest/user/pass_len.py index 346b8424fe..2324f5993f 100644 --- a/tests/pytest/user/pass_len.py +++ b/tests/pytest/user/pass_len.py @@ -26,10 +26,10 @@ class TDTestCase: def run(self): print("==============step1") try: - tdSql.execute("create user abc pass '123456'") + tdSql.execute("create user abc pass '123456rf@#'") except Exception as e: tdLog.exit(e) - print("create user abc pass '123456'") + print("create user abc pass '123456rf@#'") print("==============step2") try: diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 8d048825d0..2c2e11cfcc 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -48,7 +48,8 @@ class TDSimClient: "telemetryReporting": "0", "tqDebugflag": "135", "stDebugflag":"135", - "safetyCheckLevel":"2" + "safetyCheckLevel":"2", + "minReservedMemorySize":"1024" } def getLogDir(self): @@ -889,4 +890,4 @@ class TDDnodes: def getAsan(self): return self.asan -tdDnodes = TDDnodes() \ No newline at end of file +tdDnodes = TDDnodes() diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py index 000c907ea4..8ec8bd96c8 100644 --- a/tests/pytest/util/log.py +++ b/tests/pytest/util/log.py @@ -15,6 +15,7 @@ import sys import os import time import datetime + from distutils.log import warn as printf diff --git a/tests/script/api/asyncQuery.c b/tests/script/api/asyncQuery.c new file mode 100644 index 0000000000..d53cd23b4b --- /dev/null +++ b/tests/script/api/asyncQuery.c @@ -0,0 +1,131 @@ +// sample code to verify multiple queries with the same reqid +// to compile: gcc -o sameReqdiTest sameReqidTest.c -ltaos + +#include +#include +#include +#include +#include "taos.h" + +#define CONST_REQ_ID 12345 +#define TEST_DB "db1" + +#define CHECK_CONDITION(condition, prompt, errstr) \ + do { \ + if (!(condition)) { \ + printf("\033[31m[%s:%d] failed to " prompt ", reason: %s\033[0m\n", __func__, __LINE__, errstr); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define CHECK_RES(res, prompt) CHECK_CONDITION(taos_errno(res) == 0, prompt, taos_errstr(res)) +#define CHECK_CODE(code, prompt) CHECK_CONDITION(code == 0, prompt, taos_errstr(NULL)) + +int64_t errTimes = 0, finQueries = 0; + +static TAOS* getNewConnection() { + const char* host = "127.0.0.1"; + const char* user = "root"; + const char* passwd = "taosdata"; + TAOS* taos = NULL; + + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); + taos = taos_connect(host, user, passwd, "", 0); + CHECK_CONDITION(taos != NULL, "connect to db", taos_errstr(NULL)); + return taos; +} + +static int32_t printResult(TAOS_RES* res, int32_t nlimit) { + TAOS_ROW row = NULL; + TAOS_FIELD* fields = NULL; + int32_t numFields = 0; + int32_t nRows = 0; + + numFields = taos_num_fields(res); + fields = taos_fetch_fields(res); + while ((nlimit-- > 0) && (row = taos_fetch_row(res))) { + nRows++; + } + return nRows; +} + +void retrieveCallback(void* param, TAOS_RES* res, int32_t nrows) { + if (nrows < 0) { + taos_free_result(res); + atomic_add_fetch_64(&finQueries, 1); + atomic_add_fetch_64(&errTimes, 1); + } else if (nrows == 0) { + taos_free_result(res); + atomic_add_fetch_64(&finQueries, 1); + } else { + printResult(res, nrows); + taos_fetch_rows_a(res, retrieveCallback, param); + } +} + +void selectCallback(void* param, TAOS_RES* res, int32_t code) { + if (code) { + atomic_add_fetch_64(&errTimes, 1); + taos_free_result(res); + atomic_add_fetch_64(&finQueries, 1); + return; + } + taos_fetch_rows_a(res, retrieveCallback, param); +} + + +#if 0 +static void verifyQueryAsync(TAOS* taos) { + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2023-01-01' and ts <= '2024-12-01' interval(1s) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2023-01-01' and ts <= '2024-12-01' interval(1s) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2023-01-01' and ts <= '2024-12-01' interval(1s) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2023-01-01' and ts <= '2024-12-01' interval(1s) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2023-01-01' and ts <= '2024-12-01' interval(1s) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2023-01-01' and ts <= '2024-12-01' interval(1s) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2023-01-01' and ts <= '2024-12-01' interval(1s) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2023-01-01' and ts <= '2024-12-01' interval(1s) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2023-01-01' and ts <= '2024-12-01' interval(1s) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;;", selectCallback, NULL); + taos_query_a(taos, "select twa(c1) from stb interval(10s);", selectCallback, NULL); +} +#else +static void verifyQueryAsync(TAOS* taos) { + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2024-11-01' and ts <= '2024-12-01' interval(1m) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2024-11-01' and ts <= '2024-12-01' interval(1m) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2024-11-01' and ts <= '2024-12-01' interval(1m) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2024-11-01' and ts <= '2024-12-01' interval(1m) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2024-11-01' and ts <= '2024-12-01' interval(1m) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2024-11-01' and ts <= '2024-12-01' interval(1m) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2024-11-01' and ts <= '2024-12-01' interval(1m) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2024-11-01' and ts <= '2024-12-01' interval(1m) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;", selectCallback, NULL); + taos_query_a(taos, "select * from (select cast(count(*) as binary(100)) a, rand() b from tbx where ts >= '2024-11-01' and ts <= '2024-12-01' interval(1m) fill(value, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')) t1 group by t1.a, t1.b;", selectCallback, NULL); + taos_query_a(taos, "select twa(c1) from tbx interval(10s);", selectCallback, NULL); +} +#endif + +int main(int argc, char* argv[]) { + TAOS* taos = NULL; + int32_t code = 0; + + taos = getNewConnection(); + taos_select_db(taos, TEST_DB); + CHECK_CODE(code, "switch to database"); + + printf("************ prepare data *************\n"); + + //create table tbx (ts timestamp, f1 int); insert into tbx values('2024-11-11 00:00:00', 1); + + for (int64_t i = 0; i < 1000000000; ++i) { + verifyQueryAsync(taos); + printf("%llu queries launched, errTimes:%lld \n", i * 10, errTimes); + while ((i * 10 - atomic_load_64(&finQueries)) > 100) { + printf("left queries:%llu\n", (i * 10 - atomic_load_64(&finQueries))); + taosMsleep(2000); + } + printf("\n"); + } + + taos_close(taos); + taos_cleanup(); + + return 0; +} diff --git a/tests/script/api/last-query-ws.cpp b/tests/script/api/last-query-ws.cpp new file mode 100644 index 0000000000..d81ed5bc5e --- /dev/null +++ b/tests/script/api/last-query-ws.cpp @@ -0,0 +1,212 @@ +// g++ --std=c++17 -o multiQueryLastrow multiQueryLastrow.cpp -ltaos -lpthread -ltaosws + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "taos.h" +#include "taosws.h" + +int numThreads = 5; +int numQuerys = 100; +int queryType = 0; +int numConnections = 1; +bool useWebSocket = 0; + +using namespace std; + +const std::string dbName = "iot"; +const std::string sTableName = "m"; +int maxTableIndex = 50000; + +std::mutex mtx; +std::condition_variable cv; +vector taosArray; +vector wtaosArray; + +std::atomic finishCounter; +std::chrono::system_clock::time_point startTime; +std::chrono::system_clock::time_point stopTime; +unordered_map consumeHash; + +static void query(int numQuerys, int id, int type); + +void threadFunction(int id) { + // std::unique_lock lock(mtx); + // cv.wait(lock); + + // lock.unlock(); + + //auto startQueryTime = std::chrono::system_clock::now(); + + query(numQuerys, id, queryType); + + //consumeHash[id] = std::chrono::system_clock::now() - startQueryTime; + + // int counter = finishCounter.fetch_add(1); + // if (counter == numThreads - 1) { + // stopTime = std::chrono::system_clock::now(); + // } +} + +void createThreads(const int numThreads, std::vector* pThreads) { + for (int i = 0; i < numThreads; ++i) { + pThreads->emplace_back(threadFunction, i); + } + + std::cout << "2. Threads created\n"; +} + +void connect() { + void* res = NULL; + + for (auto i = 0; i < numConnections; i++) { + if (useWebSocket) { + const char* dsn = "taos+ws://localhost:6041"; + WS_TAOS* wtaos = ws_connect(dsn); + int32_t code = 0; + if (wtaos == NULL) { + code = ws_errno(NULL); + const char* errstr = ws_errstr(NULL); + std::cout << "Connection failed[" << code << "]: " << errstr << "\n"; + return; + } + code = ws_select_db(wtaos, dbName.c_str()); + const char* errstr = ws_errstr(wtaos); + if (code) { + std::cout << "Connection failed on select db[" << code << "]: " << errstr << "\n"; + return; + } + wtaosArray.push_back(wtaos); + } else { + TAOS* taos = taos_connect("127.0.0.1", "root", "taosdata", dbName.c_str(), 0); + if (!taos) { + std::cerr << "Failed to connect to TDengine\n"; + return; + } + taosArray.push_back(taos); + } + } + + std::cout << "1. Success to connect to TDengine\n"; +} + +void query(int numQuerys, int id, int type) { + int connIdx = id % numConnections; + + for (int i = 0; i < numQuerys; i++) { + std::string sql; + if (type == 0) { + sql = "select last_row(ts) from " + sTableName + std::to_string((i * numThreads + id) % maxTableIndex); + } else { + sql = "select first(ts) from " + sTableName + std::to_string((i * numThreads + id) % maxTableIndex); + } + + if (!useWebSocket) { + TAOS* taos = taosArray[connIdx]; + + TAOS_RES* res = taos_query(taos, sql.c_str()); + if (!res) { + std::cerr << "Failed to query TDengine\n"; + return; + } + + if (taos_errno(res) != 0) { + std::cerr << "Failed to query TDengine since: " << taos_errstr(res) << "\n"; + return; + } + taos_free_result(res); + } else { + WS_TAOS* wtaos = wtaosArray[connIdx]; + + WS_RES* wres = ws_query(wtaos, sql.c_str()); + if (!wres) { + std::cerr << "Failed to query TDengine\n"; + return; + } + + int32_t code = ws_errno(wres); + if (code != 0) { + std::cerr << "Failed to query TDengine since: " << ws_errstr(wres) << "\n"; + return; + } + ws_free_result(wres); + } + } +} + +void printHelp() { + std::cout << "./multiQueryLastrow {numThreads} {numQuerys} {queryType} {numConnections} {useWebSocket}\n"; + exit(-1); +} + +int main(int argc, char* argv[]) { + if (argc != 6) { + printHelp(); + } + + numThreads = atoi(argv[1]); + numQuerys = atoi(argv[2]); + queryType = atoi(argv[3]); + numConnections = atoi(argv[4]); + useWebSocket = atoi(argv[5]); + + std::string queryTypeStr = (queryType == 0) ? "last_row(ts)" : "first(ts)"; + std::cout << "numThreads:" << numThreads << ", queryTimes:" << numQuerys << ", queryType:" << queryTypeStr + << ", numConnections:" << numConnections << ", useWebSocket:" << useWebSocket << "\n"; + + finishCounter.store(0); + + connect(); + + //startTime = std::chrono::system_clock::now(); + + std::vector threads; + createThreads(numThreads, &threads); + + //std::this_thread::sleep_for(std::chrono::seconds(1)); + + std::cout << "3. Start quering\n"; + + startTime = std::chrono::system_clock::now(); + + //cv.notify_all(); + + for (auto& t : threads) { + t.join(); + } + + stopTime = std::chrono::system_clock::now(); + + for (auto& taos : taosArray) { + taos_close(taos); + } + + for (auto& wtaos : wtaosArray) { + ws_close(wtaos); + } + + std::cout << "4. All job done\n"; + + int64_t totalQueryConsumeMs = 0; + for (auto& res : consumeHash) { + totalQueryConsumeMs += res.second.count() /1000000; + } + + std::chrono::nanoseconds elp = stopTime - startTime; + int64_t elpMs = elp.count() / 1000000; + int64_t totalQueryCount = numThreads * numQuerys; + + std::cout << totalQueryCount << " queries finished in " << elpMs << " ms\n"; + std::cout << (float)totalQueryCount * 1000 / elpMs << "q/s\n"; + std::cout << "avg cost:" << totalQueryConsumeMs / totalQueryCount << " ms/q\n"; + + return 0; +} diff --git a/tests/script/api/makefile b/tests/script/api/makefile index ce5980b37a..b871c5f3ff 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -22,6 +22,7 @@ exe: gcc $(CFLAGS) ./insert_stb.c -o $(ROOT)insert_stb $(LFLAGS) gcc $(CFLAGS) ./tmqViewTest.c -o $(ROOT)tmqViewTest $(LFLAGS) gcc $(CFLAGS) ./stmtQuery.c -o $(ROOT)stmtQuery $(LFLAGS) + gcc $(CFLAGS) ./asyncQuery.c -o $(ROOT)asyncQuery $(LFLAGS) # gcc $(CFLAGS) ./stmt.c -o $(ROOT)stmt $(LFLAGS) # gcc $(CFLAGS) ./stmt2.c -o $(ROOT)stmt2 $(LFLAGS) # gcc $(CFLAGS) ./stmt2-example.c -o $(ROOT)stmt2-example $(LFLAGS) diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c index 259d3bec8e..ca8392771e 100644 --- a/tests/script/api/passwdTest.c +++ b/tests/script/api/passwdTest.c @@ -267,7 +267,7 @@ void createUsers(TAOS *taos, const char *host, char *qstr) { } // alter pass for users - sprintf(qstr, "alter user %s pass 'taos'", users[i]); + sprintf(qstr, "alter user %s pass 'taos@123'", users[i]); queryDB(taos, qstr); } } @@ -302,7 +302,7 @@ void passVerTestMulti(const char *host, char *qstr) { queryDB(taos[0], "create table if not exists demo2.stb (ts timestamp, c1 int) tags(t1 int)"); queryDB(taos[0], "create table if not exists demo3.stb (ts timestamp, c1 int) tags(t1 int)"); - strcpy(qstr, "alter user root pass 'taos'"); + strcpy(qstr, "alter user root pass 'taos@123'"); queryDB(taos[0], qstr); // calculate the nPassVerNotified for root and users @@ -345,7 +345,7 @@ void sysInfoTest(TAOS *taosRoot, const char *host, char *qstr) { char userName[USER_LEN] = "user0"; for (int i = 0; i < nRoot; ++i) { - taos[i] = taos_connect(host, "user0", "taos", NULL, 0); + taos[i] = taos_connect(host, "user0", "taos@123", NULL, 0); if (taos[i] == NULL) { fprintf(stderr, "failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/); exit(1); @@ -427,7 +427,7 @@ _loop: printf("\n\n%s:%d LOOP %d, nTestUsers:%d\n", __func__, __LINE__, nLoop, nTestUsers); for (int i = 0; i < nTestUsers; ++i) { // sprintf(users[i], "user%d", i); - taosu[i] = taos_connect(host, users[i], "taos", NULL, 0); + taosu[i] = taos_connect(host, users[i], "taos@123", NULL, 0); if (taosu[i] == NULL) { printf("failed to connect to server, user:%s, reason:%s\n", users[i], "null taos" /*taos_errstr(taos)*/); exit(1); @@ -476,7 +476,7 @@ _loop: nUserDropped = 0; for (int i = 0; i < nTestUsers; ++i) { sprintf(users[i], "user%d", i); - sprintf(qstr, "CREATE USER %s PASS 'taos'", users[i]); + sprintf(qstr, "CREATE USER %s PASS 'taos@123'", users[i]); fprintf(stderr, "%s:%d create user:%s\n", __func__, __LINE__, users[i]); queryDB(taos, qstr); } diff --git a/tests/script/api/stmt2-example.c b/tests/script/api/stmt2-example.c index 692ff90a06..6b59ba3118 100644 --- a/tests/script/api/stmt2-example.c +++ b/tests/script/api/stmt2-example.c @@ -33,7 +33,7 @@ void do_stmt(TAOS* taos) { char* tbs[2] = {"tb", "tb2"}; int t1_val[2] = {0, 1}; - int t2_len[2] = {10, 10}; + int t2_len[2] = {5, 5}; int t3_len[2] = {sizeof(int), sizeof(int)}; TAOS_STMT2_BIND tags[2][2] = {{{0, &t1_val[0], &t3_len[0], NULL, 0}, {0, "after1", &t2_len[0], NULL, 0}}, {{0, &t1_val[1], &t3_len[1], NULL, 0}, {0, "after2", &t2_len[1], NULL, 0}}}; @@ -87,7 +87,7 @@ void do_stmt(TAOS* taos) { taos_stmt2_close(stmt); return; } - + taos_stmt2_free_stb_fields(stmt, pFields); taos_stmt2_close(stmt); } diff --git a/tests/script/api/stmt2-performance.c b/tests/script/api/stmt2-performance.c new file mode 100644 index 0000000000..aa8e5b9450 --- /dev/null +++ b/tests/script/api/stmt2-performance.c @@ -0,0 +1,209 @@ +#include +#include +#include +#include +#include +#include "taos.h" + +int CTB_NUMS = 1000; +int ROW_NUMS = 10; +int CYC_NUMS = 5; + +void do_query(TAOS* taos, const char* sql) { + TAOS_RES* result = taos_query(taos, sql); + int code = taos_errno(result); + if (code) { + printf("failed to query: %s, reason:%s\n", sql, taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); +} + +void do_ins(TAOS* taos, const char* sql) { + TAOS_RES* result = taos_query(taos, sql); + int code = taos_errno(result); + if (code) { + printf("failed to query: %s, reason:%s\n", sql, taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); +} + +void createCtb(TAOS* taos, const char* tbname) { + char* tmp = (char*)malloc(sizeof(char) * 100); + sprintf(tmp, "create table db.%s using db.stb tags(0, 'after')", tbname); + do_query(taos, tmp); +} + +void initEnv(TAOS* taos) { + do_query(taos, "drop database if exists db"); + do_query(taos, "create database db"); + do_query(taos, "create table db.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))"); + do_query(taos, "use db"); +} + +void do_stmt(TAOS* taos, const char* sql) { + initEnv(taos); + + TAOS_STMT2_OPTION option = {0, true, true, NULL, NULL}; + + TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); + int code = taos_stmt2_prepare(stmt, sql, 0); + if (code != 0) { + printf("failed to execute taos_stmt2_prepare. error:%s\n", taos_stmt2_error(stmt)); + taos_stmt2_close(stmt); + return; + } + int fieldNum = 0; + TAOS_FIELD_STB* pFields = NULL; + // code = taos_stmt2_get_stb_fields(stmt, &fieldNum, &pFields); + // if (code != 0) { + // printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt)); + // } else { + // printf("col nums:%d\n", fieldNum); + // for (int i = 0; i < fieldNum; i++) { + // printf("field[%d]: %s, data_type:%d, field_type:%d\n", i, pFields[i].name, pFields[i].type, + // pFields[i].field_type); + // } + // } + + // tbname + char** tbs = (char**)malloc(CTB_NUMS * sizeof(char*)); + for (int i = 0; i < CTB_NUMS; i++) { + tbs[i] = (char*)malloc(sizeof(char) * 20); + sprintf(tbs[i], "ctb_%d", i); + createCtb(taos, tbs[i]); + } + for (int r = 0; r < CYC_NUMS; r++) { + // col params + int64_t** ts = (int64_t**)malloc(CTB_NUMS * sizeof(int64_t*)); + char** b = (char**)malloc(CTB_NUMS * sizeof(char*)); + int* ts_len = (int*)malloc(ROW_NUMS * sizeof(int)); + int* b_len = (int*)malloc(ROW_NUMS * sizeof(int)); + for (int i = 0; i < ROW_NUMS; i++) { + ts_len[i] = sizeof(int64_t); + b_len[i] = 1; + } + for (int i = 0; i < CTB_NUMS; i++) { + ts[i] = (int64_t*)malloc(ROW_NUMS * sizeof(int64_t)); + b[i] = (char*)malloc(ROW_NUMS * sizeof(char)); + for (int j = 0; j < ROW_NUMS; j++) { + ts[i][j] = 1591060628000 + r * 100000 + j; + b[i][j] = 'a' + j; + } + } + // tag params + int t1 = 0; + int t1len = sizeof(int); + int t2len = 3; + // TAOS_STMT2_BIND* tagv[2] = {&tags[0][0], &tags[1][0]}; + + clock_t start, end; + double cpu_time_used; + + // bind params + TAOS_STMT2_BIND** paramv = (TAOS_STMT2_BIND**)malloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*)); + TAOS_STMT2_BIND** tags = (TAOS_STMT2_BIND**)malloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*)); + for (int i = 0; i < CTB_NUMS; i++) { + // create tags + tags[i] = (TAOS_STMT2_BIND*)malloc(2 * sizeof(TAOS_STMT2_BIND)); + tags[i][0] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_INT, &t1, &t1len, NULL, 0}; + tags[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, "after", &t2len, NULL, 0}; + + // create col params + paramv[i] = (TAOS_STMT2_BIND*)malloc(2 * sizeof(TAOS_STMT2_BIND)); + paramv[i][0] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_TIMESTAMP, &ts[i][0], &ts_len[0], NULL, ROW_NUMS}; + paramv[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, &b[i][0], &b_len[0], NULL, ROW_NUMS}; + } + // bind + start = clock(); + TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, tags, paramv}; + if (taos_stmt2_bind_param(stmt, &bindv, -1)) { + printf("failed to execute taos_stmt2_bind_param statement.error:%s\n", taos_stmt2_error(stmt)); + taos_stmt2_close(stmt); + return; + } + end = clock(); + cpu_time_used = ((double)(end - start)) / CLOCKS_PER_SEC; + printf("stmt2-bind [%s] insert Time used: %f seconds\n", sql, cpu_time_used); + start = clock(); + // exec + if (taos_stmt2_exec(stmt, NULL)) { + printf("failed to execute taos_stmt2_exec statement.error:%s\n", taos_stmt2_error(stmt)); + taos_stmt2_close(stmt); + return; + } + end = clock(); + cpu_time_used = ((double)(end - start)) / CLOCKS_PER_SEC; + printf("stmt2-exec [%s] insert Time used: %f seconds\n", sql, cpu_time_used); + } + + // taos_stmt2_free_fields(stmt, pFields); + taos_stmt2_close(stmt); +} + +void do_taosc(TAOS* taos) { + initEnv(taos); + // cols + int64_t** ts = (int64_t*)malloc(CTB_NUMS * sizeof(int64_t)); + char** b = (char**)malloc(CTB_NUMS * sizeof(char*)); + // ctbnames + char** tbs = (char**)malloc(CTB_NUMS * sizeof(char*)); + + // create table before insert + for (int i = 0; i < CTB_NUMS; i++) { + tbs[i] = (char*)malloc(sizeof(char) * 20); + sprintf(tbs[i], "ctb_%d", i); + createCtb(taos, tbs[i]); + } + + for (int i = 0; i < CTB_NUMS; i++) { + ts[i] = (int64_t*)malloc(ROW_NUMS * sizeof(int64_t)); + b[i] = (char*)malloc(ROW_NUMS * sizeof(char)); + for (int j = 0; j < ROW_NUMS; j++) { + ts[i][j] = 1591060628000 + j; + b[i][j] = 'a' + j; + } + } + for (int r = 0; r < CYC_NUMS; r++) { + clock_t start, end; + double cpu_time_used; + char* tsc_sql = malloc(sizeof(char) * 1000000); + sprintf(tsc_sql, "insert into db.stb(tbname,ts,b,t1,t2) values"); + + for (int i = 0; i < CTB_NUMS; i++) { + snprintf(tbs[i], sizeof(tbs[i]), "ctb_%d", i); + for (int j = 0; j < ROW_NUMS; j++) { + if (i == CTB_NUMS - 1 && j == ROW_NUMS - 1) { + sprintf(tsc_sql + strlen(tsc_sql), "('%s',%lld,'%c',0,'after')", tbs[i], ts[i][j] + r * 10000, b[i][j]); + } else { + sprintf(tsc_sql + strlen(tsc_sql), "('%s',%lld,'%c',0,'after'),", tbs[i], ts[i][j] + r * 10000, b[i][j]); + } + } + } + + start = clock(); + // printf("%s", tsc_sql); + do_ins(taos, tsc_sql); + end = clock(); + cpu_time_used = ((double)(end - start)) / CLOCKS_PER_SEC; + printf("taosc insert Time used: %f seconds\n", cpu_time_used); + } +} + +int main() { + TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 0); + if (!taos) { + printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); + exit(1); + } + + sleep(3); + do_stmt(taos, "insert into db.stb(tbname,ts,b,t1,t2) values(?,?,?,?,?)"); + // do_stmt(taos, "insert into db.? using db.stb tags(?,?)values(?,?)"); + do_taosc(taos); + taos_close(taos); + taos_cleanup(); +} diff --git a/tests/script/test.sh b/tests/script/test.sh index b10865dd65..26c01a6c09 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -111,6 +111,7 @@ echo "wal 0" >> $TAOS_CFG echo "asyncLog 0" >> $TAOS_CFG echo "locale en_US.UTF-8" >> $TAOS_CFG echo "enableCoreFile 1" >> $TAOS_CFG +echo "minReservedMemorySize 1024" >> $TAOS_CFG echo " " >> $TAOS_CFG ulimit -n 600000 diff --git a/tests/script/tsim/alter/dnode.sim b/tests/script/tsim/alter/dnode.sim index f9b794924b..0ee08b8735 100644 --- a/tests/script/tsim/alter/dnode.sim +++ b/tests/script/tsim/alter/dnode.sim @@ -5,10 +5,10 @@ sql connect print ======== step1 sql alter dnode 1 'resetlog' -sql alter dnode 1 'monitor' '1' -sql alter dnode 1 'monitor' '0' -sql alter dnode 1 'monitor 1' -sql alter dnode 1 'monitor 0' +sql alter all dnodes 'monitor' '1' +sql alter all dnodes 'monitor' '0' +sql alter all dnodes 'monitor 1' +sql alter all dnodes 'monitor 0' print ======== step2 sql_error alter dnode 1 'resetquerycache' diff --git a/tests/script/tsim/mnode/basic2.sim b/tests/script/tsim/mnode/basic2.sim index 5be29e88a6..71714a11b8 100644 --- a/tests/script/tsim/mnode/basic2.sim +++ b/tests/script/tsim/mnode/basic2.sim @@ -67,7 +67,7 @@ if $data(2)[2] != follower then endi print =============== create user -sql create user user1 PASS 'user1' +sql create user user1 PASS 'user1@#xy' sql select * from information_schema.ins_users if $rows != 2 then return -1 diff --git a/tests/script/tsim/mnode/basic3.sim b/tests/script/tsim/mnode/basic3.sim index ff7c44b67d..e19b4e9443 100644 --- a/tests/script/tsim/mnode/basic3.sim +++ b/tests/script/tsim/mnode/basic3.sim @@ -68,7 +68,7 @@ if $leaderExist != 1 then endi print =============== step3: create user -sql create user user1 PASS 'user1' +sql create user user1 PASS 'user121$*' sql select * from information_schema.ins_users if $rows != 2 then return -1 diff --git a/tests/script/tsim/query/sys_tbname.sim b/tests/script/tsim/query/sys_tbname.sim index a9b27f1389..f8c1d6a986 100644 --- a/tests/script/tsim/query/sys_tbname.sim +++ b/tests/script/tsim/query/sys_tbname.sim @@ -58,7 +58,7 @@ endi sql select tbname from information_schema.ins_tables; print $rows $data00 -if $rows != 44 then +if $rows != 45 then return -1 endi if $data00 != @ins_tables@ then diff --git a/tests/script/tsim/query/tableCount.sim b/tests/script/tsim/query/tableCount.sim index a22bdc8b24..49e9126361 100644 --- a/tests/script/tsim/query/tableCount.sim +++ b/tests/script/tsim/query/tableCount.sim @@ -53,7 +53,7 @@ sql select stable_name,count(table_name) from information_schema.ins_tables grou if $rows != 3 then return -1 endi -if $data01 != 41 then +if $data01 != 42 then return -1 endi if $data11 != 10 then @@ -72,7 +72,7 @@ endi if $data11 != 5 then return -1 endi -if $data21 != 35 then +if $data21 != 36 then return -1 endi if $data31 != 5 then @@ -97,7 +97,7 @@ endi if $data42 != 3 then return -1 endi -if $data52 != 35 then +if $data52 != 36 then return -1 endi if $data62 != 5 then diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index fbf9d50c25..1bd55cbdfe 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -8,7 +8,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c udf -v 1 system sh/exec.sh -n dnode1 -s start sql connect -sql alter user root pass 'taosdata2' +sql alter user root pass '12s34(*&xx' system sh/exec.sh -n dnode1 -s stop system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index 4b2e33b45e..4cabf907c6 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -230,7 +230,7 @@ endi sql_error show create stable t0; sql show variables; -if $rows != 9 then +if $rows != 94 then return -1 endi diff --git a/tests/script/tsim/sync/mnodeLeaderTransfer.sim b/tests/script/tsim/sync/mnodeLeaderTransfer.sim index ed21ac19c3..a1d364fa7c 100644 --- a/tests/script/tsim/sync/mnodeLeaderTransfer.sim +++ b/tests/script/tsim/sync/mnodeLeaderTransfer.sim @@ -33,7 +33,7 @@ sql create mnode on dnode 3 sleep 3000 print =============== create user -sql create user user1 PASS 'user1' +sql create user user1 PASS 'usersdf1$*' sql select * from information_schema.ins_users if $rows != 2 then return -1 diff --git a/tests/script/tsim/trans/lossdata1.sim b/tests/script/tsim/trans/lossdata1.sim index 82e5923468..5b29d2b193 100644 --- a/tests/script/tsim/trans/lossdata1.sim +++ b/tests/script/tsim/trans/lossdata1.sim @@ -11,8 +11,8 @@ system sh/exec.sh -n dnode1 -s start sql connect print =============== create user1 -sql create user user1 PASS 'user1' -sql create user user2 PASS 'user2' +sql create user user1 PASS 'use@##r1$*' +sql create user user2 PASS 'use&*r2$*' sql select * from information_schema.ins_users if $rows != 3 then return -1 diff --git a/tests/script/tsim/user/password.sim b/tests/script/tsim/user/password.sim index d26b9dbc2e..729097e7e1 100644 --- a/tests/script/tsim/user/password.sim +++ b/tests/script/tsim/user/password.sim @@ -4,8 +4,8 @@ system sh/exec.sh -n dnode1 -s start sql connect print ============= step1 -sql create user u_read pass 'taosdata1' -sql create user u_write pass 'taosdata1' +sql create user u_read pass 'tbx12F132!' +sql create user u_write pass 'tbx12145&*' sql alter user u_read pass 'taosdata' sql alter user u_write pass 'taosdata' @@ -15,6 +15,164 @@ if $rows != 3 then return -1 endi +# invalid password format + +sql_error create user user_p1 pass 'taosdata1' +sql_error create user user_p1 pass 'taosdata2' +sql_error create user user_p1 pass '!@#$%^&3' +sql_error create user user_p1 pass '1234564' +sql_error create user user_p1 pass 'taosdataa' +sql_error create user user_p1 pass 'taosdatab' +sql_error create user user_p1 pass '!@#$%^&c' +sql_error create user user_p1 pass '123456d' +sql_error create user user_p1 pass 'taosdataE' +sql_error create user user_p1 pass 'taosdataF' +sql_error create user user_p1 pass '!@#$%^&G' +sql_error create user user_p1 pass '12333315H' +sql_error create user user_p1 pass 'aaaaaaaat1' +sql_error create user user_p1 pass 'TTTTTTTTT2' +sql_error create user user_p1 pass '!@#$%^&!3' +sql_error create user user_p1 pass '12345654' +sql_error create user user_p1 pass 'taosdatata' +sql_error create user user_p1 pass 'TAOSDATATb' +sql_error create user user_p1 pass '!@#$%^&!c' +sql_error create user user_p1 pass '1234565d' +sql_error create user user_p1 pass 'taosdatatE' +sql_error create user user_p1 pass 'TAOSDATATF' +sql_error create user user_p1 pass '!@#$$*!G' +sql_error create user user_p1 pass '1234565H' +sql_error create user user_p1 pass 'taosdataaosdata!' +sql_error create user user_p1 pass 'taosdataaosdata@' +sql_error create user user_p1 pass '!@#$%^&@*#' +sql_error create user user_p1 pass '!@#$%^&' +sql_error create user user_p1 pass '!@#$%^&@*#@' +sql_error create user user_p1 pass '!@#$%^&@*##' +sql_error create user user_p1 pass '!@#$%^&@*#$' +sql_error create user user_p1 pass '!@#$%^&@*#%' +sql_error create user user_p1 pass '!@#$%^&@*#^' +sql_error create user user_p1 pass '!@#$%^&@*#&' +sql_error create user user_p1 pass '!@#$%^&@*#*' +sql_error create user user_p1 pass '!@#$%^&@*#(' +sql_error create user user_p1 pass '!@#$%^&@*#)' +sql_error create user user_p1 pass '!@#$%^&@*#-' +sql_error create user user_p1 pass '!@#$%^&@*#_' +sql_error create user user_p1 pass '!@#$%^&@*#+' +sql_error create user user_p1 pass '!@#$%^&@*#=' +sql_error create user user_p1 pass '!@#$%^&@*#[' +sql_error create user user_p1 pass '!@#$%^&@*#]' +sql_error create user user_p1 pass '!@#$%^&@*#{' +sql_error create user user_p1 pass '!@#$%^&@*#}' +sql_error create user user_p1 pass '!@#$%^&@*#:' +sql_error create user user_p1 pass '!@#$%^&@*#;' +sql_error create user user_p1 pass '!@#$%^&@*#>' +sql_error create user user_p1 pass '!@#$%^&@*#<' +sql_error create user user_p1 pass '!@#$%^&@*#?' +sql_error create user user_p1 pass '!@#$%^&@*#|' +sql_error create user user_p1 pass '!@#$%^&@*#~' +sql_error create user user_p1 pass '!@#$%^&@*#,' +sql_error create user user_p1 pass '!@#$%^&@*#.' +sql_error create user user_p1 pass 'tbd1234TTT\' +sql_error create user user_p1 pass 'tbd1234TTT/' +sql_error create user user_p1 pass 'tbd1234TTT`' +sql_error create user user_p1 pass 'taosdatax' +sql_error create user user_p1 pass 'taosdatay' + +sql_error create user user_p1 pass 'abcd!@1' +sql create user user_p2 pass 'abcd!@12' +sql create user user_p3 pass 'abcd!@123' +sql create user user_p4 pass 'abcd!@1234' +sql create user user_p5 pass 'abcd!@12345' +sql create user user_p6 pass 'abcd!@123456' +sql create user user_p7 pass 'abcd!@1234567' +sql create user user_p8 pass 'abcd!@123456789' +sql create user user_p9 pass 'abcd!@1234567890' +sql_error create user user_p10 pass 'abcd!@1234567890T' +sql drop user user_p2 +sql drop user user_p3 +sql drop user user_p4 +sql drop user user_p5 +sql drop user user_p6 +sql drop user user_p7 +sql drop user user_p8 +sql drop user user_p9 + +sql create user user_p1 pass 'xt12!@cd' + +sql_error alter user user_p1 pass 'abcd!@1' +sql alter user user_p1 pass 'abcd!@12' +sql alter user user_p1 pass 'abcd!@123' +sql alter user user_p1 pass 'abcd!@1234' +sql alter user user_p1 pass 'abcd!@12345' +sql alter user user_p1 pass 'abcd!@123456' +sql alter user user_p1 pass 'abcd!@1234567' +sql alter user user_p1 pass 'abcd!@123456789' +sql alter user user_p1 pass 'abcd!@1234567890' +sql_error user user_p1 pass 'abcd!@1234567890T' +sql_error alter user user_p1 pass 'taosdata1' +sql_error alter user user_p1 pass 'taosdata2' +sql_error alter user user_p1 pass '!@#$%^&3' +sql_error alter user user_p1 pass '1234564' +sql_error alter user user_p1 pass 'taosdataa' +sql_error alter user user_p1 pass 'taosdatab' +sql_error alter user user_p1 pass '!@#$%^&c' +sql_error alter user user_p1 pass '123456d' +sql_error alter user user_p1 pass 'taosdataE' +sql_error alter user user_p1 pass 'taosdataF' +sql_error alter user user_p1 pass '!@#$%^&G' +sql_error alter user user_p1 pass '12334515H' +sql_error alter user user_p1 pass 'aasfdsft1' +sql_error alter user user_p1 pass 'TAOSDATAT2' +sql_error alter user user_p1 pass '!@#$%^&!3' +sql_error alter user user_p1 pass '12345654' +sql_error alter user user_p1 pass 'taosdatata' +sql_error alter user user_p1 pass 'TAOSDATATb' +sql_error alter user user_p1 pass '!@#$%^&!c' +sql_error alter user user_p1 pass '1234565d' +sql_error alter user user_p1 pass 'taosdatatE' +sql_error alter user user_p1 pass 'TAOSDATATF' +sql_error alter user user_p1 pass '*%^^%###!G' +sql_error alter user user_p1 pass '1234565H' +sql_error alter user user_p1 pass 'taosdataaosdata!' +sql_error alter user user_p1 pass 'taosdataaosdata@' +sql_error alter user user_p1 pass '!@#$%^&@*#' +sql_error alter user user_p1 pass '!@#$%^&' +sql_error alter user user_p1 pass '!@#$%^&@*#@' +sql_error alter user user_p1 pass '!@#$%^&@*##' +sql_error alter user user_p1 pass '!@#$%^&@*#$' +sql_error alter user user_p1 pass '!@#$%^&@*#%' +sql_error alter user user_p1 pass '!@#$%^&@*#^' +sql_error alter user user_p1 pass '!@#$%^&@*#&' +sql_error alter user user_p1 pass '!@#$%^&@*#*' +sql_error alter user user_p1 pass '!@#$%^&@*#(' +sql_error alter user user_p1 pass '!@#$%^&@*#)' +sql_error alter user user_p1 pass '!@#$%^&@*#-' +sql_error alter user user_p1 pass '!@#$%^&@*#_' +sql_error alter user user_p1 pass '!@#$%^&@*#+' +sql_error alter user user_p1 pass '!@#$%^&@*#=' +sql_error alter user user_p1 pass '!@#$%^&@*#[' +sql_error alter user user_p1 pass '!@#$%^&@*#]' +sql_error alter user user_p1 pass '!@#$%^&@*#{' +sql_error alter user user_p1 pass '!@#$%^&@*#}' +sql_error alter user user_p1 pass '!@#$%^&@*#:' +sql_error alter user user_p1 pass '!@#$%^&@*#;' +sql_error alter user user_p1 pass '!@#$%^&@*#>' +sql_error alter user user_p1 pass '!@#$%^&@*#<' +sql_error alter user user_p1 pass '!@#$%^&@*#?' +sql_error alter user user_p1 pass '!@#$%^&@*#|' +sql_error alter user user_p1 pass '!@#$%^&@*#~' +sql_error alter user user_p1 pass '!@#$%^&@*#,' +sql_error alter user user_p1 pass '!@#$%^&@*#.' +sql_error alter user user_p1 pass 'tbd1234TTT\' +sql_error alter user user_p1 pass 'tbd1234TTT/' +sql_error alter user user_p1 pass 'tbd1234TTT`' +sql_error alter user user_p1 pass 'taosdatax' +sql_error alter user user_p1 pass 'taosdatay' + +sql drop user user_p1 + +sql create user user_px pass 'taosdata' +sql drop user user_px + print ============= step2 print user u_read login sql close @@ -54,7 +212,7 @@ sql create user oroot pass 'taosdata' sql_error create user $user PASS 'abcd012345678901234567891234567890abcd012345678901234567891234567890abcd012345678901234567891234567890abcd012345678901234567891234567890123' sql_error create userabcd012345678901234567891234567890abcd01234567890123456789123456789 PASS 'taosdata' sql_error create user abcd0123456789012345678901234567890111 PASS '123' -sql create user abc01234567890123456789 PASS '123' +sql create user abc01234567890123456789 PASS '123xyzYDE' sql show users if $rows != 5 then @@ -84,4 +242,34 @@ sql_error REVOKE all ON *.* from o_root; sql_error GRANT read,write ON *.* to o_root; sql_error REVOKE read,write ON *.* from o_root; + +sql create user u01 pass 'taosdata1!' +sql create user u02 pass 'taosdata1@' +sql create user u03 pass 'taosdata1#' +# sql create user u04 pass 'taosdata1$' +sql create user u05 pass 'taosdata1%' +sql create user u06 pass 'taosdata1^' +sql create user u07 pass 'taosdata1&' +sql create user u08 pass 'taosdata1*' +sql create user u09 pass 'taosdata1(' +sql create user u10 pass 'taosdata1)' +sql create user u11 pass 'taosdata1-' +sql create user u12 pass 'taosdata1_' +sql create user u13 pass 'taosdata1+' +sql create user u14 pass 'taosdata1=' +sql create user u15 pass 'taosdata1[' +sql create user u16 pass 'taosdata1]' +sql create user u17 pass 'taosdata1{' +sql create user u18 pass 'taosdata1}' +sql create user u19 pass 'taosdata1:' +sql create user u20 pass 'taosdata1;' +sql create user u21 pass 'taosdata1>' +sql create user u22 pass 'taosdata1<' +sql create user u23 pass 'taosdata1?' +sql create user u24 pass 'taosdata1|' +sql create user u25 pass 'taosdata1~' +sql create user u26 pass 'taosdata1,' +sql create user u27 pass 'taosdata1.' + +return system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/user/privilege_db.sim b/tests/script/tsim/user/privilege_db.sim index 50eaa12108..fb0f9e4566 100644 --- a/tests/script/tsim/user/privilege_db.sim +++ b/tests/script/tsim/user/privilege_db.sim @@ -17,8 +17,8 @@ if $rows != 5 then endi print =============== create users -sql create user user1 PASS 'user1' -sql create user user2 PASS 'user2' +sql create user user1 PASS '123124(*&xx)' +sql create user user2 PASS '1234(*&xx' sql select * from information_schema.ins_users if $rows != 3 then return -1 diff --git a/tests/script/tsim/user/whitelist.sim b/tests/script/tsim/user/whitelist.sim index 4722c00efa..5f98b92bda 100644 --- a/tests/script/tsim/user/whitelist.sim +++ b/tests/script/tsim/user/whitelist.sim @@ -4,8 +4,8 @@ system sh/exec.sh -n dnode1 -s start sql connect print ============= step1 -sql create user u_read pass 'taosdata1' host '127.0.0.1/24','192.168.1.0/24' -sql create user u_write pass 'taosdata1' host '127.0.0.1','192.168.1.0' +sql create user u_read pass 'taosdata1xad@#' host '127.0.0.1/24','192.168.1.0/24' +sql create user u_write pass 'taosdata1TadBD' host '127.0.0.1','192.168.1.0' sql alter user u_read add host '3.3.3.4/24' sql_error alter user u_write drop host '4.4.4.5/25' @@ -16,8 +16,8 @@ if $rows != 3 then endi print ============= step2 -sql_error create user read1 pass 'taosdata1' host '127.0.0/24' -sql_error create user write1 pass 'taosdata1' host '4.4.4.4/33' +sql_error create user read1 pass 'taosdata1XR' host '127.0.0/24' +sql_error create user write1 pass 'TZtaosdata1' host '4.4.4.4/33' sql show users if $rows != 3 then diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index 5629fbb4bd..b7b04a030f 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -120,7 +120,7 @@ if $rows != 3 then endi sql show variables; -if $rows != 9 then +if $rows != 94 then return -1 endi diff --git a/tests/system-test/0-others/compact_vgroups.py b/tests/system-test/0-others/compact_vgroups.py new file mode 100644 index 0000000000..18ad36359f --- /dev/null +++ b/tests/system-test/0-others/compact_vgroups.py @@ -0,0 +1,157 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +from util.autogen import * + +import taos + +class TDTestCase: + # Test cases ====================== + def compactVgroupsErrorTest(self): + # invalid sql + sql = "compact vgroups;" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + # invalid sql + sql = "compact vgroups in" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + # invalid sql + sql = "compact vgroups in ()" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + # error without using database + sql = "compact vgroups in (2)" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + # error with duplicate vgroup + sql = "compact db1.vgroups in (2, 2)" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + # error with invalid vgroup id + sql = "compact db1.vgroups in (0, -1)" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + # error to compact vgroups not in the same dat + sql = "compact db1.vgroups in (7, 8)" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + # error to compact vgroups not in the same database + sql = "compact db1.vgroups in (2, 5, 8)" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + def waitCompactFinish(self): + while True: + sql = 'show compacts' + rows = tdSql.query(sql) + if rows == 0: + break + time.sleep(1) + + def compactVgroupsSqlTest(self): + # make sure there is no compacts + sql = 'show compacts' + rows = tdSql.query(sql) + tdSql.checkEqual(rows, 0) + + # use db1 and compact with db name should be ok + sql = 'use db1' + tdLog.info(f'expect success SQL: {sql}') + tdSql.execute(sql) + + sql = 'compact vgroups in (2)' + tdLog.info(f'expect success SQL: {sql}') + tdSql.execute(sql) + + # check there should be one row in compacts + sql = 'show compacts' + rows = tdSql.query(sql) + tdSql.checkEqual(rows, 1) + + compactId = tdSql.getData(0, 0) + + # query the compact status + sql = f'show compact {compactId}' + tdLog.info(f'expect success SQL: {sql}') + rows = tdSql.query(sql) + tdSql.checkEqual(rows, 1) + tdSql.checkEqual(tdSql.getData(0, 0), compactId) # compact_id + tdSql.checkEqual(tdSql.getData(0, 1), 2) # vgroup_id + + # wait for compact finish + self.waitCompactFinish() + + # start a new compact + sql = 'compact db2.vgroups in (7, 10)' + tdLog.info(f'expect success SQL: {sql}') + tdSql.execute(sql) + + sql = 'show compacts' + rows = tdSql.query(sql) + tdSql.checkEqual(rows, 1) + + compactId = tdSql.getData(0, 0) + sql = f'show compact {compactId}' + tdLog.info(f'expect success SQL: {sql}') + rows = tdSql.query(sql) + tdSql.checkEqual(rows, 2) + tdSql.checkEqual(tdSql.getData(0, 1) in (7, 10), True) + tdSql.checkEqual(tdSql.getData(1, 1) in (7, 10), True) + tdSql.checkEqual(tdSql.getData(0, 1) != tdSql.getData(1, 1), True) + + # wait for compact finish + self.waitCompactFinish() + + + # Test Framework Apis + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to execute {__file__}") + + # init sql + tdSql.init(conn.cursor(), True) + + def run(self): + # create database db1 + sql = "create database db1 vgroups 5" + tdLog.info(sql) + tdSql.execute(sql) + + # create database db2 + sql = "create database db2 vgroups 5" + tdLog.info(sql) + tdSql.execute(sql) + + # error test + self.compactVgroupsErrorTest() + + # success to compact vgroups + self.compactVgroupsSqlTest() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 45d3342571..af0dd6d949 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -61,7 +61,7 @@ class TDTestCase: self.ins_list = ['ins_dnodes','ins_mnodes','ins_qnodes','ins_snodes','ins_cluster','ins_databases','ins_functions',\ 'ins_indexes','ins_stables','ins_tables','ins_tags','ins_columns','ins_users','ins_grants','ins_vgroups','ins_configs','ins_dnode_variables',\ 'ins_topics','ins_subscriptions','ins_streams','ins_stream_tasks','ins_vnodes','ins_user_privileges','ins_views', - 'ins_compacts', 'ins_compact_details', 'ins_grants_full','ins_grants_logs', 'ins_machines', 'ins_arbgroups', 'ins_tsmas', "ins_encryptions", "ins_anodes", "ins_anodes_full", "ins_disk_usage"] + 'ins_compacts', 'ins_compact_details', 'ins_grants_full','ins_grants_logs', 'ins_machines', 'ins_arbgroups', 'ins_tsmas', "ins_encryptions", "ins_anodes", "ins_anodes_full", "ins_disk_usagea", "ins_filesets"] self.perf_list = ['perf_connections','perf_queries','perf_consumers','perf_trans','perf_apps'] def insert_data(self,column_dict,tbname,row_num): insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) @@ -222,7 +222,7 @@ class TDTestCase: tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") tdLog.info(len(tdSql.queryResult)) - tdSql.checkEqual(True, len(tdSql.queryResult) in range(292, 293)) + tdSql.checkEqual(True, len(tdSql.queryResult) in range(303, 304)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(60, len(tdSql.queryResult)) diff --git a/tests/system-test/0-others/persisit_config.py b/tests/system-test/0-others/persisit_config.py new file mode 100644 index 0000000000..fa9f476f7d --- /dev/null +++ b/tests/system-test/0-others/persisit_config.py @@ -0,0 +1,105 @@ +import sys +import taos +import os +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + +class TDTestCase: + updatecfgDict = { + "supportVnodes":"1000", + "tmqMaxTopicNum":"30", + "maxShellConns":"1000", + "monitorFqdn":"localhost:9033", + "tmqRowSize":"1000", + "ttlChangeOnWrite":"1", + "compressor":"1", + "statusInterval":"4", + } + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + self.index = 1 + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def cli_get_param_value(self, config_name): + tdSql.query("show dnode 1 variables;") + for row in tdSql.queryResult: + if config_name == row[1]: + tdLog.debug("Found variable '{}'".format(row[0])) + return row[2] + + def cfg(self, option, value): + cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + if os.system(cmd) != 0 : + tdLog.exit(cmd) + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + + cfgPath = buildPath + "/../sim/dnode%d/cfg" % (self.index) + self.cfgPath = cfgPath + + dataPath = buildPath + "/../sim/dnode%d/data" % (self.index) + self.dataPath = dataPath + + logPath = buildPath + "/../sim/dnode%d/log" % (self.index) + self.logPath = logPath + + tdLog.info("start to check cfg value load from cfg file {0}".format(cfgPath)) + + for name,expValue in self.updatecfgDict.items(): + actValue = self.cli_get_param_value(name) + tdLog.debug("Get {} value: {} Expect value: {}".format(name, actValue,expValue)) + assert str(actValue) == str(expValue) + + tdLog.info("rm -rf {0}".format(cfgPath)) + os.system("rm -rf {0}/*".format(cfgPath)) + + tdLog.info("rebuilt cfg file {0}".format(cfgPath)) + cfgName = cfgPath+"/taos.cfg" + os.system("touch {0}".format(cfgName)) + os.system("echo 'fqdn localhost' >> {0}".format(cfgName)) + os.system("echo 'dataDir {0}' >> {1}".format(dataPath, cfgName)) + os.system("echo 'logDir {0}' >> {1}".format(logPath, cfgName)) + + tdDnodes.stop(1) + tdLog.info("restart taosd") + tdDnodes.start(1) + + tdLog.info("start to check cfg value load from mnd sdb") + + for name,expValue in self.updatecfgDict.items(): + actValue = self.cli_get_param_value(name) + tdLog.debug("Get {} value: {} Expect value: {}".format(name, actValue,expValue)) + assert str(actValue) == str(expValue) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) + diff --git a/tests/system-test/0-others/qmemCtrl.py b/tests/system-test/0-others/qmemCtrl.py new file mode 100644 index 0000000000..a2eedbec0e --- /dev/null +++ b/tests/system-test/0-others/qmemCtrl.py @@ -0,0 +1,134 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import re +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + updatecfgDict = {'forceReadConfig':'1','queryUseMemoryPool':'true','minReservedMemorySize':1025, 'singleQueryMaxMemorySize': 0} + def init(self, conn, logSql, replicaVar=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + self.perf_param_list = ['apps','connections','consumers','queries','trans'] + self.dbname = "db" + self.vgroups = 4 + self.stbname = f'`{tdCom.getLongName(5)}`' + self.tbname = f'`{tdCom.getLongName(3)}`' + self.db_param = { + "database":f"{self.dbname}", + "buffer":100, + "vgroups":self.vgroups, + "stt_trigger":1, + "tsdb_pagesize":16 + } + + def update_cfg(self, use_mpool = 1, min_rsize = 0, single_msize = 0): + updatecfgDict = {'queryUseMemoryPool':f'{use_mpool}','minReservedMemorySize': min_rsize, 'singleQueryMaxMemorySize': single_msize} + tdDnodes.stop(1) + tdDnodes.deploy(1, updatecfgDict) + tdDnodes.starttaosd(1) + + def alter_cfg(self, use_mpool = 1, min_rsize = 0, single_msize = 0): + tdSql.error(f"alter dnode 1 'queryUseMemoryPool' '{use_mpool}'") + tdSql.error(f"alter dnode 1 'minReservedMemorySize' '{min_rsize}'") + tdSql.error(f"alter dnode 1 'singleQueryMaxMemorySize' '{single_msize}'") + + def variables_check(self, err_case = 0, use_mpool = 1, min_rsize = 0, single_msize = 0): + if err_case == 1: + tdSql.error("show dnode 1 variables like 'queryUseMemoryPool'") + else: + tdSql.query("show dnode 1 variables like 'queryUseMemoryPool'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'queryUseMemoryPool') + tdSql.checkData(0, 2, use_mpool) + + if err_case == 1: + tdSql.error("show dnode 1 variables like 'minReservedMemorySize'") + else: + tdSql.query("show dnode 1 variables like 'minReservedMemorySize'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'minReservedMemorySize') + tdSql.checkData(0, 2, min_rsize) + + if err_case == 1: + tdSql.error("show dnode 1 variables like 'singleQueryMaxMemorySize'") + else: + tdSql.query("show dnode 1 variables like 'singleQueryMaxMemorySize'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'singleQueryMaxMemorySize') + tdSql.checkData(0, 2, single_msize) + + + def cfg_check(self): + tdLog.info(f'[disable pool] start') + self.update_cfg(0, 1024, 0) + self.variables_check(0, 0, 1024, 0) + + tdLog.info(f'[enable pool + up limit] start') + self.update_cfg(1, 1024, 1000000000) + self.variables_check(0, 1, 1024, 1000000000) + + tdLog.info(f'[enable pool + reserve limit] start') + self.update_cfg(1, 1000000000, 3000) + self.variables_check(0, 1, 1000000000, 3000) + + tdLog.info(f'[enable pool + out of reserve] start') + self.update_cfg(1, 1000000001) + self.variables_check(1) + + tdLog.info(f'[enable pool + out of single] start') + self.update_cfg(1, 1000, 1000000001) + self.variables_check(1) + + tdLog.info(f'[out of pool] start') + self.update_cfg(2) + self.variables_check(1) + + def alter_check(self): + tdLog.info(f'[alter] start') + self.update_cfg(1, 1024, 0) + self.alter_cfg(1, 1024, 0); + + def single_up_check(self): + tdLog.info(f'[low single] start') + self.update_cfg(1, 1024, 1) + tdSql.error("select *, repeat('aaaaaaaaaa',1000) from information_schema.ins_tables") + + tdLog.info(f'[normal single] start') + self.update_cfg(1, 1024, 100) + tdSql.query("select *, repeat('aaaaaaaaaa',1000) from information_schema.ins_tables") + + def too_big_reserve(self): + tdLog.info(f'[too big reserve] start') + self.update_cfg(1, 1024000) + + def run(self): + self.cfg_check() + self.alter_check() + self.single_up_check() + self.too_big_reserve() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/show.py b/tests/system-test/0-others/show.py index 032e5eebe1..e6fa7bf16b 100644 --- a/tests/system-test/0-others/show.py +++ b/tests/system-test/0-others/show.py @@ -288,7 +288,7 @@ class TDTestCase: vals = ['9223372036854775807', '9223372036854775807.1', '9223372036854775806', '9223372036854775808', '9223372036854775808.1', '9223372036854775807.0', '9223372036854775806.1'] expected_vals = ['9223372036854775807', 'err', '9223372036854775806', 'err', 'err', 'err', 'err'] for val_str, expected_val in zip(vals, expected_vals): - sql = f'ALTER dnode 1 "{var}" "{val_str}"' + sql = f'ALTER all dnodes "{var}" "{val_str}"' if expected_val == 'err': tdSql.error(sql) else: diff --git a/tests/system-test/0-others/subscribe_stream_privilege.py b/tests/system-test/0-others/subscribe_stream_privilege.py index b477af9f57..a447e43e8b 100644 --- a/tests/system-test/0-others/subscribe_stream_privilege.py +++ b/tests/system-test/0-others/subscribe_stream_privilege.py @@ -114,7 +114,7 @@ class TDTestCase: consumer_dict = { "group.id": "g1", "td.connect.user": self.user_name, - "td.connect.pass": "test", + "td.connect.pass": "123456rf@#", "auto.offset.reset": "earliest" } consumer = Consumer(consumer_dict) @@ -167,7 +167,7 @@ class TDTestCase: def create_user(self): tdSql.execute(f'create topic {self.topic_name} as database {self.dbnames[0]}') - tdSql.execute(f'create user {self.user_name} pass "test"') + tdSql.execute(f'create user {self.user_name} pass "123456rf@#"') def run(self): self.prepare_data() diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py index 91e9f2fb89..b43723161f 100644 --- a/tests/system-test/0-others/taosShell.py +++ b/tests/system-test/0-others/taosShell.py @@ -158,7 +158,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() # time.sleep(2) - tdSql.query("create user testpy pass 'testpy'") + tdSql.query("create user testpy pass 'testpy243#@'") tdSql.query("alter user testpy createdb 1") #hostname = socket.gethostname() @@ -175,7 +175,7 @@ class TDTestCase: checkNetworkStatus = ['0: unavailable', '1: network ok', '2: service ok', '3: service degraded', '4: exiting'] netrole = ['client', 'server'] - keyDict = {'h':'', 'P':'6030', 'p':'testpy', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ + keyDict = {'h':'', 'P':'6030', 'p':'testpy243#@', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ 'k':'', 't':'', 'n':'', 'l':'1024', 'N':'100', 'V':'', 'd':'db', 'w':'30', '-help':'', '-usage':'', '?':''} keyDict['h'] = self.hostname diff --git a/tests/system-test/0-others/taosShellError.py b/tests/system-test/0-others/taosShellError.py index 5e6a590806..e7221b6409 100644 --- a/tests/system-test/0-others/taosShellError.py +++ b/tests/system-test/0-others/taosShellError.py @@ -134,7 +134,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() # time.sleep(2) - tdSql.query("create user testpy pass 'testpy'") + tdSql.query("create user testpy pass 'testpy243#@'") #hostname = socket.gethostname() #tdLog.info ("hostname: %s" % hostname) @@ -150,7 +150,7 @@ class TDTestCase: checkNetworkStatus = ['0: unavailable', '1: network ok', '2: service ok', '3: service degraded', '4: exiting'] netrole = ['client', 'server'] - keyDict = {'h':'', 'P':'6030', 'p':'testpy', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ + keyDict = {'h':'', 'P':'6030', 'p':'testpy243#@', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ 'k':'', 't':'', 'n':'', 'l':'1024', 'N':'100', 'V':'', 'd':'db', 'w':'30', '-help':'', '-usage':'', '?':''} keyDict['h'] = self.hostname diff --git a/tests/system-test/0-others/taosShellNetChk.py b/tests/system-test/0-others/taosShellNetChk.py index d2efa5d9fe..37d82b7e84 100644 --- a/tests/system-test/0-others/taosShellNetChk.py +++ b/tests/system-test/0-others/taosShellNetChk.py @@ -133,7 +133,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() - tdSql.query("create user testpy pass 'testpy'") + tdSql.query("create user testpy pass 'testpy243#@'") buildPath = self.getBuildPath() if (buildPath == ""): @@ -146,7 +146,7 @@ class TDTestCase: checkNetworkStatus = ['0: unavailable', '1: network ok', '2: service ok', '3: service degraded', '4: exiting'] netrole = ['client', 'server'] - keyDict = {'h':'', 'P':'6030', 'p':'testpy', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ + keyDict = {'h':'', 'P':'6030', 'p':'testpy243#@', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ 'k':'', 't':'', 'n':'', 'l':'1024', 'N':'100', 'V':'', 'd':'db', 'w':'30', '-help':'', '-usage':'', '?':''} keyDict['h'] = self.hostname diff --git a/tests/system-test/0-others/taosdShell.py b/tests/system-test/0-others/taosdShell.py index 9b0628ec12..8f30930957 100644 --- a/tests/system-test/0-others/taosdShell.py +++ b/tests/system-test/0-others/taosdShell.py @@ -132,7 +132,7 @@ class TDTestCase: def preData(self): # database\stb\tb\chiild-tb\rows\topics - tdSql.execute("create user testpy pass 'testpy'") + tdSql.execute("create user testpy pass 'testpy243#@'") tdSql.execute("drop database if exists db0;") tdSql.execute("create database db0 wal_retention_period 3600;") tdSql.execute("use db0;") diff --git a/tests/system-test/0-others/test_alter_config.py b/tests/system-test/0-others/test_alter_config.py new file mode 100644 index 0000000000..14e77bb489 --- /dev/null +++ b/tests/system-test/0-others/test_alter_config.py @@ -0,0 +1,548 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import subprocess +import random +import time +import os +import platform +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from frame.srvCtl import * +from frame import * + +class TDTestCase: + """This test case is used to veirfy hot refresh configurations + """ + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + self.configration_dic = { + "cli": [ + ], + "svr": [ + { + "name": "audit", + "value": 0, + "category": "global" + }, + { + "name": "asyncLog", + "value": 0, + "category": "local" + }, + { + "name": "disableStream", + "value": 1, + "category": "global" + }, + { + "name": "enableWhiteList", + "value": 1, + "category": "global" + }, + { + "name": "statusInterval", + "value": 3, + "category": "global" + }, + { + "name": "telemetryReporting", + "value" : 1, + "category" : "global" + }, + { + "name" : "monitor", + "value" : 0, + "category" : "global" + }, + { + "name" : "monitorInterval", + "value" : 3, + "category" : "global" + }, + { + "name" : "monitorComp", + "value" : 1, + "category" : "global" + }, + { + "name" : "monitorForceV2", + "value" : 0, + "category" : "global" + }, + { + "name" : "monitorLogProtocol", + "value" : 1, + "category" : "global" + }, + { + "name": "monitorMaxLogs", + "value": 1000, + "category": "global" + }, + { + "name": "auditCreateTable", + "value": 0, + "category": "global" + }, + { + "name": "auditInterval", + "value": 4000, + "category": "global" + }, + { + "name": "slowLogThreshold", + "value": 20, + "category": "global" + }, + { + "name": "compressMsgSize", + "value": 0, + "category": "global" + }, + { + "name": "compressor", + "value": "GZIP_COMPRESSOR", + "category": "global" + }, + { + "name": "curRange", + "value": 200, + "category": "global" + }, + { + "name": "fPrecision", + "value": "1000.000000", + "category": "global" + }, + { + "name": "dPrecision", + "value": "1000.000000", + "category": "global" + }, + { + "name": "ifAdtFse", + "value": 1, + "category": "global" + }, + { + "name": "maxRange", + "value": 1000, + "category": "global" + }, + { + "name": "maxTsmaNum", + "value": 2, + "category": "global" + }, + { + "name": "queryRsmaTolerance", + "value": 2000, + "category": "global" + }, + { + "name": "uptimeInterval", + "value": 600, + "category": "global" + }, + { + "name": "slowLogMaxLen", + "value": 8192, + "category": "global" + }, + { + "name": "slowLogScope", + "value": "insert", + "category": "global" + }, + { + "name": "slowLogExceptDb", + "value": "db1", + "category": "global" + }, + { + "name": "mndSdbWriteDelta", + "value": 1000, + "category": "global" + }, + { + "name": "minDiskFreeSize", + "value": 100*1024*1024, + "category": "global" + }, + { + "name": "randErrorChance", + "value": 5, + "category": "global" + }, + { + "name": "randErrorDivisor", + "value": 20001, + "category": "global" + }, + { + "name": "randErrorScope", + "value": 8, + "category": "global" + }, + { + "name": "syncLogBufferMemoryAllowed", + "value": 1024 * 1024 * 20 * 10, + "category": "global" + }, + { + "name": "resolveFQDNRetryTime", + "value": 500, + "category": "global" + }, + { + "name": "syncElectInterval", + "value": 50000, + "category": "global" + }, + { + "name": "syncHeartbeatInterval", + "value": 3000, + "category": "global" + }, + { + "name": "syncHeartbeatTimeout", + "value": 40000, + "category": "global" + }, + { + "name": "syncSnapReplMaxWaitN", + "value": 32, + "category": "global" + }, + { + "name": "walFsyncDataSizeLimit", + "value": 200 * 1024 * 1024, + "category": "global" + }, + { + "name": "numOfCores", + "value": "30.000000", + "category": "global" + }, + { + "name": "enableCoreFile", + "value": 0, + "category": "global" + }, + { + "name": "telemetryInterval", + "value": 6000, + "category": "global" + }, + { + "name": "cacheLazyLoadThreshold", + "value": 1000, + "category": "global" + }, + { + "name": "checkpointInterval", + "value": 120, + "category": "global" + }, + { + "name": "concurrentCheckpoint", + "value": 3, + "category": "global" + }, + { + "name": "retentionSpeedLimitMB", + "value": 24, + "category": "global" + }, + { + "name": "ttlChangeOnWrite", + "value": 1, + "category": "global" + }, + { + "name": "logKeepDays", + "value": 30, + "category": "local" + }, + { + "name": "maxStreamBackendCache", + "value": 256, + "category": "global" + }, + { + "name": "mqRebalanceInterval", + "value": 30, + "category": "global" + }, + { + "name": "numOfLogLines", + "value": 20000000, + "category": "local" + }, + { + "name": "queryRspPolicy", + "value": 1, + "category": "global" + }, + { + "name": "timeseriesThreshold", + "value": 100, + "category": "global" + }, + { + "name": "tmqMaxTopicNum", + "value": 30, + "category": "global" + }, + { + "name": "tmqRowSize", + "value": 8192, + "category": "global" + }, + { + "name": "transPullupInterval", + "value": 4, + "category": "global" + }, + { + "name": "compactPullupInterval", + "value": 20, + "category": "global" + }, + { + "name": "trimVDbIntervalSec", + "value": 7200, + "category": "global" + }, + { + "name": "ttlBatchDropNum", + "value": 20000, + "category": "global" + }, + { + "name": "ttlFlushThreshold", + "value": 200, + "category": "global" + }, + { + "name": "ttlPushInterval", + "value": 20, + "category": "global" + }, + { + "name": "ttlUnit", + "value": 86500, + "category": "global" + }, + { + "name": "udf", + "value": 0, + "category": "global" + }, + # { + # "name": "udfdLdLibPath", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "udfdResFuncs", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "s3Accesskey", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "s3BucketName", + # "value": 1000, + # "category": "global" + # }, + # { + # "name": "s3Endpoint", + # "value": 1000, + # "category": "global" + # }, + { + "name": "s3MigrateIntervalSec", + "value": 1800, + "category": "global" + }, + { + "name": "s3MigrateEnabled", + "value": 1, + "category": "global" + }, + # { + # "name": "s3BlockCacheSize", + # "value": 32, + # "category": "global" + # }, + { + "name": "s3PageCacheSize", + "value": 8192, + "category": "global" + }, + { + "name": "s3UploadDelaySec", + "value": 30, + "category": "global" + }, + { + "name": "mndLogRetention", + "value": 1000, + "category": "global" + }, + { + "name": "supportVnodes", + "value": 128, + "category": "global" + }, + { + "name": "experimental", + "value": 0, + "category": "global" + }, + { + "name": "maxTsmaNum", + "value": 2, + "category": "global" + }, + { + "name": "maxShellConns", + "value": 25000, + "category": "global" + }, + { + "name": "numOfRpcSessions", + "value": 15000, + "category": "global" + }, + { + "name": "numOfRpcThreads", + "value": 2, + "category": "global" + }, + { + "name": "rpcQueueMemoryAllowed", + "value": 1024*1024*20*10, + "category": "global" + }, + { + "name": "shellActivityTimer", + "value": 2, + "category": "global" + }, + { + "name": "timeToGetAvailableConn", + "value": 200000, + "category": "global" + }, + { + "name": "readTimeout", + "value": 800, + "category": "global" + }, + { + "name": "safetyCheckLevel", + "value": 2, + "category": "global" + }, + { + "name": "bypassFlag", + "value": 4, + "category": "global" + } + ] + } + def cli_get_param_value(self, config_name): + tdSql.query("show local variables;") + for row in tdSql.queryResult: + if config_name == row[0]: + return row[1] + + def svr_get_param_value(self, config_name): + tdSql.query("show dnode 1 variables;") + for row in tdSql.queryResult: + if config_name == row[1]: + return row[2] + + def configuration_alter(self): + for key in self.configration_dic: + if "svr" == key: + for item in self.configration_dic[key]: + name = item["name"] + value = item["value"] + category = item["category"] + if category == "global": + tdSql.execute(f'alter all dnodes "{name} {value}";') + else: + tdSql.execute(f'alter dnode 1 "{name} {value}";') + elif "cli" == key: + for item in self.configration_dic[key]: + name = item["name"] + value = item["value"] + category = item["category"] + if category == "global": + tdSql.execute(f'alter all dnodes "{name} {value}";') + else: + tdSql.execute(f'alter local "{name} {value}";') + else: + raise Exception(f"unknown key: {key}") + + def run(self): + self.configuration_alter() + for key in self.configration_dic: + if "cli" == key: + for item in self.configration_dic[key]: + actVal = self.cli_get_param_value(item["name"]) + assert str(actVal) == str(item["value"]), f"tem name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + elif "svr" == key: + for item in self.configration_dic[key]: + actVal = self.svr_get_param_value(item["name"]) + assert str(actVal) == str(item["value"]), f"tem name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + else: + raise Exception(f"unknown key: {key}") + + tdLog.info("stop and restart taosd") + tdDnodes.stop(1) + tdDnodes.start(1) + + for key in self.configration_dic: + if "cli" == key: + for item in self.configration_dic[key]: + actVal = self.cli_get_param_value(item["name"]) + assert str(actVal) == str(item["oldVal"]), f"item name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + elif "svr" == key: + for item in self.configration_dic[key]: + actVal = self.svr_get_param_value(item["name"]) + assert str(actVal) == str(item["value"]), f"item name: {item['name']}, Expected value: {item['value']}, actual value: {actVal}" + else: + raise Exception(f"unknown key: {key}") + + def stop(self): + tdSql.close() + # tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index da218162d4..47f5209940 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -224,7 +224,8 @@ class TDTestCase: for i in range(len(values)): v = values[i] dnode = random.choice(p_list) - tdSql.execute(f'alter {dnode} "{name} {v}";') + tdSql.execute(f'alter all dnodes "{name} {v}";') + sleep(0.5) value = self.svr_get_param_value(name) tdLog.debug(f"value: {value}") if check_values: diff --git a/tests/system-test/0-others/test_show_disk_usage.py b/tests/system-test/0-others/test_show_disk_usage.py index abaecffec4..eb5bdf1aa7 100644 --- a/tests/system-test/0-others/test_show_disk_usage.py +++ b/tests/system-test/0-others/test_show_disk_usage.py @@ -191,7 +191,7 @@ class TDTestCase: self.value_check(icache, cacheRdbSize, 64) self.value_check(itableMeta,tableMetaSize, 64) self.value_check(itsdbSize, tsdbSize, 64) - self.value_check(iwal, walSize, 128) + self.value_check(iwal, walSize, 256) #if abs(icache - cacheRdbSize) > 12: # tdLog.error("cache_rdb size is not equal") diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index c4d24582e4..33b263c155 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -182,14 +182,14 @@ class TDTestCase: for i in range(self.users_count): user = User() user.name = f"user_test{i}" - user.passwd = f"taosdata{i}" + user.passwd = f"taosdata@1{i}" user.db_set = set() self.users.append(user) return self.users @property def __passwd_list(self): - return [f"taosdata{i}" for i in range(self.users_count) ] + return [f"taosdata@1{i}" for i in range(self.users_count) ] @property def __privilege(self): @@ -210,34 +210,34 @@ class TDTestCase: def create_user_err(self): sqls = [ - "create users u1 pass 'u1passwd' ", - "create user '' pass 'u1passwd' ", - "create user pass 'u1passwd' ", - "create user u1 pass u1passwd ", - "create user u1 password 'u1passwd' ", - "create user u1 pass u1passwd ", + "create users u1 pass 'u1Passwd' ", + "create user '' pass 'u1Passwd' ", + "create user pass 'u1Passwd' ", + "create user u1 pass u1Passwd ", + "create user u1 password 'u1Passwd' ", + "create user u1 pass u1Passwd ", "create user u1 pass '' ", "create user u1 pass ' ' ", "create user u1 pass ", - "create user u1 u2 pass 'u1passwd' 'u2passwd' ", - "create user u1 u2 pass 'u1passwd', 'u2passwd' ", - "create user u1, u2 pass 'u1passwd', 'u2passwd' ", - "create user u1, u2 pass 'u1passwd' 'u2passwd' ", + "create user u1 u2 pass 'u1Passwd' 'u2passwd' ", + "create user u1 u2 pass 'u1Passwd', 'u2passwd' ", + "create user u1, u2 pass 'u1Passwd', 'u2passwd' ", + "create user u1, u2 pass 'u1Passwd' 'u2passwd' ", # length of user_name must <= 23 - "create user u12345678901234567890123 pass 'u1passwd' " , + "create user u12345678901234567890123 pass 'u1Passwd' " , # length of passwd must <= 128 "create user u1 pass 'u12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678' " , # password must have not " ' ~ ` \ - "create user u1 pass 'u1passwd\\' " , - "create user u1 pass 'u1passwd~' " , - "create user u1 pass 'u1passwd\"' " , - "create user u1 pass 'u1passwd\'' " , - "create user u1 pass 'u1passwd`' " , + "create user u1 pass 'u1Passwd\\' " , + "create user u1 pass 'u1Passwd~' " , + "create user u1 pass 'u1Passwd\"' " , + "create user u1 pass 'u1Passwd\'' " , + "create user u1 pass 'u1Passwd`' " , # must after create a user named u1 - "create user u1 pass 'u1passwd' " , + "create user u1 pass 'u1Passwd' " , ] - tdSql.execute("create user u1 pass 'u1passwd' ") + tdSql.execute("create user u1 pass 'u1Passwd' ") for sql in sqls: tdSql.error(sql) @@ -258,12 +258,12 @@ class TDTestCase: def alter_pass_err(self): # sourcery skip: remove-redundant-fstring sqls = [ - f"alter users {self.__user_list[0]} pass 'newpass' " , + f"alter users {self.__user_list[0]} pass 'newpassT1' " , f"alter user {self.__user_list[0]} pass '' " , f"alter user {self.__user_list[0]} pass ' ' " , - f"alter user anyuser pass 'newpass' " , + f"alter user anyuser pass 'newpassT1' " , f"alter user {self.__user_list[0]} pass " , - f"alter user {self.__user_list[0]} password 'newpass' " , + f"alter user {self.__user_list[0]} password 'newpassT1' " , ] for sql in sqls: tdSql.error(sql) @@ -649,7 +649,7 @@ class TDTestCase: # user = conn # 不能创建用户 tdLog.printNoPrefix("==========step4.1: normal user can not create user") - user.error("create use utest1 pass 'utest1pass'") + user.error("create use utest1 pass 'utest1Pass'") # 可以查看用户 tdLog.printNoPrefix("==========step4.2: normal user can show user") user.query("show users") diff --git a/tests/system-test/0-others/user_manage.py b/tests/system-test/0-others/user_manage.py index 6f90a2873a..e48231b059 100644 --- a/tests/system-test/0-others/user_manage.py +++ b/tests/system-test/0-others/user_manage.py @@ -80,16 +80,16 @@ class TDTestCase: for user_name in ['jiacy1_all', 'jiacy1_read', 'jiacy1_write', 'jiacy1_none', 'jiacy0_all', 'jiacy0_read', 'jiacy0_write', 'jiacy0_none']: if 'jiacy1' in user_name.lower(): - tdSql.execute(f'create user {user_name} pass "123" sysinfo 1') + tdSql.execute(f'create user {user_name} pass "123abc!@#" sysinfo 1') elif 'jiacy0' in user_name.lower(): - tdSql.execute(f'create user {user_name} pass "123" sysinfo 0') + tdSql.execute(f'create user {user_name} pass "123abc!@#" sysinfo 0') for user_name in ['jiacy1_all', 'jiacy1_read', 'jiacy0_all', 'jiacy0_read']: tdSql.execute(f'grant read on db to {user_name}') for user_name in ['jiacy1_all', 'jiacy1_write', 'jiacy0_all', 'jiacy0_write']: tdSql.execute(f'grant write on db to {user_name}') def user_privilege_check(self): - jiacy1_read_conn = taos.connect(user='jiacy1_read', password='123') + jiacy1_read_conn = taos.connect(user='jiacy1_read', password='123abc!@#') sql = "create table ntb (ts timestamp,c0 int)" expectErrNotOccured = True try: @@ -107,14 +107,14 @@ class TDTestCase: pass def drop_topic(self): - jiacy1_all_conn = taos.connect(user='jiacy1_all', password='123') - jiacy1_read_conn = taos.connect(user='jiacy1_read', password='123') - jiacy1_write_conn = taos.connect(user='jiacy1_write', password='123') - jiacy1_none_conn = taos.connect(user='jiacy1_none', password='123') - jiacy0_all_conn = taos.connect(user='jiacy0_all', password='123') - jiacy0_read_conn = taos.connect(user='jiacy0_read', password='123') - jiacy0_write_conn = taos.connect(user='jiacy0_write', password='123') - jiacy0_none_conn = taos.connect(user='jiacy0_none', password='123') + jiacy1_all_conn = taos.connect(user='jiacy1_all', password='123abc!@#') + jiacy1_read_conn = taos.connect(user='jiacy1_read', password='123abc!@#') + jiacy1_write_conn = taos.connect(user='jiacy1_write', password='123abc!@#') + jiacy1_none_conn = taos.connect(user='jiacy1_none', password='123abc!@#') + jiacy0_all_conn = taos.connect(user='jiacy0_all', password='123abc!@#') + jiacy0_read_conn = taos.connect(user='jiacy0_read', password='123abc!@#') + jiacy0_write_conn = taos.connect(user='jiacy0_write', password='123abc!@#') + jiacy0_none_conn = taos.connect(user='jiacy0_none', password='123abc!@#') tdSql.execute('create topic root_db as select * from db.stb') for user in [jiacy1_all_conn, jiacy1_read_conn, jiacy0_all_conn, jiacy0_read_conn]: user.execute(f'create topic db_jiacy as select * from db.stb') @@ -149,7 +149,7 @@ class TDTestCase: tdSql.execute('create topic db_topic as select * from db.stb') tdSql.execute('grant subscribe on db_topic to jiacy1_all') print("build consumer") - tmq = Consumer({"group.id": "tg2", "td.connect.user": "jiacy1_all", "td.connect.pass": "123", + tmq = Consumer({"group.id": "tg2", "td.connect.user": "jiacy1_all", "td.connect.pass": "123abc!@#", "enable.auto.commit": "true"}) print("build topic list") tmq.subscribe(["db_topic"]) diff --git a/tests/system-test/0-others/user_privilege.py b/tests/system-test/0-others/user_privilege.py index a731e85ddb..809881589a 100644 --- a/tests/system-test/0-others/user_privilege.py +++ b/tests/system-test/0-others/user_privilege.py @@ -58,7 +58,7 @@ class TDTestCase: self.stbnum_grant = 200 def create_user(self): - tdSql.execute(f'create user {self.user_name} pass "test"') + tdSql.execute(f'create user {self.user_name} pass "test123@#$"') tdSql.execute(f'grant read on {self.dbnames[0]}.{self.stbname} with t2 = "Beijing" to {self.user_name}') tdSql.execute(f'grant write on {self.dbnames[1]}.{self.stbname} with t1 = 2 to {self.user_name}') @@ -75,7 +75,7 @@ class TDTestCase: tdSql.execute(f'create table {self.stbname}_grant_{i} (ts timestamp, c0 int) tags(t0 int)') def user_read_privilege_check(self, dbname): - testconn = taos.connect(user='test', password='test') + testconn = taos.connect(user='test', password='test123@#$') expectErrNotOccured = False try: @@ -94,7 +94,7 @@ class TDTestCase: pass def user_write_privilege_check(self, dbname): - testconn = taos.connect(user='test', password='test') + testconn = taos.connect(user='test', password='test123@#$') expectErrNotOccured = False try: @@ -110,7 +110,7 @@ class TDTestCase: pass def user_privilege_error_check(self): - testconn = taos.connect(user='test', password='test') + testconn = taos.connect(user='test', password='test123@#$') expectErrNotOccured = False sql_list = [f"alter talbe {self.dbnames[0]}.stb_1 set t2 = 'Wuhan'", diff --git a/tests/system-test/0-others/user_privilege_all.py b/tests/system-test/0-others/user_privilege_all.py index 846b76317e..76f952f572 100644 --- a/tests/system-test/0-others/user_privilege_all.py +++ b/tests/system-test/0-others/user_privilege_all.py @@ -21,7 +21,7 @@ class TDTestCase: self.setsql = TDSetSql() # user info self.username = 'test' - self.password = 'test' + self.password = 'test123@#$' # db info self.dbname = "user_privilege_all_db" self.stbname = 'stb' diff --git a/tests/system-test/0-others/user_privilege_multi_users.py b/tests/system-test/0-others/user_privilege_multi_users.py index 53ff136e63..69ad32b756 100644 --- a/tests/system-test/0-others/user_privilege_multi_users.py +++ b/tests/system-test/0-others/user_privilege_multi_users.py @@ -19,7 +19,7 @@ class TDTestCase: # user info self.userNum = 100 self.basic_username = "user" - self.password = "pwd" + self.password = "test123@#$" # db info self.dbname = "user_privilege_multi_users" diff --git a/tests/system-test/0-others/user_privilege_show.py b/tests/system-test/0-others/user_privilege_show.py index 9f49778ba8..41a920967d 100644 --- a/tests/system-test/0-others/user_privilege_show.py +++ b/tests/system-test/0-others/user_privilege_show.py @@ -20,7 +20,7 @@ class TDTestCase: self.setsql = TDSetSql() # user info self.username = 'test' - self.password = 'test' + self.password = 'test123@#$' # db info self.dbname = "user_privilege_show" self.stbname = 'stb' diff --git a/tests/system-test/0-others/view/non_marterial_view/test_view.py b/tests/system-test/0-others/view/non_marterial_view/test_view.py index 3b6f774788..7e062143d2 100644 --- a/tests/system-test/0-others/view/non_marterial_view/test_view.py +++ b/tests/system-test/0-others/view/non_marterial_view/test_view.py @@ -231,7 +231,7 @@ class TDTestCase: """ self.prepare_data() username = "view_test" - password = "test" + password = "test123@#$" self.create_user(username, password) # grant all db permission to user tdSql.execute("grant all on view_db.* to view_test;") @@ -271,7 +271,7 @@ class TDTestCase: """This test case is used to verify the view permission with db write and view all """ username = "view_test" - password = "test" + password = "test123@#$" self.create_user(username, password) conn = taos.connect(user=username, password=password) self.prepare_data(conn) @@ -302,7 +302,7 @@ class TDTestCase: """This test case is used to verify the view permission with db write and view read """ username = "view_test" - password = "test" + password = "test123@#$" self.create_user(username, password) conn = taos.connect(user=username, password=password) self.prepare_data() @@ -338,7 +338,7 @@ class TDTestCase: """This test case is used to verify the view permission with db write and view alter """ username = "view_test" - password = "test" + password = "test123@#$" self.create_user(username, password) conn = taos.connect(user=username, password=password) self.prepare_data() @@ -362,7 +362,7 @@ class TDTestCase: """This test case is used to verify the view permission with db read and view all """ username = "view_test" - password = "test" + password = "test123@#$" self.create_user(username, password) conn = taos.connect(user=username, password=password) self.prepare_data() @@ -388,7 +388,7 @@ class TDTestCase: """This test case is used to verify the view permission with db read and view alter """ username = "view_test" - password = "test" + password = "test123@#$" self.create_user(username, password) conn = taos.connect(user=username, password=password) self.prepare_data() @@ -413,7 +413,7 @@ class TDTestCase: """This test case is used to verify the view permission with db read and view read """ username = "view_test" - password = "test" + password = "test123@#$" self.create_user(username, password) conn = taos.connect(user=username, password=password) self.prepare_data() diff --git a/tests/system-test/0-others/walFileIdex.py b/tests/system-test/0-others/walFileIdex.py index f8309519cd..8ef0c7c181 100644 --- a/tests/system-test/0-others/walFileIdex.py +++ b/tests/system-test/0-others/walFileIdex.py @@ -40,7 +40,7 @@ class TDTestCase: def preData(self): # database\stb\tb\chiild-tb\rows\topics - tdSql.execute("create user testpy pass 'testpy'") + tdSql.execute("create user testpy pass 'test123@#$'") tdSql.execute("drop database if exists db0;") tdSql.execute("create database db0 WAL_RETENTION_PERIOD -1 WAL_RETENTION_SIZE -1 ;") tdSql.execute("use db0;") diff --git a/tests/system-test/1-insert/boundary.py b/tests/system-test/1-insert/boundary.py index 4476236ca6..25782fd0c3 100644 --- a/tests/system-test/1-insert/boundary.py +++ b/tests/system-test/1-insert/boundary.py @@ -33,7 +33,7 @@ class TDTestCase: self.colname_length_boundary = self.boundary.COL_KEY_MAX_LENGTH self.tagname_length_boundary = self.boundary.TAG_KEY_MAX_LENGTH self.username_length_boundary = 23 - self.password_length_boundary = 31 + self.password_length_boundary = 14 def dbname_length_check(self): dbname_length = randint(1,self.dbname_length_boundary-1) for dbname in [tdCom.get_long_name(self.dbname_length_boundary),tdCom.get_long_name(dbname_length)]: @@ -120,27 +120,31 @@ class TDTestCase: def username_length_check(self): username_length = randint(1,self.username_length_boundary-1) for username in [tdCom.get_long_name(username_length),tdCom.get_long_name(self.username_length_boundary)]: - tdSql.execute(f'create user {username} pass "123"') + tdSql.execute(f'create user {username} pass "test123@#$"') tdSql.query('show users') for user in tdSql.queryResult: if user[0].lower() != 'root': tdSql.checkEqual(user[0],username) tdSql.execute(f'drop user {username}') username = tdCom.get_long_name(self.username_length_boundary+1) - tdSql.error(f'create user {username} pass "123"') + tdSql.error(f'create user {username} pass "test123@#$"') if "Name or password too long" in tdSql.error_info: tdLog.info("error info is true!") else: tdLog.exit("error info is not true") def password_length_check(self): - password_length = randint(1,self.password_length_boundary-1) + password_length = randint(8,self.password_length_boundary-1) + index = 0 for password in [tdCom.get_long_name(password_length),tdCom.get_long_name(self.password_length_boundary)]: - username = tdCom.get_long_name(3) - tdSql.execute(f'create user {username} pass "{password}"') + index += 1 + username = tdCom.get_long_name(12) + str(index) + tdSql.execute(f'create user {username} pass "{password}@1"') + index += 1 + username = tdCom.get_long_name(12) + str(index) password = tdCom.get_long_name(self.password_length_boundary+1) - tdSql.error(f'create user {username} pass "{password}"') - if "Name or password too long" in tdSql.error_info: + tdSql.error(f'create user {username} pass "{password}@1"') + if "Invalid password format" in tdSql.error_info: tdLog.info("error info is true!") else: tdLog.exit("error info is not true") diff --git a/tests/system-test/2-query/columnLenUpdated.py b/tests/system-test/2-query/columnLenUpdated.py index 4c92236fca..7470bd907c 100644 --- a/tests/system-test/2-query/columnLenUpdated.py +++ b/tests/system-test/2-query/columnLenUpdated.py @@ -104,7 +104,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() # time.sleep(2) - tdSql.query("create user testpy pass 'testpy'") + tdSql.query("create user testpy pass 'test123@#$'") buildPath = self.getBuildPath() if (buildPath == ""): @@ -117,7 +117,7 @@ class TDTestCase: checkNetworkStatus = ['0: unavailable', '1: network ok', '2: service ok', '3: service degraded', '4: exiting'] netrole = ['client', 'server'] - keyDict = {'h':'', 'P':'6030', 'p':'testpy', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ + keyDict = {'h':'', 'P':'6030', 'p':'test123@#$', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ 'k':'', 't':'', 'n':'', 'l':'1024', 'N':'100', 'V':'', 'd':'db', 'w':'30', '-help':'', '-usage':'', '?':''} keyDict['h'] = self.hostname diff --git a/tests/system-test/2-query/countAlwaysReturnValue.py b/tests/system-test/2-query/countAlwaysReturnValue.py index a6a064ddfd..a9b71c793f 100644 --- a/tests/system-test/2-query/countAlwaysReturnValue.py +++ b/tests/system-test/2-query/countAlwaysReturnValue.py @@ -168,7 +168,7 @@ class TDTestCase: tdLog.printNoPrefix("==========step1:prepare data ==============") self.prepare_data() - tdSql.execute('alter local "countAlwaysReturnValue" "0"') + tdSql.execute('alter local "countAlwaysReturnValue" "0"') tdLog.printNoPrefix("==========step2:test results ==============") diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py index ee6b517061..0c5c9773c8 100644 --- a/tests/system-test/2-query/db.py +++ b/tests/system-test/2-query/db.py @@ -47,7 +47,7 @@ class TDTestCase: def case2(self): tdSql.query("show variables") - tdSql.checkRows(9) + tdSql.checkRows(94) for i in range(self.replicaVar): tdSql.query("show dnode %d variables like 'debugFlag'" % (i + 1)) diff --git a/tests/system-test/2-query/select_null.py b/tests/system-test/2-query/select_null.py index bd92e4cf5c..4a6fb3a583 100755 --- a/tests/system-test/2-query/select_null.py +++ b/tests/system-test/2-query/select_null.py @@ -500,7 +500,7 @@ class TDTestCase: tdSql.execute('create stable sel_null.join_stable(`时间戳` timestamp, c1 int) tags(`标签1` int)', queryTimes=1) tdSql.query('select a.值 from sel_null.stable1 a join sel_null.join_stable b on a.ts = 时间戳;', queryTimes=1) tdSql.query('select a.值 from sel_null.stable1 a join sel_null.join_stable b on a.ts = b.时间戳;', queryTimes=1) - tdSql.execute('create user user1 pass "asd"', queryTimes=1) + tdSql.execute('create user user1 pass "asdxtz@#12"', queryTimes=1) tdSql.execute('grant write on sel_null.stable1 with 标签1 = 1 to user1',queryTimes=1) tdSql.execute('select count(*) from sel_null.stable1 state_window(值)', queryTimes=1) diff --git a/tests/system-test/2-query/union.py b/tests/system-test/2-query/union.py index 24ccac5d90..a6cccb8aaa 100644 --- a/tests/system-test/2-query/union.py +++ b/tests/system-test/2-query/union.py @@ -434,7 +434,7 @@ class TDTestCase: tdSql.checkRows(2) sql = "select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, stable_name `TABLE_NAME`, 'TABLE' `TABLE_TYPE`, table_comment `REMARKS` from information_schema.ins_stables union all select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, table_name `TABLE_NAME`, case when `type`='SYSTEM_TABLE' then 'TABLE' when `type`='NORMAL_TABLE' then 'TABLE' when `type`='CHILD_TABLE' then 'TABLE' else 'UNKNOWN' end `TABLE_TYPE`, table_comment `REMARKS` from information_schema.ins_tables union all select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, view_name `TABLE_NAME`, 'VIEW' `TABLE_TYPE`, NULL `REMARKS` from information_schema.ins_views" tdSql.query(sql, queryTimes=1) - tdSql.checkRows(48) + tdSql.checkRows(49) def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py b/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py index 2941a643fd..e3e5d25e5d 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py @@ -111,7 +111,7 @@ class TDTestCase: "batchNum": 5000 } username="user1" - passwd="123" + passwd="test123@#$" dnodeNumbers=int(dnodeNumbers) mnodeNums=int(mnodeNums) diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py index 1d2644c65f..ca1bc44fbb 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py @@ -163,15 +163,15 @@ class TDTestCase: threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) for i in range(5): - clusterComCreate.createUser(newTdSql,f"user{i}",f"pass{i}") - userTdSql=tdCom.newTdSql(user=f"user{i}",password=f"pass{i}") - clusterComCreate.alterUser(userTdSql,f"user{i}",f"pass{i+1}") + clusterComCreate.createUser(newTdSql,f"user{i}",f"passwd@{i}") + userTdSql=tdCom.newTdSql(user=f"user{i}",password=f"passwd@{i}") + clusterComCreate.alterUser(userTdSql,f"user{i}",f"passwd@{i+1}") clusterComCreate.deleteUser(newTdSql,f"user{i}") for j in range(5): i=100 - clusterComCreate.createUser(newTdSql,f"user{i}",f"pass{i}") - userTdSql=tdCom.newTdSql(user=f"user{i}",password=f"pass{i}") - clusterComCreate.alterUser(userTdSql,f"user{i}",f"pass{i+1}") + clusterComCreate.createUser(newTdSql,f"user{i}",f"passwd@{i}") + userTdSql=tdCom.newTdSql(user=f"user{i}",password=f"passwd@{i}") + clusterComCreate.alterUser(userTdSql,f"user{i}",f"passwd@{i+1}") clusterComCreate.deleteUser(newTdSql,f"user{i}") for tr in threads: diff --git a/tests/system-test/7-tmq/tmqMaxTopic.py b/tests/system-test/7-tmq/tmqMaxTopic.py index 4f25c39627..d2f7cab0a7 100644 --- a/tests/system-test/7-tmq/tmqMaxTopic.py +++ b/tests/system-test/7-tmq/tmqMaxTopic.py @@ -26,22 +26,6 @@ class TDTestCase: tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), True) - def modifyMaxTopics(self, tmqMaxTopicNum): - # single dnode - cfgDir = tdDnodes.dnodes[0].cfgDir - - # cluster dnodes - # tdDnodes[1].dataDir - # tdDnodes[1].logDir - # tdDnodes[1].cfgDir - - cfgFile = f"%s/taos.cfg"%(cfgDir) - shellCmd = 'echo tmqMaxTopicNum %d >> %s'%(tmqMaxTopicNum, cfgFile) - tdLog.info(" shell cmd: %s"%(shellCmd)) - os.system(shellCmd) - tdDnodes.stoptaosd(1) - tdDnodes.starttaosd(1) - time.sleep(5) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") @@ -169,8 +153,10 @@ class TDTestCase: # tdDnodes.starttaosd(1) # time.sleep(5) + alterSql = "alter all dnodes 'tmqMaxTopicNum' '22'" + tdLog.info("alter all dnodes 'tmqMaxTopicNum' '22'") + tdSql.execute(alterSql) newTmqMaxTopicNum = 22 - self.modifyMaxTopics(newTmqMaxTopicNum) sqlString = "create topic %s%s as %s" %(topicNamePrefix, 'x', queryString) tdLog.info("create topic sql: %s"%sqlString) @@ -190,8 +176,10 @@ class TDTestCase: if topicNum != newTmqMaxTopicNum: tdLog.exit("show topics %d not equal expect num: %d"%(topicNum, newTmqMaxTopicNum)) - newTmqMaxTopicNum = 18 - self.modifyMaxTopics(newTmqMaxTopicNum) + alterSql = "alter all dnodes 'tmqMaxTopicNum' '18'" + tdLog.info("alter all dnodes 'tmqMaxTopicNum' '18'") + tdSql.execute(alterSql) + i = 0 sqlString = "drop topic %s%d" %(topicNamePrefix, i) diff --git a/tests/system-test/99-TDcase/TS-5130.py b/tests/system-test/99-TDcase/TS-5130.py index 504500fa0e..994053ac13 100644 --- a/tests/system-test/99-TDcase/TS-5130.py +++ b/tests/system-test/99-TDcase/TS-5130.py @@ -13,9 +13,9 @@ class TDTestCase: self.conn = conn tdSql.init(conn.cursor(), False) self.passwd = {'root':'taosdata', - 'test':'test'} + 'test':'test123@#$'} def prepare_user(self): - tdSql.execute(f"create user test pass 'test' sysinfo 1") + tdSql.execute(f"create user test pass 'test123@#$' sysinfo 1") def test_connect_user(self, uname): try: diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index bf04352232..d922a6454e 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1436,7 +1436,7 @@ int sml_td22900_Test() { int sml_td24070_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "CREATE user test_db pass 'test'"); + TAOS_RES *pRes = taos_query(taos, "CREATE user test_db pass 'test@123'"); ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); @@ -1460,7 +1460,7 @@ int sml_td24070_Test() { // test db privilege - taos = taos_connect("localhost", "test_db", "test", NULL, 0); + taos = taos_connect("localhost", "test_db", "test@123", NULL, 0); const char* sql[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299372000"}; pRes = taos_query(taos, "use td24070_read"); @@ -1491,11 +1491,11 @@ int sml_td24070_Test() { // test stable privilege taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - pRes = taos_query(taos, "CREATE user test_stb_read pass 'test'"); + pRes = taos_query(taos, "CREATE user test_stb_read pass 'test@123'"); ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); - pRes = taos_query(taos, "CREATE user test_stb_write pass 'test'"); + pRes = taos_query(taos, "CREATE user test_stb_write pass 'test@123'"); ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); @@ -1508,7 +1508,7 @@ int sml_td24070_Test() { taos_free_result(pRes); taos_close(taos); - taos = taos_connect("localhost", "test_stb_read", "test", "td24070_write", 0); + taos = taos_connect("localhost", "test_stb_read", "test@123", "td24070_write", 0); const char* sql1[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299373000"}; pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, @@ -1520,7 +1520,7 @@ int sml_td24070_Test() { taos_free_result(pRes); taos_close(taos); - taos = taos_connect("localhost", "test_stb_write", "test", "td24070_write", 0); + taos = taos_connect("localhost", "test_stb_write", "test@123", "td24070_write", 0); const char* sql2[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299373000"}; pRes = taos_schemaless_insert(taos, (char **)sql2, sizeof(sql2) / sizeof(sql2[0]), TSDB_SML_LINE_PROTOCOL, @@ -1536,11 +1536,11 @@ int sml_td24070_Test() { // test table privilege taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - pRes = taos_query(taos, "CREATE user test_tb_read pass 'test'"); + pRes = taos_query(taos, "CREATE user test_tb_read pass 'test@123'"); ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); - pRes = taos_query(taos, "CREATE user test_tb_write pass 'test'"); + pRes = taos_query(taos, "CREATE user test_tb_write pass 'test@123'"); ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); @@ -1553,7 +1553,7 @@ int sml_td24070_Test() { taos_free_result(pRes); taos_close(taos); - taos = taos_connect("localhost", "test_tb_read", "test", "td24070_write", 0); + taos = taos_connect("localhost", "test_tb_read", "test@123", "td24070_write", 0); const char* sql3[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299374000"}; @@ -1566,7 +1566,7 @@ int sml_td24070_Test() { taos_free_result(pRes); taos_close(taos); - taos = taos_connect("localhost", "test_tb_write", "test", "td24070_write", 0); + taos = taos_connect("localhost", "test_tb_write", "test@123", "td24070_write", 0); const char* sql4[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299374000"}; pRes = taos_schemaless_insert(taos, (char **)sql4, sizeof(sql4) / sizeof(sql4[0]), TSDB_SML_LINE_PROTOCOL,