diff --git a/.gitignore b/.gitignore index 8f461f2b02..07003bda4c 100644 --- a/.gitignore +++ b/.gitignore @@ -99,6 +99,7 @@ tests/examples/JDBC/JDBCDemo/.classpath tests/examples/JDBC/JDBCDemo/.project tests/examples/JDBC/JDBCDemo/.settings/ source/libs/parser/inc/sql.* +source/os/src/timezone/ tests/script/tmqResult.txt tests/system-test/case_to_run.txt tests/develop-test/case_to_run.txt @@ -162,3 +163,12 @@ geos_c.h source/libs/parser/src/sql.c include/common/ttokenauto.h !packaging/smokeTest/pytest_require.txt +tdengine-test-dir/ +localtime.c +private.h +strftime.c +tzdir.h +tzfile.h +coverage.info +taos +taosd diff --git a/cmake/tz_CMakeLists.txt.in b/cmake/tz_CMakeLists.txt.in new file mode 100644 index 0000000000..a2e3636d9a --- /dev/null +++ b/cmake/tz_CMakeLists.txt.in @@ -0,0 +1,15 @@ +# timezone +ExternalProject_Add(tz + GIT_REPOSITORY https://github.com/eggert/tz.git + GIT_TAG main + SOURCE_DIR "${TD_CONTRIB_DIR}/tz" + BINARY_DIR "" + CONFIGURE_COMMAND "" + #BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + GIT_SHALLOW true + GIT_PROGRESS true + BUILD_COMMAND "" +) + diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 7741ae4e14..78eded3928 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -106,6 +106,10 @@ cat("${TD_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) # cJson cat("${TD_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +if(NOT ${TD_WINDOWS}) + cat("${TD_SUPPORT_DIR}/tz_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif(NOT ${TD_WINDOWS}) + # xz # cat("${TD_SUPPORT_DIR}/xz_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -651,6 +655,35 @@ if(${TD_LINUX} AND ${BUILD_WITH_S3}) add_subdirectory(azure-cmake EXCLUDE_FROM_ALL) endif() +IF(TD_LINUX) + SET(TZ_OUTPUT_PATH /usr/share/zoneinfo) +ELSEIF(TD_DARWIN) + SET(TZ_OUTPUT_PATH /var/db/timezone/zoneinfo) +ENDIF() + + +if(NOT ${TD_WINDOWS}) + MESSAGE(STATUS "timezone file path: " ${TZ_OUTPUT_PATH}) + + execute_process( + COMMAND make TZDIR=${TZ_OUTPUT_PATH}/ clean tzdir.h + WORKING_DIRECTORY "${TD_CONTRIB_DIR}/tz" + ) + + set(TZ_SRC_DIR "${TD_SOURCE_DIR}/source/os/src/timezone") + file(REMOVE_RECURSE ${TZ_SRC_DIR}) + file(MAKE_DIRECTORY ${TZ_SRC_DIR}) + file(COPY ${TD_CONTRIB_DIR}/tz/private.h ${TD_CONTRIB_DIR}/tz/tzdir.h ${TD_CONTRIB_DIR}/tz/tzfile.h + ${TD_CONTRIB_DIR}/tz/localtime.c ${TD_CONTRIB_DIR}/tz/strftime.c + DESTINATION ${TZ_SRC_DIR}) +endif(NOT ${TD_WINDOWS}) + +#if(NOT ${TD_WINDOWS}) +# execute_process( +# COMMAND make CFLAGS+=-fPIC CFLAGS+=-g TZDIR=${TZ_OUTPUT_PATH} clean libtz.a +# WORKING_DIRECTORY "${TD_CONTRIB_DIR}/tz" +# ) +#endif(NOT ${TD_WINDOWS}) # ================================================================================================ # Build test # ================================================================================================ diff --git a/docs/en/06-advanced/05-data-in/07-mqtt.md b/docs/en/06-advanced/05-data-in/07-mqtt.md index 3f6eed5834..73ef3b534c 100644 --- a/docs/en/06-advanced/05-data-in/07-mqtt.md +++ b/docs/en/06-advanced/05-data-in/07-mqtt.md @@ -112,14 +112,14 @@ Fill in the example data from the MQTT message body in **Message Body**. JSON data supports JSONObject or JSONArray, and the json parser can parse the following data: -``` json +```json {"id": 1, "message": "hello-word"} {"id": 2, "message": "hello-word"} ``` or -``` json +```json [{"id": 1, "message": "hello-word"},{"id": 2, "message": "hello-word"}] ``` diff --git a/docs/en/06-advanced/05-data-in/08-kafka.md b/docs/en/06-advanced/05-data-in/08-kafka.md index bdd652a5e7..b3bdca4cf9 100644 --- a/docs/en/06-advanced/05-data-in/08-kafka.md +++ b/docs/en/06-advanced/05-data-in/08-kafka.md @@ -109,7 +109,7 @@ In addition, the [Kerberos](https://web.mit.edu/kerberos/) authentication servic After configuration, you can use the [kcat](https://github.com/edenhill/kcat) tool to verify Kafka topic consumption: -```bash +```shell kcat \ -b \ -G kcat \ @@ -171,14 +171,14 @@ Enter sample data from the Kafka message body in **Message Body**. JSON data supports JSONObject or JSONArray, and the following data can be parsed using a JSON parser: -``` json +```json {"id": 1, "message": "hello-word"} {"id": 2, "message": "hello-word"} ``` or -``` json +```json [{"id": 1, "message": "hello-word"},{"id": 2, "message": "hello-word"}] ``` diff --git a/docs/en/06-advanced/05-data-in/index.md b/docs/en/06-advanced/05-data-in/index.md index 59a3c6da7b..5221aa2002 100644 --- a/docs/en/06-advanced/05-data-in/index.md +++ b/docs/en/06-advanced/05-data-in/index.md @@ -83,7 +83,7 @@ Parsing is the process of parsing unstructured strings into structured data. The JSON parsing supports JSONObject or JSONArray. The following JSON sample data can automatically parse fields: `groupid`, `voltage`, `current`, `ts`, `inuse`, `location`. -``` json +```json {"groupid": 170001, "voltage": "221V", "current": 12.3, "ts": "2023-12-18T22:12:00", "inuse": true, "location": "beijing.chaoyang.datun"} {"groupid": 170001, "voltage": "220V", "current": 12.2, "ts": "2023-12-18T22:12:02", "inuse": true, "location": "beijing.chaoyang.datun"} {"groupid": 170001, "voltage": "216V", "current": 12.5, "ts": "2023-12-18T22:12:04", "inuse": false, "location": "beijing.chaoyang.datun"} @@ -91,7 +91,7 @@ JSON parsing supports JSONObject or JSONArray. The following JSON sample data ca Or -``` json +```json [{"groupid": 170001, "voltage": "221V", "current": 12.3, "ts": "2023-12-18T22:12:00", "inuse": true, "location": "beijing.chaoyang.datun"}, {"groupid": 170001, "voltage": "220V", "current": 12.2, "ts": "2023-12-18T22:12:02", "inuse": true, "location": "beijing.chaoyang.datun"}, {"groupid": 170001, "voltage": "216V", "current": 12.5, "ts": "2023-12-18T22:12:04", "inuse": false, "location": "beijing.chaoyang.datun"}] @@ -101,7 +101,7 @@ Subsequent examples will only explain using JSONObject. The following nested JSON data can automatically parse fields `groupid`, `data_voltage`, `data_current`, `ts`, `inuse`, `location_0_province`, `location_0_city`, `location_0_datun`, and you can also choose which fields to parse and set aliases for the parsed fields. -``` json +```json {"groupid": 170001, "data": { "voltage": "221V", "current": 12.3 }, "ts": "2023-12-18T22:12:00", "inuse": true, "location": [{"province": "beijing", "city":"chaoyang", "street": "datun"}]} ``` @@ -114,7 +114,7 @@ The following nested JSON data can automatically parse fields `groupid`, `data_v You can use **named capture groups** in regular expressions to extract multiple fields from any string (text) field. As shown in the figure, extract fields such as access IP, timestamp, and accessed URL from nginx logs. -``` re +```regex (?\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b)\s-\s-\s\[(?\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}\s\+\d{4})\]\s"(?[A-Z]+)\s(?[^\s"]+).*(?\d{3})\s(?\d+) ``` @@ -133,7 +133,7 @@ Custom rhai syntax scripts for parsing input data (refer to `https://rhai.rs/boo For example, for data reporting three-phase voltage values, which are entered into three subtables respectively, such data needs to be parsed -``` json +```json { "ts": "2024-06-27 18:00:00", "voltage": "220.1,220.3,221.1", @@ -164,7 +164,7 @@ The final parsing result is shown below: The parsed data may still not meet the data requirements of the target table. For example, the original data collected by a smart meter is as follows (in json format): -``` json +```json {"groupid": 170001, "voltage": "221V", "current": 12.3, "ts": "2023-12-18T22:12:00", "inuse": true, "location": "beijing.chaoyang.datun"} {"groupid": 170001, "voltage": "220V", "current": 12.2, "ts": "2023-12-18T22:12:02", "inuse": true, "location": "beijing.chaoyang.datun"} {"groupid": 170001, "voltage": "216V", "current": 12.5, "ts": "2023-12-18T22:12:04", "inuse": false, "location": "beijing.chaoyang.datun"} diff --git a/docs/en/07-develop/01-connect.md b/docs/en/07-develop/01-connect.md index 97e96f98a4..66bd9b6b16 100644 --- a/docs/en/07-develop/01-connect.md +++ b/docs/en/07-develop/01-connect.md @@ -164,9 +164,6 @@ If you are using Maven to manage your project, simply add the following dependen pip3 install taospy[ws] ``` - - - - **Installation Verification** @@ -199,8 +196,8 @@ import taosws + - Edit `go.mod` to add the `driver-go` dependency. diff --git a/docs/en/07-develop/02-sql.md b/docs/en/07-develop/02-sql.md index 57376e615d..d32ff340f3 100644 --- a/docs/en/07-develop/02-sql.md +++ b/docs/en/07-develop/02-sql.md @@ -83,14 +83,14 @@ Next, create a supertable (STABLE) named `meters`, whose table structure include Create Database -```bash +```shell curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql' \ --data 'CREATE DATABASE IF NOT EXISTS power' ``` Create Table, specify the database as `power` in the URL -```bash +```shell curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql/power' \ --data 'CREATE STABLE IF NOT EXISTS meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))' ``` @@ -167,7 +167,7 @@ NOW is an internal system function, defaulting to the current time of the client Write data -```bash +```shell curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql' \ --data 'INSERT INTO power.d1001 USING power.meters TAGS(2,'\''California.SanFrancisco'\'') VALUES (NOW + 1a, 10.30000, 219, 0.31000) (NOW + 2a, 12.60000, 218, 0.33000) (NOW + 3a, 12.30000, 221, 0.31000) power.d1002 USING power.meters TAGS(3, '\''California.SanFrancisco'\'') VALUES (NOW + 1a, 10.30000, 218, 0.25000)' ``` @@ -247,7 +247,7 @@ Rust connector also supports using **serde** for deserializing to get structured Query Data -```bash +```shell curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql' \ --data 'SELECT ts, current, location FROM power.meters limit 100' ``` @@ -329,7 +329,7 @@ Below are code examples of setting reqId to execute SQL in various language conn Query data, specify reqId as 3 -```bash +```shell curl --location -uroot:taosdata 'http://127.0.0.1:6041/rest/sql?req_id=3' \ --data 'SELECT ts, current, location FROM power.meters limit 1' ``` diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index 795556c225..0e91dd09db 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -273,19 +273,19 @@ To better operate the above data structures, some convenience functions are prov Create table: -```bash +```shell create table battery(ts timestamp, vol1 float, vol2 float, vol3 float, deviceId varchar(16)); ``` Create custom function: -```bash +```shell create aggregate function max_vol as '/root/udf/libmaxvol.so' outputtype binary(64) bufsize 10240 language 'C'; ``` Use custom function: -```bash +```shell select max_vol(vol1, vol2, vol3, deviceid) from battery; ``` @@ -334,7 +334,7 @@ When developing UDFs in Python, you need to implement the specified interface fu The interface for scalar functions is as follows. -```Python +```python def process(input: datablock) -> tuple[output_type]: ``` @@ -347,7 +347,7 @@ The main parameters are as follows: The interface for aggregate functions is as follows. -```Python +```python def start() -> bytes: def reduce(inputs: datablock, buf: bytes) -> bytes def finish(buf: bytes) -> output_type: @@ -365,7 +365,7 @@ Finally, when all row data blocks have been processed, the finish function is ca The interfaces for initialization and destruction are as follows. -```Python +```python def init() def destroy() ``` @@ -381,7 +381,7 @@ Parameter description: The template for developing scalar functions in Python is as follows. -```Python +```python def init(): # initialization def destroy(): @@ -393,7 +393,7 @@ def process(input: datablock) -> tuple[output_type]: The template for developing aggregate functions in Python is as follows. -```Python +```python def init(): #initialization def destroy(): @@ -828,7 +828,7 @@ Through this example, we learned how to define aggregate functions and print cus
pybitand.py -```Python +```python {{#include tests/script/sh/pybitand.py}} ``` diff --git a/docs/en/08-operation/04-maintenance.md b/docs/en/08-operation/04-maintenance.md index 5e7fca1f08..970ee40d18 100644 --- a/docs/en/08-operation/04-maintenance.md +++ b/docs/en/08-operation/04-maintenance.md @@ -15,7 +15,7 @@ TDengine is designed for various writing scenarios, and many of these scenarios ### Syntax -```SQL +```sql COMPACT DATABASE db_name [start with 'XXXX'] [end with 'YYYY']; SHOW COMPACTS [compact_id]; KILL COMPACT compact_id; @@ -41,7 +41,7 @@ KILL COMPACT compact_id; When one or more nodes in a multi-replica cluster restart due to upgrades or other reasons, it may lead to an imbalance in the load among the various dnodes in the cluster. In extreme cases, all vgroup leaders may be located on the same dnode. To solve this problem, you can use the following commands, which were first released in version 3.0.4.0. It is recommended to use the latest version as much as possible. -```SQL +```sql balance vgroup leader; # Rebalance all vgroup leaders balance vgroup leader on ; # Rebalance a vgroup leader balance vgroup leader database ; # Rebalance all vgroup leaders within a database diff --git a/docs/en/08-operation/12-multi.md b/docs/en/08-operation/12-multi.md index f2f464be1c..1d0b8ad6cb 100644 --- a/docs/en/08-operation/12-multi.md +++ b/docs/en/08-operation/12-multi.md @@ -121,7 +121,7 @@ The cost of using object storage services is related to the amount of data store When the TSDB time-series data exceeds the time specified by the `s3_keeplocal` parameter, the related data files will be split into multiple file blocks, each with a default size of 512 MB (`s3_chunkpages * tsdb_pagesize`). Except for the last file block, which is retained on the local file system, the rest of the file blocks are uploaded to the object storage service. -```math +```text Upload Count = Data File Size / (s3_chunkpages * tsdb_pagesize) - 1 ``` @@ -135,7 +135,7 @@ During query operations, if data in object storage needs to be accessed, TSDB do Adjacent multiple data pages are downloaded as a single data block from object storage to reduce the number of downloads. The size of each data page is specified by the `tsdb_pagesize` parameter when creating the database, with a default of 4 KB. -```math +```text Download Count = Number of Data Blocks Needed for Query - Number of Cached Data Blocks ``` @@ -155,7 +155,7 @@ For deployment methods, please refer to the [Flexify](https://azuremarketplace.m In the configuration file /etc/taos/taos.cfg, add parameters for S3 access: -```cfg +```text s3EndPoint http //20.191.157.23,http://20.191.157.24,http://20.191.157.25 s3AccessKey FLIOMMNL0:uhRNdeZMLD4wo,ABCIOMMN:uhRNdeZMD4wog,DEFOMMNL049ba:uhRNdeZMLD4wogXd s3BucketName td-test diff --git a/docs/en/08-operation/14-user.md b/docs/en/08-operation/14-user.md index 95c00bd21f..5aa8b2e211 100644 --- a/docs/en/08-operation/14-user.md +++ b/docs/en/08-operation/14-user.md @@ -12,19 +12,20 @@ TDengine is configured by default with only one root user, who has the highest p Only the root user can perform the operation of creating users, with the syntax as follows. ```sql -create user user_name pass'password' [sysinfo {1|0}] +create user user_name pass'password' [sysinfo {1|0}] [createdb {1|0}] ``` The parameters are explained as follows. - user_name: Up to 23 B long. -- password: Up to 128 B long, valid characters include letters and numbers as well as special characters other than single and double quotes, apostrophes, backslashes, and spaces, and it cannot be empty. +- password: The password must be between 8 and 16 characters long and include at least three types of characters from the following: uppercase letters, lowercase letters, numbers, and special characters. Special characters include `! @ # $ % ^ & * ( ) - _ + = [ ] { } : ; > < ? | ~ , .`. - sysinfo: Whether the user can view system information. 1 means they can view it, 0 means they cannot. System information includes server configuration information, various node information such as dnode, query node (qnode), etc., as well as storage-related information, etc. The default is to view system information. +- createdb: Whether the user can create databases. 1 means they can create databases, 0 means they cannot. The default value is 0. // Supported starting from TDengine Enterprise version 3.3.2.0 -The following SQL can create a user named test with the password 123456 who can view system information. +The following SQL can create a user named test with the password abc123!@# who can view system information. ```sql -create user test pass '123456' sysinfo 1 +create user test pass 'abc123!@#' sysinfo 1 ``` ### Viewing Users @@ -51,6 +52,7 @@ alter_user_clause: { pass 'literal' | enable value | sysinfo value + | createdb value } ``` @@ -59,6 +61,7 @@ The parameters are explained as follows. - pass: Modify the user's password. - enable: Whether to enable the user. 1 means to enable this user, 0 means to disable this user. - sysinfo: Whether the user can view system information. 1 means they can view system information, 0 means they cannot. +- createdb: Whether the user can create databases. 1 means they can create databases, 0 means they cannot. // Supported starting from TDengine Enterprise version 3.3.2.0 The following SQL disables the user test. diff --git a/docs/en/10-third-party/01-collection/09-emq-broker.md b/docs/en/10-third-party/01-collection/09-emq-broker.md index 24cb12997f..4f3eadcca6 100644 --- a/docs/en/10-third-party/01-collection/09-emq-broker.md +++ b/docs/en/10-third-party/01-collection/09-emq-broker.md @@ -140,7 +140,7 @@ Finally, click the "Create" button at the bottom left to save the rule. ## Write a Mock Test Program -```javascript +```js {{#include docs/examples/other/mock.js}} ``` diff --git a/docs/en/10-third-party/01-collection/11-kafka.md b/docs/en/10-third-party/01-collection/11-kafka.md index 2627e33b65..25ba7fd54a 100644 --- a/docs/en/10-third-party/01-collection/11-kafka.md +++ b/docs/en/10-third-party/01-collection/11-kafka.md @@ -95,7 +95,7 @@ curl http://localhost:8083/connectors If all components have started successfully, the following output will be displayed: -```txt +```text [] ``` @@ -181,7 +181,7 @@ If the above command is executed successfully, the following output will be disp Prepare a text file with test data, content as follows: -```txt title="test-data.txt" +```text title="test-data.txt" meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249000000 meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611250000000 meters,location=California.LosAngeles,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249000000 @@ -303,7 +303,7 @@ kafka-console-consumer.sh --bootstrap-server localhost:9092 --from-beginning --t Output: -```txt +```text ...... meters,location="California.SanFrancisco",groupid=2i32 current=10.3f32,voltage=219i32,phase=0.31f32 1538548685000000000 meters,location="California.SanFrancisco",groupid=2i32 current=12.6f32,voltage=218i32,phase=0.33f32 1538548695000000000 diff --git a/docs/en/10-third-party/03-visual/01-grafana.mdx b/docs/en/10-third-party/03-visual/01-grafana.md similarity index 99% rename from docs/en/10-third-party/03-visual/01-grafana.mdx rename to docs/en/10-third-party/03-visual/01-grafana.md index bd2d313def..ec857f7795 100644 --- a/docs/en/10-third-party/03-visual/01-grafana.mdx +++ b/docs/en/10-third-party/03-visual/01-grafana.md @@ -60,7 +60,7 @@ Click `Save & Test` to test, if successful, it will prompt: `TDengine Data sourc For users using Grafana version 7.x or configuring with [Grafana Provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/), you can use the installation script on the Grafana server to automatically install the plugin and add the data source Provisioning configuration file. -```sh +```shell bash -c "$(curl -fsSL \ https://raw.githubusercontent.com/taosdata/grafanaplugin/master/install.sh)" -- \ -a http://localhost:6041 \ @@ -77,7 +77,7 @@ Save the script and execute `./install.sh --help` to view detailed help document Use the [`grafana-cli` command line tool](https://grafana.com/docs/grafana/latest/administration/cli/) to install the plugin [installation](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation). -```bash +```shell grafana-cli plugins install tdengine-datasource # with sudo sudo -u grafana grafana-cli plugins install tdengine-datasource @@ -85,7 +85,7 @@ sudo -u grafana grafana-cli plugins install tdengine-datasource Alternatively, download the .zip file from [GitHub](https://github.com/taosdata/grafanaplugin/releases/tag/latest) or [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) to your local machine and unzip it into the Grafana plugins directory. Example command line download is as follows: -```bash +```shell GF_VERSION=3.5.1 # from GitHub wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/tdengine-datasource-$GF_VERSION.zip @@ -95,13 +95,13 @@ wget -O tdengine-datasource-$GF_VERSION.zip https://grafana.com/api/plugins/tden For CentOS 7.2 operating system, unzip the plugin package into the /var/lib/grafana/plugins directory and restart Grafana. -```bash +```shell sudo unzip tdengine-datasource-$GF_VERSION.zip -d /var/lib/grafana/plugins/ ``` If Grafana is running in a Docker environment, you can use the following environment variable to set up automatic installation of the TDengine data source plugin: -```bash +```shell GF_INSTALL_PLUGINS=tdengine-datasource ``` @@ -120,7 +120,7 @@ Click `Save & Test` to test, if successful, it will prompt: `TDengine Data sourc Refer to [Grafana containerized installation instructions](https://grafana.com/docs/grafana/next/setup-grafana/installation/docker/#install-plugins-in-the-docker-container). Use the following command to start a container and automatically install the TDengine plugin: -```bash +```shell docker run -d \ -p 3000:3000 \ --name=grafana \ diff --git a/docs/en/14-reference/01-components/04-taosx.md b/docs/en/14-reference/01-components/04-taosx.md index b53d9f6777..8ce16f471d 100644 --- a/docs/en/14-reference/01-components/04-taosx.md +++ b/docs/en/14-reference/01-components/04-taosx.md @@ -31,7 +31,7 @@ The following parameter descriptions and examples use `` as a placehold In command line mode, taosX uses DSN to represent a data source (source or destination), a typical DSN is as follows: -```bash +```shell # url-like [+]://[[:@]:][/][?=[&=]] |------|------------|---|-----------|-----------|------|------|----------|-----------------------| @@ -390,7 +390,7 @@ You can view the log files or use the `journalctl` command to view the logs of ` The command to view logs under Linux using `journalctl` is as follows: -```bash +```shell journalctl -u taosx [-f] ``` @@ -572,7 +572,7 @@ uint32_t len: The binary length of this string (excluding `\0`). **Return Value**: -``` c +```c struct parser_resp_t { int e; // 0 if success. void* p; // Success if contains. @@ -589,7 +589,7 @@ When creation is successful, e = 0, p is the parser object. Parse the input payload and return the result in JSON format [u8]. The returned JSON will be fully decoded using the default JSON parser (expanding the root array and all objects). -``` c +```c const char* parser_mutate( void* parser, const uint8_t* in_ptr, uint32_t in_len, diff --git a/docs/en/14-reference/01-components/05-taosx-agent.md b/docs/en/14-reference/01-components/05-taosx-agent.md index 9353d45de3..3e8b4f4d63 100644 --- a/docs/en/14-reference/01-components/05-taosx-agent.md +++ b/docs/en/14-reference/01-components/05-taosx-agent.md @@ -26,7 +26,7 @@ The default configuration file for `Agent` is located at `/etc/taos/agent.toml`, As shown below: -```TOML +```toml # taosX service endpoint # #endpoint = "http://localhost:6055" @@ -83,7 +83,7 @@ You don't need to be confused about how to set up the configuration file. Read a On Linux systems, the `Agent` can be started with the Systemd command: -```bash +```shell systemctl start taosx-agent ``` @@ -95,6 +95,6 @@ You can view the log files or use the `journalctl` command to view the logs of t The command to view logs with `journalctl` on Linux is as follows: -```bash +```shell journalctl -u taosx-agent [-f] ``` diff --git a/docs/en/14-reference/01-components/06-taoskeeper.md b/docs/en/14-reference/01-components/06-taoskeeper.md index 570cf7305f..78556d888a 100644 --- a/docs/en/14-reference/01-components/06-taoskeeper.md +++ b/docs/en/14-reference/01-components/06-taoskeeper.md @@ -143,13 +143,13 @@ For details on TDengine monitoring configuration, please refer to: [TDengine Mon After installation, please use the `systemctl` command to start the taoskeeper service process. -```bash +```shell systemctl start taoskeeper ``` Check if the service is working properly: -```bash +```shell systemctl status taoskeeper ``` @@ -261,7 +261,7 @@ Query OK, 14 row(s) in set (0.006542s) You can view the most recent report record of a supertable, such as: -``` shell +```shell taos> select last_row(*) from taosd_dnodes_info; last_row(_ts) | last_row(disk_engine) | last_row(system_net_in) | last_row(vnodes_num) | last_row(system_net_out) | last_row(uptime) | last_row(has_mnode) | last_row(io_read_disk) | last_row(error_log_count) | last_row(io_read) | last_row(cpu_cores) | last_row(has_qnode) | last_row(has_snode) | last_row(disk_total) | last_row(mem_engine) | last_row(info_log_count) | last_row(cpu_engine) | last_row(io_write_disk) | last_row(debug_log_count) | last_row(disk_used) | last_row(mem_total) | last_row(io_write) | last_row(masters) | last_row(cpu_system) | last_row(trace_log_count) | last_row(mem_free) | ====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== diff --git a/docs/en/14-reference/01-components/07-explorer.md b/docs/en/14-reference/01-components/07-explorer.md index 726b07d050..a9a2ebb356 100644 --- a/docs/en/14-reference/01-components/07-explorer.md +++ b/docs/en/14-reference/01-components/07-explorer.md @@ -14,7 +14,7 @@ taosExplorer does not require separate installation. Starting from TDengine vers Before starting taosExplorer, please make sure the content in the configuration file is correct. -```TOML +```toml # This is an automatically generated configuration file for Explorer in [TOML](https://toml.io/) format. # # Here is a full list of available options. @@ -148,7 +148,7 @@ Description: Then start taosExplorer, you can directly execute taos-explorer in the command line or use the systemctl command: -```bash +```shell systemctl start taos-explorer # Linux sc.exe start taos-explorer # Windows ``` diff --git a/docs/en/14-reference/01-components/12-tdinsight.mdx b/docs/en/14-reference/01-components/12-tdinsight.md similarity index 99% rename from docs/en/14-reference/01-components/12-tdinsight.mdx rename to docs/en/14-reference/01-components/12-tdinsight.md index 463ceaaca6..12423c512d 100644 --- a/docs/en/14-reference/01-components/12-tdinsight.mdx +++ b/docs/en/14-reference/01-components/12-tdinsight.md @@ -248,13 +248,13 @@ The new version of the plugin uses the Grafana unified alerting feature, the `-E Assuming you start the TDengine database on the host `tdengine` with HTTP API port `6041`, user `root1`, and password `pass5ord`. Execute the script: -```bash +```shell ./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord ``` If you want to monitor multiple TDengine clusters, you need to set up multiple TDinsight dashboards. Setting up a non-default TDinsight requires some changes: the `-n` `-i` `-t` options need to be changed to non-default names, and if using the built-in SMS alert feature, `-N` and `-L` should also be changed. -```bash +```shell sudo ./TDengine.sh -n TDengine-Env1 -a http://another:6041 -u root -p taosdata -i tdinsight-env1 -t 'TDinsight Env1' ``` diff --git a/docs/en/14-reference/02-tools/08-taos-cli.md b/docs/en/14-reference/02-tools/08-taos-cli.md index 1dd77de186..9309da831d 100644 --- a/docs/en/14-reference/02-tools/08-taos-cli.md +++ b/docs/en/14-reference/02-tools/08-taos-cli.md @@ -10,7 +10,7 @@ The TDengine command line program (hereinafter referred to as TDengine CLI) is t To enter the TDengine CLI, simply execute `taos` in the terminal. -```bash +```shell taos ``` @@ -81,7 +81,7 @@ There are many other parameters: Example: -```bash +```shell taos -h h1.taos.com -s "use db; show tables;" ``` diff --git a/docs/en/14-reference/02-tools/10-taosbenchmark.md b/docs/en/14-reference/02-tools/10-taosbenchmark.md index c41cff49e1..09227f210b 100644 --- a/docs/en/14-reference/02-tools/10-taosbenchmark.md +++ b/docs/en/14-reference/02-tools/10-taosbenchmark.md @@ -28,7 +28,7 @@ taosBenchmark supports comprehensive performance testing for TDengine, and the T Execute the following command to quickly experience taosBenchmark performing a write performance test on TDengine based on the default configuration. -```bash +```shell taosBenchmark ``` @@ -38,7 +38,7 @@ When running without parameters, taosBenchmark by default connects to the TDengi When running taosBenchmark using command line parameters and controlling its behavior, the `-f ` parameter cannot be used. All configuration parameters must be specified through the command line. Below is an example of using command line mode to test the write performance of taosBenchmark. -```bash +```shell taosBenchmark -I stmt -n 200 -t 100 ``` @@ -50,7 +50,7 @@ The taosBenchmark installation package includes examples of configuration files, Use the following command line to run taosBenchmark and control its behavior through a configuration file. -```bash +```shell taosBenchmark -f ``` diff --git a/docs/en/14-reference/03-taos-sql/03-table.md b/docs/en/14-reference/03-taos-sql/03-table.md index 030d88cf63..b7a8bb5b1c 100644 --- a/docs/en/14-reference/03-taos-sql/03-table.md +++ b/docs/en/14-reference/03-taos-sql/03-table.md @@ -170,7 +170,7 @@ ALTER TABLE [db_name.]tb_name alter_table_clause alter_table_clause: { alter_table_options - | SET tag tag_name = new_tag_value,tag_name2=new_tag2_value... + | SET tag tag_name = new_tag_value, tag_name2=new_tag2_value ... } alter_table_options: @@ -194,7 +194,7 @@ alter_table_option: { ### Modify Subtable Tag Value ```sql -ALTER TABLE tb_name SET tag tag_name=new_tag_value; +ALTER TABLE tb_name SET TAG tag_name1=new_tag_value1, tag_name2=new_tag_value2 ...; ``` ### Modify Table Lifespan diff --git a/docs/en/14-reference/03-taos-sql/06-select.md b/docs/en/14-reference/03-taos-sql/06-select.md index 5c71697110..c33fef95fb 100644 --- a/docs/en/14-reference/03-taos-sql/06-select.md +++ b/docs/en/14-reference/03-taos-sql/06-select.md @@ -210,19 +210,19 @@ However, renaming individual columns is not supported for `first(*)`, `last(*)`, Retrieve all subtable names and related tag information from a supertable: -```mysql +```sql SELECT TAGS TBNAME, location FROM meters; ``` It is recommended that users query the subtable tag information of supertables using the INS_TAGS system table under INFORMATION_SCHEMA, for example, to get all subtable names and tag values of the supertable meters: -```mysql +```sql SELECT table_name, tag_name, tag_type, tag_value FROM information_schema.ins_tags WHERE stable_name='meters'; ``` Count the number of subtables under a supertable: -```mysql +```sql SELECT COUNT(*) FROM (SELECT DISTINCT TBNAME FROM meters); ``` @@ -385,7 +385,7 @@ SELECT CURRENT_USER(); ### Syntax -```txt +```text WHERE (column|tbname) match/MATCH/nmatch/NMATCH _regex_ ``` @@ -403,7 +403,7 @@ The length of the regular match string cannot exceed 128 bytes. You can set and ### Syntax -```txt +```text CASE value WHEN compare_value THEN result [WHEN compare_value THEN result ...] [ELSE result] END CASE WHEN condition THEN result [WHEN condition THEN result ...] [ELSE result] END ``` @@ -493,7 +493,7 @@ SELECT ... FROM (SELECT ... FROM ...) ...; ## UNION ALL Clause -```txt title=Syntax +```text title=Syntax SELECT ... UNION ALL SELECT ... [UNION ALL SELECT ...] diff --git a/docs/en/14-reference/03-taos-sql/08-delete-data.mdx b/docs/en/14-reference/03-taos-sql/08-delete-data.md similarity index 100% rename from docs/en/14-reference/03-taos-sql/08-delete-data.mdx rename to docs/en/14-reference/03-taos-sql/08-delete-data.md diff --git a/docs/en/14-reference/03-taos-sql/10-function.md b/docs/en/14-reference/03-taos-sql/10-function.md index fa6b8d333a..4eb7b083fb 100644 --- a/docs/en/14-reference/03-taos-sql/10-function.md +++ b/docs/en/14-reference/03-taos-sql/10-function.md @@ -417,7 +417,7 @@ MOD(expr1, expr2) **Example**: -``` sql +```sql taos> select mod(10,3); mod(10,3) | ============================ @@ -454,7 +454,7 @@ RAND([seed]) **Example**: -``` sql +```sql taos> select rand(); rand() | ============================ diff --git a/docs/en/14-reference/03-taos-sql/22-meta.md b/docs/en/14-reference/03-taos-sql/22-meta.md index ebc357c7f5..3a17353b02 100644 --- a/docs/en/14-reference/03-taos-sql/22-meta.md +++ b/docs/en/14-reference/03-taos-sql/22-meta.md @@ -317,28 +317,29 @@ Configuration parameters for each dnode in the system. Users with SYSINFO attrib Note: Users with SYSINFO property set to 0 cannot view this table. -| # | **Column Name** | **Data Type** | **Description** | -| --- | :-------------: | -------------- | ------------------------------------ | -| 1 | user_name | VARCHAR(24) | Username | -| 2 | privilege | VARCHAR(10) | Permission description | -| 3 | db_name | VARCHAR(65) | Database name | -| 4 | table_name | VARCHAR(193) | Table name | -| 5 | condition | VARCHAR(49152) | Subtable permission filter condition | +| # | **Column Name** | **Data Type** | **Description** | +|:-----|:----------------|:---------------|:-------------------------------------| +| 1 | user_name | VARCHAR(24) | Username | +| 2 | privilege | VARCHAR(10) | Permission description | +| 3 | db_name | VARCHAR(65) | Database name | +| 4 | table_name | VARCHAR(193) | Table name | +| 5 | condition | VARCHAR(49152) | Subtable permission filter condition | + ## INS_DISK_USAGE -| # | **Column Name** | **Data type** | **Description**| -| --- | :----------: | ------------ | ------------------------| -| 1 | db_name | VARCHAR(32) | Database name -| 2 | vgroup_id | INT | vgroup ID -| 3 | wal | BIGINT | WAL file size, in KB -| 4 | data1 | BIGINT | Data file size on primary storage, in KB -| 5 | data2 | BIGINT | Data file size on secondary storage, in KB -| 6 | data3 | BIGINT | Data file size on tertiary storage, in KB -| 7 | cache_rdb | BIGINT | Size of last/last_row files, in KB -| 8 | table_meta | BIGINT | Size of meta files, in KB -| 9 | s3 | BIGINT | Size occupied on S3, in KB -| 10 | raw_data | BIGINT | Estimated size of raw data, in KB +| # | **Column Name** | **Data type** | **Description**| +|:----|:-----------|:-----------|:--------------------| +| 1 | db_name | VARCHAR(32) | Database name | +| 2 | vgroup_id | INT | vgroup ID | +| 3 | wal | BIGINT | WAL file size, in KB | +| 4 | data1 | BIGINT | Data file size on primary storage, in KB | +| 5 | data2 | BIGINT | Data file size on secondary storage, in KB | +| 6 | data3 | BIGINT | Data file size on tertiary storage, in KB | +| 7 | cache_rdb | BIGINT | Size of last/last_row files, in KB | +| 8 | table_meta | BIGINT | Size of meta files, in KB | +| 9 | s3 | BIGINT | Size occupied on S3, in KB | +| 10 | raw_data | BIGINT | Estimated size of raw data, in KB | note: diff --git a/docs/en/14-reference/03-taos-sql/25-user.md b/docs/en/14-reference/03-taos-sql/25-user.md index 6eebf9513e..04313a5da3 100644 --- a/docs/en/14-reference/03-taos-sql/25-user.md +++ b/docs/en/14-reference/03-taos-sql/25-user.md @@ -8,19 +8,21 @@ User and permission management is a feature of TDengine Enterprise Edition. This ## Create User ```sql -CREATE USER user_name PASS 'password' [SYSINFO {1|0}]; +CREATE USER user_name PASS 'password' [SYSINFO {1|0}] [CREATEDB {1|0}]; ``` The username can be up to 23 bytes long. -The password can be up to 31 bytes long. The password can include letters, numbers, and special characters except for single quotes, double quotes, backticks, backslashes, and spaces, and it cannot be an empty string. +The password must be between 8 and 16 characters long and include at least three types of characters from the following: uppercase letters, lowercase letters, numbers, and special characters. Special characters include `! @ # $ % ^ & * ( ) - _ + = [ ] { } : ; > < ? | ~ , .`. `SYSINFO` indicates whether the user can view system information. `1` means they can view, `0` means they have no permission to view. System information includes service configuration, dnode, vnode, storage, etc. The default value is `1`. -In the example below, we create a user with the password `123456` who can view system information. +`CREATEDB` indicates whether the user can create databases. `1` means they can create databases, `0` means they have no permission to create databases. The default value is `0`. // Supported starting from TDengine Enterprise version 3.3.2.0 + +In the example below, we create a user with the password `abc123!@#` who can view system information. ```sql -taos> create user test pass '123456' sysinfo 1; +taos> create user test pass 'abc123!@#' sysinfo 1; Query OK, 0 of 0 rows affected (0.001254s) ``` @@ -76,7 +78,7 @@ alter_user_clause: { - PASS: Change the password, followed by the new password - ENABLE: Enable or disable the user, `1` means enable, `0` means disable - SYSINFO: Allow or prohibit viewing system information, `1` means allow, `0` means prohibit -- CREATEDB: Allow or prohibit creating databases, `1` means allow, `0` means prohibit +- CREATEDB: Allow or prohibit creating databases, `1` means allow, `0` means prohibit. // Supported starting from TDengine Enterprise version 3.3.2.0 The following example disables the user named `test`: diff --git a/docs/en/14-reference/03-taos-sql/32-compress.md b/docs/en/14-reference/03-taos-sql/32-compress.md index efe1959b1a..30b107b632 100644 --- a/docs/en/14-reference/03-taos-sql/32-compress.md +++ b/docs/en/14-reference/03-taos-sql/32-compress.md @@ -28,13 +28,14 @@ In this document, it specifically refers to the internal levels of the second-le - Default compression algorithms list and applicable range for each data type -| Data Type | Available Encoding Algorithms | Default Encoding Algorithm | Available Compression Algorithms|Default Compression Algorithm| Default Compression Level| -| :-----------:|:----------:|:-------:|:-------:|:----------:|:----:| -| tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| -| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium| -|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium| -|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium| -|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium| +| Data Type |Available Encoding Algorithms | Default Encoding Algorithm | Available Compression Algorithms | Default Compression Algorithm | Default Compression Level | +|:------------------------------------:|:-------------------------:|:-----------:|:--------------------:|:----:|:------:| +| int/uint | disabled/simple8b | simple8b | lz4/zlib/zstd/xz | lz4 | medium | +| tinyint/untinyint/smallint/usmallint | disabled/simple8b | simple8b | lz4/zlib/zstd/xz | zlib | medium | +| bigint/ubigint/timestamp | disabled/simple8b/delta-i | delta-i | lz4/zlib/zstd/xz | lz4 | medium | +| float/double | disabled/delta-d | delta-d | lz4/zlib/zstd/xz/tsz | lz4 | medium | +| binary/nchar | disabled | disabled | lz4/zlib/zstd/xz | zstd | medium | +| bool | disabled/bit-packing | bit-packing | lz4/zlib/zstd/xz | zstd | medium | ## SQL Syntax diff --git a/docs/en/14-reference/05-connector/10-cpp.mdx b/docs/en/14-reference/05-connector/10-cpp.md similarity index 98% rename from docs/en/14-reference/05-connector/10-cpp.mdx rename to docs/en/14-reference/05-connector/10-cpp.md index ef3026bd0c..7e643304c3 100644 --- a/docs/en/14-reference/05-connector/10-cpp.mdx +++ b/docs/en/14-reference/05-connector/10-cpp.md @@ -688,6 +688,27 @@ The basic API is used to establish database connections and provide a runtime en - `arg`: [Input] Setting item value. - **Return Value**: `0`: Success, `-1`: Failure. +- `int taos_options_connection(TAOS *taos, TSDB_OPTION_CONNECTION option, const void *arg, ...)` + - **description**:Set each connection option on the client side. Currently, it supports character set setting(`TSDB_OPTION_CONNECTION_CHARSET`), time zone setting(`TSDB_OPTION_CONNECTION_TIMEZONE`), user IP setting(`TSDB_OPTION_CONNECTION_USER_IP`), and user APP setting(`TSDB_OPTION_CONNECTION_USER_APP`). + - **input**: + - `taos`: returned by taos_connect. + - `option`: option name. + - `arg`: option value. + - **return**: + - `0`: success. + - `others`: fail. + - **notice**: + - The character set and time zone default to the current settings of the operating system, and Windows does not support connection level time zone settings. + - When arg is NULL, it means resetting the option. + - This interface is only valid for the current connection and will not affect other connections. + - If the same parameter is called multiple times, the latter shall prevail and can be used as a modification method. + - The option of TSDB_OPTION_CONNECTION_CLEAR is used to reset all connection options. + - After resetting the time zone and character set, using the operating system settings, the user IP and user app will be reset to empty. + - The values of the connection options are all string type, and the maximum value of the user app parameter is 23, which will be truncated if exceeded; Error reported when other parameters are illegal. + - If time zone value can not be used to find a time zone file or can not be interpreted as a direct specification, UTC is used, which is the same as the operating system time zone rules. Please refer to the tzset function description for details. You can view the current time zone of the connection by sql:select timezone(). + - Time zones and character sets only work on the client side and do not affect related behaviors on the server side. + - The time zone file uses the operating system time zone file and can be updated by oneself. If there is an error when setting the time zone, please check if the time zone file or path (mac:/var/db/timezone/zoneinfo, Linux:/var/share/zoneinfo) is correct. + - `char *taos_get_client_info()` - **Interface Description**: Gets client version information. - **Return Value**: Returns client version information. diff --git a/docs/en/14-reference/05-connector/14-java.mdx b/docs/en/14-reference/05-connector/14-java.md similarity index 100% rename from docs/en/14-reference/05-connector/14-java.mdx rename to docs/en/14-reference/05-connector/14-java.md diff --git a/docs/en/14-reference/05-connector/20-go.mdx b/docs/en/14-reference/05-connector/20-go.md similarity index 99% rename from docs/en/14-reference/05-connector/20-go.mdx rename to docs/en/14-reference/05-connector/20-go.md index b6313ae3b1..3342066a3d 100644 --- a/docs/en/14-reference/05-connector/20-go.mdx +++ b/docs/en/14-reference/05-connector/20-go.md @@ -108,7 +108,7 @@ For the source code of the example programs, please refer to: [Example Programs] The Data Source Name has a generic format, similar to [PEAR DB](http://pear.php.net/manual/en/package.database.db.intro-dsn.php), but without the type prefix (brackets indicate optional): -``` text +```text [username[:password]@][protocol[(address)]]/[dbname][?param1=value1&...¶mN=valueN] ``` diff --git a/docs/en/14-reference/05-connector/26-rust.mdx b/docs/en/14-reference/05-connector/26-rust.md similarity index 100% rename from docs/en/14-reference/05-connector/26-rust.mdx rename to docs/en/14-reference/05-connector/26-rust.md diff --git a/docs/en/14-reference/05-connector/30-python.mdx b/docs/en/14-reference/05-connector/30-python.md similarity index 100% rename from docs/en/14-reference/05-connector/30-python.mdx rename to docs/en/14-reference/05-connector/30-python.md diff --git a/docs/en/14-reference/05-connector/35-node.mdx b/docs/en/14-reference/05-connector/35-node.md similarity index 100% rename from docs/en/14-reference/05-connector/35-node.mdx rename to docs/en/14-reference/05-connector/35-node.md diff --git a/docs/en/14-reference/05-connector/40-csharp.mdx b/docs/en/14-reference/05-connector/40-csharp.md similarity index 100% rename from docs/en/14-reference/05-connector/40-csharp.mdx rename to docs/en/14-reference/05-connector/40-csharp.md diff --git a/docs/en/14-reference/05-connector/43-r-lang.mdx b/docs/en/14-reference/05-connector/43-r-lang.md similarity index 100% rename from docs/en/14-reference/05-connector/43-r-lang.mdx rename to docs/en/14-reference/05-connector/43-r-lang.md diff --git a/docs/en/14-reference/05-connector/45-php.mdx b/docs/en/14-reference/05-connector/45-php.md similarity index 100% rename from docs/en/14-reference/05-connector/45-php.mdx rename to docs/en/14-reference/05-connector/45-php.md diff --git a/docs/en/14-reference/05-connector/50-odbc.mdx b/docs/en/14-reference/05-connector/50-odbc.md similarity index 100% rename from docs/en/14-reference/05-connector/50-odbc.mdx rename to docs/en/14-reference/05-connector/50-odbc.md diff --git a/docs/en/14-reference/05-connector/60-rest-api.mdx b/docs/en/14-reference/05-connector/60-rest-api.md similarity index 99% rename from docs/en/14-reference/05-connector/60-rest-api.mdx rename to docs/en/14-reference/05-connector/60-rest-api.md index a63022f547..88e53f7618 100644 --- a/docs/en/14-reference/05-connector/60-rest-api.mdx +++ b/docs/en/14-reference/05-connector/60-rest-api.md @@ -21,7 +21,7 @@ Below is an example using the `curl` tool in an Ubuntu environment (please confi The following example lists all databases, please replace `h1.tdengine.com` and 6041 (default value) with the actual running TDengine service FQDN and port number: -```bash +```shell curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" \ -d "select name, ntables, status from information_schema.ins_databases;" \ h1.tdengine.com:6041/rest/sql @@ -100,13 +100,13 @@ The BODY of the HTTP request contains a complete SQL statement. The data table i Use `curl` to initiate an HTTP Request with custom authentication as follows: -```bash +```shell curl -L -H "Authorization: Basic " -d "" :/rest/sql/[db_name][?tz=timezone[&req_id=req_id][&row_with_meta=true]] ``` Or, -```bash +```shell curl -L -u username:password -d "" :/rest/sql/[db_name][?tz=timezone[&req_id=req_id][&row_with_meta=true]] ``` @@ -279,7 +279,7 @@ Column types use the following strings: Prepare data -```bash +```shell create database demo use demo create table t(ts timestamp,c1 varbinary(20),c2 geometry(100)) @@ -288,7 +288,7 @@ insert into t values(now,'\x7f8290','point(100 100)') Execute query -```bash +```shell curl --location 'http://:/rest/sql' \ --header 'Content-Type: text/plain' \ --header 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' \ @@ -428,7 +428,7 @@ Data Query Return Example HTTP requests need to include an authorization code ``, used for identity verification. The authorization code is usually provided by the administrator and can be simply obtained by sending an `HTTP GET` request as follows: -```bash +```shell curl http://:/rest/login// ``` @@ -440,7 +440,7 @@ Here, `fqdn` is the FQDN or IP address of the TDengine database, `port` is the p Example of obtaining an authorization code: -```bash +```shell curl http://192.168.0.1:6041/rest/login/root/taosdata ``` @@ -457,7 +457,7 @@ Return value: - Query all records of table d1001 in the demo database: - ```bash + ```shell curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "select * from demo.d1001" 192.168.0.1:6041/rest/sql curl -L -H "Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04" -d "select * from demo.d1001" 192.168.0.1:6041/rest/sql ``` @@ -509,7 +509,7 @@ Return value: - Create database demo: - ```bash + ```shell curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "create database demo" 192.168.0.1:6041/rest/sql curl -L -H "Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04" -d "create database demo" 192.168.0.1:6041/rest/sql ``` @@ -560,7 +560,7 @@ Return value: #### TDengine 2.x response codes and message bodies -```JSON +```json { "status": "succ", "head": [ @@ -624,7 +624,7 @@ Return value: #### TDengine 3.0 Response Codes and Message Body -```JSON +```json { "code": 0, "column_meta": [ diff --git a/docs/en/14-reference/09-error-code.md b/docs/en/14-reference/09-error-code.md index 84504395b5..5c7fdf2f1f 100644 --- a/docs/en/14-reference/09-error-code.md +++ b/docs/en/14-reference/09-error-code.md @@ -129,7 +129,7 @@ This document details the server error codes that may be encountered when using | 0x80000350 | User already exists | Create user, duplicate creation | Confirm if the operation is correct | | 0x80000351 | Invalid user | User does not exist | Confirm if the operation is correct | | 0x80000352 | Invalid user format | Incorrect format | Confirm if the operation is correct | -| 0x80000353 | Invalid password format | Incorrect format | Confirm if the operation is correct | +| 0x80000353 | Invalid password format | The password must be between 8 and 16 characters long and include at least three types of characters from the following: uppercase letters, lowercase letters, numbers, and special characters. | Confirm the format of the password string | | 0x80000354 | Can not get user from conn | Internal error | Report issue | | 0x80000355 | Too many users | (Enterprise only) Exceeding user limit | Adjust configuration | | 0x80000357 | Authentication failure | Incorrect password | Confirm if the operation is correct | @@ -251,6 +251,7 @@ This document details the server error codes that may be encountered when using | 0x80000529 | Vnode is stopped | Vnode is closed | Report issue | | 0x80000530 | Duplicate write request | Duplicate write request, internal error | Report issue | | 0x80000531 | Vnode query is busy | Query is busy | Report issue | +| 0x80000540 | Vnode already exist but Dbid not match | Internal error | Report issue | ## tsdb diff --git a/docs/en/27-train-faq/index.md b/docs/en/27-train-faq/index.md index 3b3f7a8e17..a5ec977e29 100644 --- a/docs/en/27-train-faq/index.md +++ b/docs/en/27-train-faq/index.md @@ -90,7 +90,7 @@ Batch insertion. Each insert statement can insert multiple records into one tabl When inserting nchar type data containing Chinese characters on Windows, first ensure that the system's regional settings are set to China (this can be set in the Control Panel). At this point, the `taos` client in cmd should already be working properly; if developing a Java application in an IDE, such as Eclipse or IntelliJ, ensure that the file encoding in the IDE is set to GBK (which is the default encoding type for Java), then initialize the client configuration when creating the Connection, as follows: -```JAVA +```java Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.LOCALE_KEY, "UTF-8"); @@ -145,7 +145,7 @@ Version 3.0 of TDengine includes a standalone component developed in Go called ` The Go language version requirement is 1.14 or higher. If there are Go compilation errors, often due to issues accessing Go mod in China, they can be resolved by setting Go environment variables: -```sh +```shell go env -w GO111MODULE=on go env -w GOPROXY=https://goproxy.cn,direct ``` @@ -196,7 +196,7 @@ Here are the solutions: 1. Create a file /Library/LaunchDaemons/limit.maxfiles.plist, write the following content (the example changes limit and maxfiles to 100,000, modify as needed): -```plist +```xml diff --git a/docs/zh/05-basic/03-query.md b/docs/zh/05-basic/03-query.md index b6f172829a..f3c9eb099e 100644 --- a/docs/zh/05-basic/03-query.md +++ b/docs/zh/05-basic/03-query.md @@ -303,13 +303,12 @@ Query OK, 5 row(s) in set (0.016812s) #### FILL 子句 -FILL 子句，用于指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种： 1. 不进行填充：NONE（默认填充模式）。 -2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL（VALUE， 1.23）。这里需要注意，最终填充的值受由相应列的类型决定，如 FILL（VALUE， 1.23），相应列为 INT 类型，则填充值为 1。 -3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL（PREV）。 -4. NULL 填充：使用 NULL 填充数据。例如：FILL（NULL）。 -5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL（LINEAR）。 -6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL（NEXT）。 +2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如 FILL(VALUE, 1.23)，相应列为 INT 类型，则填充值为 1, 若查询列表中有多列需要 FILL, 则需要给每一个 FILL 列指定 VALUE, 如 `SELECT _wstart, min(c1), max(c1) FROM ... FILL(VALUE, 0, 0)`, 注意, SELECT 表达式中只有包含普通列时才需要指定 FILL VALUE, 如 `_wstart`, `_wstart+1a`, `now`, `1+1` 以及使用 partition by 时的 partition key (如 tbname)都不需要指定 VALUE, 如 `timediff(last(ts), _wstart)` 则需要指定VALUE。 +3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。 +4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。 +5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。 +6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。 以上填充模式中，除了 NONE 模式默认不填充值之外，其他模式在查询的整个时间范围内如果没有数据 FILL 子句将被忽略，即不产生填充数据，查询结果为空。这种行为在部分模式（PREV、NEXT、LINEAR）下具有合理性，因为在这些模式下没有数据意味着无法产生填充数值。 diff --git a/docs/zh/07-develop/01-connect/index.md b/docs/zh/07-develop/01-connect/index.md index 94f55967ec..2381c49d93 100644 --- a/docs/zh/07-develop/01-connect/index.md +++ b/docs/zh/07-develop/01-connect/index.md @@ -37,7 +37,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 关键不同点在于： -1. 使用 原生连接，需要保证客户端的驱动程序 taosc 和服务端的 TDengine 版本配套。 +1. 使用 原生连接，需要保证客户端的驱动程序 taosc 和服务端的 TDengine 版本保持一致。 2. 使用 REST 连接，用户无需安装客户端驱动程序 taosc，具有跨平台易用的优势，但是无法体验数据订阅和二进制数据类型等功能。另外与 原生连接 和 WebSocket 连接相比，REST连接的性能最低。REST 接口是无状态的。在使用 REST 连接时，需要在 SQL 中指定表、超级表的数据库名称。 3. 使用 WebSocket 连接，用户也无需安装客户端驱动程序 taosc。 4. 连接云服务实例，必须使用 REST 连接 或 WebSocket 连接。 diff --git a/docs/zh/08-operation/12-multi.md b/docs/zh/08-operation/12-multi.md index 1e81a7ff1e..d18957d7d2 100644 --- a/docs/zh/08-operation/12-multi.md +++ b/docs/zh/08-operation/12-multi.md @@ -68,19 +68,19 @@ dataDir /mnt/data6 2 0 在配置文件 /etc/taos/taos.cfg 中，添加用于 S3 访问的参数： -|参数名称 | 参数含义 | -|:-------------|:-----------------------------------------------| -|s3EndPoint | 用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 的保持一致，否则无法访问。 | -|s3AccessKey |冒号分隔的用户 SecretId:SecretKey。例如：AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E | -|s3BucketName | 存储桶名称，减号后面是用户注册 COS 服务的 AppId。其中 AppId 是 COS 特有，AWS 和阿里云都没有，配置时需要作为 bucket name 的一部分，使用减号分隔。参数值均为字符串类型，但不需要引号。例如：test0711-1309024725 | -|s3UploadDelaySec | data 文件持续多长时间不再变动后上传至 s3，单位：秒。最小值：1；最大值：2592000 (30天），默认值 60 秒 | -|s3PageCacheSize |s3 page cache 缓存页数目，单位：页。最小值：4；最大值：1024*1024\*1024。 ，默认值 4096| -|s3MigrateIntervalSec | 本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600 | -|s3MigrateEnabled | 是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1。 | +| 参数名称 | 参数含义 | +|:---------------------|:-----------------------------------------------| +| s3EndPoint | 用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 的保持一致，否则无法访问。 | +| s3AccessKey | 冒号分隔的用户 SecretId:SecretKey。例如：AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E | +| s3BucketName | 存储桶名称，减号后面是用户注册 COS 服务的 AppId。其中 AppId 是 COS 特有，AWS 和阿里云都没有，配置时需要作为 bucket name 的一部分，使用减号分隔。参数值均为字符串类型，但不需要引号。例如：test0711-1309024725 | +| s3UploadDelaySec | data 文件持续多长时间不再变动后上传至 s3，单位：秒。最小值：1；最大值：2592000（30天），默认值 60 秒 | +| s3PageCacheSize | S3 page cache 缓存页数目，单位：页。最小值：4；最大值：1024*1024*1024。 ，默认值 4096| +| s3MigrateIntervalSec | 本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600 | +| s3MigrateEnabled | 是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1。 | ### 检查配置参数可用性 -在 taos.cfg 中完成对 s3 的配置后，通过 taosd 命令的 checks3 参数可以检查所配置的 S3 服务是否可用： +在 taos.cfg 中完成对 S3 的配置后，通过 taosd 命令的 checks3 参数可以检查所配置的 S3 服务是否可用： ``` taosd --checks3 @@ -106,11 +106,11 @@ s3migrate database ; 详细的 DB 参数见下表： -| # | 参数 | 默认值 | 最小值 | 最大值 | 描述 | -| :--- | :----------- | :----- | :----- | :------ | :----------------------------------------------------------- | -| 1 | s3_keeplocal | 365 | 1 | 365000 | 数据在本地保留的天数，即 data 文件在本地磁盘保留多长时间后可以上传到 S3。默认单位：天，支持 m（分钟）、h（小时）和 d（天）三个单位 | -| 2 | s3_chunkpages | 131072 | 131072 | 1048576 | 上传对象的大小阈值，与 tsdb_pagesize 参数一样，不可修改，单位为 TSDB 页 | -| 3 | s3_compact | 1 | 0 | 1 | TSDB 文件组首次上传 S3 时，是否自动进行 compact 操作。 | +| # | 参数 | 默认值 | 最小值 | 最大值 | 描述 | +|:--|:--------------|:-------|:------ |:------- | :----------------------------------------------------------- | +| 1 | s3_keeplocal | 365 | 1 | 365000 | 数据在本地保留的天数，即 data 文件在本地磁盘保留多长时间后可以上传到 S3。默认单位：天，支持 m（分钟）、h（小时）和 d（天）三个单位 | +| 2 | s3_chunkpages | 131072 | 131072 | 1048576 | 上传对象的大小阈值，与 tsdb_pagesize 参数一样，不可修改，单位为 TSDB 页 | +| 3 | s3_compact | 1 | 0 | 1 | TSDB 文件组首次上传 S3 时，是否自动进行 compact 操作 | ### 对象存储读写次数估算 @@ -168,10 +168,10 @@ s3BucketName td-test 用户界面同 S3，不同的地方在于下面三个参数的配置： -| # | 参数 | 示例值 | 描述 | -| :--- | :----------- | :--------------------------------------- | :----------------------------------------------------------- | -| 1 | s3EndPoint | https://fd2d01c73.blob.core.windows.net | Blob URL | -| 2 | s3AccessKey | fd2d01c73:veUy/iRBeWaI2YAerl+AStw6PPqg== | 冒号分隔的用户 accountId:accountKey | -| 3 | s3BucketName | test-container | Container name | +| # | 参数 | 示例值 | 描述 | +|:--|:-------------|:-----------------------------------------|:----------------------------------| +| 1 | s3EndPoint | https://fd2d01c73.blob.core.windows.net | Blob URL | +| 2 | s3AccessKey | fd2d01c73:veUy/iRBeWaI2YAerl+AStw6PPqg== | 冒号分隔的用户 accountId:accountKey | +| 3 | s3BucketName | test-container | Container name | 其中 fd2d01c73 是账户 ID；微软 Blob 存储服务只支持 Https 协议，不支持 Http。 diff --git a/docs/zh/08-operation/14-user.md b/docs/zh/08-operation/14-user.md index 03a838462f..a894570b46 100644 --- a/docs/zh/08-operation/14-user.md +++ b/docs/zh/08-operation/14-user.md @@ -12,18 +12,19 @@ TDengine 默认仅配置了一个 root 用户，该用户拥有最高权限。TD 创建用户的操作只能由 root 用户进行，语法如下。 ```sql -create user user_name pass'password' [sysinfo {1|0}] +create user user_name pass'password' [sysinfo {1|0}] [createdb {1|0}] ``` 相关参数说明如下。 -- user_name：最长为 23 B。 -- password：最长为 128 B，合法字符包括字母和数字以及单双引号、撇号、反斜杠和空格以外的特殊字符，且不可以为空。 +- user_name：用户名最长不超过 23 个字节。 +- password：密码长度必须为 8 到 16 位，并且至少包含大写字母、小写字母、数字、特殊字符中的三类。特殊字符包括 `! @ # $ % ^ & * ( ) - _ + = [ ] { } : ; > < ? | ~ , .`。 - sysinfo ：用户是否可以查看系统信息。1 表示可以查看，0 表示不可以查看。系统信息包括服务端配置信息、服务端各种节点信息，如 dnode、查询节点（qnode）等，以及与存储相关的信息等。默认为可以查看系统信息。 +- createdb：用户是否可以创建数据库。1 表示可以创建，0 表示不可以创建。缺省值为 0。// 从 TDengine 企业版 3.3.2.0 开始支持 -如下 SQL 可以创建密码为 123456 且可以查看系统信息的用户 test。 +如下 SQL 可以创建密码为 abc123!@# 且可以查看系统信息的用户 test。 ```sql -create user test pass '123456' sysinfo 1 +create user test pass 'abc123!@#' sysinfo 1 ``` ### 查看用户 @@ -47,6 +48,7 @@ alter_user_clause: { pass 'literal' | enable value | sysinfo value + | createdb value } ``` @@ -54,6 +56,7 @@ alter_user_clause: { - pass：修改用户密码。 - enable：是否启用用户。1 表示启用此用户，0 表示禁用此用户。 - sysinfo ：用户是否可查看系统信息。1 表示可以查看系统信息，0 表示不可以查看系统信息 +- createdb：用户是否可创建数据库。1 表示可以创建数据库，0 表示不可以创建数据库。// 从 TDengine 企业版 3.3.2.0 开始支持 如下 SQL 禁用 test 用户。 ```sql diff --git a/docs/zh/14-reference/01-components/12-tdinsight/assets/Alert-Used-Only.webp b/docs/zh/14-reference/01-components/12-tdinsight/assets/Alert-Used-Only.webp new file mode 100644 index 0000000000..8824ee5889 Binary files /dev/null and b/docs/zh/14-reference/01-components/12-tdinsight/assets/Alert-Used-Only.webp differ diff --git a/docs/zh/14-reference/01-components/12-tdinsight/assets/Alert-rules.webp b/docs/zh/14-reference/01-components/12-tdinsight/assets/Alert-rules.webp new file mode 100644 index 0000000000..e9f91c0488 Binary files /dev/null and b/docs/zh/14-reference/01-components/12-tdinsight/assets/Alert-rules.webp differ diff --git a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDengine-Alert.webp b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDengine-Alert.webp new file mode 100644 index 0000000000..a0d9b695f4 Binary files /dev/null and b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDengine-Alert.webp differ diff --git a/docs/zh/14-reference/01-components/12-tdinsight/index.mdx b/docs/zh/14-reference/01-components/12-tdinsight/index.mdx index e5f44a1080..0f6ea46ffd 100644 --- a/docs/zh/14-reference/01-components/12-tdinsight/index.mdx +++ b/docs/zh/14-reference/01-components/12-tdinsight/index.mdx @@ -145,6 +145,44 @@ TDinsight 仪表盘旨在提供 TDengine 相关资源的使用情况和状态， 还有上述分类的细分维度折线图。 +### 预配置告警规则自动导入 + +涛思总结用户使用经验，整理出 14 个常用的告警规则（alert rule），能够对集群关键指标进行监测并及时上报指标异常、超限等告警信息。 +从 TDengine-server 3.3.4.3 版本（tdengine-datasource 3.6.3）开始，TDengine Datasource 支持预配置告警规则自动导入功能，用户可将 14 个告警规则一键导入 Grafana（11 及以上版本），直接使用。 +预配置告警规则导入方法如下图所示，在 tdengine-datasource setting 界面，打开 “Load Tengine Alert” 开关，点击 “Save & test” 按钮后，插件会自动加载上述告警规则, 规则会放入以数据源名称 + “-alert” 的 grafana 告警目录中。如不需要，关闭 “Load TDengine Alert” 开关，点击 “Clear TDengine Alert” 旁边的按钮则会清除此数据源已导入的所有告警规则。 + +![TDengine Alert](./assets/TDengine-Alert.webp) + +导入后，点击 Grafana 左侧 “Alert rules” ，可查看当前所有告警规则。 +用户只需配置联络点（Contact points），即可获取告警通知。联络点配置方法见[告警配置](https://docs.taosdata.com/third-party/visual/grafana/#%E5%91%8A%E8%AD%A6%E9%85%8D%E7%BD%AE)。 + +![Alert-rules](./assets/Alert-rules.webp) + +14 个告警规则具体配置如下： + +| 规则名称| 规则阈值| 无监控数据时的行为 | 数据扫描间隔 |持续时间 | 执行SQL | +| ------ | --------- | ---------------- | ----------- |------- |----------------------| +|dnode 节点的CPU负载|均值 > 80%|触发告警|5分钟|5分钟 |`select now(), dnode_id, last(cpu_system) as cup_use from log.taosd_dnodes_info where _ts >= (now- 5m) and _ts < now partition by dnode_id having first(_ts) > 0 `| +|dnode 节点的的内存 |均值 > 60%|触发告警|5分钟|5分钟|`select now(), dnode_id, last(mem_engine) / last(mem_total) * 100 as taosd from log.taosd_dnodes_info where _ts >= (now- 5m) and _ts 80%|触发告警|5分钟|5分钟|`select now(), dnode_id, data_dir_level, data_dir_name, last(used) / last(total) * 100 as used from log.taosd_dnodes_data_dirs where _ts >= (now - 5m) and _ts < now partition by dnode_id, data_dir_level, data_dir_name`| +|集群授权到期 |< 60天|触发告警|1天|0秒|`select now(), cluster_id, last(grants_expire_time) / 86400 as expire_time from log.taosd_cluster_info where _ts >= (now - 24h) and _ts < now partition by cluster_id having first(_ts) > 0 `| +|测点数达到授权测点数|>= 90%|触发告警|1天|0秒|`select now(), cluster_id, CASE WHEN max(grants_timeseries_total) > 0.0 THEN max(grants_timeseries_used) /max(grants_timeseries_total) * 100.0 ELSE 0.0 END AS result from log.taosd_cluster_info where _ts >= (now - 30s) and _ts < now partition by cluster_id having timetruncate(first(_ts), 1m) > 0`| +|查询并发请求数 | > 100|不触发报警|1分钟|0秒|`select now() as ts, count(*) as slow_count from performance_schema.perf_queries`| +|慢查询执行最长时间 （无时间窗口） |> 300秒|不触发报警|1分钟|0秒|`select now() as ts, count(*) as slow_count from performance_schema.perf_queries where exec_usec>300000000`| +|dnode下线 |total != alive|触发告警|30秒|0秒|`select now(), cluster_id, last(dnodes_total) - last(dnodes_alive) as dnode_offline from log.taosd_cluster_info where _ts >= (now -30s) and _ts < now partition by cluster_id having first(_ts) > 0`| +|vnode下线 |total != alive|触发告警|30秒|0秒|`select now(), cluster_id, last(vnodes_total) - last(vnodes_alive) as vnode_offline from log.taosd_cluster_info where _ts >= (now - 30s) and _ts < now partition by cluster_id having first(_ts) > 0 `| +|数据删除请求数 |> 0|不触发报警|30秒|0秒|``select now(), count(`count`) as `delete_count` from log.taos_sql_req where sql_type = 'delete' and _ts >= (now -30s) and _ts < now``| +|Adapter RESTful 请求失败 |> 5|不触发报警|30秒|0秒|``select now(), sum(`fail`) as `Failed` from log.adapter_requests where req_type=0 and ts >= (now -30s) and ts < now``| +|Adapter WebSocket 请求失败 |> 5|不触发报警|30秒|0秒|``select now(), sum(`fail`) as `Failed` from log.adapter_requests where req_type=1 and ts >= (now -30s) and ts < now``| +|dnode 数据上报缺少 |< 3|触发告警|180秒|0秒|`select now(), cluster_id, count(*) as dnode_report from log.taosd_cluster_info where _ts >= (now -180s) and _ts < now partition by cluster_id having timetruncate(first(_ts), 1h) > 0`| +|dnode 重启 |max(update_time) > last(update_time)|触发告警|90秒|0秒|`select now(), dnode_id, max(uptime) - last(uptime) as dnode_restart from log.taosd_dnodes_info where _ts >= (now - 90s) and _ts < now partition by dnode_id`| + +用户可参考上述告警规则，根据自己业务需求进行修改与完善。 +Grafana7.5 及以下版本，Dashboards 与 Alert rules 功能合在一起，而之后的新版本两个功能是分开的。为兼容 Grafana7.5 及以下版本，TDinsight 面板中增加了 Alert Used Only 面板，仅 Grafana7.5 及以下版本需要使用。 + +![Alert Used Only](./assets/Alert-Used-Only.webp) + + ## 升级 下面三种方式都可以进行升级： - 用图形界面，若有新版本，可以在 ”TDengine Datasource“ 插件页面点击 update 升级。 @@ -155,10 +193,11 @@ TDinsight 仪表盘旨在提供 TDengine 相关资源的使用情况和状态， 针对不同的安装方式，卸载时： - 用图形界面，在 ”TDengine Datasource“ 插件页面点击 ”Uninstall“ 卸载。 - 通过 `TDinsight.sh` 脚本安装的 TDinsight，可以使用命令行 `TDinsight.sh -R` 清理相关资源。 -- 手动安装的 TDinsight，要完全卸载，需要清理以下内容： +- 手动安装的 TDinsight，要完全卸载，需要按照顺序清理以下内容： 1. Grafana 中的 TDinsight Dashboard。 - 2. Grafana 中的 Data Source 数据源。 - 3. 从插件安装目录删除 `tdengine-datasource` 插件。 + 2. Grafana 中的 Alert rules 告警规则。 + 3. Grafana 中的 Data Source 数据源。 + 4. 从插件安装目录删除 `tdengine-datasource` 插件。 ## 附录 diff --git a/docs/zh/14-reference/02-tools/08-taos-cli.md b/docs/zh/14-reference/02-tools/08-taos-cli.md index adecc5f760..5b204da4c2 100644 --- a/docs/zh/14-reference/02-tools/08-taos-cli.md +++ b/docs/zh/14-reference/02-tools/08-taos-cli.md @@ -54,7 +54,7 @@ taos> SET MAX_BINARY_DISPLAY_WIDTH ; - -h HOST: 要连接的 TDengine 服务端所在服务器的 FQDN, 默认为连接本地服务 - -P PORT: 指定服务端所用端口号 - -u USER: 连接时使用的用户名 -- -p PASSWORD: 连接服务端时使用的密码 +- -p PASSWORD: 连接服务端时使用的密码，特殊字符如 `! & ( ) < > ; |` 需使用字符 `\` 进行转义处理 - -?, --help: 打印出所有命令行参数 还有更多其他参数： diff --git a/docs/zh/14-reference/03-taos-sql/02-database.md b/docs/zh/14-reference/03-taos-sql/02-database.md index 9856089713..b75129eed7 100644 --- a/docs/zh/14-reference/03-taos-sql/02-database.md +++ b/docs/zh/14-reference/03-taos-sql/02-database.md @@ -64,7 +64,8 @@ database_option: { - DURATION：数据文件存储数据的时间跨度。可以使用加单位的表示形式，如 DURATION 100h、DURATION 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。不加时间单位时默认单位为天，如 DURATION 50 表示 50 天。 - MAXROWS：文件块中记录的最大条数，默认为 4096 条。 - MINROWS：文件块中记录的最小条数，默认为 100 条。 -- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 3 倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据从而释放存储空间。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。了解更多，请点击 [关于主键时间戳](https://docs.taosdata.com/reference/taos-sql/insert/) +- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 3 倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据从而释放存储空间。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](../../operation/planning/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。了解更多，请点击 [关于主键时间戳](https://docs.taosdata.com/reference/taos-sql/insert/) + - KEEP_TIME_OFFSET：自 3.2.0.0 版本生效。删除或迁移保存时间超过 KEEP 值的数据的延迟执行时间，默认值为 0 (小时)。在数据文件保存时间超过 KEEP 后，删除或迁移操作不会立即执行，而会额外等待本参数指定的时间间隔，以实现与业务高峰期错开的目的。 - STT_TRIGGER：表示落盘文件触发文件合并的个数。开源版本固定为 1，企业版本可设置范围为 1 到 16。对于少表高频写入场景，此参数建议使用默认配置；而对于多表低频写入场景，此参数建议配置较大的值。 - SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。 @@ -232,6 +233,3 @@ SHOW db_name.disk_info; 该命令本质上等同于 `select sum(data1 + data2 + data3)/sum(raw_data), sum(data1 + data2 + data3) from information_schema.ins_disk_usage where db_name="dbname"` - - - diff --git a/docs/zh/14-reference/03-taos-sql/03-table.md b/docs/zh/14-reference/03-taos-sql/03-table.md index 4bf1bd5b87..ca2170db8b 100644 --- a/docs/zh/14-reference/03-taos-sql/03-table.md +++ b/docs/zh/14-reference/03-taos-sql/03-table.md @@ -171,7 +171,7 @@ ALTER TABLE [db_name.]tb_name alter_table_clause alter_table_clause: { alter_table_options - | SET TAG tag_name = new_tag_value,tag_name2=new_tag2_value... + | SET TAG tag_name = new_tag_value, tag_name2=new_tag2_value ... } alter_table_options: @@ -195,7 +195,7 @@ alter_table_option: { ### 修改子表标签值 ``` -ALTER TABLE tb_name SET TAG tag_name=new_tag_value; +ALTER TABLE tb_name SET TAG tag_name1=new_tag_value1, tag_name2=new_tag_value2 ...; ``` ### 修改表生命周期 diff --git a/docs/zh/14-reference/03-taos-sql/08-delete-data.mdx b/docs/zh/14-reference/03-taos-sql/08-delete-data.mdx index 3f9c431338..ea03552795 100644 --- a/docs/zh/14-reference/03-taos-sql/08-delete-data.mdx +++ b/docs/zh/14-reference/03-taos-sql/08-delete-data.mdx @@ -6,7 +6,7 @@ title: "删除数据" 删除数据是 TDengine 提供的根据指定时间段删除指定表或超级表中数据记录的功能，方便用户清理由于设备故障等原因产生的异常数据。 -**注意**：删除数据并不会立即释放该表所占用的磁盘空间，而是把该表的数据标记为已删除，在查询时这些数据将不会再出现，但释放磁盘空间会延迟到系统自动或用户手动进行数据重整时。 +**注意**：删除数据并不会立即释放该表所占用的磁盘空间，而是把该表的数据标记为已删除，在查询时这些数据将不会再出现，但释放磁盘空间会延迟到系统自动清理（建库参数 keep 生效）或用户手动进行数据重整时（企业版功能 compact）。 **语法：** diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index 1d96db0969..7799e6f50e 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -1817,7 +1817,7 @@ ignore_null_values: { } ``` -**功能说明**：返回指定时间截面指定列的记录值或插值。ignore_null_values 参数的值可以是 0 或 1，为 1 时表示忽略 NULL 值, 缺省值为0。 +**功能说明**：返回指定时间截面指定列的记录值或插值。ignore_null_values 参数的值可以是 0 或 1，为 1 时表示忽略 NULL 值, 缺省值为 0。 **返回数据类型**：同字段类型。 @@ -1838,9 +1838,9 @@ ignore_null_values: { - INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(3.0.2.0 版本以后支持)。 - INTERP 可以与伪列 _isfilled 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(3.0.3.0 版本以后支持)。 - INTERP 对于带复合主键的表的查询，若存在相同时间戳的数据，则只有对应的复合主键最小的数据参与运算。 -- INTERP 查询支持NEAR FILL模式, 即当需要FILL时, 使用距离当前时间点最近的数据进行插值, 当前后时间戳与当前时间断面一样近时, FILL 前一行的值. 此模式在流计算中和窗口查询中不支持。例如: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', '2023-01-01 00:10:00') FILL(NEAR)。(3.3.4.9版本及以后支持)。 -- INTERP 只有在使用FILL PREV/NEXT/NEAR 模式时才可以使用伪列 `_irowts_origin`。`_irowts_origin`在3.3.4.9版本及以后支持。 -- INTERP `RANEG`子句支持时间范围的扩展(3.3.4.9版本及以后支持), 如`RANGE('2023-01-01 00:00:00', 10s)`表示在时间点'2023-01-01 00:00:00'查找前后10s的数据进行插值, FILL PREV/NEXT/NEAR分别表示从时间点向前/向后/前后查找数据, 若时间点周围没有数据, 则使用FILL指定的值进行插值, 因此此时FILL子句必须指定值。例如: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', 10s) FILL(PREV, 1). 目前仅支持时间点和时间范围的组合, 不支持时间区间和时间范围的组合, 即不支持RANGE('2023-01-01 00:00:00', '2023-02-01 00:00:00', 1h)。所指定的时间范围规则与EVERY类似, 单位不能是年或月, 值不能为0, 不能带引号。使用该扩展时, 不支持除FILL PREV/NEXT/NEAR外的其他FILL模式, 且不能指定EVERY子句。 +- INTERP 查询支持 NEAR FILL 模式, 即当需要 FILL 时, 使用距离当前时间点最近的数据进行插值, 当前后时间戳与当前时间断面一样近时, FILL 前一行的值. 此模式在流计算中和窗口查询中不支持。例如: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', '2023-01-01 00:10:00') FILL(NEAR)(3.3.4.9 版本及以后支持)。 +- INTERP 只有在使用 FILL PREV/NEXT/NEAR 模式时才可以使用伪列 `_irowts_origin`。`_irowts_origin`在 3.3.4.9 版本及以后支持。 +- INTERP `RANGE`子句支持时间范围的扩展(3.3.4.9 版本及以后支持), 如`RANGE('2023-01-01 00:00:00', 10s)`表示在时间点 '2023-01-01 00:00:00' 查找前后 10s 的数据进行插值, FILL PREV/NEXT/NEAR 分别表示从时间点向前/向后/前后查找数据, 若时间点周围没有数据, 则使用 FILL 指定的值进行插值, 因此此时 FILL 子句必须指定值。例如: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', 10s) FILL(PREV, 1)。目前仅支持时间点和时间范围的组合, 不支持时间区间和时间范围的组合, 即不支持 RANGE('2023-01-01 00:00:00', '2023-02-01 00:00:00', 1h)。所指定的时间范围规则与 EVERY 类似, 单位不能是年或月, 值不能为 0, 不能带引号。使用该扩展时, 不支持除FILL PREV/NEXT/NEAR外的其他 FILL 模式, 且不能指定 EVERY 子句。 ### LAST diff --git a/docs/zh/14-reference/03-taos-sql/12-distinguished.md b/docs/zh/14-reference/03-taos-sql/12-distinguished.md index 4945e37779..ef8faff85a 100644 --- a/docs/zh/14-reference/03-taos-sql/12-distinguished.md +++ b/docs/zh/14-reference/03-taos-sql/12-distinguished.md @@ -124,7 +124,7 @@ INTERVAL 子句允许使用 AUTO 关键字来指定窗口偏移量，此时如 ```sql -- 有起始时间限制，从 '2018-10-03 14:38:05' 切分时间窗口 -SELECT COUNT(*) FROM meters WHERE _rowts >= '2018-10-03 14:38:05' INTERVAL (1m， AUTO); +SELECT COUNT(*) FROM meters WHERE _rowts >= '2018-10-03 14:38:05' INTERVAL (1m, AUTO); -- 无起始时间限制，不生效，仍以 0 为偏移量 SELECT COUNT(*) FROM meters WHERE _rowts < '2018-10-03 15:00:00' INTERVAL (1m, AUTO); diff --git a/docs/zh/14-reference/03-taos-sql/22-meta.md b/docs/zh/14-reference/03-taos-sql/22-meta.md index 0f7e14e3fb..b269453ed2 100644 --- a/docs/zh/14-reference/03-taos-sql/22-meta.md +++ b/docs/zh/14-reference/03-taos-sql/22-meta.md @@ -303,24 +303,24 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 ## INS_STREAMS -| # | **列名** | **数据类型** | **说明** | -| --- | :----------: | ------------ | -------------------------------------------------------------------------------------------------------------------- | -| 1 | stream_name | VARCHAR(64) | 流计算名称 | -| 2 | create_time | TIMESTAMP | 创建时间 | -| 3 | sql | VARCHAR(1024) | 创建流计算时提供的 SQL 语句 | -| 4 | status | VARCHAR(20) | 流当前状态 | -| 5 | source_db | VARCHAR(64) | 源数据库 | -| 6 | target_db | VARCHAR(64) | 目的数据库 | -| 7 | target_table | VARCHAR(192) | 流计算写入的目标表 | -| 8 | watermark | BIGINT | watermark，详见 SQL 手册流式计算。需要注意，`watermark` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 9 | trigger | INT | 计算结果推送模式，详见 SQL 手册流式计算。需要注意，`trigger` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| # | **列名** | **数据类型** | **说明** | +|:----|:-----------|:------------|:--------| +| 1 | stream_name | VARCHAR(64) | 流计算名称 | +| 2 | create_time | TIMESTAMP | 创建时间 | +| 3 | sql | VARCHAR(1024) | 创建流计算时提供的 SQL 语句 | +| 4 | status | VARCHAR(20) | 流当前状态 | +| 5 | source_db | VARCHAR(64) | 源数据库 | +| 6 | target_db | VARCHAR(64) | 目的数据库 | +| 7 | target_table | VARCHAR(192) | 流计算写入的目标表 | +| 8 | watermark | BIGINT | watermark，详见 SQL 手册流式计算。需要注意，`watermark` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 9 | trigger | INT | 计算结果推送模式，详见 SQL 手册流式计算。需要注意，`trigger` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。| ## INS_USER_PRIVILEGES 注：SYSINFO 属性为 0 的用户不能查看此表。 -| # | **列名** | **数据类型** | **说明** | -| --- | :----------: | ------------ | -------------------------------------------------------------------------------------------------------------------- | +| # | **列名** | **数据类型** | **说明** | +|:----|:-----------|:------------|:--------| | 1 | user_name | VARCHAR(24) | 用户名 | 2 | privilege | VARCHAR(10) | 权限描述 | 3 | db_name | VARCHAR(65) | 数据库名称 @@ -329,18 +329,18 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 ## INS_DISK_USAGE -| # | **列名** | **数据类型** | **说明** | -| --- | :----------: | ------------ | -------------------------------------------------------------------------------------------------------------------- | -| 1 | db_name | VARCHAR(32) | 数据库名称 -| 2 | vgroup_id | INT | vgroup 的 ID -| 3 | wal | BIGINT | wal 文件大小, 单位为 K -| 4 | data1 | BIGINT | 一级存储上数据文件的大小,单位为KB -| 5 | data2 | BIGINT | 二级存储上数据文件的大小,单位为 KB -| 6 | data3 | BIGINT | 三级存储上数据文件的大小, 单位为KB -| 7 | cache_rdb | BIGINT | last/last_row 文件的大小,单位为KB -| 8 | table_meta | BIGINT | meta 文件的大小, 单位为KB -| 9 | s3 | BIGINT | s3 上占用的大小, 单位为KB -| 10 | raw_data | BIGINT | 预估的原始数据的大小, 单位为KB +| # | **列名** | **数据类型** | **说明** | +|:----|:-----------|:------------|:--------| +| 1 | db_name | VARCHAR(32) | 数据库名称 | +| 2 | vgroup_id | INT | vgroup 的 ID | +| 3 | wal | BIGINT | wal 文件大小, 单位为 K | +| 4 | data1 | BIGINT | 一级存储上数据文件的大小,单位为KB | +| 5 | data2 | BIGINT | 二级存储上数据文件的大小,单位为 KB | +| 6 | data3 | BIGINT | 三级存储上数据文件的大小, 单位为KB | +| 7 | cache_rdb | BIGINT | last/last_row 文件的大小,单位为KB | +| 8 | table_meta | BIGINT | meta 文件的大小, 单位为KB | +| 9 | s3 | BIGINT | s3 上占用的大小, 单位为KB | +| 10 | raw_data | BIGINT | 预估的原始数据的大小, 单位为KB | ## INS_FILESETS diff --git a/docs/zh/14-reference/03-taos-sql/25-user.md b/docs/zh/14-reference/03-taos-sql/25-user.md index a77a5d6a67..3ddde297de 100644 --- a/docs/zh/14-reference/03-taos-sql/25-user.md +++ b/docs/zh/14-reference/03-taos-sql/25-user.md @@ -9,19 +9,21 @@ description: 本节讲述基本的用户管理功能 ## 创建用户 ```sql -CREATE USER user_name PASS 'password' [SYSINFO {1|0}]; +CREATE USER user_name PASS 'password' [SYSINFO {1|0}] [CREATEDB {1|0}]; ``` 用户名最长不超过 23 个字节。 -密码最长不超过 31 个字节。密码可以包含字母、数字以及除单引号、双引号、反引号、反斜杠和空格以外的特殊字符，密码不能为空字符串。 +密码长度必须为 8 到 16 位，并且至少包含大写字母、小写字母、数字、特殊字符中的三类。特殊字符包括 `! @ # $ % ^ & * ( ) - _ + = [ ] { } : ; > < ? | ~ , .`。 `SYSINFO` 表示该用户是否能够查看系统信息。`1` 表示可以查看，`0` 表示无权查看。系统信息包括服务配置、dnode、vnode、存储等信息。缺省值为 `1`。 -在下面的示例中，我们创建一个密码为 `123456` 且可以查看系统信息的用户。 +`CREATEDB` 表示该用户是否能够创建数据库。`1` 表示可以创建，`0` 表示无权创建。缺省值为 `0`。// 从 TDengine 企业版 3.3.2.0 开始支持 + +在下面的示例中，我们创建一个密码为 `abc123!@#` 且可以查看系统信息的用户。 ```sql -taos> create user test pass '123456' sysinfo 1; +taos> create user test pass 'abc123!@#' sysinfo 1; Query OK, 0 of 0 rows affected (0.001254s) ``` @@ -77,7 +79,7 @@ alter_user_clause: { - PASS: 修改密码，后跟新密码 - ENABLE: 启用或禁用该用户，`1` 表示启用，`0` 表示禁用 - SYSINFO: 允许或禁止查看系统信息，`1` 表示允许，`0` 表示禁止 -- CREATEDB: 允许或禁止创建数据库，`1` 表示允许，`0` 表示禁止 +- CREATEDB: 允许或禁止创建数据库，`1` 表示允许，`0` 表示禁止。// 从 TDengine 企业版 3.3.2.0 开始支持 下面的示例禁用了名为 `test` 的用户: diff --git a/docs/zh/14-reference/03-taos-sql/32-compress.md b/docs/zh/14-reference/03-taos-sql/32-compress.md index 51650c9123..60d03e0cc0 100644 --- a/docs/zh/14-reference/03-taos-sql/32-compress.md +++ b/docs/zh/14-reference/03-taos-sql/32-compress.md @@ -29,14 +29,15 @@ description: 可配置压缩算法 - 各个数据类型的默认压缩算法列表和适用范围 -| 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法|压缩算法默认值| 压缩等级默认值| -| :-----------:|:----------:|:-------:|:-------:|:----------:|:----:| -| int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| -| tinyint/untinyint/smallint/usmallint | simple8b| simple8b | lz4/zlib/zstd/xz| zlib | medium| -| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium| -|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium| -|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| zstd| medium| -|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| zstd| medium| +| 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法 | 压缩算法默认值 |压缩等级默认值| +|:------------------------------------:|:-------------------------:|:-----------:|:--------------------:|:----:|:------:| +| int/uint | disabled/simple8b | simple8b | lz4/zlib/zstd/xz | lz4 | medium | +| tinyint/untinyint/smallint/usmallint | disabled/simple8b | simple8b | lz4/zlib/zstd/xz | zlib | medium | +| bigint/ubigint/timestamp | disabled/simple8b/delta-i | delta-i | lz4/zlib/zstd/xz | lz4 | medium | +| float/double | disabled/delta-d | delta-d | lz4/zlib/zstd/xz/tsz | lz4 | medium | +| binary/nchar | disabled | disabled | lz4/zlib/zstd/xz | zstd | medium | +| bool | disabled/bit-packing | bit-packing | lz4/zlib/zstd/xz | zstd | medium | + ## SQL 语法 diff --git a/docs/zh/14-reference/05-connector/10-cpp.mdx b/docs/zh/14-reference/05-connector/10-cpp.mdx index 7164baad2a..a836234af2 100644 --- a/docs/zh/14-reference/05-connector/10-cpp.mdx +++ b/docs/zh/14-reference/05-connector/10-cpp.mdx @@ -680,12 +680,31 @@ TDengine 客户端驱动的版本号与 TDengine 服务端的版本号是一一 - **接口说明**：清理运行环境，应用退出前应调用。 - `int taos_options(TSDB_OPTION option, const void * arg, ...)` - - **接口说明**：设置客户端选项，目前支持区域设置（`TSDB_OPTION_LOCALE`）、字符集设置（`TSDB_OPTION_CHARSET`）、时区设置（`TSDB_OPTION_TIMEZONE`）、配置文件路径设置（`TSDB_OPTION_CONFIGDIR`）。区域设置、字符集、时区默认为操作系统当前设置。 + - **接口说明**：设置客户端选项，支持区域设置（`TSDB_OPTION_LOCALE`）、字符集设置（`TSDB_OPTION_CHARSET`）、时区设置（`TSDB_OPTION_TIMEZONE`）、配置文件路径设置（`TSDB_OPTION_CONFIGDIR`）。区域设置、字符集、时区默认为操作系统当前设置。 - **参数说明**： - `option`：[入参] 设置项类型。 - `arg`：[入参] 设置项值。 - **返回值**：`0`：成功，`-1`：失败。 +- `int taos_options_connection(TAOS *taos, TSDB_OPTION_CONNECTION option, const void *arg, ...)` + - **接口说明**：设置客户端连接选项，目前支持字符集设置（`TSDB_OPTION_CONNECTION_CHARSET`）、时区设置（`TSDB_OPTION_CONNECTION_TIMEZONE`）、用户 IP 设置（`TSDB_OPTION_CONNECTION_USER_IP`）、用户 APP 设置（`TSDB_OPTION_CONNECTION_USER_APP`）。 + - **参数说明**： + - `taos`: [入参] taos_connect 返回的连接句柄。 + - `option`：[入参] 设置项类型。 + - `arg`：[入参] 设置项值。 + - **返回值**：`0`：成功，`非0`：失败。 + - **说明**： + - 字符集、时区默认为操作系统当前设置，windows 不支持连接级别的时区设置。 + - arg 为 NULL 时表示重置该选项。 + - 该接口只对当前连接有效，不会影响其他连接。 + - 同样参数多次调用该接口，以后面的为准，可以作为修改的方法。 + - TSDB_OPTION_CONNECTION_CLEAR 选项用于重置所有连接选项。 + - 时区和字符集重置后，使用系统的设置，user ip 和 user app 重置后为空。 + - 连接选项的值都是 string 类型，user app 参数值最大长度为 23，超过该长度会被截断；其他参数非法时报错。 + - 时区配置找不到时区文件或者不能按照规范解释时，默认为 UTC，和操作系统时区规则相同，详见 tzset 函数说明。可通过 select timezone() 查看当前连接的时区。 + - 时区和字符集只在 client 侧起作用，对于在服务端的相关行为不起作用。 + - 时区文件使用操作系统时区文件，可以自行更新操作系统时区文件。如果设置时区报错，请检查是否有时区文件或路径（mac:/var/db/timezone/zoneinfo, linux:/usr/share/zoneinfo）是否正确。 + - `char *taos_get_client_info()` - **接口说明**：获取客户端版本信息。 - **返回值**：返回客户端版本信息。 diff --git a/docs/zh/14-reference/05-connector/14-java.mdx b/docs/zh/14-reference/05-connector/14-java.mdx index e8554ae668..be04a9e2dc 100644 --- a/docs/zh/14-reference/05-connector/14-java.mdx +++ b/docs/zh/14-reference/05-connector/14-java.mdx @@ -172,7 +172,7 @@ WKB规范请参考[Well-Known Binary (WKB)](https://libgeos.org/specifications/w **原因**：程序没有找到依赖的本地函数库 taos。 -**解决方法**：Windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，Linux 下将建立如下软链 `ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可，macOS 下需要建立软链 `ln -s /usr/local/lib/libtaos.dylib`。 +**解决方法**：Windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，Linux 下将建立如下软链 `ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可，macOS 下需要建立软链 `ln -s /usr/local/lib/libtaos.dylib /usr/lib/libtaos.dylib`。 3. java.lang.UnsatisfiedLinkError: taos.dll Can't load AMD 64 bit on a IA 32-bit platform diff --git a/docs/zh/14-reference/09-error-code.md b/docs/zh/14-reference/09-error-code.md index 00e6a72b6f..91792f6afc 100644 --- a/docs/zh/14-reference/09-error-code.md +++ b/docs/zh/14-reference/09-error-code.md @@ -136,7 +136,7 @@ description: TDengine 服务端的错误码列表和详细说明 | 0x80000350 | User already exists | Create user, 重复创建 | 确认操作是否正确 | | 0x80000351 | Invalid user | 用户不存在 | 确认操作是否正确 | | 0x80000352 | Invalid user format | 格式不正确 | 确认操作是否正确 | -| 0x80000353 | Invalid password format | 格式不正确 | 确认操作是否正确 | +| 0x80000353 | Invalid password format | 密码长度必须为 8 到 16 位，并且至少包含大写字母、小写字母、数字、特殊字符中的三类 | 确认密码字符串的格式 | | 0x80000354 | Can not get user from conn | 内部错误 | 上报issue | | 0x80000355 | Too many users | （仅企业版）用户数量超限 | 调整配置 | | 0x80000357 | Authentication failure | 密码不正确 | 确认操作是否正确 | @@ -261,6 +261,7 @@ description: TDengine 服务端的错误码列表和详细说明 | 0x80000529 | Vnode is stopped | Vnode 已经关闭 | 上报问题 | | 0x80000530 | Duplicate write request | 重复写入请求，内部错误 | 上报问题 | | 0x80000531 | Vnode query is busy | 查询忙碌 | 上报问题 | +| 0x80000540 | Vnode already exist but Dbid not match | 内部错误 | 上报问题 | ## tsdb diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md index af8468411c..b7d20cac5a 100644 --- a/docs/zh/27-train-faq/01-faq.md +++ b/docs/zh/27-train-faq/01-faq.md @@ -280,4 +280,12 @@ TDinsight插件中展示的数据是通过taosKeeper和taosAdapter服务收集 https://docs.taosdata.com/reference/components/taosd/#%E7%9B%91%E6%8E%A7%E7%9B%B8%E5%85%B3 您可以随时关闭该参数，只需要在taos.cfg 中修改telemetryReporting为 0，然后重启数据库服务即可。 代码位于:https://github.com/taosdata/TDengine/blob/62e609c558deb764a37d1a01ba84bc35115a85a4/source/dnode/mnode/impl/src/mndTelem.c -此外，对于安全性要求极高的企业版 TDengine Enterprise 来说，此参数不会工作。 +此外，对于安全性要求极高的企业版 TDengine Enterprise 来说，此参数不会工作。 +### 31 第一次连接集群时遇到“Sync leader is unreachable”怎么办？ +报这个错，说明第一次向集群的连接是成功的，但第一次访问的IP不是mnode的leader节点，客户端试图与leader建立连接时发生错误。客户端通过EP，也就是指定的fqdn与端口号寻找leader节点，常见的报错原因有两个： + +- 集群中其他节点的端口没有打开 +- 客户端的hosts未正确配置 + +因此用户首先要检查服务端，集群的所有端口（原生连接默认6030，http连接默认6041）有无打开；其次是客户端的hosts文件中是否配置了集群所有节点的fqdn与IP信息。 +如仍无法解决，则需要联系涛思技术人员支持。 diff --git a/include/client/taos.h b/include/client/taos.h index 360a58ca2b..e59ea9d418 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -64,6 +64,15 @@ typedef enum { TSDB_MAX_OPTIONS } TSDB_OPTION; +typedef enum { + TSDB_OPTION_CONNECTION_CLEAR = -1, // means clear all option in this connection + TSDB_OPTION_CONNECTION_CHARSET, // charset, Same as the scope supported by the system + TSDB_OPTION_CONNECTION_TIMEZONE, // timezone, Same as the scope supported by the system + TSDB_OPTION_CONNECTION_USER_IP, // user ip + TSDB_OPTION_CONNECTION_USER_APP, // user app, max lengthe is 23, truncated if longer than 23 + TSDB_MAX_OPTIONS_CONNECTION +} TSDB_OPTION_CONNECTION; + typedef enum { TSDB_SML_UNKNOWN_PROTOCOL = 0, TSDB_SML_LINE_PROTOCOL = 1, @@ -174,11 +183,12 @@ typedef struct TAOS_STMT_OPTIONS { DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); +DLL_EXPORT int taos_options_connection(TAOS *taos, TSDB_OPTION_CONNECTION option, const void *arg, ...); DLL_EXPORT setConfRet taos_set_config(const char *config); DLL_EXPORT int taos_init(void); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); -DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); -DLL_EXPORT void taos_close(TAOS *taos); +DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); +DLL_EXPORT void taos_close(TAOS *taos); DLL_EXPORT const char *taos_data_type(int type); diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index f899fc5589..cb05f98f45 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -154,7 +154,7 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag); int32_t tTagToValArray(const STag *pTag, SArray **ppArray); void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove -int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf); +int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf, void *charsetCxt); // SColData ================================ typedef struct { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d9c262705d..fbd4f5a553 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1245,7 +1245,7 @@ typedef struct { } STsBufInfo; typedef struct { - int32_t tz; // query client timezone + void* timezone; char intervalUnit; char slidingUnit; char offsetUnit; @@ -3472,6 +3472,8 @@ typedef struct { SAppHbReq app; SQueryHbReqBasic* query; SHashObj* info; // hash + char userApp[TSDB_APP_NAME_LEN]; + uint32_t userIp; } SClientHbReq; typedef struct { @@ -3893,7 +3895,7 @@ typedef struct { int8_t igExists; int8_t intervalUnit; int8_t slidingUnit; - int8_t timezone; + int8_t timezone; // int8_t is not enough, timezone is unit of second int32_t dstVgId; // for stream int64_t interval; int64_t offset; diff --git a/include/common/ttime.h b/include/common/ttime.h index a54f8b2490..47875752d5 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -61,22 +61,9 @@ static FORCE_INLINE int64_t taosGetTimestamp(int32_t precision) { * precision == TSDB_TIME_PRECISION_MILLI, it returns timestamp in millisecond. * precision == TSDB_TIME_PRECISION_NANO, it returns timestamp in nanosecond. */ -static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { - int64_t factor = (precision == TSDB_TIME_PRECISION_MILLI) ? 1000 - : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000 - : 1000000000; - time_t t; - (void) taosTime(&t); - struct tm tm; - (void) taosLocalTime(&t, &tm, NULL, 0); - tm.tm_hour = 0; - tm.tm_min = 0; - tm.tm_sec = 0; +int64_t taosGetTimestampToday(int32_t precision, timezone_t tz); - return (int64_t)taosMktime(&tm) * factor; -} - -int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); +int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision, timezone_t tz); int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval); int64_t taosTimeGetIntervalEnd(int64_t ts, const SInterval* pInterval); @@ -86,13 +73,12 @@ void calcIntervalAutoOffset(SInterval* interval); int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision, bool negativeAllow); -int32_t taosParseTime(const char* timestr, int64_t* pTime, int32_t len, int32_t timePrec, int8_t dayligth); -void deltaToUtcInitOnce(); +int32_t taosParseTime(const char* timestr, int64_t* pTime, int32_t len, int32_t timePrec, timezone_t tz); char getPrecisionUnit(int32_t precision); int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision); int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes); -int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal); +int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal, timezone_t tz, void* charsetCxt); int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision); int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision); @@ -102,8 +88,8 @@ struct STm { int64_t fsec; // in NANOSECOND }; -int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm); -int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision); +int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm, timezone_t tz); +int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision, timezone_t tz); /// @brief convert a timestamp to a formatted string /// @param format the timestamp format, must null terminated @@ -112,7 +98,7 @@ int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision); /// formats array; If not NULL, [formats] will be used instead of [format] to skip parse formats again. /// @param out output buffer, should be initialized by memset /// @notes remember to free the generated formats -int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen); +int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen, timezone_t tz); /// @brief convert a formatted timestamp string to a timestamp /// @param format must null terminated /// @param [in, out] formats, see taosTs2Char @@ -120,7 +106,7 @@ int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t pr /// @retval 0 for success, otherwise error occured /// @notes remember to free the generated formats even when error occured int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg, - int32_t errMsgLen); + int32_t errMsgLen, timezone_t tz); int32_t TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out, int32_t outLen); int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const char* tsStr); diff --git a/include/libs/command/command.h b/include/libs/command/command.h index 9fb2ca40b9..dd2441a115 100644 --- a/include/libs/command/command.h +++ b/include/libs/command/command.h @@ -22,7 +22,7 @@ typedef struct SExplainCtx SExplainCtx; -int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode *pStmt, SRetrieveTableRsp **pRsp, int8_t biMode); +int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode *pStmt, SRetrieveTableRsp **pRsp, int8_t biMode, void* charsetCxt); int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp); int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs); diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 51d9e752a4..0ca1962b4e 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -292,8 +292,16 @@ struct SScalarParam { void *param; // other parameter, such as meta handle from vnode, to extract table name/tag value int32_t numOfRows; int32_t numOfQualified; // number of qualified elements in the final results + timezone_t tz; + void *charsetCxt; }; +static inline void setTzCharset(SScalarParam* param, timezone_t tz, void* charsetCxt){ + if (param == NULL) return; + param->tz = tz; + param->charsetCxt = charsetCxt; +} + #define cleanupResultRowEntry(p) p->initialized = false #define isRowEntryCompleted(p) (p->complete) #define isRowEntryInitialized(p) (p->initialized) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 667d610eab..5e4e8b6292 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -129,8 +129,10 @@ typedef struct SValueNode { double d; char* p; } datum; - int64_t typeData; - int8_t unit; + int64_t typeData; + int8_t unit; + timezone_t tz; + void *charsetCxt; } SValueNode; typedef struct SLeftValueNode { @@ -159,6 +161,8 @@ typedef struct SOperatorNode { EOperatorType opType; SNode* pLeft; SNode* pRight; + timezone_t tz; + void* charsetCxt; } SOperatorNode; typedef struct SLogicConditionNode { @@ -190,7 +194,9 @@ typedef struct SFunctionNode { bool hasOriginalFunc; int32_t originalFuncId; ETrimType trimType; - bool dual; // whether select stmt without from stmt, true for without. + bool dual; // whether select stmt without from stmt, true for without. + timezone_t tz; + void *charsetCxt; } SFunctionNode; typedef struct STableNode { @@ -332,6 +338,7 @@ typedef struct SIntervalWindowNode { SNode* pSliding; // SValueNode SNode* pFill; STimeWindow timeRange; + void* timezone; } SIntervalWindowNode; typedef struct SEventWindowNode { @@ -401,6 +408,8 @@ typedef struct SCaseWhenNode { SNode* pCase; SNode* pElse; SNodeList* pWhenThenList; + timezone_t tz; + void* charsetCxt; } SCaseWhenNode; typedef struct SWindowOffsetNode { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index fbbe7cc40c..82985457a8 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -101,6 +101,8 @@ typedef struct SParseContext { int8_t biMode; SArray* pSubMetaList; setQueryFn setQueryFp; + timezone_t timezone; + void *charsetCxt; } SParseContext; int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery); @@ -139,28 +141,28 @@ void qDestroyStmtDataBlock(STableDataCxt* pBlock); STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock); int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData); -int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx); +int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, void *charsetCxt); int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery); int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, - STSchema** pTSchema, SBindInfo* pBindInfos); -int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); + STSchema** pTSchema, SBindInfo* pBindInfos, void* charsetCxt); +int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt); int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, - int32_t colIdx, int32_t rowNum); + int32_t colIdx, int32_t rowNum, void* charsetCxt); int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields); int32_t qBuildStmtStbColFields(void* pBlock, void* boundTags, bool hasCtbName, int32_t* fieldNum, TAOS_FIELD_STB** fields); int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields); int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, - TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); + TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt); -int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx); +int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx, void* charsetCxt); int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, - STSchema** pTSchema, SBindInfo2* pBindInfos); -int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen); + STSchema** pTSchema, SBindInfo2* pBindInfos, void *charsetCxt); +int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void *charsetCxt); int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, - int32_t colIdx, int32_t rowNum); + int32_t colIdx, int32_t rowNum, void *charsetCxt); int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, - TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen); + TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void *charsetCxt); void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, @@ -170,13 +172,13 @@ void qDestroyBoundColInfo(void* pInfo); int32_t smlInitHandle(SQuery** query); int32_t smlBuildRow(STableDataCxt* pTableCxt); -int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index); +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index, void* charsetCxt); int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt); void clearColValArraySml(SArray* pCols); int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, - char* msgBuf, int32_t msgBufLen); + char* msgBuf, int32_t msgBufLen, void* charsetCxt); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields, int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw); diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index a78fc4d2a6..92417bbf32 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -46,6 +46,7 @@ typedef struct SPlanContext { int64_t allocatorId; bool destHasPrimaryKey; bool sourceHasPrimaryKey; + void* timezone; } SPlanContext; // Create the physical plan for the query, according to the AST. diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index d2f714f400..1d4542b531 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -194,6 +194,7 @@ typedef struct SBoundColInfo { int32_t numOfCols; int32_t numOfBound; bool hasBoundCols; + bool mixTagsCols; } SBoundColInfo; typedef struct STableColsData { @@ -337,7 +338,7 @@ SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* nam void destroyQueryExecRes(SExecResult* pRes); int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_t bufSize, int32_t* len); -void parseTagDatatoJson(void* p, char** jsonStr); +void parseTagDatatoJson(void* p, char** jsonStr, void *charsetCxt); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 4b89a6a439..fd936dd087 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -105,7 +105,6 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); -int32_t timeZoneStrLen(); int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t weekdayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t dayofweekFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index 97bb6f53ba..d595c687b7 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -48,13 +48,13 @@ void atomic_store_8(int8_t volatile *ptr, int8_t val); void atomic_store_16(int16_t volatile *ptr, int16_t val); void atomic_store_32(int32_t volatile *ptr, int32_t val); void atomic_store_64(int64_t volatile *ptr, int64_t val); -double atomic_store_double(double volatile *ptr, double val); +void atomic_store_double(double volatile *ptr, double val); void atomic_store_ptr(void *ptr, void *val); int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val); int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val); int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val); int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val); -double atomic_exchange_double(double volatile *ptr, int64_t val); +double atomic_exchange_double(double volatile *ptr, double val); void *atomic_exchange_ptr(void *ptr, void *val); int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval); int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval); @@ -71,7 +71,7 @@ int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val); double atomic_fetch_add_double(double volatile *ptr, double val); -void *atomic_fetch_add_ptr(void *ptr, void *val); +void *atomic_fetch_add_ptr(void *ptr, int64_t val); int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val); @@ -82,37 +82,37 @@ int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val); double atomic_fetch_sub_double(double volatile *ptr, double val); -void *atomic_fetch_sub_ptr(void *ptr, void *val); +void *atomic_fetch_sub_ptr(void *ptr, int64_t val); int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val); -void *atomic_and_fetch_ptr(void *ptr, void *val); +void *atomic_and_fetch_ptr(void *ptr, int64_t val); int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val); -void *atomic_fetch_and_ptr(void *ptr, void *val); +void *atomic_fetch_and_ptr(void *ptr, int64_t val); int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val); -void *atomic_or_fetch_ptr(void *ptr, void *val); +void *atomic_or_fetch_ptr(void *ptr, int64_t val); int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val); -void *atomic_fetch_or_ptr(void *ptr, void *val); +void *atomic_fetch_or_ptr(void *ptr, int64_t val); int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val); -void *atomic_xor_fetch_ptr(void *ptr, void *val); +void *atomic_xor_fetch_ptr(void *ptr, int64_t val); int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val); -void *atomic_fetch_xor_ptr(void *ptr, void *val); +void *atomic_fetch_xor_ptr(void *ptr, int64_t val); #ifdef _MSC_VER #define tmemory_barrier(order) MemoryBarrier() diff --git a/include/os/osEnv.h b/include/os/osEnv.h index f235d3f235..85fdb7c38c 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -25,10 +25,9 @@ extern "C" { extern char tsOsName[]; extern char tsTimezoneStr[]; -extern enum TdTimezone tsTimezone; extern char tsCharset[]; +extern void *tsCharsetCxt; extern char tsLocale[]; -extern int8_t tsDaylight; extern bool tsEnableCoreFile; extern int64_t tsPageSizeKB; extern int64_t tsOpenMax; @@ -67,8 +66,7 @@ bool osDataSpaceSufficient(); bool osTempSpaceSufficient(); int32_t osSetTimezone(const char *timezone); -void osSetSystemLocale(const char *inLocale, const char *inCharSet); -void osSetProcPath(int32_t argc, char **argv); +void osSetProcPath(int32_t argc, char **argv); #ifdef __cplusplus } diff --git a/include/os/osLocale.h b/include/os/osLocale.h index 5f2a1c35de..78d5f948b0 100644 --- a/include/os/osLocale.h +++ b/include/os/osLocale.h @@ -28,9 +28,9 @@ extern "C" { #define setlocale SETLOCALE_FUNC_TAOS_FORBID #endif -char *taosCharsetReplace(char *charsetstr); -void taosGetSystemLocale(char *outLocale, char *outCharset); -int32_t taosSetSystemLocale(const char *inLocale, const char *inCharSet); +char *taosCharsetReplace(char *charsetstr); +void taosGetSystemLocale(char *outLocale, char *outCharset); +int32_t taosSetSystemLocale(const char *inLocale); #ifdef __cplusplus } diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 49acf285ee..5d95ff308c 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -26,8 +26,8 @@ #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID -#define inet_addr INET_ADDR_FUNC_TAOS_FORBID #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID +#define inet_addr INET_ADDR_FUNC_TAOS_FORBID #endif #if defined(WINDOWS) @@ -55,10 +55,10 @@ #define __BIG_ENDIAN BIG_ENDIAN #define __LITTLE_ENDIAN LITTLE_ENDIAN #define __PDP_ENDIAN PDP_ENDIAN - #else #include #include +#include #if defined(_TD_DARWIN_64) #include @@ -162,10 +162,10 @@ int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *a int32_t taosBlockSIGPIPE(); int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip); int32_t taosGetFqdn(char *); -void tinet_ntoa(char *ipstr, uint32_t ip); -uint32_t ip2uint(const char *const ip_addr); +void taosInetNtoa(char *ipstr, uint32_t ip); +uint32_t taosInetAddr(const char *ipstr); int32_t taosIgnSIGPIPE(); -const char *taosInetNtoa(struct in_addr ipInt, char *dstStr, int32_t len); +const char *taosInetNtop(struct in_addr ipInt, char *dstStr, int32_t len); uint64_t taosHton64(uint64_t val); uint64_t taosNtoh64(uint64_t val); diff --git a/include/os/osString.h b/include/os/osString.h index 995aa4a591..a841771a46 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -22,12 +22,24 @@ extern "C" { typedef wchar_t TdWchar; typedef int32_t TdUcs4; -#if !defined(DISALLOW_NCHAR_WITHOUT_ICONV) && defined(DARWIN) +#if !defined(DISALLOW_NCHAR_WITHOUT_ICONV)// && defined(DARWIN) #include "iconv.h" #else -typedef void *iconv_t; +typedef void *iconv_t; #endif -typedef enum { M2C = 0, C2M } ConvType; +typedef enum { M2C = 0, C2M, CM_NUM } ConvType; + +typedef struct { + iconv_t conv; + int8_t inUse; +} SConv; + +typedef struct { + SConv *gConv[CM_NUM]; + int32_t convUsed[CM_NUM]; + int32_t gConvMaxNum[CM_NUM]; + char charset[TD_CHARSET_LEN]; +} SConvInfo; // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. @@ -55,36 +67,39 @@ typedef enum { M2C = 0, C2M } ConvType; #ifdef strndup #undef strndup #endif -#define strndup STR_TO_F_FUNC_TAOS_FORBID +#define strndup STR_TO_F_FUNC_TAOS_FORBID #endif #define tstrncpy(dst, src, size) \ do { \ (void)strncpy((dst), (src), (size)); \ - (dst)[(size) - 1] = 0; \ + (dst)[(size)-1] = 0; \ } while (0) int64_t tsnprintf(char *dst, int64_t size, const char *format, ...); -#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src)) +#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src)) #define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size)) -#define TAOS_STRCAT(_dst, _src) ((void)strcat(_dst, _src)) -#define TAOS_STRNCAT(_dst, _src, len) ((void)strncat(_dst, _src, len)) +#define TAOS_STRCAT(_dst, _src) ((void)strcat(_dst, _src)) +#define TAOS_STRNCAT(_dst, _src, len) ((void)strncat(_dst, _src, len)) char *tstrdup(const char *src); int32_t taosUcs4len(TdUcs4 *ucs4); int32_t taosStr2int64(const char *str, int64_t *val); -int32_t taosStr2int16(const char *str, int16_t *val); int32_t taosStr2int32(const char *str, int32_t *val); +int32_t taosStr2int16(const char *str, int16_t *val); int32_t taosStr2int8(const char *str, int8_t *val); -int32_t taosConvInit(void); -void taosConvDestroy(); -iconv_t taosAcquireConv(int32_t *idx, ConvType type); -void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type); -int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs); +int32_t taosStr2Uint64(const char *str, uint64_t *val); +int32_t taosStr2Uint32(const char *str, uint32_t *val); +int32_t taosStr2Uint16(const char *str, uint16_t *val); +int32_t taosStr2Uint8(const char *str, uint8_t *val); + +iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt); +void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetCxt); +int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt); int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv); -bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len); +bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt); int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); bool taosValidateEncodec(const char *encodec); @@ -112,9 +127,9 @@ float taosStr2Float(const char *str, char **pEnd); int32_t taosHex2Ascii(const char *z, uint32_t n, void **data, uint32_t *size); int32_t taosAscii2Hex(const char *z, uint32_t n, void **data, uint32_t *size); char *taosStrndup(const char *s, int n); -//int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size); -bool isHex(const char* z, uint32_t n); -bool isValidateHex(const char* z, uint32_t n); +// int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size); +bool isHex(const char *z, uint32_t n); +bool isValidateHex(const char *z, uint32_t n); #ifdef __cplusplus } diff --git a/include/os/osTime.h b/include/os/osTime.h index 7a65efe28d..69ae8f5e2e 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -24,6 +24,7 @@ extern "C" { // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define strptime STRPTIME_FUNC_TAOS_FORBID +#define strftime STRFTIME_FUNC_TAOS_FORBID #define gettimeofday GETTIMEOFDAY_FUNC_TAOS_FORBID #define localtime LOCALTIME_FUNC_TAOS_FORBID #define localtime_s LOCALTIMES_FUNC_TAOS_FORBID @@ -42,6 +43,7 @@ extern "C" { #define MILLISECOND_PER_SECOND ((int64_t)1000LL) #endif +#include "osTimezone.h" #define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60) #define MILLISECOND_PER_HOUR (MILLISECOND_PER_MINUTE * 60) #define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24) @@ -91,13 +93,17 @@ static FORCE_INLINE int64_t taosGetMonoTimestampMs() { } char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm); -struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf, int32_t bufSize); -struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst); +size_t taosStrfTime(char *s, size_t maxsize, char const *format, struct tm const *t); +struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf, int32_t bufSize, timezone_t tz); +struct tm *taosGmTimeR(const time_t *timep, struct tm *result); +time_t taosTimeGm(struct tm *tmp); int32_t taosTime(time_t *t); -time_t taosMktime(struct tm *timep); +time_t taosMktime(struct tm *timep, timezone_t tz); int64_t user_mktime64(const uint32_t year, const uint32_t mon, const uint32_t day, const uint32_t hour, const uint32_t min, const uint32_t sec, int64_t time_zone); +//struct tm *taosLocalTimeRz(timezone_t state, const time_t *timep, struct tm *result); +//time_t taosMktimeRz(timezone_t state, struct tm *timep); #ifdef __cplusplus } #endif diff --git a/include/os/osTimezone.h b/include/os/osTimezone.h index 2a8d5a442d..f770a9dba0 100644 --- a/include/os/osTimezone.h +++ b/include/os/osTimezone.h @@ -20,43 +20,29 @@ extern "C" { #endif -// If the error is in a third-party library, place this header file under the third-party library header file. -// When you want to use this feature, you should find or add the same function in the following section. -#ifndef ALLOW_FORBID_FUNC -#define tzset TZSET_FUNC_TAOS_FORBID +#define TdEastZone8 8*60*60 +#define TZ_UNKNOWN "n/a" + +extern void* pTimezoneNameMap; + +#ifdef WINDOWS +typedef void *timezone_t; +#else +typedef struct state *timezone_t; + +struct tm* localtime_rz(timezone_t , time_t const *, struct tm *); +time_t mktime_z(timezone_t, struct tm *); +timezone_t tzalloc(char const *); +void tzfree(timezone_t); +void getTimezoneStr(char *tz); + #endif -enum TdTimezone { - TdWestZone12 = -12, - TdWestZone11, - TdWestZone10, - TdWestZone9, - TdWestZone8, - TdWestZone7, - TdWestZone6, - TdWestZone5, - TdWestZone4, - TdWestZone3, - TdWestZone2, - TdWestZone1, - TdZeroZone, - TdEastZone1, - TdEastZone2, - TdEastZone3, - TdEastZone4, - TdEastZone5, - TdEastZone6, - TdEastZone7, - TdEastZone8, - TdEastZone9, - TdEastZone10, - TdEastZone11, - TdEastZone12 -}; - -int32_t taosGetSystemTimezone(char *outTimezone, enum TdTimezone *tsTimezone); -int32_t taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight, enum TdTimezone *tsTimezone); +int32_t taosGetLocalTimezoneOffset(); +int32_t taosGetSystemTimezone(char *outTimezone); +int32_t taosSetGlobalTimezone(const char *tz); +int32_t taosFormatTimezoneStr(time_t t, const char* tzStr, timezone_t sp, char *outTimezoneStr); #ifdef __cplusplus } #endif diff --git a/include/util/taoserror.h b/include/util/taoserror.h index d72e8855d5..3f3bd93284 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -210,6 +210,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0234) #define TSDB_CODE_TSC_FAIL_GENERATE_JSON TAOS_DEF_ERROR_CODE(0, 0x0235) #define TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR TAOS_DEF_ERROR_CODE(0, 0x0236) +#define TSDB_CODE_NOT_SUPPORTTED_IN_WINDOWS TAOS_DEF_ERROR_CODE(0, 0x0237) #define TSDB_CODE_TSC_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x02FF) // mnode-common @@ -481,7 +482,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_MNODE_STOPPED TAOS_DEF_ERROR_CODE(0, 0x042A) #define TSDB_CODE_DNODE_INVALID_COMPACT_TASKS TAOS_DEF_ERROR_CODE(0, 0x042B) -// anode +// anode #define TSDB_CODE_MND_ANODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0430) #define TSDB_CODE_MND_ANODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0431) #define TSDB_CODE_MND_ANODE_TOO_LONG_URL TAOS_DEF_ERROR_CODE(0, 0x0432) @@ -565,6 +566,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_VND_ARB_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0537) // internal #define TSDB_CODE_VND_WRITE_DISABLED TAOS_DEF_ERROR_CODE(0, 0x0538) // internal #define TSDB_CODE_VND_TTL_FLUSH_INCOMPLETION TAOS_DEF_ERROR_CODE(0, 0x0539) // internal +#define TSDB_CODE_VND_ALREADY_EXIST_BUT_NOT_MATCH TAOS_DEF_ERROR_CODE(0, 0x0540) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 3fc247982f..947ef67902 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -34,7 +34,8 @@ typedef enum { CFG_STYPE_APOLLO_URL, CFG_STYPE_ARG_LIST, CFG_STYPE_TAOS_OPTIONS, - CFG_STYPE_ALTER_CMD, + CFG_STYPE_ALTER_CLIENT_CMD, + CFG_STYPE_ALTER_SERVER_CMD, } ECfgSrcType; typedef enum { diff --git a/include/util/tconv.h b/include/util/tconv.h new file mode 100644 index 0000000000..c886b38f51 --- /dev/null +++ b/include/util/tconv.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TCONV_H +#define TDENGINE_TCONV_H + +#ifdef __cplusplus +extern "C" { +#endif + +//#include "osString.h" +// +//bool taosValidateEncodec(const char *encodec); +//int32_t taosUcs4len(TdUcs4 *ucs4); +void* taosConvInit(const char* charset); +void taosConvDestroy(); +//iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt); +//void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetCxt); +//int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt); +//int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv); +//bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt); +//int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); +//int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TCONV_H diff --git a/include/util/tdef.h b/include/util/tdef.h index 4c1b263755..99504f93c4 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -295,6 +295,8 @@ typedef enum ELogicConditionType { #define TSDB_MAX_JSON_KEY_LEN 256 #define TSDB_AUTH_LEN 16 +#define TSDB_PASSWORD_MIN_LEN 8 +#define TSDB_PASSWORD_MAX_LEN 16 #define TSDB_PASSWORD_LEN 32 #define TSDB_USET_PASSWORD_LEN 129 #define TSDB_VERSION_LEN 32 diff --git a/include/util/tencode.h b/include/util/tencode.h index b66e79fa60..270d9e75df 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -413,21 +413,26 @@ static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint3 static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) { TAOS_CHECK_RETURN(tDecodeBinary(pCoder, (uint8_t**)val, len)); - (*len) -= 1; + if (*len > 0) { // notice!!! *len maybe 0 + (*len) -= 1; + } return 0; } static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, char** val) { - uint32_t len; + uint32_t len = 0; return tDecodeCStrAndLen(pCoder, val, &len); } static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) { - char* pStr; - uint32_t len; + char* pStr = NULL; + uint32_t len = 0; TAOS_CHECK_RETURN(tDecodeCStrAndLen(pCoder, &pStr, &len)); - TAOS_MEMCPY(val, pStr, len + 1); + if (len < pCoder->size) { + TAOS_MEMCPY(val, pStr, len + 1); + } + return 0; } @@ -479,12 +484,14 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, u static FORCE_INLINE int32_t tDecodeCStrAndLenAlloc(SDecoder* pCoder, char** val, uint64_t* len) { TAOS_CHECK_RETURN(tDecodeBinaryAlloc(pCoder, (void**)val, len)); - (*len) -= 1; + if (*len > 0){ + (*len) -= 1; + } return 0; } static FORCE_INLINE int32_t tDecodeCStrAlloc(SDecoder* pCoder, char** val) { - uint64_t len; + uint64_t len = 0; return tDecodeCStrAndLenAlloc(pCoder, val, &len); } diff --git a/include/util/tutil.h b/include/util/tutil.h index c1b6b2505b..01977e27b6 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -48,11 +48,6 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]); int32_t tintToHex(uint64_t val, char hex[]); int32_t titoa(uint64_t val, size_t radix, char str[]); -char *taosIpStr(uint32_t ipInt); -uint32_t ip2uint(const char *const ip_addr); -void taosIp2String(uint32_t ip, char *str); -void taosIpPort2String(uint32_t ip, uint16_t port, char *str); - void *tmemmem(const char *haystack, int hlen, const char *needle, int nlen); int32_t parseCfgReal(const char *str, float *out); @@ -232,6 +227,11 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, #define TAOS_UNUSED(expr) (void)(expr) +bool taosIsBigChar(char c); +bool taosIsSmallChar(char c); +bool taosIsNumberChar(char c); +bool taosIsSpecialChar(char c); + #ifdef __cplusplus } #endif diff --git a/packaging/delete_ref_lock.py b/packaging/delete_ref_lock.py index cf0e4cdd05..7200246a8a 100644 --- a/packaging/delete_ref_lock.py +++ b/packaging/delete_ref_lock.py @@ -1,59 +1,80 @@ import subprocess import re -# 执行 git fetch 命令并捕获输出 def git_fetch(): result = subprocess.run(['git', 'fetch'], capture_output=True, text=True) return result -# 解析分支名称 +def git_prune(): + # git remote prune origin + print("git remote prune origin") + result = subprocess.run(['git', 'remote', 'prune', 'origin'], capture_output=True, text=True) + return result + def parse_branch_name_type1(error_output): - # 使用正则表达式匹配 'is at' 前的分支名称 + # error: cannot lock ref 'refs/remotes/origin/fix/3.0/TD-32817': is at 7af5 but expected eaba + # match the branch name before ‘is at’ with a regular expression match = re.search(r"error: cannot lock ref '(refs/remotes/origin/[^']+)': is at", error_output) if match: return match.group(1) return None -# 解析第二种错误中的分支名称 def parse_branch_name_type2(error_output): - # 使用正则表达式匹配 'exists' 前的第一个引号内的分支名称 + # match the branch name before ‘exists; cannot create’ with a regular expression match = re.search(r"'(refs/remotes/origin/[^']+)' exists;", error_output) if match: return match.group(1) return None -# 执行 git update-ref -d 命令 +# parse branch name from error output of git remote prune origin +def parse_branch_name_type3(error_output): + # match the branch name before the first single quote before 'Unable to' with a regular expression + # git error: could not delete references: cannot lock ref 'refs/remotes/origin/test/3.0/TS-4893': Unable to create 'D:/workspace/main/TDinternal/community/.git/refs/remotes/origin/test/3.0/TS-4893.lock': File exists + match = re.search(r"references: cannot lock ref '(refs/remotes/origin/[^']+)': Unable to", error_output) + if match: + return match.group(1) + return None + + +# execute git update-ref -d to delete the ref def git_update_ref(branch_name): if branch_name: subprocess.run(['git', 'update-ref', '-d', f'{branch_name}'], check=True) -# 解析错误类型并执行相应的修复操作 +# parse error type and execute corresponding repair operation def handle_error(error_output): - # 错误类型1：本地引用的提交ID与远程不一致 - if "is at" in error_output and "but expected" in error_output: - branch_name = parse_branch_name_type1(error_output) - if branch_name: - print(f"Detected error type 1, attempting to delete ref for branch: {branch_name}") - git_update_ref(branch_name) - else: - print("Error parsing branch name for type 1.") - # 错误类型2：尝试创建新的远程引用时，本地已经存在同名的引用 - elif "exists; cannot create" in error_output: - branch_name = parse_branch_name_type2(error_output) - if branch_name: - print(f"Detected error type 2, attempting to delete ref for branch: {branch_name}") - git_update_ref(branch_name) - else: - print("Error parsing branch name for type 2.") + error_types = [ + ("is at", "but expected", parse_branch_name_type1, "type 1"), + ("exists; cannot create", None, parse_branch_name_type2, "type 2"), + ("Unable to create", "File exists", parse_branch_name_type3, "type 3") + ] + + for error_type in error_types: + if error_type[0] in error_output and (error_type[1] is None or error_type[1] in error_output): + branch_name = error_type[2](error_output) + if branch_name: + print(f"Detected error {error_type[3]}, attempting to delete ref for branch: {branch_name}") + git_update_ref(branch_name) + else: + print(f"Error parsing branch name for {error_type[3]}.") + break -# 主函数 def main(): fetch_result = git_fetch() - if fetch_result.returncode != 0: # 如果 git fetch 命令失败 + if fetch_result.returncode != 0: error_output = fetch_result.stderr handle_error(error_output) else: print("Git fetch successful.") + prune_result = git_prune() + print(prune_result.returncode) + if prune_result.returncode != 0: + error_output = prune_result.stderr + print(error_output) + handle_error(error_output) + else: + print("Git prune successful.") + if __name__ == "__main__": main() \ No newline at end of file diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 0874433e94..0bc82cdbd1 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -52,6 +52,7 @@ else installDir="/usr/local/taos" fi fi + install_main_dir=${installDir} bin_dir="${installDir}/bin" cfg_dir="${installDir}/cfg" diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index da56dcf75e..bc80cb2673 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -153,6 +153,13 @@ typedef struct { __taos_notify_fn_t fp; } SWhiteListInfo; +typedef struct { + timezone_t timezone; + void *charsetCxt; + char userApp[TSDB_APP_NAME_LEN]; + uint32_t userIp; +}SOptionInfo; + typedef struct STscObj { char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; @@ -175,6 +182,7 @@ typedef struct STscObj { SPassInfo passInfo; SWhiteListInfo whiteListInfo; STscNotifyInfo userDroppedInfo; + SOptionInfo optionInfo; } STscObj; typedef struct STscDbg { @@ -211,6 +219,7 @@ typedef struct SReqResultInfo { int32_t precision; int32_t payloadLen; char* convertJson; + void* charsetCxt; } SReqResultInfo; typedef struct SRequestSendRecvBody { @@ -338,6 +347,7 @@ extern int32_t clientReqRefPool; extern int32_t clientConnRefPool; extern int32_t timestampDeltaLimit; extern int64_t lastClusterId; +extern SHashObj* pTimezoneMap; __async_send_cb_fn_t getMsgRspHandle(int32_t msgType); @@ -437,6 +447,9 @@ void stopAllQueries(SRequestObj* pRequest); void doRequestCallback(SRequestObj* pRequest, int32_t code); void freeQueryParam(SSyncQueryParam* param); +int32_t tzInit(); +void tzCleanup(); + #ifdef TD_ENTERPRISE int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effeciveUser, SParseSqlRes* pRes); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 072cbee2f4..a3ffa7f3c2 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -36,6 +36,7 @@ #include "tsched.h" #include "ttime.h" #include "tversion.h" +#include "tconv.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -74,6 +75,7 @@ int64_t lastClusterId = 0; int32_t clientReqRefPool = -1; int32_t clientConnRefPool = -1; int32_t clientStop = -1; +SHashObj* pTimezoneMap = NULL; int32_t timestampDeltaLimit = 900; // s @@ -559,6 +561,7 @@ int32_t createRequest(uint64_t connId, int32_t type, int64_t reqid, SRequestObj (*pRequest)->metric.start = taosGetTimestampUs(); (*pRequest)->body.resInfo.convertUcs4 = true; // convert ucs4 by default + (*pRequest)->body.resInfo.charsetCxt = pTscObj->optionInfo.charsetCxt; (*pRequest)->type = type; (*pRequest)->allocatorRefId = -1; @@ -956,25 +959,31 @@ void taos_init_imp(void) { return; } taosHashSetFreeFp(appInfo.pInstMap, destroyAppInst); - deltaToUtcInitOnce(); const char *logName = CUS_PROMPT "slog"; ENV_ERR_RET(taosInitLogOutput(&logName), "failed to init log output"); if (taosCreateLog(logName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { (void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logName, strerror(errno), configDir); - tscInitRes = -1; + tscInitRes = terrno; return; } ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1), "failed to init cfg"); initQueryModuleMsgHandle(); - ENV_ERR_RET(taosConvInit(), "failed to init conv"); + if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL){ + tscInitRes = terrno; + tscError("failed to init conv"); + return; + } +#ifndef WINDOWS + ENV_ERR_RET(tzInit(), "failed to init timezone"); +#endif ENV_ERR_RET(monitorInit(), "failed to init monitor"); ENV_ERR_RET(rpcInit(), "failed to init rpc"); if (InitRegexCache() != 0) { - tscInitRes = -1; + tscInitRes = terrno; (void)printf("failed to init regex cache\n"); return; } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 07be4bb596..23b3b0315f 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -1194,6 +1194,9 @@ int32_t hbGatherAllInfo(SAppHbMgr *pAppHbMgr, SClientHbBatchReq **pBatchReq) { continue; } + tstrncpy(pOneReq->userApp, pTscObj->optionInfo.userApp, sizeof(pOneReq->userApp)); + pOneReq->userIp = pTscObj->optionInfo.userIp; + pOneReq = taosArrayPush((*pBatchReq)->reqs, pOneReq); if (NULL == pOneReq) { releaseTscObj(connKey->tscRid); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 2c98ecac1b..e5e304bc79 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -300,7 +300,9 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC .svrVer = pTscObj->sVer, .nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes), .isStmtBind = pRequest->isStmtBind, - .setQueryFp = setQueryRequest}; + .setQueryFp = setQueryRequest, + .timezone = pTscObj->optionInfo.timezone, + .charsetCxt = pTscObj->optionInfo.charsetCxt,}; cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog); @@ -331,7 +333,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { SRetrieveTableRsp* pRsp = NULL; int8_t biMode = atomic_load_8(&pRequest->pTscObj->biMode); - int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp, biMode); + int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp, biMode, pRequest->pTscObj->optionInfo.charsetCxt); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4); } @@ -369,7 +371,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { } int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp, - atomic_load_8(&pRequest->pTscObj->biMode)); + atomic_load_8(&pRequest->pTscObj->biMode), pRequest->pTscObj->optionInfo.charsetCxt); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4); } @@ -507,6 +509,7 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pUser = pRequest->pTscObj->user, + .timezone = pRequest->pTscObj->optionInfo.timezone, .sysInfo = pRequest->pTscObj->sysInfo}; return qCreateQueryPlan(&cxt, pPlan, pNodeList); @@ -1361,6 +1364,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pUser = pRequest->pTscObj->user, .sysInfo = pRequest->pTscObj->sysInfo, + .timezone = pRequest->pTscObj->optionInfo.timezone, .allocatorId = pRequest->allocatorRefId}; if (TSDB_CODE_SUCCESS == code) { code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); @@ -2086,7 +2090,7 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) { int32_t idx = -1; - iconv_t conv = taosAcquireConv(&idx, C2M); + iconv_t conv = taosAcquireConv(&idx, C2M, pResultInfo->charsetCxt); if (conv == (iconv_t)-1) return TSDB_CODE_TSC_INTERNAL_ERROR; for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { @@ -2096,7 +2100,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) { if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) { char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); if (p == NULL) { - taosReleaseConv(idx, conv, C2M); + taosReleaseConv(idx, conv, C2M, pResultInfo->charsetCxt); return terrno; } @@ -2113,7 +2117,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) { "doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + " "colLength[i]):%p", len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i])); - taosReleaseConv(idx, conv, C2M); + taosReleaseConv(idx, conv, C2M, pResultInfo->charsetCxt); return TSDB_CODE_TSC_INTERNAL_ERROR; } @@ -2127,7 +2131,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) { pResultInfo->row[i] = pResultInfo->pCol[i].pData; } } - taosReleaseConv(idx, conv, C2M); + taosReleaseConv(idx, conv, C2M, pResultInfo->charsetCxt); return TSDB_CODE_SUCCESS; } @@ -2292,7 +2296,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo) { varDataSetLen(dst, strlen(varDataVal(dst))); } else if (tTagIsJson(data)) { char* jsonString = NULL; - parseTagDatatoJson(data, &jsonString); + parseTagDatatoJson(data, &jsonString, pResultInfo->charsetCxt); if (jsonString == NULL) { tscError("doConvertJson error: parseTagDatatoJson failed"); return terrno; @@ -2302,9 +2306,10 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo) { } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" *(char*)varDataVal(dst) = '\"'; int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), - varDataVal(dst) + CHAR_BYTES); + varDataVal(dst) + CHAR_BYTES, pResultInfo->charsetCxt); if (length <= 0) { - tscError("charset:%s to %s. convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset); + tscError("charset:%s to %s. convert failed.", DEFAULT_UNICODE_ENCODEC, + pResultInfo->charsetCxt != NULL ? ((SConvInfo *)(pResultInfo->charsetCxt))->charset : tsCharset); length = 0; } varDataSetLen(dst, length + CHAR_BYTES * 2); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 1f41a9b0b0..d80c7df2e4 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -30,6 +30,7 @@ #include "tref.h" #include "trpc.h" #include "version.h" +#include "tconv.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -38,11 +39,13 @@ static int32_t sentinel = TSC_VAR_NOT_RELEASE; static int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt, SSqlCallbackWrapper *pWrapper); int taos_options(TSDB_OPTION option, const void *arg, ...) { + if (arg == NULL) { + return TSDB_CODE_INVALID_PARA; + } static int32_t lock = 0; for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) { if (i % 1000 == 0) { - tscInfo("haven't acquire lock after spin %d times.", i); (void)sched_yield(); } } @@ -51,6 +54,167 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) { atomic_store_32(&lock, 0); return ret; } + +#ifndef WINDOWS +static void freeTz(void *p){ + timezone_t tz = *(timezone_t *)p; + tzfree(tz); +} + +int32_t tzInit(){ + pTimezoneMap = taosHashInit(0, MurmurHash3_32, false, HASH_ENTRY_LOCK); + if (pTimezoneMap == NULL) { + return terrno; + } + taosHashSetFreeFp(pTimezoneMap, freeTz); + + pTimezoneNameMap = taosHashInit(0, taosIntHash_64, false, HASH_ENTRY_LOCK); + if (pTimezoneNameMap == NULL) { + return terrno; + } + return 0; +} + +void tzCleanup(){ + taosHashCleanup(pTimezoneMap); + taosHashCleanup(pTimezoneNameMap); +} + +static timezone_t setConnnectionTz(const char* val){ + timezone_t tz = NULL; + timezone_t *tmp = taosHashGet(pTimezoneMap, val, strlen(val)); + if (tmp != NULL && *tmp != NULL){ + tz = *tmp; + goto END; + } + + tscDebug("set timezone to %s", val); + tz = tzalloc(val); + if (tz == NULL) { + tscWarn("%s unknown timezone %s change to UTC", __func__, val); + tz = tzalloc("UTC"); + if (tz == NULL) { + tscError("%s set timezone UTC error", __func__); + terrno = TAOS_SYSTEM_ERROR(errno); + goto END; + } + } + int32_t code = taosHashPut(pTimezoneMap, val, strlen(val), &tz, sizeof(timezone_t)); + if (code != 0){ + tscError("%s put timezone to tz map error:%d", __func__, code); + tzfree(tz); + tz = NULL; + goto END; + } + + time_t tx1 = taosGetTimestampSec(); + char output[TD_TIMEZONE_LEN] = {0}; + taosFormatTimezoneStr(tx1, val, tz, output); + code = taosHashPut(pTimezoneNameMap, &tz, sizeof(timezone_t), output, strlen(output) + 1); + if (code != 0){ + tscError("failed to put timezone %s to map", val); + } + +END: + return tz; +} +#endif + +static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, const char* val){ + if (taos == NULL) { + return TSDB_CODE_INVALID_PARA; + } + +#ifdef WINDOWS + if (option == TSDB_OPTION_CONNECTION_TIMEZONE){ + return TSDB_CODE_NOT_SUPPORTTED_IN_WINDOWS; + } +#endif + + if (option < TSDB_OPTION_CONNECTION_CLEAR || option >= TSDB_MAX_OPTIONS_CONNECTION){ + return TSDB_CODE_INVALID_PARA; + } + + int32_t code = taos_init(); + // initialize global config + if (code != 0) { + return code; + } + + STscObj *pObj = acquireTscObj(*(int64_t *)taos); + if (NULL == pObj) { + tscError("invalid parameter for %s", __func__); + return terrno; + } + + if (option == TSDB_OPTION_CONNECTION_CLEAR){ + val = NULL; + } + + if (option == TSDB_OPTION_CONNECTION_CHARSET || option == TSDB_OPTION_CONNECTION_CLEAR) { + if (val != NULL) { + if (!taosValidateEncodec(val)) { + code = terrno; + goto END; + } + void *tmp = taosConvInit(val); + if (tmp == NULL) { + code = terrno; + goto END; + } + pObj->optionInfo.charsetCxt = tmp; + }else{ + pObj->optionInfo.charsetCxt = NULL; + } + } + + if (option == TSDB_OPTION_CONNECTION_TIMEZONE || option == TSDB_OPTION_CONNECTION_CLEAR) { +#ifndef WINDOWS + if (val != NULL){ + if (val[0] == 0){ + val = "UTC"; + } + timezone_t tz = setConnnectionTz(val); + if (tz == NULL){ + code = terrno; + goto END; + } + pObj->optionInfo.timezone = tz; + } else { + pObj->optionInfo.timezone = NULL; + } +#endif + } + + if (option == TSDB_OPTION_CONNECTION_USER_APP || option == TSDB_OPTION_CONNECTION_CLEAR) { + if (val != NULL) { + tstrncpy(pObj->optionInfo.userApp, val, sizeof(pObj->optionInfo.userApp)); + } else { + pObj->optionInfo.userApp[0] = 0; + } + } + + if (option == TSDB_OPTION_CONNECTION_USER_IP || option == TSDB_OPTION_CONNECTION_CLEAR) { + if (val != NULL) { + pObj->optionInfo.userIp = taosInetAddr(val); + if (pObj->optionInfo.userIp == INADDR_NONE){ + code = TSDB_CODE_INVALID_PARA; + goto END; + } + } else { + pObj->optionInfo.userIp = INADDR_NONE; + } + } + +END: + releaseTscObj(*(int64_t *)taos); + return code; +} + +int taos_options_connection(TAOS *taos, TSDB_OPTION_CONNECTION option, const void *arg, ...){ + return setConnectionOption(taos, option, (const char *)arg); +} + // this function may be called by user or system, or by both simultaneously. void taos_cleanup(void) { tscDebug("start to cleanup client environment"); @@ -73,6 +237,9 @@ void taos_cleanup(void) { tscWarn("failed to cleanup task queue"); } +#ifndef WINDOWS + tzCleanup(); +#endif tmqMgmtClose(); int32_t id = clientReqRefPool; @@ -1244,7 +1411,9 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt, SS .allocatorId = pRequest->allocatorRefId, .parseSqlFp = clientParseSql, .parseSqlParam = pWrapper, - .setQueryFp = setQueryRequest}; + .setQueryFp = setQueryRequest, + .timezone = pTscObj->optionInfo.timezone, + .charsetCxt = pTscObj->optionInfo.charsetCxt}; int8_t biMode = atomic_load_8(&((STscObj *)pTscObj)->biMode); (*pCxt)->biMode = biMode; return TSDB_CODE_SUCCESS; diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 3a23d38375..76aec26ff4 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -52,6 +52,22 @@ #define TMQ_META_VERSION "1.0" +static bool tmqAddJsonObjectItem(cJSON *object, const char *string, cJSON *item){ + bool ret = cJSON_AddItemToObject(object, string, item); + if (!ret){ + cJSON_Delete(item); + } + return ret; +} +static bool tmqAddJsonArrayItem(cJSON *array, cJSON *item){ + bool ret = cJSON_AddItemToArray(array, item); + if (!ret){ + cJSON_Delete(item); + } + return ret; +} + + static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen); static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); } static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t, @@ -68,41 +84,43 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche cJSON* type = cJSON_CreateString("create"); RAW_NULL_CHECK(type); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "type", type)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "type", type)); cJSON* tableType = cJSON_CreateString(t == TSDB_NORMAL_TABLE ? "normal" : "super"); RAW_NULL_CHECK(tableType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableType", tableType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableType", tableType)); cJSON* tableName = cJSON_CreateString(name); RAW_NULL_CHECK(tableName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableName", tableName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableName", tableName)); cJSON* columns = cJSON_CreateArray(); RAW_NULL_CHECK(columns); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "columns", columns)); + for (int i = 0; i < schemaRow->nCols; i++) { cJSON* column = cJSON_CreateObject(); RAW_NULL_CHECK(column); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(columns, column)); SSchema* s = schemaRow->pSchema + i; cJSON* cname = cJSON_CreateString(s->name); RAW_NULL_CHECK(cname); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "name", cname)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "name", cname)); cJSON* ctype = cJSON_CreateNumber(s->type); RAW_NULL_CHECK(ctype); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "type", ctype)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "type", ctype)); if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = s->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "length", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "length", cbytes)); } else if (s->type == TSDB_DATA_TYPE_NCHAR) { int32_t length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "length", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "length", cbytes)); } cJSON* isPk = cJSON_CreateBool(s->flags & COL_IS_KEY); RAW_NULL_CHECK(isPk); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "isPrimarykey", isPk)); - RAW_FALSE_CHECK(cJSON_AddItemToArray(columns, column)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "isPrimarykey", isPk)); if (pColCmprRow == NULL) { continue; @@ -124,44 +142,44 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche cJSON* encodeJson = cJSON_CreateString(encode); RAW_NULL_CHECK(encodeJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "encode", encodeJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "encode", encodeJson)); cJSON* compressJson = cJSON_CreateString(compress); RAW_NULL_CHECK(compressJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "compress", compressJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "compress", compressJson)); cJSON* levelJson = cJSON_CreateString(level); RAW_NULL_CHECK(levelJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(column, "level", levelJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(column, "level", levelJson)); } - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "columns", columns)); cJSON* tags = cJSON_CreateArray(); RAW_NULL_CHECK(tags); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tags", tags)); + for (int i = 0; schemaTag && i < schemaTag->nCols; i++) { cJSON* tag = cJSON_CreateObject(); RAW_NULL_CHECK(tag); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(tags, tag)); SSchema* s = schemaTag->pSchema + i; cJSON* tname = cJSON_CreateString(s->name); RAW_NULL_CHECK(tname); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "name", tname)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "name", tname)); cJSON* ttype = cJSON_CreateNumber(s->type); RAW_NULL_CHECK(ttype); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "type", ttype)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "type", ttype)); if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = s->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "length", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "length", cbytes)); } else if (s->type == TSDB_DATA_TYPE_NCHAR) { int32_t length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "length", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "length", cbytes)); } - RAW_FALSE_CHECK(cJSON_AddItemToArray(tags, tag)); } - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tags", tags)); end: *pJson = json; @@ -175,7 +193,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) { RAW_NULL_CHECK(encodeStr); cJSON* encodeJson = cJSON_CreateString(encodeStr); RAW_NULL_CHECK(encodeJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "encode", encodeJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "encode", encodeJson)); return code; } uint8_t compress = COMPRESS_L2_TYPE_U32(para); @@ -184,7 +202,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) { RAW_NULL_CHECK(compressStr); cJSON* compressJson = cJSON_CreateString(compressStr); RAW_NULL_CHECK(compressJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "compress", compressJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "compress", compressJson)); return code; } uint8_t level = COMPRESS_L2_TYPE_LEVEL_U32(para); @@ -193,7 +211,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) { RAW_NULL_CHECK(levelStr); cJSON* levelJson = cJSON_CreateString(levelStr); RAW_NULL_CHECK(levelJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "level", levelJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "level", levelJson)); return code; } @@ -214,19 +232,19 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** RAW_NULL_CHECK(json); cJSON* type = cJSON_CreateString("alter"); RAW_NULL_CHECK(type); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "type", type)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "type", type)); SName name = {0}; RAW_RETURN_CHECK(tNameFromString(&name, req.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)); cJSON* tableType = cJSON_CreateString("super"); RAW_NULL_CHECK(tableType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableType", tableType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableType", tableType)); cJSON* tableName = cJSON_CreateString(name.tname); RAW_NULL_CHECK(tableName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableName", tableName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableName", tableName)); cJSON* alterType = cJSON_CreateNumber(req.alterType); RAW_NULL_CHECK(alterType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "alterType", alterType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "alterType", alterType)); switch (req.alterType) { case TSDB_ALTER_TABLE_ADD_TAG: case TSDB_ALTER_TABLE_ADD_COLUMN: { @@ -234,22 +252,22 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** RAW_NULL_CHECK(field); cJSON* colName = cJSON_CreateString(field->name); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colType = cJSON_CreateNumber(field->type); RAW_NULL_CHECK(colType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colType", colType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colType", colType)); if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } else if (field->type == TSDB_DATA_TYPE_NCHAR) { int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } break; } @@ -258,22 +276,22 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** RAW_NULL_CHECK(field); cJSON* colName = cJSON_CreateString(field->name); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colType = cJSON_CreateNumber(field->type); RAW_NULL_CHECK(colType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colType", colType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colType", colType)); if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } else if (field->type == TSDB_DATA_TYPE_NCHAR) { int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } RAW_RETURN_CHECK(setCompressOption(json, field->compress)); break; @@ -284,7 +302,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** RAW_NULL_CHECK(field); cJSON* colName = cJSON_CreateString(field->name); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); break; } case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: @@ -293,21 +311,21 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** RAW_NULL_CHECK(field); cJSON* colName = cJSON_CreateString(field->name); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colType = cJSON_CreateNumber(field->type); RAW_NULL_CHECK(colType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colType", colType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colType", colType)); if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } else if (field->type == TSDB_DATA_TYPE_NCHAR) { int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } break; } @@ -319,10 +337,10 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** RAW_NULL_CHECK(newField); cJSON* colName = cJSON_CreateString(oldField->name); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colNewName = cJSON_CreateString(newField->name); RAW_NULL_CHECK(colNewName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colNewName", colNewName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colNewName", colNewName)); break; } case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: { @@ -330,7 +348,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** RAW_NULL_CHECK(field); cJSON* colName = cJSON_CreateString(field->name); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); RAW_RETURN_CHECK(setCompressOption(json, field->bytes)); break; } @@ -391,51 +409,47 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { int64_t id = pCreateReq->uid; uint8_t tagNum = pCreateReq->ctb.tagNum; int32_t code = 0; - cJSON* tags = NULL; + SArray* pTagVals = NULL; + char* pJson = NULL; + cJSON* tableName = cJSON_CreateString(name); RAW_NULL_CHECK(tableName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableName", tableName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableName", tableName)); cJSON* using = cJSON_CreateString(sname); RAW_NULL_CHECK(using); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "using", using)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "using", using)); cJSON* tagNumJson = cJSON_CreateNumber(tagNum); RAW_NULL_CHECK(tagNumJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tagNum", tagNumJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tagNum", tagNumJson)); - tags = cJSON_CreateArray(); + cJSON* tags = cJSON_CreateArray(); RAW_NULL_CHECK(tags); - SArray* pTagVals = NULL; + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tags", tags)); RAW_RETURN_CHECK(tTagToValArray(pTag, &pTagVals)); - if (tTagIsJson(pTag)) { STag* p = (STag*)pTag; if (p->nTag == 0) { uError("p->nTag == 0"); goto end; } - char* pJson = NULL; - parseTagDatatoJson(pTag, &pJson); - if (pJson == NULL) { - uError("parseTagDatatoJson failed, pJson == NULL"); - goto end; - } + parseTagDatatoJson(pTag, &pJson, NULL); + RAW_NULL_CHECK(pJson); cJSON* tag = cJSON_CreateObject(); RAW_NULL_CHECK(tag); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(tags, tag)); STagVal* pTagVal = taosArrayGet(pTagVals, 0); RAW_NULL_CHECK(pTagVal); char* ptname = taosArrayGet(tagName, 0); RAW_NULL_CHECK(ptname); cJSON* tname = cJSON_CreateString(ptname); RAW_NULL_CHECK(tname); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "name", tname)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "name", tname)); cJSON* ttype = cJSON_CreateNumber(TSDB_DATA_TYPE_JSON); RAW_NULL_CHECK(ttype); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "type", ttype)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "type", ttype)); cJSON* tvalue = cJSON_CreateString(pJson); RAW_NULL_CHECK(tvalue); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "value", tvalue)); - RAW_FALSE_CHECK(cJSON_AddItemToArray(tags, tag)); - taosMemoryFree(pJson); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "value", tvalue)); goto end; } @@ -444,36 +458,34 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { RAW_NULL_CHECK(pTagVal); cJSON* tag = cJSON_CreateObject(); RAW_NULL_CHECK(tag); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(tags, tag)); char* ptname = taosArrayGet(tagName, i); RAW_NULL_CHECK(ptname); cJSON* tname = cJSON_CreateString(ptname); RAW_NULL_CHECK(tname); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "name", tname)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "name", tname)); cJSON* ttype = cJSON_CreateNumber(pTagVal->type); RAW_NULL_CHECK(ttype); - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "type", ttype)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "type", ttype)); cJSON* tvalue = NULL; if (IS_VAR_DATA_TYPE(pTagVal->type)) { - char* buf = NULL; int64_t bufSize = 0; if (pTagVal->type == TSDB_DATA_TYPE_VARBINARY) { bufSize = pTagVal->nData * 2 + 2 + 3; } else { bufSize = pTagVal->nData + 3; } - buf = taosMemoryCalloc(bufSize, 1); - + char* buf = taosMemoryCalloc(bufSize, 1); RAW_NULL_CHECK(buf); - if (!buf) goto end; if (dataConverToStr(buf, bufSize, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL) != TSDB_CODE_SUCCESS) { taosMemoryFree(buf); goto end; } tvalue = cJSON_CreateString(buf); - RAW_NULL_CHECK(tvalue); taosMemoryFree(buf); + RAW_NULL_CHECK(tvalue); } else { double val = 0; GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64); @@ -481,12 +493,11 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { RAW_NULL_CHECK(tvalue); } - RAW_FALSE_CHECK(cJSON_AddItemToObject(tag, "value", tvalue)); - RAW_FALSE_CHECK(cJSON_AddItemToArray(tags, tag)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(tag, "value", tvalue)); } end: - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tags", tags)); + taosMemoryFree(pJson); taosArrayDestroy(pTagVals); } @@ -497,22 +508,23 @@ static void buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs, cJSO RAW_NULL_CHECK(json); cJSON* type = cJSON_CreateString("create"); RAW_NULL_CHECK(type); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "type", type)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "type", type)); cJSON* tableType = cJSON_CreateString("child"); RAW_NULL_CHECK(tableType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableType", tableType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableType", tableType)); buildChildElement(json, pCreateReq); cJSON* createList = cJSON_CreateArray(); RAW_NULL_CHECK(createList); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "createList", createList)); + for (int i = 0; nReqs > 1 && i < nReqs; i++) { cJSON* create = cJSON_CreateObject(); RAW_NULL_CHECK(create); buildChildElement(create, pCreateReq + i); - RAW_FALSE_CHECK(cJSON_AddItemToArray(createList, create)); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(createList, create)); } - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "createList", createList)); end: *pJson = json; @@ -619,62 +631,62 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { RAW_NULL_CHECK(json); cJSON* type = cJSON_CreateString("alter"); RAW_NULL_CHECK(type); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "type", type)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "type", type)); cJSON* tableType = cJSON_CreateString(vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_TAG_VAL || vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_MULTI_TAG_VAL ? "child" : "normal"); RAW_NULL_CHECK(tableType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableType", tableType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableType", tableType)); cJSON* tableName = cJSON_CreateString(vAlterTbReq.tbName); RAW_NULL_CHECK(tableName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableName", tableName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableName", tableName)); cJSON* alterType = cJSON_CreateNumber(vAlterTbReq.action); RAW_NULL_CHECK(alterType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "alterType", alterType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "alterType", alterType)); switch (vAlterTbReq.action) { case TSDB_ALTER_TABLE_ADD_COLUMN: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type); RAW_NULL_CHECK(colType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colType", colType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colType", colType)); if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_VARBINARY || vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR) { int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } break; } case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type); RAW_NULL_CHECK(colType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colType", colType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colType", colType)); if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_VARBINARY || vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR) { int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } RAW_RETURN_CHECK(setCompressOption(json, vAlterTbReq.compress)); break; @@ -682,43 +694,43 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { case TSDB_ALTER_TABLE_DROP_COLUMN: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); break; } case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colType = cJSON_CreateNumber(vAlterTbReq.colModType); RAW_NULL_CHECK(colType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colType", colType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colType", colType)); if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_VARBINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } else if (vAlterTbReq.colModType == TSDB_DATA_TYPE_NCHAR) { int32_t length = (vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); RAW_NULL_CHECK(cbytes); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colLength", cbytes)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colLength", cbytes)); } break; } case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); cJSON* colNewName = cJSON_CreateString(vAlterTbReq.colNewName); RAW_NULL_CHECK(colNewName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colNewName", colNewName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colNewName", colNewName)); break; } case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: { cJSON* tagName = cJSON_CreateString(vAlterTbReq.tagName); RAW_NULL_CHECK(tagName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", tagName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", tagName)); bool isNull = vAlterTbReq.isNull; if (vAlterTbReq.tagType == TSDB_DATA_TYPE_JSON) { @@ -733,7 +745,7 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { uError("processAlterTable isJson false"); goto end; } - parseTagDatatoJson(vAlterTbReq.pTagVal, &buf); + parseTagDatatoJson(vAlterTbReq.pTagVal, &buf, NULL); if (buf == NULL) { uError("parseTagDatatoJson failed, buf == NULL"); goto end; @@ -757,12 +769,12 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { cJSON* colValue = cJSON_CreateString(buf); taosMemoryFree(buf); RAW_NULL_CHECK(colValue); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colValue", colValue)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colValue", colValue)); } cJSON* isNullCJson = cJSON_CreateBool(isNull); RAW_NULL_CHECK(isNullCJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colValueNull", isNullCJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colValueNull", isNullCJson)); break; } case TSDB_ALTER_TABLE_UPDATE_MULTI_TAG_VAL: { @@ -774,14 +786,17 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { cJSON* tags = cJSON_CreateArray(); RAW_NULL_CHECK(tags); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tags", tags)); + for (int32_t i = 0; i < nTags; i++) { cJSON* member = cJSON_CreateObject(); RAW_NULL_CHECK(member); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(tags, member)); SMultiTagUpateVal* pTagVal = taosArrayGet(vAlterTbReq.pMultiTag, i); cJSON* tagName = cJSON_CreateString(pTagVal->tagName); RAW_NULL_CHECK(tagName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(member, "colName", tagName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(member, "colName", tagName)); if (pTagVal->tagType == TSDB_DATA_TYPE_JSON) { uError("processAlterTable isJson false"); @@ -789,14 +804,13 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { } bool isNull = pTagVal->isNull; if (!isNull) { - char* buf = NULL; int64_t bufSize = 0; if (pTagVal->tagType == TSDB_DATA_TYPE_VARBINARY) { bufSize = pTagVal->nTagVal * 2 + 2 + 3; } else { bufSize = pTagVal->nTagVal + 3; } - buf = taosMemoryCalloc(bufSize, 1); + char* buf = taosMemoryCalloc(bufSize, 1); RAW_NULL_CHECK(buf); if (dataConverToStr(buf, bufSize, pTagVal->tagType, pTagVal->pTagVal, pTagVal->nTagVal, NULL) != TSDB_CODE_SUCCESS) { @@ -806,21 +820,19 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { cJSON* colValue = cJSON_CreateString(buf); taosMemoryFree(buf); RAW_NULL_CHECK(colValue); - RAW_FALSE_CHECK(cJSON_AddItemToObject(member, "colValue", colValue)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(member, "colValue", colValue)); } cJSON* isNullCJson = cJSON_CreateBool(isNull); RAW_NULL_CHECK(isNullCJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(member, "colValueNull", isNullCJson)); - RAW_FALSE_CHECK(cJSON_AddItemToArray(tags, member)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(member, "colValueNull", isNullCJson)); } - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tags", tags)); break; } case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); RAW_NULL_CHECK(colName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "colName", colName)); RAW_RETURN_CHECK(setCompressOption(json, vAlterTbReq.compress)); break; } @@ -858,13 +870,13 @@ static void processDropSTable(SMqMetaRsp* metaRsp, cJSON** pJson) { RAW_NULL_CHECK(json); cJSON* type = cJSON_CreateString("drop"); RAW_NULL_CHECK(type); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "type", type)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "type", type)); cJSON* tableType = cJSON_CreateString("super"); RAW_NULL_CHECK(tableType); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableType", tableType)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableType", tableType)); cJSON* tableName = cJSON_CreateString(req.name); RAW_NULL_CHECK(tableName); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableName", tableName)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableName", tableName)); end: uDebug("processDropSTable return"); @@ -897,10 +909,10 @@ static void processDeleteTable(SMqMetaRsp* metaRsp, cJSON** pJson) { RAW_NULL_CHECK(json); cJSON* type = cJSON_CreateString("delete"); RAW_NULL_CHECK(type); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "type", type)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "type", type)); cJSON* sqlJson = cJSON_CreateString(sql); RAW_NULL_CHECK(sqlJson); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", sqlJson)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "sql", sqlJson)); end: uDebug("processDeleteTable return"); @@ -928,16 +940,17 @@ static void processDropTable(SMqMetaRsp* metaRsp, cJSON** pJson) { RAW_NULL_CHECK(json); cJSON* type = cJSON_CreateString("drop"); RAW_NULL_CHECK(type); - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "type", type)); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "type", type)); cJSON* tableNameList = cJSON_CreateArray(); RAW_NULL_CHECK(tableNameList); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(json, "tableNameList", tableNameList)); + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { SVDropTbReq* pDropTbReq = req.pReqs + iReq; cJSON* tableName = cJSON_CreateString(pDropTbReq->name); RAW_NULL_CHECK(tableName); - RAW_FALSE_CHECK(cJSON_AddItemToArray(tableNameList, tableName)); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(tableNameList, tableName)); } - RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableNameList", tableNameList)); end: uDebug("processDropTable return"); @@ -2183,6 +2196,8 @@ static void processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp, char** string) { RAW_FALSE_CHECK(cJSON_AddStringToObject(pJson, "tmq_meta_version", TMQ_META_VERSION)); cJSON* pMetaArr = cJSON_CreateArray(); RAW_NULL_CHECK(pMetaArr); + RAW_FALSE_CHECK(tmqAddJsonObjectItem(pJson, "metas", pMetaArr)); + int32_t num = taosArrayGetSize(rsp.batchMetaReq); for (int32_t i = 0; i < num; i++) { int32_t* len = taosArrayGet(rsp.batchMetaLen, i); @@ -2198,10 +2213,9 @@ static void processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp, char** string) { cJSON* pItem = NULL; processSimpleMeta(&metaRsp, &pItem); tDeleteMqMetaRsp(&metaRsp); - RAW_FALSE_CHECK(cJSON_AddItemToArray(pMetaArr, pItem)); + RAW_FALSE_CHECK(tmqAddJsonArrayItem(pMetaArr, pItem)); } - RAW_FALSE_CHECK(cJSON_AddItemToObject(pJson, "metas", pMetaArr)); tDeleteMqBatchMetaRsp(&rsp); char* fullStr = cJSON_PrintUnformatted(pJson); cJSON_Delete(pJson); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 0b1637bebf..cc96d61b18 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -267,7 +267,7 @@ bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { goto END; } // bind data - int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1); + int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1, info->taos->optionInfo.charsetCxt); if (unlikely(ret != TSDB_CODE_SUCCESS)) { uDebug("smlBuildCol error, retry"); goto END; @@ -411,8 +411,8 @@ int32_t smlParseEndTelnetJsonFormat(SSmlHandle *info, SSmlLineInfo *elements, SS int32_t code = 0; int32_t lino = 0; uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__ , kvTs->i); - SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0)); - SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1)); + SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0, info->taos->optionInfo.charsetCxt)); + SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1, info->taos->optionInfo.charsetCxt)); SML_CHECK_CODE(smlBuildRow(info->currTableDataCtx)); END: @@ -438,7 +438,7 @@ END: int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) { if (info->dataFormat) { uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__, kvTs->i); - int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); + int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0, info->taos->optionInfo.charsetCxt); if (ret == TSDB_CODE_SUCCESS) { ret = smlBuildRow(info->currTableDataCtx); } @@ -1486,7 +1486,7 @@ static int32_t smlInsertData(SSmlHandle *info) { SML_CHECK_CODE(smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, (*pMeta)->tableMeta, tableData->childTableName, measure, measureLen, info->ttl, info->msgBuf.buf, - info->msgBuf.len)); + info->msgBuf.len, info->taos->optionInfo.charsetCxt)); taosMemoryFreeClear(measure); oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index b1f5af4593..9cad219614 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -1071,7 +1071,7 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { tscDebug("start to bind stmt tag values"); STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName, pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, - pStmt->exec.pRequest->msgBufLen)); + pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt)); return TSDB_CODE_SUCCESS; } @@ -1239,7 +1239,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { } if (STMT_TYPE_QUERY == pStmt->sql.type) { - STMT_ERR_RET(qStmtBindParams(pStmt->sql.pQuery, bind, colIdx)); + STMT_ERR_RET(qStmtBindParams(pStmt->sql.pQuery, bind, colIdx, pStmt->taos->optionInfo.charsetCxt)); SParseContext ctx = {.requestId = pStmt->exec.pRequest->requestId, .acctId = pStmt->taos->acctId, @@ -1325,10 +1325,10 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { if (pStmt->sql.stbInterlaceMode) { (*pDataBlock)->pData->flags = 0; code = qBindStmtStbColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, - pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo); + pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo, pStmt->taos->optionInfo.charsetCxt); } else { code = - qBindStmtColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); + qBindStmtColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt); } if (code) { @@ -1353,7 +1353,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { } code = qBindStmtSingleColValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, - pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum); + pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum, pStmt->taos->optionInfo.charsetCxt); if (code) { tscError("qBindStmtSingleColValue failed, error:%s", tstrerror(code)); STMT_ERR_RET(code); diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index 2eedab2289..f03879f199 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1015,7 +1015,7 @@ int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) { tscDebug("start to bind stmt tag values"); STMT_ERR_RET(qBindStmtTagsValue2(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName, pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, - pStmt->exec.pRequest->msgBufLen)); + pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt)); return TSDB_CODE_SUCCESS; } @@ -1094,6 +1094,13 @@ static int stmtFetchStbColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIEL } STMT_ERR_RET(qBuildStmtStbColFields(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.preCtbname, fieldNum, fields)); + if (pStmt->bInfo.tbType == TSDB_SUPER_TABLE) { + pStmt->bInfo.needParse = true; + if (taosHashRemove(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)) != 0) { + tscError("get fileds %s remove exec blockHash fail", pStmt->bInfo.tbFName); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); + } + } return TSDB_CODE_SUCCESS; } @@ -1324,7 +1331,7 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) { } if (STMT_TYPE_QUERY == pStmt->sql.type) { - STMT_ERR_RET(qStmtBindParams2(pStmt->sql.pQuery, bind, colIdx)); + STMT_ERR_RET(qStmtBindParams2(pStmt->sql.pQuery, bind, colIdx, pStmt->taos->optionInfo.charsetCxt)); SParseContext ctx = {.requestId = pStmt->exec.pRequest->requestId, .acctId = pStmt->taos->acctId, @@ -1408,10 +1415,10 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) { if (pStmt->sql.stbInterlaceMode) { (*pDataBlock)->pData->flags = 0; code = qBindStmtStbColsValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, - pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo); + pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo, pStmt->taos->optionInfo.charsetCxt); } else { code = - qBindStmtColsValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); + qBindStmtColsValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt); } if (code) { @@ -1436,7 +1443,7 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) { } code = qBindStmtSingleColValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, - pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum); + pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum, pStmt->taos->optionInfo.charsetCxt); if (code) { tscError("qBindStmtSingleColValue failed, error:%s", tstrerror(code)); STMT_ERR_RET(code); diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 9393bfc449..fecddbbff4 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -11,6 +11,12 @@ TARGET_LINK_LIBRARIES( os util common transport parser catalog scheduler gtest ${TAOS_LIB_STATIC} qcom executor function ) +ADD_EXECUTABLE(connectOptionsTest connectOptionsTest.cpp) +TARGET_LINK_LIBRARIES( + connectOptionsTest + os util common transport parser catalog scheduler gtest ${TAOS_LIB_STATIC} qcom executor function +) + ADD_EXECUTABLE(tmqTest tmqTest.cpp) TARGET_LINK_LIBRARIES( tmqTest @@ -41,11 +47,21 @@ TARGET_INCLUDE_DIRECTORIES( PRIVATE "${TD_SOURCE_DIR}/source/client/inc" ) +TARGET_INCLUDE_DIRECTORIES( + connectOptionsTest + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" +) + IF(${TD_LINUX}) add_test( NAME clientTest COMMAND clientTest ) + add_test( + NAME connectOptionsTest + COMMAND connectOptionsTest + ) ENDIF () TARGET_INCLUDE_DIRECTORIES( @@ -80,3 +96,4 @@ add_test( NAME userOperTest COMMAND userOperTest ) + diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 307ef7e06f..5a4393bfed 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -300,7 +300,13 @@ void* doConsumeData(void* param) { int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc > 1) { - numOfThreads = atoi(argv[1]); + //numOfThreads = atoi(argv[1]); + int32_t code = taosStr2int32(argv[1], &numOfThreads); + if (code != 0) { + return code; + } + + } numOfThreads = TMAX(numOfThreads, 1); @@ -1609,5 +1615,4 @@ TEST(clientCase, timezone_Test) { taos_close(pConn); } } - #pragma GCC diagnostic pop diff --git a/source/client/test/connectOptionsTest.cpp b/source/client/test/connectOptionsTest.cpp new file mode 100644 index 0000000000..95596e9ed3 --- /dev/null +++ b/source/client/test/connectOptionsTest.cpp @@ -0,0 +1,946 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "executor.h" +#include "taos.h" +#include "clientInt.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + + return RUN_ALL_TESTS(); +} + +TAOS* getConnWithGlobalOption(const char *tz){ + int code = taos_options(TSDB_OPTION_TIMEZONE, tz); + ASSERT(code == 0); + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT(pConn != nullptr); + return pConn; +} + +TAOS* getConnWithOption(const char *tz){ + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT(pConn != nullptr); + if (tz != NULL){ + int code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_TIMEZONE, tz); + ASSERT(code == 0); + } + return pConn; +} + +void execQuery(TAOS* pConn, const char *sql){ + TAOS_RES* pRes = taos_query(pConn, sql); + ASSERT(taos_errno(pRes) == TSDB_CODE_SUCCESS); + taos_free_result(pRes); +} + +void execQueryFail(TAOS* pConn, const char *sql){ + printf("execQueryFail: %s\n", sql); + TAOS_RES* pRes = taos_query(pConn, sql); +#ifndef WINDOWS + ASSERT(taos_errno(pRes) != TSDB_CODE_SUCCESS); +#endif + taos_free_result(pRes); +} + +void checkRows(TAOS* pConn, const char *sql, int32_t expectedRows){ + printf("checkRows sql:%s,rows:%d\n", sql, expectedRows); + TAOS_RES* pRes = taos_query(pConn, sql); + ASSERT(taos_errno(pRes) == TSDB_CODE_SUCCESS); + TAOS_ROW pRow = NULL; + int rows = 0; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + rows++; + } + ASSERT(rows == expectedRows); + taos_free_result(pRes); +} + +void check_timezone(TAOS* pConn, const char *sql, const char* tz){ + TAOS_RES *pRes = taos_query(pConn, sql); + ASSERT(taos_errno(pRes) == 0); + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(pRes)) != NULL) { + if (strcmp((const char*)row[0], "timezone") == 0){ + ASSERT(strstr((const char*)row[1], tz) != NULL); + } + } + taos_free_result(pRes); +} + +void check_sql_result_partial(TAOS* pConn, const char *sql, const char* result){ + TAOS_RES *pRes = taos_query(pConn, sql); + ASSERT(taos_errno(pRes) == 0); + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(pRes)) != NULL) { + ASSERT(strstr((const char*)row[0], result) != NULL); + } + taos_free_result(pRes); +} + +int64_t get_sql_result(TAOS* pConn, const char *sql){ + int64_t ts = 0; + TAOS_RES *pRes = taos_query(pConn, sql); + ASSERT(taos_errno(pRes) == 0); + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(pRes)) != NULL) { + ts = *(int64_t*)row[0]; + } + taos_free_result(pRes); + return ts; +} + +void check_sql_result(TAOS* pConn, const char *sql, const char* result){ + printf("check_sql_result sql:%s,result:%s\n", sql, result); + TAOS_RES *pRes = taos_query(pConn, sql); + ASSERT(taos_errno(pRes) == 0); + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(pRes)) != NULL) { +#ifndef WINDOWS + ASSERT (memcmp((const char*)row[0], result, strlen(result)) == 0); +#endif + } + taos_free_result(pRes); +} + +void check_sql_result_integer(TAOS* pConn, const char *sql, int64_t result){ + printf("check_sql_result_integer sql:%s,result:%ld\n", sql, result); + TAOS_RES *pRes = taos_query(pConn, sql); + ASSERT(taos_errno(pRes) == 0); + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(pRes)) != NULL) { +#ifndef WINDOWS + ASSERT (*(int64_t*)row[0] == result); +#endif + } + taos_free_result(pRes); +} + +void check_set_timezone(TAOS* optionFunc(const char *tz)){ + { + TAOS* pConn = optionFunc("UTC-8"); + check_timezone(pConn, "show local variables", "UTC-8"); + + execQuery(pConn, "drop database if exists db1"); + execQuery(pConn, "create database db1"); + execQuery(pConn, "create table db1.t1 (ts timestamp, v int)"); + + execQuery(pConn, "insert into db1.t1 values('2023-09-16 17:00:00', 1)"); + checkRows(pConn, "select * from db1.t1 where ts == '2023-09-16 17:00:00'", 1); + + taos_close(pConn); + } + + { + TAOS* pConn = optionFunc("UTC+8"); + check_timezone(pConn, "show local variables", "UTC+8"); + checkRows(pConn, "select * from db1.t1 where ts == '2023-09-16 01:00:00'", 1); + execQuery(pConn, "insert into db1.t1 values('2023-09-16 17:00:01', 1)"); + + taos_close(pConn); + } + + { + TAOS* pConn = optionFunc("UTC+0"); + check_timezone(pConn, "show local variables", "UTC+0"); + checkRows(pConn, "select * from db1.t1 where ts == '2023-09-16 09:00:00'", 1); + checkRows(pConn, "select * from db1.t1 where ts == '2023-09-17 01:00:01'", 1); + + taos_close(pConn); + } +} + +#define CHECK_TAOS_OPTION_POINTER(taos, option, isnull) \ + { \ + STscObj* pObj = acquireTscObj(*(int64_t*)taos); \ + ASSERT(pObj != nullptr); \ + if (isnull) { \ + ASSERT(pObj->optionInfo.option == nullptr); \ + } else { \ + ASSERT(pObj->optionInfo.option != nullptr); \ + } \ + } + +#define CHECK_TAOS_OPTION_APP(taos, option, val) \ + { \ + STscObj* pObj = acquireTscObj(*(int64_t*)taos); \ + ASSERT(pObj != nullptr); \ + ASSERT(strcmp(pObj->optionInfo.option, val) == 0); \ + } + +#define CHECK_TAOS_OPTION_IP_ERROR(taos, option, val) \ + { \ + STscObj* pObj = acquireTscObj(*(int64_t*)taos); \ + ASSERT(pObj != nullptr); \ + ASSERT(pObj->optionInfo.option == val); \ + } + +#define CHECK_TAOS_OPTION_IP(taos, option, val) \ + { \ + STscObj* pObj = acquireTscObj(*(int64_t*)taos); \ + ASSERT(pObj != nullptr); \ + char ip[TD_IP_LEN] = {0}; \ + taosInetNtoa(ip, pObj->optionInfo.option); \ + ASSERT(strcmp(ip, val) == 0); \ + } + +TEST(connectionCase, setConnectionOption_Test) { + int32_t code = taos_options_connection(NULL, TSDB_OPTION_CONNECTION_CHARSET, NULL); + ASSERT(code != 0); + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT(pConn != nullptr); + + code = taos_options_connection(pConn, TSDB_MAX_OPTIONS_CONNECTION, NULL); + ASSERT(code != 0); + + // test charset + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_CHARSET, ""); + ASSERT(code != 0); + + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_CHARSET, NULL); + ASSERT(code == 0); + CHECK_TAOS_OPTION_POINTER(pConn, charsetCxt, true); + + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_CHARSET, "Asia/Shanghai"); + ASSERT(code != 0); + + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_CHARSET, "gbk"); + ASSERT(code == 0); + CHECK_TAOS_OPTION_POINTER(pConn, charsetCxt, false); + +#ifndef WINDOWS + // test timezone + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_TIMEZONE, ""); + ASSERT(code == 0); + CHECK_TAOS_OPTION_POINTER(pConn, timezone, false); + check_sql_result(pConn, "select timezone()", "UTC (UTC, +0000)"); + + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_TIMEZONE, NULL); + ASSERT(code == 0); + CHECK_TAOS_OPTION_POINTER(pConn, timezone, true); + check_sql_result(pConn, "select timezone()", "Asia/Shanghai (CST, +0800)"); + + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_TIMEZONE, "UTC"); + ASSERT(code == 0); + CHECK_TAOS_OPTION_POINTER(pConn, timezone, false); + check_sql_result(pConn, "select timezone()", "UTC (UTC, +0000)"); + + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_TIMEZONE, "Asia/Kolkata"); + ASSERT(code == 0); + CHECK_TAOS_OPTION_POINTER(pConn, timezone, false); + check_sql_result(pConn, "select timezone()", "Asia/Kolkata (IST, +0530)"); + + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_TIMEZONE, "adbc"); + ASSERT(code == 0); + CHECK_TAOS_OPTION_POINTER(pConn, timezone, false); + check_sql_result(pConn, "select timezone()", "adbc (UTC, +0000)"); +#endif + + // test user APP + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_USER_APP, ""); + ASSERT(code == 0); + CHECK_TAOS_OPTION_APP(pConn, userApp, ""); + + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_USER_APP, NULL); + ASSERT(code == 0); + CHECK_TAOS_OPTION_APP(pConn, userApp, ""); + + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_USER_APP, "aaaaaaaaaaaaaaaaaaaaaabbbbbbb"); + ASSERT(code == 0); + CHECK_TAOS_OPTION_APP(pConn, userApp, "aaaaaaaaaaaaaaaaaaaaaab"); + + + // test user IP + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_USER_IP, ""); + ASSERT(code != 0); + CHECK_TAOS_OPTION_IP_ERROR(pConn, userIp, INADDR_NONE); + + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_USER_IP, NULL); + ASSERT(code == 0); + CHECK_TAOS_OPTION_IP_ERROR(pConn, userIp, INADDR_NONE); + + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_USER_IP, "aaaaaaaaaaaaaaaaaaaaaabbbbbbb"); + ASSERT(code != 0); + CHECK_TAOS_OPTION_IP_ERROR(pConn, userIp, INADDR_NONE); + + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_USER_IP, "1292.168.0.2"); + ASSERT(code != 0); + CHECK_TAOS_OPTION_IP_ERROR(pConn, userIp, INADDR_NONE); + + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_USER_IP, "192.168.0.2"); + ASSERT(code == 0); + CHECK_TAOS_OPTION_IP(pConn, userIp, "192.168.0.2"); + + taosMsleep(2 * HEARTBEAT_INTERVAL); + + //test user APP and user IP + check_sql_result_integer(pConn, "select count(*) from performance_schema.perf_connections where user_app = 'aaaaaaaaaaaaaaaaaaaaaab'", 1); + check_sql_result_integer(pConn, "select count(*) from performance_schema.perf_connections where user_ip = '192.168.0.2'", 1); + + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_USER_IP, "192.168.1.2"); + ASSERT(code == 0); + CHECK_TAOS_OPTION_IP(pConn, userIp, "192.168.1.2"); + + + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_USER_APP, "user"); + ASSERT(code == 0); + CHECK_TAOS_OPTION_APP(pConn, userApp, "user"); + + taosMsleep(2 * HEARTBEAT_INTERVAL); + + check_sql_result_integer(pConn, "select count(*) from performance_schema.perf_connections where user_app = 'user'", 1); + check_sql_result_integer(pConn, "select count(*) from performance_schema.perf_connections where user_ip = '192.168.1.2'", 1); + + // test clear + code = taos_options_connection(pConn, TSDB_OPTION_CONNECTION_CLEAR, "192.168.0.2"); + ASSERT(code == 0); + CHECK_TAOS_OPTION_POINTER(pConn, charsetCxt, true); + +#ifndef WINDOWS + CHECK_TAOS_OPTION_POINTER(pConn, timezone, true); + check_sql_result(pConn, "select timezone()", "Asia/Shanghai (CST, +0800)"); +#endif + + CHECK_TAOS_OPTION_APP(pConn, userApp, ""); + CHECK_TAOS_OPTION_IP_ERROR(pConn, userIp, INADDR_NONE); + + taos_close(pConn); +} + +TEST(charsetCase, charset_Test) { + // 1. build connection with different charset + TAOS* pConnGbk = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT(pConnGbk != nullptr); + + TAOS* pConnUTF8 = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT(pConnUTF8 != nullptr); + + TAOS* pConnDefault = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT(pConnDefault != nullptr); + + int32_t code = taos_options_connection(pConnGbk, TSDB_OPTION_CONNECTION_CHARSET, "gbk"); + ASSERT(code == 0); + CHECK_TAOS_OPTION_POINTER(pConnGbk, charsetCxt, false); + + code = taos_options_connection(pConnUTF8, TSDB_OPTION_CONNECTION_CHARSET, "UTF-8"); + ASSERT(code == 0); + CHECK_TAOS_OPTION_POINTER(pConnUTF8, charsetCxt, false); + + // 2. build test string + char sqlTag[256] = {0}; + char sqlCol[256] = {0}; + + // 芬 gbk encode is 0xB7D2, UTF-8 encode is 0xE88AAC + // 中国 gbk encode is 0xD6D0B9FA, UTF-8 encode is 0xE4B8ADE59BBD + char fenUtf8[32] = {0}; + char fenGbk[32] = {0}; + char zhongGbk[32] = {0}; + char zhongguoUtf8[32] = {0}; + char guoUtf8[32] = {0}; + char zhongguoGbk[32] = {0}; + snprintf(fenUtf8, sizeof(fenUtf8), "%c%c%c", 0xE8, 0x8A, 0xAC); + snprintf(fenGbk, sizeof(fenGbk), "%c%c", 0xB7, 0xD2); + snprintf(zhongguoUtf8, sizeof(zhongguoUtf8), "%c%c%c%c%c%c", 0xE4, 0xB8, 0xAD, 0xE5, 0x9B, 0xBD); + snprintf(guoUtf8, sizeof(guoUtf8), "%c%c%c", 0xE5, 0x9B, 0xBD); + snprintf(zhongguoGbk, sizeof(zhongguoGbk), "%c%c%c%c", 0xD6, 0xD0, 0xB9, 0xFA); + snprintf(zhongGbk, sizeof(zhongGbk), "%c%c", 0xD6, 0xD0); + + // 3. create stable + execQuery(pConnGbk, "drop database if exists db1"); + execQuery(pConnGbk, "create database db1"); + execQuery(pConnGbk, "create table db1.stb (ts timestamp, c1 nchar(32), c2 int) tags(t1 timestamp, t2 nchar(32), t3 int)"); + + // 4. test tag with different charset + snprintf(sqlTag, sizeof(sqlTag), "create table db1.ctb1 using db1.stb tags('2023-09-16 17:00:00+05:00', '%s', 1)", fenUtf8); + execQueryFail(pConnGbk, sqlTag); + + snprintf(sqlTag, sizeof(sqlTag), "create table db1.ctb1 using db1.stb tags('2023-09-16 17:00:00+05:00', '%s', 1)", fenGbk); + execQuery(pConnGbk, sqlTag); + + // 5. test column with different charset + snprintf(sqlCol, sizeof(sqlCol), "insert into db1.ctb1 values(1732178775133, '%s', 1)", zhongguoUtf8); + execQueryFail(pConnGbk, sqlCol); + + snprintf(sqlCol, sizeof(sqlCol), "insert into db1.ctb1 values(1732178775133, '%s', 1)", zhongguoGbk); + execQuery(pConnGbk, sqlCol); + + // 6. check result with different charset + check_sql_result(pConnGbk, "select t2 from db1.ctb1", fenGbk); + check_sql_result(pConnUTF8, "select t2 from db1.ctb1", fenUtf8); + + check_sql_result(pConnGbk, "select c1 from db1.ctb1", zhongguoGbk); + check_sql_result(pConnUTF8, "select c1 from db1.ctb1", zhongguoUtf8); + + // 7. test function with different charset + // 7.1 concat + char zhongguofenGbk[32] = {0}; + snprintf(zhongguofenGbk, sizeof(zhongguofenGbk), "%s%s", zhongguoGbk, fenGbk); + char sql[256] = {0}; + snprintf(sql, sizeof(sql), "select concat(c1, '%s') from db1.ctb1", fenGbk); + execQueryFail(pConnGbk, sql); + snprintf(sql, sizeof(sql), "select concat(c1, '%s') from db1.ctb1", fenUtf8); + check_sql_result(pConnGbk, sql, zhongguofenGbk); + + // 7.2 trim + snprintf(sql, sizeof(sql), "select trim(LEADING c1 from '%s') from db1.ctb1", zhongguofenGbk); + check_sql_result(pConnGbk, sql, zhongguofenGbk); + char zhongguofenUtf8[32] = {0}; + snprintf(zhongguofenUtf8, sizeof(zhongguofenUtf8), "%s%s", zhongguoUtf8, fenUtf8); + snprintf(sql, sizeof(sql), "select trim(LEADING c1 from '%s') from db1.ctb1", zhongguofenUtf8); + check_sql_result(pConnGbk, sql, fenUtf8); + + check_sql_result(pConnGbk, "select char(c1) from db1.ctb1", ""); + + check_sql_result_integer(pConnGbk, "select ascii(c1) from db1.ctb1", 0xE4); + check_sql_result_integer(pConnUTF8, "select ascii(c1) from db1.ctb1", 0xE4); + check_sql_result_integer(pConnGbk, "select LENGTH(c1) from db1.ctb1", 8); + check_sql_result_integer(pConnUTF8, "select LENGTH(c1) from db1.ctb1", 8); + check_sql_result_integer(pConnGbk, "select CHAR_LENGTH(c1) from db1.ctb1", 2); + check_sql_result_integer(pConnUTF8, "select CHAR_LENGTH(c1) from db1.ctb1", 2); + + execQuery(pConnGbk, "select LOWER(c1) from db1.ctb1"); + execQuery(pConnGbk, "select UPPER(c1) from db1.ctb1"); + + snprintf(sql, sizeof(sql), "select position(c1 in '%s') from db1.ctb1", zhongguofenGbk); + check_sql_result_integer(pConnGbk, sql, 0); + + snprintf(sql, sizeof(sql), "select position('%s' in c1) from db1.ctb1", guoUtf8); + check_sql_result_integer(pConnUTF8, sql, 2); + + snprintf(sql, sizeof(sql), "select replace(c1, '%s', 'a') from db1.ctb1", zhongguoGbk); + execQueryFail(pConnGbk, sql); + + snprintf(sql, sizeof(sql), "select replace(c1, '%s', 'a') from db1.ctb1", zhongguoUtf8); + check_sql_result(pConnUTF8, sql, "a"); + + snprintf(sql, sizeof(sql), "%s%s", zhongguoGbk, zhongguoGbk); + check_sql_result(pConnGbk, "select repeat(c1, 2) from db1.ctb1", sql); + + check_sql_result(pConnGbk, "select cast(c1 as binary(32)) from db1.ctb1", zhongguoUtf8); + + check_sql_result(pConnUTF8, "select substr(c1,2,1) from db1.ctb1", guoUtf8); + + snprintf(sql, sizeof(sql), "select SUBSTRING_INDEX(c1,'%s',1) from db1.ctb1", guoUtf8); + check_sql_result(pConnGbk, sql, zhongGbk); + + // 8. test default charset + snprintf(sqlCol, sizeof(sqlCol), "insert into db1.ctb1 values(1732178775134, '%s', 1)", zhongguoUtf8); + execQuery(pConnDefault, sqlCol); + check_sql_result(pConnDefault, "select c1 from db1.ctb1 where ts = 1732178775134", zhongguoUtf8); + + // 9. test json tag with different charset + execQuery(pConnUTF8, "create table db1.jsta (ts timestamp, c1 nchar(32), c2 int) tags(t1 json)"); + snprintf(sqlCol, sizeof(sqlCol), "create table db1.jsta1 using db1.jsta tags('{\"k\":\"%s\"}')", fenUtf8); + execQuery(pConnUTF8, sqlCol); + snprintf(sqlCol, sizeof(sqlCol), "insert into db1.jsta1 values(1732178775133, '%s', 1)", zhongguoUtf8); + execQuery(pConnUTF8, sqlCol); + + char resJsonTag[32] = {0}; + snprintf(resJsonTag, sizeof(resJsonTag), "{\"k\":\"%s\"}", fenGbk); + check_sql_result(pConnGbk, "select t1 from db1.jsta1", resJsonTag); + + // 10. reset charset to default(utf-8 + code = taos_options_connection(pConnGbk, TSDB_OPTION_CONNECTION_CHARSET, NULL); + ASSERT(code == 0); + CHECK_TAOS_OPTION_POINTER(pConnGbk, charsetCxt, true); + check_sql_result(pConnGbk, "select t2 from db1.ctb1 where ts = 1732178775134", fenUtf8); + check_sql_result(pConnGbk, "select c1 from db1.ctb1 where ts = 1732178775134", zhongguoUtf8); + + taos_close(pConnGbk); + taos_close(pConnUTF8); + taos_close(pConnDefault); + +} + +TEST(charsetCase, alter_charset_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT(pConn != nullptr); + + execQueryFail(pConn, "alter dnode 1 'charset gbk'"); + execQueryFail(pConn, "local 'charset gbk'"); + + taos_close(pConn); +} + +#ifndef WINDOWS +TEST(timezoneCase, set_timezone_Test) { + check_set_timezone(getConnWithGlobalOption); + check_set_timezone(getConnWithOption); +} + +TEST(timezoneCase, alter_timezone_Test) { + TAOS* pConn = getConnWithGlobalOption("UTC-8"); + check_timezone(pConn, "show local variables", "UTC-8"); + + execQuery(pConn, "alter local 'timezone Asia/Kolkata'"); + check_timezone(pConn, "show local variables", "Asia/Kolkata"); + + execQuery(pConn, "alter local 'timezone Asia/Shanghai'"); + check_timezone(pConn, "show local variables", "Asia/Shanghai"); + + execQueryFail(pConn, "alter dnode 1 'timezone Asia/Kolkata'"); + + taos_close(pConn); +} + +char *tz_test[] = { + "2023-09-16 17:00:00+", + "2023-09-16 17:00:00+a", + "2023-09-16 17:00:00+8", + "2023-09-16 17:00:00+832", + "2023-09-16 17:00:00+8323", + "2023-09-16 17:00:00+:", + "2023-09-16 17:00:00+8:", + "2023-09-16 17:00:00++:", + "2023-09-16 17:00:00+d:", + "2023-09-16 17:00:00+09:", + "2023-09-16 17:00:00+8f:", + "2023-09-16 17:00:00+080:", + "2023-09-16 17:00:00+:30", + "2023-09-16 17:00:00+:3", + "2023-09-16 17:00:00+:093", + "2023-09-16 17:00:00+:-30", + "2023-09-16 17:00:00++:-30", + "2023-09-16 17:00:00+8:8", + "2023-09-16 17:00:00+8:2a", + "2023-09-16 17:00:00+8:08", + "2023-09-16 17:00:00+8:038", + "2023-09-16 17:00:00+08:8", + "2023-09-16 17:00:00+09:3a", + "2023-09-16 17:00:00+09:abc", + "2023-09-16 17:00:00+09:001", +}; + +void do_insert_failed(){ + TAOS* pConn = getConnWithGlobalOption("UTC-8"); + + for (unsigned int i = 0; i < sizeof (tz_test) / sizeof (tz_test[0]); ++i){ + char sql[1024] = {0}; + (void)snprintf(sql, sizeof(sql), "insert into db1.ctb1 values('%s', '%s', 1)", tz_test[i], tz_test[i]); + + execQueryFail(pConn, sql); + } + taos_close(pConn); +} + +struct insert_params +{ + const char *tz; + const char *tbname; + const char *t1; + const char *t2; +}; + +struct insert_params params1[] = { + {"UTC", "ntb", "2023-09-16 17:00:00", "2023-09-16 17:00:00+08:00"}, + {"UTC", "ctb1", "2023-09-16 17:00:00", "2023-09-16 17:00:00+08:00"}, +}; + +struct insert_params params2[] = { + {"UTC+9", "ntb", "2023-09-16 08:00:00", "2023-09-16 08:00:00-01:00"}, + {"UTC+9", "ctb1", "2023-09-16 08:00:00", "2023-09-16 11:00:00+02:00"}, +}; + +void do_insert(struct insert_params params){ + TAOS* pConn = getConnWithOption(params.tz); + char sql[1024] = {0}; + (void)snprintf(sql, sizeof(sql), "insert into db1.%s values('%s', '%s', 1)", params.tbname, params.t1, params.t2); + execQuery(pConn, sql); + taos_close(pConn); +} + +void do_select(struct insert_params params){ + TAOS* pConn = getConnWithOption(params.tz); + char sql[1024] = {0}; + (void)snprintf(sql, sizeof(sql), "select * from db1.%s where ts == '%s' and c1 == '%s'", params.tbname, params.t1, params.t2); + checkRows(pConn, sql, 1); + taos_close(pConn); +} + +// test insert string and integer to timestamp both normal table and child table(and tag) +TEST(timezoneCase, insert_with_timezone_Test) { + /* + * 1. prepare data, create db and tables + */ + TAOS* pConn1 = getConnWithOption("UTC+2"); + execQuery(pConn1, "drop database if exists db1"); + execQuery(pConn1, "create database db1"); + execQuery(pConn1, "create table db1.ntb (ts timestamp, c1 timestamp, c2 int)"); + execQuery(pConn1, "create table db1.stb (ts timestamp, c1 timestamp, c2 int) tags(t1 timestamp, t2 timestamp, t3 int)"); + execQuery(pConn1, "create table db1.ctb1 using db1.stb tags(\"2023-09-16 17:00:00+05:00\", \"2023-09-16 17:00:00\", 1)"); + execQuery(pConn1, "create table db1.ctb2 using db1.stb tags(1732178775000, 1732178775000, 1)"); + execQuery(pConn1, "insert into db1.ntb values(1732178775133, 1732178775133, 1)"); + execQuery(pConn1, "insert into db1.ctb1 values(1732178775133, 1732178775133, 1)"); //2024-11-21 10:46:15.133+02:00 + execQuery(pConn1, "insert into db1.ctb2 values(1732178775133, 1732178775133, 1)"); + + /* + * 2. test tag and timestamp with integer format + */ + TAOS* pConn2 = getConnWithOption("UTC-2"); + checkRows(pConn2, "select * from db1.stb where t1 == '2023-09-16 17:00:00+05:00' and t2 == '2023-09-16 21:00:00'", 1); + checkRows(pConn2, "select * from db1.stb where t1 == '2024-11-21 16:46:15+08:00' and t2 == '2024-11-21 09:46:15+01:00'", 1); + checkRows(pConn2, "select * from db1.ntb where ts == '2024-11-21 09:46:15.133+01:00' and c1 == '2024-11-21 10:46:15.133'", 1); + checkRows(pConn2, "select * from db1.ctb1 where ts == '2024-11-21 09:46:15.133+01:00' and c1 == '2024-11-21 10:46:15.133'", 1); + + check_sql_result(pConn2, "select TO_ISO8601(ts) from db1.ctb1", "2024-11-21T10:46:15.133+0200"); // 2024-01-01 23:00:00+0200 + + + /* + * 3. test timestamp with string format + */ + for (unsigned int i = 0; i < sizeof (params1) / sizeof (params1[0]); ++i){ + do_insert(params1[i]); + do_select(params1[i]); + do_select(params2[i]); + } + + do_insert_failed(); + /* + * 4. test NULL timezone, use default timezone UTC-8 + */ + TAOS* pConn3 = getConnWithOption(NULL); + checkRows(pConn3, "select * from db1.stb where t1 == '2023-09-16 20:00:00' and t2 == '2023-09-17 03:00:00'", 2); + checkRows(pConn3, "select * from db1.stb where t1 == 1732178775000 and t2 == 1732178775000", 1); + checkRows(pConn3, "select * from db1.ntb where ts == '2024-11-21 16:46:15.133' and c1 == '2024-11-21 16:46:15.133'", 1); + checkRows(pConn3, "select * from db1.ctb1 where ts == '2023-09-17 01:00:00' and c1 == '2023-09-16 17:00:00'", 1); + + /* + * 5. test multi connection with different timezone + */ + checkRows(pConn2, "select * from db1.ctb1 where ts == '2024-11-21 09:46:15.133+01:00' and c1 == '2024-11-21 10:46:15.133'", 1); + checkRows(pConn1, "select * from db1.ctb1 where ts == '2024-11-21 09:46:15.133+01:00' and c1 == '2024-11-21 06:46:15.133'", 1); + + taos_close(pConn1); + taos_close(pConn2); + taos_close(pConn3); +} + +TEST(timezoneCase, func_timezone_Test) { + TAOS* pConn = getConnWithGlobalOption("UTC+8"); + check_sql_result(pConn, "select timezone()", "UTC+8 (UTC, -0800)"); + taos_close(pConn); + + pConn = getConnWithOption("UTC-2"); + + execQuery(pConn, "drop database if exists db1"); + execQuery(pConn, "create database db1"); + execQuery(pConn, "create table db1.ntb (ts timestamp, c1 binary(32), c2 int)"); + execQuery(pConn, "insert into db1.ntb values(1704142800000, '2024-01-01 23:00:00', 1)"); // 2024-01-01 23:00:00+0200 + + // test timezone + check_sql_result(pConn, "select timezone()", "UTC-2 (UTC, +0200)"); + + // test timetruncate + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-01 23:00:00', 1d, 0))", "2024-01-01T02:00:00.000+0200"); + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-01 01:00:00', 1d, 0))", "2023-12-31T02:00:00.000+0200"); + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-01 01:00:00+0300', 1d, 0))", "2023-12-31T02:00:00.000+0200"); + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-01 01:00:00-0300', 1d, 0))", "2024-01-01T02:00:00.000+0200"); + + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-04 23:00:00', 1w, 0))", "2024-01-04T02:00:00.000+0200"); + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-04 01:00:00', 1w, 0))", "2023-12-28T02:00:00.000+0200"); + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-04 01:00:00+0300', 1w, 0))", "2023-12-28T02:00:00.000+0200"); + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-04 01:00:00-0300', 1w, 0))", "2024-01-04T02:00:00.000+0200"); + + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-01 23:00:00', 1d, 1))", "2024-01-01T00:00:00.000+0200"); + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-01 01:00:00', 1d, 1))", "2024-01-01T00:00:00.000+0200"); + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-01 01:00:00+0500', 1d, 1))", "2023-12-31T00:00:00.000+0200"); + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-01 01:00:00-0300', 1d, 1))", "2024-01-01T00:00:00.000+0200"); + + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-04 23:00:00', 1w, 1))", "2024-01-04T00:00:00.000+0200"); + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-04 01:00:00', 1w, 1))", "2024-01-04T00:00:00.000+0200"); + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-04 01:00:00+0500', 1w, 1))", "2023-12-28T00:00:00.000+0200"); + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE('2024-01-04 01:00:00-0300', 1w, 1))", "2024-01-04T00:00:00.000+0200"); + + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE(1704142800000, 1d, 0))", "2024-01-01T02:00:00.000+0200"); // 2024-01-01 23:00:00+0200 + check_sql_result(pConn, "select TO_ISO8601(TIMETRUNCATE(ts, 1w, 1)) from db1.ntb", "2023-12-28T00:00:00.000+0200"); // 2024-01-01 23:00:00+0200 + + // TODAY + check_sql_result_partial(pConn, "select TO_ISO8601(today())", "T00:00:00.000+0200"); + + // NOW + check_sql_result_partial(pConn, "select TO_ISO8601(now())", "+0200"); + + // WEEKDAY + check_sql_result_integer(pConn, "select WEEKDAY('2024-01-01')", 0); + check_sql_result_integer(pConn, "select WEEKDAY('2024-01-01 03:00:00')", 0); + check_sql_result_integer(pConn, "select WEEKDAY('2024-01-01 23:00:00+0200')", 0); + check_sql_result_integer(pConn, "select WEEKDAY('2024-01-01 23:00:00-1100')", 1); + check_sql_result_integer(pConn, "select WEEKDAY(1704142800000)", 0); + check_sql_result_integer(pConn, "select WEEKDAY(ts) from db1.ntb", 1); + + // DAYOFWEEK + check_sql_result_integer(pConn, "select DAYOFWEEK('2024-01-01')", 2); + check_sql_result_integer(pConn, "select DAYOFWEEK('2024-01-01 03:00:00')", 2); + check_sql_result_integer(pConn, "select DAYOFWEEK('2024-01-01 23:00:00+0200')", 2); + check_sql_result_integer(pConn, "select DAYOFWEEK('2024-01-01 23:00:00-1100')", 3); + check_sql_result_integer(pConn, "select DAYOFWEEK(1704142800000)", 2); + check_sql_result_integer(pConn, "select DAYOFWEEK(ts) from db1.ntb", 3); + + // WEEK + check_sql_result_integer(pConn, "select WEEK('2024-01-07')", 1); + check_sql_result_integer(pConn, "select WEEK('2024-01-07 02:00:00')", 1); + check_sql_result_integer(pConn, "select WEEK('2024-01-07 02:00:00+0200')", 1); + check_sql_result_integer(pConn, "select WEEK('2024-01-07 02:00:00+1100')", 0); + check_sql_result_integer(pConn, "select WEEK(1704142800000)", 0); // 2024-01-01 23:00:00+0200 + check_sql_result_integer(pConn, "select WEEK(ts) from db1.ntb", 0); // 2024-01-01 23:00:00+0200 + + check_sql_result_integer(pConn, "select WEEK('2024-01-07', 3)", 1); + check_sql_result_integer(pConn, "select WEEK('2024-01-07 02:00:00', 3)", 1); + check_sql_result_integer(pConn, "select WEEK('2024-01-07 02:00:00+0200', 3)", 1); + check_sql_result_integer(pConn, "select WEEK('2024-01-01 02:00:00+1100', 3)", 52); + check_sql_result_integer(pConn, "select WEEK(1704142800000, 3)", 1); // 2024-01-01 23:00:00+0200 + check_sql_result_integer(pConn, "select WEEK(ts, 3) from db1.ntb", 1); // 2024-01-01 23:00:00+0200 + + // WEEKOFYEAR + check_sql_result_integer(pConn, "select WEEKOFYEAR('2024-01-07')", 1); + check_sql_result_integer(pConn, "select WEEKOFYEAR('2024-01-07 02:00:00')", 1); + check_sql_result_integer(pConn, "select WEEKOFYEAR('2024-01-07 02:00:00+0200')", 1); + check_sql_result_integer(pConn, "select WEEKOFYEAR('2024-01-01 02:00:00+1100')", 52); + check_sql_result_integer(pConn, "select WEEKOFYEAR(1704142800000)", 1); // 2024-01-01 23:00:00+0200 + check_sql_result_integer(pConn, "select WEEKOFYEAR(ts) from db1.ntb", 1); // 2024-01-01 23:00:00+0200 + + // TO_ISO8601 + check_sql_result(pConn, "select TO_ISO8601(ts) from db1.ntb", "2024-01-01T23:00:00.000+0200"); + check_sql_result(pConn, "select TO_ISO8601(ts,'-08') from db1.ntb", "2024-01-01T13:00:00.000-08"); + check_sql_result(pConn, "select TO_ISO8601(1)", "1970-01-01T02:00:00.001+0200"); + check_sql_result(pConn, "select TO_ISO8601(1,'+0800')", "1970-01-01T08:00:00.001+0800"); + + // TO_UNIXTIMESTAMP + check_sql_result_integer(pConn, "select TO_UNIXTIMESTAMP(c1) from db1.ntb", 1704121200000); // use timezone in server UTC-8 + check_sql_result_integer(pConn, "select TO_UNIXTIMESTAMP('2024-01-01T23:00:00.000+0200')", 1704142800000); + check_sql_result_integer(pConn, "select TO_UNIXTIMESTAMP('2024-01-01T13:00:00.000-08')", 1704142800000); + check_sql_result_integer(pConn, "select TO_UNIXTIMESTAMP('2024-01-01T23:00:00.001')", 1704142800001); + + // TO_TIMESTAMP + check_sql_result_integer(pConn, "select TO_TIMESTAMP(c1,'yyyy-mm-dd hh24:mi:ss') from db1.ntb", 1704121200000); // use timezone in server UTC-8 + check_sql_result_integer(pConn, "select TO_TIMESTAMP('2024-01-01 23:00:00+02:00', 'yyyy-mm-dd hh24:mi:ss tzh')", 1704142800000); + check_sql_result_integer(pConn, "select TO_TIMESTAMP('2024-01-01T13:00:00-08', 'yyyy-mm-ddThh24:mi:ss tzh')", 1704142800000); + check_sql_result_integer(pConn, "select TO_TIMESTAMP('2024/01/01 23:00:00', 'yyyy/mm/dd hh24:mi:ss')", 1704142800000); + + // TO_CHAR + check_sql_result(pConn, "select TO_CHAR(ts,'yyyy-mm-dd hh24:mi:ss') from db1.ntb", "2024-01-02 05:00:00"); // use timezone in server UTC-8 + check_sql_result(pConn, "select TO_CHAR(cast(1704142800000 as timestamp), 'yyyy-mm-dd hh24:mi:ss tzh')", "2024-01-01 23:00:00 +02"); + check_sql_result(pConn, "select TO_CHAR(cast(1704142800000 as timestamp), 'yyyy-mm-dd hh24:mi:ss')", "2024-01-01 23:00:00"); + + // TIMEDIFF + check_sql_result_integer(pConn, "select TIMEDIFF(c1, '2024-01-01T23:00:00.001+02') from db1.ntb", -21600001); // use timezone in server UTC-8 + check_sql_result_integer(pConn, "select TIMEDIFF(c1, '2024-01-01T23:00:00.001') from db1.ntb", -1); // use timezone in server UTC-8 + check_sql_result_integer(pConn, "select TIMEDIFF('2024-01-01T23:00:00.001', '2024-01-01T13:00:00.000-08')", 1); + + // CAST + check_sql_result_integer(pConn, "select CAST(c1 as timestamp) from db1.ntb", 1704121200000); + check_sql_result_integer(pConn, "select CAST('2024-01-01T23:00:00.000+02' as timestamp)", 1704142800000); + check_sql_result_integer(pConn, "select CAST('2024-01-01T23:00:00.000' as timestamp)", 1704142800000); + + taos_close(pConn); + + // hash join + pConn = getConnWithOption("UTC+1"); + + execQuery(pConn, "drop database if exists db1"); + execQuery(pConn, "create database db1"); + execQuery(pConn, "create table db1.ntb (ts timestamp, c1 binary(32), c2 int)"); + execQuery(pConn, "create table db1.ntb1 (ts timestamp, c1 binary(32), c2 int)"); + execQuery(pConn, "insert into db1.ntb values(1703987400000, '2023-12-31 00:50:00', 1)"); // 2023-12-31 00:50:00-0100 + execQuery(pConn, "insert into db1.ntb1 values(1704070200000, '2023-12-31 23:50:00', 11)"); // 2023-12-31 23:50:00-0100 + checkRows(pConn, "select a.ts,b.ts from db1.ntb a join db1.ntb1 b on timetruncate(a.ts, 1d) = timetruncate(b.ts, 1d)", 1); + + // operator +1n +1y + check_sql_result(pConn, "select TO_ISO8601(CAST('2023-01-31T00:00:00.000-01' as timestamp) + 1n)", "2023-02-28T00:00:00.000-0100"); + check_sql_result(pConn, "select TO_ISO8601(CAST('2024-01-31T00:00:00.000-01' as timestamp) + 1n)", "2024-02-29T00:00:00.000-0100"); + check_sql_result(pConn, "select TO_ISO8601(CAST('2024-02-29T00:00:00.000-01' as timestamp) + 1y)", "2025-02-28T00:00:00.000-0100"); + check_sql_result(pConn, "select TO_ISO8601(CAST('2024-01-31T00:00:00.000-01' as timestamp) + 1y)", "2025-01-31T00:00:00.000-0100"); + + check_sql_result(pConn, "select TO_ISO8601(CAST('2024-01-01T00:00:00.000+01' as timestamp) + 1n)", "2024-01-31T22:00:00.000-0100"); + check_sql_result(pConn, "select TO_ISO8601(CAST('2024-01-01T00:00:00.000+01' as timestamp) + 1y)", "2024-12-31T22:00:00.000-0100"); + + // case when + check_sql_result_integer(pConn, "select case CAST('2024-01-01T00:00:00.000+01' as timestamp) when 1704063600000 then 1 end", 1); + check_sql_result_integer(pConn, "select case CAST('2024-01-01T00:00:00.000' as timestamp) when 1704070800000 then 1 end", 1); + + taos_close(pConn); + +} + +time_t time_winter = 1731323281; // 2024-11-11 19:08:01+0800 +time_t time_summer = 1731323281 - 120 * 24 * 60 * 60; + +struct test_times +{ + const char *name; + time_t t; + const char *timezone; +} test_tz[] = { + {"", time_winter, " (UTC, +0000)"}, + {"America/New_York", time_winter, "America/New_York (EST, -0500)"}, // 2024-11-11 19:08:01+0800 + {"America/New_York", time_summer, "America/New_York (EDT, -0400)"}, + {"Asia/Kolkata", time_winter, "Asia/Kolkata (IST, +0530)"}, + {"Asia/Shanghai", time_winter, "Asia/Shanghai (CST, +0800)"}, + {"Europe/London", time_winter, "Europe/London (GMT, +0000)"}, + {"Europe/London", time_summer, "Europe/London (BST, +0100)"} +}; + +void timezone_str_test(const char* tz, time_t t, const char* tzStr) { + int code = setenv("TZ", tz, 1); + ASSERT(-1 != code); + tzset(); + + char str1[TD_TIMEZONE_LEN] = {0}; + ASSERT(taosFormatTimezoneStr(t, tz, NULL, str1) == 0); + ASSERT_STREQ(str1, tzStr); +} + +void timezone_rz_str_test(const char* tz, time_t t, const char* tzStr) { + timezone_t sp = tzalloc(tz); + ASSERT(sp); + + char str1[TD_TIMEZONE_LEN] = {0}; + ASSERT(taosFormatTimezoneStr(t, tz, sp, str1) == 0); + ASSERT_STREQ(str1, tzStr); + tzfree(sp); +} + +TEST(timezoneCase, format_timezone_Test) { + for (unsigned int i = 0; i < sizeof (test_tz) / sizeof (test_tz[0]); ++i){ + timezone_str_test(test_tz[i].name, test_tz[i].t, test_tz[i].timezone); + timezone_str_test(test_tz[i].name, test_tz[i].t, test_tz[i].timezone); + } +} + +TEST(timezoneCase, get_tz_Test) { + { + char tz[TD_TIMEZONE_LEN] = {0}; + getTimezoneStr(tz); + ASSERT_STREQ(tz, "Asia/Shanghai"); + +// getTimezoneStr(tz); +// ASSERT_STREQ(tz, "Asia/Shanghai"); +// +// getTimezoneStr(tz); +// ASSERT_STREQ(tz, TZ_UNKNOWN); + } +} + +struct { + const char * env; + time_t expected; +} test_mk[] = { + {"MST", 832935315}, + {"", 832910115}, + {":UTC", 832910115}, + {"UTC", 832910115}, + {"UTC0", 832910115} +}; + + +TEST(timezoneCase, mktime_Test){ + struct tm tm; + time_t t; + + memset (&tm, 0, sizeof (tm)); + tm.tm_isdst = 0; + tm.tm_year = 96; /* years since 1900 */ + tm.tm_mon = 4; + tm.tm_mday = 24; + tm.tm_hour = 3; + tm.tm_min = 55; + tm.tm_sec = 15; + + for (unsigned int i = 0; i < sizeof (test_mk) / sizeof (test_mk[0]); ++i) + { + setenv ("TZ", test_mk[i].env, 1); + t = taosMktime (&tm, NULL); + ASSERT (t == test_mk[i].expected); + } +} + +TEST(timezoneCase, mktime_rz_Test){ + struct tm tm; + time_t t; + + memset (&tm, 0, sizeof (tm)); + tm.tm_isdst = 0; + tm.tm_year = 96; /* years since 1900 */ + tm.tm_mon = 4; + tm.tm_mday = 24; + tm.tm_hour = 3; + tm.tm_min = 55; + tm.tm_sec = 15; + + for (unsigned int i = 0; i < sizeof (test_mk) / sizeof (test_mk[0]); ++i) + { + timezone_t tz = tzalloc(test_mk[i].env); + ASSERT(tz); + t = taosMktime(&tm, tz); + ASSERT (t == test_mk[i].expected); + tzfree(tz); + } +} + +TEST(timezoneCase, localtime_performance_Test) { + timezone_t sp = tzalloc("Asia/Shanghai"); + ASSERT(sp); + + int cnt = 1000000; + int times = 10; + int64_t time_localtime = 0; + int64_t time_localtime_rz = 0; +// int cnt = 1000000; + for (int i = 0; i < times; ++i) { + int64_t t1 = taosGetTimestampNs(); + for (int j = 0; j < cnt; ++j) { + time_t t = time_winter - j; + struct tm tm1; + ASSERT (taosLocalTime(&t, &tm1, NULL, 0, NULL)); + } + int64_t tmp = taosGetTimestampNs() - t1; + printf("localtime cost:%" PRId64 " ns, run %d times", tmp, cnt); + time_localtime += tmp/cnt; + + printf("\n"); + + + + int64_t t2 = taosGetTimestampNs(); + for (int j = 0; j < cnt; ++j) { + time_t t = time_winter - j; + struct tm tm1; + ASSERT (taosLocalTime(&t, &tm1, NULL, 0, sp)); + } + tmp = taosGetTimestampNs() - t2; + printf("localtime_rz cost:%" PRId64 " ns, run %d times", tmp, cnt); + time_localtime_rz += tmp/cnt; + printf("\n\n"); + } + printf("average: localtime cost:%" PRId64 " ns, localtime_rz cost:%" PRId64 " ns\n", time_localtime/times, time_localtime_rz/times); + tzfree(sp); +} +#endif + +#pragma GCC diagnostic pop + diff --git a/source/common/src/cos.c b/source/common/src/cos.c index de4f65f2ad..35ab328815 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -261,7 +261,7 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro for (int i = 0; i < error->extraDetailsCount; i++) { if (elen - len > 0) { len += tsnprintf(&(cbd->err_msg[len]), elen - len, " %s: %s\n", error->extraDetails[i].name, - error->extraDetails[i].value); + error->extraDetails[i].value); } } } @@ -742,9 +742,9 @@ upload: TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit); } n = tsnprintf(buf, sizeof(buf), - "%d" - "%s", - i + 1, manager.etags[i]); + "%d" + "%s", + i + 1, manager.etags[i]); size += growbuffer_append(&(manager.gb), buf, n); } size += growbuffer_append(&(manager.gb), "", strlen("")); @@ -908,10 +908,10 @@ upload: int n; for (int i = 0; i < cp.part_num; ++i) { n = tsnprintf(buf, sizeof(buf), - "%d" - "%s", - // i + 1, manager.etags[i]); - cp.parts[i].index + 1, cp.parts[i].etag); + "%d" + "%s", + // i + 1, manager.etags[i]); + cp.parts[i].index + 1, cp.parts[i].etag); size += growbuffer_append(&(manager.gb), buf, n); } size += growbuffer_append(&(manager.gb), "", strlen("")); @@ -1916,7 +1916,8 @@ void s3EvictCache(const char *path, long object_size) { } long s3Size(const char *object_name) { - long size = 0; + int32_t code = 0; + long size = 0; cos_pool_t *p = NULL; int is_cname = 0; @@ -1941,7 +1942,10 @@ long s3Size(const char *object_name) { if (cos_status_is_ok(s)) { char *content_length_str = (char *)apr_table_get(resp_headers, COS_CONTENT_LENGTH); if (content_length_str != NULL) { - size = atol(content_length_str); + code = taosStr2Int64(content_length_str, &size); + if (code != 0) { + cos_warn_log("parse content length failed since %s\n", tstrerror(code)); + } } cos_warn_log("head object succeeded: %ld\n", size); } else { diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index 26ac61c42a..c3a6d44ae5 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -302,6 +302,8 @@ static int32_t tSerializeSClientHbReq(SEncoder *pEncoder, const SClientHbReq *pR TAOS_CHECK_RETURN(tEncodeSKv(pEncoder, kv)); pIter = taosHashIterate(pReq->info, pIter); } + TAOS_CHECK_RETURN(tEncodeU32(pEncoder, pReq->userIp)); + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pReq->userApp)); return 0; } @@ -399,6 +401,10 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq) return terrno = code; } } + if (!tDecodeIsEnd(pDecoder)) { + TAOS_CHECK_RETURN(tDecodeU32(pDecoder, &pReq->userIp)); + TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pReq->userApp)); + } return 0; } diff --git a/source/common/src/systable.c b/source/common/src/systable.c index d86e5bf326..944dcd07dc 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -530,6 +530,8 @@ static const SSysDbTableSchema connectionsSchema[] = { {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, {.name = "login_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, + {.name = "user_app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, + {.name = "user_ip", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysDbTableSchema consumerSchema[] = { @@ -571,6 +573,8 @@ static const SSysDbTableSchema querySchema[] = { {.name = "sub_num", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "sub_status", .bytes = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "user_app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "user_ip", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysDbTableSchema appSchema[] = { diff --git a/source/common/src/tcol.c b/source/common/src/tcol.c index 55a4b21208..41075b48e9 100644 --- a/source/common/src/tcol.c +++ b/source/common/src/tcol.c @@ -267,7 +267,7 @@ bool checkColumnEncode(char encode[TSDB_CL_COMPRESS_OPTION_LEN]) { } bool checkColumnEncodeOrSetDefault(uint8_t type, char encode[TSDB_CL_COMPRESS_OPTION_LEN]) { if (0 == strlen(encode)) { - strncpy(encode, getDefaultEncodeStr(type), TSDB_CL_COMPRESS_OPTION_LEN); + tstrncpy(encode, getDefaultEncodeStr(type), TSDB_CL_COMPRESS_OPTION_LEN); return true; } return checkColumnEncode(encode) && validColEncode(type, columnEncodeVal(encode)); @@ -284,7 +284,7 @@ bool checkColumnCompress(char compress[TSDB_CL_COMPRESS_OPTION_LEN]) { } bool checkColumnCompressOrSetDefault(uint8_t type, char compress[TSDB_CL_COMPRESS_OPTION_LEN]) { if (0 == strlen(compress)) { - strncpy(compress, getDefaultCompressStr(type), TSDB_CL_COMPRESS_OPTION_LEN); + tstrncpy(compress, getDefaultCompressStr(type), TSDB_CL_COMPRESS_OPTION_LEN); return true; } @@ -306,7 +306,7 @@ bool checkColumnLevel(char level[TSDB_CL_COMPRESS_OPTION_LEN]) { } bool checkColumnLevelOrSetDefault(uint8_t type, char level[TSDB_CL_COMPRESS_OPTION_LEN]) { if (0 == strlen(level)) { - strncpy(level, getDefaultLevelStr(type), TSDB_CL_COMPRESS_OPTION_LEN); + tstrncpy(level, getDefaultLevelStr(type), TSDB_CL_COMPRESS_OPTION_LEN); return true; } return checkColumnLevel(level) && validColCompressLevel(type, columnLevelVal(level)); @@ -330,7 +330,7 @@ void setColLevel(uint32_t* compress, uint8_t level) { int32_t setColCompressByOption(uint8_t type, uint8_t encode, uint16_t compressType, uint8_t level, bool check, uint32_t* compress) { - if(compress == NULL) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; + if (compress == NULL) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; if (check && !validColEncode(type, encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; setColEncode(compress, encode); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 94ebcc7033..bd18c9ceb9 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2494,12 +2494,12 @@ static int32_t formatTimestamp(char* buf, size_t cap, int64_t val, int precision } } struct tm ptm = {0}; - if (taosLocalTime(&tt, &ptm, buf, cap) == NULL) { + if (taosLocalTime(&tt, &ptm, buf, cap, NULL) == NULL) { code = TSDB_CODE_INTERNAL_ERROR; TSDB_CHECK_CODE(code, lino, _end); } - size_t pos = strftime(buf, cap, "%Y-%m-%d %H:%M:%S", &ptm); + size_t pos = taosStrfTime(buf, cap, "%Y-%m-%d %H:%M:%S", &ptm); if (pos == 0) { code = TSDB_CODE_OUT_OF_BUFFER; TSDB_CHECK_CODE(code, lino, _end); @@ -2641,7 +2641,7 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf char* pData = colDataGetVarData(pColInfoData, j); int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); memset(pBuf, 0, sizeof(pBuf)); - code = taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); + code = taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf, NULL); if (code < 0) { uError("func %s failed to convert to ucs charset since %s", __func__, tstrerror(code)); lino = __LINE__; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index db43ffd931..fc06f6fff3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -330,6 +330,7 @@ bool tsExperimental = true; int32_t tsMaxTsmaNum = 3; int32_t tsMaxTsmaCalcDelay = 600; int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24; // in ms, default to 1d +void* pTimezoneNameMap = NULL; #define TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, pName) \ if ((pItem = cfgGetItem(pCfg, pName)) == NULL) { \ @@ -1364,34 +1365,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { static int32_t taosSetSystemCfg(SConfig *pCfg) { SConfigItem *pItem = NULL; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "timezone"); - if (0 == strlen(pItem->str)) { - uError("timezone is not set"); - } else { - TAOS_CHECK_RETURN(osSetTimezone(pItem->str)); - uDebug("timezone format changed from %s to %s", pItem->str, tsTimezoneStr); - } - TAOS_CHECK_RETURN(cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype, true)); - - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "locale"); - const char *locale = pItem->str; - - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "charset"); - const char *charset = pItem->str; - - int32_t code = taosSetSystemLocale(locale, charset); - if (TSDB_CODE_SUCCESS != code) { - uError("failed to set locale:%s, since: %s", locale, tstrerror(code)); - char curLocale[TD_LOCALE_LEN] = {0}; - char curCharset[TD_CHARSET_LEN] = {0}; - taosGetSystemLocale(curLocale, curCharset); - if (0 != strlen(curLocale) && 0 != strlen(curCharset)) { - uInfo("current locale: %s, charset: %s", curLocale, curCharset); - } - } - - osSetSystemLocale(locale, charset); - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "enableCoreFile"); tsEnableCoreFile = pItem->bval; taosSetCoreDump(tsEnableCoreFile); @@ -2126,6 +2099,9 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; + if (strcasecmp("charset", name) == 0 || strcasecmp("timezone", name) == 0) { + goto _out; + } cfgLock(pCfg); SConfigItem *pItem = cfgGetItem(pCfg, name); @@ -2217,18 +2193,15 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { case 'l': { if (strcasecmp("locale", name) == 0) { SConfigItem *pLocaleItem = cfgGetItem(pCfg, "locale"); - SConfigItem *pCharsetItem = cfgGetItem(pCfg, "charset"); - if (pLocaleItem == NULL || pCharsetItem == NULL) { - uError("failed to get locale or charset from cfg"); + if (pLocaleItem == NULL) { + uError("failed to get locale from cfg"); code = TSDB_CODE_CFG_NOT_FOUND; goto _out; } const char *locale = pLocaleItem->str; - const char *charset = pCharsetItem->str; - TAOS_CHECK_GOTO(taosSetSystemLocale(locale, charset), &lino, _out); - osSetSystemLocale(locale, charset); - uInfo("locale set to '%s', charset set to '%s'", locale, charset); + TAOS_CHECK_GOTO(taosSetSystemLocale(locale), &lino, _out); + uInfo("locale set to '%s'", locale); matched = true; } break; @@ -2303,13 +2276,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { break; } case 't': { - if (strcasecmp("timezone", name) == 0) { - TAOS_CHECK_GOTO(osSetTimezone(pItem->str), &lino, _out); - uInfo("%s set from %s to %s", name, tsTimezoneStr, pItem->str); - - TAOS_CHECK_GOTO(cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype, false), &lino, _out); - matched = true; - } else if (strcasecmp("tempDir", name) == 0) { + if (strcasecmp("tempDir", name) == 0) { uInfo("%s set from %s to %s", name, tsTempDir, pItem->str); tstrncpy(tsTempDir, pItem->str, PATH_MAX); TAOS_CHECK_GOTO(taosExpandDir(tsTempDir, tsTempDir, PATH_MAX), &lino, _out); diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index 8988fab56a..5ed0c78569 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -22,12 +22,15 @@ int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) { pEp->port = 0; memset(pEp->fqdn, 0, TSDB_FQDN_LEN); - strncpy(pEp->fqdn, ep, TSDB_FQDN_LEN - 1); + tstrncpy(pEp->fqdn, ep, TSDB_FQDN_LEN); char* temp = strchr(pEp->fqdn, ':'); if (temp) { *temp = 0; - pEp->port = atoi(temp + 1); + pEp->port = taosStr2UInt16(temp + 1, NULL, 10); + if (pEp->port < 0) { + return TSDB_CODE_INVALID_PARA; + } } if (pEp->port == 0) { @@ -282,7 +285,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { locked = 1; while ((pItem = cfgNextIter(pIter)) != NULL) { -_start: + _start: col = startCol; // GRANT_CFG_SKIP; @@ -297,11 +300,11 @@ _start: TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, name, false), NULL, _exit); - char value[TSDB_CONFIG_PATH_LEN + VARSTR_HEADER_SIZE] = {0}; - int32_t valueLen = 0; + char value[TSDB_CONFIG_PATH_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t valueLen = 0; SDiskCfg* pDiskCfg = NULL; if (strcasecmp(pItem->name, "dataDir") == 0 && exSize > 0) { - char* buf = &value[VARSTR_HEADER_SIZE]; + char* buf = &value[VARSTR_HEADER_SIZE]; pDiskCfg = taosArrayGet(pItem->array, index); valueLen = tsnprintf(buf, TSDB_CONFIG_PATH_LEN, "%s", pDiskCfg->dir); index++; @@ -334,7 +337,7 @@ _start: if (strcasecmp(pItem->name, "dataDir") == 0 && pDiskCfg) { char* buf = &info[VARSTR_HEADER_SIZE]; valueLen = tsnprintf(buf, TSDB_CONFIG_INFO_LEN, "level %d primary %d disabled %" PRIi8, pDiskCfg->level, - pDiskCfg->primary, pDiskCfg->disable); + pDiskCfg->primary, pDiskCfg->disable); } else { valueLen = 0; } @@ -351,7 +354,7 @@ _start: if (index > 0 && index <= exSize) { goto _start; } -} + } pBlock->info.rows = numOfRows; _exit: if (locked) cfgUnLock(pConf); diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 964741e2c4..9ced37eb38 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -20,73 +20,6 @@ #define VALID_NAME_TYPE(x) ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T) -#if 0 -int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, int64_t intervalTime, char timeUnit, int16_t precision) { - if (slidingTime == 0) { - return startTime; - } - int64_t start = startTime; - if (timeUnit == 'n' || timeUnit == 'y') { - start /= 1000; - if (precision == TSDB_TIME_PRECISION_MICRO) { - start /= 1000; - } - struct tm tm; - time_t t = (time_t)start; - taosLocalTime(&t, &tm, NULL, 0); - tm.tm_sec = 0; - tm.tm_min = 0; - tm.tm_hour = 0; - tm.tm_mday = 1; - - if (timeUnit == 'y') { - tm.tm_mon = 0; - tm.tm_year = (int)(tm.tm_year / slidingTime * slidingTime); - } else { - int mon = tm.tm_year * 12 + tm.tm_mon; - mon = (int)(mon / slidingTime * slidingTime); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - } - - start = mktime(&tm) * 1000L; - if (precision == TSDB_TIME_PRECISION_MICRO) { - start *= 1000L; - } - } else { - int64_t delta = startTime - intervalTime; - int32_t factor = delta > 0? 1:-1; - - start = (delta / slidingTime + factor) * slidingTime; - - if (timeUnit == 'd' || timeUnit == 'w') { - /* - * here we revised the start time of day according to the local time zone, - * but in case of DST, the start time of one day need to be dynamically decided. - */ - // todo refactor to extract function that is available for Linux/Windows/Mac platform -#if defined(WINDOWS) && _MSC_VER >= 1900 - // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 - int64_t timezone = _timezone; - int32_t daylight = _daylight; - char** tzname = _tzname; -#endif - - int64_t t = (precision == TSDB_TIME_PRECISION_MILLI) ? MILLISECOND_PER_SECOND : MILLISECOND_PER_SECOND * 1000L; - start += timezone * t; - } - - int64_t end = start + intervalTime - 1; - if (end < startTime) { - start += slidingTime; - } - } - - return start; -} - -#endif - void toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) { if (pName == NULL){ return; diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index b24ded5c0c..9746fea034 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -25,42 +25,26 @@ #include "tlog.h" -// ==== mktime() kernel code =================// -static int64_t m_deltaUtc = 0; - -void deltaToUtcInitOnce() { - struct tm tm = {0}; - if (taosStrpTime("1970-01-01 00:00:00", (const char*)("%Y-%m-%d %H:%M:%S"), &tm) == NULL) { - uError("failed to parse time string"); - } - m_deltaUtc = (int64_t)taosMktime(&tm); - // printf("====delta:%lld\n\n", seconds); -} - static int32_t parseFraction(char* str, char** end, int32_t timePrec, int64_t* pFraction); static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim); -static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); -static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); +static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim, timezone_t tz); static char* forwardToTimeStringEnd(char* str); static bool checkTzPresent(const char* str, int32_t len); static int32_t parseTimezone(char* str, int64_t* tzOffset); -static int32_t (*parseLocaltimeFp[])(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) = { - parseLocaltime, parseLocaltimeDst}; - -int32_t taosParseTime(const char* timestr, int64_t* utime, int32_t len, int32_t timePrec, int8_t day_light) { +int32_t taosParseTime(const char* timestr, int64_t* utime, int32_t len, int32_t timePrec, timezone_t tz) { /* parse datatime string in with tz */ if (strnchr(timestr, 'T', len, false) != NULL) { if (checkTzPresent(timestr, len)) { return parseTimeWithTz(timestr, utime, timePrec, 'T'); } else { - return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 'T'); + return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 'T', tz); } } else { if (checkTzPresent(timestr, len)) { return parseTimeWithTz(timestr, utime, timePrec, 0); } else { - return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 0); + return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 0, tz); } } } @@ -146,8 +130,15 @@ int32_t parseFraction(char* str, char** end, int32_t timePrec, int64_t* pFractio TAOS_RETURN(TSDB_CODE_SUCCESS); } +#define PARSE(str,len,result) \ + if (len != 2) {\ + TAOS_RETURN(TSDB_CODE_INVALID_PARA);\ + }\ + result = strnatoi(str, len); + int32_t parseTimezone(char* str, int64_t* tzOffset) { int64_t hour = 0; + int64_t minute = 0; int32_t i = 0; if (str[i] != '+' && str[i] != '-') { @@ -168,27 +159,29 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { char* sep = strchr(&str[i], ':'); if (sep != NULL) { - int32_t len = (int32_t)(sep - &str[i]); + int32_t hourSize = (int32_t)(sep - &str[i]); + PARSE(&str[i], hourSize, hour); - hour = strnatoi(&str[i], len); - i += len + 1; + i += hourSize + 1; + size_t minSize = strlen(&str[i]); + PARSE(&str[i], minSize, minute); } else { - hour = strnatoi(&str[i], 2); - i += 2; + size_t hourSize = strlen(&str[i]); + if (hourSize > 2){ + hourSize = 2; + } + PARSE(&str[i], hourSize, hour) + i += hourSize; + size_t minSize = strlen(&str[i]); + if (minSize > 0){ + PARSE(&str[i], minSize, minute); + } } - if (hour > 12 || hour < 0) { + if (hour > 13 || hour < 0) { TAOS_RETURN(TSDB_CODE_INVALID_PARA); } - - // return error if there're illegal charaters after min(2 Digits) - char* minStr = &str[i]; - if (minStr[1] != '\0' && minStr[2] != '\0') { - TAOS_RETURN(TSDB_CODE_INVALID_PARA); - } - - int64_t minute = strnatoi(&str[i], 2); - if (minute > 59 || (hour == 12 && minute > 0)) { + if (minute > 59 || minute < 0) { TAOS_RETURN(TSDB_CODE_INVALID_PARA); } @@ -245,7 +238,10 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch int64_t seconds = user_mktime64(tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, 0); // int64_t seconds = gmtime(&tm); #else - int64_t seconds = timegm(&tm); + int64_t seconds = taosTimeGm(&tm); + if (seconds == -1){ + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); + } #endif int64_t fraction = 0; @@ -310,48 +306,7 @@ static FORCE_INLINE bool validateTm(struct tm* pTm) { return true; } -int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) { - *utime = 0; - struct tm tm = {0}; - - char* str; - if (delim == 'T') { - str = taosStrpTime(timestr, "%Y-%m-%dT%H:%M:%S", &tm); - } else if (delim == 0) { - str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm); - } else { - str = NULL; - } - - if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { - // if parse failed, try "%Y-%m-%d" format - str = taosStrpTime(timestr, "%Y-%m-%d", &tm); - if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { - TAOS_RETURN(TSDB_CODE_INVALID_PARA); - } - } - -#ifdef _MSC_VER -#if _MSC_VER >= 1900 - int64_t timezone = _timezone; -#endif -#endif - - int64_t seconds = - user_mktime64(tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, timezone); - - int64_t fraction = 0; - - if (*str == '.') { - /* parse the second fraction part */ - TAOS_CHECK_RETURN(parseFraction(str + 1, &str, timePrec, &fraction)); - } - - *utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction; - TAOS_RETURN(TSDB_CODE_SUCCESS); -} - -int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) { +int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim, timezone_t tz) { *utime = 0; struct tm tm = {0}; tm.tm_isdst = -1; @@ -373,8 +328,7 @@ int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t ti } } - /* mktime will be affected by TZ, set by using taos_options */ - int64_t seconds = taosMktime(&tm); + int64_t seconds = taosMktime(&tm, tz); int64_t fraction = 0; if (*str == '.') { @@ -535,7 +489,7 @@ int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char TAOS_RETURN(TSDB_CODE_SUCCESS); } -int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal) { +int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal, timezone_t tz, void* charsetCxt) { int32_t charLen = varDataLen(inputData); char* newColData; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) { @@ -544,7 +498,7 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec TAOS_RETURN(terrno); } (void)memcpy(newColData, varDataVal(inputData), charLen); - int32_t ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight); + int32_t ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tz); if (ret != TSDB_CODE_SUCCESS) { taosMemoryFree(newColData); TAOS_RETURN(TSDB_CODE_INVALID_TIMESTAMP); @@ -555,13 +509,13 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec if (NULL == newColData) { TAOS_RETURN(terrno); } - int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData); + int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData, charsetCxt); if (len < 0) { taosMemoryFree(newColData); TAOS_RETURN(TSDB_CODE_FAILED); } newColData[len] = 0; - int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tsDaylight); + int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tz); if (ret != TSDB_CODE_SUCCESS) { taosMemoryFree(newColData); TAOS_RETURN(ret); @@ -678,7 +632,7 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati static bool taosIsLeapYear(int32_t year) { return (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)); } -int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { +int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision, timezone_t tz) { if (duration == 0) { return t; } @@ -693,7 +647,10 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { struct tm tm; time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision)); - struct tm* ptm = taosLocalTime(&tt, &tm, NULL, 0); + if(taosLocalTime(&tt, &tm, NULL, 0, tz) == NULL) { + uError("failed to convert time to gm time, code:%d", errno); + return t; + } int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)numOfMonth; tm.tm_year = mon / 12; tm.tm_mon = mon % 12; @@ -704,7 +661,13 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { if (tm.tm_mday > daysOfMonth[tm.tm_mon]) { tm.tm_mday = daysOfMonth[tm.tm_mon]; } - return (int64_t)(taosMktime(&tm) * TSDB_TICK_PER_SECOND(precision) + fraction); + + tt = taosMktime(&tm, tz); + if (tt == -1){ + uError("failed to convert gm time to time, code:%d", errno); + return t; + } + return (int64_t)(tt * TSDB_TICK_PER_SECOND(precision) + fraction); } /** @@ -743,7 +706,7 @@ int32_t taosTimeCountIntervalForFill(int64_t skey, int64_t ekey, int64_t interva ekey = skey; skey = tmp; } - int32_t ret; + int32_t ret = 0; if (unit != 'n' && unit != 'y') { ret = (int32_t)((ekey - skey) / interval); @@ -754,11 +717,17 @@ int32_t taosTimeCountIntervalForFill(int64_t skey, int64_t ekey, int64_t interva struct tm tm; time_t t = (time_t)skey; - struct tm* ptm = taosLocalTime(&t, &tm, NULL, 0); + if (taosLocalTime(&t, &tm, NULL, 0, NULL) == NULL) { + uError("%s failed to convert time to local time, code:%d", __FUNCTION__, errno); + return ret; + } int32_t smon = tm.tm_year * 12 + tm.tm_mon; t = (time_t)ekey; - ptm = taosLocalTime(&t, &tm, NULL, 0); + if (taosLocalTime(&t, &tm, NULL, 0, NULL) == NULL) { + uError("%s failed to convert time to local time, code:%d", __FUNCTION__, errno); + return ret; + } int32_t emon = tm.tm_year * 12 + tm.tm_mon; if (unit == 'y') { @@ -782,7 +751,10 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { start /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; time_t tt = (time_t)start; - struct tm* ptm = taosLocalTime(&tt, &tm, NULL, 0); + if (taosLocalTime(&tt, &tm, NULL, 0, pInterval->timezone) == NULL){ + uError("%s failed to convert time to local time, code:%d", __FUNCTION__, errno); + return ts; + } tm.tm_sec = 0; tm.tm_min = 0; tm.tm_hour = 0; @@ -798,7 +770,12 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { tm.tm_mon = mon % 12; } - start = (int64_t)(taosMktime(&tm) * TSDB_TICK_PER_SECOND(precision)); + tt = taosMktime(&tm, pInterval->timezone); + if (tt == -1){ + uError("%s failed to convert local time to time, code:%d", __FUNCTION__, errno); + return ts; + } + start = (int64_t)(tt * TSDB_TICK_PER_SECOND(precision)); } else { if (IS_CALENDAR_TIME_DURATION(pInterval->intervalUnit)) { int64_t news = (ts / pInterval->sliding) * pInterval->sliding; @@ -812,12 +789,12 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { start = news; if (news <= ts) { int64_t prev = news; - int64_t newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + int64_t newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision, pInterval->timezone) - 1; if (newe < ts) { // move towards the greater endpoint while (newe < ts && news < ts) { news += pInterval->sliding; - newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision, pInterval->timezone) - 1; } prev = news; @@ -825,7 +802,7 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { while (newe >= ts) { prev = news; news -= pInterval->sliding; - newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision, pInterval->timezone) - 1; } } @@ -847,8 +824,6 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { // see // https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 int64_t timezone = _timezone; - int32_t daylight = _daylight; - char** tzname = _tzname; #endif start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); @@ -858,7 +833,7 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { // not enough time range if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { - end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; + end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision, pInterval->timezone) - 1; while (end < ts) { // move forward to the correct time window start += pInterval->sliding; @@ -877,15 +852,15 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { if (pInterval->offset > 0) { // try to move current window to the left-hande-side, due to the offset effect. - int64_t newe = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; + int64_t newe = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision, pInterval->timezone) - 1; int64_t slidingStart = start; while (newe >= ts) { start = slidingStart; - slidingStart = taosTimeAdd(slidingStart, -pInterval->sliding, pInterval->slidingUnit, precision); - int64_t news = taosTimeAdd(slidingStart, pInterval->offset, pInterval->offsetUnit, precision); - newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + slidingStart = taosTimeAdd(slidingStart, -pInterval->sliding, pInterval->slidingUnit, precision, pInterval->timezone); + int64_t news = taosTimeAdd(slidingStart, pInterval->offset, pInterval->offsetUnit, precision, pInterval->timezone); + newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision, pInterval->timezone) - 1; } - start = taosTimeAdd(start, pInterval->offset, pInterval->offsetUnit, precision); + start = taosTimeAdd(start, pInterval->offset, pInterval->offsetUnit, precision, pInterval->timezone); } return start; @@ -893,7 +868,7 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { // used together with taosTimeTruncate. when offset is great than zero, slide-start/slide-end is the anchor point int64_t taosTimeGetIntervalEnd(int64_t intervalStart, const SInterval* pInterval) { - return taosTimeAdd(intervalStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + return taosTimeAdd(intervalStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision, pInterval->timezone) - 1; } void calcIntervalAutoOffset(SInterval* interval) { @@ -912,7 +887,7 @@ void calcIntervalAutoOffset(SInterval* interval) { TSKEY news = start; while (news <= skey) { start = news; - news = taosTimeAdd(start, interval->sliding, interval->slidingUnit, interval->precision); + news = taosTimeAdd(start, interval->sliding, interval->slidingUnit, interval->precision, interval->timezone); if (news < start) { // overflow happens uError("%s failed and skip, skey [%" PRId64 "], inter[%" PRId64 "(%c)], slid[%" PRId64 "(%c)], precision[%d]", @@ -938,15 +913,15 @@ const char* fmtts(int64_t ts) { if (ts > -62135625943 && ts < 32503651200) { time_t t = (time_t)ts; - if (taosLocalTime(&t, &tm, buf, sizeof(buf)) == NULL) { + if (taosLocalTime(&t, &tm, buf, sizeof(buf), NULL) == NULL) { return buf; } - pos += strftime(buf + pos, sizeof(buf), "s=%Y-%m-%d %H:%M:%S", &tm); + pos += taosStrfTime(buf + pos, sizeof(buf), "s=%Y-%m-%d %H:%M:%S", &tm); } if (ts > -62135625943000 && ts < 32503651200000) { time_t t = (time_t)(ts / 1000); - if (taosLocalTime(&t, &tm, buf, sizeof(buf)) == NULL) { + if (taosLocalTime(&t, &tm, buf, sizeof(buf), NULL) == NULL) { return buf; } if (pos > 0) { @@ -954,13 +929,13 @@ const char* fmtts(int64_t ts) { buf[pos++] = '|'; buf[pos++] = ' '; } - pos += strftime(buf + pos, sizeof(buf), "ms=%Y-%m-%d %H:%M:%S", &tm); + pos += taosStrfTime(buf + pos, sizeof(buf), "ms=%Y-%m-%d %H:%M:%S", &tm); pos += sprintf(buf + pos, ".%03d", (int32_t)(ts % 1000)); } { time_t t = (time_t)(ts / 1000000); - if (taosLocalTime(&t, &tm, buf, sizeof(buf)) == NULL) { + if (taosLocalTime(&t, &tm, buf, sizeof(buf), NULL) == NULL) { return buf; } if (pos > 0) { @@ -968,7 +943,7 @@ const char* fmtts(int64_t ts) { buf[pos++] = '|'; buf[pos++] = ' '; } - pos += strftime(buf + pos, sizeof(buf), "us=%Y-%m-%d %H:%M:%S", &tm); + pos += taosStrfTime(buf + pos, sizeof(buf), "us=%Y-%m-%d %H:%M:%S", &tm); pos += sprintf(buf + pos, ".%06d", (int32_t)(ts % 1000000)); } @@ -1014,28 +989,28 @@ int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precisio TAOS_RETURN(TSDB_CODE_INVALID_PARA); } - if (NULL == taosLocalTime(", &ptm, buf, bufLen)) { + if (NULL == taosLocalTime(", &ptm, buf, bufLen, NULL)) { TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } - int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm); + int32_t length = (int32_t)taosStrfTime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm); length += tsnprintf(ts + length, fractionLen, format, mod); - length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm); + length += (int32_t)taosStrfTime(ts + length, 40 - length, "%z", &ptm); tstrncpy(buf, ts, bufLen); TAOS_RETURN(TSDB_CODE_SUCCESS); } -int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm) { +int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm, timezone_t tz) { tm->fsec = ts % TICK_PER_SECOND[precision] * (TICK_PER_SECOND[TSDB_TIME_PRECISION_NANO] / TICK_PER_SECOND[precision]); time_t t = ts / TICK_PER_SECOND[precision]; - if (NULL == taosLocalTime(&t, &tm->tm, NULL, 0)) { + if (NULL == taosLocalTime(&t, &tm->tm, NULL, 0, tz)) { TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } return TSDB_CODE_SUCCESS; } -int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision) { - *ts = taosMktime(&tm->tm); +int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision, timezone_t tz) { + *ts = taosMktime(&tm->tm, tz); *ts *= TICK_PER_SECOND[precision]; *ts += tm->fsec / (TICK_PER_SECOND[TSDB_TIME_PRECISION_NANO] / TICK_PER_SECOND[precision]); return TSDB_CODE_SUCCESS; @@ -1502,10 +1477,17 @@ static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int (void)sprintf(s, "%09" PRId64, tm->fsec); s += 9; break; - case TSFKW_TZH: - (void)sprintf(s, "%s%02d", tsTimezone < 0 ? "-" : "+", tsTimezone); + case TSFKW_TZH:{ +#ifdef WINDOWS + int32_t gmtoff = -_timezone; +#else + int32_t gmtoff = tm->tm.tm_gmtoff; +#endif + (void)sprintf(s, "%c%02d", (gmtoff >= 0) ? '+' : '-', + abs(gmtoff) / 3600); s += strlen(s); break; + } case TSFKW_YYYY: (void)sprintf(s, "%04d", tm->tm.tm_year + 1900); s += strlen(s); @@ -1635,13 +1617,13 @@ static bool needMoreDigits(SArray* formats, int32_t curIdx) { /// @retval -2 if datetime err, like 2023-13-32 25:61:69 /// @retval -3 if not supported static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t precision, const char** sErrPos, - int32_t* fErrIdx) { + int32_t* fErrIdx, timezone_t tz) { int32_t size = taosArrayGetSize(formats); int32_t pm = 0; // default am int32_t hour12 = 0; // default HH24 int32_t year = 0, mon = 0, yd = 0, md = 1, wd = 0; int32_t hour = 0, min = 0, sec = 0, us = 0, ms = 0, ns = 0; - int32_t tzSign = 1, tz = tsTimezone; + int32_t tzHour = 0; int32_t err = 0; bool withYD = false, withMD = false; @@ -1768,8 +1750,7 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec } } break; case TSFKW_TZH: { - tzSign = *s == '-' ? -1 : 1; - const char* newPos = tsFormatStr2Int32(&tz, s, -1, needMoreDigits(formats, i)); + const char* newPos = tsFormatStr2Int32(&tzHour, s, -1, needMoreDigits(formats, i)); if (NULL == newPos) err = -1; else { @@ -1922,14 +1903,21 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec tm.tm.tm_min = min; tm.tm.tm_sec = sec; if (!checkTm(&tm.tm)) return -2; - if (tz < -12 || tz > 12) return -2; + if (tzHour < -13 || tzHour > 13) return -2; tm.fsec = ms * 1000000 + us * 1000 + ns; - int32_t ret = taosTm2Ts(&tm, ts, precision); - *ts += 60 * 60 * (tsTimezone - tz) * TICK_PER_SECOND[precision]; + int32_t ret = taosTm2Ts(&tm, ts, precision, tz); + if (tzHour != 0) { +#ifdef WINDOWS + int32_t gmtoff = -_timezone; +#else + int32_t gmtoff = tm.tm.tm_gmtoff; +#endif + *ts += (gmtoff - tzHour * 3600) * TICK_PER_SECOND[precision]; + } return ret; } -int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen) { +int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen, timezone_t tz) { if (!*formats) { *formats = taosArrayInit(8, sizeof(TSFormatNode)); if (!*formats) { @@ -1938,12 +1926,12 @@ int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t pr TAOS_CHECK_RETURN(parseTsFormat(format, *formats)); } struct STm tm; - TAOS_CHECK_RETURN(taosTs2Tm(ts, precision, &tm)); + TAOS_CHECK_RETURN(taosTs2Tm(ts, precision, &tm, tz)); return tm2char(*formats, &tm, out, outLen); } int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, - char* errMsg, int32_t errMsgLen) { + char* errMsg, int32_t errMsgLen, timezone_t tz) { const char* sErrPos; int32_t fErrIdx; if (!*formats) { @@ -1953,7 +1941,7 @@ int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int } TAOS_CHECK_RETURN(parseTsFormat(format, *formats)); } - int32_t code = char2ts(tsStr, *formats, ts, precision, &sErrPos, &fErrIdx); + int32_t code = char2ts(tsStr, *formats, ts, precision, &sErrPos, &fErrIdx, tz); if (code == -1) { TSFormatNode* fNode = (taosArrayGet(*formats, fErrIdx)); snprintf(errMsg, errMsgLen, "mismatch format for: %s and %s", sErrPos, @@ -1978,7 +1966,7 @@ int32_t TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* ou } TAOS_CHECK_RETURN(parseTsFormat(format, formats)); struct STm tm; - TAOS_CHECK_GOTO(taosTs2Tm(ts, precision, &tm), NULL, _exit); + TAOS_CHECK_GOTO(taosTs2Tm(ts, precision, &tm, NULL), NULL, _exit); TAOS_CHECK_GOTO(tm2char(formats, &tm, out, outLen), NULL, _exit); _exit: @@ -1991,7 +1979,7 @@ int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const c int32_t fErrIdx; SArray* formats = taosArrayInit(4, sizeof(TSFormatNode)); TAOS_CHECK_RETURN(parseTsFormat(format, formats)); - int32_t code = char2ts(tsStr, formats, ts, precision, &sErrPos, &fErrIdx); + int32_t code = char2ts(tsStr, formats, ts, precision, &sErrPos, &fErrIdx, NULL); if (code == -1) { (void)printf("failed position: %s\n", sErrPos); (void)printf("failed format: %s\n", ((TSFormatNode*)taosArrayGet(formats, fErrIdx))->key->name); @@ -2107,3 +2095,30 @@ bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t i } return true; } + +int64_t taosGetTimestampToday(int32_t precision, timezone_t tz) { + int64_t factor = (precision == TSDB_TIME_PRECISION_SECONDS) ? 1 + : (precision == TSDB_TIME_PRECISION_MILLI) ? 1000 + : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000 + : 1000000000; + time_t t; + int32_t code = taosTime(&t); + if (code != 0) { + return -1; + } + struct tm tm; + if (taosLocalTime(&t, &tm, NULL, 0, tz) == NULL){ + uError("%s failed to get local time, code:%d", __FUNCTION__, errno); + return t; + } + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; + + time_t tmp = taosMktime(&tm, tz); + if (tmp == (time_t)-1) { + uError("%s failed to get timestamp of today, code:%d", __FUNCTION__, errno); + return t; + } + return (int64_t)tmp * factor; +} \ No newline at end of file diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index b85af42d1c..a6ffe2cd40 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -429,9 +429,9 @@ void test_timestamp_tm_conversion(int64_t ts, int32_t precision, int32_t y, int3 struct STm tm; taosFormatUtcTime(buf, 128, ts, precision); printf("formated ts of %ld, precision: %d is: %s\n", ts, precision, buf); - taosTs2Tm(ts, precision, &tm); + taosTs2Tm(ts, precision, &tm, NULL); check_tm(&tm, y, mon, d, h, m, s, fsec); - taosTm2Ts(&tm, &ts_tmp, precision); + taosTm2Ts(&tm, &ts_tmp, precision, NULL); ASSERT_EQ(ts, ts_tmp); } @@ -442,15 +442,15 @@ TEST(timeTest, timestamp2tm) { int64_t ts, tmp_ts = 0; struct STm tm; - ASSERT_EQ(TSDB_CODE_SUCCESS, taosParseTime(ts_str_ns, &ts, strlen(ts_str_ns), TSDB_TIME_PRECISION_NANO, 0)); + ASSERT_EQ(TSDB_CODE_SUCCESS, taosParseTime(ts_str_ns, &ts, strlen(ts_str_ns), TSDB_TIME_PRECISION_NANO, NULL)); test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_NANO, 2023 - 1900, 9 /* mon start from 0*/, 12, 11, 29, 0, 775726171L); - ASSERT_EQ(TSDB_CODE_SUCCESS, taosParseTime(ts_str_us, &ts, strlen(ts_str_us), TSDB_TIME_PRECISION_MICRO, 0)); + ASSERT_EQ(TSDB_CODE_SUCCESS, taosParseTime(ts_str_us, &ts, strlen(ts_str_us), TSDB_TIME_PRECISION_MICRO, NULL)); test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MICRO, 2023 - 1900, 9 /* mon start from 0*/, 12, 11, 29, 0, 775726000L); - ASSERT_EQ(TSDB_CODE_SUCCESS, taosParseTime(ts_str_ms, &ts, strlen(ts_str_ms), TSDB_TIME_PRECISION_MILLI, 0)); + ASSERT_EQ(TSDB_CODE_SUCCESS, taosParseTime(ts_str_ms, &ts, strlen(ts_str_ms), TSDB_TIME_PRECISION_MILLI, NULL)); test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MILLI, 2023 - 1900, 9 /* mon start from 0*/, 12, 11, 29, 0, 775000000L); @@ -477,7 +477,7 @@ void test_ts2char(int64_t ts, const char* format, int32_t precison, const char* TEST(timeTest, ts2char) { osDefaultInit(); - if (tsTimezone != TdEastZone8) GTEST_SKIP(); + if (taosGetLocalTimezoneOffset() != TdEastZone8) GTEST_SKIP(); int64_t ts; const char* format = "YYYY-MM-DD"; ts = 0; @@ -529,7 +529,7 @@ TEST(timeTest, ts2char) { TEST(timeTest, char2ts) { osDefaultInit(); - if (tsTimezone != TdEastZone8) GTEST_SKIP(); + if (taosGetLocalTimezoneOffset() != TdEastZone8) GTEST_SKIP(); int64_t ts; int32_t code = TEST_char2ts("YYYY-DD-MM HH12:MI:SS:MSPM", &ts, TSDB_TIME_PRECISION_MILLI, "2023-10-10 12:00:00.000AM"); @@ -630,7 +630,7 @@ TEST(timeTest, char2ts) { // default to 1970-1-1 00:00:00+08 -> 1969-12-31 16:00:00+00 ASSERT_EQ(0, TEST_char2ts("YYYY", &ts, TSDB_TIME_PRECISION_SECONDS, "1970")); - ASSERT_EQ(ts, -1 * tsTimezone * 60 * 60); + ASSERT_EQ(ts, -1 * taosGetLocalTimezoneOffset()); ASSERT_EQ(0, TEST_char2ts("yyyyMM1/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "210001/2")); ASSERT_EQ(ts, 4102502400000000LL); diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 4aa76d9284..c5aab38835 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -20,6 +20,7 @@ #include "tconfig.h" #include "tglobal.h" #include "version.h" +#include "tconv.h" #ifdef TD_JEMALLOC_ENABLED #include "jemalloc/jemalloc.h" #endif @@ -278,7 +279,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { printf("ERROR: Encrypt key overflow, it should be at most %d characters\n", ENCRYPT_KEY_LEN); return TSDB_CODE_INVALID_CFG; } - tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN); + tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN + 1); } else { printf("'-y' requires a parameter\n"); return TSDB_CODE_INVALID_CFG; @@ -477,7 +478,7 @@ int mainWindows(int argc, char **argv) { return code; } - if ((code = taosConvInit()) != 0) { + if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL) { dError("failed to init conv"); taosCloseLog(); taosCleanupArgs(); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index ba124739f1..566ebd8270 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -114,7 +114,7 @@ static void dmMayShouldUpdateAnalFunc(SDnodeMgmt *pMgmt, int64_t newVer) { .pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_RETRIEVE_ANAL_ALGO, - .info.ahandle = (void *)0x9527, + .info.ahandle = 0, .info.refId = 0, .info.noResp = 0, .info.handle = 0, @@ -197,7 +197,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { req.clusterCfg.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold; tstrncpy(req.clusterCfg.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN); char timestr[32] = "1970-01-01 00:00:00.00"; - if (taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0) != 0) { + if (taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, NULL) != 0) { dError("failed to parse time since %s", tstrerror(code)); } memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); @@ -380,7 +380,7 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { SConfig *pCfg = taosGetCfg(); - code = cfgSetItem(pCfg, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true); + code = cfgSetItem(pCfg, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_SERVER_CMD, true); if (code != 0) { if (strncasecmp(cfgReq.config, "resetlog", strlen("resetlog")) == 0) { code = 0; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index fd30555c3b..8e8d70118b 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -186,7 +186,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { #if defined(TD_ENTERPRISE) pCfg->tsdbCfg.encryptAlgorithm = pCreate->encryptAlgorithm; if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) { - strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN + 1); } #else pCfg->tsdbCfg.encryptAlgorithm = 0; @@ -202,7 +202,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { #if defined(TD_ENTERPRISE) pCfg->walCfg.encryptAlgorithm = pCreate->encryptAlgorithm; if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) { - strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN + 1); } #else pCfg->walCfg.encryptAlgorithm = 0; @@ -378,12 +378,11 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId); - if (vnodeCreate(path, &vnodeCfg, diskPrimary, pMgmt->pTfs) < 0) { - dError("vgId:%d, failed to create vnode since %s", req.vgId, terrstr()); + if ((code = vnodeCreate(path, &vnodeCfg, diskPrimary, pMgmt->pTfs)) < 0) { + dError("vgId:%d, failed to create vnode since %s", req.vgId, tstrerror(code)); vmReleaseVnode(pMgmt, pVnode); vmCleanPrimaryDisk(pMgmt, req.vgId); (void)tFreeSCreateVnodeReq(&req); - code = terrno != 0 ? terrno : -1; return code; } @@ -884,7 +883,7 @@ int32_t vmProcessArbHeartBeatReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { size_t size = taosArrayGetSize(arbHbReq.hbMembers); arbHbRsp.dnodeId = pMgmt->pData->dnodeId; - strncpy(arbHbRsp.arbToken, arbHbReq.arbToken, TSDB_ARB_TOKEN_SIZE); + tstrncpy(arbHbRsp.arbToken, arbHbReq.arbToken, TSDB_ARB_TOKEN_SIZE); arbHbRsp.hbMembers = taosArrayInit(size, sizeof(SVArbHbRspMember)); if (arbHbRsp.hbMembers == NULL) { goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 6a61cc3859..537c1f6297 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -94,7 +94,7 @@ static void vmUnRegisterCreatingState(SVnodeMgmt *pMgmt, int32_t vgId) { dTrace("vgId:%d, remove from creating Hash", vgId); r = taosHashRemove(pMgmt->creatingHash, &vgId, sizeof(int32_t)); if (r != 0) { - dError("vgId:%d, failed to remove vnode from hash", vgId); + dError("vgId:%d, failed to remove vnode from creatingHash", vgId); } (void)taosThreadRwlockUnlock(&pMgmt->lock); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 9f1c292a90..85d891c8d8 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -22,6 +22,7 @@ #include "tglobal.h" #include "tgrant.h" #include "tstream.h" +#include "tconv.h" static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); @@ -179,7 +180,7 @@ int32_t dmInitVars(SDnode *pDnode) { //code = taosGetCryptKey(tsAuthCode, pData->machineId, tsCryptKey); code = 0; - strncpy(tsEncryptKey, tsAuthCode, 16); + tstrncpy(tsEncryptKey, tsAuthCode, 16); if (code != 0) { if(code == -1){ @@ -220,6 +221,7 @@ int32_t dmInitVars(SDnode *pDnode) { } extern SMonVloadInfo tsVinfo; + void dmClearVars(SDnode *pDnode) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 1da13f72cd..26f45d2fb8 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -230,7 +230,7 @@ static int32_t dmWriteCheckCodeFile(char *file, char *realfile, char *key, bool } SCryptOpts opts; - strncpy(opts.key, key, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, key, ENCRYPT_KEY_LEN + 1); opts.len = len; opts.source = DM_KEY_INDICATOR; opts.result = result; @@ -349,7 +349,7 @@ static int32_t dmCompareEncryptKey(char *file, char *key, bool toLogFile) { } SCryptOpts opts = {0}; - strncpy(opts.key, key, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, key, ENCRYPT_KEY_LEN + 1); opts.len = len; opts.source = content; opts.result = result; @@ -551,7 +551,7 @@ int32_t dmGetEncryptKey() { goto _OVER; } - strncpy(tsEncryptKey, encryptKey, ENCRYPT_KEY_LEN); + strncpy(tsEncryptKey, encryptKey, ENCRYPT_KEY_LEN + 1); taosMemoryFreeClear(encryptKey); tsEncryptionKeyChksum = taosCalcChecksum(0, tsEncryptKey, strlen(tsEncryptKey)); tsEncryptionKeyStat = ENCRYPT_KEY_STAT_LOADED; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 2346e060f6..b733ee9cb2 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -493,7 +493,7 @@ typedef struct { int64_t dstTbUid; int8_t intervalUnit; int8_t slidingUnit; - int8_t timezone; + int8_t timezone; // int8_t is not enough, timezone is unit of second int32_t dstVgId; // for stream int64_t interval; int64_t offset; diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c index 0192044e67..3879e8e6e2 100644 --- a/source/dnode/mnode/impl/src/mndArbGroup.c +++ b/source/dnode/mnode/impl/src/mndArbGroup.c @@ -1315,7 +1315,7 @@ static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock continue; } char dbNameInGroup[TSDB_DB_FNAME_LEN]; - strncpy(dbNameInGroup, pVgObj->dbName, TSDB_DB_FNAME_LEN); + tstrncpy(dbNameInGroup, pVgObj->dbName, TSDB_DB_FNAME_LEN); sdbRelease(pSdb, pVgObj); char dbname[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index 82fc96a34c..ade7b63afa 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -244,7 +244,7 @@ int32_t mndCompactGetDbName(SMnode *pMnode, int32_t compactId, char *dbname, int TAOS_RETURN(code); } - (void)strncpy(dbname, pCompact->dbname, len); + tstrncpy(dbname, pCompact->dbname, len); mndReleaseCompact(pMnode, pCompact); TAOS_RETURN(code); } @@ -321,7 +321,7 @@ int32_t mndRetrieveCompact(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, TAOS_CHECK_GOTO(tNameFromString(&name, pCompact->dbname, T_NAME_ACCT | T_NAME_DB), &lino, _OVER); (void)tNameGetDbName(&name, varDataVal(tmpBuf)); } else { - (void)strncpy(varDataVal(tmpBuf), pCompact->dbname, TSDB_SHOW_SQL_LEN); + tstrncpy(varDataVal(tmpBuf), pCompact->dbname, TSDB_SHOW_SQL_LEN); } varDataSetLen(tmpBuf, strlen(varDataVal(tmpBuf))); RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)tmpBuf, false), pCompact, &lino, _OVER); @@ -516,11 +516,14 @@ int32_t mndProcessKillCompactReq(SRpcMsg *pReq) { code = TSDB_CODE_ACTION_IN_PROGRESS; - char obj[TSDB_INT32_ID_LEN] = {0}; - (void)sprintf(obj, "%d", pCompact->compactId); - - auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen); - + char obj[TSDB_INT32_ID_LEN] = {0}; + int32_t nBytes = snprintf(obj, sizeof(obj), "%d", pCompact->compactId); + if ((uint32_t)nBytes < sizeof(obj)) { + auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, + killCompactReq.sqlLen); + } else { + mError("compact:%" PRId32 " failed to audit since %s", pCompact->compactId, tstrerror(TSDB_CODE_OUT_OF_RANGE)); + } _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("failed to kill compact %" PRId32 " since %s", killCompactReq.compactId, terrstr()); @@ -643,7 +646,7 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) { char detail[1024] = {0}; int32_t len = tsnprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d", - TMSG_INFO(TDMT_VND_QUERY_COMPACT_PROGRESS), epSet.numOfEps, epSet.inUse); + TMSG_INFO(TDMT_VND_QUERY_COMPACT_PROGRESS), epSet.numOfEps, epSet.inUse); for (int32_t i = 0; i < epSet.numOfEps; ++i) { len += tsnprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); } @@ -954,7 +957,7 @@ static int32_t mndCompactDispatch(SRpcMsg *pReq) { continue; } - int64_t remainder = ((curMin + (int64_t)pDb->cfg.compactTimeOffset * 60LL) % pDb->cfg.compactInterval); + int64_t remainder = ((curMin - (int64_t)pDb->cfg.compactTimeOffset * 60LL) % pDb->cfg.compactInterval); if (remainder != 0) { mDebug("db:%p,%s, current time:%" PRIi64 "m is not divisible by compact interval:%dm, offset:%" PRIi8 "h, remainder:%" PRIi64 "m, skip", diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 2347de6cbc..0732f8742c 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -428,7 +428,12 @@ static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) { return TSDB_CODE_MND_INVALID_DB; } - int32_t acctId = atoi(dbName); + int32_t acctId; + int32_t code = taosStr2int32(dbName, &acctId); + if (code != 0) { + return code; + } + if (acctId != pUser->acctId) { return TSDB_CODE_MND_INVALID_DB_ACCT; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 02359bccfe..e785d44bc5 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -462,7 +462,7 @@ int32_t mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) { dInfo.isMnode = 0; } - if(taosArrayPush(pDnodeInfo, &dInfo) == NULL){ + if (taosArrayPush(pDnodeInfo, &dInfo) == NULL) { code = terrno; sdbCancelFetch(pSdb, pIter); break; @@ -471,12 +471,12 @@ int32_t mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) { TAOS_RETURN(code); } -#define CHECK_MONITOR_PARA(para,err) \ -if (pCfg->monitorParas.para != para) { \ - mError("dnode:%d, para:%d inconsistent with cluster:%d", pDnode->id, pCfg->monitorParas.para, para); \ - terrno = err; \ - return err;\ -} +#define CHECK_MONITOR_PARA(para, err) \ + if (pCfg->monitorParas.para != para) { \ + mError("dnode:%d, para:%d inconsistent with cluster:%d", pDnode->id, pCfg->monitorParas.para, para); \ + terrno = err; \ + return err; \ + } static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const SClusterCfg *pCfg) { CHECK_MONITOR_PARA(tsEnableMonitor, DND_REASON_STATUS_MONITOR_SWITCH_NOT_MATCH); @@ -486,7 +486,8 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const S CHECK_MONITOR_PARA(tsSlowLogScope, DND_REASON_STATUS_MONITOR_SLOW_LOG_SCOPE_NOT_MATCH); if (0 != strcasecmp(pCfg->monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb)) { - mError("dnode:%d, tsSlowLogExceptDb:%s inconsistent with cluster:%s", pDnode->id, pCfg->monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb); + mError("dnode:%d, tsSlowLogExceptDb:%s inconsistent with cluster:%s", pDnode->id, + pCfg->monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb); terrno = TSDB_CODE_DNODE_INVALID_MONITOR_PARAS; return DND_REASON_STATUS_MONITOR_NOT_MATCH; } @@ -582,8 +583,8 @@ static bool mndUpdateMnodeState(SMnodeObj *pObj, SMnodeLoad *pMload) { return stateChanged; } -extern char* tsMonFwUri; -extern char* tsMonSlowLogUri; +extern char *tsMonFwUri; +extern char *tsMonSlowLogUri; static int32_t mndProcessStatisReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStatisReq statisReq = {0}; @@ -595,9 +596,9 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { mInfo("process statis req,\n %s", statisReq.pCont); } - if (statisReq.type == MONITOR_TYPE_COUNTER){ + if (statisReq.type == MONITOR_TYPE_COUNTER) { monSendContent(statisReq.pCont, tsMonFwUri); - }else if(statisReq.type == MONITOR_TYPE_SLOW_LOG){ + } else if (statisReq.type == MONITOR_TYPE_SLOW_LOG) { monSendContent(statisReq.pCont, tsMonSlowLogUri); } @@ -1057,27 +1058,27 @@ _OVER: TAOS_RETURN(code); } -static void getSlowLogScopeString(int32_t scope, char* result){ - if(scope == SLOW_LOG_TYPE_NULL) { +static void getSlowLogScopeString(int32_t scope, char *result) { + if (scope == SLOW_LOG_TYPE_NULL) { (void)strcat(result, "NONE"); return; } - while(scope > 0){ - if(scope & SLOW_LOG_TYPE_QUERY) { + while (scope > 0) { + if (scope & SLOW_LOG_TYPE_QUERY) { (void)strcat(result, "QUERY"); scope &= ~SLOW_LOG_TYPE_QUERY; - } else if(scope & SLOW_LOG_TYPE_INSERT) { + } else if (scope & SLOW_LOG_TYPE_INSERT) { (void)strcat(result, "INSERT"); scope &= ~SLOW_LOG_TYPE_INSERT; - } else if(scope & SLOW_LOG_TYPE_OTHERS) { + } else if (scope & SLOW_LOG_TYPE_OTHERS) { (void)strcat(result, "OTHERS"); scope &= ~SLOW_LOG_TYPE_OTHERS; - } else{ + } else { (void)printf("invalid slow log scope:%d", scope); return; } - if(scope > 0) { + if (scope > 0) { (void)strcat(result, "|"); } } @@ -1439,7 +1440,7 @@ _OVER: static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { int32_t code = 0; - char *p = pMCfgReq->config; + char *p = pMCfgReq->config; while (*p) { if (*p == ' ') { break; @@ -1448,7 +1449,7 @@ static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { } size_t optLen = p - pMCfgReq->config; - (void)strncpy(pDCfgReq->config, pMCfgReq->config, optLen); + tstrncpy(pDCfgReq->config, pMCfgReq->config, optLen + 1); pDCfgReq->config[optLen] = 0; if (' ' == pMCfgReq->config[optLen]) { @@ -1540,7 +1541,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { tstrncpy(dcfgReq.config, cfgReq.config, TSDB_DNODE_CONFIG_LEN); tstrncpy(dcfgReq.value, cfgReq.value, TSDB_DNODE_VALUE_LEN); } else { - TAOS_CHECK_GOTO (mndMCfg2DCfg(&cfgReq, &dcfgReq), NULL, _err_out); + TAOS_CHECK_GOTO(mndMCfg2DCfg(&cfgReq, &dcfgReq), NULL, _err_out); if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) { mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId); code = TSDB_CODE_INVALID_CFG; @@ -1883,11 +1884,19 @@ static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32 if (' ' == pMCfgReq->config[optLen]) { // 'key value' if (strlen(pMCfgReq->value) != 0) goto _err; - *pOutValue = atoi(pMCfgReq->config + optLen + 1); + code = taosStr2int32(pMCfgReq->config + optLen + 1, pOutValue); + if (code != 0) { + mError("dnode:%d, failed to get cfg since %s", pMCfgReq->dnodeId, tstrerror(code)); + goto _err; + } } else { // 'key' 'value' if (strlen(pMCfgReq->value) == 0) goto _err; - *pOutValue = atoi(pMCfgReq->value); + code = taosStr2int32(pMCfgReq->value, pOutValue); + if (code != 0) { + mError("dnode:%d, failed to get cfg since %s", pMCfgReq->dnodeId, tstrerror(code)); + goto _err; + } } TAOS_RETURN(code); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 9dd43225b1..aaa4446ca8 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -15,8 +15,8 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" -#include "mndArbGroup.h" #include "mndAnode.h" +#include "mndArbGroup.h" #include "mndCluster.h" #include "mndCompact.h" #include "mndCompactDetail.h" @@ -254,7 +254,7 @@ static void mndIncreaseUpTime(SMnode *pMnode) { .pCont = pReq, .contLen = contLen, .info.notFreeAhandle = 1, - .info.ahandle = (void *)0x9527}; + .info.ahandle = 0}; // TODO check return value if (tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg) < 0) { mError("failed to put into write-queue since %s, line:%d", terrstr(), __LINE__); @@ -530,7 +530,7 @@ static int32_t mndInitWal(SMnode *pMnode) { code = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; TAOS_RETURN(code); } else { - (void)strncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN + 1); } } #endif @@ -713,7 +713,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { } char timestr[24] = "1970-01-01 00:00:00.00"; - code = taosParseTime(timestr, &pMnode->checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + code = taosParseTime(timestr, &pMnode->checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, NULL); if (code < 0) { mError("failed to parse time since %s", tstrerror(code)); (void)taosThreadRwlockDestroy(&pMnode->lock); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 6b1c97b399..c89fc26fb5 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -14,10 +14,10 @@ */ #define _DEFAULT_SOURCE +#include "mndMnode.h" #include "audit.h" #include "mndCluster.h" #include "mndDnode.h" -#include "mndMnode.h" #include "mndPrivilege.h" #include "mndShow.h" #include "mndSync.h" @@ -722,10 +722,13 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { code = mndCreateMnode(pMnode, pReq, pDnode, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; - char obj[40] = {0}; - sprintf(obj, "%d", createReq.dnodeId); - - auditRecord(pReq, pMnode->clusterId, "createMnode", "", obj, createReq.sql, createReq.sqlLen); + char obj[40] = {0}; + int32_t bytes = snprintf(obj, sizeof(obj), "%d", createReq.dnodeId); + if ((uint32_t)bytes < sizeof(obj)) { + auditRecord(pReq, pMnode->clusterId, "createMnode", "", obj, createReq.sql, createReq.sqlLen); + } else { + mError("mnode:%d, failed to audit create req since %s", createReq.dnodeId, tstrerror(TSDB_CODE_OUT_OF_RANGE)); + } _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index fd02367f6d..e691ecb904 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -14,12 +14,12 @@ */ #define _DEFAULT_SOURCE +#include "mndProfile.h" #include "audit.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" #include "mndPrivilege.h" -#include "mndProfile.h" #include "mndQnode.h" #include "mndShow.h" #include "mndSma.h" @@ -45,6 +45,8 @@ typedef struct { int32_t numOfQueries; SRWLatch queryLock; SArray *pQueries; // SArray + char userApp[TSDB_APP_NAME_LEN]; + uint32_t userIp; } SConnObj; typedef struct { @@ -135,6 +137,13 @@ void mndCleanupProfile(SMnode *pMnode) { } } +static void setUserInfo2Conn(SConnObj* connObj, char* userApp, uint32_t userIp){ + if (connObj == NULL){ + return; + } + tstrncpy(connObj->userApp, userApp, sizeof(connObj->userApp)); + connObj->userIp = userIp; +} static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType, uint32_t ip, uint16_t port, int32_t pid, const char *app, int64_t startTime) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; @@ -232,7 +241,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { SConnObj *pConn = NULL; int32_t code = 0; SConnectReq connReq = {0}; - char ip[24] = {0}; + char ip[TD_IP_LEN] = {0}; const STraceId *trace = &pReq->info.traceId; if ((code = tDeserializeSConnectReq(pReq->pCont, pReq->contLen, &connReq)) != 0) { @@ -244,7 +253,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto _OVER; } - taosIp2String(pReq->info.conn.clientIp, ip); + taosInetNtoa(ip, pReq->info.conn.clientIp); if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONNECT)) != 0) { mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, tstrerror(code)); goto _OVER; @@ -335,10 +344,13 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { code = 0; - char detail[1000] = {0}; - (void)sprintf(detail, "app:%s", connReq.app); - - auditRecord(pReq, pMnode->clusterId, "login", "", "", detail, strlen(detail)); + char detail[1000] = {0}; + int32_t nBytes = snprintf(detail, sizeof(detail), "app:%s", connReq.app); + if ((uint32_t)nBytes < sizeof(detail)) { + auditRecord(pReq, pMnode->clusterId, "login", "", "", detail, strlen(detail)); + } else { + mError("failed to audit logic since %s", tstrerror(TSDB_CODE_OUT_OF_RANGE)); + } _OVER: @@ -513,6 +525,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } } + setUserInfo2Conn(pConn, pHbReq->userApp, pHbReq->userIp); SQueryHbRspBasic *rspBasic = taosMemoryCalloc(1, sizeof(SQueryHbRspBasic)); if (rspBasic == NULL) { mndReleaseConn(pMnode, pConn, true); @@ -896,9 +909,10 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl return code; } - char endpoint[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; - (void)sprintf(&endpoint[VARSTR_HEADER_SIZE], "%s:%d", taosIpStr(pConn->ip), pConn->port); - varDataLen(endpoint) = strlen(&endpoint[VARSTR_HEADER_SIZE]); + char endpoint[TD_IP_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; + taosInetNtoa(varDataVal(endpoint), pConn->ip); + (void)sprintf(varDataVal(endpoint) + strlen(varDataVal(endpoint)), ":%d", pConn->port); + varDataLen(endpoint) = strlen(varDataVal(endpoint)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)endpoint, false); if (code != 0) { @@ -920,6 +934,27 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl return code; } + char userApp[TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE]; + STR_TO_VARSTR(userApp, pConn->userApp); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)userApp, false); + if (code != 0) { + mError("failed to set user app since %s", tstrerror(code)); + return code; + } + + char userIp[TD_IP_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; + if (pConn->userIp != 0 && pConn->userIp != INADDR_NONE){ + taosInetNtoa(varDataVal(userIp), pConn->userIp); + varDataLen(userIp) = strlen(varDataVal(userIp)); + } + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)userIp, false); + if (code != 0) { + mError("failed to set user ip since %s", tstrerror(code)); + return code; + } + numOfRows++; } @@ -1006,8 +1041,9 @@ static int32_t packQueriesIntoBlock(SShowObj *pShow, SConnObj *pConn, SSDataBloc return code; } - char endpoint[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; - (void)sprintf(&endpoint[VARSTR_HEADER_SIZE], "%s:%d", taosIpStr(pConn->ip), pConn->port); + char endpoint[TD_IP_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; + taosInetNtoa(varDataVal(endpoint), pConn->ip); + (void)sprintf(varDataVal(endpoint) + strlen(varDataVal(endpoint)), ":%d", pConn->port); varDataLen(endpoint) = strlen(&endpoint[VARSTR_HEADER_SIZE]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, curRowIndex, (const char *)endpoint, false); @@ -1091,6 +1127,29 @@ static int32_t packQueriesIntoBlock(SShowObj *pShow, SConnObj *pConn, SSDataBloc return code; } + char userApp[TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE]; + STR_TO_VARSTR(userApp, pConn->userApp); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, curRowIndex, (const char *)userApp, false); + if (code != 0) { + mError("failed to set user app since %s", tstrerror(code)); + taosRUnLockLatch(&pConn->queryLock); + return code; + } + + char userIp[TD_IP_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; + if (pConn->userIp != 0 && pConn->userIp != INADDR_NONE){ + taosInetNtoa(varDataVal(userIp), pConn->userIp); + varDataLen(userIp) = strlen(varDataVal(userIp)); + } + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, curRowIndex, (const char *)userIp, false); + if (code != 0) { + mError("failed to set user ip since %s", tstrerror(code)); + taosRUnLockLatch(&pConn->queryLock); + return code; + } + pBlock->info.rows++; } @@ -1165,9 +1224,9 @@ static int32_t mndRetrieveApps(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo return code; } - char ip[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; - (void)sprintf(&ip[VARSTR_HEADER_SIZE], "%s", taosIpStr(pApp->ip)); - varDataLen(ip) = strlen(&ip[VARSTR_HEADER_SIZE]); + char ip[TD_IP_LEN + VARSTR_HEADER_SIZE] = {0}; + taosInetNtoa(varDataVal(ip), pApp->ip); + varDataLen(ip) = strlen(varDataVal(ip)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); code = colDataSetVal(pColInfo, numOfRows, (const char *)ip, false); if (code != 0) { diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index a54c7f1b14..9db7ae0156 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndSma.h" +#include "functionMgt.h" #include "mndDb.h" #include "mndDnode.h" #include "mndIndex.h" @@ -31,7 +32,6 @@ #include "mndVgroup.h" #include "parser.h" #include "tname.h" -#include "functionMgt.h" #define TSDB_SMA_VER_NUMBER 1 #define TSDB_SMA_RESERVE_SIZE 64 @@ -48,8 +48,8 @@ static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq); static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndDestroySmaObj(SSmaObj *pSmaObj); -static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq); -static int32_t mndProcessDropTSMAReq(SRpcMsg* pReq); +static int32_t mndProcessCreateTSMAReq(SRpcMsg *pReq); +static int32_t mndProcessDropTSMAReq(SRpcMsg *pReq); // sma and tag index comm func static int32_t mndProcessDropIdxReq(SRpcMsg *pReq); @@ -61,11 +61,11 @@ static void mndCancelRetrieveTSMA(SMnode *pMnode, void *pIter); static int32_t mndProcessGetTbTSMAReq(SRpcMsg *pReq); typedef struct SCreateTSMACxt { - SMnode * pMnode; + SMnode *pMnode; const SRpcMsg *pRpcReq; union { const SMCreateSmaReq *pCreateSmaReq; - const SMDropSmaReq * pDropSmaReq; + const SMDropSmaReq *pDropSmaReq; }; SDbObj *pDb; SStbObj *pSrcStb; @@ -298,16 +298,13 @@ void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma) { sdbRelease(pSdb, pSma); } -SDbObj *mndAcquireDbBySma(SMnode *pMnode, const char *db) { - - return mndAcquireDb(pMnode, db); -} +SDbObj *mndAcquireDbBySma(SMnode *pMnode, const char *db) { return mndAcquireDb(pMnode, db); } static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) { SEncoder encoder = {0}; int32_t contLen = 0; SName name = {0}; - int32_t code = tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + int32_t code = tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); if (TSDB_CODE_SUCCESS != code) { return NULL; } @@ -316,7 +313,7 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm req.version = 0; req.intervalUnit = pSma->intervalUnit; req.slidingUnit = pSma->slidingUnit; - req.timezoneInt = pSma->timezone; +// req.timezoneInt = pSma->timezone; tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); req.exprLen = pSma->exprLen; req.tagsFilterLen = pSma->tagsFilterLen; @@ -368,7 +365,7 @@ static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, SEncoder encoder = {0}; int32_t contLen; SName name = {0}; - int32_t code = tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + int32_t code = tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); if (TSDB_CODE_SUCCESS != code) { terrno = code; return NULL; @@ -424,9 +421,9 @@ static int32_t mndSetCreateSmaRedoLogs(SMnode *pMnode, STrans *pTrans, SSmaObj * TAOS_RETURN(code); } -static int32_t mndSetCreateSmaUndoLogs(SMnode* pMnode, STrans* pTrans, SSmaObj* pSma) { - int32_t code = 0; - SSdbRaw * pUndoRaw = mndSmaActionEncode(pSma); +static int32_t mndSetCreateSmaUndoLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + int32_t code = 0; + SSdbRaw *pUndoRaw = mndSmaActionEncode(pSma); if (!pUndoRaw) { code = TSDB_CODE_MND_RETURN_VALUE_NULL; if (terrno != 0) code = terrno; @@ -617,9 +614,9 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea smaObj.intervalUnit = pCreate->intervalUnit; smaObj.slidingUnit = pCreate->slidingUnit; #if 0 - smaObj.timezone = pCreate->timezone; +// smaObj.timezone = pCreate->timezone; #endif - smaObj.timezone = tsTimezone; // use timezone of server +// smaObj.timezone = taosGetLocalTimezoneOffset(); // use timezone of server smaObj.interval = pCreate->interval; smaObj.offset = pCreate->offset; smaObj.sliding = pCreate->sliding; @@ -670,7 +667,8 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea // check the maxDelay if (streamObj.conf.triggerParam < TSDB_MIN_ROLLUP_MAX_DELAY) { int64_t msInterval = -1; - int32_t code = convertTimeFromPrecisionToUnit(pCreate->interval, pDb->cfg.precision, TIME_UNIT_MILLISECOND, &msInterval); + int32_t code = + convertTimeFromPrecisionToUnit(pCreate->interval, pDb->cfg.precision, TIME_UNIT_MILLISECOND, &msInterval); if (TSDB_CODE_SUCCESS != code) { mError("sma:%s, failed to create since convert time failed: %s", smaObj.name, tstrerror(code)); return code; @@ -792,12 +790,12 @@ static int32_t mndCheckCreateSmaReq(SMCreateSmaReq *pCreate) { } static int32_t mndGetStreamNameFromSmaName(char *streamName, char *smaName) { - SName n; + SName n; int32_t code = tNameFromString(&n, smaName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); if (TSDB_CODE_SUCCESS != code) { return code; } - sprintf(streamName, "%d.%s", n.acctId, n.tname); + snprintf(streamName, TSDB_TABLE_FNAME_LEN,"%d.%s", n.acctId, n.tname); return TSDB_CODE_SUCCESS; } @@ -984,10 +982,10 @@ static int32_t mndSetDropSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SD } static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *pSma) { - int32_t code = -1; - SVgObj *pVgroup = NULL; - SStbObj *pStb = NULL; - STrans *pTrans = NULL; + int32_t code = -1; + SVgObj *pVgroup = NULL; + SStbObj *pStb = NULL; + STrans *pTrans = NULL; SStreamObj *pStream = NULL; pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId); @@ -1023,7 +1021,6 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p goto _OVER; } - code = mndAcquireStream(pMnode, streamName, &pStream); if (pStream == NULL || pStream->smaId != pSma->uid || code != 0) { sdbRelease(pMnode->pSdb, pStream); @@ -1124,8 +1121,8 @@ _OVER: int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { int32_t code = 0; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; while (1) { SSmaObj *pSma = NULL; @@ -1222,7 +1219,7 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp memcpy(rsp->dbFName, pSma->db, sizeof(pSma->db)); memcpy(rsp->tblFName, pSma->stb, sizeof(pSma->stb)); - strcpy(rsp->indexType, TSDB_INDEX_TYPE_SMA); + tstrncpy(rsp->indexType, TSDB_INDEX_TYPE_SMA, TSDB_INDEX_TYPE_LEN); SNodeList *pList = NULL; int32_t extOffset = 0; @@ -1232,7 +1229,7 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp FOREACH(node, pList) { SFunctionNode *pFunc = (SFunctionNode *)node; extOffset += tsnprintf(rsp->indexExts + extOffset, sizeof(rsp->indexExts) - extOffset - 1, "%s%s", - (extOffset ? "," : ""), pFunc->functionName); + (extOffset ? "," : ""), pFunc->functionName); } *exist = true; @@ -1255,8 +1252,8 @@ int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool return TSDB_CODE_SUCCESS; } - strcpy(rsp->dbFName, pStb->db); - strcpy(rsp->tbName, pStb->name + strlen(pStb->db) + 1); + tstrncpy(rsp->dbFName, pStb->db, TSDB_DB_FNAME_LEN); + tstrncpy(rsp->tbName, pStb->name + strlen(pStb->db) + 1, TSDB_TABLE_NAME_LEN); rsp->suid = pStb->uid; rsp->version = pStb->smaVer; mndReleaseStb(pMnode, pStb); @@ -1439,8 +1436,8 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc SName smaName = {0}; SName stbName = {0}; - char n2[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - char n3[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char n2[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char n3[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; code = tNameFromString(&smaName, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; if (TSDB_CODE_SUCCESS == code) { @@ -1448,7 +1445,7 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc STR_TO_VARSTR(n2, (char *)mndGetDbStr(pSma->db)); code = tNameFromString(&stbName, pSma->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); } - SColumnInfoData* pColInfo = NULL; + SColumnInfoData *pColInfo = NULL; if (TSDB_CODE_SUCCESS == code) { STR_TO_VARSTR(n3, (char *)tNameGetTableName(&stbName)); @@ -1540,7 +1537,7 @@ static void mndCancelRetrieveIdx(SMnode *pMnode, void *pIter) { taosMemoryFree(p); } -static void initSMAObj(SCreateTSMACxt* pCxt) { +static void initSMAObj(SCreateTSMACxt *pCxt) { memcpy(pCxt->pSma->name, pCxt->pCreateSmaReq->name, TSDB_TABLE_FNAME_LEN); memcpy(pCxt->pSma->stb, pCxt->pCreateSmaReq->stb, TSDB_TABLE_FNAME_LEN); memcpy(pCxt->pSma->db, pCxt->pDb->name, TSDB_DB_FNAME_LEN); @@ -1549,12 +1546,12 @@ static void initSMAObj(SCreateTSMACxt* pCxt) { pCxt->pSma->uid = mndGenerateUid(pCxt->pCreateSmaReq->name, TSDB_TABLE_FNAME_LEN); memcpy(pCxt->pSma->dstTbName, pCxt->targetStbFullName, TSDB_TABLE_FNAME_LEN); - pCxt->pSma->dstTbUid = 0; // not used + pCxt->pSma->dstTbUid = 0; // not used pCxt->pSma->stbUid = pCxt->pSrcStb ? pCxt->pSrcStb->uid : pCxt->pCreateSmaReq->normSourceTbUid; pCxt->pSma->dbUid = pCxt->pDb->uid; pCxt->pSma->interval = pCxt->pCreateSmaReq->interval; pCxt->pSma->intervalUnit = pCxt->pCreateSmaReq->intervalUnit; - pCxt->pSma->timezone = tsTimezone; +// pCxt->pSma->timezone = taosGetLocalTimezoneOffset(); pCxt->pSma->version = 1; pCxt->pSma->exprLen = pCxt->pCreateSmaReq->exprLen; @@ -1598,7 +1595,7 @@ static int32_t mndCreateTSMABuildCreateStreamReq(SCreateTSMACxt *pCxt) { if (!pCxt->pCreateStreamReq->pTags) { return terrno; } - SField f = {0}; + SField f = {0}; int32_t code = 0; if (pCxt->pSrcStb) { for (int32_t idx = 0; idx < pCxt->pCreateStreamReq->numOfTags - 1; ++idx) { @@ -1626,13 +1623,13 @@ static int32_t mndCreateTSMABuildCreateStreamReq(SCreateTSMACxt *pCxt) { if (TSDB_CODE_SUCCESS == code) { // construct output cols - SNode* pNode; + SNode *pNode; FOREACH(pNode, pCxt->pProjects) { - SExprNode* pExprNode = (SExprNode*)pNode; + SExprNode *pExprNode = (SExprNode *)pNode; f.bytes = pExprNode->resType.bytes; f.type = pExprNode->resType.type; f.flags = COL_SMA_ON; - strcpy(f.name, pExprNode->userAlias); + tstrncpy(f.name, pExprNode->userAlias, TSDB_COL_NAME_LEN); if (NULL == taosArrayPush(pCxt->pCreateStreamReq->pCols, &f)) { code = terrno; break; @@ -1642,7 +1639,7 @@ static int32_t mndCreateTSMABuildCreateStreamReq(SCreateTSMACxt *pCxt) { return code; } -static int32_t mndCreateTSMABuildDropStreamReq(SCreateTSMACxt* pCxt) { +static int32_t mndCreateTSMABuildDropStreamReq(SCreateTSMACxt *pCxt) { tstrncpy(pCxt->pDropStreamReq->name, pCxt->streamName, TSDB_STREAM_FNAME_LEN); pCxt->pDropStreamReq->igNotExists = false; pCxt->pDropStreamReq->sql = taosStrdup(pCxt->pDropSmaReq->name); @@ -1685,14 +1682,13 @@ static int32_t mndSetUpdateDbTsmaVersionCommitLogs(SMnode *pMnode, STrans *pTran TAOS_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); } -static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { +static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt *pCxt) { int32_t code = -1; STransAction createStreamRedoAction = {0}; STransAction createStreamUndoAction = {0}; STransAction dropStbUndoAction = {0}; SMDropStbReq dropStbReq = {0}; - STrans *pTrans = - mndTransCreate(pCxt->pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TSMA, pCxt->pRpcReq, "create-tsma"); + STrans *pTrans = mndTransCreate(pCxt->pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TSMA, pCxt->pRpcReq, "create-tsma"); if (!pTrans) { code = terrno; goto _OVER; @@ -1713,7 +1709,9 @@ static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { code = terrno; goto _OVER; } - if (createStreamRedoAction.contLen != tSerializeSCMCreateStreamReq(createStreamRedoAction.pCont, createStreamRedoAction.contLen, pCxt->pCreateStreamReq)) { + if (createStreamRedoAction.contLen != tSerializeSCMCreateStreamReq(createStreamRedoAction.pCont, + createStreamRedoAction.contLen, + pCxt->pCreateStreamReq)) { mError("sma: %s, failed to create due to create stream req encode failure", pCxt->pCreateSmaReq->name); code = TSDB_CODE_INVALID_MSG; goto _OVER; @@ -1728,14 +1726,15 @@ static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { code = terrno; goto _OVER; } - if (createStreamUndoAction.contLen != tSerializeSMDropStreamReq(createStreamUndoAction.pCont, createStreamUndoAction.contLen, pCxt->pDropStreamReq)) { + if (createStreamUndoAction.contLen != + tSerializeSMDropStreamReq(createStreamUndoAction.pCont, createStreamUndoAction.contLen, pCxt->pDropStreamReq)) { mError("sma: %s, failed to create due to drop stream req encode failure", pCxt->pCreateSmaReq->name); code = TSDB_CODE_INVALID_MSG; goto _OVER; } dropStbReq.igNotExists = true; - strncpy(dropStbReq.name, pCxt->targetStbFullName, TSDB_TABLE_FNAME_LEN); + tstrncpy(dropStbReq.name, pCxt->targetStbFullName, TSDB_TABLE_FNAME_LEN); dropStbUndoAction.epSet = createStreamRedoAction.epSet; dropStbUndoAction.acceptableCode = TSDB_CODE_MND_STB_NOT_EXIST; dropStbUndoAction.retryCode = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; @@ -1746,7 +1745,8 @@ static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { code = terrno; goto _OVER; } - if (dropStbUndoAction.contLen != tSerializeSMDropStbReq(dropStbUndoAction.pCont, dropStbUndoAction.contLen, &dropStbReq)) { + if (dropStbUndoAction.contLen != + tSerializeSMDropStbReq(dropStbUndoAction.pCont, dropStbUndoAction.contLen, &dropStbReq)) { mError("sma: %s, failed to create due to drop stb req encode failure", pCxt->pCreateSmaReq->name); code = TSDB_CODE_INVALID_MSG; goto _OVER; @@ -1781,7 +1781,7 @@ static int32_t mndCreateTSMA(SCreateTSMACxt *pCxt) { pCxt->pSma = &sma; initSMAObj(pCxt); - SNodeList* pProjects = NULL; + SNodeList *pProjects = NULL; code = nodesStringToList(pCxt->pCreateSmaReq->expr, &pProjects); if (TSDB_CODE_SUCCESS != code) { goto _OVER; @@ -1830,28 +1830,28 @@ _OVER: TAOS_RETURN(code); } -static int32_t mndTSMAGenerateOutputName(const char* tsmaName, char* streamName, char* targetStbName) { - SName smaName; +static int32_t mndTSMAGenerateOutputName(const char *tsmaName, char *streamName, char *targetStbName) { + SName smaName; int32_t code = tNameFromString(&smaName, tsmaName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); if (TSDB_CODE_SUCCESS != code) { return code; } - sprintf(streamName, "%d.%s", smaName.acctId, smaName.tname); + snprintf(streamName, TSDB_TABLE_FNAME_LEN, "%d.%s", smaName.acctId, smaName.tname); snprintf(targetStbName, TSDB_TABLE_FNAME_LEN, "%s"TSMA_RES_STB_POSTFIX, tsmaName); return TSDB_CODE_SUCCESS; } -static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq) { +static int32_t mndProcessCreateTSMAReq(SRpcMsg *pReq) { #ifdef WINDOWS TAOS_RETURN(TSDB_CODE_MND_INVALID_PLATFORM); #endif - SMnode * pMnode = pReq->info.node; + SMnode *pMnode = pReq->info.node; int32_t code = -1; - SDbObj * pDb = NULL; - SStbObj * pStb = NULL; - SSmaObj * pSma = NULL; - SSmaObj * pBaseTsma = NULL; - SStreamObj * pStream = NULL; + SDbObj *pDb = NULL; + SStbObj *pStb = NULL; + SSmaObj *pSma = NULL; + SSmaObj *pBaseTsma = NULL; + SStreamObj *pStream = NULL; int64_t mTraceId = TRACE_GET_ROOTID(&pReq->info.traceId); SMCreateSmaReq createReq = {0}; @@ -1941,16 +1941,16 @@ static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq) { } SCreateTSMACxt cxt = { - .pMnode = pMnode, - .pCreateSmaReq = &createReq, - .pCreateStreamReq = NULL, - .streamName = streamName, - .targetStbFullName = streamTargetStbFullName, - .pDb = pDb, - .pRpcReq = pReq, - .pSma = NULL, - .pBaseSma = pBaseTsma, - .pSrcStb = pStb, + .pMnode = pMnode, + .pCreateSmaReq = &createReq, + .pCreateStreamReq = NULL, + .streamName = streamName, + .targetStbFullName = streamTargetStbFullName, + .pDb = pDb, + .pRpcReq = pReq, + .pSma = NULL, + .pBaseSma = pBaseTsma, + .pSrcStb = pStb, }; code = mndCreateTSMA(&cxt); @@ -1971,10 +1971,10 @@ _OVER: TAOS_RETURN(code); } -static int32_t mndDropTSMA(SCreateTSMACxt* pCxt) { - int32_t code = -1; +static int32_t mndDropTSMA(SCreateTSMACxt *pCxt) { + int32_t code = -1; STransAction dropStreamRedoAction = {0}; - STrans *pTrans = mndTransCreate(pCxt->pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TSMA, pCxt->pRpcReq, "drop-tsma"); + STrans *pTrans = mndTransCreate(pCxt->pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TSMA, pCxt->pRpcReq, "drop-tsma"); if (!pTrans) { code = terrno; goto _OVER; @@ -2021,7 +2021,8 @@ static int32_t mndDropTSMA(SCreateTSMACxt* pCxt) { code = terrno; goto _OVER; } - if (dropStbRedoAction.contLen != tSerializeSMDropStbReq(dropStbRedoAction.pCont, dropStbRedoAction.contLen, &dropStbReq)) { + if (dropStbRedoAction.contLen != + tSerializeSMDropStbReq(dropStbRedoAction.pCont, dropStbRedoAction.contLen, &dropStbReq)) { mError("tsma: %s, failedto drop due to drop stb req encode failure", pCxt->pDropSmaReq->name); code = TSDB_CODE_INVALID_MSG; goto _OVER; @@ -2044,9 +2045,9 @@ _OVER: TAOS_RETURN(code); } -static bool hasRecursiveTsmasBasedOnMe(SMnode* pMnode, const SSmaObj* pSma) { +static bool hasRecursiveTsmasBasedOnMe(SMnode *pMnode, const SSmaObj *pSma) { SSmaObj *pSmaObj = NULL; - void * pIter = NULL; + void *pIter = NULL; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void **)&pSmaObj); if (pIter == NULL) break; @@ -2060,25 +2061,25 @@ static bool hasRecursiveTsmasBasedOnMe(SMnode* pMnode, const SSmaObj* pSma) { return false; } -static int32_t mndProcessDropTSMAReq(SRpcMsg* pReq) { +static int32_t mndProcessDropTSMAReq(SRpcMsg *pReq) { int32_t code = -1; SMDropSmaReq dropReq = {0}; - SSmaObj * pSma = NULL; - SDbObj * pDb = NULL; - SMnode * pMnode = pReq->info.node; + SSmaObj *pSma = NULL; + SDbObj *pDb = NULL; + SMnode *pMnode = pReq->info.node; if (tDeserializeSMDropSmaReq(pReq->pCont, pReq->contLen, &dropReq) != TSDB_CODE_SUCCESS) { code = TSDB_CODE_INVALID_MSG; goto _OVER; } - char streamName[TSDB_TABLE_FNAME_LEN] = {0}; - char streamTargetStbFullName[TSDB_TABLE_FNAME_LEN] = {0}; + char streamName[TSDB_TABLE_FNAME_LEN] = {0}; + char streamTargetStbFullName[TSDB_TABLE_FNAME_LEN] = {0}; code = mndTSMAGenerateOutputName(dropReq.name, streamName, streamTargetStbFullName); if (TSDB_CODE_SUCCESS != code) { goto _OVER; } - SStbObj* pStb = mndAcquireStb(pMnode, streamTargetStbFullName); + SStbObj *pStb = mndAcquireStb(pMnode, streamTargetStbFullName); pSma = mndAcquireSma(pMnode, dropReq.name); if (!pSma && dropReq.igNotExists) { @@ -2113,13 +2114,13 @@ static int32_t mndProcessDropTSMAReq(SRpcMsg* pReq) { } SCreateTSMACxt cxt = { - .pDb = pDb, - .pMnode = pMnode, - .pRpcReq = pReq, - .pSma = pSma, - .streamName = streamName, - .targetStbFullName = streamTargetStbFullName, - .pDropSmaReq = &dropReq, + .pDb = pDb, + .pMnode = pMnode, + .pRpcReq = pReq, + .pSma = pSma, + .streamName = streamName, + .targetStbFullName = streamTargetStbFullName, + .pDropSmaReq = &dropReq, }; code = mndDropTSMA(&cxt); @@ -2134,10 +2135,10 @@ _OVER: } static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { - SDbObj * pDb = NULL; + SDbObj *pDb = NULL; int32_t numOfRows = 0; - SSmaObj * pSma = NULL; - SMnode * pMnode = pReq->info.node; + SSmaObj *pSma = NULL; + SMnode *pMnode = pReq->info.node; int32_t code = 0; SColumnInfoData *pColInfo; if (pShow->pIter == NULL) { @@ -2153,7 +2154,7 @@ static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo while (numOfRows < rows) { pIter->pSmaIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter->pSmaIter, (void **)&pSma); if (pIter->pSmaIter == NULL) break; - SDbObj* pSrcDb = mndAcquireDb(pMnode, pSma->db); + SDbObj *pSrcDb = mndAcquireDb(pMnode, pSma->db); if ((pDb && pSma->dbUid != pDb->uid) || !pSrcDb) { sdbRelease(pMnode->pSdb, pSma); @@ -2176,7 +2177,7 @@ static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo if (TSDB_CODE_SUCCESS == code) { STR_TO_VARSTR(db, (char *)mndGetDbStr(pSma->db)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - code = colDataSetVal(pColInfo, numOfRows, (const char*)db, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)db, false); } if (TSDB_CODE_SUCCESS == code) { @@ -2186,12 +2187,12 @@ static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo if (TSDB_CODE_SUCCESS == code) { STR_TO_VARSTR(srcTb, (char *)tNameGetTableName(&n)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - code = colDataSetVal(pColInfo, numOfRows, (const char*)srcTb, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)srcTb, false); } if (TSDB_CODE_SUCCESS == code) { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - code = colDataSetVal(pColInfo, numOfRows, (const char*)db, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)db, false); } if (TSDB_CODE_SUCCESS == code) { @@ -2200,29 +2201,29 @@ static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo if (TSDB_CODE_SUCCESS == code) { char targetTb[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(targetTb, (char*)tNameGetTableName(&n)); + STR_TO_VARSTR(targetTb, (char *)tNameGetTableName(&n)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - code = colDataSetVal(pColInfo, numOfRows, (const char*)targetTb, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)targetTb, false); } if (TSDB_CODE_SUCCESS == code) { // stream name pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - code = colDataSetVal(pColInfo, numOfRows, (const char*)smaName, false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)smaName, false); } if (TSDB_CODE_SUCCESS == code) { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - code = colDataSetVal(pColInfo, numOfRows, (const char*)(&pSma->createdTime), false); + code = colDataSetVal(pColInfo, numOfRows, (const char *)(&pSma->createdTime), false); } // interval - char interval[64 + VARSTR_HEADER_SIZE] = {0}; + char interval[64 + VARSTR_HEADER_SIZE] = {0}; int32_t len = 0; if (TSDB_CODE_SUCCESS == code) { if (!IS_CALENDAR_TIME_DURATION(pSma->intervalUnit)) { len = tsnprintf(interval + VARSTR_HEADER_SIZE, 64, "%" PRId64 "%c", pSma->interval, - getPrecisionUnit(pSrcDb->cfg.precision)); + getPrecisionUnit(pSrcDb->cfg.precision)); } else { len = tsnprintf(interval + VARSTR_HEADER_SIZE, 64, "%" PRId64 "%c", pSma->interval, pSma->intervalUnit); } @@ -2243,17 +2244,17 @@ static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo // func list len = 0; SNode *pNode = NULL, *pFunc = NULL; - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code) { code = nodesStringToNode(pSma->ast, &pNode); } if (TSDB_CODE_SUCCESS == code) { - char * start = buf + VARSTR_HEADER_SIZE; + char *start = buf + VARSTR_HEADER_SIZE; FOREACH(pFunc, ((SSelectStmt *)pNode)->pProjectionList) { if (nodeType(pFunc) == QUERY_NODE_FUNCTION) { SFunctionNode *pFuncNode = (SFunctionNode *)pFunc; if (!fmIsTSMASupportedFunc(pFuncNode->funcId)) continue; len += tsnprintf(start, TSDB_MAX_SAVED_SQL_LEN - len, "%s%s", start != buf + VARSTR_HEADER_SIZE ? "," : "", - ((SExprNode *)pFunc)->userAlias); + ((SExprNode *)pFunc)->userAlias); if (len >= TSDB_MAX_SAVED_SQL_LEN) { len = TSDB_MAX_SAVED_SQL_LEN; break; @@ -2297,7 +2298,8 @@ static void mndCancelRetrieveTSMA(SMnode *pMnode, void *pIter) { taosMemoryFree(p); } -int32_t dumpTSMAInfoFromSmaObj(const SSmaObj* pSma, const SStbObj* pDestStb, STableTSMAInfo* pInfo, const SSmaObj* pBaseTsma) { +int32_t dumpTSMAInfoFromSmaObj(const SSmaObj *pSma, const SStbObj *pDestStb, STableTSMAInfo *pInfo, + const SSmaObj *pBaseTsma) { int32_t code = 0; pInfo->interval = pSma->interval; pInfo->unit = pSma->intervalUnit; @@ -2336,7 +2338,7 @@ int32_t dumpTSMAInfoFromSmaObj(const SSmaObj* pSma, const SStbObj* pDestStb, STa SSelectStmt *pSelect = (SSelectStmt *)pNode; FOREACH(pFunc, pSelect->pProjectionList) { STableTSMAFuncInfo funcInfo = {0}; - SFunctionNode * pFuncNode = (SFunctionNode *)pFunc; + SFunctionNode *pFuncNode = (SFunctionNode *)pFunc; if (!fmIsTSMASupportedFunc(pFuncNode->funcId)) continue; funcInfo.funcId = pFuncNode->funcId; funcInfo.colId = ((SColumnNode *)pFuncNode->pParameterList->pHead->pNode)->colId; @@ -2383,9 +2385,9 @@ int32_t dumpTSMAInfoFromSmaObj(const SSmaObj* pSma, const SStbObj* pDestStb, STa } // @note remember to mndReleaseSma(*ppOut) -static int32_t mndGetDeepestBaseForTsma(SMnode* pMnode, SSmaObj* pSma, SSmaObj** ppOut) { - int32_t code = 0; - SSmaObj* pRecursiveTsma = NULL; +static int32_t mndGetDeepestBaseForTsma(SMnode *pMnode, SSmaObj *pSma, SSmaObj **ppOut) { + int32_t code = 0; + SSmaObj *pRecursiveTsma = NULL; if (pSma->baseSmaName[0]) { pRecursiveTsma = mndAcquireSma(pMnode, pSma->baseSmaName); if (!pRecursiveTsma) { @@ -2393,7 +2395,7 @@ static int32_t mndGetDeepestBaseForTsma(SMnode* pMnode, SSmaObj* pSma, SSmaObj** return TSDB_CODE_MND_SMA_NOT_EXIST; } while (pRecursiveTsma->baseSmaName[0]) { - SSmaObj* pTmpSma = pRecursiveTsma; + SSmaObj *pTmpSma = pRecursiveTsma; pRecursiveTsma = mndAcquireSma(pMnode, pTmpSma->baseSmaName); if (!pRecursiveTsma) { mError("base tsma: %s for tsma: %s not found", pTmpSma->baseSmaName, pTmpSma->name); @@ -2407,7 +2409,6 @@ static int32_t mndGetDeepestBaseForTsma(SMnode* pMnode, SSmaObj* pSma, SSmaObj** return code; } - static int32_t mndGetTSMA(SMnode *pMnode, char *tsmaFName, STableTSMAInfoRsp *rsp, bool *exist) { int32_t code = -1; SSmaObj *pSma = NULL; @@ -2450,17 +2451,17 @@ static int32_t mndGetTSMA(SMnode *pMnode, char *tsmaFName, STableTSMAInfoRsp *rs TAOS_RETURN(code); } -typedef bool (*tsmaFilter)(const SSmaObj* pSma, void* param); +typedef bool (*tsmaFilter)(const SSmaObj *pSma, void *param); -static int32_t mndGetSomeTsmas(SMnode* pMnode, STableTSMAInfoRsp* pRsp, tsmaFilter filtered, void* param, bool* exist) { - int32_t code = 0; - SSmaObj * pSma = NULL; - SSmaObj * pBaseTsma = NULL; - SSdb * pSdb = pMnode->pSdb; - void * pIter = NULL; - SStreamObj * pStream = NULL; - SStbObj * pStb = NULL; - bool shouldRetry = false; +static int32_t mndGetSomeTsmas(SMnode *pMnode, STableTSMAInfoRsp *pRsp, tsmaFilter filtered, void *param, bool *exist) { + int32_t code = 0; + SSmaObj *pSma = NULL; + SSmaObj *pBaseTsma = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SStreamObj *pStream = NULL; + SStbObj *pStb = NULL; + bool shouldRetry = false; while (1) { pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); @@ -2479,14 +2480,14 @@ static int32_t mndGetSomeTsmas(SMnode* pMnode, STableTSMAInfoRsp* pRsp, tsmaFilt } SName smaName; - char streamName[TSDB_TABLE_FNAME_LEN] = {0}; + char streamName[TSDB_TABLE_FNAME_LEN] = {0}; code = tNameFromString(&smaName, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); if (TSDB_CODE_SUCCESS != code) { sdbRelease(pSdb, pSma); mndReleaseStb(pMnode, pStb); TAOS_RETURN(code); } - sprintf(streamName, "%d.%s", smaName.acctId, smaName.tname); + snprintf(streamName, TSDB_TABLE_FNAME_LEN, "%d.%s", smaName.acctId, smaName.tname); pStream = NULL; code = mndAcquireStream(pMnode, streamName, &pStream); @@ -2541,8 +2542,8 @@ static int32_t mndGetSomeTsmas(SMnode* pMnode, STableTSMAInfoRsp* pRsp, tsmaFilt return TSDB_CODE_SUCCESS; } -static bool tsmaTbFilter(const SSmaObj* pSma, void* param) { - const char* tbFName = param; +static bool tsmaTbFilter(const SSmaObj *pSma, void *param) { + const char *tbFName = param; return pSma->stb[0] != tbFName[0] || strcmp(pSma->stb, tbFName) != 0; } @@ -2550,7 +2551,7 @@ static int32_t mndGetTableTSMA(SMnode *pMnode, char *tbFName, STableTSMAInfoRsp return mndGetSomeTsmas(pMnode, pRsp, tsmaTbFilter, tbFName, exist); } -static bool tsmaDbFilter(const SSmaObj* pSma, void* param) { +static bool tsmaDbFilter(const SSmaObj *pSma, void *param) { uint64_t *dbUid = param; return pSma->dbUid != *dbUid; } @@ -2564,7 +2565,7 @@ static int32_t mndProcessGetTbTSMAReq(SRpcMsg *pReq) { int32_t code = -1; STableTSMAInfoReq tsmaReq = {0}; bool exist = false; - SMnode * pMnode = pReq->info.node; + SMnode *pMnode = pReq->info.node; TAOS_CHECK_GOTO(tDeserializeTableTSMAInfoReq(pReq->pCont, pReq->contLen, &tsmaReq), NULL, _OVER); @@ -2635,12 +2636,12 @@ static int32_t mkNonExistTSMAInfo(const STSMAVersion *pTsmaVer, STableTSMAInfo * int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t numOfTsmas, void **ppRsp, int32_t *pRspLen) { - int32_t code = -1; - STSMAHbRsp hbRsp = {0}; - int32_t rspLen = 0; - void * pRsp = NULL; - char tsmaFName[TSDB_TABLE_FNAME_LEN] = {0}; - STableTSMAInfo * pTsmaInfo = NULL; + int32_t code = -1; + STSMAHbRsp hbRsp = {0}; + int32_t rspLen = 0; + void *pRsp = NULL; + char tsmaFName[TSDB_TABLE_FNAME_LEN] = {0}; + STableTSMAInfo *pTsmaInfo = NULL; hbRsp.pTsmas = taosArrayInit(numOfTsmas, POINTER_BYTES); if (!hbRsp.pTsmas) { @@ -2649,13 +2650,13 @@ int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t } for (int32_t i = 0; i < numOfTsmas; ++i) { - STSMAVersion* pTsmaVer = &pTsmaVersions[i]; + STSMAVersion *pTsmaVer = &pTsmaVersions[i]; pTsmaVer->dbId = be64toh(pTsmaVer->dbId); pTsmaVer->tsmaId = be64toh(pTsmaVer->tsmaId); pTsmaVer->version = ntohl(pTsmaVer->version); snprintf(tsmaFName, sizeof(tsmaFName), "%s.%s", pTsmaVer->dbFName, pTsmaVer->name); - SSmaObj* pSma = mndAcquireSma(pMnode, tsmaFName); + SSmaObj *pSma = mndAcquireSma(pMnode, tsmaFName); if (!pSma) { code = mkNonExistTSMAInfo(pTsmaVer, &pTsmaInfo); if (code) goto _OVER; @@ -2683,7 +2684,7 @@ int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t continue; } - SStbObj* pDestStb = mndAcquireStb(pMnode, pSma->dstTbName); + SStbObj *pDestStb = mndAcquireStb(pMnode, pSma->dstTbName); if (!pDestStb) { mInfo("tsma: %s.%" PRIx64 " dest stb: %s not found, maybe dropped", tsmaFName, pTsmaVer->tsmaId, pSma->dstTbName); code = mkNonExistTSMAInfo(pTsmaVer, &pTsmaInfo); @@ -2698,7 +2699,7 @@ int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t } // dump smaObj into rsp - STableTSMAInfo * pInfo = NULL; + STableTSMAInfo *pInfo = NULL; pInfo = taosMemoryCalloc(1, sizeof(STableTSMAInfo)); if (!pInfo) { code = terrno; @@ -2707,7 +2708,7 @@ int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t goto _OVER; } - SSmaObj* pBaseSma = NULL; + SSmaObj *pBaseSma = NULL; code = mndGetDeepestBaseForTsma(pMnode, pSma, &pBaseSma); if (code == 0) code = dumpTSMAInfoFromSmaObj(pSma, pDestStb, pInfo, pBaseSma); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5d41e1506c..0a107518df 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1931,11 +1931,6 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange sdbCancelFetch(pSdb, pIter); return terrno = code; } - - code = mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_UPDATE_NAME, pStream->uid); - if (code) { - mError("failed to register trans, transId:%d, and continue", pTrans->id); - } } if (!includeAllNodes) { @@ -1951,6 +1946,12 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid, pStream->name, pTrans->id); + // NOTE: for each stream, we register one trans entry for task update + code = mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_UPDATE_NAME, pStream->uid); + if (code) { + mError("failed to register trans, transId:%d, and continue", pTrans->id); + } + code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans); // todo: not continue, drop all and retry again diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index 905a73ad48..a1e104aeca 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -35,7 +35,11 @@ int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt) size_t keyLen = 0; void *pIter = NULL; SArray *pList = taosArrayInit(4, sizeof(SKeyInfo)); - int32_t num = 0; + int32_t numOfChkpt = 0; + + if (pNumOfActiveChkpt != NULL) { + *pNumOfActiveChkpt = 0; + } if (pList == NULL) { return terrno; @@ -50,15 +54,15 @@ int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt) void *pKey = taosHashGetKey(pEntry, &keyLen); // key is the name of src/dst db name SKeyInfo info = {.pKey = pKey, .keyLen = keyLen}; - mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name, - pEntry->startTime); + mDebug("transId:%d stream:0x%" PRIx64 " %s startTs:%" PRId64 " cleared since finished", pEntry->transId, + pEntry->streamId, pEntry->name, pEntry->startTime); void* p = taosArrayPush(pList, &info); if (p == NULL) { return terrno; } } else { if (strcmp(pEntry->name, MND_STREAM_CHECKPOINT_NAME) == 0) { - num++; + numOfChkpt++; } mndReleaseTrans(pMnode, pTrans); } @@ -78,48 +82,34 @@ int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt) } } - mDebug("clear %d finished stream-trans, remained:%d, active checkpoint trans:%d", size, - taosHashGetSize(execInfo.transMgmt.pDBTrans), num); + mDebug("clear %d finished stream-trans, active trans:%d, active checkpoint trans:%d", size, + taosHashGetSize(execInfo.transMgmt.pDBTrans), numOfChkpt); taosArrayDestroy(pList); if (pNumOfActiveChkpt != NULL) { - *pNumOfActiveChkpt = num; + *pNumOfActiveChkpt = numOfChkpt; } return 0; } -// * Transactions of different streams are not related. Here only check the conflict of transaction for a given stream. -// For a given stream: -// 1. checkpoint trans is conflict with any other trans except for the drop and reset trans. -// 2. create/drop/reset/update trans are conflict with any other trans. -int32_t mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock) { - if (lock) { - streamMutexLock(&execInfo.lock); - } - +static int32_t doStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName) { int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans); if (num <= 0) { - if (lock) { - streamMutexUnlock(&execInfo.lock); - } return 0; } + // if any task updates exist, any other stream trans are not allowed to be created int32_t code = mndStreamClearFinishedTrans(pMnode, NULL); if (code) { - mError("failed to clear finish trans, code:%s", tstrerror(code)); + mError("failed to clear finish trans, code:%s, and continue", tstrerror(code)); } SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId)); if (pEntry != NULL) { SStreamTransInfo tInfo = *pEntry; - if (lock) { - streamMutexUnlock(&execInfo.lock); - } - if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0) { if ((strcmp(pTransName, MND_STREAM_DROP_NAME) != 0) && (strcmp(pTransName, MND_STREAM_TASK_RESET_NAME) != 0) && (strcmp(pTransName, MND_STREAM_RESTART_NAME) != 0)) { @@ -141,11 +131,25 @@ int32_t mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char mDebug("stream:0x%" PRIx64 " no conflict trans existed, continue create trans", streamId); } + return TSDB_CODE_SUCCESS; +} + +// * Transactions of different streams are not related. Here only check the conflict of transaction for a given stream. +// For a given stream: +// 1. checkpoint trans is conflict with any other trans except for the drop and reset trans. +// 2. create/drop/reset/update trans are conflict with any other trans. +int32_t mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock) { + if (lock) { + streamMutexLock(&execInfo.lock); + } + + int32_t code = doStreamTransConflictCheck(pMnode, streamId, pTransName); + if (lock) { streamMutexUnlock(&execInfo.lock); } - return 0; + return code; } int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId) { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index e51500bf34..718b7d0df6 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -14,13 +14,13 @@ */ #define _DEFAULT_SOURCE -#include "mndTrans.h" #include "mndDb.h" #include "mndPrivilege.h" #include "mndShow.h" #include "mndStb.h" #include "mndSubscribe.h" #include "mndSync.h" +#include "mndTrans.h" #include "mndUser.h" #define TRANS_VER1_NUMBER 1 @@ -52,10 +52,17 @@ static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans, bool t static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool topHalf); static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans, bool topHalf); -static bool mndCannotExecuteTransAction(SMnode *pMnode, bool topHalf) { - return (!pMnode->deploy && !mndIsLeader(pMnode)) || !topHalf; +static inline bool mndTransIsInSyncContext(bool topHalf) { return !topHalf; } + +static bool mndCannotExecuteTrans(SMnode *pMnode, bool topHalf) { + bool isLeader = mndIsLeader(pMnode); + bool ret = (!pMnode->deploy && !isLeader) || mndTransIsInSyncContext(topHalf); + if (ret) mDebug("cannot execute trans action, deploy:%d, isLeader:%d, topHalf:%d", pMnode->deploy, isLeader, topHalf); + return ret; } +static inline char *mndStrExecutionContext(bool topHalf) { return topHalf ? "transContext" : "syncContext"; } + static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); static int32_t mndProcessTransTimer(SRpcMsg *pReq); static int32_t mndProcessTtl(SRpcMsg *pReq); @@ -216,7 +223,7 @@ SSdbRaw *mndTransEncode(STrans *pTrans) { SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) - terrno = 0; + terrno = 0; _OVER: if (terrno != 0) { @@ -287,8 +294,8 @@ _OVER: SSdbRow *mndTransDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_INVALID_MSG; - int32_t code = 0; - int32_t lino = 0; + int32_t code = 0; + int32_t lino = 0; SSdbRow *pRow = NULL; STrans *pTrans = NULL; char *pData = NULL; @@ -890,7 +897,7 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { if (pNew->conflict == TRN_CONFLICT_ARBGROUP) { if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; if (pTrans->conflict == TRN_CONFLICT_ARBGROUP) { - void* pGidIter = taosHashIterate(pNew->arbGroupIds, NULL); + void *pGidIter = taosHashIterate(pNew->arbGroupIds, NULL); while (pGidIter != NULL) { int32_t groupId = *(int32_t *)pGidIter; if (taosHashGet(pTrans->arbGroupIds, &groupId, sizeof(int32_t)) != NULL) { @@ -1033,7 +1040,7 @@ static int32_t mndTransCheckCommitActions(SMnode *pMnode, STrans *pTrans) { int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { int32_t code = 0; if (pTrans == NULL) { - return TSDB_CODE_INVALID_PARA; + return TSDB_CODE_INVALID_PARA; } TAOS_CHECK_RETURN(mndTransCheckConflict(pMnode, pTrans)); @@ -1305,6 +1312,14 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi TAOS_RETURN(TSDB_CODE_MND_TRANS_CTX_SWITCH); } + if (pAction->pRaw->type >= SDB_MAX) { + pAction->rawWritten = true; + pAction->errCode = 0; + mndSetTransLastAction(pTrans, pAction); + mInfo("skip sdb raw type:%d since it is not supported", pAction->pRaw->type); + TAOS_RETURN(TSDB_CODE_SUCCESS); + } + int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pAction->pRaw); if (code == 0 || terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { pAction->rawWritten = true; @@ -1327,7 +1342,7 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi // execute in trans context static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->msgSent) return 0; - if (mndCannotExecuteTransAction(pMnode, topHalf)) { + if (mndCannotExecuteTrans(pMnode, topHalf)) { TAOS_RETURN(TSDB_CODE_MND_TRANS_CTX_SWITCH); } @@ -1348,10 +1363,10 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio char detail[1024] = {0}; int32_t len = tsnprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d", TMSG_INFO(pAction->msgType), - pAction->epSet.numOfEps, pAction->epSet.inUse); + pAction->epSet.numOfEps, pAction->epSet.inUse); for (int32_t i = 0; i < pAction->epSet.numOfEps; ++i) { len += tsnprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, pAction->epSet.eps[i].fqdn, - pAction->epSet.eps[i].port); + pAction->epSet.eps[i].port); } int32_t code = tmsgSendReq(&pAction->epSet, &rpcMsg); @@ -1454,9 +1469,9 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA for (int32_t action = 0; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pArray, action); - mDebug("trans:%d, %s:%d Sent:%d, Received:%d, errCode:0x%x, acceptableCode:0x%x, retryCode:0x%x", - pTrans->id, mndTransStr(pAction->stage), pAction->id, pAction->msgSent, pAction->msgReceived, - pAction->errCode, pAction->acceptableCode, pAction->retryCode); + mDebug("trans:%d, %s:%d Sent:%d, Received:%d, errCode:0x%x, acceptableCode:0x%x, retryCode:0x%x", pTrans->id, + mndTransStr(pAction->stage), pAction->id, pAction->msgSent, pAction->msgReceived, pAction->errCode, + pAction->acceptableCode, pAction->retryCode); if (pAction->msgSent) { if (pAction->msgReceived) { if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) { @@ -1473,8 +1488,8 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { - mError("trans:%d, failed to execute redoActions since:%s, code:0x%x, topHalf(TransContext):%d", pTrans->id, - terrstr(), terrno, topHalf); + mError("trans:%d, failed to execute redoActions since:%s, code:0x%x, in %s", pTrans->id, terrstr(), terrno, + mndStrExecutionContext(topHalf)); } return code; } @@ -1482,8 +1497,8 @@ static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans, bool t static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { - mError("trans:%d, failed to execute undoActions since %s. topHalf(TransContext):%d", pTrans->id, terrstr(), - topHalf); + mError("trans:%d, failed to execute undoActions since %s. in %s", pTrans->id, terrstr(), + mndStrExecutionContext(topHalf)); } return code; } @@ -1491,8 +1506,8 @@ static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans, bool t static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->commitActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { - mError("trans:%d, failed to execute commitActions since %s. topHalf(TransContext):%d", pTrans->id, terrstr(), - topHalf); + mError("trans:%d, failed to execute commitActions since %s. in %s", pTrans->id, terrstr(), + mndStrExecutionContext(topHalf)); } return code; } @@ -1512,7 +1527,7 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr for (int32_t action = pTrans->actionPos; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pActions, action); - mInfo("trans:%d, current action:%d, stage:%s, actionType(0:log,1:msg):%d", pTrans->id, pTrans->actionPos, + mInfo("trans:%d, current action:%d, stage:%s, actionType(1:msg,2:log):%d", pTrans->id, pTrans->actionPos, mndTransStr(pAction->stage), pAction->actionType); code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf); @@ -1543,11 +1558,11 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr } mndSetTransLastAction(pTrans, pAction); - if (mndCannotExecuteTransAction(pMnode, topHalf)) { + if (mndCannotExecuteTrans(pMnode, topHalf)) { pTrans->lastErrorNo = code; pTrans->code = code; - mInfo("trans:%d, %s:%d, topHalf(TransContext):%d, not execute next action, code:%s", pTrans->id, - mndTransStr(pAction->stage), action, topHalf, tstrerror(code)); + mInfo("trans:%d, %s:%d, cannot execute next action in %s, code:%s", pTrans->id, mndTransStr(pAction->stage), + action, mndStrExecutionContext(topHalf), tstrerror(code)); break; } @@ -1648,20 +1663,21 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool code = mndTransExecuteRedoActions(pMnode, pTrans, topHalf); } - if (mndCannotExecuteTransAction(pMnode, topHalf)) { + if (code != 0 && code != TSDB_CODE_MND_TRANS_CTX_SWITCH && mndTransIsInSyncContext(topHalf)) { pTrans->lastErrorNo = code; pTrans->code = code; - bool continueExec = true; - if (code != 0 && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { - taosMsleep(100); - continueExec = true; - } else { - continueExec = false; + mInfo( + "trans:%d, failed to execute, will retry redo action stage in 100 ms , in %s, " + "continueExec:%d, code:%s", + pTrans->id, mndStrExecutionContext(topHalf), continueExec, tstrerror(code)); + taosMsleep(100); + return true; + } else { + if (mndCannotExecuteTrans(pMnode, topHalf)) { + mInfo("trans:%d, cannot continue to execute redo action stage in %s, continueExec:%d, code:%s", pTrans->id, + mndStrExecutionContext(topHalf), continueExec, tstrerror(code)); + return false; } - mInfo("trans:%d, cannot execute redo action stage, topHalf(TransContext):%d, continueExec:%d, code:%s", pTrans->id, - topHalf, continueExec, tstrerror(code)); - - return continueExec; } terrno = code; @@ -1704,9 +1720,9 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool return continueExec; } -// in trans context +// execute in trans context static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { - if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; + if (mndCannotExecuteTrans(pMnode, topHalf)) return false; bool continueExec = true; int32_t code = mndTransCommit(pMnode, pTrans); @@ -1760,7 +1776,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans, bool code = mndTransExecuteUndoActions(pMnode, pTrans, topHalf); } - if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; + if (mndCannotExecuteTrans(pMnode, topHalf)) return false; terrno = code; if (code == 0) { @@ -1781,7 +1797,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans, bool // in trans context static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { - if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; + if (mndCannotExecuteTrans(pMnode, topHalf)) return false; bool continueExec = true; int32_t code = mndTransRollback(pMnode, pTrans); @@ -1798,8 +1814,9 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool to return continueExec; } +// excute in trans context static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { - if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; + if (mndCannotExecuteTrans(pMnode, topHalf)) return false; bool continueExec = true; int32_t code = mndTransPreFinish(pMnode, pTrans); @@ -1842,8 +1859,8 @@ void mndTransExecuteImp(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = true; while (continueExec) { - mInfo("trans:%d, continue to execute, stage:%s createTime:%" PRId64 " topHalf(TransContext):%d", pTrans->id, - mndTransStr(pTrans->stage), pTrans->createdTime, topHalf); + mInfo("trans:%d, continue to execute stage:%s in %s, createTime:%" PRId64 "", pTrans->id, + mndTransStr(pTrans->stage), mndStrExecutionContext(topHalf), pTrans->createdTime); pTrans->lastExecTime = taosGetTimestampMs(); switch (pTrans->stage) { case TRN_STAGE_PREPARE: @@ -2038,11 +2055,11 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl char lastInfo[TSDB_TRANS_ERROR_LEN + VARSTR_HEADER_SIZE] = {0}; char detail[TSDB_TRANS_ERROR_LEN + 1] = {0}; int32_t len = tsnprintf(detail, sizeof(detail), "action:%d code:0x%x(%s) ", pTrans->lastAction, - pTrans->lastErrorNo & 0xFFFF, tstrerror(pTrans->lastErrorNo)); + pTrans->lastErrorNo & 0xFFFF, tstrerror(pTrans->lastErrorNo)); SEpSet epset = pTrans->lastEpset; if (epset.numOfEps > 0) { len += tsnprintf(detail + len, sizeof(detail) - len, "msgType:%s numOfEps:%d inUse:%d ", - TMSG_INFO(pTrans->lastMsgType), epset.numOfEps, epset.inUse); + TMSG_INFO(pTrans->lastMsgType), epset.numOfEps, epset.inUse); for (int32_t i = 0; i < pTrans->lastEpset.numOfEps; ++i) { len += tsnprintf(detail + len, sizeof(detail) - len, "ep:%d-%s:%u ", i, epset.eps[i].fqdn, epset.eps[i].port); } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 63390d4772..6a9d67fc83 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1706,8 +1706,8 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate if (pCreate->isImport != 1) { taosEncryptPass_c((uint8_t *)pCreate->pass, strlen(pCreate->pass), userObj.pass); } else { - // mInfo("pCreate->pass:%s", pCreate->pass) - strncpy(userObj.pass, pCreate->pass, TSDB_PASSWORD_LEN); + // mInfo("pCreate->pass:%s", pCreate->eass) + memcpy(userObj.pass, pCreate->pass, TSDB_PASSWORD_LEN); } tstrncpy(userObj.user, pCreate->user, TSDB_USER_LEN); tstrncpy(userObj.acct, acct, TSDB_USER_LEN); @@ -1803,6 +1803,43 @@ _OVER: TAOS_RETURN(code); } +static int32_t mndCheckPasswordFmt(const char *pwd) { + int32_t len = strlen(pwd); + if (len < TSDB_PASSWORD_MIN_LEN || len > TSDB_PASSWORD_MAX_LEN) { + return -1; + } + + if (strcmp(pwd, "taosdata") == 0) { + return 0; + } + + bool charTypes[4] = {0}; + for (int32_t i = 0; i < len; ++i) { + if (taosIsBigChar(pwd[i])) { + charTypes[0] = true; + } else if (taosIsSmallChar(pwd[i])) { + charTypes[1] = true; + } else if (taosIsNumberChar(pwd[i])) { + charTypes[2] = true; + } else if (taosIsSpecialChar(pwd[i])) { + charTypes[3] = true; + } else { + return -1; + } + } + + int32_t numOfTypes = 0; + for (int32_t i = 0; i < 4; ++i) { + numOfTypes += charTypes[i]; + } + + if (numOfTypes < 3) { + return -1; + } + + return 0; +} + static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = 0; @@ -1836,7 +1873,7 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_USER_FORMAT, &lino, _OVER); } - if (createReq.pass[0] == 0) { + if (mndCheckPasswordFmt(createReq.pass) != 0) { TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_PASS_FORMAT, &lino, _OVER); } @@ -2325,8 +2362,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_USER_FORMAT, &lino, _OVER); } - if (TSDB_ALTER_USER_PASSWD == alterReq.alterType && - (alterReq.pass[0] == 0 || strlen(alterReq.pass) >= TSDB_PASSWORD_LEN)) { + if (TSDB_ALTER_USER_PASSWD == alterReq.alterType && mndCheckPasswordFmt(alterReq.pass) != 0) { TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_PASS_FORMAT, &lino, _OVER); } diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 474b22cca0..42bdfbb56f 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -370,7 +370,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { opts.source = pRaw->pData; opts.result = plantContent; opts.unitLen = 16; - strncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN + 1); count = CBC_Decrypt(&opts); @@ -388,6 +388,11 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { goto _OVER; } + if (pRaw->type >= SDB_MAX) { + mInfo("skip sdb raw type:%d since it is not supported", pRaw->type); + continue; + } + code = sdbWriteWithoutFree(pSdb, pRaw); if (code != 0) { mError("failed to read sdb file:%s since %s", file, terrstr()); @@ -510,7 +515,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb, int32_t skip_type) { opts.source = pRaw->pData; opts.result = newData; opts.unitLen = 16; - strncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN + 1); int32_t count = CBC_Encrypt(&opts); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 9a5bea33e3..5351554631 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -138,6 +138,7 @@ static int32_t metaOpenImpl(SVnode *pVnode, SMeta **ppMeta, const char *metaDir, int32_t code = 0; int32_t lino; int32_t offset; + int32_t pathLen = 0; char path[TSDB_FILENAME_LEN] = {0}; char indexFullPath[128] = {0}; @@ -150,14 +151,15 @@ static int32_t metaOpenImpl(SVnode *pVnode, SMeta **ppMeta, const char *metaDir, taosRemoveDir(path); } - if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + strlen(path) + 1)) == NULL) { + pathLen = strlen(path) + 1; + if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + pathLen)) == NULL) { TSDB_CHECK_CODE(code = terrno, lino, _exit); } metaInitLock(pMeta); pMeta->path = (char *)&pMeta[1]; - strcpy(pMeta->path, path); + tstrncpy(pMeta->path, path, pathLen); int32_t ret = taosRealPath(pMeta->path, NULL, strlen(path) + 1); pMeta->pVnode = pVnode; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index e2ba8d9ccb..87beb8842b 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1328,7 +1328,7 @@ int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) { TAOS_CHECK_GOTO(terrno, NULL, END); } - if (false == taosMbsToUcs4(tagData, nTagData, (TdUcs4 *)buf, maxSize, &maxSize)) { + if (false == taosMbsToUcs4(tagData, nTagData, (TdUcs4 *)buf, maxSize, &maxSize, NULL)) { TAOS_CHECK_GOTO(terrno, NULL, END); } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 65e520bb4a..c32d4b30b0 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -153,7 +153,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const if (val == NULL) { TAOS_CHECK_GOTO(terrno, NULL, _exception); } - int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); + int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE, NULL); if (len < 0) { TAOS_CHECK_GOTO(len, NULL, _exception); } @@ -237,7 +237,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche if (val == NULL) { TAOS_CHECK_GOTO(terrno, NULL, _exception); } - int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); + int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE, NULL); if (len < 0) { TAOS_CHECK_GOTO(len, NULL, _exception); } @@ -1189,7 +1189,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs (*pMetaRsp)->tableType = TSDB_CHILD_TABLE; (*pMetaRsp)->tuid = pReq->uid; (*pMetaRsp)->suid = pReq->ctb.suid; - strcpy((*pMetaRsp)->tbName, pReq->name); + tstrncpy((*pMetaRsp)->tbName, pReq->name, strlen(pReq->name) + 1); } else { ret = metaUpdateMetaRsp(pReq->uid, pReq->name, &pReq->ntb.schemaRow, *pMetaRsp); if (ret < 0) { @@ -1834,7 +1834,8 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type = pAlterTbReq->type; pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].flags = pAlterTbReq->flags; pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId = entry.ntbEntry.ncid++; - strcpy(pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].name, pAlterTbReq->colName); + tstrncpy(pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].name, pAlterTbReq->colName, + strlen(pAlterTbReq->colName) + 1); ++pMeta->pVnode->config.vndStats.numOfNTimeSeries; metaTimeSeriesNotifyCheck(pMeta); @@ -1943,7 +1944,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl goto _err; } pSchema->version++; - strcpy(pColumn->name, pAlterTbReq->colNewName); + tstrncpy(pColumn->name, pAlterTbReq->colNewName, strlen(pAlterTbReq->colNewName) + 1); break; } diff --git a/source/dnode/vnode/src/meta/metaTtl.c b/source/dnode/vnode/src/meta/metaTtl.c index 4077e9fa5d..2b67c27234 100644 --- a/source/dnode/vnode/src/meta/metaTtl.c +++ b/source/dnode/vnode/src/meta/metaTtl.c @@ -49,18 +49,20 @@ const char *ttlV1Tbname = "ttlv1.idx"; int32_t ttlMgrOpen(STtlManger **ppTtlMgr, TDB *pEnv, int8_t rollback, const char *logPrefix, int32_t flushThreshold) { int32_t code = TSDB_CODE_SUCCESS; int64_t startNs = taosGetTimestampNs(); + int32_t pathLen = 0; *ppTtlMgr = NULL; STtlManger *pTtlMgr = (STtlManger *)tdbOsCalloc(1, sizeof(*pTtlMgr)); if (pTtlMgr == NULL) TAOS_RETURN(terrno); - char *logBuffer = (char *)tdbOsCalloc(1, strlen(logPrefix) + 1); + pathLen = strlen(logPrefix) + 1; + char *logBuffer = (char *)tdbOsCalloc(1, pathLen); if (logBuffer == NULL) { tdbOsFree(pTtlMgr); TAOS_RETURN(terrno); } - (void)strcpy(logBuffer, logPrefix); + tstrncpy(logBuffer, logPrefix, pathLen); pTtlMgr->logPrefix = logBuffer; pTtlMgr->flushThreshold = flushThreshold; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index bbc58004d9..5a515b1440 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -254,11 +254,12 @@ static void tdRSmaTaskInit(SStreamMeta *pMeta, SRSmaInfoItem *pItem, SStreamTask } static void tdRSmaTaskRemove(SStreamMeta *pMeta, int64_t streamId, int32_t taskId) { + streamMetaWLock(pMeta); + int32_t code = streamMetaUnregisterTask(pMeta, streamId, taskId); if (code != 0) { smaError("vgId:%d, rsma task:%" PRIi64 ",%d drop failed since %s", pMeta->vgId, streamId, taskId, tstrerror(code)); } - streamMetaWLock(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); if (streamMetaCommit(pMeta) < 0) { // persist to disk diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a234777441..bb50499ea8 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1122,91 +1122,76 @@ _OVER: return code; } +// always return success to mnode +//todo: handle failure of build and send msg to mnode +static void doSendChkptSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, int32_t code, + int32_t taskId) { + SRpcMsg rsp = {0}; + int32_t ret = streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &rsp, code); + if (ret) { // suppress the error in build checkpoint source rsp + tqError("s-task:0x%x failed to build checkpoint-source rsp, code:%s", taskId, tstrerror(ret)); + } + tmsgSendRsp(&rsp); // error occurs +} + // no matter what kinds of error happened, make sure the mnode will receive the success execution code. int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) { - int32_t vgId = TD_VID(pTq->pVnode); - SStreamMeta* pMeta = pTq->pStreamMeta; - char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t len = pMsg->contLen - sizeof(SMsgHead); - int32_t code = 0; + int32_t vgId = TD_VID(pTq->pVnode); + SStreamMeta* pMeta = pTq->pStreamMeta; + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + int32_t code = 0; + SStreamCheckpointSourceReq req = {0}; + SDecoder decoder = {0}; + SStreamTask* pTask = NULL; + int64_t checkpointId = 0; // disable auto rsp to mnode pRsp->info.handle = NULL; - SStreamCheckpointSourceReq req = {0}; - SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, len); if (tDecodeStreamCheckpointSourceReq(&decoder, &req) < 0) { code = TSDB_CODE_MSG_DECODE_ERROR; tDecoderClear(&decoder); tqError("vgId:%d failed to decode checkpoint-source msg, code:%s", vgId, tstrerror(code)); - - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:0x%x failed to build checkpoint-source rsp, code:%s", req.taskId, tstrerror(code)); - } - - tmsgSendRsp(&rsp); // error occurs - return TSDB_CODE_SUCCESS; // always return success to mnode, todo: handle failure of build and send msg to mnode + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); + return TSDB_CODE_SUCCESS; // always return success to mnode, } + tDecoderClear(&decoder); if (!vnodeIsRoleLeader(pTq->pVnode)) { tqDebug("vgId:%d not leader, ignore checkpoint-source msg, s-task:0x%x", vgId, req.taskId); - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:0x%x failed to build checkpoint-source rsp, code:%s", req.taskId, tstrerror(code)); - } - - tmsgSendRsp(&rsp); // error occurs - return TSDB_CODE_SUCCESS; // always return success to mnode, todo: handle failure of build and send msg to mnode + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); + return TSDB_CODE_SUCCESS; // always return success to mnode } if (!pTq->pVnode->restored) { tqDebug("vgId:%d checkpoint-source msg received during restoring, checkpointId:%" PRId64 ", transId:%d s-task:0x%x ignore it", vgId, req.checkpointId, req.transId, req.taskId); - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:0x%x failed to build checkpoint-source rsp, code:%s", req.taskId, tstrerror(code)); - } - - tmsgSendRsp(&rsp); // error occurs - return TSDB_CODE_SUCCESS; // always return success to mnode, , todo: handle failure of build and send msg to mnode + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); + return TSDB_CODE_SUCCESS; // always return success to mnode } - SStreamTask* pTask = NULL; code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); if (pTask == NULL || code != 0) { tqError("vgId:%d failed to find s-task:0x%x, ignore checkpoint msg. checkpointId:%" PRId64 " transId:%d it may have been destroyed", vgId, req.taskId, req.checkpointId, req.transId); - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:%s failed to build checkpoint-source rsp, code:%s", pTask->id.idStr, tstrerror(code)); - } - tmsgSendRsp(&rsp); // error occurs + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); return TSDB_CODE_SUCCESS; } if (pTask->status.downstreamReady != 1) { - streamTaskSetFailedChkptInfo(pTask, req.transId, req.checkpointId); // record the latest failed checkpoint id + // record the latest failed checkpoint id + streamTaskSetFailedChkptInfo(pTask, req.transId, req.checkpointId); tqError("s-task:%s not ready for checkpoint, since downstream not ready, ignore this checkpointId:%" PRId64 ", transId:%d set it failed", pTask->id.idStr, req.checkpointId, req.transId); + streamMetaReleaseTask(pMeta, pTask); - - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:%s failed to build checkpoint-source rsp, code:%s", pTask->id.idStr, tstrerror(code)); - } - - tmsgSendRsp(&rsp); // error occurs + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); return TSDB_CODE_SUCCESS; // todo retry handle error } @@ -1221,14 +1206,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); - - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:%s failed to build checkpoint-source rsp, code:%s", pTask->id.idStr, tstrerror(code)); - } - - tmsgSendRsp(&rsp); // error occurs + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); return TSDB_CODE_SUCCESS; } } else { @@ -1240,7 +1218,6 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) // check if the checkpoint msg already sent or not. if (status == TASK_STATUS__CK) { - int64_t checkpointId = 0; streamTaskGetActiveCheckpointInfo(pTask, NULL, &checkpointId); tqWarn("s-task:%s repeatly recv checkpoint-source msg checkpointId:%" PRId64 @@ -1249,7 +1226,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); - + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SYN_PROPOSE_NOT_READY, req.taskId); return TSDB_CODE_SUCCESS; } else { // checkpoint already finished, and not in checkpoint status if (req.checkpointId <= pTask->chkInfo.checkpointId) { @@ -1259,15 +1236,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); - - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:%s failed to build checkpoint-source rsp, code:%s", pTask->id.idStr, tstrerror(code)); - } - - tmsgSendRsp(&rsp); // error occurs - + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); return TSDB_CODE_SUCCESS; } } @@ -1278,7 +1247,9 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) if (code) { qError("s-task:%s (vgId:%d) failed to process checkpoint-source req, code:%s", pTask->id.idStr, vgId, tstrerror(code)); - return code; + streamMetaReleaseTask(pMeta, pTask); + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); + return TSDB_CODE_SUCCESS; } if (req.mndTrigger) { @@ -1293,13 +1264,8 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) code = streamAddCheckpointSourceRspMsg(&req, &pMsg->info, pTask); if (code != TSDB_CODE_SUCCESS) { - SRpcMsg rsp = {0}; - int32_t ret = streamTaskBuildCheckpointSourceRsp(&req, &pMsg->info, &rsp, TSDB_CODE_SUCCESS); - if (ret) { // suppress the error in build checkpointsource rsp - tqError("s-task:%s failed to build checkpoint-source rsp, code:%s", pTask->id.idStr, tstrerror(code)); - } - tmsgSendRsp(&rsp); // error occurs - return TSDB_CODE_SUCCESS; + streamTaskSetCheckpointFailed(pTask); // set the checkpoint failed + doSendChkptSourceRsp(&req, &pMsg->info, TSDB_CODE_SUCCESS, req.taskId); } streamMetaReleaseTask(pMeta, pTask); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 3f4ff7f3d9..5405ace89b 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -13,9 +13,7 @@ * along with this program. If not, see . */ -#include #include "tcommon.h" -#include "tmsg.h" #include "tq.h" #define IS_NEW_SUBTB_RULE(_t) (((_t)->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER) && ((_t)->subtableWithoutMd5 != 1)) @@ -50,7 +48,7 @@ static int32_t doPutSinkTableInfoIntoCache(SSHashObj* pSinkTableMap, STableSinkI static bool doGetSinkTableInfoFromCache(SSHashObj* pTableInfoMap, uint64_t groupId, STableSinkInfo** pInfo); static int32_t doRemoveSinkTableInfoInCache(SSHashObj* pSinkTableMap, uint64_t groupId, const char* id); static int32_t checkTagSchema(SStreamTask* pTask, SVnode* pVnode); -static void reubuildAndSendMultiResBlock(SStreamTask* pTask, const SArray* pBlocks, SVnode* pVnode, int64_t earlyTs); +static void rebuildAndSendMultiResBlock(SStreamTask* pTask, const SArray* pBlocks, SVnode* pVnode, int64_t earlyTs); static int32_t handleResultBlockMsg(SStreamTask* pTask, SSDataBlock* pDataBlock, int32_t index, SVnode* pVnode, int64_t earlyTs); static int32_t doWaitForDstTableDropped(SVnode* pVnode, SStreamTask* pTask, const char* dstTableName); @@ -1187,7 +1185,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { return; } - reubuildAndSendMultiResBlock(pTask, pBlocks, pVnode, earlyTs); + rebuildAndSendMultiResBlock(pTask, pBlocks, pVnode, earlyTs); } } @@ -1290,7 +1288,7 @@ int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* return TSDB_CODE_SUCCESS; } -void reubuildAndSendMultiResBlock(SStreamTask* pTask, const SArray* pBlocks, SVnode* pVnode, int64_t earlyTs) { +void rebuildAndSendMultiResBlock(SStreamTask* pTask, const SArray* pBlocks, SVnode* pVnode, int64_t earlyTs) { int32_t code = 0; const char* id = pTask->id.idStr; int32_t vgId = pTask->pMeta->vgId; diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 29372c5da7..bc7e2e28e3 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -17,19 +17,20 @@ #include "vnd.h" #define MAX_REPEAT_SCAN_THRESHOLD 3 -#define SCAN_WAL_IDLE_DURATION 100 +#define SCAN_WAL_IDLE_DURATION 500 // idle for 500ms to do next wal scan typedef struct SBuildScanWalMsgParam { int64_t metaId; int32_t numOfTasks; } SBuildScanWalMsgParam; -static int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle); +static int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta); static int32_t setWalReaderStartOffset(SStreamTask* pTask, int32_t vgId); static bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver); static bool taskReadyForDataFromWal(SStreamTask* pTask); static int32_t doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems, bool* pSucc); static int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration); +static int32_t doScanWalAsync(STQ* pTq, bool ckPause); // extract data blocks(submit/delete) from WAL, and add them into the input queue for all the sources tasks. int32_t tqScanWal(STQ* pTq) { @@ -37,12 +38,11 @@ int32_t tqScanWal(STQ* pTq) { int32_t vgId = pMeta->vgId; int64_t st = taosGetTimestampMs(); int32_t numOfTasks = 0; - bool shouldIdle = true; tqDebug("vgId:%d continue to check if data in wal are available, scanCounter:%d", vgId, pMeta->scanInfo.scanCounter); // check all tasks - int32_t code = doScanWalForAllTasks(pMeta, &shouldIdle); + int32_t code = doScanWalForAllTasks(pMeta); if (code) { tqError("vgId:%d failed to start all tasks, try next time, code:%s", vgId, tstrerror(code)); return code; @@ -133,10 +133,9 @@ int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) { } int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { - int32_t vgId = TD_VID(pTq->pVnode); SStreamMeta* pMeta = pTq->pStreamMeta; bool alreadyRestored = pTq->pVnode->restored; - int32_t numOfTasks = 0; + int32_t code = 0; // do not launch the stream tasks, if it is a follower or not restored vnode. if (!(vnodeIsRoleLeader(pTq->pVnode) && alreadyRestored)) { @@ -144,47 +143,8 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { } streamMetaWLock(pMeta); - - numOfTasks = taosArrayGetSize(pMeta->pTaskList); - if (numOfTasks == 0) { - tqDebug("vgId:%d no stream tasks existed to run", vgId); - streamMetaWUnLock(pMeta); - return 0; - } - - if (pMeta->startInfo.startAllTasks) { - tqTrace("vgId:%d in restart procedure, not scan wal", vgId); - streamMetaWUnLock(pMeta); - return 0; - } - - pMeta->scanInfo.scanCounter += 1; - if (pMeta->scanInfo.scanCounter > MAX_REPEAT_SCAN_THRESHOLD) { - pMeta->scanInfo.scanCounter = MAX_REPEAT_SCAN_THRESHOLD; - } - - if (pMeta->scanInfo.scanCounter > 1) { - tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->scanInfo.scanCounter); - streamMetaWUnLock(pMeta); - return 0; - } - - int32_t numOfPauseTasks = pMeta->numOfPausedTasks; - if (ckPause && numOfTasks == numOfPauseTasks) { - tqDebug("vgId:%d ignore all submit, all streams had been paused, reset the walScanCounter", vgId); - - // reset the counter value, since we do not launch the scan wal operation. - pMeta->scanInfo.scanCounter = 0; - streamMetaWUnLock(pMeta); - return 0; - } - - tqDebug("vgId:%d create msg to start wal scan to launch stream tasks, numOfTasks:%d, vnd restored:%d", vgId, - numOfTasks, alreadyRestored); - - int32_t code = streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); + code = doScanWalAsync(pTq, ckPause); streamMetaWUnLock(pMeta); - return code; } @@ -368,11 +328,8 @@ int32_t doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfIt return code; } -int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { - *pScanIdle = true; - bool noDataInWal = true; +int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta) { int32_t vgId = pStreamMeta->vgId; - int32_t numOfTasks = taosArrayGetSize(pStreamMeta->pTaskList); if (numOfTasks == 0) { return TSDB_CODE_SUCCESS; @@ -410,8 +367,6 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { continue; } - *pScanIdle = false; - // seek the stored version and extract data from WAL code = setWalReaderStartOffset(pTask, vgId); if (code != TSDB_CODE_SUCCESS) { @@ -437,7 +392,6 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { streamMutexUnlock(&pTask->lock); if ((numOfItems > 0) || hasNewData) { - noDataInWal = false; code = streamTrySchedExec(pTask); if (code != TSDB_CODE_SUCCESS) { streamMetaReleaseTask(pStreamMeta, pTask); @@ -449,11 +403,47 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { streamMetaReleaseTask(pStreamMeta, pTask); } - // all wal are checked, and no new data available in wal. - if (noDataInWal) { - *pScanIdle = true; - } - taosArrayDestroy(pTaskList); return TSDB_CODE_SUCCESS; } + +int32_t doScanWalAsync(STQ* pTq, bool ckPause) { + SStreamMeta* pMeta = pTq->pStreamMeta; + bool alreadyRestored = pTq->pVnode->restored; + int32_t vgId = pMeta->vgId; + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + + if (numOfTasks == 0) { + tqDebug("vgId:%d no stream tasks existed to run", vgId); + return 0; + } + + if (pMeta->startInfo.startAllTasks) { + tqTrace("vgId:%d in restart procedure, not scan wal", vgId); + return 0; + } + + pMeta->scanInfo.scanCounter += 1; + if (pMeta->scanInfo.scanCounter > MAX_REPEAT_SCAN_THRESHOLD) { + pMeta->scanInfo.scanCounter = MAX_REPEAT_SCAN_THRESHOLD; + } + + if (pMeta->scanInfo.scanCounter > 1) { + tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->scanInfo.scanCounter); + return 0; + } + + int32_t numOfPauseTasks = pMeta->numOfPausedTasks; + if (ckPause && numOfTasks == numOfPauseTasks) { + tqDebug("vgId:%d ignore all submit, all streams had been paused, reset the walScanCounter", vgId); + + // reset the counter value, since we do not launch the scan wal operation. + pMeta->scanInfo.scanCounter = 0; + return 0; + } + + tqDebug("vgId:%d create msg to start wal scan to launch stream tasks, numOfTasks:%d, vnd restored:%d", vgId, + numOfTasks, alreadyRestored); + + return streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); +} diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 6e27677b54..06b7b33cd8 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -718,8 +718,6 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen } } - streamMetaWUnLock(pMeta); - // drop the related fill-history task firstly if (hTaskId.taskId != 0 && hTaskId.streamId != 0) { tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId); @@ -737,7 +735,6 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen } // commit the update - streamMetaWLock(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); tqDebug("vgId:%d task:0x%x dropped, remain tasks:%d", vgId, pReq->taskId, numOfTasks); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index ef2766acda..17cbcc86b2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -396,7 +396,7 @@ static int32_t tsdbFSAddEntryToFileObjHash(STFileHash *hash, const char *fname) STFileHashEntry *entry = taosMemoryMalloc(sizeof(*entry)); if (entry == NULL) return terrno; - strncpy(entry->fname, fname, TSDB_FILENAME_LEN); + tstrncpy(entry->fname, fname, TSDB_FILENAME_LEN); uint32_t idx = MurmurHash3_32(fname, strlen(fname)) % hash->numBucket; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 53e1c57f14..6dba1825ad 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -175,7 +175,7 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e opts.result = PacketData; opts.unitLen = 128; // strncpy(opts.key, tsEncryptKey, 16); - strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN + 1); NewLen = CBC_Encrypt(&opts); @@ -249,7 +249,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor opts.result = PacketData; opts.unitLen = 128; // strncpy(opts.key, tsEncryptKey, 16); - strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN + 1); NewLen = CBC_Decrypt(&opts); diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 7c789e84ae..2ceeeca160 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -265,7 +265,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tsEncryptKey[0] == 0) { return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; } else { - strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN + 1); } } #endif @@ -292,7 +292,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tsEncryptKey[0] == 0) { return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; } else { - strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN + 1); } } #endif diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 2d2446415e..b91abe93af 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -67,8 +67,17 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs SVnodeInfo oldInfo = {0}; oldInfo.config = vnodeCfgDefault; if (vnodeLoadInfo(dir, &oldInfo) == 0) { - vWarn("vgId:%d, vnode config info already exists at %s.", oldInfo.config.vgId, dir); - return (oldInfo.config.dbId == info.config.dbId) ? 0 : -1; + code = (oldInfo.config.dbId == info.config.dbId) ? 0 : TSDB_CODE_VND_ALREADY_EXIST_BUT_NOT_MATCH; + if (code == 0) { + vWarn("vgId:%d, vnode config info already exists at %s.", oldInfo.config.vgId, dir); + } else { + vError("vgId:%d, vnode config info already exists at %s. oldDbId:%" PRId64 "(%s) at cluster:%" PRId64 + ", newDbId:%" PRId64 "(%s) at cluser:%" PRId64 ", code:%s", + oldInfo.config.vgId, dir, oldInfo.config.dbId, oldInfo.config.dbname, + oldInfo.config.syncCfg.nodeInfo[oldInfo.config.syncCfg.myIndex].clusterId, info.config.dbId, + info.config.dbname, info.config.syncCfg.nodeInfo[info.config.syncCfg.myIndex].clusterId, tstrerror(code)); + } + return code; } vInfo("vgId:%d, save config while create", info.config.vgId); @@ -190,7 +199,14 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr char *tsdbFilePrefixPos = strstr(oldRname, tsdbFilePrefix); if (tsdbFilePrefixPos == NULL) continue; - int32_t tsdbFileVgId = atoi(tsdbFilePrefixPos + prefixLen); + int32_t tsdbFileVgId = 0; // atoi(tsdbFilePrefixPos + prefixLen); + ret = taosStr2int32(tsdbFilePrefixPos + prefixLen, &tsdbFileVgId); + if (ret != 0) { + vError("vgId:%d, failed to get tsdb file vgid since %s", dstVgId, tstrerror(ret)); + tfsClosedir(tsdbDir); + return ret; + } + if (tsdbFileVgId == srcVgId) { char *tsdbFileSurfixPos = tsdbFilePrefixPos + prefixLen + vnodeVgroupIdLen(srcVgId); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 328c2c1585..b1c206bdae 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -977,6 +977,11 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { extern int32_t vnodeAsyncRetention(SVnode *pVnode, int64_t now); static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { + if (!pVnode->restored) { + vInfo("vgId:%d, ignore trim req during restoring. ver:%" PRId64, TD_VID(pVnode), ver); + return 0; + } + int32_t code = 0; SVTrimDbReq trimReq = {0}; @@ -1088,7 +1093,7 @@ static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode *pVnode, int64_t ver, void expiredTb.suid = *uid; terrno = metaReaderGetTableEntryByUid(&mr, *uid); if (terrno < 0) goto _end; - strncpy(buf, mr.me.name, TSDB_TABLE_NAME_LEN); + tstrncpy(buf, mr.me.name, TSDB_TABLE_NAME_LEN); void *p = taosArrayPush(pNames, buf); if (p == NULL) { goto _end; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 0ea1d98312..c194b51b07 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -447,7 +447,7 @@ int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, } char* pJson = NULL; - parseTagDatatoJson(pTag, &pJson); + parseTagDatatoJson(pTag, &pJson, NULL); if(NULL == pJson) { taosArrayDestroy(pTagVals); CTG_ERR_JRET(terrno); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 9bfb4102aa..e3f7ad4811 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -2340,7 +2340,7 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* } char* pJson = NULL; - parseTagDatatoJson(pTag, &pJson); + parseTagDatatoJson(pTag, &pJson, NULL); if (NULL == pJson) { taosArrayDestroy(pTagVals); CTG_ERR_JRET(terrno); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 33c0a2abee..a9d960176d 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -581,7 +581,7 @@ static void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) { } } -static int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { +static int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg, void* charsetCxt) { int32_t code = TSDB_CODE_SUCCESS; SArray* pTagVals = NULL; STag* pTag = (STag*)pCfg->pTags; @@ -593,7 +593,7 @@ static int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { if (tTagIsJson(pTag)) { char* pJson = NULL; - parseTagDatatoJson(pTag, &pJson); + parseTagDatatoJson(pTag, &pJson, charsetCxt); if (NULL == pJson) { qError("failed to parse tag to json, pJson is NULL"); return terrno; @@ -734,7 +734,7 @@ static void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STab } } -static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* pDbCfg, char* tbName, STableCfg* pCfg) { +static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* pDbCfg, char* tbName, STableCfg* pCfg, void* charsetCxt) { int32_t code = TSDB_CODE_SUCCESS; QRY_ERR_RET(blockDataEnsureCapacity(pBlock, 1)); pBlock->info.rows = 1; @@ -768,7 +768,7 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p appendTagNameFields(buf2, &len, pCfg); len += tsnprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), ") TAGS ("); - code = appendTagValues(buf2, &len, pCfg); + code = appendTagValues(buf2, &len, pCfg, charsetCxt); TAOS_CHECK_ERRNO(code); len += snprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), ")"); @@ -825,11 +825,11 @@ static int32_t setCreateViewResultIntoDataBlock(SSDataBlock* pBlock, SShowCreate return code; } -static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) { +static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp, void* charsetCxt) { SSDataBlock* pBlock = NULL; int32_t code = buildCreateTbResultDataBlock(&pBlock); if (TSDB_CODE_SUCCESS == code) { - code = setCreateTBResultIntoDataBlock(pBlock, pStmt->pDbCfg, pStmt->tableName, pStmt->pTableCfg); + code = setCreateTBResultIntoDataBlock(pBlock, pStmt->pDbCfg, pStmt->tableName, pStmt->pTableCfg, charsetCxt); } if (TSDB_CODE_SUCCESS == code) { code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_TB_RESULT_COLS, pRsp); @@ -838,14 +838,14 @@ static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRs return code; } -static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) { +static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp, void* charsetCxt) { STableCfg* pCfg = (STableCfg*)pStmt->pTableCfg; if (TSDB_SUPER_TABLE != pCfg->tableType) { terrno = TSDB_CODE_TSC_NOT_STABLE_ERROR; return terrno; } - return execShowCreateTable(pStmt, pRsp); + return execShowCreateTable(pStmt, pRsp, charsetCxt); } static int32_t execAlterCmd(char* cmd, char* value, bool* processed) { @@ -917,7 +917,7 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { return terrno; } - if (cfgSetItem(tsCfg, pStmt->config, pStmt->value, CFG_STYPE_ALTER_CMD, true)) { + if (cfgSetItem(tsCfg, pStmt->config, pStmt->value, CFG_STYPE_ALTER_CLIENT_CMD, true)) { return terrno; } @@ -1066,7 +1066,7 @@ static int32_t execShowCreateView(SShowCreateViewStmt* pStmt, SRetrieveTableRsp* return code; } -int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp, int8_t biMode) { +int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp, int8_t biMode, void* charsetCxt) { switch (nodeType(pStmt)) { case QUERY_NODE_DESCRIBE_STMT: return execDescribe(sysInfoUser, pStmt, pRsp, biMode); @@ -1075,9 +1075,9 @@ int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode* pStmt, SRetrieve case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: return execShowCreateDatabase((SShowCreateDatabaseStmt*)pStmt, pRsp); case QUERY_NODE_SHOW_CREATE_TABLE_STMT: - return execShowCreateTable((SShowCreateTableStmt*)pStmt, pRsp); + return execShowCreateTable((SShowCreateTableStmt*)pStmt, pRsp, charsetCxt); case QUERY_NODE_SHOW_CREATE_STABLE_STMT: - return execShowCreateSTable((SShowCreateTableStmt*)pStmt, pRsp); + return execShowCreateSTable((SShowCreateTableStmt*)pStmt, pRsp, charsetCxt); case QUERY_NODE_SHOW_CREATE_VIEW_STMT: return execShowCreateView((SShowCreateViewStmt*)pStmt, pRsp); case QUERY_NODE_ALTER_LOCAL_STMT: diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index d8ef097ce7..ccb7005dbd 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2418,7 +2418,7 @@ void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, b w->skey = key; } - w->ekey = taosTimeAdd(w->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + w->ekey = taosTimeAdd(w->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL) - 1; } } @@ -2473,15 +2473,15 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI TSKEY getNextTimeWindowStart(const SInterval* pInterval, TSKEY start, int32_t order) { int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - TSKEY nextStart = taosTimeAdd(start, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision); - nextStart = taosTimeAdd(nextStart, factor * pInterval->sliding, pInterval->slidingUnit, pInterval->precision); - nextStart = taosTimeAdd(nextStart, pInterval->offset, pInterval->offsetUnit, pInterval->precision); + TSKEY nextStart = taosTimeAdd(start, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision, NULL); + nextStart = taosTimeAdd(nextStart, factor * pInterval->sliding, pInterval->slidingUnit, pInterval->precision, NULL); + nextStart = taosTimeAdd(nextStart, pInterval->offset, pInterval->offsetUnit, pInterval->precision, NULL); return nextStart; } void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order) { tw->skey = getNextTimeWindowStart(pInterval, tw->skey, order); - tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL) - 1; } bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo) { diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index d6a518ccc4..bcbd3f7563 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -568,7 +568,7 @@ static void reviseFillStartAndEndKey(SFillOperatorInfo* pInfo, int32_t order) { next = ekey; while (next < pInfo->win.ekey) { next = taosTimeAdd(ekey, pInfo->pFillInfo->interval.sliding, pInfo->pFillInfo->interval.slidingUnit, - pInfo->pFillInfo->interval.precision); + pInfo->pFillInfo->interval.precision, NULL); ekey = next > pInfo->win.ekey ? ekey : next; } pInfo->win.ekey = ekey; @@ -577,7 +577,7 @@ static void reviseFillStartAndEndKey(SFillOperatorInfo* pInfo, int32_t order) { next = skey; while (next < pInfo->win.skey) { next = taosTimeAdd(skey, pInfo->pFillInfo->interval.sliding, pInfo->pFillInfo->interval.slidingUnit, - pInfo->pFillInfo->interval.precision); + pInfo->pFillInfo->interval.precision, NULL); skey = next > pInfo->win.skey ? skey : next; } taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey); diff --git a/source/libs/executor/src/hashjoinoperator.c b/source/libs/executor/src/hashjoinoperator.c index 06498d73e7..27efd59c12 100644 --- a/source/libs/executor/src/hashjoinoperator.c +++ b/source/libs/executor/src/hashjoinoperator.c @@ -114,7 +114,7 @@ int32_t hJoinLaunchPrimExpr(SSDataBlock* pBlock, SHJoinTableCtx* pTable, int32_t SColumnInfoData* pPrimOut = taosArrayGet(pBlock->pDataBlock, pTable->primCtx.targetSlotId); if (0 != pCtx->timezoneUnit) { for (int32_t i = startIdx; i <= endIdx; ++i) { - ((int64_t*)pPrimOut->pData)[i] = ((int64_t*)pPrimIn->pData)[i] - (((int64_t*)pPrimIn->pData)[i] - pCtx->timezoneUnit) % pCtx->truncateUnit; + ((int64_t*)pPrimOut->pData)[i] = ((int64_t*)pPrimIn->pData)[i] - (((int64_t*)pPrimIn->pData)[i] + pCtx->timezoneUnit) % pCtx->truncateUnit; } } else { for (int32_t i = startIdx; i <= endIdx; ++i) { diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index c992bd15b7..ffaa62721e 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -281,12 +281,12 @@ static void calcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillCo } static void setFillInfoStart(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { - ts = taosTimeAdd(ts, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + ts = taosTimeAdd(ts, pInterval->sliding, pInterval->slidingUnit, pInterval->precision, NULL); pFillInfo->start = ts; } static void setFillInfoEnd(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { - ts = taosTimeAdd(ts, pInterval->sliding * -1, pInterval->slidingUnit, pInterval->precision); + ts = taosTimeAdd(ts, pInterval->sliding * -1, pInterval->slidingUnit, pInterval->precision, NULL); pFillInfo->end = ts; } @@ -303,7 +303,7 @@ void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillS } TSKEY realStart = taosTimeAdd(pFillSup->prev.key, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); + pFillSup->interval.precision, NULL); pFillInfo->needFill = true; pFillInfo->start = realStart; @@ -542,7 +542,7 @@ static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* QUERY_CHECK_CODE(code, lino, _end); } pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); + pFillSup->interval.precision, NULL); } _end: @@ -564,7 +564,7 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* if ((pFillSup->hasDelete && !ckRes) || !inWinRange(&pFillSup->winRange, &st)) { pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); + pFillSup->interval.precision, NULL); pFillInfo->pLinearInfo->winIndex++; continue; } @@ -610,7 +610,7 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* } } pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); + pFillSup->interval.precision, NULL); pBlock->info.rows++; } @@ -987,7 +987,14 @@ _end: } void resetStreamFillSup(SStreamFillSupporter* pFillSup) { - tSimpleHashClear(pFillSup->pResMap); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + SSHashObj* pNewMap = tSimpleHashInit(16, hashFn); + if (pNewMap != NULL) { + tSimpleHashCleanup(pFillSup->pResMap); + pFillSup->pResMap = pNewMap; + } else { + tSimpleHashClear(pFillSup->pResMap); + } pFillSup->hasDelete = false; } void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) { @@ -1336,7 +1343,7 @@ static int32_t doStreamForceFillImpl(SOperatorInfo* pOperator) { break; } resTs = taosTimeAdd(resTs, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); + pFillSup->interval.precision, NULL); } } } @@ -1406,6 +1413,7 @@ static int32_t doStreamForceFillNext(SOperatorInfo* pOperator, SSDataBlock** ppR } pInfo->stateStore.streamStateClearExpiredState(pInfo->pState); + resetStreamFillInfo(pInfo); setStreamOperatorCompleted(pOperator); (*ppRes) = NULL; goto _end; @@ -1477,6 +1485,7 @@ static int32_t doStreamForceFillNext(SOperatorInfo* pOperator, SSDataBlock** ppR if ((*ppRes) == NULL) { pInfo->stateStore.streamStateClearExpiredState(pInfo->pState); + resetStreamFillInfo(pInfo); setStreamOperatorCompleted(pOperator); } diff --git a/source/libs/executor/src/streamintervalsliceoperator.c b/source/libs/executor/src/streamintervalsliceoperator.c index cd02f07143..d038e4d82c 100644 --- a/source/libs/executor/src/streamintervalsliceoperator.c +++ b/source/libs/executor/src/streamintervalsliceoperator.c @@ -120,6 +120,36 @@ void initIntervalSlicePoint(SStreamAggSupporter* pAggSup, STimeWindow* pTWin, in pPoint->pLastRow = POINTER_SHIFT(pPoint->pFinished, sizeof(bool)); } +int32_t getIntervalSlicePrevStateBuf(SStreamAggSupporter* pAggSup, SInterval* pInterval, SWinKey* pCurKey, + SInervalSlicePoint* pPrevPoint) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SWinKey prevKey = {.groupId = pCurKey->groupId}; + SET_WIN_KEY_INVALID(prevKey.ts); + int32_t prevVLen = 0; + int32_t prevWinCode = TSDB_CODE_SUCCESS; + code = pAggSup->stateStore.streamStateGetPrev(pAggSup->pState, pCurKey, &prevKey, (void**)&pPrevPoint->pResPos, + &prevVLen, &prevWinCode); + QUERY_CHECK_CODE(code, lino, _end); + + if (prevWinCode == TSDB_CODE_SUCCESS) { + STimeWindow prevSTW = {.skey = prevKey.ts}; + prevSTW.ekey = taosTimeGetIntervalEnd(prevSTW.skey, pInterval); + initIntervalSlicePoint(pAggSup, &prevSTW, pCurKey->groupId, pPrevPoint); + qDebug("===stream=== set stream twa prev point buf.ts:%" PRId64 ", groupId:%" PRIu64 ", res:%d", + pPrevPoint->winKey.win.skey, pPrevPoint->winKey.groupId, prevWinCode); + } else { + SET_WIN_KEY_INVALID(pPrevPoint->winKey.win.skey); + SET_WIN_KEY_INVALID(pPrevPoint->winKey.win.ekey); + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; +} + static int32_t getIntervalSliceCurStateBuf(SStreamAggSupporter* pAggSup, SInterval* pInterval, bool needPrev, STimeWindow* pTWin, int64_t groupId, SInervalSlicePoint* pCurPoint, SInervalSlicePoint* pPrevPoint, int32_t* pWinCode) { int32_t code = TSDB_CODE_SUCCESS; @@ -136,24 +166,8 @@ static int32_t getIntervalSliceCurStateBuf(SStreamAggSupporter* pAggSup, SInterv initIntervalSlicePoint(pAggSup, pTWin, groupId, pCurPoint); if (needPrev) { - SWinKey prevKey = {.groupId = groupId}; - SET_WIN_KEY_INVALID(prevKey.ts); - int32_t prevVLen = 0; - int32_t prevWinCode = TSDB_CODE_SUCCESS; - code = pAggSup->stateStore.streamStateGetPrev(pAggSup->pState, &curKey, &prevKey, (void**)&pPrevPoint->pResPos, - &prevVLen, &prevWinCode); + code = getIntervalSlicePrevStateBuf(pAggSup, pInterval, &curKey, pPrevPoint); QUERY_CHECK_CODE(code, lino, _end); - - if (prevWinCode == TSDB_CODE_SUCCESS) { - STimeWindow prevSTW = {.skey = prevKey.ts}; - prevSTW.ekey = taosTimeGetIntervalEnd(prevSTW.skey, pInterval); - initIntervalSlicePoint(pAggSup, &prevSTW, groupId, pPrevPoint); - qDebug("===stream=== set stream twa prev point buf.ts:%" PRId64 ", groupId:%" PRIu64 ", res:%d", pPrevPoint->winKey.win.skey, - pPrevPoint->winKey.groupId, prevWinCode); - } else { - SET_WIN_KEY_INVALID(pPrevPoint->winKey.win.skey); - SET_WIN_KEY_INVALID(pPrevPoint->winKey.win.ekey); - } } _end: @@ -265,13 +279,15 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc STimeWindow curWin = getActiveTimeWindow(NULL, pResultRowInfo, curTs, &pInfo->interval, TSDB_ORDER_ASC); while (1) { - if (curTs > pInfo->endTs) { - break; - } - int32_t winCode = TSDB_CODE_SUCCESS; - code = getIntervalSliceCurStateBuf(&pInfo->streamAggSup, &pInfo->interval, pInfo->hasInterpoFunc, &curWin, groupId, &curPoint, &prevPoint, &winCode); - QUERY_CHECK_CODE(code, lino, _end); + if (curTs <= pInfo->endTs) { + code = getIntervalSliceCurStateBuf(&pInfo->streamAggSup, &pInfo->interval, pInfo->hasInterpoFunc, &curWin, groupId, &curPoint, &prevPoint, &winCode); + QUERY_CHECK_CODE(code, lino, _end); + } else if (pInfo->hasInterpoFunc) { + SWinKey curKey = {.ts = curWin.skey, .groupId = groupId}; + code = getIntervalSlicePrevStateBuf(&pInfo->streamAggSup, &pInfo->interval, &curKey, &prevPoint); + QUERY_CHECK_CODE(code, lino, _end); + } if (pInfo->hasInterpoFunc && IS_VALID_WIN_KEY(prevPoint.winKey.win.skey) && isInterpoWindowFinished(&prevPoint) == false) { code = setIntervalSliceOutputBuf(&prevPoint, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); @@ -288,6 +304,12 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc code = saveWinResult(&prevKey, prevPoint.pResPos, pInfo->pUpdatedMap); QUERY_CHECK_CODE(code, lino, _end); setInterpoWindowFinished(&prevPoint); + } else if (IS_VALID_WIN_KEY(prevPoint.winKey.win.skey)) { + releaseOutputBuf(pInfo->streamAggSup.pState, prevPoint.pResPos, &pInfo->streamAggSup.stateStore); + } + + if (curTs > pInfo->endTs) { + break; } code = setIntervalSliceOutputBuf(&curPoint, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); @@ -300,7 +322,7 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, curWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); int32_t prevEndPos = (forwardRows - 1) + startPos; - if (pInfo->hasInterpoFunc && winCode != TSDB_CODE_SUCCESS) { + if (pInfo->hasInterpoFunc) { int32_t endRowId = getQualifiedRowNumDesc(pSup, pBlock, tsCols, prevEndPos, false); TSKEY endRowTs = tsCols[endRowId]; transBlockToSliceResultRow(pBlock, endRowId, endRowTs, curPoint.pLastRow, 0, NULL, NULL, pInfo->pOffsetInfo); diff --git a/source/libs/executor/src/streamtimesliceoperator.c b/source/libs/executor/src/streamtimesliceoperator.c index 8611678e5a..44004a4c6b 100644 --- a/source/libs/executor/src/streamtimesliceoperator.c +++ b/source/libs/executor/src/streamtimesliceoperator.c @@ -467,7 +467,7 @@ static void fillNormalRange(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFi QUERY_CHECK_CODE(code, lino, _end); // } pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); + pFillSup->interval.precision, NULL); } _end: @@ -527,7 +527,7 @@ static void fillLinearRange(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFi } } pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); + pFillSup->interval.precision, NULL); if (ckRes) { pBlock->info.rows++; } @@ -547,14 +547,14 @@ static void setFillKeyInfo(TSKEY start, TSKEY end, SInterval* pInterval, SStream static TSKEY adustPrevTsKey(TSKEY pointTs, TSKEY rowTs, SInterval* pInterval) { if (rowTs >= pointTs) { - pointTs = taosTimeAdd(pointTs, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + pointTs = taosTimeAdd(pointTs, pInterval->sliding, pInterval->slidingUnit, pInterval->precision, NULL); } return pointTs; } static TSKEY adustEndTsKey(TSKEY pointTs, TSKEY rowTs, SInterval* pInterval) { if (rowTs <= pointTs) { - pointTs = taosTimeAdd(pointTs, pInterval->sliding * -1, pInterval->slidingUnit, pInterval->precision); + pointTs = taosTimeAdd(pointTs, pInterval->sliding * -1, pInterval->slidingUnit, pInterval->precision, NULL); } return pointTs; } @@ -954,7 +954,7 @@ static int32_t getPointInfoFromState(SStreamAggSupporter* pAggSup, SStreamFillSu } } else { pNextPoint->key.ts = taosTimeAdd(pCurPoint->key.ts, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); + pFillSup->interval.precision, NULL); code = pAggSup->stateStore.streamStateFillAddIfNotExist(pState, &pNextPoint->key, (void**)&pNextPoint->pResPos, &nextVLen, &tmpRes); QUERY_CHECK_CODE(code, lino, _end); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 039baca54c..e576e0d920 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -358,7 +358,7 @@ static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp void* chIds = taosHashGet(pPullDataMap, pWinKey, sizeof(SWinKey)); STimeWindow win = { .skey = pWinKey->ts, - .ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1, + .ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL) - 1, }; if (isCloseWindow(&win, pTwSup)) { if (chIds && pPullDataMap) { @@ -391,7 +391,7 @@ _end: STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) { STimeWindow w = {.skey = ts, .ekey = INT64_MAX}; - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL) - 1; return w; } @@ -851,7 +851,7 @@ static int32_t processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pF } } } - winTs = taosTimeAdd(winTs, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + winTs = taosTimeAdd(winTs, pInterval->sliding, pInterval->slidingUnit, pInterval->precision, NULL); } } if (pBeOver) { diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 464ec40eac..9fabda9ef5 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -981,7 +981,7 @@ int32_t convertTagDataToStr(char* str, int32_t strBuffLen, int type, void* buf, return TSDB_CODE_TSC_INVALID_VALUE; } - int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str); + int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str, NULL); if (length <= 0) { return TSDB_CODE_TSC_INVALID_VALUE; } @@ -1132,7 +1132,7 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, if (tagData != NULL) { if (tagType == TSDB_DATA_TYPE_JSON) { char* tagJson = NULL; - parseTagDatatoJson(tagData, &tagJson); + parseTagDatatoJson(tagData, &tagJson, NULL); if (tagJson == NULL) { code = terrno; goto _end; diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 190b327522..ecbb389f8a 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -143,7 +143,7 @@ bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnI // TODO: include endpoint SInterval* pInterval = &pFillInfo->interval; int64_t windowEnd = - taosTimeAdd(pFillInfo->currentKey, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + taosTimeAdd(pFillInfo->currentKey, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL); code = colDataSetVal(pDstColInfoData, rowIndex, (const char*)&windowEnd, false); QUERY_CHECK_CODE(code, lino, _end); return true; @@ -264,7 +264,7 @@ static void doFillOneRow(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* // setTagsValue(pFillInfo, data, index); SInterval* pInterval = &pFillInfo->interval; pFillInfo->currentKey = - taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision); + taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision, NULL); pBlock->info.rows += 1; pFillInfo->numOfCurrent++; @@ -484,7 +484,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t // set the tag value for final result SInterval* pInterval = &pFillInfo->interval; pFillInfo->currentKey = - taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision); + taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision, NULL); pBlock->info.rows += 1; pFillInfo->index += 1; diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 6a3052c75c..3a639772c8 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -901,7 +901,7 @@ static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pS doKeepLinearInfo(pSliceInfo, pBlock, i); pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL); if (checkWindowBoundReached(pSliceInfo)) { break; @@ -927,7 +927,7 @@ static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pS break; } else { pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL); } } @@ -955,7 +955,7 @@ static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pS break; } else { pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL); } } @@ -965,7 +965,7 @@ static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pS QUERY_CHECK_CODE(code, lino, _end); pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL); } doKeepPrevRows(pSliceInfo, pBlock, i); @@ -1003,7 +1003,7 @@ static void genInterpAfterDataBlock(STimeSliceOperatorInfo* pSliceInfo, SOperato while (pSliceInfo->current <= pSliceInfo->win.ekey) { (void)genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, NULL, index, false, pOperator->pTaskInfo); pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL); } } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 00c0e9e61f..c2aa07dacc 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1922,7 +1922,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR STimeWindow win = {0}; win.skey = miaInfo->curTs; - win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL) - 1; int32_t ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup); if (ret != TSDB_CODE_SUCCESS || miaInfo->pResultRow == NULL) { @@ -1949,7 +1949,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR miaInfo->curTs = tsCols[currPos]; currWin.skey = miaInfo->curTs; - currWin.ekey = taosTimeAdd(currWin.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + currWin.ekey = taosTimeAdd(currWin.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL) - 1; startPos = currPos; ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup); diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 69097ce755..5d397be623 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -592,7 +592,7 @@ void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) { memset(pTmp, 'A' + taosRand() % 26, *pLen); *ppVal = taosMemoryCalloc(1, *pLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); assert(*ppVal); - assert(taosMbsToUcs4(pTmp, *pLen, (TdUcs4 *)varDataVal(*ppVal), *pLen * TSDB_NCHAR_SIZE, NULL)); + assert(taosMbsToUcs4(pTmp, *pLen, (TdUcs4 *)varDataVal(*ppVal), *pLen * TSDB_NCHAR_SIZE, NULL, NULL)); *pLen *= TSDB_NCHAR_SIZE; varDataSetLen(*ppVal, *pLen); taosMemoryFree(pTmp); diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index b2313f35a1..ce2da01b66 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -93,7 +93,7 @@ SSDataBlock* getSingleColDummyBlock(void* param) { char strOri[128] = {0}; taosRandStr(strOri, size); int32_t len = 0; - bool ret = taosMbsToUcs4(strOri, size, (TdUcs4*)varDataVal(str), size * TSDB_NCHAR_SIZE, &len); + bool ret = taosMbsToUcs4(strOri, size, (TdUcs4*)varDataVal(str), size * TSDB_NCHAR_SIZE, &len, NULL); if (!ret) { (void) printf("error\n"); return NULL; @@ -331,7 +331,7 @@ TEST(testCase, external_mem_sort_Test) { if(pInfo[i].type == TSDB_DATA_TYPE_NCHAR){ char buf[128] = {0}; - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(v), varDataLen(v), buf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(v), varDataLen(v), buf, NULL); printf("%d: %s\n", row++, buf); }else if(pInfo[i].type == TSDB_DATA_TYPE_BINARY || pInfo[i]->type == TSDB_DATA_TYPE_GEOMETRY){ char buf[128] = {0}; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 9fe01370f9..95a332ac05 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -186,16 +186,16 @@ static int32_t countTrailingSpaces(const SValueNode* pVal, bool isLtrim) { return numOfSpaces; } -static int32_t addTimezoneParam(SNodeList* pList) { - char buf[TD_TIME_STR_LEN] = {0}; - time_t t; +static int32_t addTimezoneParam(SNodeList* pList, timezone_t tz) { + char buf[TD_TIME_STR_LEN] = {0}; + time_t t; int32_t code = taosTime(&t); if (code != 0) { return code; } struct tm tmInfo; - if (taosLocalTime(&t, &tmInfo, buf, sizeof(buf)) != NULL) { - (void)strftime(buf, sizeof(buf), "%z", &tmInfo); + if (taosLocalTime(&t, &tmInfo, buf, sizeof(buf), tz) != NULL) { + (void)taosStrfTime(buf, sizeof(buf), "%z", &tmInfo); } int32_t len = (int32_t)strlen(buf); @@ -1445,7 +1445,7 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "Invalid timzone format"); } } else { // add default client timezone - int32_t code = addTimezoneParam(pFunc->pParameterList); + int32_t code = addTimezoneParam(pFunc->pParameterList, pFunc->tz); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1500,7 +1500,7 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_ } // add client timezone as param - code = addTimezoneParam(pFunc->pParameterList); + code = addTimezoneParam(pFunc->pParameterList, pFunc->tz); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1657,9 +1657,18 @@ static int32_t translateOutVarchar(SFunctionNode* pFunc, char* pErrBuf, int32_t case FUNCTION_TYPE_TBNAME: bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE; break; - case FUNCTION_TYPE_TIMEZONE: - bytes = timeZoneStrLen(); + case FUNCTION_TYPE_TIMEZONE:{ + if (pFunc->tz == NULL) { + bytes = VARSTR_HEADER_SIZE + strlen(tsTimezoneStr); + }else{ + char *tzName = (char*)taosHashGet(pTimezoneNameMap, &pFunc->tz, sizeof(timezone_t)); + if (tzName == NULL){ + tzName = TZ_UNKNOWN; + } + bytes = strlen(tzName) + VARSTR_HEADER_SIZE; + } break; + } case FUNCTION_TYPE_IRATE_PARTIAL: bytes = getIrateInfoSize((pFunc->hasPk) ? pFunc->pkBytes : 0) + VARSTR_HEADER_SIZE; break; @@ -4943,7 +4952,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .paramInfoPattern = 0, .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_ALL_TYPE}}, .translateFunc = translateSelectValue, - .getEnvFunc = getSelectivityFuncEnv, + .getEnvFunc = getGroupKeyFuncEnv, .initFunc = functionSetup, .processFunc = groupConstValueFunction, .finalizeFunc = groupConstValueFinalize, diff --git a/source/libs/monitorfw/src/taos_monitor_util.c b/source/libs/monitorfw/src/taos_monitor_util.c index b0eff27507..1240526d48 100644 --- a/source/libs/monitorfw/src/taos_monitor_util.c +++ b/source/libs/monitorfw/src/taos_monitor_util.c @@ -41,10 +41,11 @@ void taos_monitor_split_str_metric(char** arr, taos_metric_t* metric, const char memset(name, 0, size + 1); memcpy(name, metric->name, size); - char* s = strtok(name, del); + char* saveptr; + char* s = strtok_r(name, del, &saveptr); while (s != NULL) { *arr++ = s; - s = strtok(NULL, del); + s = strtok_r(NULL, del, &saveptr); } *buf = name; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 6ead98df0f..27c642470d 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -26,9 +26,9 @@ (pDst)->fldname = (pSrc)->fldname; \ } while (0) -#define COPY_CHAR_ARRAY_FIELD(fldname) \ - do { \ - strcpy((pDst)->fldname, (pSrc)->fldname); \ +#define COPY_CHAR_ARRAY_FIELD(fldname) \ + do { \ + tstrncpy((pDst)->fldname, (pSrc)->fldname, sizeof((pDst)->fldname)); \ } while (0) #define COPY_OBJECT_FIELD(fldname, size) \ diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 466275b397..bd3baea518 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2362,7 +2362,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode) { return NULL; } - sprintf(buf, "%s", pNode->datum.b ? "true" : "false"); + snprintf(buf, MAX_NUM_STR_SIZE, "%s", pNode->datum.b ? "true" : "false"); return buf; } case TSDB_DATA_TYPE_TINYINT: @@ -2375,7 +2375,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode) { return NULL; } - sprintf(buf, "%" PRId64, pNode->datum.i); + snprintf(buf, MAX_NUM_STR_SIZE, "%" PRId64, pNode->datum.i); return buf; } case TSDB_DATA_TYPE_UTINYINT: @@ -2387,7 +2387,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode) { return NULL; } - sprintf(buf, "%" PRIu64, pNode->datum.u); + snprintf(buf, MAX_NUM_STR_SIZE, "%" PRIu64, pNode->datum.u); return buf; } case TSDB_DATA_TYPE_FLOAT: @@ -2397,7 +2397,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode) { return NULL; } - sprintf(buf, "%e", pNode->datum.d); + snprintf(buf, MAX_NUM_STR_SIZE, "%e", pNode->datum.d); return buf; } case TSDB_DATA_TYPE_NCHAR: @@ -2553,7 +2553,7 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN } if (pCol->projRefIdx > 0) { len = taosHashBinary(name, strlen(name)); - len += sprintf(name + len, "_%d", pCol->projRefIdx); + len += tsnprintf(name + len, TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN - len, "_%d", pCol->projRefIdx); } SNode** pNodeFound = taosHashGet(pCxt->pColHash, name, len); if (pNodeFound == NULL) { @@ -2973,17 +2973,16 @@ int32_t nodesValueNodeToVariant(const SValueNode* pNode, SVariant* pVal) { case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: - pVal->pz = taosMemoryMalloc(pVal->nLen + 1); + pVal->pz = taosMemoryCalloc(1, pVal->nLen + 1); if (pVal->pz) { - memcpy(pVal->pz, pNode->datum.p, pVal->nLen); - pVal->pz[pVal->nLen] = 0; + memcpy(pVal->pz, pNode->datum.p, varDataTLen(pNode->datum.p)); } else { code = terrno; } break; case TSDB_DATA_TYPE_JSON: pVal->nLen = getJsonValueLen(pNode->datum.p); - pVal->pz = taosMemoryMalloc(pVal->nLen); + pVal->pz = taosMemoryCalloc(1, pVal->nLen); if (pVal->pz) { memcpy(pVal->pz, pNode->datum.p, pVal->nLen); } else { diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 7298b04eb0..aec5fb4cd6 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -136,7 +136,7 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName); int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf, int8_t type); int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SSchema* pTagSchema, SToken* pToken, - SArray* pTagName, SArray* pTagVals, STag** pTag); + SArray* pTagName, SArray* pTagVals, STag** pTag, timezone_t tz, void *charsetCxt); int32_t parseTbnameToken(SMsgBuf* pMsgBuf, char* tname, SToken* pToken, bool* pFoundCtbName); int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq); diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 9e8b083578..a13472620b 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -142,13 +142,13 @@ static int32_t parseEndpoint(SAstCreateContext* pCxt, const SToken* pEp, char* p (void)strdequote(ep); (void)strtrim(ep); if (NULL == pPort) { - strcpy(pFqdn, ep); + tstrncpy(pFqdn, ep, TSDB_FQDN_LEN); return TSDB_CODE_SUCCESS; } char* pColon = strchr(ep, ':'); if (NULL == pColon) { *pPort = tsServerPort; - strcpy(pFqdn, ep); + tstrncpy(pFqdn, ep, TSDB_FQDN_LEN); return TSDB_CODE_SUCCESS; } strncpy(pFqdn, ep, pColon - ep); @@ -320,12 +320,12 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { if (nodesIsExprNode(pRealizedExpr)) { SExprNode* pExpr = (SExprNode*)pRealizedExpr; if (QUERY_NODE_COLUMN == nodeType(pExpr)) { - strcpy(pExpr->aliasName, ((SColumnNode*)pExpr)->colName); - strcpy(pExpr->userAlias, ((SColumnNode*)pExpr)->colName); + tstrncpy(pExpr->aliasName, ((SColumnNode*)pExpr)->colName, TSDB_COL_NAME_LEN); + tstrncpy(pExpr->userAlias, ((SColumnNode*)pExpr)->colName, TSDB_COL_NAME_LEN); } else if (pRawExpr->isPseudoColumn) { // all pseudo column are translate to function with same name - strcpy(pExpr->userAlias, ((SFunctionNode*)pExpr)->functionName); - strcpy(pExpr->aliasName, ((SFunctionNode*)pExpr)->functionName); + tstrncpy(pExpr->userAlias, ((SFunctionNode*)pExpr)->functionName, TSDB_COL_NAME_LEN); + tstrncpy(pExpr->aliasName, ((SFunctionNode*)pExpr)->functionName, TSDB_COL_NAME_LEN); } else { int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); @@ -333,9 +333,9 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { // Len of pRawExpr->p could be larger than len of aliasName[TSDB_COL_NAME_LEN]. // If aliasName is truncated, hash value of aliasName could be the same. uint64_t hashVal = MurmurHash3_64(pRawExpr->p, pRawExpr->n); - sprintf(pExpr->aliasName, "%" PRIu64, hashVal); + snprintf(pExpr->aliasName, TSDB_COL_NAME_LEN, "%" PRIu64, hashVal); strncpy(pExpr->userAlias, pRawExpr->p, len); - pExpr->userAlias[len] = '\0'; + pExpr->userAlias[len] = 0; } } pRawExpr->pNode = NULL; @@ -420,6 +420,8 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* val->node.resType.precision = TSDB_TIME_PRECISION_MILLI; } val->translate = false; + val->tz = pCxt->pQueryCxt->timezone; + val->charsetCxt = pCxt->pQueryCxt->charsetCxt; return (SNode*)val; _err: return NULL; @@ -946,11 +948,11 @@ SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pL goto _err; } if ('+' == pVal->literal[0]) { - sprintf(pNewLiteral, "-%s", pVal->literal + 1); + snprintf(pNewLiteral, strlen(pVal->literal) + 2, "-%s", pVal->literal + 1); } else if ('-' == pVal->literal[0]) { - sprintf(pNewLiteral, "%s", pVal->literal + 1); + snprintf(pNewLiteral, strlen(pVal->literal) + 2, "%s", pVal->literal + 1); } else { - sprintf(pNewLiteral, "-%s", pVal->literal); + snprintf(pNewLiteral, strlen(pVal->literal) + 2, "-%s", pVal->literal); } taosMemoryFree(pVal->literal); pVal->literal = pNewLiteral; @@ -963,6 +965,8 @@ SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pL op->opType = type; op->pLeft = pLeft; op->pRight = pRight; + op->tz = pCxt->pQueryCxt->timezone; + op->charsetCxt = pCxt->pQueryCxt->charsetCxt; return (SNode*)op; _err: nodesDestroyNode(pLeft); @@ -1021,7 +1025,7 @@ static SNode* createPrimaryKeyCol(SAstCreateContext* pCxt, const SToken* pFuncNa CHECK_MAKE_NODE(pCol); pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; if (NULL == pFuncName) { - strcpy(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME); + tstrncpy(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME, TSDB_COL_NAME_LEN); } else { strncpy(pCol->colName, pFuncName->z, pFuncName->n); } @@ -1041,6 +1045,8 @@ SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNod CHECK_MAKE_NODE(func); COPY_STRING_FORM_ID_TOKEN(func->functionName, pFuncName); func->pParameterList = pParameterList; + func->tz = pCxt->pQueryCxt->timezone; + func->charsetCxt = pCxt->pQueryCxt->charsetCxt; return (SNode*)func; _err: nodesDestroyList(pParameterList); @@ -1052,7 +1058,7 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); - strcpy(func->functionName, "cast"); + tstrncpy(func->functionName, "cast", TSDB_FUNC_NAME_LEN); func->node.resType = dt; if (TSDB_DATA_TYPE_VARCHAR == dt.type || TSDB_DATA_TYPE_GEOMETRY == dt.type || TSDB_DATA_TYPE_VARBINARY == dt.type) { func->node.resType.bytes = func->node.resType.bytes + VARSTR_HEADER_SIZE; @@ -1061,6 +1067,9 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d } pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); CHECK_PARSER_STATUS(pCxt); + func->tz = pCxt->pQueryCxt->timezone; + func->charsetCxt = pCxt->pQueryCxt->charsetCxt; + return (SNode*)func; _err: nodesDestroyNode((SNode*)func); @@ -1073,7 +1082,7 @@ SNode* createPositionFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SNode* CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); - strcpy(func->functionName, "position"); + tstrncpy(func->functionName, "position", TSDB_FUNC_NAME_LEN); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr2); @@ -1091,10 +1100,11 @@ SNode* createTrimFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, ETrimType t CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); - strcpy(func->functionName, "trim"); + tstrncpy(func->functionName, "trim", TSDB_FUNC_NAME_LEN); func->trimType = type; pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); CHECK_PARSER_STATUS(pCxt); + func->charsetCxt = pCxt->pQueryCxt->charsetCxt; return (SNode*)func; _err: nodesDestroyNode((SNode*)func); @@ -1107,12 +1117,13 @@ SNode* createTrimFunctionNodeExt(SAstCreateContext* pCxt, SNode* pExpr, SNode* p CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); - strcpy(func->functionName, "trim"); + tstrncpy(func->functionName, "trim", TSDB_FUNC_NAME_LEN); func->trimType = type; pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr2); CHECK_PARSER_STATUS(pCxt); + func->charsetCxt = pCxt->pQueryCxt->charsetCxt; return (SNode*)func; _err: nodesDestroyNode((SNode*)func); @@ -1126,7 +1137,7 @@ SNode* createSubstrFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SNode* pE CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); - strcpy(func->functionName, "substr"); + tstrncpy(func->functionName, "substr", TSDB_FUNC_NAME_LEN); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr2); @@ -1144,7 +1155,7 @@ SNode* createSubstrFunctionNodeExt(SAstCreateContext* pCxt, SNode* pExpr, SNode* CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); CHECK_MAKE_NODE(func); - strcpy(func->functionName, "substr"); + tstrncpy(func->functionName, "substr", TSDB_FUNC_NAME_LEN); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr2); @@ -1228,10 +1239,10 @@ SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, SToken* pT taosRandStr(tempTable->table.tableAlias, 8); } if (QUERY_NODE_SELECT_STMT == nodeType(pSubquery)) { - strcpy(((SSelectStmt*)pSubquery)->stmtName, tempTable->table.tableAlias); + tstrncpy(((SSelectStmt*)pSubquery)->stmtName, tempTable->table.tableAlias, TSDB_TABLE_NAME_LEN); ((SSelectStmt*)pSubquery)->isSubquery = true; } else if (QUERY_NODE_SET_OPERATOR == nodeType(pSubquery)) { - strcpy(((SSetOperator*)pSubquery)->stmtName, tempTable->table.tableAlias); + tstrncpy(((SSetOperator*)pSubquery)->stmtName, tempTable->table.tableAlias, TSDB_TABLE_NAME_LEN); } return (SNode*)tempTable; _err: @@ -1400,6 +1411,7 @@ SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode interval->pSliding = pSliding; interval->pFill = pFill; interval->timeRange = TSWINDOW_INITIALIZER; + interval->timezone = pCxt->pQueryCxt->timezone; return (SNode*)interval; _err: nodesDestroyNode((SNode*)interval); @@ -1435,7 +1447,7 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { fill->pWStartTs = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&(fill->pWStartTs)); CHECK_MAKE_NODE(fill->pWStartTs); - strcpy(((SFunctionNode*)fill->pWStartTs)->functionName, "_wstart"); + tstrncpy(((SFunctionNode*)fill->pWStartTs)->functionName, "_wstart", TSDB_FUNC_NAME_LEN); return (SNode*)fill; _err: nodesDestroyNode((SNode*)fill); @@ -1514,6 +1526,8 @@ SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhe pCaseWhen->pCase = pCase; pCaseWhen->pWhenThenList = pWhenThenList; pCaseWhen->pElse = pElse; + pCaseWhen->tz = pCxt->pQueryCxt->timezone; + pCaseWhen->charsetCxt = pCxt->pQueryCxt->charsetCxt; return (SNode*)pCaseWhen; _err: nodesDestroyNode(pCase); @@ -1765,7 +1779,7 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* setSubquery(setOp->pLeft); setOp->pRight = pRight; setSubquery(setOp->pRight); - sprintf(setOp->stmtName, "%p", setOp); + snprintf(setOp->stmtName, TSDB_TABLE_NAME_LEN, "%p", setOp); return (SNode*)setOp; _err: nodesDestroyNode(pLeft); @@ -2376,8 +2390,8 @@ SNode* createCreateTableStmt(SAstCreateContext* pCxt, bool ignoreExists, SNode* SCreateTableStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_TABLE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); pStmt->ignoreExists = ignoreExists; pStmt->pCols = pCols; pStmt->pTags = pTags; @@ -2398,10 +2412,10 @@ SNode* createCreateSubTableClause(SAstCreateContext* pCxt, bool ignoreExists, SN SCreateSubTableClause* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_SUBTABLE_CLAUSE, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); - strcpy(pStmt->useDbName, ((SRealTableNode*)pUseRealTable)->table.dbName); - strcpy(pStmt->useTableName, ((SRealTableNode*)pUseRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pStmt->useDbName, ((SRealTableNode*)pUseRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->useTableName, ((SRealTableNode*)pUseRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); pStmt->ignoreExists = ignoreExists; pStmt->pSpecificTags = pSpecificTags; pStmt->pValsOfTags = pValsOfTags; @@ -2424,8 +2438,8 @@ SNode* createCreateSubTableFromFileClause(SAstCreateContext* pCxt, bool ignoreEx SCreateSubTableFromFileClause* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->useDbName, ((SRealTableNode*)pUseRealTable)->table.dbName); - strcpy(pStmt->useTableName, ((SRealTableNode*)pUseRealTable)->table.tableName); + tstrncpy(pStmt->useDbName, ((SRealTableNode*)pUseRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->useTableName, ((SRealTableNode*)pUseRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); pStmt->ignoreExists = ignoreExists; pStmt->pSpecificTags = pSpecificTags; if (TK_NK_STRING == pFilePath->type) { @@ -2459,8 +2473,8 @@ SNode* createDropTableClause(SAstCreateContext* pCxt, bool ignoreNotExists, SNod SDropTableClause* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_DROP_TABLE_CLAUSE, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); pStmt->ignoreNotExists = ignoreNotExists; nodesDestroyNode(pRealTable); return (SNode*)pStmt; @@ -2487,8 +2501,8 @@ SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool withOpt, bool igno SDropSuperTableStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_DROP_SUPER_TABLE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); pStmt->ignoreNotExists = ignoreNotExists; pStmt->withOpt = withOpt; nodesDestroyNode(pRealTable); @@ -2499,8 +2513,8 @@ _err: } static SNode* createAlterTableStmtFinalize(SNode* pRealTable, SAlterTableStmt* pStmt) { - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); nodesDestroyNode(pRealTable); return (SNode*)pStmt; } @@ -2819,8 +2833,8 @@ SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* SShowCreateTableStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(type, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); nodesDestroyNode(pRealTable); return (SNode*)pStmt; _err: @@ -2833,8 +2847,8 @@ SNode* createShowCreateViewStmt(SAstCreateContext* pCxt, ENodeType type, SNode* SShowCreateViewStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(type, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->viewName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->viewName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); nodesDestroyNode(pRealTable); return (SNode*)pStmt; _err: @@ -2847,8 +2861,8 @@ SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable SShowTableDistributedStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); nodesDestroyNode(pRealTable); return (SNode*)pStmt; _err: @@ -2926,13 +2940,16 @@ static int32_t getIpV4RangeFromWhitelistItem(char* ipRange, SIpV4Range* pIpRange *slash = '\0'; struct in_addr addr; if (uv_inet_pton(AF_INET, ipCopy, &addr) == 0) { - int prefix = atoi(slash + 1); - if (prefix < 0 || prefix > 32) { - code = TSDB_CODE_PAR_INVALID_IP_RANGE; - } else { - pIpRange->ip = addr.s_addr; - pIpRange->mask = prefix; - code = TSDB_CODE_SUCCESS; + int32_t prefix = 0; + code = taosStr2int32(slash + 1, &prefix); + if (code == 0) { + if (prefix < 0 || prefix > 32) { + code = TSDB_CODE_PAR_INVALID_IP_RANGE; + } else { + pIpRange->ip = addr.s_addr; + pIpRange->mask = prefix; + code = TSDB_CODE_SUCCESS; + } } } else { code = TSDB_CODE_PAR_INVALID_IP_RANGE; @@ -3004,7 +3021,7 @@ SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const ST pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_USER_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); - strcpy(pStmt->password, password); + tstrncpy(pStmt->password, password, TSDB_USET_PASSWORD_LEN); pStmt->sysinfo = sysinfo; pStmt->createDb = createDb; pStmt->isImport = is_import; @@ -3026,7 +3043,7 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t al char password[TSDB_USET_PASSWORD_LEN] = {0}; SToken* pVal = pAlterInfo; CHECK_NAME(checkPassword(pCxt, pVal, password)); - strcpy(pStmt->password, password); + tstrncpy(pStmt->password, password, TSDB_USET_PASSWORD_LEN); break; } case TSDB_ALTER_USER_ENABLE: { @@ -3336,8 +3353,8 @@ SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, pStmt->withMeta = withMeta; pStmt->pWhere = pWhere; - strcpy(pStmt->subDbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->subDbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); nodesDestroyNode(pRealTable); return (SNode*)pStmt; _err: @@ -3450,8 +3467,8 @@ SNode* createDescribeStmt(SAstCreateContext* pCxt, SNode* pRealTable) { SDescribeStmt* pStmt = NULL; pCxt->errCode = nodesMakeNode(QUERY_NODE_DESCRIBE_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); nodesDestroyNode(pRealTable); return (SNode*)pStmt; _err: @@ -3531,8 +3548,8 @@ SNode* createCreateViewStmt(SAstCreateContext* pCxt, bool orReplace, SNode* pVie } pStmt->pQuerySql = tstrdup(pAs->z + i); CHECK_OUT_OF_MEM(pStmt->pQuerySql); - strcpy(pStmt->dbName, ((SViewNode*)pView)->table.dbName); - strcpy(pStmt->viewName, ((SViewNode*)pView)->table.tableName); + tstrncpy(pStmt->dbName, ((SViewNode*)pView)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->viewName, ((SViewNode*)pView)->table.tableName, TSDB_VIEW_NAME_LEN); nodesDestroyNode(pView); pStmt->orReplace = orReplace; pStmt->pQuery = pQuery; @@ -3550,8 +3567,8 @@ SNode* createDropViewStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pCxt->errCode = nodesMakeNode(QUERY_NODE_DROP_VIEW_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); pStmt->ignoreNotExists = ignoreNotExists; - strcpy(pStmt->dbName, ((SViewNode*)pView)->table.dbName); - strcpy(pStmt->viewName, ((SViewNode*)pView)->table.tableName); + tstrncpy(pStmt->dbName, ((SViewNode*)pView)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->viewName, ((SViewNode*)pView)->table.tableName, TSDB_VIEW_NAME_LEN); nodesDestroyNode(pView); return (SNode*)pStmt; _err: @@ -3636,8 +3653,8 @@ SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken pCxt->errCode = nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT, (SNode**)&pStmt); CHECK_MAKE_NODE(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->streamName, pStreamName); - strcpy(pStmt->targetDbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->targetTabName, ((SRealTableNode*)pRealTable)->table.tableName); + tstrncpy(pStmt->targetDbName, ((SRealTableNode*)pRealTable)->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pStmt->targetTabName, ((SRealTableNode*)pRealTable)->table.tableName, TSDB_TABLE_NAME_LEN); nodesDestroyNode(pRealTable); pStmt->ignoreExists = ignoreExists; pStmt->pOptions = (SStreamOptions*)pOptions; @@ -3891,9 +3908,9 @@ SNode* createInsertStmt(SAstCreateContext* pCxt, SNode* pTable, SNodeList* pCols pStmt->pCols = pCols; pStmt->pQuery = pQuery; if (QUERY_NODE_SELECT_STMT == nodeType(pQuery)) { - strcpy(((SSelectStmt*)pQuery)->stmtName, ((STableNode*)pTable)->tableAlias); + tstrncpy(((SSelectStmt*)pQuery)->stmtName, ((STableNode*)pTable)->tableAlias, TSDB_TABLE_NAME_LEN); } else if (QUERY_NODE_SET_OPERATOR == nodeType(pQuery)) { - strcpy(((SSetOperator*)pQuery)->stmtName, ((STableNode*)pTable)->tableAlias); + tstrncpy(((SSetOperator*)pQuery)->stmtName, ((STableNode*)pTable)->tableAlias, TSDB_TABLE_NAME_LEN); } return (SNode*)pStmt; _err: diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 44706afcb5..e9debf47aa 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -481,8 +481,8 @@ static int32_t collectMetaKeyFromExplain(SCollectMetaKeyCxt* pCxt, SExplainStmt* static int32_t collectMetaKeyFromDescribe(SCollectMetaKeyCxt* pCxt, SDescribeStmt* pStmt) { SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(name.dbname, pStmt->dbName); - strcpy(name.tname, pStmt->tableName); + tstrncpy(name.dbname, pStmt->dbName, TSDB_DB_NAME_LEN); + tstrncpy(name.tname, pStmt->tableName, TSDB_TABLE_NAME_LEN); int32_t code = catalogRemoveTableMeta(pCxt->pParseCxt->pCatalog, &name); #ifdef TD_ENTERPRISE if (TSDB_CODE_SUCCESS == code) { @@ -796,8 +796,8 @@ static int32_t collectMetaKeyFromShowCreateDatabase(SCollectMetaKeyCxt* pCxt, SS static int32_t collectMetaKeyFromShowCreateTable(SCollectMetaKeyCxt* pCxt, SShowCreateTableStmt* pStmt) { SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(name.dbname, pStmt->dbName); - strcpy(name.tname, pStmt->tableName); + tstrncpy(name.dbname, pStmt->dbName, TSDB_DB_NAME_LEN); + tstrncpy(name.tname, pStmt->tableName, TSDB_TABLE_NAME_LEN); int32_t code = catalogRemoveTableMeta(pCxt->pParseCxt->pCatalog, &name); if (TSDB_CODE_SUCCESS == code) { code = reserveTableCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); @@ -814,8 +814,8 @@ static int32_t collectMetaKeyFromShowCreateTable(SCollectMetaKeyCxt* pCxt, SShow static int32_t collectMetaKeyFromShowCreateView(SCollectMetaKeyCxt* pCxt, SShowCreateViewStmt* pStmt) { SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(name.dbname, pStmt->dbName); - strcpy(name.tname, pStmt->viewName); + tstrncpy(name.dbname, pStmt->dbName, TSDB_DB_NAME_LEN); + tstrncpy(name.tname, pStmt->viewName, TSDB_TABLE_NAME_LEN); char dbFName[TSDB_DB_FNAME_LEN]; (void)tNameGetFullDbName(&name, dbFName); int32_t code = catalogRemoveViewMeta(pCxt->pParseCxt->pCatalog, dbFName, 0, pStmt->viewName, 0); @@ -856,8 +856,8 @@ static int32_t collectMetaKeyFromInsert(SCollectMetaKeyCxt* pCxt, SInsertStmt* p static int32_t collectMetaKeyFromShowBlockDist(SCollectMetaKeyCxt* pCxt, SShowTableDistributedStmt* pStmt) { SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; - strcpy(name.dbname, pStmt->dbName); - strcpy(name.tname, pStmt->tableName); + tstrncpy(name.dbname, pStmt->dbName, TSDB_DB_NAME_LEN); + tstrncpy(name.tname, pStmt->tableName, TSDB_TABLE_NAME_LEN); int32_t code = catalogRemoveTableMeta(pCxt->pParseCxt->pCatalog, &name); if (TSDB_CODE_SUCCESS == code) { code = collectMetaKeyFromRealTableImpl(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ); diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index c8dfc87602..87ad0d62b7 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -134,8 +134,8 @@ EDealRes rewriteAuthTable(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; SAuthRewriteCxt* pCxt = (SAuthRewriteCxt*)pContext; - strcpy(pCol->tableName, pCxt->pTarget->tableName); - strcpy(pCol->tableAlias, pCxt->pTarget->tableAlias); + tstrncpy(pCol->tableName, pCxt->pTarget->tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableAlias, pCxt->pTarget->tableAlias, TSDB_TABLE_NAME_LEN); } return DEAL_RES_CONTINUE; diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 911cb2a915..99be9b07b0 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -178,14 +178,14 @@ static EDealRes doFindAndReplaceNode(SNode** pNode, void* pContext) { SCalcConstContext* pCxt = pContext; if (pCxt->replaceCxt.pTarget == *pNode) { char aliasName[TSDB_COL_NAME_LEN] = {0}; - strcpy(aliasName, ((SExprNode*)*pNode)->aliasName); + tstrncpy(aliasName, ((SExprNode*)*pNode)->aliasName, TSDB_COL_NAME_LEN); nodesDestroyNode(*pNode); *pNode = NULL; pCxt->code = nodesCloneNode(pCxt->replaceCxt.pNew, pNode); if (NULL == *pNode) { return DEAL_RES_ERROR; } - strcpy(((SExprNode*)*pNode)->aliasName, aliasName); + tstrncpy(((SExprNode*)*pNode)->aliasName, aliasName, TSDB_COL_NAME_LEN); pCxt->replaceCxt.replaced = true; return DEAL_RES_END; @@ -228,14 +228,14 @@ static int32_t calcConstProject(SCalcConstContext* pCxt, SNode* pProject, bool d SAssociationNode* pAssNode = taosArrayGet(pAssociation, i); SNode** pCol = pAssNode->pPlace; if (*pCol == pAssNode->pAssociationNode) { - strcpy(aliasName, ((SExprNode*)*pCol)->aliasName); + tstrncpy(aliasName, ((SExprNode*)*pCol)->aliasName, TSDB_COL_NAME_LEN); SArray* pOrigAss = NULL; TSWAP(((SExprNode*)*pCol)->pAssociation, pOrigAss); nodesDestroyNode(*pCol); *pCol = NULL; code = nodesCloneNode(*pNew, pCol); if (TSDB_CODE_SUCCESS == code) { - strcpy(((SExprNode*)*pCol)->aliasName, aliasName); + tstrncpy(((SExprNode*)*pCol)->aliasName, aliasName, TSDB_COL_NAME_LEN); TSWAP(pOrigAss, ((SExprNode*)*pCol)->pAssociation); } taosArrayDestroy(pOrigAss); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index b5cdf1e4ee..22d1f7edda 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -98,7 +98,7 @@ end: * @return int32_t */ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName, - SMsgBuf* msg) { + SMsgBuf* msg, void* charsetCxt) { SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); if (!pTagArray) { return terrno; @@ -142,7 +142,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem code = terrno; goto end; } - if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), kv->length * TSDB_NCHAR_SIZE, &output)) { + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), kv->length * TSDB_NCHAR_SIZE, &output, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { taosMemoryFree(p); code = generateSyntaxErrMsg(msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name); @@ -221,7 +221,7 @@ int32_t smlBuildRow(STableDataCxt* pTableCxt) { return TSDB_CODE_SUCCESS; } -int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32_t index) { +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32_t index, void* charsetCxt) { int ret = TSDB_CODE_SUCCESS; SSchema* pColSchema = schema + index; SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); @@ -256,7 +256,7 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 ret = terrno; goto end; } - if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, size, &len)) { + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, size, &len, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { taosMemoryFree(pUcs4); ret = TSDB_CODE_PAR_VALUE_TOO_LONG; @@ -291,7 +291,7 @@ end: int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, - char* msgBuf, int32_t msgBufLen) { + char* msgBuf, int32_t msgBufLen, void* charsetCxt) { SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SSchema* pTagsSchema = getTableTagSchema(pTableMeta); @@ -313,7 +313,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc STag* pTag = NULL; - ret = smlBuildTagRow(tags, &bindTags, pTagsSchema, &pTag, &tagName, &pBuf); + ret = smlBuildTagRow(tags, &bindTags, pTagsSchema, &pTag, &tagName, &pBuf, charsetCxt); if (ret != TSDB_CODE_SUCCESS) { goto end; } @@ -411,7 +411,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc ret = terrno; goto end; } - if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) { + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { uError("sml bind taosMbsToUcs4 error, kv length:%d, bytes:%d, kv->value:%s", (int)kv->length, pColSchema->bytes, kv->value); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 65ee0de80e..c5e9ad4ae4 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -248,13 +248,13 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, E } static int32_t parseTimestampOrInterval(const char** end, SToken* pToken, int16_t timePrec, int64_t* ts, int64_t* interval, - SMsgBuf* pMsgBuf, bool* isTs) { + SMsgBuf* pMsgBuf, bool* isTs, timezone_t tz) { if (pToken->type == TK_NOW) { *isTs = true; *ts = taosGetTimestamp(timePrec); } else if (pToken->type == TK_TODAY) { *isTs = true; - *ts = taosGetTimestampToday(timePrec); + *ts = taosGetTimestampToday(timePrec, tz); } else if (pToken->type == TK_NK_INTEGER) { *isTs = true; if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, ts)) { @@ -268,7 +268,7 @@ static int32_t parseTimestampOrInterval(const char** end, SToken* pToken, int16_ } } else { // parse the RFC-3339/ISO-8601 timestamp format string *isTs = true; - if (taosParseTime(pToken->z, ts, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { + if (taosParseTime(pToken->z, ts, pToken->n, timePrec, tz) != TSDB_CODE_SUCCESS) { if ((pToken->n == 0) || (pToken->type != TK_NK_STRING && pToken->type != TK_NK_HEX && pToken->type != TK_NK_BIN)) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); @@ -278,7 +278,7 @@ static int32_t parseTimestampOrInterval(const char** end, SToken* pToken, int16_ *ts = taosGetTimestamp(timePrec); } else if (IS_TODAY_STR(pToken->z, pToken->n)) { *isTs = true; - *ts = taosGetTimestampToday(timePrec); + *ts = taosGetTimestampToday(timePrec, tz); } else if (TSDB_CODE_SUCCESS == toIntegerPure(pToken->z, pToken->n, 10, ts)) { *isTs = true; } else { @@ -290,7 +290,7 @@ static int32_t parseTimestampOrInterval(const char** end, SToken* pToken, int16_ return TSDB_CODE_SUCCESS; } -static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) { +static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf, timezone_t tz) { int32_t index = 0, i = 0; int64_t interval = 0, tempInterval = 0; int64_t ts = 0, tempTs = 0; @@ -298,7 +298,7 @@ static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t const char* pTokenEnd = *end; if (TSDB_CODE_SUCCESS != - parseTimestampOrInterval(&pTokenEnd, pToken, timePrec, &ts, &interval, pMsgBuf, &firstIsTS)) { + parseTimestampOrInterval(&pTokenEnd, pToken, timePrec, &ts, &interval, pMsgBuf, &firstIsTS, tz)) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); } @@ -372,7 +372,7 @@ static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t } if (TSDB_CODE_SUCCESS != - parseTimestampOrInterval(&pTokenEnd, &valueToken, timePrec, &tempTs, &tempInterval, pMsgBuf, &secondIsTs)) { + parseTimestampOrInterval(&pTokenEnd, &valueToken, timePrec, &tempTs, &tempInterval, pMsgBuf, &secondIsTs, tz)) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); } @@ -478,7 +478,7 @@ static int32_t parseVarbinary(SToken* pToken, uint8_t** pData, uint32_t* nData, } static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val, - SMsgBuf* pMsgBuf) { + SMsgBuf* pMsgBuf, timezone_t tz, void *charsetCxt) { int64_t iv; uint64_t uv; char* endptr = NULL; @@ -686,13 +686,13 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, if (p == NULL) { return terrno; } - if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), realLen, &output)) { + if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), realLen, &output, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { taosMemoryFree(p); return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } char buf[512] = {0}; - snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(terrno)); + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s %d %d", strerror(terrno), errno, EILSEQ); taosMemoryFree(p); return buildSyntaxErrMsg(pMsgBuf, buf, pToken->z); } @@ -701,7 +701,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, break; } case TSDB_DATA_TYPE_TIMESTAMP: { - if (parseTime(end, pToken, timePrec, &iv, pMsgBuf) != TSDB_CODE_SUCCESS) { + if (parseTime(end, pToken, timePrec, &iv, pMsgBuf, tz) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z); } @@ -733,7 +733,7 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm } int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SSchema* pTagSchema, SToken* pToken, - SArray* pTagName, SArray* pTagVals, STag** pTag) { + SArray* pTagName, SArray* pTagVals, STag** pTag, timezone_t tz, void *charsetCxt) { bool isNull = isNullValue(pTagSchema->type, pToken); if (!isNull && pTagName) { if (NULL == taosArrayPush(pTagName, pTagSchema->name)) { @@ -749,14 +749,14 @@ int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SS if (isNull) { return tTagNew(pTagVals, 1, true, pTag); } else { - return parseJsontoTagData(pToken->z, pTagVals, pTag, pMsgBuf); + return parseJsontoTagData(pToken->z, pTagVals, pTag, pMsgBuf, charsetCxt); } } if (isNull) return 0; STagVal val = {0}; - int32_t code = parseTagToken(pSql, pToken, pTagSchema, precision, &val, pMsgBuf); + int32_t code = parseTagToken(pSql, pToken, pTagSchema, precision, &val, pMsgBuf, tz, charsetCxt); if (TSDB_CODE_SUCCESS == code) { if (NULL == taosArrayPush(pTagVals, &val)) { code = terrno; @@ -979,7 +979,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt code = buildSyntaxErrMsg(&pCxt->msg, "not expected tags values ", token.z); } if (TSDB_CODE_SUCCESS == code) { - code = parseTagValue(&pCxt->msg, &pStmt->pSql, precision, pTagSchema, &token, pTagName, pTagVals, &pTag); + code = parseTagValue(&pCxt->msg, &pStmt->pSql, precision, pTagSchema, &token, pTagName, pTagVals, &pTag, pCxt->pComCxt->timezone, pCxt->pComCxt->charsetCxt); } } @@ -1387,7 +1387,16 @@ static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS } char tbFName[TSDB_TABLE_FNAME_LEN]; - int32_t code = tNameExtractFullName(&pStmt->targetTableName, tbFName); + int32_t code = 0; + if (pCxt->preCtbname) { + tstrncpy(pStmt->targetTableName.tname, pStmt->usingTableName.tname, sizeof(pStmt->targetTableName.tname)); + tstrncpy(pStmt->targetTableName.dbname, pStmt->usingTableName.dbname, sizeof(pStmt->targetTableName.dbname)); + pStmt->targetTableName.type = TSDB_SUPER_TABLE; + pStmt->pTableMeta->tableType = TSDB_SUPER_TABLE; + } + + code = tNameExtractFullName(&pStmt->targetTableName, tbFName); + if (TSDB_CODE_SUCCESS != code) { return code; } @@ -1621,7 +1630,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, if (NULL == pUcs4) { return terrno; } - if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)pUcs4, realLen, &len)) { + if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)pUcs4, realLen, &len, pCxt->pComCxt->charsetCxt)) { taosMemoryFree(pUcs4); if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); @@ -1676,7 +1685,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_TIMESTAMP: { - if (parseTime(pSql, pToken, timePrec, &pVal->value.val, &pCxt->msg) != TSDB_CODE_SUCCESS) { + if (parseTime(pSql, pToken, timePrec, &pVal->value.val, &pCxt->msg, pCxt->pComCxt->timezone) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z); } break; @@ -1789,7 +1798,7 @@ static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModif if (code == TSDB_CODE_SUCCESS) { code = parseTagValue(&pCxt->msg, NULL, precision, pTagSchema, pTagToken, pStbRowsCxt->aTagNames, - pStbRowsCxt->aTagVals, &pStbRowsCxt->pTag); + pStbRowsCxt->aTagVals, &pStbRowsCxt->pTag, pCxt->pComCxt->timezone, pCxt->pComCxt->charsetCxt); } } if (code == TSDB_CODE_SUCCESS && !pStbRowsCxt->isJsonTag) { @@ -1812,8 +1821,10 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* SArray* pTagVals = pStbRowsCxt->aTagVals; bool canParseTagsAfter = !pStbRowsCxt->pTagCond && !pStbRowsCxt->hasTimestampTag; int32_t numOfCols = getNumOfColumns(pStbRowsCxt->pStbMeta); + int32_t numOfTags = getNumOfTags(pStbRowsCxt->pStbMeta); int32_t tbnameIdx = getTbnameSchemaIndex(pStbRowsCxt->pStbMeta); uint8_t precision = getTableInfo(pStbRowsCxt->pStbMeta).precision; + int idx = 0; for (int i = 0; i < pCols->numOfBound && (code) == TSDB_CODE_SUCCESS; ++i) { const char* pTmpSql = *ppSql; bool ignoreComma = false; @@ -1831,13 +1842,37 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* if (TK_NK_QUESTION == pToken->type) { pCxt->isStmtBind = true; + pStmt->usingTableProcessing = true; if (pCols->pColIndex[i] == tbnameIdx) { - pCxt->preCtbname = false; - *bFoundTbName = true; - } - if (NULL == pCxt->pComCxt->pStmtCb) { - code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", pToken->z); - break; + char* tbName = NULL; + if ((*pCxt->pComCxt->pStmtCb->getTbNameFn)(pCxt->pComCxt->pStmtCb->pStmt, &tbName) == TSDB_CODE_SUCCESS) { + tstrncpy(pStbRowsCxt->ctbName.tname, tbName, sizeof(pStbRowsCxt->ctbName.tname)); + tstrncpy(pStmt->usingTableName.tname, pStmt->targetTableName.tname, sizeof(pStmt->usingTableName.tname)); + tstrncpy(pStmt->targetTableName.tname, tbName, sizeof(pStmt->targetTableName.tname)); + + tstrncpy(pStmt->usingTableName.dbname, pStmt->targetTableName.dbname, sizeof(pStmt->usingTableName.dbname)); + pStmt->usingTableName.type = 1; + + *bFoundTbName = true; + } + } else if (pCols->pColIndex[i] < numOfCols) { + // bind column + } else if (pCols->pColIndex[i] < tbnameIdx) { + if (pCxt->tags.pColIndex == NULL) { + pCxt->tags.pColIndex = taosMemoryCalloc(numOfTags, sizeof(int16_t)); + if (NULL == pCxt->tags.pColIndex) { + return terrno; + } + } + if (!(idx < numOfTags)) { + return buildInvalidOperationMsg(&pCxt->msg, "not expected numOfTags"); + } + pCxt->tags.pColIndex[idx++] = pCols->pColIndex[i] - numOfCols; + pCxt->tags.mixTagsCols = true; + pCxt->tags.numOfBound++; + pCxt->tags.numOfCols++; + } else { + return buildInvalidOperationMsg(&pCxt->msg, "not expected numOfBound"); } } else { if (pCols->pColIndex[i] < numOfCols) { @@ -1860,7 +1895,7 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* } if (code == TSDB_CODE_SUCCESS) { code = parseTagValue(&pCxt->msg, ppSql, precision, (SSchema*)pTagSchema, pToken, pTagNames, pTagVals, - &pStbRowsCxt->pTag); + &pStbRowsCxt->pTag, pCxt->pComCxt->timezone, pCxt->pComCxt->charsetCxt); } } } else if (pCols->pColIndex[i] == tbnameIdx) { @@ -1882,6 +1917,7 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* } } } + return code; } @@ -1901,13 +1937,17 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS code = doGetStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pToken, pCols, pSchemas, tagTokens, tagSchemas, &numOfTagTokens, &bFoundTbName); - if (code == TSDB_CODE_SUCCESS && !bFoundTbName) { - code = buildSyntaxErrMsg(&pCxt->msg, "tbname value expected", pOrigSql); + if (code != TSDB_CODE_SUCCESS) { + return code; } - if (code == TSDB_CODE_SUCCESS && pStbRowsCxt->ctbName.tname[0] == '\0') { - *pGotRow = true; - return TSDB_CODE_TSC_STMT_TBNAME_ERROR; + if (!bFoundTbName) { + if (!pCxt->isStmtBind) { + code = buildSyntaxErrMsg(&pCxt->msg, "tbname value expected", pOrigSql); + } else { + *pGotRow = true; + return TSDB_CODE_TSC_STMT_TBNAME_ERROR; + } } bool ctbFirst = true; @@ -2050,14 +2090,24 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt code = processCtbAutoCreationAndCtbMeta(pCxt, pStmt, pStbRowsCxt); } if (code == TSDB_CODE_SUCCESS) { - code = - insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), - pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, false, true); + if (pCxt->isStmtBind) { + char ctbFName[TSDB_TABLE_FNAME_LEN]; + code = tNameExtractFullName(&pStbRowsCxt->ctbName, ctbFName); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = insGetTableDataCxt(pStmt->pTableBlockHashObj, ctbFName, strlen(ctbFName), pStbRowsCxt->pCtbMeta, + &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, true, true); + } else { + code = + insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), + pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, false, true); + } } if (code == TSDB_CODE_SUCCESS) { code = initTableColSubmitData(*ppTableDataCxt); } - if (code == TSDB_CODE_SUCCESS) { + if (code == TSDB_CODE_SUCCESS && !pCxt->isStmtBind) { SRow** pRow = taosArrayReserve((*ppTableDataCxt)->pData->aRowP, 1); code = tRowBuild(pStbRowsCxt->aColVals, (*ppTableDataCxt)->pSchema, pRow); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 22b84e09dd..b3c89a6b1c 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -123,7 +123,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash } int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, - TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { + TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; int32_t code = TSDB_CODE_SUCCESS; @@ -183,7 +183,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch goto end; } memcpy(tmp, bind[c].buffer, colLen); - code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf); + code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf, charsetCxt); taosMemoryFree(tmp); if (code != TSDB_CODE_SUCCESS) { goto end; @@ -208,7 +208,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch code = terrno; goto end; } - if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output)) { + if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { taosMemoryFree(p); code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name); @@ -266,7 +266,7 @@ end: return code; } -int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* src, TAOS_MULTI_BIND* dst) { +int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* src, TAOS_MULTI_BIND* dst, void* charsetCxt) { int32_t output = 0; int32_t newBuflen = (pSchema->bytes - VARSTR_HEADER_SIZE) * src->num; if (dst->buffer_length < newBuflen) { @@ -292,7 +292,7 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* } if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i], - (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) { + (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } @@ -312,7 +312,7 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* } int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, - STSchema** pTSchema, SBindInfo* pBindInfos) { + STSchema** pTSchema, SBindInfo* pBindInfos, void* charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; @@ -349,7 +349,7 @@ int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind } if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { - code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind); + code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt); if (code) { goto _return; } @@ -380,7 +380,7 @@ _return: return code; } -int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { +int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; @@ -406,7 +406,7 @@ int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, c } if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { - code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind); + code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt); if (code) { goto _return; } @@ -434,7 +434,7 @@ _return: } int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, - int32_t colIdx, int32_t rowNum) { + int32_t colIdx, int32_t rowNum, void* charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; @@ -459,7 +459,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bi } if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { - code = convertStmtNcharCol(&pBuf, pColSchema, bind, &ncharBind); + code = convertStmtNcharCol(&pBuf, pColSchema, bind, &ncharBind, charsetCxt); if (code) { goto _return; } @@ -483,7 +483,7 @@ _return: } int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, - TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen) { + TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; int32_t code = TSDB_CODE_SUCCESS; @@ -543,7 +543,7 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c goto end; } memcpy(tmp, bind[c].buffer, colLen); - code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf); + code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf, charsetCxt); taosMemoryFree(tmp); if (code != TSDB_CODE_SUCCESS) { goto end; @@ -568,7 +568,7 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c code = terrno; goto end; } - if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output)) { + if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { taosMemoryFree(p); code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name); @@ -626,7 +626,7 @@ end: return code; } -static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst) { +static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst, void *charsetCxt) { int32_t output = 0; const int32_t max_buf_len = pSchema->bytes - VARSTR_HEADER_SIZE; @@ -648,7 +648,7 @@ static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_ continue; } - if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output)) { + if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } @@ -670,7 +670,7 @@ static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_ } int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, - STSchema** pTSchema, SBindInfo2* pBindInfos) { + STSchema** pTSchema, SBindInfo2* pBindInfos, void *charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; @@ -720,7 +720,7 @@ int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bin } if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { - code = convertStmtStbNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind); + code = convertStmtStbNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt); if (code) { goto _return; } @@ -755,7 +755,7 @@ _return: return code; } -static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst) { +static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst, void *charsetCxt) { int32_t output = 0; const int32_t max_buf_len = pSchema->bytes - VARSTR_HEADER_SIZE; @@ -785,7 +785,7 @@ static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STM /*if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i], (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) {*/ - if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output)) { + if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } @@ -806,7 +806,7 @@ static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STM return TSDB_CODE_SUCCESS; } -int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen) { +int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void *charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; @@ -836,7 +836,7 @@ int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, } if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { - code = convertStmtNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind); + code = convertStmtNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt); if (code) { goto _return; } @@ -864,7 +864,7 @@ _return: } int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, - int32_t colIdx, int32_t rowNum) { + int32_t colIdx, int32_t rowNum, void *charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; @@ -889,7 +889,7 @@ int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* b } if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { - code = convertStmtNcharCol2(&pBuf, pColSchema, bind, &ncharBind); + code = convertStmtNcharCol2(&pBuf, pColSchema, bind, &ncharBind, charsetCxt); if (code) { goto _return; } @@ -927,7 +927,7 @@ int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSc for (int32_t i = 0; i < numOfBound; ++i) { schema = &pSchema[boundColumns[i]]; - strcpy((*fields)[i].name, schema->name); + tstrncpy((*fields)[i].name, schema->name, 65); (*fields)[i].type = schema->type; (*fields)[i].bytes = schema->bytes; } @@ -941,8 +941,8 @@ int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSc int32_t buildStbBoundFields(SBoundColInfo boundColsInfo, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_STB** fields, STableMeta* pMeta, void* boundTags, bool preCtbname) { SBoundColInfo* tags = (SBoundColInfo*)boundTags; - int32_t numOfBound = boundColsInfo.numOfBound + tags->numOfBound + (preCtbname ? 1 : 0); - int32_t idx = 0; + int32_t numOfBound = boundColsInfo.numOfBound + (tags->mixTagsCols ? 0 : tags->numOfBound) + (preCtbname ? 1 : 0); + int32_t idx = 0; if (fields != NULL) { *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_STB)); if (NULL == *fields) { @@ -957,13 +957,13 @@ int32_t buildStbBoundFields(SBoundColInfo boundColsInfo, SSchema* pSchema, int32 idx++; } - if (tags->numOfBound > 0) { - SSchema* pSchema = getTableTagSchema(pMeta); + if (tags->numOfBound > 0 && !tags->mixTagsCols) { + SSchema* tagSchema = getTableTagSchema(pMeta); for (int32_t i = 0; i < tags->numOfBound; ++i) { (*fields)[idx].field_type = TAOS_FIELD_TAG; - SSchema* schema = &pSchema[tags->pColIndex[i]]; + SSchema* schema = &tagSchema[tags->pColIndex[i]]; tstrncpy((*fields)[idx].name, schema->name, sizeof((*fields)[i].name)); (*fields)[idx].type = schema->type; (*fields)[idx].bytes = schema->bytes; @@ -1017,11 +1017,11 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA if (NULL == tags) { return TSDB_CODE_APP_ERROR; } - /* + if (pDataBlock->pMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pMeta->tableType != TSDB_CHILD_TABLE) { return TSDB_CODE_TSC_STMT_API_ERROR; } - */ + SSchema* pSchema = getTableTagSchema(pDataBlock->pMeta); if (tags->numOfBound <= 0) { *fieldNum = 0; diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index bcb560ab5e..502dbb57dd 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -191,6 +191,7 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { pInfo->numOfCols = numOfBound; pInfo->numOfBound = numOfBound; pInfo->hasBoundCols = false; + pInfo->mixTagsCols = false; pInfo->pColIndex = taosMemoryCalloc(numOfBound, sizeof(int16_t)); if (NULL == pInfo->pColIndex) { return terrno; @@ -204,6 +205,7 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { void insResetBoundColsInfo(SBoundColInfo* pInfo) { pInfo->numOfBound = pInfo->numOfCols; pInfo->hasBoundCols = false; + pInfo->mixTagsCols = false; for (int32_t i = 0; i < pInfo->numOfCols; ++i) { pInfo->pColIndex[i] = i; } @@ -739,7 +741,7 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks, bool STableDataCxt* pTableCxt = *(STableDataCxt**)p; if (colFormat) { SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, 0); - if (pCol->nVal <= 0) { + if (pCol && pCol->nVal <= 0) { p = taosHashIterate(pTableHash, p); continue; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0275a6eac7..3205d4e99b 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1312,15 +1312,15 @@ static bool hasPkInTable(const STableMeta* pTableMeta) { static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* pColSchema, int32_t tagFlag, SColumnNode* pCol) { - strcpy(pCol->dbName, pTable->table.dbName); - strcpy(pCol->tableAlias, pTable->table.tableAlias); - strcpy(pCol->tableName, pTable->table.tableName); - strcpy(pCol->colName, pColSchema->name); + tstrncpy(pCol->dbName, pTable->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pCol->tableAlias, pTable->table.tableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableName, pTable->table.tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->colName, pColSchema->name, TSDB_COL_NAME_LEN); if ('\0' == pCol->node.aliasName[0]) { - strcpy(pCol->node.aliasName, pColSchema->name); + tstrncpy(pCol->node.aliasName, pColSchema->name, TSDB_COL_NAME_LEN); } if ('\0' == pCol->node.userAlias[0]) { - strcpy(pCol->node.userAlias, pColSchema->name); + tstrncpy(pCol->node.userAlias, pColSchema->name, TSDB_COL_NAME_LEN); } pCol->tableId = pTable->pMeta->uid; pCol->tableType = pTable->pMeta->tableType; @@ -1351,7 +1351,7 @@ static int32_t setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SCo return terrno; } - strcpy(pCol->tableAlias, pTable->table.tableAlias); + tstrncpy(pCol->tableAlias, pTable->table.tableAlias, TSDB_TABLE_NAME_LEN); pCol->isPrimTs = isPrimaryKeyImpl((SNode*)pExpr); pCol->colId = pCol->isPrimTs ? PRIMARYKEY_TIMESTAMP_COL_ID : 0; if (QUERY_NODE_COLUMN == nodeType(pExpr)) { @@ -1359,12 +1359,12 @@ static int32_t setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SCo // strcpy(pCol->dbName, ((SColumnNode*)pExpr)->dbName); // strcpy(pCol->tableName, ((SColumnNode*)pExpr)->tableName); } - strcpy(pCol->colName, pExpr->aliasName); + tstrncpy(pCol->colName, pExpr->aliasName, TSDB_COL_NAME_LEN); if ('\0' == pCol->node.aliasName[0]) { - strcpy(pCol->node.aliasName, pCol->colName); + tstrncpy(pCol->node.aliasName, pExpr->aliasName, TSDB_COL_NAME_LEN); } if ('\0' == pCol->node.userAlias[0]) { - strcpy(pCol->node.userAlias, pExpr->userAlias); + tstrncpy(pCol->node.userAlias, pExpr->aliasName, TSDB_COL_NAME_LEN); } pCol->node.resType = pExpr->resType; return TSDB_CODE_SUCCESS; @@ -1652,7 +1652,7 @@ static void biMakeAliasNameInMD5(char* pExprStr, int32_t len, char* pAlias) { tMD5Final(&ctx); char* p = pAlias; for (uint8_t i = 0; i < tListLen(ctx.digest); ++i) { - sprintf(p, "%02x", ctx.digest[i]); + snprintf(p, len + 1 - 2 * i, "%02x", ctx.digest[i]); p += 2; } } @@ -1691,7 +1691,7 @@ static int32_t biMakeTbnameProjectAstNode(char* funcName, char* tableAlias, SNod if (TSDB_CODE_SUCCESS == code) { snprintf(tbNameFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), (tableAlias) ? "%s.tbname" : "%stbname", (tableAlias) ? tableAlias : ""); - strncpy(tbNameFunc->node.aliasName, tbNameFunc->functionName, TSDB_COL_NAME_LEN); + tstrncpy(tbNameFunc->node.aliasName, tbNameFunc->functionName, TSDB_COL_NAME_LEN); if (funcName == NULL) { *pOutNode = (SNode*)tbNameFunc; return code; @@ -1709,7 +1709,7 @@ static int32_t biMakeTbnameProjectAstNode(char* funcName, char* tableAlias, SNod if (tsKeepColumnName) { snprintf(multiResFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), (tableAlias) ? "%s.tbname" : "%stbname", (tableAlias) ? tableAlias : ""); - strcpy(multiResFunc->node.aliasName, tbNameFunc->functionName); + tstrncpy(multiResFunc->node.aliasName, tbNameFunc->functionName, TSDB_COL_NAME_LEN); } else { snprintf(multiResFunc->node.userAlias, sizeof(multiResFunc->node.userAlias), tableAlias ? "%s(%s.tbname)" : "%s(%stbname)", funcName, tableAlias ? tableAlias : ""); @@ -1836,8 +1836,8 @@ int32_t biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRe (SNode**)&tbnameFuncNode); if (TSDB_CODE_SUCCESS != code) return code; tbnameFuncNode->node.resType = pCol->node.resType; - strcpy(tbnameFuncNode->node.aliasName, pCol->node.aliasName); - strcpy(tbnameFuncNode->node.userAlias, pCol->node.userAlias); + tstrncpy(tbnameFuncNode->node.aliasName, pCol->node.aliasName, TSDB_COL_NAME_LEN); + tstrncpy(tbnameFuncNode->node.userAlias, pCol->node.userAlias, TSDB_COL_NAME_LEN); nodesDestroyNode(*ppNode); *ppNode = (SNode*)tbnameFuncNode; @@ -1974,7 +1974,7 @@ static int32_t parseTimeFromValueNode(STranslateContext* pCxt, SValueNode* pVal) return TSDB_CODE_SUCCESS; } else if (IS_VAR_DATA_TYPE(pVal->node.resType.type) || TSDB_DATA_TYPE_TIMESTAMP == pVal->node.resType.type) { if (TSDB_CODE_SUCCESS == taosParseTime(pVal->literal, &pVal->datum.i, pVal->node.resType.bytes, - pVal->node.resType.precision, tsDaylight)) { + pVal->node.resType.precision, pVal->tz)) { return TSDB_CODE_SUCCESS; } char* pEnd = NULL; @@ -2176,7 +2176,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, int32_t len = 0; if (!taosMbsToUcs4(pVal->literal, strlen(pVal->literal), (TdUcs4*)varDataVal(pVal->datum.p), - targetDt.bytes - VARSTR_HEADER_SIZE, &len)) { + targetDt.bytes - VARSTR_HEADER_SIZE, &len, pCxt->pParseCxt->charsetCxt)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } varDataSetLen(pVal->datum.p, len); @@ -2843,7 +2843,8 @@ static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFu } } if (tsKeepColumnName && 1 == LIST_LENGTH(pFunc->pParameterList) && !pFunc->node.asAlias && !pFunc->node.asParam) { - strcpy(pFunc->node.userAlias, ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->userAlias); + tstrncpy(pFunc->node.userAlias, ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->userAlias, + TSDB_COL_NAME_LEN); } return TSDB_CODE_SUCCESS; } @@ -2918,8 +2919,8 @@ static int32_t rewriteFuncToValue(STranslateContext* pCxt, char** pLiteral, SNod if (TSDB_CODE_SUCCESS != code) { return code; } - strcpy(pVal->node.aliasName, ((SExprNode*)*pNode)->aliasName); - strcpy(pVal->node.userAlias, ((SExprNode*)*pNode)->userAlias); + tstrncpy(pVal->node.aliasName, ((SExprNode*)*pNode)->aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pVal->node.userAlias, ((SExprNode*)*pNode)->userAlias, TSDB_COL_NAME_LEN); pVal->node.resType = ((SExprNode*)*pNode)->resType; if (NULL == pLiteral || NULL == *pLiteral) { pVal->isNull = true; @@ -3034,10 +3035,10 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode if (0 != LIST_LENGTH(pFunc->pParameterList)) { SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); pCol->node.resType = pOldExpr->resType; - strcpy(pCol->tableAlias, pVal->literal); - strcpy(pCol->colName, pFunc->functionName); - strcpy(pCol->node.aliasName, pCol->colName); - strcpy(pCol->node.userAlias, pCol->colName); + tstrncpy(pCol->node.aliasName, pVal->literal, TSDB_COL_NAME_LEN); + tstrncpy(pCol->colName, pFunc->functionName, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.aliasName, pCol->colName, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.userAlias, pCol->colName, TSDB_COL_NAME_LEN); nodesDestroyNode(*ppNode); *ppNode = (SNode*)pCol; @@ -3045,9 +3046,9 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode } } pCol->node.resType = pOldExpr->resType; - strcpy(pCol->node.aliasName, pOldExpr->aliasName); - strcpy(pCol->node.userAlias, pOldExpr->userAlias); - strcpy(pCol->colName, pOldExpr->aliasName); + tstrncpy(pCol->node.aliasName, pOldExpr->aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.userAlias, pOldExpr->userAlias, TSDB_COL_NAME_LEN); + tstrncpy(pCol->colName, pOldExpr->aliasName, TSDB_COL_NAME_LEN); nodesDestroyNode(*ppNode); *ppNode = (SNode*)pCol; @@ -3306,7 +3307,7 @@ static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType d if (TSDB_CODE_SUCCESS != code) { return code; } - strcpy(pFunc->functionName, "cast"); + tstrncpy(pFunc->functionName, "cast", TSDB_FUNC_NAME_LEN); pFunc->node.resType = dt; if (TSDB_CODE_SUCCESS != nodesListMakeAppend(&pFunc->pParameterList, pExpr)) { nodesDestroyNode((SNode*)pFunc); @@ -3522,9 +3523,9 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode pCxt->errCode = code; return DEAL_RES_ERROR; } - strcpy(pFunc->functionName, "_select_value"); - strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName); - strcpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias); + tstrncpy(pFunc->functionName, "_select_value", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias, TSDB_COL_NAME_LEN); pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); if (TSDB_CODE_SUCCESS == pCxt->errCode) { pCxt->errCode = getFuncInfo(pCxt, pFunc); @@ -3546,9 +3547,9 @@ static EDealRes rewriteExprToGroupKeyFunc(STranslateContext* pCxt, SNode** pNode return DEAL_RES_ERROR; } - strcpy(pFunc->functionName, "_group_key"); - strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName); - strcpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias); + tstrncpy(pFunc->functionName, "_group_key", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias, TSDB_COL_NAME_LEN); pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); if (TSDB_CODE_SUCCESS == pCxt->errCode) { *pNode = (SNode*)pFunc; @@ -3565,9 +3566,9 @@ static EDealRes rewriteExprToSelectTagFunc(STranslateContext* pCxt, SNode** pNod return DEAL_RES_ERROR; } - strcpy(pFunc->functionName, "_group_const_value"); - strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName); - strcpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias); + tstrncpy(pFunc->functionName, "_group_const_value", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias, TSDB_COL_NAME_LEN); pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); if (TSDB_CODE_SUCCESS == pCxt->errCode) { *pNode = (SNode*)pFunc; @@ -4345,7 +4346,7 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo int32_t len = tsnprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, pRealTable->table.tableName); len = taosCreateMD5Hash(buf, len); - strncpy(tsmaTargetTbName.tname, buf, MD5_OUTPUT_LEN); + tstrncpy(tsmaTargetTbName.tname, buf, TSDB_TABLE_NAME_LEN); code = collectUseTable(&tsmaTargetTbName, pCxt->pTargetTables); if (TSDB_CODE_SUCCESS == code) code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &tsmaTargetTbName, &vgInfo, &exists); @@ -4389,7 +4390,7 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo } } if (code == TSDB_CODE_SUCCESS) { - sprintf(ctbInfo.tableName, "%s", tsmaTargetTbName.tname); + snprintf(ctbInfo.tableName, TSDB_TABLE_NAME_LEN, "%s", tsmaTargetTbName.tname); ctbInfo.uid = pTableMeta->uid; taosMemoryFree(pTableMeta); } else if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { @@ -4451,10 +4452,9 @@ static EDealRes doTranslateTbName(SNode** pNode, void* pContext) { return DEAL_RES_ERROR; } varDataSetLen(pVal->datum.p, tbLen); - strncpy(varDataVal(pVal->datum.p), pVal->literal, tbLen); - strcpy(pVal->node.userAlias, pFunc->node.userAlias); - strcpy(pVal->node.aliasName, pFunc->node.aliasName); - + tstrncpy(varDataVal(pVal->datum.p), pVal->literal, tbLen + 1); + tstrncpy(pVal->node.userAlias, pFunc->node.userAlias, TSDB_COL_NAME_LEN); + tstrncpy(pVal->node.aliasName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); nodesDestroyNode(*pNode); *pNode = (SNode*)pVal; } @@ -4519,9 +4519,9 @@ static int32_t addPrimJoinEqCond(SNode** pCond, SRealTableNode* leftTable, SReal pLeft->tableId = pLMeta->uid; pLeft->colId = pLMeta->schema[0].colId; pLeft->colType = COLUMN_TYPE_COLUMN; - strcpy(pLeft->tableName, leftTable->table.tableName); - strcpy(pLeft->tableAlias, leftTable->table.tableAlias); - strcpy(pLeft->colName, pLMeta->schema[0].name); + tstrncpy(pLeft->tableName, leftTable->table.tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pLeft->tableAlias, leftTable->table.tableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pLeft->colName, pLMeta->schema[0].name, TSDB_COL_NAME_LEN); pOp->pLeft = (SNode*)pLeft; @@ -4536,9 +4536,9 @@ static int32_t addPrimJoinEqCond(SNode** pCond, SRealTableNode* leftTable, SReal pRight->tableId = pRMeta->uid; pRight->colId = pRMeta->schema[0].colId; pRight->colType = COLUMN_TYPE_COLUMN; - strcpy(pRight->tableName, rightTable->table.tableName); - strcpy(pRight->tableAlias, rightTable->table.tableAlias); - strcpy(pRight->colName, pRMeta->schema[0].name); + tstrncpy(pRight->tableName, rightTable->table.tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pRight->tableAlias, rightTable->table.tableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pRight->colName, pRMeta->schema[0].name, TSDB_COL_NAME_LEN); pOp->pRight = (SNode*)pRight; @@ -5141,29 +5141,29 @@ static int32_t createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr, SNo pFunc->node.resType = pExpr->resType; pFunc->funcId = pSrcFunc->funcId; pFunc->funcType = pSrcFunc->funcType; - strcpy(pFunc->functionName, pSrcFunc->functionName); + tstrncpy(pFunc->functionName, pSrcFunc->functionName, TSDB_FUNC_NAME_LEN); char buf[TSDB_FUNC_NAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN + TSDB_NAME_DELIMITER_LEN + 3] = {0}; int32_t len = 0; if (QUERY_NODE_COLUMN == nodeType(pExpr)) { SColumnNode* pCol = (SColumnNode*)pExpr; if (tsKeepColumnName) { - strcpy(pFunc->node.userAlias, pCol->colName); - strcpy(pFunc->node.aliasName, pCol->colName); + tstrncpy(pFunc->node.userAlias, pCol->colName, TSDB_COL_NAME_LEN); + tstrncpy(pFunc->node.aliasName, pCol->colName, TSDB_COL_NAME_LEN); } else { len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s.%s)", pSrcFunc->functionName, pCol->tableAlias, pCol->colName); (void)taosHashBinary(buf, len); - strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); + tstrncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN); len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pCol->colName); // note: userAlias could be truncated here - strncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN - 1); + tstrncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN); } } else { len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pExpr->aliasName); (void)taosHashBinary(buf, len); - strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); + tstrncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN); len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pExpr->userAlias); // note: userAlias could be truncated here - strncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN - 1); + tstrncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN); } *ppNodeOut = (SNode*)pFunc; return code; @@ -5493,9 +5493,9 @@ static int32_t rewriteProjectAlias(SNodeList* pProjectionList) { FOREACH(pProject, pProjectionList) { SExprNode* pExpr = (SExprNode*)pProject; if ('\0' == pExpr->userAlias[0]) { - strcpy(pExpr->userAlias, pExpr->aliasName); + tstrncpy(pExpr->userAlias, pExpr->aliasName, TSDB_COL_NAME_LEN); } - sprintf(pExpr->aliasName, "#expr_%d", no++); + snprintf(pExpr->aliasName, TSDB_COL_NAME_LEN,"#expr_%d", no++); } return TSDB_CODE_SUCCESS; } @@ -5681,7 +5681,11 @@ static int32_t getTimeRange(SNode** pPrimaryKeyCond, STimeWindow* pTimeRange, bo int32_t code = scalarCalculateConstants(*pPrimaryKeyCond, &pNew); if (TSDB_CODE_SUCCESS == code) { *pPrimaryKeyCond = pNew; - code = filterGetTimeRange(*pPrimaryKeyCond, pTimeRange, pIsStrict); + if (nodeType(pNew) == QUERY_NODE_VALUE) { + *pTimeRange = TSWINDOW_INITIALIZER; + } else { + code = filterGetTimeRange(*pPrimaryKeyCond, pTimeRange, pIsStrict); + } } return code; } @@ -5947,6 +5951,7 @@ void tryCalcIntervalAutoOffset(SIntervalWindowNode *pInterval) { .slidingUnit = (pSliding != NULL) ? pSliding->unit : pInter->unit, .offset = pOffset->datum.i, .precision = precision, + .timezone = pInterval->timezone, .timeRange = pInterval->timeRange}; /** @@ -6236,7 +6241,7 @@ static int32_t createDefaultFillNode(STranslateContext* pCxt, SNode** pOutput) { return code; } pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - strcpy(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME); + tstrncpy(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME, TSDB_COL_NAME_LEN); pFill->pWStartTs = (SNode*)pCol; *pOutput = (SNode*)pFill; @@ -6844,7 +6849,7 @@ static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTa int32_t len = tsnprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name, pRealTable->table.tableName); len = taosCreateMD5Hash(buf, len); - strncpy(tsmaTargetTbName.tname, buf, MD5_OUTPUT_LEN); + tstrncpy(tsmaTargetTbName.tname, buf, TSDB_TABLE_NAME_LEN); STsmaTargetTbInfo ctbInfo = {0}; if (!pRealTable->tsmaTargetTbInfo) { pRealTable->tsmaTargetTbInfo = taosArrayInit(pRealTable->pTsmas->size, sizeof(STsmaTargetTbInfo)); @@ -6853,7 +6858,7 @@ static int32_t replaceToChildTableQuery(STranslateContext* pCxt, SEqCondTbNameTa break; } } - sprintf(ctbInfo.tableName, "%s", tsmaTargetTbName.tname); + snprintf(ctbInfo.tableName, TSDB_TABLE_NAME_LEN, "%s", tsmaTargetTbName.tname); ctbInfo.uid = pMeta->uid; if (NULL == taosArrayPush(pRealTable->tsmaTargetTbInfo, &ctbInfo)) { @@ -6918,7 +6923,7 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* code = terrno; break; } - sprintf(pNewTbName, "%s.%s_%s", pTsma->dbFName, pTsma->name, pTbName); + snprintf(pNewTbName, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN + 1, "%s.%s_%s", pTsma->dbFName, pTsma->name, pTbName); int32_t len = taosCreateMD5Hash(pNewTbName, strlen(pNewTbName)); } if (TSDB_CODE_SUCCESS == code) { @@ -7007,7 +7012,7 @@ static int32_t createPrimaryKeyColByTable(STranslateContext* pCxt, STableNode* p return code; } pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - strcpy(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME); + tstrncpy(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME, TSDB_COL_NAME_LEN); bool found = false; code = findAndSetColumn(pCxt, &pCol, pTable, &found, true); if (TSDB_CODE_SUCCESS != code || !found) { @@ -7106,7 +7111,7 @@ static int32_t createPkColByTable(STranslateContext* pCxt, SRealTableNode* pTabl return code; } pCol->colId = pTable->pMeta->schema[1].colId; - strcpy(pCol->colName, pTable->pMeta->schema[1].name); + tstrncpy(pCol->colName, pTable->pMeta->schema[1].name, TSDB_COL_NAME_LEN); bool found = false; code = findAndSetColumn(pCxt, &pCol, (STableNode*)pTable, &found, true); if (TSDB_CODE_SUCCESS != code || !found) { @@ -8716,7 +8721,7 @@ static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray, bool calB field.bytes = pCol->dataType.bytes; } - strcpy(field.name, pCol->colName); + tstrncpy(field.name, pCol->colName, TSDB_COL_NAME_LEN); if (pCol->pOptions) { setColEncode(&field.compress, columnEncodeVal(((SColumnOptions*)pCol->pOptions)->encode)); setColCompress(&field.compress, columnCompressVal(((SColumnOptions*)pCol->pOptions)->compress)); @@ -8754,7 +8759,7 @@ static int32_t tagDefNodeToField(SNodeList* pList, SArray** pArray, bool calByte } else { field.bytes = pCol->dataType.bytes; } - strcpy(field.name, pCol->colName); + tstrncpy(field.name, pCol->colName, TSDB_COL_NAME_LEN); if (pCol->sma) { field.flags |= COL_SMA_ON; } @@ -9159,7 +9164,7 @@ static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchem pSchema->type = pCol->dataType.type; pSchema->bytes = calcTypeBytes(pCol->dataType); pSchema->flags = flags; - strcpy(pSchema->name, pCol->colName); + tstrncpy(pSchema->name, pCol->colName, TSDB_COL_NAME_LEN); } typedef struct SSampleAstInfo { @@ -9196,8 +9201,8 @@ static int32_t addWstartToSampleProjects(SNodeList* pProjectionList) { if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "_wstart"); - strcpy(pFunc->node.userAlias, "_wstart"); + tstrncpy(pFunc->functionName, "_wstart", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.userAlias, "_wstart", TSDB_FUNC_NAME_LEN); return nodesListPushFront(pProjectionList, (SNode*)pFunc); } @@ -9207,8 +9212,8 @@ static int32_t addWendToSampleProjects(SNodeList* pProjectionList) { if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "_wend"); - strcpy(pFunc->node.userAlias, "_wend"); + tstrncpy(pFunc->functionName, "_wend", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.userAlias, "_wend", TSDB_FUNC_NAME_LEN); return nodesListAppend(pProjectionList, (SNode*)pFunc); } @@ -9218,8 +9223,8 @@ static int32_t addWdurationToSampleProjects(SNodeList* pProjectionList) { if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "_wduration"); - strcpy(pFunc->node.userAlias, "_wduration"); + tstrncpy(pFunc->functionName, "_wduration", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.userAlias, "_wduration", TSDB_FUNC_NAME_LEN); return nodesListAppend(pProjectionList, (SNode*)pFunc); } @@ -9239,7 +9244,7 @@ static int32_t buildProjectsForSampleAst(SSampleAstInfo* pInfo, SNodeList** pLis SNode* pProject = NULL; if (pProjectionTotalLen) *pProjectionTotalLen = 0; FOREACH(pProject, pProjectionList) { - sprintf(((SExprNode*)pProject)->aliasName, "#%p", pProject); + snprintf(((SExprNode*)pProject)->aliasName, TSDB_COL_NAME_LEN, "#%p", pProject); if (pProjectionTotalLen) *pProjectionTotalLen += ((SExprNode*)pProject)->resType.bytes; } *pList = pProjectionList; @@ -9272,7 +9277,7 @@ static int32_t buildIntervalForSampleAst(SSampleAstInfo* pInfo, SNode** pOutput) return code; } ((SColumnNode*)pInterval->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - strcpy(((SColumnNode*)pInterval->pCol)->colName, ROWTS_PSEUDO_COLUMN_NAME); + tstrncpy(((SColumnNode*)pInterval->pCol)->colName, ROWTS_PSEUDO_COLUMN_NAME, TSDB_COL_NAME_LEN); *pOutput = (SNode*)pInterval; return TSDB_CODE_SUCCESS; } @@ -9284,7 +9289,7 @@ static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, ch if (NULL == pSelect) { return code; } - sprintf(pSelect->stmtName, "%p", pSelect); + snprintf(pSelect->stmtName, TSDB_TABLE_NAME_LEN, "%p", pSelect); code = buildTableForSampleAst(pInfo, &pSelect->pFromTable); if (TSDB_CODE_SUCCESS == code) { @@ -9352,7 +9357,7 @@ static int32_t createColumnFromDef(SColumnDefNode* pDef, SNode** ppCol) { if (NULL == pCol) { return code; } - strcpy(pCol->colName, pDef->colName); + tstrncpy(pCol->colName, pDef->colName, TSDB_COL_NAME_LEN); *ppCol = (SNode*)pCol; return code; } @@ -9445,9 +9450,9 @@ static int32_t createTbnameFunction(SFunctionNode** ppFunc) { if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "tbname"); - strcpy(pFunc->node.aliasName, "tbname"); - strcpy(pFunc->node.userAlias, "tbname"); + tstrncpy(pFunc->functionName, "tbname", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, "tbname", TSDB_COL_NAME_LEN); + tstrncpy(pFunc->node.userAlias, "tbname", TSDB_COL_NAME_LEN); *ppFunc = pFunc; return code; } @@ -9644,7 +9649,7 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: { TAOS_FIELD field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)}; - strcpy(field.name, pStmt->colName); + tstrncpy(field.name, pStmt->colName, 65); if (NULL == taosArrayPush(pAlterReq->pFields, &field)) { return terrno; } @@ -9653,12 +9658,12 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { TAOS_FIELD oldField = {0}; - strcpy(oldField.name, pStmt->colName); + tstrncpy(oldField.name, pStmt->colName, 65); if (NULL == taosArrayPush(pAlterReq->pFields, &oldField)) { return terrno; } TAOS_FIELD newField = {0}; - strcpy(newField.name, pStmt->newColName); + tstrncpy(newField.name, pStmt->newColName, 65); if (NULL == taosArrayPush(pAlterReq->pFields, &newField)) { return terrno; } @@ -9666,7 +9671,7 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* } case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: { TAOS_FIELD field = {0}; - strcpy(field.name, pStmt->colName); + tstrncpy(field.name, pStmt->colName, 65); if (!checkColumnEncode(pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; if (!checkColumnCompress(pStmt->pColOptions->compress)) return TSDB_CODE_TSC_COMPRESS_PARAM_ERROR; if (!checkColumnLevel(pStmt->pColOptions->compressLevel)) return TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR; @@ -9689,7 +9694,7 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* if (!pAlterReq->pFields) return terrno; SFieldWithOptions field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)}; // TAOS_FIELD field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)}; - strcpy(field.name, pStmt->colName); + tstrncpy(field.name, pStmt->colName, TSDB_COL_NAME_LEN); if (pStmt->pColOptions != NULL) { if (!checkColumnEncodeOrSetDefault(pStmt->dataType.type, pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; @@ -9930,12 +9935,12 @@ static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pSt if ((code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "sysinfo", pStmt->sysinfo, 0, 1, false))) { return code; } - strcpy(createReq.user, pStmt->userName); + tstrncpy(createReq.user, pStmt->userName, TSDB_USER_LEN); createReq.createType = 0; createReq.superUser = 0; createReq.sysInfo = pStmt->sysinfo; createReq.enable = 1; - strcpy(createReq.pass, pStmt->password); + tstrncpy(createReq.pass, pStmt->password, TSDB_USET_PASSWORD_LEN); createReq.isImport = pStmt->isImport; createReq.createDb = pStmt->createDb; @@ -9974,7 +9979,7 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt if ((code = checkAlterUser(pCxt, pStmt))) { return code; } - strcpy(alterReq.user, pStmt->userName); + tstrncpy(alterReq.user, pStmt->userName, TSDB_USER_LEN); alterReq.alterType = pStmt->alterType; alterReq.superUser = 0; alterReq.enable = pStmt->enable; @@ -10000,7 +10005,7 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt static int32_t translateDropUser(STranslateContext* pCxt, SDropUserStmt* pStmt) { SDropUserReq dropReq = {0}; - strcpy(dropReq.user, pStmt->userName); + tstrncpy(dropReq.user, pStmt->userName, TSDB_USER_LEN); int32_t code = buildCmdMsg(pCxt, TDMT_MND_DROP_USER, (FSerializeFunc)tSerializeSDropUserReq, &dropReq); tFreeSDropUserReq(&dropReq); @@ -10046,7 +10051,7 @@ static int32_t translateUpdateAnode(STranslateContext* pCxt, SUpdateAnodeStmt* p static int32_t translateCreateDnode(STranslateContext* pCxt, SCreateDnodeStmt* pStmt) { SCreateDnodeReq createReq = {0}; - strcpy(createReq.fqdn, pStmt->fqdn); + tstrncpy(createReq.fqdn, pStmt->fqdn, TSDB_FQDN_LEN); createReq.port = pStmt->port; int32_t code = buildCmdMsg(pCxt, TDMT_MND_CREATE_DNODE, (FSerializeFunc)tSerializeSCreateDnodeReq, &createReq); @@ -10057,7 +10062,7 @@ static int32_t translateCreateDnode(STranslateContext* pCxt, SCreateDnodeStmt* p static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt) { SDropDnodeReq dropReq = {0}; dropReq.dnodeId = pStmt->dnodeId; - strcpy(dropReq.fqdn, pStmt->fqdn); + tstrncpy(dropReq.fqdn, pStmt->fqdn, TSDB_FQDN_LEN); dropReq.port = pStmt->port; dropReq.force = pStmt->force; dropReq.unsafe = pStmt->unsafe; @@ -10073,8 +10078,8 @@ static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pStmt) { SMCfgDnodeReq cfgReq = {0}; cfgReq.dnodeId = pStmt->dnodeId; - strcpy(cfgReq.config, pStmt->config); - strcpy(cfgReq.value, pStmt->value); + tstrncpy(cfgReq.config, pStmt->config, TSDB_DNODE_CONFIG_LEN); + tstrncpy(cfgReq.value, pStmt->value, TSDB_DNODE_VALUE_LEN); int32_t code = 0; @@ -10148,8 +10153,8 @@ static int32_t translateRestoreDnode(STranslateContext* pCxt, SRestoreComponentN static int32_t translateAlterCluster(STranslateContext* pCxt, SAlterClusterStmt* pStmt) { SMCfgClusterReq cfgReq = {0}; - strcpy(cfgReq.config, pStmt->config); - strcpy(cfgReq.value, pStmt->value); + tstrncpy(cfgReq.config, pStmt->config, TSDB_DNODE_CONFIG_LEN); + tstrncpy(cfgReq.value, pStmt->value, TSDB_CLUSTER_VALUE_LEN); int32_t code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_CLUSTER, (FSerializeFunc)tSerializeSMCfgClusterReq, &cfgReq); tFreeSMCfgClusterReq(&cfgReq); @@ -10297,7 +10302,7 @@ static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt return code; } -int32_t createIntervalFromCreateSmaIndexStmt(SCreateIndexStmt* pStmt, SInterval* pInterval) { +int32_t createIntervalFromCreateSmaIndexStmt(SCreateIndexStmt* pStmt, SInterval* pInterval, void* timezone) { pInterval->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i; pInterval->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit; pInterval->offset = NULL != pStmt->pOptions->pOffset ? ((SValueNode*)pStmt->pOptions->pOffset)->datum.i : 0; @@ -10310,6 +10315,7 @@ int32_t createIntervalFromCreateSmaIndexStmt(SCreateIndexStmt* pStmt, SInterval* parserError("%s failed for invalid interval offset %" PRId64, __func__, pInterval->offset); return TSDB_CODE_INVALID_PARA; } + pInterval->timezone = timezone; return TSDB_CODE_SUCCESS; } @@ -10321,7 +10327,7 @@ int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, SS STranslateContext pCxt = {0}; code = initTranslateContext(pParseCxt, NULL, &pCxt); if (TSDB_CODE_SUCCESS == code) { - code = createIntervalFromCreateSmaIndexStmt(pStmt, &interval); + code = createIntervalFromCreateSmaIndexStmt(pStmt, &interval, pParseCxt->timezone); } if (TSDB_CODE_SUCCESS == code) { @@ -10624,9 +10630,9 @@ static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* if (NULL == col) { return code; } - strcpy(col->colName, column->name); - strcpy(col->node.aliasName, col->colName); - strcpy(col->node.userAlias, col->colName); + tstrncpy(col->colName, column->name, TSDB_COL_NAME_LEN); + tstrncpy(col->node.aliasName, col->colName, TSDB_COL_NAME_LEN); + tstrncpy(col->node.userAlias, col->colName, TSDB_COL_NAME_LEN); code = addTagList(&colCxt.pTags, (SNode*)col); if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode((SNode*)col); @@ -10663,9 +10669,9 @@ static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt if (TSDB_CODE_SUCCESS == code) { code = nodesMakeNode(QUERY_NODE_REAL_TABLE, (SNode**)&realTable); if (realTable) { - strcpy(realTable->table.dbName, pStmt->subDbName); - strcpy(realTable->table.tableName, pStmt->subSTbName); - strcpy(realTable->table.tableAlias, pStmt->subSTbName); + tstrncpy(realTable->table.dbName, pStmt->subDbName, TSDB_DB_NAME_LEN); + tstrncpy(realTable->table.tableName, pStmt->subSTbName, TSDB_TABLE_NAME_LEN); + tstrncpy(realTable->table.tableAlias, pStmt->subSTbName, TSDB_TABLE_NAME_LEN); code = createSelectStmtImpl(true, pProjection, (SNode*)realTable, NULL, ppSelect); } if (TSDB_CODE_SUCCESS == code) { @@ -10738,7 +10744,7 @@ static int32_t translateDropCGroup(STranslateContext* pCxt, SDropCGroupStmt* pSt if (TSDB_CODE_SUCCESS != code) return code; (void)tNameGetFullDbName(&name, dropReq.topic); dropReq.igNotExists = pStmt->ignoreNotExists; - strcpy(dropReq.cgroup, pStmt->cgroup); + tstrncpy(dropReq.cgroup, pStmt->cgroup, TSDB_CGROUP_LEN); return buildCmdMsg(pCxt, TDMT_MND_TMQ_DROP_CGROUP, (FSerializeFunc)tSerializeSMDropCgroupReq, &dropReq); } @@ -10933,7 +10939,7 @@ static int32_t translateKillCompact(STranslateContext* pCxt, SKillStmt* pStmt) { static int32_t translateKillQuery(STranslateContext* pCxt, SKillQueryStmt* pStmt) { SKillQueryReq killReq = {0}; - strcpy(killReq.queryStrId, pStmt->queryId); + tstrncpy(killReq.queryStrId, pStmt->queryId, TSDB_QUERY_ID_LEN); return buildCmdMsg(pCxt, TDMT_MND_KILL_QUERY, (FSerializeFunc)tSerializeSKillQueryReq, &killReq); } @@ -10989,7 +10995,7 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt static void getSourceDatabase(SNode* pStmt, int32_t acctId, char* pDbFName) { SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; - strcpy(name.dbname, ((SRealTableNode*)(((SSelectStmt*)pStmt)->pFromTable))->table.dbName); + tstrncpy(name.dbname, ((SRealTableNode*)(((SSelectStmt*)pStmt)->pFromTable))->table.dbName, TSDB_DB_NAME_LEN); (void)tNameGetFullDbName(&name, pDbFName); } @@ -11048,7 +11054,7 @@ static int32_t addIrowTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSelect SColumnDefNode* pColDef = NULL; code = nodesMakeNode(QUERY_NODE_COLUMN_DEF, (SNode**)&pColDef); if (TSDB_CODE_SUCCESS == code) { - strcpy(pColDef->colName, pFunc->node.aliasName); + tstrncpy(pColDef->colName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); pColDef->dataType = pFunc->node.resType; pColDef->sma = true; code = setColumnDefNodePrimaryKey(pColDef, false); @@ -11072,8 +11078,8 @@ static int32_t addWstartTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSele if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "_wstart"); - strcpy(pFunc->node.userAlias, "_irowts"); + tstrncpy(pFunc->functionName, "_wstart", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.userAlias, "_irowts", TSDB_COL_NAME_LEN); char* defaultName[] = {"_wstart", "ts", NULL}; getStreamQueryFirstProjectAliasName(pUserAliasSet, pFunc->node.aliasName, sizeof(pFunc->node.aliasName), defaultName); code = getFuncInfo(pCxt, pFunc); @@ -11085,7 +11091,7 @@ static int32_t addWstartTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSele SColumnDefNode* pColDef = NULL; code = nodesMakeNode(QUERY_NODE_COLUMN_DEF, (SNode**)&pColDef); if (TSDB_CODE_SUCCESS == code) { - strcpy(pColDef->colName, pFunc->node.aliasName); + tstrncpy(pColDef->colName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); pColDef->dataType = pFunc->node.resType; pColDef->sma = true; code = setColumnDefNodePrimaryKey(pColDef, false); @@ -11245,7 +11251,7 @@ static int32_t addColDefNodeByProj(SNodeList** ppCols, const SNode* pProject, in SColumnDefNode* pColDef = NULL; int32_t code = nodesMakeNode(QUERY_NODE_COLUMN_DEF, (SNode**)&pColDef); if (TSDB_CODE_SUCCESS != code) return code; - strcpy(pColDef->colName, pExpr->userAlias); + tstrncpy(pColDef->colName, pExpr->userAlias, TSDB_COL_NAME_LEN); pColDef->dataType = pExpr->resType; pColDef->sma = flags & COL_SMA_ON; code = setColumnDefNodePrimaryKey(pColDef, flags & COL_IS_KEY); @@ -11546,7 +11552,7 @@ static int32_t adjustDataTypeOfProjections(STranslateContext* pCxt, const STable SColumnDefNode* pColDef = NULL; int32_t code = nodesMakeNode(QUERY_NODE_COLUMN_DEF, (SNode**)&pColDef); if (TSDB_CODE_SUCCESS != code) return code; - strcpy(pColDef->colName, pSchema->name); + tstrncpy(pColDef->colName, pSchema->name, TSDB_COL_NAME_LEN); pColDef->dataType = dt; pColDef->sma = pSchema->flags & COL_SMA_ON; code = setColumnDefNodePrimaryKey(pColDef, pSchema->flags & COL_IS_KEY); @@ -11589,7 +11595,7 @@ static int32_t addProjToProjColPos(STranslateContext* pCxt, const SSchema* pSche if (!dataTypeEqual(&dt, &((SExprNode*)pNewProj)->resType)) { SNode* pFunc = NULL; code = createCastFunc(pCxt, pNewProj, dt, &pFunc); - strcpy(((SExprNode*)pFunc)->userAlias, ((SExprNode*)pNewProj)->userAlias); + tstrncpy(((SExprNode*)pFunc)->userAlias, ((SExprNode*)pNewProj)->userAlias, TSDB_COL_NAME_LEN); pNewProj = pFunc; } if (TSDB_CODE_SUCCESS == code) { @@ -11683,9 +11689,6 @@ static int32_t adjustOrderOfProjections(STranslateContext* pCxt, SNodeList** ppC code = nodesMakeList(&pNewProjections); if (TSDB_CODE_SUCCESS != code) return code; code = nodesMakeList(&pNewCols); - if (TSDB_CODE_SUCCESS != code) { - code = code; - } for (int32_t i = 0; TSDB_CODE_SUCCESS == code && i < num; ++i) { SProjColPos* pPos = taosArrayGet(pProjColPos, i); code = nodesListStrictAppend(pNewProjections, pPos->pProj); @@ -11791,9 +11794,6 @@ static int32_t adjustOrderOfTags(STranslateContext* pCxt, SNodeList* pTags, cons int32_t numOfTags = getNumOfTags(pMeta); const SSchema* pTagsSchema = getTableTagSchema(pMeta); code = nodesMakeList(&pNewTagExprs); - if (NULL == pNewTagExprs) { - code = code; - } for (int32_t i = 0; TSDB_CODE_SUCCESS == code && i < numOfTags; ++i) { const SSchema* pTagSchema = pTagsSchema + i; if (indexOfBoundTags < numOfBoundTags) { @@ -12158,8 +12158,8 @@ static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt* (void)tNameGetFullDbName(&name, pReq->name); if ('\0' != pStmt->targetTabName[0]) { - strcpy(name.dbname, pStmt->targetDbName); - strcpy(name.tname, pStmt->targetTabName); + tstrncpy(name.dbname, pStmt->targetDbName, TSDB_DB_NAME_LEN); + tstrncpy(name.tname, pStmt->targetTabName, TSDB_TABLE_NAME_LEN); code = tNameExtractFullName(&name, pReq->targetStbFullName); } if (TSDB_CODE_SUCCESS == code) { @@ -12242,6 +12242,7 @@ static int32_t buildIntervalForCreateStream(SCreateStreamStmt* pStmt, SInterval* (NULL != pWindow->pSliding ? ((SValueNode*)pWindow->pSliding)->unit : pInterval->intervalUnit); pInterval->precision = ((SColumnNode*)pWindow->pCol)->node.resType.precision; pInterval->timeRange = pWindow->timeRange; + pInterval->timezone = pWindow->timezone; return code; } @@ -12291,7 +12292,7 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, SSDa if (TSDB_CODE_SUCCESS == code) { if (interval.interval > 0) { pStmt->pReq->lastTs = taosTimeAdd(taosTimeTruncate(lastTs, &interval), interval.interval, interval.intervalUnit, - interval.precision); + interval.precision, pParseCxt->timezone); } else { pStmt->pReq->lastTs = lastTs + 1; // start key of the next time window } @@ -12386,7 +12387,7 @@ static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pSt pStmt->createReq.precision = res.schemaRes.precision; pStmt->createReq.numOfCols = res.schemaRes.numOfCols; pStmt->createReq.pSchema = res.schemaRes.pSchema; - strncpy(pStmt->createReq.name, pStmt->viewName, sizeof(pStmt->createReq.name) - 1); + tstrncpy(pStmt->createReq.name, pStmt->viewName, TSDB_VIEW_NAME_LEN); tstrncpy(pStmt->createReq.dbFName, dbFName, sizeof(pStmt->createReq.dbFName)); snprintf(pStmt->createReq.fullname, sizeof(pStmt->createReq.fullname) - 1, "%s.%s", pStmt->createReq.dbFName, pStmt->viewName); @@ -12415,7 +12416,7 @@ static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt) int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); if (TSDB_CODE_SUCCESS == code) { (void)tNameGetFullDbName(&name, dropReq.dbFName); - strncpy(dropReq.name, pStmt->viewName, sizeof(dropReq.name) - 1); + tstrncpy(dropReq.name, pStmt->viewName, TSDB_VIEW_NAME_LEN); snprintf(dropReq.fullname, sizeof(dropReq.fullname) - 1, "%s.%s", dropReq.dbFName, dropReq.name); dropReq.sql = (char*)pCxt->pParseCxt->pSql; if (NULL == dropReq.sql) { @@ -12482,7 +12483,7 @@ static int32_t translateCreateFunction(STranslateContext* pCxt, SCreateFunctionS } SCreateFuncReq req = {0}; - strcpy(req.name, pStmt->funcName); + tstrncpy(req.name, pStmt->funcName, TSDB_FUNC_NAME_LEN); req.orReplace = pStmt->orReplace; req.igExists = pStmt->ignoreExists; req.funcType = pStmt->isAgg ? TSDB_FUNC_TYPE_AGGREGATE : TSDB_FUNC_TYPE_SCALAR; @@ -12501,7 +12502,7 @@ static int32_t translateCreateFunction(STranslateContext* pCxt, SCreateFunctionS static int32_t translateDropFunction(STranslateContext* pCxt, SDropFunctionStmt* pStmt) { SDropFuncReq req = {0}; - strcpy(req.name, pStmt->funcName); + tstrncpy(req.name, pStmt->funcName, TSDB_FUNC_NAME_LEN); req.igNotExists = pStmt->ignoreNotExists; return buildCmdMsg(pCxt, TDMT_MND_DROP_FUNC, (FSerializeFunc)tSerializeSDropFuncReq, &req); } @@ -12512,9 +12513,9 @@ static int32_t createRealTableForGrantTable(SGrantStmt* pStmt, SRealTableNode** if (NULL == pRealTable) { return code; } - strcpy(pRealTable->table.dbName, pStmt->objName); - strcpy(pRealTable->table.tableName, pStmt->tabName); - strcpy(pRealTable->table.tableAlias, pStmt->tabName); + tstrncpy(pRealTable->table.dbName, pStmt->objName, TSDB_DB_NAME_LEN); + tstrncpy(pRealTable->table.tableName, pStmt->tabName, TSDB_TABLE_NAME_LEN); + tstrncpy(pRealTable->table.tableAlias, pStmt->tabName, TSDB_TABLE_NAME_LEN); *pTable = pRealTable; return TSDB_CODE_SUCCESS; } @@ -12589,9 +12590,9 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { } #endif - strcpy(req.user, pStmt->userName); - sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName); - sprintf(req.tabName, "%s", pStmt->tabName); + tstrncpy(req.user, pStmt->userName, TSDB_USER_LEN); + snprintf(req.objname, TSDB_DB_FNAME_LEN, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName); + snprintf(req.tabName, TSDB_TABLE_NAME_LEN, "%s", pStmt->tabName); if (!req.isView) { code = translateGrantTagCond(pCxt, pStmt, &req); } @@ -12625,9 +12626,9 @@ static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) { } #endif - strcpy(req.user, pStmt->userName); - sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName); - sprintf(req.tabName, "%s", pStmt->tabName); + tstrncpy(req.user, pStmt->userName, TSDB_USER_LEN); + snprintf(req.objname, TSDB_DB_FNAME_LEN, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName); + snprintf(req.tabName, TSDB_TABLE_NAME_LEN, "%s", pStmt->tabName); code = buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); tFreeSAlterUserReq(&req); return code; @@ -12643,7 +12644,7 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) { SBalanceVgroupLeaderReq req = {0}; req.vgId = pStmt->vgId; - strcpy(req.db, pStmt->dbName); + tstrncpy(req.db, pStmt->dbName, TSDB_DB_FNAME_LEN); int32_t code = buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req); tFreeSBalanceVgroupLeaderReq(&req); @@ -12810,14 +12811,14 @@ static int32_t buildTSMAAstStreamSubTable(SCreateTSMAStmt* pStmt, SMCreateSmaReq code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pVal); if (TSDB_CODE_SUCCESS != code) goto _end; - sprintf(pMd5Func->functionName, "%s", "md5"); - sprintf(pConcatFunc->functionName, "%s", "concat"); + snprintf(pMd5Func->functionName, TSDB_FUNC_NAME_LEN, "%s", "md5"); + snprintf(pConcatFunc->functionName, TSDB_FUNC_NAME_LEN, "%s", "concat"); pVal->literal = taosMemoryMalloc(TSDB_TABLE_FNAME_LEN + 1); if (!pVal->literal) { code = terrno; goto _end; } - sprintf(pVal->literal, "%s_", pReq->name); + snprintf(pVal->literal, TSDB_TABLE_FNAME_LEN + 1, "%s_", pReq->name); pVal->node.resType.type = TSDB_DATA_TYPE_VARCHAR; pVal->node.resType.bytes = strlen(pVal->literal); code = nodesListMakeAppend(&pConcatFunc->pParameterList, (SNode*)pVal); @@ -12861,10 +12862,8 @@ static int32_t buildTSMAAst(STranslateContext* pCxt, SCreateTSMAStmt* pStmt, SMC // append partition by tbname code = createTbnameFunction(&pTbnameFunc); if (pTbnameFunc) { - sprintf(pTbnameFunc->node.userAlias, "tbname"); + snprintf(pTbnameFunc->node.userAlias, TSDB_COL_NAME_LEN, "tbname"); code = nodesListMakeStrictAppend(&info.pPartitionByList, (SNode*)pTbnameFunc); - } else { - code = code; } } if (TSDB_CODE_SUCCESS == code) { @@ -12929,7 +12928,7 @@ static int32_t createColumnBySchema(const SSchema* pSchema, SColumnNode** ppCol) (*ppCol)->colId = pSchema->colId; (*ppCol)->node.resType.type = pSchema->type; (*ppCol)->node.resType.bytes = pSchema->bytes; - strcpy((*ppCol)->colName, pSchema->name); + tstrncpy((*ppCol)->colName, pSchema->name, TSDB_COL_NAME_LEN); return TSDB_CODE_SUCCESS; } @@ -13162,7 +13161,7 @@ static int32_t translateCreateTSMA(STranslateContext* pCxt, SCreateTSMAStmt* pSt return code; } -static int32_t buildIntervalForCreateTSMA(SCreateTSMAStmt* pStmt, SInterval* pInterval) { +static int32_t buildIntervalForCreateTSMA(SCreateTSMAStmt* pStmt, SInterval* pInterval, void* timezone) { int32_t code = TSDB_CODE_SUCCESS; pInterval->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i; pInterval->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit; @@ -13170,6 +13169,7 @@ static int32_t buildIntervalForCreateTSMA(SCreateTSMAStmt* pStmt, SInterval* pIn pInterval->sliding = pInterval->interval; pInterval->slidingUnit = pInterval->intervalUnit; pInterval->precision = pStmt->pOptions->tsPrecision; + pInterval->timezone = timezone; return code; } @@ -13181,7 +13181,7 @@ int32_t translatePostCreateTSMA(SParseContext* pParseCxt, SQuery* pQuery, SSData int32_t code = initTranslateContext(pParseCxt, NULL, &cxt); if (TSDB_CODE_SUCCESS == code) { - code = buildIntervalForCreateTSMA(pStmt, &interval); + code = buildIntervalForCreateTSMA(pStmt, &interval, pParseCxt->timezone); } if (TSDB_CODE_SUCCESS == code) { @@ -13191,7 +13191,7 @@ int32_t translatePostCreateTSMA(SParseContext* pParseCxt, SQuery* pQuery, SSData if (TSDB_CODE_SUCCESS == code) { if (interval.interval > 0) { pStmt->pReq->lastTs = taosTimeAdd(taosTimeTruncate(lastTs, &interval), interval.interval, interval.intervalUnit, - interval.precision); + interval.precision, pParseCxt->timezone); } else { pStmt->pReq->lastTs = lastTs + 1; // start key of the next time window } @@ -13481,9 +13481,9 @@ static int32_t extractQueryResultSchema(const SNodeList* pProjections, int32_t* } (*pSchema)[index].colId = index + 1; if ('\0' != pExpr->userAlias[0]) { - strcpy((*pSchema)[index].name, pExpr->userAlias); + tstrncpy((*pSchema)[index].name, pExpr->userAlias, TSDB_COL_NAME_LEN); } else { - strcpy((*pSchema)[index].name, pExpr->aliasName); + tstrncpy((*pSchema)[index].name, pExpr->aliasName, TSDB_COL_NAME_LEN); } index += 1; } @@ -13501,7 +13501,7 @@ static int32_t extractExplainResultSchema(int32_t* numOfCols, SSchema** pSchema) } (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE; - strcpy((*pSchema)[0].name, TSDB_EXPLAIN_RESULT_COLUMN_NAME); + tstrncpy((*pSchema)[0].name, TSDB_EXPLAIN_RESULT_COLUMN_NAME, TSDB_COL_NAME_LEN); return TSDB_CODE_SUCCESS; } @@ -13515,32 +13515,32 @@ static int32_t extractDescribeResultSchema(STableMeta* pMeta, int32_t* numOfCols (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = DESCRIBE_RESULT_FIELD_LEN; - strcpy((*pSchema)[0].name, "field"); + tstrncpy((*pSchema)[0].name, "field", TSDB_COL_NAME_LEN); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[1].bytes = DESCRIBE_RESULT_TYPE_LEN; - strcpy((*pSchema)[1].name, "type"); + tstrncpy((*pSchema)[1].name, "type", TSDB_COL_NAME_LEN); (*pSchema)[2].type = TSDB_DATA_TYPE_INT; (*pSchema)[2].bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; - strcpy((*pSchema)[2].name, "length"); + tstrncpy((*pSchema)[2].name, "length", TSDB_COL_NAME_LEN); (*pSchema)[3].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[3].bytes = DESCRIBE_RESULT_NOTE_LEN; - strcpy((*pSchema)[3].name, "note"); + tstrncpy((*pSchema)[3].name, "note", TSDB_COL_NAME_LEN); if (pMeta && useCompress(pMeta->tableType)) { (*pSchema)[4].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[4].bytes = DESCRIBE_RESULT_COPRESS_OPTION_LEN; - strcpy((*pSchema)[4].name, "encode"); + tstrncpy((*pSchema)[4].name, "encode", TSDB_COL_NAME_LEN); (*pSchema)[5].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[5].bytes = DESCRIBE_RESULT_COPRESS_OPTION_LEN; - strcpy((*pSchema)[5].name, "compress"); + tstrncpy((*pSchema)[5].name, "compress", TSDB_COL_NAME_LEN); (*pSchema)[6].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[6].bytes = DESCRIBE_RESULT_COPRESS_OPTION_LEN; - strcpy((*pSchema)[6].name, "level"); + tstrncpy((*pSchema)[6].name, "level", TSDB_COL_NAME_LEN); } return TSDB_CODE_SUCCESS; @@ -13555,11 +13555,11 @@ static int32_t extractShowCreateDatabaseResultSchema(int32_t* numOfCols, SSchema (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = TSDB_DB_NAME_LEN; - strcpy((*pSchema)[0].name, "Database"); + tstrncpy((*pSchema)[0].name, "Database", TSDB_COL_NAME_LEN); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[1].bytes = TSDB_MAX_BINARY_LEN; - strcpy((*pSchema)[1].name, "Create Database"); + tstrncpy((*pSchema)[1].name, "Create Database", TSDB_COL_NAME_LEN); return TSDB_CODE_SUCCESS; } @@ -13573,11 +13573,11 @@ static int32_t extractShowCreateTableResultSchema(int32_t* numOfCols, SSchema** (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = SHOW_CREATE_TB_RESULT_FIELD1_LEN; - strcpy((*pSchema)[0].name, "Table"); + tstrncpy((*pSchema)[0].name, "Table", TSDB_COL_NAME_LEN); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[1].bytes = SHOW_CREATE_TB_RESULT_FIELD2_LEN; - strcpy((*pSchema)[1].name, "Create Table"); + tstrncpy((*pSchema)[1].name, "Create Table", TSDB_COL_NAME_LEN); return TSDB_CODE_SUCCESS; } @@ -13591,11 +13591,11 @@ static int32_t extractShowCreateViewResultSchema(int32_t* numOfCols, SSchema** p (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = SHOW_CREATE_VIEW_RESULT_FIELD1_LEN; - strcpy((*pSchema)[0].name, "View"); + tstrncpy((*pSchema)[0].name, "View", TSDB_COL_NAME_LEN); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[1].bytes = SHOW_CREATE_VIEW_RESULT_FIELD2_LEN; - strcpy((*pSchema)[1].name, "Create View"); + tstrncpy((*pSchema)[1].name, "Create View", TSDB_COL_NAME_LEN); return TSDB_CODE_SUCCESS; } @@ -13609,19 +13609,19 @@ static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pS (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = TSDB_CONFIG_OPTION_LEN; - strcpy((*pSchema)[0].name, "name"); + tstrncpy((*pSchema)[0].name, "name", TSDB_COL_NAME_LEN); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[1].bytes = TSDB_CONFIG_PATH_LEN; - strcpy((*pSchema)[1].name, "value"); + tstrncpy((*pSchema)[1].name, "value", TSDB_COL_NAME_LEN); (*pSchema)[2].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[2].bytes = TSDB_CONFIG_SCOPE_LEN; - strcpy((*pSchema)[2].name, "scope"); + tstrncpy((*pSchema)[2].name, "scope", TSDB_COL_NAME_LEN); (*pSchema)[3].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[3].bytes = TSDB_CONFIG_INFO_LEN; - strcpy((*pSchema)[3].name, "info"); + tstrncpy((*pSchema)[3].name, "info", TSDB_COL_NAME_LEN); return TSDB_CODE_SUCCESS; } @@ -13635,15 +13635,15 @@ static int32_t extractCompactDbResultSchema(int32_t* numOfCols, SSchema** pSchem (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = COMPACT_DB_RESULT_FIELD1_LEN; - strcpy((*pSchema)[0].name, "result"); + tstrncpy((*pSchema)[0].name, "result", TSDB_COL_NAME_LEN); (*pSchema)[1].type = TSDB_DATA_TYPE_INT; (*pSchema)[1].bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; - strcpy((*pSchema)[1].name, "id"); + tstrncpy((*pSchema)[1].name, "id", TSDB_COL_NAME_LEN); (*pSchema)[2].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[2].bytes = COMPACT_DB_RESULT_FIELD3_LEN; - strcpy((*pSchema)[2].name, "reason"); + tstrncpy((*pSchema)[2].name, "reason", TSDB_COL_NAME_LEN); return TSDB_CODE_SUCCESS; } @@ -13689,7 +13689,7 @@ static int32_t createStarCol(SNode** ppNode) { if (NULL == pCol) { return code; } - strcpy(pCol->colName, "*"); + tstrncpy(pCol->colName, "*", TSDB_COL_NAME_LEN); *ppNode = (SNode*)pCol; return code; } @@ -13743,7 +13743,7 @@ static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, S if (NULL == pSelect) { return code; } - sprintf(pSelect->stmtName, "%p", pSelect); + snprintf(pSelect->stmtName, TSDB_TABLE_NAME_LEN, "%p", pSelect); SRealTableNode* pRealTable = NULL; code = nodesMakeNode(QUERY_NODE_REAL_TABLE, (SNode**)&pRealTable); @@ -13839,8 +13839,8 @@ static int32_t createParOperatorNode(EOperatorType opType, const char* pLeftCol, nodesDestroyNode((SNode*)pOper); return code; } - strcpy(((SColumnNode*)pOper->pLeft)->colName, pLeftCol); - strcpy(((SColumnNode*)pOper->pRight)->colName, pRightCol); + tstrncpy(((SColumnNode*)pOper->pLeft)->colName, pLeftCol, TSDB_COL_NAME_LEN); + tstrncpy(((SColumnNode*)pOper->pRight)->colName, pRightCol, TSDB_COL_NAME_LEN); *ppResOp = (SNode*)pOper; return TSDB_CODE_SUCCESS; @@ -14107,7 +14107,7 @@ static int32_t createShowCondition(const SShowStmt* pShow, SSelectStmt* pSelect) } if (NULL != pShow->pDbName) { - strcpy(((SRealTableNode*)pSelect->pFromTable)->qualDbName, ((SValueNode*)pShow->pDbName)->literal); + tstrncpy(((SRealTableNode*)pSelect->pFromTable)->qualDbName, ((SValueNode*)pShow->pDbName)->literal, TSDB_DB_NAME_LEN); } return TSDB_CODE_SUCCESS; @@ -14180,7 +14180,7 @@ static int32_t createTagsFunction(SFunctionNode** ppNode) { if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "_tags"); + tstrncpy(pFunc->functionName, "_tags", TSDB_FUNC_NAME_LEN); *ppNode = pFunc; return code; } @@ -14278,8 +14278,8 @@ static int32_t createBlockDistInfoFunc(SFunctionNode** ppNode) { return code; } - strcpy(pFunc->functionName, "_block_dist_info"); - strcpy(pFunc->node.aliasName, "_block_dist_info"); + tstrncpy(pFunc->functionName, "_block_dist_info", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, "_block_dist_info", TSDB_COL_NAME_LEN); *ppNode = pFunc; return code; } @@ -14291,8 +14291,8 @@ static int32_t createBlockDistFunc(SFunctionNode** ppNode) { return code; } - strcpy(pFunc->functionName, "_block_dist"); - strcpy(pFunc->node.aliasName, "_block_dist"); + tstrncpy(pFunc->functionName, "_block_dist", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, "_block_dist", TSDB_COL_NAME_LEN); SFunctionNode* pFuncNew = NULL; code = createBlockDistInfoFunc(&pFuncNew); if (TSDB_CODE_SUCCESS == code) { @@ -14390,7 +14390,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* SVgroupCreateTableBatch* pBatch) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; - (void)strcpy(name.dbname, pStmt->dbName); + tstrncpy(name.dbname, pStmt->dbName, TSDB_DB_NAME_LEN); (void)tNameGetFullDbName(&name, dbFName); SVCreateTbReq req = {0}; @@ -14446,7 +14446,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* ++index; } pBatch->info = *pVgroupInfo; - (void)strcpy(pBatch->dbName, pStmt->dbName); + tstrncpy(pBatch->dbName, pStmt->dbName, TSDB_DB_NAME_LEN); pBatch->req.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq)); if (NULL == pBatch->req.pArray) { tdDestroySVCreateTbReq(&req); @@ -14622,7 +14622,7 @@ static int32_t addCreateTbReqIntoVgroup(SHashObj* pVgroupHashmap, const char* db if (pTableBatch == NULL) { SVgroupCreateTableBatch tBatch = {0}; tBatch.info = *pVgInfo; - strcpy(tBatch.dbName, dbName); + tstrncpy(tBatch.dbName, dbName, TSDB_DB_NAME_LEN); tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq)); if (!tBatch.req.pArray) { @@ -14706,7 +14706,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla if (pSchema->type == TSDB_DATA_TYPE_JSON) { isJson = true; } - code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pSchema, &token, tagName, pTagArray, ppTag); + code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pSchema, &token, tagName, pTagArray, ppTag, pCxt->pParseCxt->timezone, pCxt->pParseCxt->charsetCxt); } if (TSDB_CODE_SUCCESS == code) { @@ -14767,7 +14767,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { isJson = true; } - code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pTagSchema, &token, tagName, pTagArray, ppTag); + code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pTagSchema, &token, tagName, pTagArray, ppTag, pCxt->pParseCxt->timezone, pCxt->pParseCxt->charsetCxt); } if (TSDB_CODE_SUCCESS == code) { @@ -14963,7 +14963,7 @@ static int32_t fillVgroupInfo(SParseContext* pParseCxt, const SName* pName, SVgr return code; } -static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) { +static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt, timezone_t tz, void *charsetCxt) { int32_t code = TSDB_CODE_SUCCESS; int sz = taosArrayGetSize(pParFileCxt->aTagIndexs); int32_t numOfTags = getNumOfTags(pParFileCxt->pStbMeta); @@ -14995,7 +14995,7 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) if (TSDB_CODE_SUCCESS == code) { SArray* aTagNames = pParFileCxt->tagNameFilled ? NULL : pParFileCxt->aTagNames; code = parseTagValue(pMsgBuf, &pParFileCxt->pSql, precision, (SSchema*)pTagSchema, &token, aTagNames, - pParFileCxt->aTagVals, &pParFileCxt->pTag); + pParFileCxt->aTagVals, &pParFileCxt->pTag, tz, charsetCxt); } } else { // parse tbname @@ -15061,7 +15061,7 @@ static int32_t parseCsvFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt, SParseFi (void)strtolower(pLine, pLine); pParFileCxt->pSql = pLine; - code = parseOneStbRow(pMsgBuf, pParFileCxt); + code = parseOneStbRow(pMsgBuf, pParFileCxt, pParseCxt->timezone, pParseCxt->charsetCxt); if (TSDB_CODE_SUCCESS == code) { code = fillVgroupInfo(pParseCxt, &pParFileCxt->ctbName, &pParFileCxt->vg); @@ -15125,7 +15125,7 @@ static int32_t constructParseFileContext(SCreateSubTableFromFileClause* pStmt, S pParFileCxt->pTag = NULL; pParFileCxt->ctbName.type = TSDB_TABLE_NAME_T; pParFileCxt->ctbName.acctId = acctId; - strcpy(pParFileCxt->ctbName.dbname, pStmt->useDbName); + tstrncpy(pParFileCxt->ctbName.dbname, pStmt->useDbName, TSDB_DB_NAME_LEN); if (NULL == pParFileCxt->aTagNames) { pParFileCxt->aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN); @@ -15199,8 +15199,8 @@ static int32_t prepareReadCsvFile(STranslateContext* pCxt, SCreateSubTableFromFi } pCreateInfo->ignoreExists = pCreateStmt->ignoreExists; - strncpy(pCreateInfo->useDbName, pCreateStmt->useDbName, TSDB_DB_NAME_LEN); - strncpy(pCreateInfo->useTableName, pCreateStmt->useTableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pCreateInfo->useDbName, pCreateStmt->useDbName, TSDB_DB_NAME_LEN); + tstrncpy(pCreateInfo->useTableName, pCreateStmt->useTableName, TSDB_TABLE_NAME_LEN); } { @@ -15754,7 +15754,7 @@ static int32_t buildUpdateTagValReqImpl2(STranslateContext* pCxt, SAlterTableStm if (TSDB_CODE_SUCCESS == code) { code = parseTagValue(&pCxt->msgBuf, &tagStr, pTableMeta->tableInfo.precision, pSchema, &token, NULL, - pReq->pTagArray, &pTag); + pReq->pTagArray, &pTag, pCxt->pParseCxt->timezone, pCxt->pParseCxt->charsetCxt); if (pSchema->type == TSDB_DATA_TYPE_JSON && token.type == TK_NULL && code == TSDB_CODE_SUCCESS) { pReq->tagFree = true; } @@ -15826,7 +15826,7 @@ static int32_t buildUpdateTagValReqImpl(STranslateContext* pCxt, SAlterTableStmt if (TSDB_CODE_SUCCESS == code) { code = parseTagValue(&pCxt->msgBuf, &tagStr, pTableMeta->tableInfo.precision, pSchema, &token, NULL, - pReq->pTagArray, &pTag); + pReq->pTagArray, &pTag, pCxt->pParseCxt->timezone, pCxt->pParseCxt->charsetCxt); if (pSchema->type == TSDB_DATA_TYPE_JSON && token.type == TK_NULL && code == TSDB_CODE_SUCCESS) { pReq->tagFree = true; } @@ -16400,8 +16400,8 @@ static int32_t createParCaseWhenNode(SNode* pCase, SNodeList* pWhenThenList, SNo pCaseWhen->pWhenThenList = pWhenThenList; pCaseWhen->pElse = pElse; if (pAias) { - strcpy(pCaseWhen->node.aliasName, pAias); - strcpy(pCaseWhen->node.userAlias, pAias); + tstrncpy(pCaseWhen->node.aliasName, pAias, TSDB_COL_NAME_LEN); + tstrncpy(pCaseWhen->node.userAlias, pAias, TSDB_COL_NAME_LEN); } *ppResCaseWhen = (SNode*)pCaseWhen; return TSDB_CODE_SUCCESS; @@ -16414,9 +16414,9 @@ static int32_t createParFunctionNode(const char* pFunName, const char* pAias, SN if (TSDB_CODE_SUCCESS != code) { return code; } - strcpy(pFunc->functionName, pFunName); - strcpy(pFunc->node.aliasName, pAias); - strcpy(pFunc->node.userAlias, pAias); + tstrncpy(pFunc->functionName, pFunName, TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, pAias, TSDB_COL_NAME_LEN); + tstrncpy(pFunc->node.userAlias, pAias, TSDB_COL_NAME_LEN); pFunc->pParameterList = pParameterList; *ppResFunc = (SNode*)pFunc; return TSDB_CODE_SUCCESS; @@ -16441,7 +16441,7 @@ static int32_t createParTempTableNode(SSelectStmt* pSubquery, SNode** ppResTempT } pTempTable->pSubquery = (SNode*)pSubquery; taosRandStr(pTempTable->table.tableAlias, 8); - strcpy(pSubquery->stmtName, pTempTable->table.tableAlias); + tstrncpy(pSubquery->stmtName, pTempTable->table.tableAlias, TSDB_TABLE_NAME_LEN); pSubquery->isSubquery = true; *ppResTempTable = (SNode*)pTempTable; return TSDB_CODE_SUCCESS; @@ -16857,7 +16857,7 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) { pStmt->pProjectionList = pProjList; pStmt->pFromTable = pTempTblNode; - sprintf(pStmt->stmtName, "%p", pStmt); + snprintf(pStmt->stmtName, TSDB_TABLE_NAME_LEN, "%p", pStmt); nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = (SNode*)pStmt; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 44e44982a3..9706644324 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -250,7 +250,7 @@ int32_t generateSyntaxErrMsgExt(SMsgBuf* pBuf, int32_t errCode, const char* pFor int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) { if (pBuf->buf) { - strncpy(pBuf->buf, msg, pBuf->len); + tstrncpy(pBuf->buf, msg, pBuf->len); } return TSDB_CODE_TSC_INVALID_OPERATION; @@ -277,7 +277,7 @@ int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* } char buf[64] = {0}; // only extract part of sql string - strncpy(buf, sourceStr, tListLen(buf) - 1); + tstrncpy(buf, sourceStr, tListLen(buf)); if (additionalInfo != NULL) { snprintf(pBuf->buf, pBuf->len, msgFormat2, buf, additionalInfo); @@ -402,7 +402,7 @@ static bool isValidateTag(char* input) { return true; } -int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, void* pMsgBuf) { +int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, void* pMsgBuf, void *charsetCxt) { int32_t retCode = TSDB_CODE_SUCCESS; cJSON* root = NULL; SHashObj* keyHash = NULL; @@ -454,7 +454,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi continue; } STagVal val = {0}; - // strcpy(val.colName, colName); + // TSDB_DB_FNAME_LENme, colName); val.pKey = jsonKey; retCode = taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless @@ -471,8 +471,9 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi goto end; } val.type = TSDB_DATA_TYPE_NCHAR; - if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp, (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) { - uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, + if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp, (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen, charsetCxt)) { + uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, + charsetCxt != NULL ? ((SConvInfo *)(charsetCxt))->charset : tsCharset, jsonValue, strerror(terrno)); retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); taosMemoryFree(tmp); @@ -595,15 +596,15 @@ int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, char* pStr, bool isView) { - return sprintf(pStr, "%s*%d*%s*%s*%d*%d", pUser, acctId, pDb, (NULL == pTable || '\0' == pTable[0]) ? "``" : pTable, - type, isView); + return snprintf(pStr, USER_AUTH_KEY_MAX_LEN, "%s*%d*%s*%s*%d*%d", pUser, acctId, pDb, + (NULL == pTable || '\0' == pTable[0]) ? "``" : pTable, type, isView); } static int32_t getIntegerFromAuthStr(const char* pStart, char** pNext) { char* p = strchr(pStart, '*'); char buf[10] = {0}; if (NULL == p) { - strcpy(buf, pStart); + tstrncpy(buf, pStart, 10); *pNext = NULL; } else { strncpy(buf, pStart, p - pStart); @@ -612,10 +613,10 @@ static int32_t getIntegerFromAuthStr(const char* pStart, char** pNext) { return taosStr2Int32(buf, NULL, 10); } -static void getStringFromAuthStr(const char* pStart, char* pStr, char** pNext) { +static void getStringFromAuthStr(const char* pStart, char* pStr, uint32_t dstLen, char** pNext) { char* p = strchr(pStart, '*'); if (NULL == p) { - strcpy(pStr, pStart); + tstrncpy(pStr, pStart, dstLen); *pNext = NULL; } else { strncpy(pStr, pStart, p - pStart); @@ -628,10 +629,10 @@ static void getStringFromAuthStr(const char* pStart, char* pStr, char** pNext) { static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUserAuth) { char* p = NULL; - getStringFromAuthStr(pStr, pUserAuth->user, &p); + getStringFromAuthStr(pStr, pUserAuth->user, TSDB_USER_LEN, &p); pUserAuth->tbName.acctId = getIntegerFromAuthStr(p, &p); - getStringFromAuthStr(p, pUserAuth->tbName.dbname, &p); - getStringFromAuthStr(p, pUserAuth->tbName.tname, &p); + getStringFromAuthStr(p, pUserAuth->tbName.dbname, TSDB_DB_NAME_LEN, &p); + getStringFromAuthStr(p, pUserAuth->tbName.tname, TSDB_TABLE_NAME_LEN, &p); if (pUserAuth->tbName.tname[0]) { pUserAuth->tbName.type = TSDB_TABLE_NAME_T; } else { @@ -707,7 +708,7 @@ static int32_t buildTableReqFromDb(SHashObj* pDbsHash, SArray** pDbs) { SParseTablesMetaReq* p = taosHashIterate(pDbsHash, NULL); while (NULL != p) { STablesReq req = {0}; - strcpy(req.dbFName, p->dbFName); + tstrncpy(req.dbFName, p->dbFName, TSDB_DB_FNAME_LEN); int32_t code = buildTableReq(p->pTables, &req.pTables); if (TSDB_CODE_SUCCESS == code) { if (NULL == taosArrayPush(*pDbs, &req)) { @@ -831,7 +832,7 @@ int32_t createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* select->isDistinct = isDistinct; select->pProjectionList = pProjectionList; select->pFromTable = pTable; - sprintf(select->stmtName, "%p", select); + snprintf(select->stmtName, TSDB_TABLE_NAME_LEN, "%p", select); select->timeLineResMode = select->isDistinct ? TIME_LINE_NONE : TIME_LINE_GLOBAL; select->timeLineCurMode = TIME_LINE_GLOBAL; select->onlyHasKeepOrderFunc = true; diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index e2135bfd63..6d2822f038 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -153,7 +153,7 @@ static int32_t parseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, SParseMetaCa return code; } -static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { +static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam, void *charsetCxt) { if (!pParam || IS_NULL_TYPE(pParam->buffer_type)) { return TSDB_CODE_APP_ERROR; } @@ -200,7 +200,7 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { int32_t output = 0; if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes, - &output)) { + &output, charsetCxt)) { return terrno; } varDataSetLen(pVal->datum.p, output); @@ -221,7 +221,7 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { static EDealRes rewriteQueryExprAliasImpl(SNode* pNode, void* pContext) { if (nodesIsExprNode(pNode) && QUERY_NODE_COLUMN != nodeType(pNode)) { - sprintf(((SExprNode*)pNode)->aliasName, "#%d", *(int32_t*)pContext); + snprintf(((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN, "#%d", *(int32_t*)pContext); ++(*(int32_t*)pContext); } return DEAL_RES_CONTINUE; @@ -417,19 +417,19 @@ int32_t qInitKeywordsTable() { return taosInitKeywordsTable(); } void qCleanupKeywordsTable() { taosCleanupKeywordsTable(); } -int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx) { +int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, void *charsetCxt) { int32_t code = TSDB_CODE_SUCCESS; if (colIdx < 0) { int32_t size = taosArrayGetSize(pQuery->pPlaceholderValues); for (int32_t i = 0; i < size; ++i) { - code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i); + code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i, charsetCxt); if (TSDB_CODE_SUCCESS != code) { return code; } } } else { - code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams); + code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams, charsetCxt); } if (TSDB_CODE_SUCCESS == code && (colIdx < 0 || colIdx + 1 == pQuery->placeholderNum)) { @@ -443,7 +443,7 @@ int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx return code; } -static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam) { +static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam, void* charsetCxt) { if (!pParam || IS_NULL_TYPE(pParam->buffer_type)) { return TSDB_CODE_APP_ERROR; } @@ -490,7 +490,7 @@ static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam) { int32_t output = 0; if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes, - &output)) { + &output, charsetCxt)) { return terrno; } varDataSetLen(pVal->datum.p, output); @@ -509,28 +509,25 @@ static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam) { return TSDB_CODE_SUCCESS; } -int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx) { +int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx, void* charsetCxt) { int32_t code = TSDB_CODE_SUCCESS; if (colIdx < 0) { int32_t size = taosArrayGetSize(pQuery->pPlaceholderValues); for (int32_t i = 0; i < size; ++i) { - code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i); + code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i, charsetCxt); if (TSDB_CODE_SUCCESS != code) { return code; } } } else { - code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams); + code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams, charsetCxt); } if (TSDB_CODE_SUCCESS == code && (colIdx < 0 || colIdx + 1 == pQuery->placeholderNum)) { nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = NULL; code = nodesCloneNode(pQuery->pPrepareRoot, &pQuery->pRoot); - if (NULL == pQuery->pRoot) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { rewriteExprAlias(pQuery->pRoot); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index a9c3beb025..363aa71479 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -119,9 +119,9 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { } SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode); pCol->node.resType = pToBeRewrittenExpr->resType; - strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); - strcpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias); - strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); + tstrncpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias, TSDB_COL_NAME_LEN); + tstrncpy(pCol->colName, ((SExprNode*)pExpr)->aliasName, TSDB_COL_NAME_LEN); pCol->node.projIdx = ((SExprNode*)(*pNode))->projIdx; if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { setColumnInfo((SFunctionNode*)pExpr, pCol, pCxt->isPartitionBy); @@ -150,7 +150,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_FUNCTION: { if ('\0' == ((SExprNode*)pNode)->aliasName[0]) { - sprintf(((SExprNode*)pNode)->aliasName, "#expr_%p", pNode); + snprintf(((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN, "#expr_%p", pNode); } return DEAL_RES_IGNORE_CHILD; } @@ -311,12 +311,12 @@ static SNode* createFirstCol(SRealTableNode* pTable, const SSchema* pSchema) { pCol->tableId = pTable->pMeta->uid; pCol->colId = pSchema->colId; pCol->colType = COLUMN_TYPE_COLUMN; - strcpy(pCol->tableAlias, pTable->table.tableAlias); - strcpy(pCol->tableName, pTable->table.tableName); + tstrncpy(pCol->tableAlias, pTable->table.tableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableName, pTable->table.tableName, TSDB_TABLE_NAME_LEN); pCol->isPk = pSchema->flags & COL_IS_KEY; pCol->tableHasPk = hasPkInTable(pTable->pMeta); pCol->numOfPKs = pTable->pMeta->tableInfo.numOfPKs; - strcpy(pCol->colName, pSchema->name); + tstrncpy(pCol->colName, pSchema->name, TSDB_COL_NAME_LEN); return (SNode*)pCol; } @@ -392,8 +392,8 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT pScan->scanRange = TSWINDOW_INITIALIZER; pScan->tableName.type = TSDB_TABLE_NAME_T; pScan->tableName.acctId = pCxt->pPlanCxt->acctId; - strcpy(pScan->tableName.dbname, pRealTable->table.dbName); - strcpy(pScan->tableName.tname, pRealTable->table.tableName); + tstrncpy(pScan->tableName.dbname, pRealTable->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pScan->tableName.tname, pRealTable->table.tableName, TSDB_TABLE_NAME_LEN); pScan->showRewrite = pCxt->pPlanCxt->showRewrite; pScan->ratio = pRealTable->ratio; pScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; @@ -785,12 +785,12 @@ static int32_t addWinJoinPrimKeyToAggFuncs(SSelectStmt* pSelect, SNodeList** pLi } SSchema* pColSchema = &pProbeTable->pMeta->schema[0]; - strcpy(pCol->dbName, pProbeTable->table.dbName); - strcpy(pCol->tableAlias, pProbeTable->table.tableAlias); - strcpy(pCol->tableName, pProbeTable->table.tableName); - strcpy(pCol->colName, pColSchema->name); - strcpy(pCol->node.aliasName, pColSchema->name); - strcpy(pCol->node.userAlias, pColSchema->name); + tstrncpy(pCol->dbName, pProbeTable->table.dbName, TSDB_DB_NAME_LEN); + tstrncpy(pCol->tableAlias, pProbeTable->table.tableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableName, pProbeTable->table.tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->colName, pColSchema->name, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.aliasName, pColSchema->name, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.userAlias, pColSchema->name, TSDB_COL_NAME_LEN); pCol->tableId = pProbeTable->pMeta->uid; pCol->tableType = pProbeTable->pMeta->tableType; pCol->colId = pColSchema->colId; @@ -1557,7 +1557,7 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect if (TSDB_CODE_SUCCESS == code) { pSort->pSortKeys = NULL; code = nodesCloneList(pSelect->pOrderByList, &pSort->pSortKeys); - if (NULL != pSort->pSortKeys) { + if (TSDB_CODE_SUCCESS == code) { SNode* pNode = NULL; SOrderByExprNode* firstSortKey = (SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0); if (isPrimaryKeySort(pSelect->pOrderByList)) pSort->node.outputTsOrder = firstSortKey->order; @@ -1624,10 +1624,7 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel pProject->pProjections = NULL; code = nodesCloneList(pSelect->pProjectionList, &pProject->pProjections); - if (NULL == pProject->pProjections) { - code = code; - } - strcpy(pProject->stmtName, pSelect->stmtName); + tstrncpy(pProject->stmtName, pSelect->stmtName, TSDB_TABLE_NAME_LEN); if (TSDB_CODE_SUCCESS == code) { code = createColumnByProjections(pCxt, pSelect->stmtName, pSelect->pProjectionList, &pProject->node.pTargets); @@ -2117,7 +2114,7 @@ static int32_t createVnodeModifLogicNodeByDelete(SLogicPlanContext* pCxt, SDelet pModify->tableId = pRealTable->pMeta->uid; pModify->tableType = pRealTable->pMeta->tableType; snprintf(pModify->tableName, sizeof(pModify->tableName), "%s", pRealTable->table.tableName); - strcpy(pModify->tsColName, pRealTable->pMeta->schema->name); + tstrncpy(pModify->tsColName, pRealTable->pMeta->schema->name, TSDB_COL_NAME_LEN); pModify->deleteTimeRange = pDelete->timeRange; pModify->pAffectedRows = NULL; code = nodesCloneNode(pDelete->pCountFunc, &pModify->pAffectedRows); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 9db72d9ba4..11cf926081 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -188,7 +188,7 @@ static EDealRes optRebuildTbanme(SNode** pNode, void* pContext) { *(int32_t*)pContext = code; return DEAL_RES_ERROR; } - strcpy(pFunc->functionName, "tbname"); + tstrncpy(pFunc->functionName, "tbname", TSDB_FUNC_NAME_LEN); pFunc->funcType = FUNCTION_TYPE_TBNAME; pFunc->node.resType = ((SColumnNode*)*pNode)->node.resType; nodesDestroyNode(*pNode); @@ -1302,9 +1302,7 @@ static int32_t pdcJoinAddPreFilterColsToTarget(SOptimizeContext* pCxt, SJoinLogi SNodeList* pCondCols = NULL; code = nodesMakeList(&pCondCols); SNodeList* pTargets = NULL; - if (NULL == pCondCols) { - code = code; - } else { + if (TSDB_CODE_SUCCESS == code) { code = nodesCollectColumnsFromNode(pJoin->pColOnCond, NULL, COLLECT_COL_TYPE_ALL, &pCondCols); } if (TSDB_CODE_SUCCESS == code) { @@ -2928,7 +2926,7 @@ static int32_t smaIndexOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* return TSDB_CODE_SUCCESS; } -static bool smaIndexOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pWindow, STableIndexInfo* pIndex) { +static bool smaIndexOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pWindow, STableIndexInfo* pIndex, void* tz) { if (pWindow->interval != pIndex->interval || pWindow->intervalUnit != pIndex->intervalUnit || pWindow->offset != pIndex->offset || pWindow->sliding != pIndex->sliding || pWindow->slidingUnit != pIndex->slidingUnit) { @@ -2941,6 +2939,7 @@ static bool smaIndexOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pW .offsetUnit = TIME_UNIT_MILLISECOND, .sliding = pIndex->sliding, .slidingUnit = pIndex->slidingUnit, + .timezone = tz, .precision = pScan->node.precision}; return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval)) && (pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval)); @@ -2958,9 +2957,9 @@ static int32_t smaIndexOptCreateSmaCol(SNode* pFunc, uint64_t tableId, int32_t c pCol->tableType = TSDB_SUPER_TABLE; pCol->colId = colId; pCol->colType = COLUMN_TYPE_COLUMN; - strcpy(pCol->colName, ((SExprNode*)pFunc)->aliasName); + tstrncpy(pCol->colName, ((SExprNode*)pFunc)->aliasName, TSDB_COL_NAME_LEN); pCol->node.resType = ((SExprNode*)pFunc)->resType; - strcpy(pCol->node.aliasName, ((SExprNode*)pFunc)->aliasName); + tstrncpy(pCol->node.aliasName, ((SExprNode*)pFunc)->aliasName, TSDB_COL_NAME_LEN); *ppNode = pCol; return code; } @@ -3029,7 +3028,7 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo } SExprNode exprNode; exprNode.resType = ((SExprNode*)pWsNode)->resType; - sprintf(exprNode.aliasName, "#expr_%d", index + 1); + snprintf(exprNode.aliasName, TSDB_COL_NAME_LEN, "#expr_%d", index + 1); SColumnNode* pkNode = NULL; code = smaIndexOptCreateSmaCol((SNode*)&exprNode, tableId, PRIMARYKEY_TIMESTAMP_COL_ID, &pkNode); if (TSDB_CODE_SUCCESS != code) { @@ -3051,9 +3050,9 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo return code; } -static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols) { +static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols, void* tz) { SWindowLogicNode* pWindow = (SWindowLogicNode*)pScan->node.pParent; - if (!smaIndexOptEqualInterval(pScan, pWindow, pIndex)) { + if (!smaIndexOptEqualInterval(pScan, pWindow, pIndex, tz)) { return TSDB_CODE_SUCCESS; } SNodeList* pSmaFuncs = NULL; @@ -3084,7 +3083,7 @@ static int32_t smaIndexOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogi for (int32_t i = 0; i < nindexes; ++i) { STableIndexInfo* pIndex = taosArrayGet(pScan->pSmaIndexes, i); SNodeList* pSmaCols = NULL; - code = smaIndexOptCouldApplyIndex(pScan, pIndex, &pSmaCols); + code = smaIndexOptCouldApplyIndex(pScan, pIndex, &pSmaCols, pCxt->pPlanCxt->timezone); if (TSDB_CODE_SUCCESS == code && NULL != pSmaCols) { code = smaIndexOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols); pCxt->optimized = true; @@ -3199,7 +3198,7 @@ static void partTagsSetAlias(char* pAlias, const char* pTableAlias, const char* int32_t len = tsnprintf(name, TSDB_COL_FNAME_LEN, "%s.%s", pTableAlias, pColName); (void)taosHashBinary(name, len); - strncpy(pAlias, name, TSDB_COL_NAME_LEN - 1); + tstrncpy(pAlias, name, TSDB_COL_NAME_LEN); } static int32_t partTagsCreateWrapperFunc(const char* pFuncName, SNode* pNode, SFunctionNode** ppNode) { @@ -3217,7 +3216,7 @@ static int32_t partTagsCreateWrapperFunc(const char* pFuncName, SNode* pNode, SF SColumnNode* pCol = (SColumnNode*)pNode; partTagsSetAlias(pFunc->node.aliasName, pCol->tableAlias, pCol->colName); } else { - strcpy(pFunc->node.aliasName, ((SExprNode*)pNode)->aliasName); + tstrncpy(pFunc->node.aliasName, ((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN); } code = nodesCloneNode(pNode, &pNew); if (TSDB_CODE_SUCCESS == code) { @@ -3506,7 +3505,7 @@ static EDealRes eliminateProjOptRewriteScanTableAlias(SNode* pNode, void* pConte if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; RewriteTableAliasCxt* pCtx = (RewriteTableAliasCxt*)pContext; - strncpy(pCol->tableAlias, pCtx->newTableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableAlias, pCtx->newTableAlias, TSDB_TABLE_NAME_LEN); } return DEAL_RES_CONTINUE; } @@ -3762,7 +3761,7 @@ static int32_t rewriteTailOptCreateProjectExpr(SFunctionNode* pFunc, SNode** ppN if (NULL == pExpr) { return code; } - strcpy(((SExprNode*)pExpr)->aliasName, pFunc->node.aliasName); + tstrncpy(((SExprNode*)pExpr)->aliasName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); *ppNode = pExpr; return code; } @@ -3918,15 +3917,15 @@ static int32_t rewriteUniqueOptCreateFirstFunc(SFunctionNode* pSelectValue, SNod return code; } - strcpy(pFunc->functionName, "first"); + tstrncpy(pFunc->functionName, "first", TSDB_FUNC_NAME_LEN); if (NULL != pSelectValue) { - strcpy(pFunc->node.aliasName, pSelectValue->node.aliasName); + tstrncpy(pFunc->node.aliasName, pSelectValue->node.aliasName, TSDB_COL_NAME_LEN); } else { int64_t pointer = (int64_t)pFunc; char name[TSDB_FUNC_NAME_LEN + TSDB_POINTER_PRINT_BYTES + TSDB_NAME_DELIMITER_LEN + 1] = {0}; int32_t len = tsnprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pFunc->functionName, pointer); (void)taosHashBinary(name, len); - strncpy(pFunc->node.aliasName, name, TSDB_COL_NAME_LEN - 1); + tstrncpy(pFunc->node.aliasName, name, TSDB_COL_NAME_LEN); } SNode* pNew = NULL; code = nodesCloneNode(pCol, &pNew); @@ -4024,15 +4023,15 @@ static int32_t rewriteUniqueOptCreateProjectCol(SFunctionNode* pFunc, SNode** pp if (FUNCTION_TYPE_UNIQUE == pFunc->funcType) { SExprNode* pExpr = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0); if (QUERY_NODE_COLUMN == nodeType(pExpr)) { - strcpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias); - strcpy(pCol->colName, ((SColumnNode*)pExpr)->colName); + tstrncpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->colName, ((SColumnNode*)pExpr)->colName, TSDB_COL_NAME_LEN); } else { - strcpy(pCol->colName, pExpr->aliasName); + tstrncpy(pCol->colName, pExpr->aliasName, TSDB_COL_NAME_LEN); } } else { - strcpy(pCol->colName, pFunc->node.aliasName); + tstrncpy(pCol->colName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); } - strcpy(pCol->node.aliasName, pFunc->node.aliasName); + tstrncpy(pCol->node.aliasName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); *ppNode = (SNode*)pCol; return code; } @@ -4407,7 +4406,7 @@ static int32_t lastRowScanBuildFuncTypes(SScanLogicNode* pScan, SColumnNode* pCo return terrno; } pFuncTypeParam->pCol->colId = pColNode->colId; - strcpy(pFuncTypeParam->pCol->name, pColNode->colName); + tstrncpy(pFuncTypeParam->pCol->name, pColNode->colName, TSDB_COL_NAME_LEN); if (NULL == taosArrayPush(pScan->pFuncTypes, pFuncTypeParam)) { taosMemoryFree(pFuncTypeParam); return terrno; @@ -4484,15 +4483,15 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic if (TSDB_CODE_SUCCESS != code) { break; } - sprintf(((SColumnNode*)newColNode)->colName, "#dup_col.%p", newColNode); - sprintf(((SColumnNode*)pParamNode)->colName, "#dup_col.%p", newColNode); + snprintf(((SColumnNode*)newColNode)->colName, TSDB_COL_NAME_LEN, "#dup_col.%p", newColNode); + snprintf(((SColumnNode*)pParamNode)->colName, TSDB_COL_NAME_LEN, "#dup_col.%p", newColNode); if (FUNCTION_TYPE_LAST_ROW == funcType && ((SColumnNode*)pParamNode)->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { if (!adjLastRowTsColName) { adjLastRowTsColName = true; - strncpy(tsColName, ((SColumnNode*)pParamNode)->colName, TSDB_COL_NAME_LEN); + tstrncpy(tsColName, ((SColumnNode*)pParamNode)->colName, TSDB_COL_NAME_LEN); } else { - strncpy(((SColumnNode*)pParamNode)->colName, tsColName, TSDB_COL_NAME_LEN); + tstrncpy(((SColumnNode*)pParamNode)->colName, tsColName, TSDB_COL_NAME_LEN); nodesDestroyNode(newColNode); continue; } @@ -4616,7 +4615,7 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic ((cxt.pLastCols->length == 1 && nodesEqualNode((SNode*)pPKTsCol, nodesListGetNode(cxt.pLastCols, 0))) || (pScan->node.pTargets->length == 2 && cxt.pkBytes > 0))) { // when select last(ts),tbname,ts from ..., we add another ts to targets - sprintf(pPKTsCol->colName, "#sel_val.%p", pPKTsCol); + snprintf(pPKTsCol->colName, TSDB_COL_NAME_LEN, "#sel_val.%p", pPKTsCol); SNode* pNew = NULL; code = nodesCloneNode((SNode*)pPKTsCol, &pNew); if (TSDB_CODE_SUCCESS == code) { @@ -4633,7 +4632,7 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic if (pNonPKCol && cxt.pLastCols->length == 1 && nodesEqualNode((SNode*)pNonPKCol, nodesListGetNode(cxt.pLastCols, 0))) { // when select last(c1), c1 from ..., we add c1 to targets - sprintf(pNonPKCol->colName, "#sel_val.%p", pNonPKCol); + snprintf(pNonPKCol->colName, TSDB_COL_NAME_LEN, "#sel_val.%p", pNonPKCol); SNode* pNew = NULL; code = nodesCloneNode((SNode*)pNonPKCol, &pNew); if (TSDB_CODE_SUCCESS == code) { @@ -5405,10 +5404,10 @@ static bool tbCntScanOptIsEligibleLogicCond(STbCntScanOptInfo* pInfo, SLogicCond } if (!hasDbCond && 0 == strcmp(pCol->colName, "db_name")) { hasDbCond = true; - strcpy(pInfo->table.dbname, pVal->literal); + tstrncpy(pInfo->table.dbname, pVal->literal, TSDB_DB_NAME_LEN); } else if (!hasStbCond && 0 == strcmp(pCol->colName, "stable_name")) { hasStbCond = true; - strcpy(pInfo->table.tname, pVal->literal); + tstrncpy(pInfo->table.tname, pVal->literal, TSDB_TABLE_NAME_LEN); } else { return false; } @@ -5473,8 +5472,8 @@ static int32_t tbCntScanOptCreateTableCountFunc(SNode** ppNode) { if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "_table_count"); - strcpy(pFunc->node.aliasName, "_table_count"); + tstrncpy(pFunc->functionName, "_table_count", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, "_table_count", TSDB_COL_NAME_LEN); code = fmGetFuncInfo(pFunc, NULL, 0); if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode((SNode*)pFunc); @@ -5486,8 +5485,8 @@ static int32_t tbCntScanOptCreateTableCountFunc(SNode** ppNode) { static int32_t tbCntScanOptRewriteScan(STbCntScanOptInfo* pInfo) { pInfo->pScan->scanType = SCAN_TYPE_TABLE_COUNT; - strcpy(pInfo->pScan->tableName.dbname, pInfo->table.dbname); - strcpy(pInfo->pScan->tableName.tname, pInfo->table.tname); + tstrncpy(pInfo->pScan->tableName.dbname, pInfo->table.dbname, TSDB_DB_NAME_LEN); + tstrncpy(pInfo->pScan->tableName.tname, pInfo->table.tname, TSDB_TABLE_NAME_LEN); NODES_DESTORY_LIST(pInfo->pScan->node.pTargets); NODES_DESTORY_LIST(pInfo->pScan->pScanCols); NODES_DESTORY_NODE(pInfo->pScan->node.pConditions); @@ -5537,8 +5536,8 @@ static int32_t tbCntScanOptCreateSumFunc(SFunctionNode* pCntFunc, SNode* pParam, if (NULL == pFunc) { return code; } - strcpy(pFunc->functionName, "sum"); - strcpy(pFunc->node.aliasName, pCntFunc->node.aliasName); + tstrncpy(pFunc->functionName, "sum", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, pCntFunc->node.aliasName, TSDB_COL_NAME_LEN); code = createColumnByRewriteExpr(pParam, &pFunc->pParameterList); if (TSDB_CODE_SUCCESS == code) { code = fmGetFuncInfo(pFunc, NULL, 0); @@ -6315,29 +6314,22 @@ static int32_t stbJoinOptCreateDynQueryCtrlNode(SLogicNode* pRoot, SLogicNode* p if (TSDB_CODE_SUCCESS == code) { pDynCtrl->node.pChildren = NULL; code = nodesMakeList(&pDynCtrl->node.pChildren); - if (NULL == pDynCtrl->node.pChildren) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { pDynCtrl->stbJoin.pVgList = NULL; code = nodesMakeList(&pDynCtrl->stbJoin.pVgList); - if (NULL == pDynCtrl->stbJoin.pVgList) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { pDynCtrl->stbJoin.pUidList = NULL; code = nodesMakeList(&pDynCtrl->stbJoin.pUidList); - if (NULL == pDynCtrl->stbJoin.pUidList) { - code = code; - } } SJoinLogicNode* pHJoin = (SJoinLogicNode*)pPrev; - code = nodesListStrictAppend(pDynCtrl->stbJoin.pUidList, nodesListGetNode(pHJoin->node.pTargets, 0)); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pDynCtrl->stbJoin.pUidList, nodesListGetNode(pHJoin->node.pTargets, 0)); + } if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pDynCtrl->stbJoin.pUidList, nodesListGetNode(pHJoin->node.pTargets, 2)); } @@ -6356,9 +6348,6 @@ static int32_t stbJoinOptCreateDynQueryCtrlNode(SLogicNode* pRoot, SLogicNode* p if (TSDB_CODE_SUCCESS == code) { pDynCtrl->node.pTargets = NULL; code = nodesCloneList(pPost->pTargets, &pDynCtrl->node.pTargets); - if (!pDynCtrl->node.pTargets) { - code = code; - } } } @@ -6777,7 +6766,7 @@ typedef struct STSMAOptCtx { SNodeList** ppParentTsmaSubplans; } STSMAOptCtx; -static int32_t fillTSMAOptCtx(STSMAOptCtx* pTsmaOptCtx, SScanLogicNode* pScan) { +static int32_t fillTSMAOptCtx(STSMAOptCtx* pTsmaOptCtx, SScanLogicNode* pScan, void* tz) { int32_t code = 0; pTsmaOptCtx->pScan = pScan; pTsmaOptCtx->pParent = pScan->node.pParent; @@ -6797,7 +6786,7 @@ static int32_t fillTSMAOptCtx(STSMAOptCtx* pTsmaOptCtx, SScanLogicNode* pScan) { pTsmaOptCtx->queryInterval->sliding = pWindow->sliding; pTsmaOptCtx->queryInterval->slidingUnit = pWindow->slidingUnit; pTsmaOptCtx->queryInterval->precision = pWindow->node.precision; - pTsmaOptCtx->queryInterval->tz = tsTimezone; + pTsmaOptCtx->queryInterval->timezone = tz; pTsmaOptCtx->pAggFuncs = pWindow->pFuncs; pTsmaOptCtx->ppParentTsmaSubplans = &pWindow->pTsmaSubplans; } else { @@ -6988,7 +6977,7 @@ static int32_t tsmaInfoCompWithIntervalDesc(const void* pLeft, const void* pRigh return 0; } -static void tsmaOptInitIntervalFromTsma(SInterval* pInterval, const STableTSMAInfo* pTsma, int8_t precision) { +static void tsmaOptInitIntervalFromTsma(SInterval* pInterval, const STableTSMAInfo* pTsma, int8_t precision, void* tz) { pInterval->interval = pTsma->interval; pInterval->intervalUnit = pTsma->unit; pInterval->sliding = pTsma->interval; @@ -6996,15 +6985,16 @@ static void tsmaOptInitIntervalFromTsma(SInterval* pInterval, const STableTSMAIn pInterval->offset = 0; pInterval->offsetUnit = pTsma->unit; pInterval->precision = precision; + pInterval->timezone = tz; } static const STSMAOptUsefulTsma* tsmaOptFindUsefulTsma(const SArray* pUsefulTsmas, int32_t startIdx, int64_t startAlignInterval, int64_t endAlignInterval, - int8_t precision) { + int8_t precision, void* tz) { SInterval tsmaInterval; for (int32_t i = startIdx; i < pUsefulTsmas->size; ++i) { const STSMAOptUsefulTsma* pUsefulTsma = taosArrayGet(pUsefulTsmas, i); - tsmaOptInitIntervalFromTsma(&tsmaInterval, pUsefulTsma->pTsma, precision); + tsmaOptInitIntervalFromTsma(&tsmaInterval, pUsefulTsma->pTsma, precision, tz); if (taosTimeTruncate(startAlignInterval, &tsmaInterval) == startAlignInterval && taosTimeTruncate(endAlignInterval, &tsmaInterval) == endAlignInterval) { return pUsefulTsma; @@ -7013,7 +7003,7 @@ static const STSMAOptUsefulTsma* tsmaOptFindUsefulTsma(const SArray* pUsefulTsma return NULL; } -static int32_t tsmaOptSplitWindows(STSMAOptCtx* pTsmaOptCtx, const STimeWindow* pScanRange) { +static int32_t tsmaOptSplitWindows(STSMAOptCtx* pTsmaOptCtx, const STimeWindow* pScanRange, void* tz) { bool needTailWindow = false; bool isSkeyAlignedWithTsma = true, isEkeyAlignedWithTsma = true; int32_t code = 0; @@ -7029,17 +7019,17 @@ static int32_t tsmaOptSplitWindows(STSMAOptCtx* pTsmaOptCtx, const STimeWindow* if (pScanRange->ekey <= pScanRange->skey) return code; if (!pInterval) { - tsmaOptInitIntervalFromTsma(&interval, pTsma, pTsmaOptCtx->precision); + tsmaOptInitIntervalFromTsma(&interval, pTsma, pTsmaOptCtx->precision, tz); pInterval = &interval; } - tsmaOptInitIntervalFromTsma(&tsmaInterval, pTsma, pTsmaOptCtx->precision); + tsmaOptInitIntervalFromTsma(&tsmaInterval, pTsma, pTsmaOptCtx->precision, tz); // check for head windows if (pScanRange->skey != TSKEY_MIN) { startOfSkeyFirstWin = taosTimeTruncate(pScanRange->skey, pInterval); endOfSkeyFirstWin = - taosTimeAdd(startOfSkeyFirstWin, pInterval->interval, pInterval->intervalUnit, pTsmaOptCtx->precision); + taosTimeAdd(startOfSkeyFirstWin, pInterval->interval, pInterval->intervalUnit, pTsmaOptCtx->precision, tz); isSkeyAlignedWithTsma = taosTimeTruncate(pScanRange->skey, &tsmaInterval) == pScanRange->skey; } else { endOfSkeyFirstWin = TSKEY_MIN; @@ -7049,7 +7039,7 @@ static int32_t tsmaOptSplitWindows(STSMAOptCtx* pTsmaOptCtx, const STimeWindow* if (pScanRange->ekey != TSKEY_MAX) { startOfEkeyFirstWin = taosTimeTruncate(pScanRange->ekey, pInterval); endOfEkeyFirstWin = - taosTimeAdd(startOfEkeyFirstWin, pInterval->interval, pInterval->intervalUnit, pTsmaOptCtx->precision); + taosTimeAdd(startOfEkeyFirstWin, pInterval->interval, pInterval->intervalUnit, pTsmaOptCtx->precision, tz); isEkeyAlignedWithTsma = taosTimeTruncate(pScanRange->ekey + 1, &tsmaInterval) == (pScanRange->ekey + 1); if (startOfEkeyFirstWin > startOfSkeyFirstWin) { needTailWindow = true; @@ -7060,9 +7050,9 @@ static int32_t tsmaOptSplitWindows(STSMAOptCtx* pTsmaOptCtx, const STimeWindow* if (!isSkeyAlignedWithTsma) { scanRange.ekey = TMIN( scanRange.ekey, - taosTimeAdd(startOfSkeyFirstWin, pInterval->interval * 1, pInterval->intervalUnit, pTsmaOptCtx->precision) - 1); + taosTimeAdd(startOfSkeyFirstWin, pInterval->interval * 1, pInterval->intervalUnit, pTsmaOptCtx->precision, tz) - 1); const STSMAOptUsefulTsma* pTsmaFound = - tsmaOptFindUsefulTsma(pTsmaOptCtx->pUsefulTsmas, 1, scanRange.skey, scanRange.ekey + 1, pTsmaOptCtx->precision); + tsmaOptFindUsefulTsma(pTsmaOptCtx->pUsefulTsmas, 1, scanRange.skey, scanRange.ekey + 1, pTsmaOptCtx->precision, tz); STSMAOptUsefulTsma usefulTsma = {.pTsma = pTsmaFound ? pTsmaFound->pTsma : NULL, .scanRange = scanRange, .pTsmaScanCols = pTsmaFound ? pTsmaFound->pTsmaScanCols : NULL}; @@ -7087,7 +7077,7 @@ static int32_t tsmaOptSplitWindows(STSMAOptCtx* pTsmaOptCtx, const STimeWindow* scanRange.ekey = pScanRange->ekey; const STSMAOptUsefulTsma* pTsmaFound = tsmaOptFindUsefulTsma(pTsmaOptCtx->pUsefulTsmas, 1, scanRange.skey - startOfEkeyFirstWin, - scanRange.ekey + 1 - startOfEkeyFirstWin, pTsmaOptCtx->precision); + scanRange.ekey + 1 - startOfEkeyFirstWin, pTsmaOptCtx->precision, tz); STSMAOptUsefulTsma usefulTsma = {.pTsma = pTsmaFound ? pTsmaFound->pTsma : NULL, .scanRange = scanRange, .pTsmaScanCols = pTsmaFound ? pTsmaFound->pTsmaScanCols : NULL}; @@ -7119,10 +7109,9 @@ int32_t tsmaOptCreateTsmaScanCols(const STSMAOptUsefulTsma* pTsma, const SNodeLi pCol->tableType = TSDB_SUPER_TABLE; pCol->tableId = pTsma->targetTbUid; pCol->colType = COLUMN_TYPE_COLUMN; - strcpy(pCol->tableName, pTsma->targetTbName); - strcpy(pCol->dbName, pTsma->pTsma->targetDbFName); - strcpy(pCol->colName, pFunc->node.aliasName); - strcpy(pCol->node.aliasName, pFunc->node.aliasName); + tstrncpy(pCol->tableName, pTsma->targetTbName, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->colName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.aliasName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); pCol->node.resType.type = TSDB_DATA_TYPE_BINARY; code = nodesListMakeStrictAppend(&pScanCols, (SNode*)pCol); } @@ -7146,8 +7135,8 @@ static int32_t tsmaOptRewriteTag(const STSMAOptCtx* pTsmaOptCtx, const STSMAOptU for (int32_t i = 0; i < pTsma->pTsma->pTags->size; ++i) { const SSchema* pSchema = taosArrayGet(pTsma->pTsma->pTags, i); if (strcmp(pTagCol->colName, pSchema->name) == 0) { - strcpy(pTagCol->tableName, pTsma->targetTbName); - strcpy(pTagCol->tableAlias, pTsma->targetTbName); + tstrncpy(pTagCol->tableName, pTsma->targetTbName, TSDB_TABLE_NAME_LEN); + tstrncpy(pTagCol->tableAlias, pTsma->targetTbName, TSDB_TABLE_NAME_LEN); pTagCol->tableId = pTsma->targetTbUid; pTagCol->tableType = TSDB_SUPER_TABLE; pTagCol->colId = pSchema->colId; @@ -7172,8 +7161,8 @@ static int32_t tsmaOptRewriteTbname(const STSMAOptCtx* pTsmaOptCtx, SNode** pTbN nodesDestroyNode(*pTbNameNode); SColumnNode* pCol = (SColumnNode*)pRewrittenFunc; const SSchema* pSchema = taosArrayGet(pTsma->pTsma->pTags, pTsma->pTsma->pTags->size - 1); - strcpy(pCol->tableName, pTsma->targetTbName); - strcpy(pCol->tableAlias, pTsma->targetTbName); + tstrncpy(pCol->tableName, pTsma->targetTbName, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableAlias, pTsma->targetTbName, TSDB_TABLE_NAME_LEN); pCol->tableId = pTsma->targetTbUid; pCol->tableType = TSDB_SUPER_TABLE; pCol->colId = pSchema->colId; @@ -7298,7 +7287,7 @@ static int32_t tsmaOptRewriteScan(STSMAOptCtx* pTsmaOptCtx, SScanLogicNode* pNew if (code == TSDB_CODE_SUCCESS) { pNewScan->stableId = pTsma->pTsma->destTbUid; pNewScan->tableId = pTsma->targetTbUid; - strcpy(pNewScan->tableName.tname, pTsma->targetTbName); + tstrncpy(pNewScan->tableName.tname, pTsma->targetTbName, TSDB_TABLE_NAME_LEN); } if (code == TSDB_CODE_SUCCESS) { code = tsmaOptRewriteNodeList(pNewScan->pScanPseudoCols, pTsmaOptCtx, pTsma, true, true); @@ -7351,12 +7340,12 @@ static int32_t tsmaOptCreateWStart(int8_t precision, SFunctionNode** pWStartOut) if (NULL == pWStart) { return code; } - strcpy(pWStart->functionName, "_wstart"); + tstrncpy(pWStart->functionName, "_wstart", TSDB_FUNC_NAME_LEN); int64_t pointer = (int64_t)pWStart; char name[TSDB_COL_NAME_LEN + TSDB_POINTER_PRINT_BYTES + TSDB_NAME_DELIMITER_LEN + 1] = {0}; int32_t len = tsnprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pWStart->functionName, pointer); (void)taosHashBinary(name, len); - strncpy(pWStart->node.aliasName, name, TSDB_COL_NAME_LEN - 1); + tstrncpy(pWStart->node.aliasName, name, TSDB_COL_NAME_LEN); pWStart->node.resType.precision = precision; code = fmGetFuncInfo(pWStart, NULL, 0); @@ -7459,12 +7448,12 @@ static int32_t tsmaOptGeneratePlan(STSMAOptCtx* pTsmaOptCtx) { for (int32_t j = 0; j < pTsmaOptCtx->pScan->pTsmas->size; ++j) { if (taosArrayGetP(pTsmaOptCtx->pScan->pTsmas, j) == pTsma->pTsma) { const STsmaTargetTbInfo* ptbInfo = taosArrayGet(pTsmaOptCtx->pScan->pTsmaTargetTbInfo, j); - strcpy(pTsma->targetTbName, ptbInfo->tableName); + tstrncpy(pTsma->targetTbName, ptbInfo->tableName, TSDB_TABLE_NAME_LEN); pTsma->targetTbUid = ptbInfo->uid; } } } else { - strcpy(pTsma->targetTbName, pTsma->pTsma->targetTb); + tstrncpy(pTsma->targetTbName, pTsma->pTsma->targetTb, TSDB_TABLE_NAME_LEN); pTsma->targetTbUid = pTsma->pTsma->destTbUid; } } @@ -7526,7 +7515,7 @@ static int32_t tsmaOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan SLogicNode* pRootNode = getLogicNodeRootNode((SLogicNode*)pScan); if (getOptHint(pRootNode->pHint, HINT_SKIP_TSMA)) return code; - code = fillTSMAOptCtx(&tsmaOptCtx, pScan); + code = fillTSMAOptCtx(&tsmaOptCtx, pScan, pCxt->pPlanCxt->timezone); if (code == TSDB_CODE_SUCCESS) { // 1. extract useful tsmas code = tsmaOptFilterTsmas(&tsmaOptCtx); @@ -7535,7 +7524,7 @@ static int32_t tsmaOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan // 2. sort useful tsmas with interval taosArraySort(tsmaOptCtx.pUsefulTsmas, tsmaInfoCompWithIntervalDesc); // 3. split windows - code = tsmaOptSplitWindows(&tsmaOptCtx, tsmaOptCtx.pTimeRange); + code = tsmaOptSplitWindows(&tsmaOptCtx, tsmaOptCtx.pTimeRange, pCxt->pPlanCxt->timezone); if (TSDB_CODE_SUCCESS == code && tsmaOptIsUsingTsmas(&tsmaOptCtx)) { // 4. create logic plan code = tsmaOptGeneratePlan(&tsmaOptCtx); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 761dd42425..c60024b323 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -40,9 +40,9 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int if (!*ppKey) { return terrno; } - strcat(*ppKey, pStmtName); - strcat(*ppKey, "."); - strcat(*ppKey, pCol->node.aliasName); + TAOS_STRNCAT(*ppKey, pStmtName, TSDB_TABLE_NAME_LEN); + TAOS_STRNCAT(*ppKey, ".", 2); + TAOS_STRNCAT(*ppKey, pCol->node.aliasName, TSDB_COL_NAME_LEN); *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); return code; } else { @@ -50,7 +50,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int if (!*ppKey) { return terrno; } - strcat(*ppKey, pCol->node.aliasName); + TAOS_STRNCAT(*ppKey, pCol->node.aliasName, TSDB_COL_NAME_LEN); *pLen = strlen(*ppKey); return code; } @@ -60,7 +60,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int if (!*ppKey) { return terrno; } - strcat(*ppKey, pCol->colName); + TAOS_STRNCAT(*ppKey, pCol->colName, TSDB_COL_NAME_LEN); *pLen = strlen(*ppKey); return code; } @@ -69,9 +69,9 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int if (!*ppKey) { return terrno; } - strcat(*ppKey, pCol->tableAlias); - strcat(*ppKey, "."); - strcat(*ppKey, pCol->colName); + TAOS_STRNCAT(*ppKey, pCol->tableAlias, TSDB_TABLE_NAME_LEN); + TAOS_STRNCAT(*ppKey, ".", 2); + TAOS_STRNCAT(*ppKey, pCol->colName, TSDB_COL_NAME_LEN); *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); return code; } else if (QUERY_NODE_FUNCTION == nodeType(pNode)) { @@ -84,19 +84,20 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int if (!*ppKey) { return terrno; } - strcat(*ppKey, pStmtName); - strcat(*ppKey, "."); - strcat(*ppKey, ((SExprNode*)pNode)->aliasName); + TAOS_STRNCAT(*ppKey, pStmtName, TSDB_TABLE_NAME_LEN); + TAOS_STRNCAT(*ppKey, ".", 2); + TAOS_STRNCAT(*ppKey, ((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN); *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); return code; } - *ppKey = taosMemoryCalloc(1, strlen(pVal->literal) + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); + int32_t literalLen = strlen(pVal->literal); + *ppKey = taosMemoryCalloc(1, literalLen + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } - strcat(*ppKey, pVal->literal); - strcat(*ppKey, "."); - strcat(*ppKey, ((SExprNode*)pNode)->aliasName); + TAOS_STRNCAT(*ppKey, pVal->literal, literalLen); + TAOS_STRNCAT(*ppKey, ".", 2); + TAOS_STRNCAT(*ppKey, ((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN); *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); return code; } @@ -108,9 +109,9 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int if (!*ppKey) { return terrno; } - strcat(*ppKey, pStmtName); - strcat(*ppKey, "."); - strcat(*ppKey, ((SExprNode*)pNode)->aliasName); + TAOS_STRNCAT(*ppKey, pStmtName, TSDB_TABLE_NAME_LEN); + TAOS_STRNCAT(*ppKey, ".", 2); + TAOS_STRNCAT(*ppKey, ((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN); *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); return code; } @@ -119,7 +120,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int if (!*ppKey) { return terrno; } - strcat(*ppKey, ((SExprNode*)pNode)->aliasName); + TAOS_STRNCAT(*ppKey, ((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN); *pLen = strlen(*ppKey); return code; } @@ -228,7 +229,7 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD code = putSlotToHash(name, len, pDataBlockDesc->dataBlockId, slotId, pNode, pHash); if (TSDB_CODE_SUCCESS == code) { if (nodeType(pNode) == QUERY_NODE_COLUMN && ((SColumnNode*)pNode)->resIdx > 0) { - sprintf(name + strlen(name), "_%d", ((SColumnNode*)pNode)->resIdx); + snprintf(name + strlen(name), 16, "_%d", ((SColumnNode*)pNode)->resIdx); code = putSlotToHash(name, strlen(name), pDataBlockDesc->dataBlockId, slotId, pNode, pProjIdxDescHash); } } @@ -395,7 +396,7 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { } SSlotIndex* pIndex = NULL; if (((SColumnNode*)pNode)->projRefIdx > 0) { - sprintf(name + strlen(name), "_%d", ((SColumnNode*)pNode)->projRefIdx); + snprintf(name + strlen(name), 16, "_%d", ((SColumnNode*)pNode)->projRefIdx); pIndex = taosHashGet(pCxt->pLeftProjIdxHash, name, strlen(name)); if (!pIndex) { pIndex = taosHashGet(pCxt->pRightProdIdxHash, name, strlen(name)); @@ -566,9 +567,6 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS if (TSDB_CODE_SUCCESS == code && NULL != pScanLogicNode->pScanPseudoCols) { pScanPhysiNode->pScanPseudoCols = NULL; code = nodesCloneList(pScanLogicNode->pScanPseudoCols, &pScanPhysiNode->pScanPseudoCols); - if (NULL == pScanPhysiNode->pScanPseudoCols) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { @@ -588,9 +586,6 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS if (NULL != pScanLogicNode->pTagCond) { pSubplan->pTagCond = NULL; code = nodesCloneNode(pScanLogicNode->pTagCond, &pSubplan->pTagCond); - if (NULL == pSubplan->pTagCond) { - code = code; - } } } @@ -598,9 +593,6 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS if (NULL != pScanLogicNode->pTagIndexCond) { pSubplan->pTagIndexCond = NULL; code = nodesCloneNode(pScanLogicNode->pTagIndexCond, &pSubplan->pTagIndexCond); - if (NULL == pSubplan->pTagIndexCond) { - code = code; - } } } @@ -1691,11 +1683,11 @@ static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) SExprNode* pRewrittenExpr = (SExprNode*)pExpr; pCol->node.resType = pRewrittenExpr->resType; if ('\0' != pRewrittenExpr->aliasName[0]) { - strcpy(pCol->colName, pRewrittenExpr->aliasName); + tstrncpy(pCol->colName, pRewrittenExpr->aliasName, TSDB_COL_NAME_LEN); } else { snprintf(pRewrittenExpr->aliasName, sizeof(pRewrittenExpr->aliasName), "#expr_%d_%d", pCxt->planNodeId, pCxt->rewriteId); - strcpy(pCol->colName, pRewrittenExpr->aliasName); + tstrncpy(pCol->colName, pRewrittenExpr->aliasName, TSDB_COL_NAME_LEN); } nodesDestroyNode(*pNode); *pNode = (SNode*)pCol; @@ -1716,7 +1708,7 @@ static int32_t rewriteValueToOperator(SRewritePrecalcExprsCxt* pCxt, SNode** pNo } SValueNode* pVal = (SValueNode*)*pNode; pOper->node.resType = pVal->node.resType; - strcpy(pOper->node.aliasName, pVal->node.aliasName); + tstrncpy(pOper->node.aliasName, pVal->node.aliasName, TSDB_COL_NAME_LEN); pOper->opType = OP_TYPE_ASSIGN; pOper->pRight = *pNode; *pNode = (SNode*)pOper; @@ -1977,9 +1969,6 @@ static int32_t createInterpFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pCh pInterpFunc->rangeInterval = pFuncLogicNode->rangeInterval; pInterpFunc->rangeIntervalUnit = pFuncLogicNode->rangeIntervalUnit; code = nodesCloneNode(pFuncLogicNode->pFillValues, &pInterpFunc->pFillValues); - if (TSDB_CODE_SUCCESS != code) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { @@ -2895,7 +2884,7 @@ static int32_t createQueryInserter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNod pInserter->tableId = pModify->tableId; pInserter->stableId = pModify->stableId; pInserter->tableType = pModify->tableType; - strcpy(pInserter->tableName, pModify->tableName); + tstrncpy(pInserter->tableName, pModify->tableName, TSDB_TABLE_NAME_LEN); pInserter->explain = (QUERY_NODE_EXPLAIN_STMT == nodeType(pCxt->pPlanCxt->pAstRoot) ? true : false); if (pModify->pVgroupList) { pInserter->vgId = pModify->pVgroupList->vgroups[0].vgId; @@ -2907,9 +2896,6 @@ static int32_t createQueryInserter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNod if (TSDB_CODE_SUCCESS == code) { pInserter->sink.pInputDataBlockDesc = NULL; code = nodesCloneNode((SNode*)pSubplan->pNode->pOutputDataBlockDesc, (SNode**)&pInserter->sink.pInputDataBlockDesc); - if (NULL == pInserter->sink.pInputDataBlockDesc) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { @@ -2948,8 +2934,8 @@ static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pDeleter->tableId = pModify->tableId; pDeleter->tableType = pModify->tableType; - strcpy(pDeleter->tableFName, pModify->tableName); - strcpy(pDeleter->tsColName, pModify->tsColName); + tstrncpy(pDeleter->tableFName, pModify->tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pDeleter->tsColName, pModify->tsColName, TSDB_COL_NAME_LEN); pDeleter->deleteTimeRange = pModify->deleteTimeRange; code = setNodeSlotId(pCxt, pRoot->pOutputDataBlockDesc->dataBlockId, -1, pModify->pAffectedRows, @@ -2963,9 +2949,6 @@ static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* if (TSDB_CODE_SUCCESS == code) { pDeleter->sink.pInputDataBlockDesc = NULL; code = nodesCloneNode((SNode*)pRoot->pOutputDataBlockDesc, (SNode**)&pDeleter->sink.pInputDataBlockDesc); - if (NULL == pDeleter->sink.pInputDataBlockDesc) { - code = code; - } } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index e960c0ff5d..5c2d1efd7b 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -428,12 +428,12 @@ static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex, uint8_t pr if (NULL == pWStart) { return code; } - strcpy(pWStart->functionName, "_wstart"); + tstrncpy(pWStart->functionName, "_wstart", TSDB_FUNC_NAME_LEN); int64_t pointer = (int64_t)pWStart; char name[TSDB_COL_NAME_LEN + TSDB_POINTER_PRINT_BYTES + TSDB_NAME_DELIMITER_LEN + 1] = {0}; int32_t len = tsnprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pWStart->functionName, pointer); (void)taosHashBinary(name, len); - strncpy(pWStart->node.aliasName, name, TSDB_COL_NAME_LEN - 1); + tstrncpy(pWStart->node.aliasName, name, TSDB_COL_NAME_LEN); pWStart->node.resType.precision = precision; code = fmGetFuncInfo(pWStart, NULL, 0); @@ -460,12 +460,12 @@ static int32_t stbSplAppendWEnd(SWindowLogicNode* pWin, int32_t* pIndex) { if (NULL == pWEnd) { return code; } - strcpy(pWEnd->functionName, "_wend"); + tstrncpy(pWEnd->functionName, "_wend", TSDB_FUNC_NAME_LEN); int64_t pointer = (int64_t)pWEnd; char name[TSDB_COL_NAME_LEN + TSDB_POINTER_PRINT_BYTES + TSDB_NAME_DELIMITER_LEN + 1] = {0}; int32_t len = tsnprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pWEnd->functionName, pointer); (void)taosHashBinary(name, len); - strncpy(pWEnd->node.aliasName, name, TSDB_COL_NAME_LEN - 1); + tstrncpy(pWEnd->node.aliasName, name, TSDB_COL_NAME_LEN); code = fmGetFuncInfo(pWEnd, NULL, 0); if (TSDB_CODE_SUCCESS == code) { @@ -1135,8 +1135,8 @@ static int32_t stbSplAggNodeCreateMerge(SSplitContext* pCtx, SStableSplitInfo* p if (!nodesEqualNode(pParam, (SNode*)pCol)) continue; // use the colName of group_key func to make sure finding the right slot id for merge keys. - strcpy(pCol->colName, pFunc->node.aliasName); - strcpy(pCol->node.aliasName, pFunc->node.aliasName); + tstrncpy(pCol->colName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.aliasName, pFunc->node.aliasName, TSDB_COL_NAME_LEN); memset(pCol->tableAlias, 0, TSDB_TABLE_NAME_LEN); break; } @@ -1265,15 +1265,15 @@ static int32_t stbSplCreateColumnNode(SExprNode* pExpr, SNode** ppNode) { return code; } if (QUERY_NODE_COLUMN == nodeType(pExpr)) { - strcpy(pCol->dbName, ((SColumnNode*)pExpr)->dbName); - strcpy(pCol->tableName, ((SColumnNode*)pExpr)->tableName); - strcpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias); - strcpy(pCol->colName, ((SColumnNode*)pExpr)->colName); + tstrncpy(pCol->dbName, ((SColumnNode*)pExpr)->dbName, TSDB_DB_NAME_LEN); + tstrncpy(pCol->tableName, ((SColumnNode*)pExpr)->tableName, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->colName, ((SColumnNode*)pExpr)->colName, TSDB_COL_NAME_LEN); } else { - strcpy(pCol->colName, pExpr->aliasName); + tstrncpy(pCol->colName, pExpr->aliasName, TSDB_COL_NAME_LEN); } - strcpy(pCol->node.aliasName, pExpr->aliasName); - strcpy(pCol->node.userAlias, pExpr->userAlias); + tstrncpy(pCol->node.aliasName, pExpr->aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pCol->node.userAlias, pExpr->userAlias, TSDB_COL_NAME_LEN); pCol->node.resType = pExpr->resType; *ppNode = (SNode*)pCol; return code; @@ -1358,9 +1358,6 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOut int32_t code = TSDB_CODE_SUCCESS; SSortLogicNode* pPartSort = NULL; code = nodesCloneNode((SNode*)pSort, (SNode**)&pPartSort); - if (NULL == pPartSort) { - code = code; - } SNodeList* pMergeKeys = NULL; if (TSDB_CODE_SUCCESS == code) { @@ -1541,9 +1538,6 @@ static int32_t stbSplCreateMergeScanNode(SScanLogicNode* pScan, SLogicNode** pOu int32_t code = TSDB_CODE_SUCCESS; SScanLogicNode* pMergeScan = NULL; code = nodesCloneNode((SNode*)pScan, (SNode**)&pMergeScan); - if (NULL == pMergeScan) { - code = code; - } SNodeList* pMergeKeys = NULL; if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index fd8670c23e..f03e2d8ab0 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -68,14 +68,14 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { return DEAL_RES_ERROR; } pCol->node.resType = pExpr->resType; - strcpy(pCol->colName, pExpr->aliasName); + tstrncpy(pCol->colName, pExpr->aliasName, TSDB_COL_NAME_LEN); if (QUERY_NODE_FUNCTION == nodeType(pNode)) { SFunctionNode* pFunc = (SFunctionNode*)pNode; if (pFunc->funcType == FUNCTION_TYPE_TBNAME) { SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); if (NULL != pVal) { - strcpy(pCol->tableAlias, pVal->literal); - strcpy(pCol->tableName, pVal->literal); + tstrncpy(pCol->tableAlias, pVal->literal, TSDB_TABLE_NAME_LEN); + tstrncpy(pCol->tableName, pVal->literal, TSDB_TABLE_NAME_LEN); } } } @@ -636,9 +636,9 @@ SFunctionNode* createGroupKeyAggFunc(SColumnNode* pGroupCol) { SFunctionNode* pFunc = NULL; int32_t code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); if (pFunc) { - strcpy(pFunc->functionName, "_group_key"); - strcpy(pFunc->node.aliasName, pGroupCol->node.aliasName); - strcpy(pFunc->node.userAlias, pGroupCol->node.userAlias); + tstrncpy(pFunc->functionName, "_group_key", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, pGroupCol->node.aliasName, TSDB_COL_NAME_LEN); + tstrncpy(pFunc->node.userAlias, pGroupCol->node.userAlias, TSDB_COL_NAME_LEN); SNode* pNew = NULL; code = nodesCloneNode((SNode*)pGroupCol, &pNew); if (TSDB_CODE_SUCCESS == code) { @@ -655,7 +655,7 @@ SFunctionNode* createGroupKeyAggFunc(SColumnNode* pGroupCol) { char name[TSDB_FUNC_NAME_LEN + TSDB_NAME_DELIMITER_LEN + TSDB_POINTER_PRINT_BYTES + 1] = {0}; int32_t len = tsnprintf(name, sizeof(name) - 1, "%s.%p", pFunc->functionName, pFunc); (void)taosHashBinary(name, len); - strncpy(pFunc->node.aliasName, name, TSDB_COL_NAME_LEN - 1); + tstrncpy(pFunc->node.aliasName, name, TSDB_COL_NAME_LEN); } } if (TSDB_CODE_SUCCESS != code) { diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index da38a72953..902e254465 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -377,7 +377,7 @@ class PlannerTestBaseImpl { } void doBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx) { - DO_WITH_THROW(qStmtBindParams, pQuery, pParams, colIdx); + DO_WITH_THROW(qStmtBindParams, pQuery, pParams, colIdx, NULL); if (colIdx < 0 || pQuery->placeholderNum == colIdx + 1) { res_.boundAst_ = toString(pQuery->pRoot); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 54e92c6a1b..22141c33ee 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -397,7 +397,7 @@ int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_ } *str = '"'; - int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str + 1); + int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str + 1, NULL); if (length <= 0) { return TSDB_CODE_TSC_INVALID_VALUE; } @@ -430,7 +430,7 @@ int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_ return TSDB_CODE_SUCCESS; } -void parseTagDatatoJson(void* p, char** jsonStr) { +void parseTagDatatoJson(void* p, char** jsonStr, void *charsetCxt) { if (!p || !jsonStr) { qError("parseTagDatatoJson invalid input, line:%d", __LINE__); return; @@ -475,9 +475,10 @@ void parseTagDatatoJson(void* p, char** jsonStr) { if (tagJsonValue == NULL) { goto end; } - int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue); + int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue, charsetCxt); if (length < 0) { - qError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, + qError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, + charsetCxt != NULL ? ((SConvInfo *)(charsetCxt))->charset : tsCharset, pTagVal->pData); taosMemoryFree(tagJsonValue); goto end; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index d8622d93ee..f0325dd174 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -2266,7 +2266,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { // match/nmatch for nchar type need convert from ucs4 to mbs if (type == TSDB_DATA_TYPE_NCHAR && (unit->compare.optr == OP_TYPE_MATCH || unit->compare.optr == OP_TYPE_NMATCH)) { char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData), NULL); if (len < 0) { qError("filterInitValFieldData taosUcs4ToMbs error 1"); return TSDB_CODE_SCALAR_CONVERT_ERROR; @@ -3603,7 +3603,7 @@ int32_t filterExecuteImplMisc(void *pinfo, int32_t numOfRows, SColumnInfoData *p if (newColData == NULL) { FLT_ERR_RET(terrno); } - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData)); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData), NULL); if (len < 0) { qError("castConvert1 taosUcs4ToMbs error"); taosMemoryFreeClear(newColData); @@ -3678,7 +3678,7 @@ int32_t filterExecuteImpl(void *pinfo, int32_t numOfRows, SColumnInfoData *pRes, if (newColData == NULL) { FLT_ERR_RET(terrno); } - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData)); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData), NULL); if (len < 0) { qError("castConvert1 taosUcs4ToMbs error"); taosMemoryFreeClear(newColData); @@ -4614,7 +4614,7 @@ int32_t filterConverNcharColumns(SFilterInfo *info, int32_t rows, bool *gotNchar varDataCopy(dst, src); continue; } - bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), (TdUcs4 *)varDataVal(dst), bufSize, &len); + bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), (TdUcs4 *)varDataVal(dst), bufSize, &len, NULL); if (!ret) { qError("filterConverNcharColumns taosMbsToUcs4 error"); return TSDB_CODE_SCALAR_CONVERT_ERROR; @@ -4694,33 +4694,6 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } - - /* - if (!FILTER_GET_FLAG(stat->info->options, FLT_OPTION_TIMESTAMP)) { - return DEAL_RES_CONTINUE; - } - - if (TSDB_DATA_TYPE_BINARY != valueNode->node.resType.type && TSDB_DATA_TYPE_NCHAR != - valueNode->node.resType.type && - TSDB_DATA_TYPE_GEOMETRY != valueNode->node.resType.type) { return DEAL_RES_CONTINUE; - } - - if (stat->precision < 0) { - int32_t code = fltAddValueNodeToConverList(stat, valueNode); - if (code) { - stat->code = code; - return DEAL_RES_ERROR; - } - - return DEAL_RES_CONTINUE; - } - - int32_t code = sclConvertToTsValueNode(stat->precision, valueNode); - if (code) { - stat->code = code; - return DEAL_RES_ERROR; - } - */ return DEAL_RES_CONTINUE; } @@ -4887,15 +4860,6 @@ int32_t fltReviseNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { FLT_ERR_JRET(pStat->code); - /* - int32_t nodeNum = taosArrayGetSize(pStat->nodeList); - for (int32_t i = 0; i < nodeNum; ++i) { - SValueNode *valueNode = *(SValueNode **)taosArrayGet(pStat->nodeList, i); - - FLT_ERR_JRET(sclConvertToTsValueNode(pStat->precision, valueNode)); - } - */ - _return: taosArrayDestroy(pStat->nodeList); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 209110b014..8c6db7b8ce 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -24,7 +24,7 @@ int32_t scalarGetOperatorParamNum(EOperatorType type) { int32_t sclConvertToTsValueNode(int8_t precision, SValueNode *valueNode) { char *timeStr = valueNode->datum.p; int64_t value = 0; - int32_t code = convertStringToTimestamp(valueNode->node.resType.type, valueNode->datum.p, precision, &value); + int32_t code = convertStringToTimestamp(valueNode->node.resType.type, valueNode->datum.p, precision, &value, valueNode->tz, valueNode->charsetCxt); //todo tz if (code != TSDB_CODE_SUCCESS) { return code; } @@ -80,7 +80,8 @@ int32_t sclConvertValueToSclParam(SValueNode *pValueNode, SScalarParam *out, int if (code != TSDB_CODE_SUCCESS) { goto _exit; } - + setTzCharset(&in, pValueNode->tz, pValueNode->charsetCxt); + setTzCharset(out, pValueNode->tz, pValueNode->charsetCxt); code = vectorConvertSingleColImpl(&in, out, overflow, -1, -1); _exit: @@ -586,9 +587,11 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal SCL_ERR_JRET(sclSetOperatorValueType(node, ctx)); SCL_ERR_JRET(sclInitParam(node->pLeft, ¶mList[0], ctx, rowNum)); + setTzCharset(¶mList[0], node->tz, node->charsetCxt); if (paramNum > 1) { TSWAP(ctx->type.selfType, ctx->type.peerType); SCL_ERR_JRET(sclInitParam(node->pRight, ¶mList[1], ctx, rowNum)); + setTzCharset(¶mList[1], node->tz, node->charsetCxt); } *pParams = paramList; @@ -756,6 +759,7 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp int32_t paramNum = 0; int32_t code = 0; SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, ¶mNum, &rowNum)); + setTzCharset(params, node->tz, node->charsetCxt); if (fmIsUserDefinedFunc(node->funcId)) { code = callUdfScalarFunc(node->functionName, params, paramNum, output); @@ -958,7 +962,11 @@ int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *outp sclError("invalid when/then in whenThen list"); SCL_ERR_JRET(TSDB_CODE_INVALID_PARA); } - + setTzCharset(pCase, node->tz, node->charsetCxt); + setTzCharset(pWhen, node->tz, node->charsetCxt); + setTzCharset(pThen, node->tz, node->charsetCxt); + setTzCharset(pElse, node->tz, node->charsetCxt); + setTzCharset(output, node->tz, node->charsetCxt); if (pCase) { SCL_ERR_JRET(vectorCompare(pCase, pWhen, &comp, TSDB_ORDER_ASC, OP_TYPE_EQUAL)); @@ -1241,7 +1249,6 @@ EDealRes sclRewriteFunction(SNode **pNode, SScalarCtx *ctx) { ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } - res->node.resType.bytes = varDataTLen(output.columnData->pData); (void)memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData)); } else { ctx->code = nodesSetValueNodeValue(res, output.columnData->pData); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 788ac38d8c..1da9a8f123 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -12,8 +12,8 @@ typedef float (*_float_fn_2)(float, float); typedef double (*_double_fn)(double); typedef double (*_double_fn_2)(double, double); typedef int (*_conv_fn)(int); -typedef void (*_trim_space_fn)(char *, char *, int32_t, int32_t); -typedef int32_t (*_trim_fn)(char *, char *, char *, int32_t, int32_t); +typedef void (*_trim_space_fn)(char *, char *, int32_t, int32_t, void*); +typedef int32_t (*_trim_fn)(char *, char *, char *, int32_t, int32_t, void*); typedef int32_t (*_len_fn)(char *, int32_t, VarDataLenT *); /** Math functions **/ @@ -500,7 +500,7 @@ static int32_t tcharlength(char *input, int32_t type, VarDataLenT *len) { return TSDB_CODE_SUCCESS; } -static void tltrimspace(char *input, char *output, int32_t type, int32_t charLen) { +static void tltrimspace(char *input, char *output, int32_t type, int32_t charLen, void* charsetCxt) { int32_t numOfSpaces = 0; if (type == TSDB_DATA_TYPE_VARCHAR) { for (int32_t i = 0; i < charLen; ++i) { @@ -530,7 +530,7 @@ static void tltrimspace(char *input, char *output, int32_t type, int32_t charLen varDataSetLen(output, resLen); } -static void tlrtrimspace(char *input, char *output, int32_t type, int32_t charLen) { +static void tlrtrimspace(char *input, char *output, int32_t type, int32_t charLen, void* charsetCxt) { int32_t numOfLeftSpaces = 0; int32_t numOfRightSpaces = 0; if (type == TSDB_DATA_TYPE_VARCHAR) { @@ -584,7 +584,7 @@ static bool isCharStart(char c) { static int32_t trimHelper(char *orgStr, char* remStr, int32_t orgLen, int32_t remLen, bool trimLeft, bool isNchar) { if (trimLeft) { int32_t pos = 0; - for (int32_t i = 0; i < orgLen; i += remLen) { + for (int32_t i = 0; i < orgLen - remLen; i += remLen) { if (memcmp(orgStr + i, remStr, remLen) == 0) { if (isCharStart(orgStr[i + remLen]) || isNchar) { pos = i + remLen; @@ -615,12 +615,12 @@ static int32_t trimHelper(char *orgStr, char* remStr, int32_t orgLen, int32_t re } } -static int32_t convVarcharToNchar(char *input, char **output, int32_t inputLen, int32_t *outputLen) { +static int32_t convVarcharToNchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, void* charsetCxt) { *output = taosMemoryCalloc(inputLen * TSDB_NCHAR_SIZE, 1); if (NULL == *output) { return terrno; } - bool ret = taosMbsToUcs4(input, inputLen, (TdUcs4 *)*output, inputLen * TSDB_NCHAR_SIZE, outputLen); + bool ret = taosMbsToUcs4(input, inputLen, (TdUcs4 *)*output, inputLen * TSDB_NCHAR_SIZE, outputLen, charsetCxt); if (!ret) { taosMemoryFreeClear(*output); return TSDB_CODE_SCALAR_CONVERT_ERROR; @@ -628,12 +628,12 @@ static int32_t convVarcharToNchar(char *input, char **output, int32_t inputLen, return TSDB_CODE_SUCCESS; } -static int32_t convNcharToVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen) { +static int32_t convNcharToVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, void* charsetCxt) { *output = taosMemoryCalloc(inputLen, 1); if (NULL == *output) { return terrno; } - *outputLen = taosUcs4ToMbs((TdUcs4 *)input, inputLen, *output); + *outputLen = taosUcs4ToMbs((TdUcs4 *)input, inputLen, *output, charsetCxt); if (*outputLen < 0) { taosMemoryFree(*output); return TSDB_CODE_SCALAR_CONVERT_ERROR; @@ -641,14 +641,14 @@ static int32_t convNcharToVarchar(char *input, char **output, int32_t inputLen, return TSDB_CODE_SUCCESS; } -static int32_t convBetweenNcharAndVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, int32_t wantType) { +static int32_t convBetweenNcharAndVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, int32_t wantType, void* charsetCxt) { if (wantType == TSDB_DATA_TYPE_NCHAR) { - return convVarcharToNchar(input, output, inputLen, outputLen); + return convVarcharToNchar(input, output, inputLen, outputLen, charsetCxt); } else { - return convNcharToVarchar(input, output, inputLen, outputLen); + return convNcharToVarchar(input, output, inputLen, outputLen, charsetCxt); } } -static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType) { +static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType, void* charsetCxt) { int32_t orgLen = varDataLen(input); char *orgStr = varDataVal(input); int32_t remLen = varDataLen(remInput); @@ -661,7 +661,7 @@ static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputTy bool needFree = false; if (inputType != remType) { - SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType)); + SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType, charsetCxt)); needFree = true; } @@ -683,7 +683,7 @@ static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputTy return TSDB_CODE_SUCCESS; } -static void trtrimspace(char *input, char *output, int32_t type, int32_t charLen) { +static void trtrimspace(char *input, char *output, int32_t type, int32_t charLen, void *charsetCxt) { int32_t numOfSpaces = 0; if (type == TSDB_DATA_TYPE_VARCHAR) { for (int32_t i = charLen - 1; i >= 0; --i) { @@ -712,7 +712,7 @@ static void trtrimspace(char *input, char *output, int32_t type, int32_t charLen varDataSetLen(output, resLen); } -static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType) { +static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType, void* charsetCxt) { int32_t orgLen = varDataLen(input); char *orgStr = varDataVal(input); int32_t remLen = varDataLen(remInput); @@ -725,7 +725,7 @@ static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputTy bool needFree = false; if (inputType != remType) { - SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType)); + SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType, charsetCxt)); needFree = true; } @@ -746,7 +746,7 @@ static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputTy return TSDB_CODE_SUCCESS; } -static int32_t tlrtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType) { +static int32_t tlrtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType, void *charsetCxt) { int32_t orgLen = varDataLen(input); char *orgStr = varDataVal(input); int32_t remLen = varDataLen(remInput); @@ -759,7 +759,7 @@ static int32_t tlrtrim(char *input, char *remInput, char *output, int32_t inputT bool needFree = false; if (inputType != remType) { - SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType)); + SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType, charsetCxt)); needFree = true; } @@ -810,14 +810,14 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP return TSDB_CODE_SUCCESS; } -static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, VarDataLenT *dataLen) { +static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, VarDataLenT *dataLen, void* charsetCxt) { if (hasNchar && type == TSDB_DATA_TYPE_VARCHAR) { TdUcs4 *newBuf = taosMemoryCalloc((varDataLen(input) + 1) * TSDB_NCHAR_SIZE, 1); if (NULL == newBuf) { return terrno; } int32_t len = varDataLen(input); - bool ret = taosMbsToUcs4(varDataVal(input), len, newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, &len); + bool ret = taosMbsToUcs4(varDataVal(input), len, newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, &len, charsetCxt); if (!ret) { taosMemoryFree(newBuf); return TSDB_CODE_SCALAR_CONVERT_ERROR; @@ -905,7 +905,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k; input[i] = colDataGetData(pInputData[i], rowIdx); - SCL_ERR_JRET(concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen)); + SCL_ERR_JRET(concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen, pInput->charsetCxt)); } varDataSetLen(output, dataLen); SCL_ERR_JRET(colDataSetVal(pOutputData, k, outputBuf, false)); @@ -980,12 +980,12 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k; - SCL_ERR_JRET(concatCopyHelper(colDataGetData(pInputData[i], rowIdx), output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen)); + SCL_ERR_JRET(concatCopyHelper(colDataGetData(pInputData[i], rowIdx), output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen, pInput->charsetCxt)); if (i < inputNum - 1) { // insert the separator char *sep = (pInput[0].numOfRows == 1) ? colDataGetData(pInputData[0], 0) : colDataGetData(pInputData[0], k); - SCL_ERR_JRET(concatCopyHelper(sep, output, hasNchar, GET_PARAM_TYPE(&pInput[0]), &dataLen)); + SCL_ERR_JRET(concatCopyHelper(sep, output, hasNchar, GET_PARAM_TYPE(&pInput[0]), &dataLen, pInput->charsetCxt)); } } @@ -1092,7 +1092,7 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar continue; } SCL_ERR_JRET(trimFn(colDataGetData(pInputData[1], colIdx2), colDataGetData(pInputData[0], colIdx1), - output, GET_PARAM_TYPE(&pInput[1]), GET_PARAM_TYPE(&pInput[0]))); + output, GET_PARAM_TYPE(&pInput[1]), GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt)); SCL_ERR_JRET(colDataSetVal(pOutputData, i, output, false)); } } else { @@ -1105,7 +1105,7 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar char *input = colDataGetData(pInputData[0], i); int32_t len = varDataLen(input); int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE; - trimSpaceFn(input, output, type, charLen); + trimSpaceFn(input, output, type, charLen, pInput->charsetCxt); SCL_ERR_JRET(colDataSetVal(pOutputData, i, output, false)); } } @@ -1343,7 +1343,7 @@ int32_t charFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp if (convBuf == NULL) { SCL_ERR_RET(terrno); } - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colDataGetData(pInput[j].columnData, colIdx)), varDataLen(colDataGetData(pInput[j].columnData, colIdx)), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colDataGetData(pInput[j].columnData, colIdx)), varDataLen(colDataGetData(pInput[j].columnData, colIdx)), convBuf, pInput->charsetCxt); if (len < 0) { taosMemoryFree(convBuf); code = TSDB_CODE_SCALAR_CONVERT_ERROR; @@ -1386,7 +1386,7 @@ int32_t asciiFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut int32_t inLen = varDataLen(colDataGetData(pInputData, i)); SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData, i)), &in, varDataLen(colDataGetData(pInputData, i)), &inLen, - TSDB_DATA_TYPE_VARBINARY)); + TSDB_DATA_TYPE_VARBINARY, pInput->charsetCxt)); out[i] = (uint8_t)(in)[0]; taosMemoryFree(in); } else { @@ -1454,7 +1454,9 @@ int32_t positionFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t orgLen = varDataLen(colDataGetData(pInputData[1], colIdx2)); bool needFreeSub = false; if (GET_PARAM_TYPE(&pInput[1]) != GET_PARAM_TYPE(&pInput[0])) { - SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[0], colIdx1)), &substr, varDataLen(colDataGetData(pInputData[0], colIdx1)), &subLen, GET_PARAM_TYPE(&pInput[1]))); + SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[0], colIdx1)), &substr, + varDataLen(colDataGetData(pInputData[0], colIdx1)), &subLen, + GET_PARAM_TYPE(&pInput[1]), pInput->charsetCxt)); needFreeSub = true; } @@ -1562,13 +1564,13 @@ int32_t replaceFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pO if (GET_PARAM_TYPE(&pInput[1]) != GET_PARAM_TYPE(&pInput[0])) { SCL_ERR_JRET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[1], colIdx2)), &fromStr, varDataLen(colDataGetData(pInputData[1], colIdx2)), &fromLen, - GET_PARAM_TYPE(&pInput[0]))); + GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt)); needFreeFrom = true; } if (GET_PARAM_TYPE(&pInput[2]) != GET_PARAM_TYPE(&pInput[0])) { code = convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[2], colIdx3)), &toStr, varDataLen(colDataGetData(pInputData[2], colIdx3)), &toLen, - GET_PARAM_TYPE(&pInput[0])); + GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt); if (TSDB_CODE_SUCCESS != code) { if (needFreeFrom) { taosMemoryFree(fromStr); @@ -1674,7 +1676,7 @@ int32_t substrIdxFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * if (GET_PARAM_TYPE(&pInput[0]) != GET_PARAM_TYPE(&pInput[1])) { SCL_ERR_JRET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[1], colIdx2)), &delimStr, varDataLen(colDataGetData(pInputData[1], colIdx2)), &delimLen, - GET_PARAM_TYPE(&pInput[0]))); + GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt)); needFreeDelim = true; } @@ -1853,7 +1855,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(int8_t *)output = taosStr2Int8(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -1872,7 +1874,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(int16_t *)output = taosStr2Int16(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -1890,7 +1892,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(int32_t *)output = taosStr2Int32(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -1909,7 +1911,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(int64_t *)output = taosStr2Int64(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -1927,7 +1929,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(uint8_t *)output = taosStr2UInt8(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -1945,7 +1947,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(uint16_t *)output = taosStr2UInt16(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -1963,7 +1965,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(uint32_t *)output = taosStr2UInt32(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -1981,7 +1983,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(uint64_t *)output = taosStr2UInt64(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2000,7 +2002,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(float *)output = taosStr2Float(buf, NULL); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2018,7 +2020,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(double *)output = taosStr2Double(buf, NULL); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2036,7 +2038,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(bool *)output = taosStr2Int8(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2053,7 +2055,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { int64_t timePrec; GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); - int32_t ret = convertStringToTimestamp(inputType, input, timePrec, &timeVal); + int32_t ret = convertStringToTimestamp(inputType, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt); if (ret != TSDB_CODE_SUCCESS) { *(int64_t *)output = 0; } else { @@ -2076,7 +2078,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp (void)memcpy(varDataVal(output), varDataVal(input), len); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2111,7 +2113,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp if (inputType == TSDB_DATA_TYPE_BOOL) { char tmp[8] = {0}; len = tsnprintf(tmp, sizeof(tmp), "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false"); - bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); + bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len, pInput->charsetCxt); if (!ret) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2121,7 +2123,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } else if (inputType == TSDB_DATA_TYPE_BINARY) { len = outputCharLen > varDataLen(input) ? varDataLen(input) : outputCharLen; bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, (TdUcs4 *)varDataVal(output), - outputLen - VARSTR_HEADER_SIZE, &len); + outputLen - VARSTR_HEADER_SIZE, &len, pInput->charsetCxt); if (!ret) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2135,7 +2137,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp NUM_TO_STRING(inputType, input, bufSize, buf); len = (int32_t)strlen(buf); len = outputCharLen > len ? len : outputCharLen; - bool ret = taosMbsToUcs4(buf, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); + bool ret = taosMbsToUcs4(buf, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len, pInput->charsetCxt); if (!ret) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2240,17 +2242,14 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * if (0 != offsetOfTimezone(tz, &offset)) { goto _end; } - quot -= offset + 3600 * ((int64_t)tsTimezone); + quot -= offset; struct tm tmInfo; - int32_t len = 0; - - if (taosLocalTime((const time_t *)", &tmInfo, buf, sizeof(buf)) == NULL) { - len = (int32_t)strlen(buf); + if (taosGmTimeR((const time_t *)", &tmInfo) == NULL) { goto _end; } - len = (int32_t)strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo); + int32_t len = (int32_t)taosStrfTime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo); len += tsnprintf(buf + len, fractionLen, format, mod); @@ -2260,10 +2259,10 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * len += tzLen; } -_end: memmove(buf + VARSTR_HEADER_SIZE, buf, len); varDataSetLen(buf, len); + _end: SCL_ERR_RET(colDataSetVal(pOutput->columnData, i, buf, false)); } @@ -2286,7 +2285,7 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP char *input = colDataGetData(pInput[0].columnData, i); int64_t timeVal = 0; - int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal); + int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt); if (ret != TSDB_CODE_SUCCESS) { colDataSetNULL(pOutput->columnData, i); } else { @@ -2326,7 +2325,7 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } (void)memcpy(tmp, varDataVal(input), varDataLen(input)); tmp[varDataLen(input)] = 0; - if (parseJsontoTagData(tmp, pTagVals, &pTag, NULL)) { + if (parseJsontoTagData(tmp, pTagVals, &pTag, NULL, pInput->charsetCxt)) { code = tTagNew(pTagVals, 1, true, &pTag); if (TSDB_CODE_SUCCESS != code) { tTagFree(pTag); @@ -2381,7 +2380,7 @@ int32_t toTimestampFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam } int32_t precision = pOutput->columnData->info.precision; char errMsg[128] = {0}; - code = taosChar2Ts(format, &formats, tsStr, &ts, precision, errMsg, 128); + code = taosChar2Ts(format, &formats, tsStr, &ts, precision, errMsg, 128, pInput->tz); if (code) { qError("func to_timestamp failed %s", errMsg); SCL_ERR_JRET(code); @@ -2424,7 +2423,7 @@ int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOu } } int32_t precision = pInput[0].columnData->info.precision; - SCL_ERR_JRET(taosTs2Char(format, &formats, *(int64_t *)ts, precision, varDataVal(out), TS_FORMAT_MAX_LEN)); + SCL_ERR_JRET(taosTs2Char(format, &formats, *(int64_t *)ts, precision, varDataVal(out), TS_FORMAT_MAX_LEN, pInput->tz)); varDataSetLen(out, strlen(varDataVal(out))); SCL_ERR_JRET(colDataSetVal(pOutput->columnData, i, out, false)); } @@ -2437,11 +2436,11 @@ _return: } /** Time functions **/ -int64_t offsetFromTz(char *timezone, int64_t factor) { - char *minStr = &timezone[3]; +int64_t offsetFromTz(char *timezoneStr, int64_t factor) { + char *minStr = &timezoneStr[3]; int64_t minutes = taosStr2Int64(minStr, NULL, 10); (void)memset(minStr, 0, strlen(minStr)); - int64_t hours = taosStr2Int64(timezone, NULL, 10); + int64_t hours = taosStr2Int64(timezoneStr, NULL, 10); int64_t seconds = hours * 3600 + minutes * 60; return seconds * factor; @@ -2453,7 +2452,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara int64_t timeUnit, timePrec, timeVal = 0; bool ignoreTz = true; - char timezone[20] = {0}; + char timezoneStr[20] = {0}; GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); @@ -2465,7 +2464,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[timePrecIdx]), pInput[timePrecIdx].columnData->pData); - (void)memcpy(timezone, varDataVal(pInput[timeZoneIdx].columnData->pData), varDataLen(pInput[timeZoneIdx].columnData->pData)); + (void)memcpy(timezoneStr, varDataVal(pInput[timeZoneIdx].columnData->pData), varDataLen(pInput[timeZoneIdx].columnData->pData)); for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { @@ -2476,7 +2475,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara char *input = colDataGetData(pInput[0].columnData, i); if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ - int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal); + int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt); if (ret != TSDB_CODE_SUCCESS) { colDataSetNULL(pOutput->columnData, i); continue; @@ -2493,7 +2492,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara // truncate the timestamp to time_unit precision int64_t seconds = timeUnit / TSDB_TICK_PER_SECOND(timePrec); if (ignoreTz && (seconds == 604800 || seconds == 86400)) { - timeVal = timeVal - (timeVal + offsetFromTz(timezone, TSDB_TICK_PER_SECOND(timePrec))) % timeUnit; + timeVal = timeVal - (timeVal + offsetFromTz(timezoneStr, TSDB_TICK_PER_SECOND(timePrec))) % timeUnit; } else { timeVal = timeVal / timeUnit * timeUnit; } @@ -2536,7 +2535,7 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t type = GET_PARAM_TYPE(&pInput[k]); if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ - int32_t ret = convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k]); + int32_t ret = convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k], pInput->tz, pInput->charsetCxt); if (ret != TSDB_CODE_SUCCESS) { hasNull = true; break; @@ -2655,7 +2654,7 @@ int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut int64_t timePrec; GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[0]), pInput[0].columnData->pData); - int64_t ts = taosGetTimestampToday(timePrec); + int64_t ts = taosGetTimestampToday(timePrec, pInput->tz); for (int32_t i = 0; i < pInput->numOfRows; ++i) { colDataSetInt64(pOutput->columnData, i, &ts); } @@ -2663,14 +2662,20 @@ int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut return TSDB_CODE_SUCCESS; } -int32_t timeZoneStrLen() { - return sizeof(VarDataLenT) + strlen(tsTimezoneStr); -} - int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { char output[TD_TIMEZONE_LEN + VARSTR_HEADER_SIZE] = {0}; - (void)memcpy(varDataVal(output), tsTimezoneStr, TD_TIMEZONE_LEN); - varDataSetLen(output, strlen(tsTimezoneStr)); + char* tmp = NULL; + if (pInput->tz == NULL) { + (void)memcpy(varDataVal(output), tsTimezoneStr, TD_TIMEZONE_LEN); + } else{ + char *tzName = (char*)taosHashGet(pTimezoneNameMap, &pInput->tz, sizeof(timezone_t)); + if (tzName == NULL){ + tzName = TZ_UNKNOWN; + } + tstrncpy(varDataVal(output), tzName, strlen(tzName) + 1); + } + + varDataSetLen(output, strlen(varDataVal(output))); for (int32_t i = 0; i < pInput->numOfRows; ++i) { SCL_ERR_RET(colDataSetVal(pOutput->columnData, i, output, false)); } @@ -2694,7 +2699,7 @@ int32_t weekdayFunctionImpl(SScalarParam *pInput, int32_t inputNum, SScalarParam char *input = colDataGetData(pInput[0].columnData, i); if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ - int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal); + int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt); if (ret != TSDB_CODE_SUCCESS) { colDataSetNULL(pOutput->columnData, i); continue; @@ -2705,7 +2710,7 @@ int32_t weekdayFunctionImpl(SScalarParam *pInput, int32_t inputNum, SScalarParam GET_TYPED_DATA(timeVal, int64_t, type, input); } struct STm tm; - TAOS_CHECK_RETURN(taosTs2Tm(timeVal, timePrec, &tm)); + TAOS_CHECK_RETURN(taosTs2Tm(timeVal, timePrec, &tm, pInput->tz)); int64_t ret = startFromZero ? (tm.tm.tm_wday + 6) % 7 : tm.tm.tm_wday + 1; colDataSetInt64(pOutput->columnData, i, &ret); } @@ -2804,7 +2809,7 @@ int32_t weekFunctionImpl(SScalarParam *pInput, int32_t inputNum, SScalarParam *p char *input = colDataGetData(pInput[0].columnData, i); if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ - int32_t ret = convertStringToTimestamp(type, input, prec, &timeVal); + int32_t ret = convertStringToTimestamp(type, input, prec, &timeVal, pInput->tz, pInput->charsetCxt); if (ret != TSDB_CODE_SUCCESS) { colDataSetNULL(pOutput->columnData, i); continue; @@ -2815,7 +2820,7 @@ int32_t weekFunctionImpl(SScalarParam *pInput, int32_t inputNum, SScalarParam *p GET_TYPED_DATA(timeVal, int64_t, type, input); } struct STm tm; - SCL_ERR_RET(taosTs2Tm(timeVal, prec, &tm)); + SCL_ERR_RET(taosTs2Tm(timeVal, prec, &tm, pInput->tz)); int64_t ret = calculateWeekNum(tm.tm, weekMode(mode)); colDataSetInt64(pOutput->columnData, i, &ret); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c6c8333392..393bb947cf 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -103,7 +103,7 @@ int32_t convertNcharToDouble(const void *inData, void *outData) { if (NULL == tmp) { SCL_ERR_RET(terrno); } - int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp); + int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp, NULL); if (len < 0) { sclError("castConvert taosUcs4ToMbs error 1"); SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); @@ -242,7 +242,7 @@ int32_t getVectorBigintValueFn(int32_t srcType, _getBigintValue_fn_t *p) { static FORCE_INLINE int32_t varToTimestamp(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { int64_t value = 0; int32_t code = TSDB_CODE_SUCCESS; - if (taosParseTime(buf, &value, strlen(buf), pOut->columnData->info.precision, tsDaylight) != TSDB_CODE_SUCCESS) { + if (taosParseTime(buf, &value, strlen(buf), pOut->columnData->info.precision, pOut->tz) != TSDB_CODE_SUCCESS) { value = 0; code = TSDB_CODE_SCALAR_CONVERT_ERROR; } @@ -404,7 +404,7 @@ static FORCE_INLINE int32_t varToNchar(char *buf, SScalarParam *pOut, int32_t ro SCL_ERR_RET(terrno); } int32_t ret = - taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4 *)varDataVal(t), outputMaxLen - VARSTR_HEADER_SIZE, &len); + taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4 *)varDataVal(t), outputMaxLen - VARSTR_HEADER_SIZE, &len, pOut->charsetCxt); if (!ret) { sclError("failed to convert to NCHAR"); SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); @@ -426,7 +426,7 @@ static FORCE_INLINE int32_t ncharToVar(char *buf, SScalarParam *pOut, int32_t ro if (NULL == t) { SCL_ERR_RET(terrno); } - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t)); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t), pOut->charsetCxt); if (len < 0) { SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); } @@ -557,7 +557,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { SCL_ERR_JRET(TSDB_CODE_APP_ERROR); } - int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp); + int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp, pCtx->pIn->charsetCxt); if (len < 0) { sclError("castConvert taosUcs4ToMbs error 1"); SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); @@ -592,18 +592,19 @@ int32_t getVectorDoubleValue_JSON(void *src, int32_t index, double *out) { SCL_RET(TSDB_CODE_SUCCESS); } -int32_t ncharTobinary(void *buf, void **out) { // todo need to remove , if tobinary is nchar +int32_t ncharTobinary(void *buf, void **out, void* charsetCxt) { // todo need to remove , if tobinary is nchar int32_t inputLen = varDataTLen(buf); *out = taosMemoryCalloc(1, inputLen); if (NULL == *out) { sclError("charset:%s to %s. val:%s convert ncharTobinary failed, since memory alloc failed.", - DEFAULT_UNICODE_ENCODEC, tsCharset, (char *)varDataVal(buf)); + DEFAULT_UNICODE_ENCODEC, charsetCxt != NULL ? ((SConvInfo *)(charsetCxt))->charset : tsCharset, (char *)varDataVal(buf)); SCL_ERR_RET(terrno); } - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(*out)); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(*out), charsetCxt); if (len < 0) { - sclError("charset:%s to %s. val:%s convert ncharTobinary failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, + sclError("charset:%s to %s. val:%s convert ncharTobinary failed.", DEFAULT_UNICODE_ENCODEC, + charsetCxt != NULL ? ((SConvInfo *)(charsetCxt))->charset : tsCharset, (char *)varDataVal(buf)); taosMemoryFree(*out); SCL_ERR_RET(TSDB_CODE_SCALAR_CONVERT_ERROR); @@ -614,7 +615,7 @@ int32_t ncharTobinary(void *buf, void **out) { // todo need to remove , if tobi int32_t convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull, bool *freeLeft, - bool *freeRight, bool *result) { + bool *freeRight, bool *result, void* charsetCxt) { *result = false; if (optr == OP_TYPE_JSON_CONTAINS) { *result = true; @@ -700,13 +701,13 @@ int32_t convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_ type == TSDB_DATA_TYPE_GEOMETRY) { if (typeLeft == TSDB_DATA_TYPE_NCHAR) { char *tmpLeft = NULL; - SCL_ERR_RET(ncharTobinary(*pLeftData, (void *)&tmpLeft)); + SCL_ERR_RET(ncharTobinary(*pLeftData, (void *)&tmpLeft, charsetCxt)); *pLeftData = tmpLeft; *freeLeft = true; } if (typeRight == TSDB_DATA_TYPE_NCHAR) { char *tmpRight = NULL; - SCL_ERR_RET(ncharTobinary(*pRightData, (void *)&tmpRight)); + SCL_ERR_RET(ncharTobinary(*pRightData, (void *)&tmpRight, charsetCxt)); *pRightData = tmpRight; *freeRight = true; } @@ -1191,7 +1192,7 @@ static int32_t vectorMathAddHelper(SColumnInfoData *pLeftCol, SColumnInfoData *p } static int32_t vectorMathTsAddHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRightCol, SColumnInfoData *pOutputCol, - int32_t numOfRows, int32_t step, int32_t i) { + int32_t numOfRows, int32_t step, int32_t i, timezone_t tz) { _getBigintValue_fn_t getVectorBigintValueFnLeft; _getBigintValue_fn_t getVectorBigintValueFnRight; SCL_ERR_RET(getVectorBigintValueFn(pLeftCol->info.type, &getVectorBigintValueFnLeft)); @@ -1211,7 +1212,7 @@ static int32_t vectorMathTsAddHelper(SColumnInfoData *pLeftCol, SColumnInfoData SCL_ERR_RET(getVectorBigintValueFnLeft(pLeftCol->pData, i, &leftRes)); SCL_ERR_RET(getVectorBigintValueFnRight(pRightCol->pData, 0, &rightRes)); *output = - taosTimeAdd(leftRes, rightRes, pRightCol->info.scale, pRightCol->info.precision); + taosTimeAdd(leftRes, rightRes, pRightCol->info.scale, pRightCol->info.precision, tz); } } SCL_RET(TSDB_CODE_SUCCESS); @@ -1275,14 +1276,14 @@ int32_t vectorMathAdd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p if (pLeft->numOfRows == 1 && pRight->numOfRows == 1) { if (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP) { - SCL_ERR_JRET(vectorMathTsAddHelper(pLeftCol, pRightCol, pOutputCol, pRight->numOfRows, step, i)); + SCL_ERR_JRET(vectorMathTsAddHelper(pLeftCol, pRightCol, pOutputCol, pRight->numOfRows, step, i, pLeft->tz)); } else { - SCL_ERR_JRET(vectorMathTsAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i)); + SCL_ERR_JRET(vectorMathTsAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i, pLeft->tz)); } } else if (pLeft->numOfRows == 1) { - SCL_ERR_JRET(vectorMathTsAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i)); + SCL_ERR_JRET(vectorMathTsAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i, pLeft->tz)); } else if (pRight->numOfRows == 1) { - SCL_ERR_JRET(vectorMathTsAddHelper(pLeftCol, pRightCol, pOutputCol, pLeft->numOfRows, step, i)); + SCL_ERR_JRET(vectorMathTsAddHelper(pLeftCol, pRightCol, pOutputCol, pLeft->numOfRows, step, i, pLeft->tz)); } else if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { if (IS_NULL) { @@ -1356,7 +1357,7 @@ static int32_t vectorMathSubHelper(SColumnInfoData *pLeftCol, SColumnInfoData *p } static int32_t vectorMathTsSubHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRightCol, SColumnInfoData *pOutputCol, - int32_t numOfRows, int32_t step, int32_t factor, int32_t i) { + int32_t numOfRows, int32_t step, int32_t factor, int32_t i, timezone_t tz) { _getBigintValue_fn_t getVectorBigintValueFnLeft; _getBigintValue_fn_t getVectorBigintValueFnRight; SCL_ERR_RET(getVectorBigintValueFn(pLeftCol->info.type, &getVectorBigintValueFnLeft)); @@ -1377,7 +1378,7 @@ static int32_t vectorMathTsSubHelper(SColumnInfoData *pLeftCol, SColumnInfoData SCL_ERR_RET(getVectorBigintValueFnLeft(pLeftCol->pData, i, &leftRes)); SCL_ERR_RET(getVectorBigintValueFnRight(pRightCol->pData, 0, &rightRes)); *output = - taosTimeAdd(leftRes, -rightRes, pRightCol->info.scale, pRightCol->info.precision) * factor; + taosTimeAdd(leftRes, -rightRes, pRightCol->info.scale, pRightCol->info.precision, tz) * factor; } } SCL_RET(TSDB_CODE_SUCCESS); @@ -1408,11 +1409,11 @@ int32_t vectorMathSub(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p SCL_ERR_JRET(getVectorBigintValueFn(pRightCol->info.type, &getVectorBigintValueFnRight)); if (pLeft->numOfRows == 1 && pRight->numOfRows == 1) { - SCL_ERR_JRET(vectorMathTsSubHelper(pLeftCol, pRightCol, pOutputCol, pLeft->numOfRows, step, 1, i)); + SCL_ERR_JRET(vectorMathTsSubHelper(pLeftCol, pRightCol, pOutputCol, pLeft->numOfRows, step, 1, i, pLeft->tz)); } else if (pLeft->numOfRows == 1) { - SCL_ERR_JRET(vectorMathTsSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i)); + SCL_ERR_JRET(vectorMathTsSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i, pLeft->tz)); } else if (pRight->numOfRows == 1) { - SCL_ERR_JRET(vectorMathTsSubHelper(pLeftCol, pRightCol, pOutputCol, pLeft->numOfRows, step, 1, i)); + SCL_ERR_JRET(vectorMathTsSubHelper(pLeftCol, pRightCol, pOutputCol, pLeft->numOfRows, step, 1, i, pLeft->tz)); } else if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { if (IS_NULL) { @@ -1958,7 +1959,7 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa bool result = false; SCL_ERR_RET(convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, - &leftOut, &rightOut, &isJsonnull, &freeLeft, &freeRight, &result)); + &leftOut, &rightOut, &isJsonnull, &freeLeft, &freeRight, &result, pLeft->charsetCxt)); if (isJsonnull) { sclError("doVectorCompareImpl: invalid json null value"); @@ -2042,6 +2043,8 @@ int32_t vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara SScalarParam *param1 = NULL; SScalarParam *param2 = NULL; int32_t code = TSDB_CODE_SUCCESS; + setTzCharset(&pLeftOut, pLeft->tz, pLeft->charsetCxt); + setTzCharset(&pRightOut, pLeft->tz, pLeft->charsetCxt); if (noConvertBeforeCompare(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) { param1 = pLeft; param2 = pRight; diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 4cab644582..6a5188f208 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1450,7 +1450,7 @@ TEST(columnTest, json_column_arith_op) { SArray *tags = taosArrayInit(1, sizeof(STagVal)); ASSERT_NE(tags, nullptr); STag *row = NULL; - int32_t code = parseJsontoTagData(rightv, tags, &row, NULL); + int32_t code = parseJsontoTagData(rightv, tags, &row, NULL, NULL); ASSERT_EQ(code, TSDB_CODE_SUCCESS); const int32_t len = 8; @@ -1606,7 +1606,7 @@ void *prepareNchar(char *rightData) { int32_t inputLen = strlen(rightData); char *t = (char *)taosMemoryCalloc(1, (inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); - taosMbsToUcs4(rightData, inputLen, (TdUcs4 *)varDataVal(t), inputLen * TSDB_NCHAR_SIZE, &len); + taosMbsToUcs4(rightData, inputLen, (TdUcs4 *)varDataVal(t), inputLen * TSDB_NCHAR_SIZE, &len, NULL); varDataSetLen(t, len); return t; } @@ -1623,7 +1623,7 @@ TEST(columnTest, json_column_logic_op) { SArray *tags = taosArrayInit(1, sizeof(STagVal)); ASSERT_NE(tags, nullptr); STag *row = NULL; - code = parseJsontoTagData(rightv, tags, &row, NULL); + code = parseJsontoTagData(rightv, tags, &row, NULL, NULL); ASSERT_EQ(code, TSDB_CODE_SUCCESS); const int32_t len0 = 6; diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 9be0e3fc40..37249b5418 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -1129,7 +1129,11 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { // SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); if (NULL == schMgmt.queryMgmt) { - SCH_ERR_RET(qWorkerInit(NODE_TYPE_CLIENT, CLIENT_HANDLE, (void **)&schMgmt.queryMgmt, NULL)); + void* p = NULL; + SCH_ERR_RET(qWorkerInit(NODE_TYPE_CLIENT, CLIENT_HANDLE, &p, NULL)); + if (atomic_val_compare_exchange_ptr(&schMgmt.queryMgmt, NULL, p)) { + qWorkerDestroy(&p); + } } SArray *explainRes = NULL; diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 6a10b21c53..2b5312c0a0 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -166,10 +166,12 @@ int32_t streamStateDel_rocksdb(SStreamState* pState, const SWinKey* key); int32_t streamStateClear_rocksdb(SStreamState* pState); void streamStateCurNext_rocksdb(SStreamStateCur* pCur); int32_t streamStateGetFirst_rocksdb(SStreamState* pState, SWinKey* key); -int32_t streamStateGetGroupKVByCur_rocksdb(SStreamState* pState, SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); +int32_t streamStateGetGroupKVByCur_rocksdb(SStreamState* pState, SStreamStateCur* pCur, SWinKey* pKey, + const void** pVal, int32_t* pVLen); int32_t streamStateAddIfNotExist_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); void streamStateCurPrev_rocksdb(SStreamStateCur* pCur); -int32_t streamStateGetKVByCur_rocksdb(SStreamState* pState, SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); +int32_t streamStateGetKVByCur_rocksdb(SStreamState* pState, SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, + int32_t* pVLen); SStreamStateCur* streamStateGetAndCheckCur_rocksdb(SStreamState* pState, SWinKey* key); SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey* key); @@ -217,16 +219,15 @@ int32_t streamStateFillGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* p // partag cf int32_t streamStatePutParTag_rocksdb(SStreamState* pState, int64_t groupId, const void* tag, int32_t tagLen); int32_t streamStateGetParTag_rocksdb(SStreamState* pState, int64_t groupId, void** tagVal, int32_t* tagLen); -void streamStateParTagSeekKeyNext_rocksdb(SStreamState* pState, const int64_t groupId, SStreamStateCur* pCur); -int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGroupId, const void** pVal, int32_t* pVLen); +void streamStateParTagSeekKeyNext_rocksdb(SStreamState* pState, const int64_t groupId, SStreamStateCur* pCur); +int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGroupId, const void** pVal, + int32_t* pVLen); // parname cf int32_t streamStatePutParName_rocksdb(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]); int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, void** pVal); int32_t streamStateDeleteParName_rocksdb(SStreamState* pState, int64_t groupId); -void streamStateDestroy_rocksdb(SStreamState* pState, bool remove); - // default cf int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen); int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** pVal, int32_t* pVLen); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 427733e9ec..e4f921f04a 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -37,7 +37,7 @@ extern "C" { #define META_HB_CHECK_INTERVAL 200 #define META_HB_SEND_IDLE_COUNTER 25 // send hb every 5 sec #define STREAM_TASK_KEY_LEN ((sizeof(int64_t)) << 1) -#define STREAM_TASK_QUEUE_CAPACITY 20480 +#define STREAM_TASK_QUEUE_CAPACITY 5120 #define STREAM_TASK_QUEUE_CAPACITY_IN_SIZE (30) // clang-format off diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index ce7dc79a17..fcfda71dd4 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -153,7 +153,6 @@ void taskDbUnRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp); int32_t chkpAddExtraInfo(char* pChkpIdDir, int64_t chkpId, int64_t processId); int32_t chkpLoadExtraInfo(char* pChkpIdDir, int64_t* chkpId, int64_t* processId); -#define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX)); int32_t copyFiles(const char* src, const char* dst); uint32_t nextPow2(uint32_t x); @@ -1156,13 +1155,17 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { taosArrayDestroy(pBackend->chkpSaved); pBackend->chkpSaved = chkpDup; + chkpDup = NULL; TAOS_UNUSED(taosThreadRwlockUnlock(&pBackend->chkpDirLock)); for (int i = 0; i < taosArrayGetSize(chkpDel); i++) { int64_t id = *(int64_t*)taosArrayGet(chkpDel, i); char tbuf[256] = {0}; - sprintf(tbuf, "%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, id); + if (snprintf(tbuf, sizeof(tbuf), "%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, id) >= sizeof(tbuf)) { + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_CHECK_GOTO(code, NULL, _exception); + } stInfo("backend remove obsolete checkpoint: %s", tbuf); if (taosIsDir(tbuf)) { @@ -1187,12 +1190,17 @@ int chkpIdComp(const void* a, const void* b) { } int32_t taskDbLoadChkpInfo(STaskDbWrapper* pBackend) { int32_t code = 0; - char* pChkpDir = taosMemoryCalloc(1, 256); + int32_t nBytes = 0; + int32_t cap = 256; + char* pChkpDir = taosMemoryCalloc(1, cap); if (pChkpDir == NULL) { return terrno; } - sprintf(pChkpDir, "%s%s%s", pBackend->path, TD_DIRSEP, "checkpoints"); + nBytes = snprintf(pChkpDir, cap, "%s%s%s", pBackend->path, TD_DIRSEP, "checkpoints"); + if (nBytes >= cap) { + return TSDB_CODE_OUT_OF_RANGE; + } if (!taosIsDir(pChkpDir)) { taosMemoryFree(pChkpDir); return 0; @@ -1413,12 +1421,18 @@ int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) { if (pSnapInfo == NULL) return 0; SStreamMeta* pMeta = arg; int32_t code = 0; + int32_t cap = 256; + int32_t nBytes = 0; streamMutexLock(&pMeta->backendMutex); - char buf[128] = {0}; + char buf[256] = {0}; for (int i = 0; i < taosArrayGetSize(pSnapInfo); i++) { SStreamTaskSnap* pSnap = taosArrayGet(pSnapInfo, i); - sprintf(buf, "0x%" PRIx64 "-0x%x", pSnap->streamId, (int32_t)pSnap->taskId); + nBytes = snprintf(buf, cap, "0x%" PRIx64 "-0x%x", pSnap->streamId, (int32_t)pSnap->taskId); + if (nBytes <= 0 || nBytes >= cap) { + code = TSDB_CODE_OUT_OF_RANGE; + break; + } STaskDbWrapper** pTaskDb = taosHashGet(pMeta->pTaskDbUnique, buf, strlen(buf)); if (pTaskDb == NULL || *pTaskDb == NULL) { stWarn("stream backend:%p failed to find task db, streamId:% " PRId64 "", pMeta, pSnap->streamId); @@ -1430,7 +1444,7 @@ int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) { taskDbUnRefChkp(*pTaskDb, pSnap->chkpId); } streamMutexUnlock(&pMeta->backendMutex); - return 0; + return code; } #ifdef BUILD_NO_CALL int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId) { @@ -1685,9 +1699,6 @@ void streamBackendDelCompare(void* backend, void* arg) { taosMemoryFree(node); } } -#ifdef BUILD_NO_CALL -void streamStateDestroy_rocksdb(SStreamState* pState, bool remove) { streamStateCloseBackend(pState, remove); } -#endif void destroyRocksdbCfInst(RocksdbCfInst* inst) { int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); if (inst->pHandle) { @@ -1712,7 +1723,7 @@ void destroyRocksdbCfInst(RocksdbCfInst* inst) { } // |key|-----value------| -// |key|ttl|len|userData| +// |key|ttl|len|userData int defaultKeyComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen) { int len = aLen < bLen ? aLen : bLen; @@ -2447,43 +2458,50 @@ void taskDbDestroyChkpOpt(STaskDbWrapper* pTaskDb) { int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** stateFullPath) { int32_t code = 0; - char* statePath = taosMemoryCalloc(1, strlen(path) + 128); + int32_t cap = strlen(path) + 128, nBytes = 0; + char* statePath = NULL; + char* dbPath = NULL; + + statePath = taosMemoryCalloc(1, cap); if (statePath == NULL) { - return terrno; + TAOS_CHECK_GOTO(terrno, NULL, _err); + } + + nBytes = snprintf(statePath, cap, "%s%s%s", path, TD_DIRSEP, key); + if (nBytes < 0 || nBytes >= cap) { + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_CHECK_GOTO(code, NULL, _err); } - sprintf(statePath, "%s%s%s", path, TD_DIRSEP, key); if (!taosDirExist(statePath)) { code = taosMulMkDir(statePath); - if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); - stError("failed to create dir: %s, reason:%s", statePath, tstrerror(code)); - taosMemoryFree(statePath); - return code; - } + TAOS_CHECK_GOTO(code, NULL, _err); } - char* dbPath = taosMemoryCalloc(1, strlen(statePath) + 128); + dbPath = taosMemoryCalloc(1, cap); if (dbPath == NULL) { - taosMemoryFree(statePath); - return terrno; + TAOS_CHECK_GOTO(terrno, NULL, _err); + } + nBytes = snprintf(dbPath, cap, "%s%s%s", statePath, TD_DIRSEP, "state"); + if (nBytes < 0 || nBytes >= cap) { + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_CHECK_GOTO(code, NULL, _err); } - sprintf(dbPath, "%s%s%s", statePath, TD_DIRSEP, "state"); if (!taosDirExist(dbPath)) { code = taosMulMkDir(dbPath); - if (code != 0) { - code = TAOS_SYSTEM_ERROR(errno); - stError("failed to create dir: %s, reason:%s", dbPath, tstrerror(code)); - taosMemoryFree(statePath); - taosMemoryFree(dbPath); - return code; - } + TAOS_CHECK_GOTO(code, NULL, _err); } *dbFullPath = dbPath; *stateFullPath = statePath; return 0; +_err: + stError("failed to create dir: %s, reason:%s", dbPath, tstrerror(code)); + + taosMemoryFree(statePath); + taosMemoryFree(dbPath); + return code; } void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) { @@ -2864,6 +2882,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t int64_t streamId; int32_t taskId, dummy = 0; char suffix[64] = {0}; + int32_t code = 0; rocksdb_options_t** cfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); RocksdbCfParam* params = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam)); @@ -2873,6 +2892,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t for (int i = 0; i < nCf; i++) { char* cf = cfs[i]; char funcname[64] = {0}; + cfOpts[i] = rocksdb_options_create_copy(handle->dbOpt); if (i == 0) continue; if (3 == sscanf(cf, "0x%" PRIx64 "-%d_%s", &streamId, &taskId, funcname)) { @@ -2909,7 +2929,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t taosMemoryFree(params); taosMemoryFree(cfOpts); // fix other leak - return -1; + return TSDB_CODE_THIRDPARTY_ERROR; } else { stDebug("succ to open rocksdb cf"); } @@ -2929,8 +2949,13 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t char funcname[64] = {0}; if (3 == sscanf(cf, "0x%" PRIx64 "-%d_%s", &streamId, &taskId, funcname)) { - char idstr[128] = {0}; - sprintf(idstr, "0x%" PRIx64 "-%d", streamId, taskId); + char idstr[128] = {0}; + int32_t nBytes = snprintf(idstr, sizeof(idstr), "0x%" PRIx64 "-%d", streamId, taskId); + if (nBytes <= 0 || nBytes >= sizeof(idstr)) { + code = TSDB_CODE_OUT_OF_RANGE; + stError("failed to open cf since %s", tstrerror(code)); + return code; + } int idx = streamStateGetCfIdx(NULL, funcname); @@ -2997,117 +3022,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t taosMemoryFree(cfOpts); return 0; } -#ifdef BUILD_NO_CALL -int streamStateOpenBackend(void* backend, SStreamState* pState) { - taosAcquireRef(streamBackendId, pState->streamBackendRid); - SBackendWrapper* handle = backend; - SBackendCfWrapper* pBackendCfWrapper = taosMemoryCalloc(1, sizeof(SBackendCfWrapper)); - streamMutexLock(&handle->cfMutex); - RocksdbCfInst** ppInst = taosHashGet(handle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); - if (ppInst != NULL && *ppInst != NULL) { - RocksdbCfInst* inst = *ppInst; - pBackendCfWrapper->rocksdb = inst->db; - pBackendCfWrapper->pHandle = (void**)inst->pHandle; - pBackendCfWrapper->writeOpts = inst->wOpt; - pBackendCfWrapper->readOpts = inst->rOpt; - pBackendCfWrapper->cfOpts = (void**)(inst->cfOpt); - pBackendCfWrapper->dbOpt = handle->dbOpt; - pBackendCfWrapper->param = inst->param; - pBackendCfWrapper->pBackend = handle; - pBackendCfWrapper->pComparNode = inst->pCompareNode; - streamMutexUnlock(&handle->cfMutex); - pBackendCfWrapper->backendId = pState->streamBackendRid; - memcpy(pBackendCfWrapper->idstr, pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr)); - - int64_t id = taosAddRef(streamBackendCfWrapperId, pBackendCfWrapper); - pState->pTdbState->backendCfWrapperId = id; - pState->pTdbState->pBackendCfWrapper = pBackendCfWrapper; - stInfo("succ to open state %p on backendWrapper, %p, %s", pState, pBackendCfWrapper, pBackendCfWrapper->idstr); - - inst->pHandle = NULL; - inst->cfOpt = NULL; - inst->param = NULL; - - inst->wOpt = NULL; - inst->rOpt = NULL; - return 0; - } - streamMutexUnlock(&handle->cfMutex); - - char* err = NULL; - int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); - - RocksdbCfParam* param = taosMemoryCalloc(cfLen, sizeof(RocksdbCfParam)); - const rocksdb_options_t** cfOpt = taosMemoryCalloc(cfLen, sizeof(rocksdb_options_t*)); - for (int i = 0; i < cfLen; i++) { - cfOpt[i] = rocksdb_options_create_copy(handle->dbOpt); - // refactor later - rocksdb_block_based_table_options_t* tableOpt = rocksdb_block_based_options_create(); - rocksdb_block_based_options_set_block_cache(tableOpt, handle->cache); - rocksdb_block_based_options_set_partition_filters(tableOpt, 1); - - rocksdb_filterpolicy_t* filter = rocksdb_filterpolicy_create_bloom(15); - rocksdb_block_based_options_set_filter_policy(tableOpt, filter); - - rocksdb_options_set_block_based_table_factory((rocksdb_options_t*)cfOpt[i], tableOpt); - - param[i].tableOpt = tableOpt; - }; - - rocksdb_comparator_t** pCompare = taosMemoryCalloc(cfLen, sizeof(rocksdb_comparator_t*)); - for (int i = 0; i < cfLen; i++) { - SCfInit* cf = &ginitDict[i]; - - rocksdb_comparator_t* compare = rocksdb_comparator_create(NULL, cf->destroyCmp, cf->cmpKey, cf->cmpName); - rocksdb_options_set_comparator((rocksdb_options_t*)cfOpt[i], compare); - pCompare[i] = compare; - } - rocksdb_column_family_handle_t** cfHandle = taosMemoryCalloc(cfLen, sizeof(rocksdb_column_family_handle_t*)); - pBackendCfWrapper->rocksdb = handle->db; - pBackendCfWrapper->pHandle = (void**)cfHandle; - pBackendCfWrapper->writeOpts = rocksdb_writeoptions_create(); - pBackendCfWrapper->readOpts = rocksdb_readoptions_create(); - pBackendCfWrapper->cfOpts = (void**)cfOpt; - pBackendCfWrapper->dbOpt = handle->dbOpt; - pBackendCfWrapper->param = param; - pBackendCfWrapper->pBackend = handle; - pBackendCfWrapper->backendId = pState->streamBackendRid; - taosThreadRwlockInit(&pBackendCfWrapper->rwLock, NULL); - SCfComparator compare = {.comp = pCompare, .numOfComp = cfLen}; - pBackendCfWrapper->pComparNode = streamBackendAddCompare(handle, &compare); - rocksdb_writeoptions_disable_WAL(pBackendCfWrapper->writeOpts, 1); - memcpy(pBackendCfWrapper->idstr, pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr)); - - int64_t id = taosAddRef(streamBackendCfWrapperId, pBackendCfWrapper); - pState->pTdbState->backendCfWrapperId = id; - pState->pTdbState->pBackendCfWrapper = pBackendCfWrapper; - stInfo("succ to open state %p on backendWrapper %p %s", pState, pBackendCfWrapper, pBackendCfWrapper->idstr); - return 0; -} - -void streamStateCloseBackend(SStreamState* pState, bool remove) { - SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - SBackendWrapper* pHandle = wrapper->pBackend; - - stInfo("start to close state on backend: %p", pHandle); - - streamMutexLock(&pHandle->cfMutex); - RocksdbCfInst** ppInst = taosHashGet(pHandle->cfInst, wrapper->idstr, strlen(pState->pTdbState->idstr) + 1); - if (ppInst != NULL && *ppInst != NULL) { - RocksdbCfInst* inst = *ppInst; - taosMemoryFree(inst); - taosHashRemove(pHandle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); - } - streamMutexUnlock(&pHandle->cfMutex); - - char* status[] = {"close", "drop"}; - stInfo("start to %s state %p on backendWrapper %p %s", status[remove == false ? 0 : 1], pState, wrapper, - wrapper->idstr); - wrapper->remove |= remove; // update by other pState - taosReleaseRef(streamBackendCfWrapperId, pState->pTdbState->backendCfWrapperId); -} -#endif void streamStateDestroyCompar(void* arg) { SCfComparator* comp = (SCfComparator*)arg; for (int i = 0; i < comp->numOfComp; i++) { @@ -3386,7 +3301,8 @@ int32_t streamStateGetFirst_rocksdb(SStreamState* pState, SWinKey* key) { return streamStateDel_rocksdb(pState, &tmp); } -int32_t streamStateFillGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { +int32_t streamStateFillGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, + int32_t* pVLen) { if (!pCur) { return -1; } @@ -4343,7 +4259,7 @@ int32_t streamStatePutParTag_rocksdb(SStreamState* pState, int64_t groupId, cons void streamStateParTagSeekKeyNext_rocksdb(SStreamState* pState, const int64_t groupId, SStreamStateCur* pCur) { if (pCur == NULL) { - return ; + return; } STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; pCur->number = pState->number; @@ -4353,13 +4269,13 @@ void streamStateParTagSeekKeyNext_rocksdb(SStreamState* pState, const int64_t gr int i = streamStateGetCfIdx(pState, "partag"); if (i < 0) { stError("streamState failed to put to cf name:%s", "partag"); - return ; + return; } char buf[128] = {0}; int32_t klen = ginitDict[i].enFunc((void*)&groupId, buf); if (!streamStateIterSeekAndValid(pCur->iter, buf, klen)) { - return ; + return; } // skip ttl expired data while (rocksdb_iter_valid(pCur->iter) && iterValueIsStale(pCur->iter)) { @@ -4371,13 +4287,14 @@ void streamStateParTagSeekKeyNext_rocksdb(SStreamState* pState, const int64_t gr size_t kLen = 0; char* keyStr = (char*)rocksdb_iter_key(pCur->iter, &kLen); TAOS_UNUSED(parKeyDecode((void*)&curGroupId, keyStr)); - if (curGroupId > groupId) return ; + if (curGroupId > groupId) return; rocksdb_iter_next(pCur->iter); } } -int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGroupId, const void** pVal, int32_t* pVLen) { +int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGroupId, const void** pVal, + int32_t* pVLen) { stDebug("streamStateFillGetKVByCur_rocksdb"); if (!pCur) { return -1; @@ -4720,7 +4637,7 @@ int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) { if (fname == NULL) { return terrno; } - TAOS_UNUSED(strncpy(fname, name, len)); + tstrncpy(fname, name, strlen(name)); if (taosArrayPush(diff, &fname) == NULL) { taosMemoryFree(fname); return terrno; @@ -4744,17 +4661,32 @@ int32_t compareHashTable(SHashObj* p1, SHashObj* p2, SArray* add, SArray* del) { void hashTableToDebug(SHashObj* pTbl, char** buf) { size_t sz = taosHashGetSize(pTbl); int32_t total = 0; - char* p = taosMemoryCalloc(1, sz * 16 + 4); - void* pIter = taosHashIterate(pTbl, NULL); + int32_t cap = sz * 16 + 4; + + char* p = taosMemoryCalloc(1, cap); + if (p == NULL) { + stError("failed to alloc memory for stream snapshot debug info"); + return; + } + + void* pIter = taosHashIterate(pTbl, NULL); while (pIter) { size_t len = 0; char* name = taosHashGetKey(pIter, &len); - char* tname = taosMemoryCalloc(1, len + 1); - memcpy(tname, name, len); - total += sprintf(p + total, "%s,", tname); + if (name == NULL || len <= 0) { + pIter = taosHashIterate(pTbl, pIter); + continue; + } + int32_t left = cap - strlen(p); + int32_t nBytes = snprintf(p + total, left, "%s,", name); + if (nBytes <= 0 || nBytes >= left) { + stError("failed to debug snapshot info since %s", tstrerror(TSDB_CODE_OUT_OF_RANGE)); + taosMemoryFree(p); + return; + } pIter = taosHashIterate(pTbl, pIter); - taosMemoryFree(tname); + total += nBytes; } if (total > 0) { p[total - 1] = 0; @@ -4765,13 +4697,30 @@ void strArrayDebugInfo(SArray* pArr, char** buf) { int32_t sz = taosArrayGetSize(pArr); if (sz <= 0) return; - char* p = (char*)taosMemoryCalloc(1, 64 + sz * 64); - int32_t total = 0; + int32_t code = 0; + int32_t total = 0, nBytes = 0; + int32_t cap = 64 + sz * 64; + + char* p = (char*)taosMemoryCalloc(1, cap); + if (p == NULL) { + stError("failed to alloc memory for stream snapshot debug info"); + return; + } for (int i = 0; i < sz; i++) { - char* name = taosArrayGetP(pArr, i); - total += sprintf(p + total, "%s,", name); + char* name = taosArrayGetP(pArr, i); + int32_t left = cap - strlen(p); + nBytes = snprintf(p + total, left, "%s,", name); + if (nBytes <= 0 || nBytes >= left) { + code = TSDB_CODE_OUT_OF_RANGE; + stError("failed to debug snapshot info since %s", tstrerror(code)); + taosMemoryFree(p); + return; + } + + total += nBytes; } + p[total - 1] = 0; *buf = p; @@ -4781,16 +4730,16 @@ void dbChkpDebugInfo(SDbChkp* pDb) { char* p[4] = {NULL}; hashTableToDebug(pDb->pSstTbl[pDb->idx], &p[0]); - stTrace("chkp previous file: [%s]", p[0]); + if (p[0]) stTrace("chkp previous file: [%s]", p[0]); hashTableToDebug(pDb->pSstTbl[1 - pDb->idx], &p[1]); - stTrace("chkp curr file: [%s]", p[1]); + if (p[1]) stTrace("chkp curr file: [%s]", p[1]); strArrayDebugInfo(pDb->pAdd, &p[2]); - stTrace("chkp newly addded file: [%s]", p[2]); + if (p[2]) stTrace("chkp newly addded file: [%s]", p[2]); strArrayDebugInfo(pDb->pDel, &p[3]); - stTrace("chkp newly deleted file: [%s]", p[3]); + if (p[3]) stTrace("chkp newly deleted file: [%s]", p[3]); for (int i = 0; i < 4; i++) { taosMemoryFree(p[i]); @@ -4882,7 +4831,7 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) { return terrno; } - TAOS_UNUSED(strncpy(fname, name, len)); + tstrncpy(fname, name, strlen(name)); if (taosArrayPush(p->pAdd, &fname) == NULL) { taosMemoryFree(fname); TAOS_UNUSED(taosThreadRwlockUnlock(&p->rwLock)); @@ -5357,7 +5306,8 @@ SStreamStateCur* streamStateSeekKeyPrev_rocksdb(SStreamState* pState, const SWin return NULL; } -int32_t streamStateGetGroupKVByCur_rocksdb(SStreamState* pState, SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { +int32_t streamStateGetGroupKVByCur_rocksdb(SStreamState* pState, SStreamStateCur* pCur, SWinKey* pKey, + const void** pVal, int32_t* pVLen) { if (!pCur) { return -1; } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 64194e349c..01cd201a1b 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -613,7 +613,6 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV { // destroy the related fill-history tasks // drop task should not in the meta-lock, and drop the related fill-history task now - streamMetaWUnLock(pMeta); if (pReq->dropRelHTask) { code = streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); @@ -621,7 +620,6 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV id, vgId, pReq->taskId, numOfTasks); } - streamMetaWLock(pMeta); if (pReq->dropRelHTask) { code = streamMetaCommit(pMeta); } @@ -656,9 +654,11 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV pInfo->processedVer <= pReq->checkpointVer); if (!valid) { - stFatal("invalid checkpoint id check, current checkpointId:%" PRId64 " checkpointVer:%" PRId64 - " processedVer:%" PRId64 " req checkpointId:%" PRId64 " checkpointVer:%" PRId64, - pInfo->checkpointId, pInfo->checkpointVer, pInfo->processedVer, pReq->checkpointId, pReq->checkpointVer); + stFatal("s-task:%s invalid checkpointId update info recv, current checkpointId:%" PRId64 " checkpointVer:%" PRId64 + " processedVer:%" PRId64 " req checkpointId:%" PRId64 " checkpointVer:%" PRId64 " discard it", + id, pInfo->checkpointId, pInfo->checkpointVer, pInfo->processedVer, pReq->checkpointId, + pReq->checkpointVer); + streamMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INTERNAL_ERROR; } @@ -695,8 +695,6 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV return TSDB_CODE_SUCCESS; } - streamMetaWUnLock(pMeta); - // drop task should not in the meta-lock, and drop the related fill-history task now if (pReq->dropRelHTask) { code = streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); @@ -705,9 +703,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV (int32_t)pReq->hTaskId, numOfTasks); } - streamMetaWLock(pMeta); code = streamMetaCommit(pMeta); - return TSDB_CODE_SUCCESS; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 318720b5b0..85f287f301 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -522,7 +522,10 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int if (pItem->type == STREAM_INPUT__GET_RES) { const SStreamTrigger* pTrigger = (const SStreamTrigger*)pInput; code = qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); - + if (pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) { + stDebug("s-task:%s set force_window_close as source block, skey:%"PRId64, id, pTrigger->pBlock->info.window.skey); + (*pVer) = pTrigger->pBlock->info.window.skey; + } } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput; code = qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); @@ -671,7 +674,7 @@ static int32_t doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock doRecordThroughput(&pTask->execInfo, totalBlocks, totalSize, blockSize, st, pTask->id.idStr); - // update the currentVer if processing the submit blocks. + // update the currentVer if processing the submitted blocks. if (!(pInfo->checkpointVer <= pInfo->nextProcessVer && ver >= pInfo->checkpointVer)) { stError("s-task:%s invalid info, checkpointVer:%" PRId64 ", nextProcessVer:%" PRId64 " currentVer:%" PRId64, id, pInfo->checkpointVer, pInfo->nextProcessVer, ver); @@ -688,6 +691,34 @@ static int32_t doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock return code; } +// do nothing after sync executor state to storage backend, untill checkpoint is completed. +static int32_t doHandleChkptBlock(SStreamTask* pTask) { + int32_t code = 0; + const char* id = pTask->id.idStr; + + streamMutexLock(&pTask->lock); + SStreamTaskState pState = streamTaskGetStatus(pTask); + if (pState.state == TASK_STATUS__CK) { // todo other thread may change the status + stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name); + code = streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue + } else { // todo refactor + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + code = streamTaskSendCheckpointSourceRsp(pTask); + } else { + code = streamTaskSendCheckpointReadyMsg(pTask); + } + + if (code != TSDB_CODE_SUCCESS) { + // todo: let's retry send rsp to upstream/mnode + stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%d, code:%s", id, 0, + tstrerror(code)); + } + } + + streamMutexUnlock(&pTask->lock); + return code; +} + int32_t flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock) { const char* id = pTask->id.idStr; @@ -832,36 +863,16 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { } } - if (type != STREAM_INPUT__CHECKPOINT) { + if (type == STREAM_INPUT__CHECKPOINT) { + code = doHandleChkptBlock(pTask); + streamFreeQitem(pInput); + return code; + } else { code = doStreamTaskExecImpl(pTask, pInput, numOfBlocks); streamFreeQitem(pInput); if (code) { return code; } - } else { // todo other thread may change the status - // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. - streamMutexLock(&pTask->lock); - SStreamTaskState pState = streamTaskGetStatus(pTask); - if (pState.state == TASK_STATUS__CK) { - stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name); - code = streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue - } else { // todo refactor - if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - code = streamTaskSendCheckpointSourceRsp(pTask); - } else { - code = streamTaskSendCheckpointReadyMsg(pTask); - } - - if (code != TSDB_CODE_SUCCESS) { - // todo: let's retry send rsp to upstream/mnode - stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%d, code:%s", id, 0, - tstrerror(code)); - } - } - - streamMutexUnlock(&pTask->lock); - streamFreeQitem(pInput); - return code; } } } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index a6f87711bf..8e8e300ee6 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -99,7 +99,7 @@ int32_t metaRefMgtInit() { void metaRefMgtCleanup() { void* pIter = taosHashIterate(gMetaRefMgt.pTable, NULL); while (pIter) { - int64_t* p = *(int64_t**) pIter; + int64_t* p = *(int64_t**)pIter; taosMemoryFree(p); pIter = taosHashIterate(gMetaRefMgt.pTable, pIter); } @@ -118,14 +118,14 @@ int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) { if (p == NULL) { code = taosHashPut(gMetaRefMgt.pTable, &rid, sizeof(rid), &rid, sizeof(void*)); if (code) { - stError("vgId:%d failed to put into refId mgt, refId:%" PRId64" %p, code:%s", (int32_t)vgId, *rid, rid, + stError("vgId:%d failed to put into refId mgt, refId:%" PRId64 " %p, code:%s", (int32_t)vgId, *rid, rid, tstrerror(code)); return code; - } else { // not -// stInfo("add refId:%"PRId64" vgId:%d, %p", *rid, (int32_t)vgId, rid); + } else { // not + // stInfo("add refId:%"PRId64" vgId:%d, %p", *rid, (int32_t)vgId, rid); } } else { - stFatal("try to add refId:%"PRId64" vgId:%d, %p that already added into mgt", *rid, (int32_t) vgId, rid); + stFatal("try to add refId:%" PRId64 " vgId:%d, %p that already added into mgt", *rid, (int32_t)vgId, rid); } streamMutexUnlock(&gMetaRefMgt.mutex); @@ -292,6 +292,7 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key) void* p = taskDbAddRef(*ppBackend); if (p == NULL) { stError("s-task:0x%x failed to ref backend", pTask->id.taskId); + streamMutexUnlock(&pMeta->backendMutex); return TSDB_CODE_FAILED; } @@ -501,8 +502,6 @@ _err: void streamMetaInitBackend(SStreamMeta* pMeta) { pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId, pMeta->vgId); if (pMeta->streamBackend == NULL) { - streamMetaWUnLock(pMeta); - while (1) { streamMetaWLock(pMeta); pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId, pMeta->vgId); @@ -671,7 +670,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { int64_t refId = pTask->id.refId; int32_t ret = taosRemoveRef(streamTaskRefPool, pTask->id.refId); if (ret != 0) { - stError("s-task:0x%x failed to remove ref, refId:%"PRId64, (int32_t) id[1], refId); + stError("s-task:0x%x failed to remove ref, refId:%" PRId64, (int32_t)id[1], refId); } } else { stDebug("s-task:%s vgId:%d refId:%" PRId64 " task meta save to disk", pTask->id.idStr, vgId, pTask->id.refId); @@ -729,14 +728,14 @@ int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTa int32_t ret = taosRemoveRef(streamTaskRefPool, refId); if (ret != 0) { - stError("s-task:0x%x failed to remove ref, refId:%"PRId64, (int32_t) id.taskId, refId); + stError("s-task:0x%x failed to remove ref, refId:%" PRId64, (int32_t)id.taskId, refId); } return code; } if ((code = streamMetaSaveTask(pMeta, pTask)) != 0) { int32_t unused = taosHashRemove(pMeta->pTasksMap, &id, sizeof(id)); - void* pUnused = taosArrayPop(pMeta->pTaskList); + void* pUnused = taosArrayPop(pMeta->pTaskList); int32_t ret = taosRemoveRef(streamTaskRefPool, refId); if (ret) { @@ -747,7 +746,7 @@ int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTa if ((code = streamMetaCommit(pMeta)) != 0) { int32_t unused = taosHashRemove(pMeta->pTasksMap, &id, sizeof(id)); - void* pUnused = taosArrayPop(pMeta->pTaskList); + void* pUnused = taosArrayPop(pMeta->pTaskList); int32_t ret = taosRemoveRef(streamTaskRefPool, refId); if (ret) { @@ -785,7 +784,7 @@ int32_t streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_ SStreamTask* p = taosAcquireRef(streamTaskRefPool, *pTaskRefId); if (p == NULL) { - stDebug("s-task:%x failed to acquire task refId:%"PRId64", may have been destoried", taskId, *pTaskRefId); + stDebug("s-task:%x failed to acquire task refId:%" PRId64 ", may have been destoried", taskId, *pTaskRefId); return TSDB_CODE_STREAM_TASK_NOT_EXIST; } @@ -908,8 +907,6 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t int32_t code = 0; STaskId id = {.streamId = streamId, .taskId = taskId}; - streamMetaWLock(pMeta); - code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask); if (code == 0) { // desc the paused task counter @@ -950,18 +947,15 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t pTask->info.delaySchedParam = 0; } - int64_t refId = pTask->id.refId; int32_t ret = taosRemoveRef(streamTaskRefPool, refId); if (ret != 0) { - stError("s-task:0x%x failed to remove ref, refId:%"PRId64, (int32_t) id.taskId, refId); + stError("s-task:0x%x failed to remove ref, refId:%" PRId64, (int32_t)id.taskId, refId); } streamMetaReleaseTask(pMeta, pTask); - streamMetaWUnLock(pMeta); } else { stDebug("vgId:%d failed to find the task:0x%x, it may have been dropped already", vgId, taskId); - streamMetaWUnLock(pMeta); } return 0; @@ -1121,7 +1115,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { STaskId id = streamTaskGetTaskId(pTask); tFreeStreamTask(pTask); - void* px = taosArrayPush(pRecycleList, &id); + void* px = taosArrayPush(pRecycleList, &id); if (px == NULL) { stError("s-task:0x%x failed record the task into recycle list due to out of memory", taskId); } @@ -1171,7 +1165,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { continue; } - stInfo("s-task:0x%x vgId:%d set refId:%"PRId64, (int32_t) id.taskId, vgId, pTask->id.refId); + stInfo("s-task:0x%x vgId:%d set refId:%" PRId64, (int32_t)id.taskId, vgId, pTask->id.refId); if (pTask->info.fillHistory == 0) { int32_t val = atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1); } diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index 56bb7437c1..1f76f349ae 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -211,6 +211,7 @@ static int32_t doCreateForceWindowTrigger(SStreamTask* pTask, int32_t* pNextTrig const char* id = pTask->id.idStr; int8_t precision = pTask->info.interval.precision; SStreamTrigger* pTrigger = NULL; + bool isFull = false; while (1) { code = streamCreateForcewindowTrigger(&pTrigger, pTask->info.delaySchedParam, &pTask->info.interval, @@ -233,7 +234,6 @@ static int32_t doCreateForceWindowTrigger(SStreamTask* pTask, int32_t* pNextTrig // check whether the time window gaps exist or not int64_t now = taosGetTimestamp(precision); - int64_t ekey = pTrigger->pBlock->info.window.skey + pTask->info.interval.interval; // there are gaps, needs to be filled STimeWindow w = pTrigger->pBlock->info.window; @@ -245,13 +245,18 @@ static int32_t doCreateForceWindowTrigger(SStreamTask* pTask, int32_t* pNextTrig } pTask->status.latestForceWindow = w; - if (ekey + pTask->info.watermark + pTask->info.interval.interval > now) { - int64_t prev = convertTimePrecision(*pNextTrigger, precision, TSDB_TIME_PRECISION_MILLI); + isFull = streamQueueIsFull(pTask->inputq.queue); + + if ((w.ekey + pTask->info.watermark + pTask->info.interval.interval > now) || isFull) { + int64_t prev = convertTimePrecision(*pNextTrigger, precision, TSDB_TIME_PRECISION_MILLI); + if (!isFull) { + *pNextTrigger = w.ekey + pTask->info.watermark + pTask->info.interval.interval - now; + } - *pNextTrigger = ekey + pTask->info.watermark + pTask->info.interval.interval - now; *pNextTrigger = convertTimePrecision(*pNextTrigger, precision, TSDB_TIME_PRECISION_MILLI); - stDebug("s-task:%s generate %d time window(s), trigger delay adjust from %" PRId64 " to %d", id, num, prev, - *pNextTrigger); + pTask->chkInfo.nextProcessVer = w.ekey + pTask->info.interval.interval; + stDebug("s-task:%s generate %d time window(s), trigger delay adjust from %" PRId64 " to %d, set ver:%" PRId64, id, + num, prev, *pNextTrigger, pTask->chkInfo.nextProcessVer); return code; } else { stDebug("s-task:%s gap exist for force_window_close, current force_window_skey:%" PRId64, id, w.skey); @@ -308,7 +313,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) { } if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK) { - nextTrigger = TRIGGER_RECHECK_INTERVAL; // retry in 10 seec + nextTrigger = TRIGGER_RECHECK_INTERVAL; // retry in 10 sec stDebug("s-task:%s in checkpoint procedure, not retrieve result, next:%dms", id, TRIGGER_RECHECK_INTERVAL); } else { if (pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE && pTask->info.taskLevel == TASK_LEVEL__SOURCE) { diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 5461b5899b..89f0ea9e1f 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -120,7 +120,8 @@ SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, i SStreamTask* pStreamTask = pTask; pState->streamId = streamId; pState->taskId = taskId; - TAOS_UNUSED(tsnprintf(pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr), "0x%" PRIx64 "-0x%x", pState->streamId, pState->taskId)); + TAOS_UNUSED(tsnprintf(pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr), "0x%" PRIx64 "-0x%x", + pState->streamId, pState->taskId)); code = streamTaskSetDb(pStreamTask->pMeta, pTask, pState->pTdbState->idstr); QUERY_CHECK_CODE(code, lino, _end); @@ -539,7 +540,6 @@ int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId) { void streamStateDestroy(SStreamState* pState, bool remove) { streamFileStateDestroy(pState->pFileState); - // streamStateDestroy_rocksdb(pState, remove); tSimpleHashCleanup(pState->parNameMap); // do nothong taosMemoryFreeClear(pState->pTdbState); @@ -584,7 +584,8 @@ int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey return getCountWinResultBuff(pState->pFileState, pKey, winCount, ppVal, pVLen, pWinCode); } -int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen) { +int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, + int32_t* pVLen) { return createCountWinResultBuff(pState->pFileState, pKey, winCount, pVal, pVLen); } @@ -605,9 +606,7 @@ SStreamStateCur* streamStateGroupGetCur(SStreamState* pState) { return pCur; } -void streamStateGroupCurNext(SStreamStateCur* pCur) { - streamFileStateGroupCurNext(pCur); -} +void streamStateGroupCurNext(SStreamStateCur* pCur) { streamFileStateGroupCurNext(pCur); } int32_t streamStateGroupGetKVByCur(SStreamStateCur* pCur, int64_t* pKey, void** pVal, int32_t* pVLen) { if (pVal != NULL) { @@ -616,13 +615,9 @@ int32_t streamStateGroupGetKVByCur(SStreamStateCur* pCur, int64_t* pKey, void** return streamFileStateGroupGetKVByCur(pCur, pKey, pVal, pVLen); } -void streamStateClearExpiredState(SStreamState* pState) { - clearExpiredState(pState->pFileState); -} +void streamStateClearExpiredState(SStreamState* pState) { clearExpiredState(pState->pFileState); } -void streamStateSetFillInfo(SStreamState* pState) { - setFillInfo(pState->pFileState); -} +void streamStateSetFillInfo(SStreamState* pState) { setFillInfo(pState->pFileState); } int32_t streamStateGetPrev(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal, int32_t* pVLen, int32_t* pWinCode) { diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index dcabadb8bd..05edad0f5f 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -947,6 +947,7 @@ int32_t recoverSession(SStreamFileState* pFileState, int64_t ckId) { } SRowBuffPos* pPos = createSessionWinBuff(pFileState, &key, pVal, &vlen); + pPos->beUsed = false; winRes = putSessionWinResultBuff(pFileState, pPos); if (winRes != TSDB_CODE_SUCCESS) { break; @@ -1008,6 +1009,7 @@ int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { memcpy(pNewPos->pRowBuff, pVal, vlen); taosMemoryFreeClear(pVal); pNewPos->beFlushed = true; + pNewPos->beUsed = false; qDebug("===stream=== read checkpoint state from disc. %s", __func__); code = tSimpleHashPut(pFileState->rowStateBuff, pNewPos->pKey, pFileState->keyLen, &pNewPos, POINTER_BYTES); if (code != TSDB_CODE_SUCCESS) { @@ -1090,6 +1092,7 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) { if (vlen != pFileState->rowSize) { qError("row size mismatch, expect:%d, actual:%d", pFileState->rowSize, vlen); + destroyRowBuffPos(pNewPos); code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; taosMemoryFreeClear(pVal); QUERY_CHECK_CODE(code, lino, _end); @@ -1098,6 +1101,7 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) { memcpy(pNewPos->pRowBuff, pVal, vlen); taosMemoryFreeClear(pVal); pNewPos->beFlushed = true; + pNewPos->beUsed = false; qDebug("===stream=== read checkpoint state from disc. %s", __func__); winRes = tSimpleHashPut(pFileState->rowStateBuff, pNewPos->pKey, pFileState->keyLen, &pNewPos, POINTER_BYTES); if (winRes != TSDB_CODE_SUCCESS) { @@ -1228,6 +1232,8 @@ void setFillInfo(SStreamFileState* pFileState) { } void clearExpiredState(SStreamFileState* pFileState) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SSHashObj* pSearchBuff = pFileState->searchBuff; void* pIte = NULL; int32_t iter = 0; @@ -1246,6 +1252,13 @@ void clearExpiredState(SStreamFileState* pFileState) { } taosArrayRemoveBatch(pWinStates, 0, size - 1, NULL); } + code = clearRowBuff(pFileState); + QUERY_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } int32_t getStateSearchRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, void** pVal, int32_t* pVLen, diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 0fe074084f..b9e2d3435f 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1464,7 +1464,7 @@ int32_t syncNodeRestore(SSyncNode* pSyncNode) { // if (endIndex != lastVer + 1) return TSDB_CODE_SYN_INTERNAL_ERROR; pSyncNode->commitIndex = TMAX(pSyncNode->commitIndex, commitIndex); - sInfo("vgId:%d, restore sync until commitIndex:%" PRId64, pSyncNode->vgId, pSyncNode->commitIndex); + sInfo("vgId:%d, restore began, and keep syncing until commitIndex:%" PRId64, pSyncNode->vgId, pSyncNode->commitIndex); if (pSyncNode->fsmState != SYNC_FSM_STATE_INCOMPLETE && (code = syncLogBufferCommit(pSyncNode->pLogBuf, pSyncNode, pSyncNode->commitIndex)) < 0) { @@ -2933,7 +2933,7 @@ void syncNodeLogConfigInfo(SSyncNode* ths, SSyncCfg* cfg, char* str) { n += tsnprintf(buf + n, len - n, "%s", "{"); for (int i = 0; i < ths->peersEpset->numOfEps; i++) { n += tsnprintf(buf + n, len - n, "%s:%d%s", ths->peersEpset->eps[i].fqdn, ths->peersEpset->eps[i].port, - (i + 1 < ths->peersEpset->numOfEps ? ", " : "")); + (i + 1 < ths->peersEpset->numOfEps ? ", " : "")); } n += tsnprintf(buf + n, len - n, "%s", "}"); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 65c7f9761e..0fedd96e4c 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -62,7 +62,7 @@ bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* } char ipbuf[TD_IP_LEN] = {0}; - tinet_ntoa(ipbuf, ipv4); + taosInetNtoa(ipbuf, ipv4); raftId->addr = SYNC_ADDR(pInfo); raftId->vgId = vgId; diff --git a/source/libs/sync/test/sync_test_lib/src/syncIO.c b/source/libs/sync/test/sync_test_lib/src/syncIO.c index f5a32b98d9..088a82a009 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncIO.c +++ b/source/libs/sync/test/sync_test_lib/src/syncIO.c @@ -520,7 +520,7 @@ void syncUtilU642Addr(uint64_t u64, char *host, int64_t len, uint16_t *port) { uint32_t hostU32 = (uint32_t)((u64 >> 32) & 0x00000000FFFFFFFF); struct in_addr addr = {.s_addr = hostU32}; - taosInetNtoa(addr, host, len); + taosInetNtop(addr, host, len); *port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16); } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 02ab997f69..aeec8f93f0 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -52,7 +52,7 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i pDb->encryptAlgorithm = encryptAlgorithm; if (encryptKey != NULL) { - strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN + 1); } ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache)); diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 2753fe30d6..c2f97982f5 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -459,7 +459,7 @@ static char *tdbEncryptPage(SPager *pPager, char *pPageData, int32_t pageSize, c opts.source = pPageData + count; opts.result = packetData; opts.unitLen = 128; - strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN + 1); int32_t newLen = CBC_Encrypt(&opts); @@ -927,7 +927,7 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage opts.source = pPage->pData + count; opts.result = packetData; opts.unitLen = 128; - strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); + tstrncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN + 1); int newLen = CBC_Decrypt(&opts); diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 51fda85f0a..2c97a9ac85 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -200,8 +200,8 @@ bool tfsIsSameFile(const STfsFile *pFile1, const STfsFile *pFile2) { if (pFile1->did.level != pFile2->did.level) return false; if (pFile1->did.id != pFile2->did.id) return false; char nameBuf1[TMPNAME_LEN], nameBuf2[TMPNAME_LEN]; - (void)strncpy(nameBuf1, pFile1->rname, TMPNAME_LEN); - (void)strncpy(nameBuf2, pFile2->rname, TMPNAME_LEN); + tstrncpy(nameBuf1, pFile1->rname, TMPNAME_LEN); + tstrncpy(nameBuf2, pFile2->rname, TMPNAME_LEN); nameBuf1[TMPNAME_LEN - 1] = 0; nameBuf2[TMPNAME_LEN - 1] = 0; TAOS_UNUSED(taosRealPath(nameBuf1, NULL, TMPNAME_LEN)); @@ -573,7 +573,7 @@ static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { TAOS_RETURN(TSDB_CODE_FS_INVLD_CFG); } - strncpy(pCfg->dir, dirName, TSDB_FILENAME_LEN); + tstrncpy(pCfg->dir, dirName, TSDB_FILENAME_LEN); TAOS_RETURN(0); } diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index bdcbfeb1cd..0c0f723b8b 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -224,7 +224,7 @@ static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, return TSDB_CODE_RPC_FQDN_ERROR; } char buf[TD_IP_LEN] = {0}; - tinet_ntoa(buf, ip); + taosInetNtoa(buf, ip); int ret = uv_ip4_addr(buf, port, dest); if (ret != 0) { tError("http-report failed to get addr, reason:%s", uv_err_name(ret)); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a90c546796..ed84404d05 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1862,8 +1862,8 @@ static FORCE_INLINE int32_t cliUpdateFqdnCache(SHashObj* cache, char* fqdn) { if (v != NULL) { if (addr != *v) { char old[TSDB_FQDN_LEN] = {0}, new[TSDB_FQDN_LEN] = {0}; - tinet_ntoa(old, *v); - tinet_ntoa(new, addr); + taosInetNtoa(old, *v); + taosInetNtoa(new, addr); tWarn("update ip of fqdn:%s, old: %s, new: %s", fqdn, old, new); code = taosHashPut(cache, fqdn, len, &addr, sizeof(addr)); } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 69e70a0f47..998668beac 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -332,7 +332,7 @@ _exit: wError("vgId:%d, %s failed at line %d since %s", pWal->cfg.vgId, __func__, lino, tstrerror(code)); } - TAOS_RETURN(TSDB_CODE_SUCCESS); + TAOS_RETURN(code); } static FORCE_INLINE int32_t walCheckAndRoll(SWal *pWal) { @@ -621,7 +621,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy opts.source = newBody; opts.result = newBodyEncrypted; opts.unitLen = 16; - TAOS_UNUSED(strncpy((char *)opts.key, pWal->cfg.encryptKey, ENCRYPT_KEY_LEN)); + tstrncpy((char *)opts.key, pWal->cfg.encryptKey, ENCRYPT_KEY_LEN + 1); int32_t count = CBC_Encrypt(&opts); @@ -689,8 +689,13 @@ _exit: static int32_t walInitWriteFile(SWal *pWal) { TdFilePtr pIdxTFile, pLogTFile; + int64_t fileFirstVer = -1; + int32_t code = 0; SWalFileInfo *pRet = taosArrayGetLast(pWal->fileInfoSet); - int64_t fileFirstVer = pRet->firstVer; + if (pRet == NULL) { + fileFirstVer = pWal->vers.lastVer + 1; + } + fileFirstVer = pRet->firstVer; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, fileFirstVer, fnameStr); @@ -707,6 +712,13 @@ static int32_t walInitWriteFile(SWal *pWal) { // switch file pWal->pIdxFile = pIdxTFile; pWal->pLogFile = pLogTFile; + if (taosArrayGetSize(pWal->fileInfoSet) == 0) { + code = walRollFileInfo(pWal); + if (code < 0) { + wError("vgId:%d, failed to roll file info while init write file since %s", pWal->cfg.vgId, terrstr()); + TAOS_RETURN(code); + } + } pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; TAOS_RETURN(TSDB_CODE_SUCCESS); diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index 32609301a9..96deaa4196 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -1,5 +1,11 @@ aux_source_directory(src OS_SRC) -add_library(os STATIC ${OS_SRC}) +if(NOT ${TD_WINDOWS}) + add_definitions(-DTHREAD_SAFE=1) + aux_source_directory(src/timezone OS_TZ) + add_library(os STATIC ${OS_SRC} ${OS_TZ}) +else() + add_library(os STATIC ${OS_SRC}) +endif(NOT ${TD_WINDOWS}) target_include_directories( os PUBLIC "${TD_SOURCE_DIR}/include/os" @@ -70,6 +76,11 @@ IF (JEMALLOC_ENABLED) target_link_libraries(os PUBLIC -L${CMAKE_BINARY_DIR}/build/lib -ljemalloc) ENDIF () +#if(NOT ${TD_WINDOWS}) +# find_library(tz libtz.a "${TD_SOURCE_DIR}/contrib/tz") +# target_link_libraries(os PUBLIC ${tz}) +#endif(NOT ${TD_WINDOWS}) + if(${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index c891ee4579..5da2307cb3 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -19,7 +19,6 @@ typedef union { volatile int64_t i; volatile double d; - //double d; } double_number; #ifdef WINDOWS @@ -345,7 +344,7 @@ void atomic_store_64(int64_t volatile* ptr, int64_t val) { #endif } -double atomic_store_double(double volatile *ptr, double val){ +void atomic_store_double(double volatile* ptr, double val) { for (;;) { double_number old_num = {0}; old_num.d = *ptr; // current old value @@ -354,9 +353,9 @@ double atomic_store_double(double volatile *ptr, double val){ new_num.d = val; double_number ret_num = {0}; - ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); - if (ret_num.i == old_num.i) return ret_num.d; + if (ret_num.i == old_num.i) return; } } @@ -414,16 +413,16 @@ int64_t atomic_exchange_64(int64_t volatile* ptr, int64_t val) { #endif } -double atomic_exchange_double(double volatile *ptr, int64_t val){ +double atomic_exchange_double(double volatile* ptr, double val) { for (;;) { double_number old_num = {0}; old_num.d = *ptr; // current old value double_number new_num = {0}; - int64_t iNew = val; + new_num.d = val; double_number ret_num = {0}; - ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); if (ret_num.i == old_num.i) { return ret_num.d; @@ -589,7 +588,7 @@ int64_t atomic_fetch_add_64(int64_t volatile* ptr, int64_t val) { #endif } -double atomic_fetch_add_double(double volatile *ptr, double val){ +double atomic_fetch_add_double(double volatile* ptr, double val) { for (;;) { double_number old_num = {0}; old_num.d = *ptr; // current old value @@ -598,13 +597,13 @@ double atomic_fetch_add_double(double volatile *ptr, double val){ new_num.d = old_num.d + val; double_number ret_num = {0}; - ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); if (ret_num.i == old_num.i) return ret_num.d; } } -void* atomic_fetch_add_ptr(void* ptr, void* val) { +void* atomic_fetch_add_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) @@ -710,7 +709,7 @@ int64_t atomic_fetch_sub_64(int64_t volatile* ptr, int64_t val) { #endif } -double atomic_fetch_sub_double(double volatile *ptr, double val){ +double atomic_fetch_sub_double(double volatile* ptr, double val) { for (;;) { double_number old_num = {0}; old_num.d = *ptr; // current old value @@ -719,13 +718,13 @@ double atomic_fetch_sub_double(double volatile *ptr, double val){ new_num.d = old_num.d - val; double_number ret_num = {0}; - ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); if (ret_num.i == old_num.i) return ret_num.d; } } -void* atomic_fetch_sub_ptr(void* ptr, void* val) { +void* atomic_fetch_sub_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return interlocked_fetch_sub_ptr(ptr, val); #elif defined(_TD_NINGSI_60) @@ -777,7 +776,7 @@ int64_t atomic_and_fetch_64(int64_t volatile* ptr, int64_t val) { #endif } -void* atomic_and_fetch_ptr(void* ptr, void* val) { +void* atomic_and_fetch_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return interlocked_and_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) @@ -829,7 +828,7 @@ int64_t atomic_fetch_and_64(int64_t volatile* ptr, int64_t val) { #endif } -void* atomic_fetch_and_ptr(void* ptr, void* val) { +void* atomic_fetch_and_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return interlocked_fetch_and_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) @@ -881,7 +880,7 @@ int64_t atomic_or_fetch_64(int64_t volatile* ptr, int64_t val) { #endif } -void* atomic_or_fetch_ptr(void* ptr, void* val) { +void* atomic_or_fetch_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return interlocked_or_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) @@ -933,7 +932,7 @@ int64_t atomic_fetch_or_64(int64_t volatile* ptr, int64_t val) { #endif } -void* atomic_fetch_or_ptr(void* ptr, void* val) { +void* atomic_fetch_or_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return interlocked_fetch_or_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) @@ -985,7 +984,7 @@ int64_t atomic_xor_fetch_64(int64_t volatile* ptr, int64_t val) { #endif } -void* atomic_xor_fetch_ptr(void* ptr, void* val) { +void* atomic_xor_fetch_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return interlocked_xor_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) @@ -1037,7 +1036,7 @@ int64_t atomic_fetch_xor_64(int64_t volatile* ptr, int64_t val) { #endif } -void* atomic_fetch_xor_ptr(void* ptr, void* val) { +void* atomic_fetch_xor_ptr(void* ptr, int64_t val) { #ifdef WINDOWS return interlocked_fetch_xor_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 25245d2df9..410f9da623 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -39,7 +39,7 @@ enum { WRDE_SYNTAX /* Shell syntax error. */ }; -int wordexp(char *words, wordexp_t *pwordexp, int flags) { +int32_t wordexp(char *words, wordexp_t *pwordexp, int32_t flags) { pwordexp->we_offs = 0; pwordexp->we_wordc = 1; pwordexp->we_wordv[0] = pwordexp->wordPos; @@ -197,14 +197,10 @@ int32_t taosMulMkDir(const char *dirname) { } } - if (code < 0 && errno == EEXIST) { - return 0; - } - - return code; + return 0; } -int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) { +int32_t taosMulModeMkDir(const char *dirname, int32_t mode, bool checkAccess) { if (dirname == NULL || strlen(dirname) >= TDDIRMAXLEN) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -326,7 +322,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { int32_t days = (int32_t)(TABS(sec - fileSec) / 86400 + 1); if (days > keepDays) { TAOS_UNUSED(taosRemoveFile(filename)); - uInfo("file:%s is removed, days:%d keepDays:%d, sed:%" PRId64, filename, days, keepDays, fileSec); + // printf("file:%s is removed, days:%d keepDays:%d, sed:%" PRId64, filename, days, keepDays, fileSec); } else { // printf("file:%s won't be removed, days:%d keepDays:%d", filename, days, keepDays); } @@ -340,7 +336,9 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { OS_PARAM_CHECK(dirname); OS_PARAM_CHECK(outname); - wordexp_t full_path; + if (dirname[0] == 0) return 0; + + wordexp_t full_path = {0}; int32_t code = wordexp(dirname, &full_path, 0); switch (code) { case 0: @@ -357,13 +355,12 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { } wordfree(&full_path); - return 0; } int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) { OS_PARAM_CHECK(dirname); - OS_PARAM_CHECK(realPath); + char tmp[PATH_MAX] = {0}; #ifdef WINDOWS if (_fullpath(tmp, dirname, maxlen) != NULL) { @@ -377,12 +374,14 @@ int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) { tstrncpy(realPath, tmp, maxlen); } return 0; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; } + } else { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; } - - terrno = TAOS_SYSTEM_ERROR(errno); - - return terrno; } bool taosIsDir(const char *dirname) { @@ -454,7 +453,7 @@ TdDirPtr taosOpenDir(const char *dirname) { } #ifdef WINDOWS - char szFind[MAX_PATH]; //这是要找的 + char szFind[MAX_PATH]; // 这是要找的 HANDLE hFind; TdDirPtr pDir = taosMemoryMalloc(sizeof(TdDir)); @@ -462,7 +461,7 @@ TdDirPtr taosOpenDir(const char *dirname) { return NULL; } - snprintf(szFind, sizeof(szFind), "%s%s", dirname, "\\*.*"); //利用通配符找这个目录下的所以文件，包括目录 + snprintf(szFind, sizeof(szFind), "%s%s", dirname, "\\*.*"); // 利用通配符找这个目录下的所以文件，包括目录 pDir->hFind = FindFirstFile(szFind, &(pDir->dirEntry.findFileData)); if (INVALID_HANDLE_VALUE == pDir->hFind) { @@ -585,14 +584,14 @@ void taosGetCwd(char *buf, int32_t len) { #endif } -int taosGetDirSize(const char *path, int64_t *size) { - int32_t code; +int32_t taosGetDirSize(const char *path, int64_t *size) { + int32_t code = 0; + char fullPath[PATH_MAX + 100] = {0}; + TdDirPtr pDir = taosOpenDir(path); if (pDir == NULL) { return code = terrno; } - int32_t nBytes = 0; - char fullPath[1024] = {0}; int64_t totalSize = 0; TdDirEntryPtr de = NULL; @@ -601,31 +600,23 @@ int taosGetDirSize(const char *path, int64_t *size) { if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) { continue; } - nBytes = snprintf(fullPath, sizeof(fullPath), "%s%s%s", path, TD_DIRSEP, name); - if (nBytes <= 0 || nBytes >= sizeof(fullPath)) { - TAOS_UNUSED(taosCloseDir(&pDir)); - return TSDB_CODE_OUT_OF_RANGE; - } + (void)snprintf(fullPath, sizeof(fullPath), "%s%s%s", path, TD_DIRSEP, name); int64_t subSize = 0; if (taosIsDir(fullPath)) { code = taosGetDirSize(fullPath, &subSize); - if (code != 0) { - TAOS_UNUSED(taosCloseDir(&pDir)); - return code; - } } else { code = taosStatFile(fullPath, &subSize, NULL, NULL); - if (code != 0) { - TAOS_UNUSED(taosCloseDir(&pDir)); - return code; - } } + + if (code != 0) goto _OVER; + totalSize += subSize; fullPath[0] = 0; } +_OVER: *size = totalSize; TAOS_UNUSED(taosCloseDir(&pDir)); - return 0; + return code; } diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 832deb574d..d5ed46cf9e 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -25,10 +25,9 @@ SDiskSpace tsLogSpace = {0}; SDiskSpace tsTempSpace = {0}; char tsOsName[16] = {0}; char tsTimezoneStr[TD_TIMEZONE_LEN] = {0}; -enum TdTimezone tsTimezone = TdZeroZone; char tsLocale[TD_LOCALE_LEN] = {0}; char tsCharset[TD_CHARSET_LEN] = {0}; -int8_t tsDaylight = 0; +void *tsCharsetCxt = NULL; bool tsEnableCoreFile = 1; int64_t tsPageSizeKB = 0; int64_t tsOpenMax = 0; @@ -49,14 +48,7 @@ int32_t osDefaultInit() { taosSeedRand(taosSafeRand()); taosGetSystemLocale(tsLocale, tsCharset); - code = taosGetSystemTimezone(tsTimezoneStr, &tsTimezone); - if(code != 0) { - return code; - } - if (strlen(tsTimezoneStr) > 0) { // ignore empty timezone - if ((code = taosSetSystemTimezone(tsTimezoneStr, tsTimezoneStr, &tsDaylight, &tsTimezone)) != TSDB_CODE_SUCCESS) - return code; - } + (void)taosGetSystemTimezone(tsTimezoneStr); taosGetSystemInfo(); @@ -124,12 +116,7 @@ bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reser bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reserved; } -int32_t osSetTimezone(const char *tz) { return taosSetSystemTimezone(tz, tsTimezoneStr, &tsDaylight, &tsTimezone); } - -void osSetSystemLocale(const char *inLocale, const char *inCharSet) { - if (inLocale) (void)memcpy(tsLocale, inLocale, strlen(inLocale) + 1); - if (inCharSet) (void)memcpy(tsCharset, inCharSet, strlen(inCharSet) + 1); -} +int32_t osSetTimezone(const char *tz) { return taosSetGlobalTimezone(tz); } void osSetProcPath(int32_t argc, char **argv) { if (argv == NULL || argc < 1) { diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 21f781c7e4..484436fafe 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -78,19 +78,16 @@ char *taosCharsetReplace(char *charsetstr) { * * In case that the setLocale failed to be executed, the right charset needs to be set. */ -int32_t taosSetSystemLocale(const char *inLocale, const char *inCharSet) { - OS_PARAM_CHECK(inLocale); - OS_PARAM_CHECK(inCharSet); - if (!taosValidateEncodec(inCharSet)) { - return terrno; - } +int32_t taosSetSystemLocale(const char *inLocale) { char *locale = setlocale(LC_CTYPE, inLocale); if (NULL == locale) { terrno = TSDB_CODE_INVALID_PARA; + uError("failed to set locale:%s", inLocale); return terrno; } + tstrncpy(tsLocale, locale, TD_LOCALE_LEN); return 0; } @@ -102,59 +99,6 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { tstrncpy(outLocale, locale, TD_LOCALE_LEN); } tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN); - -#elif defined(_TD_DARWIN_64) - /* - * originally from src/os/src/detail/osSysinfo.c - * POSIX format locale string: - * (Language Strings)_(Country/Region Strings).(code_page) - * - * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, - * - * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. - * - * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page - * for libiconv that is employed to convert string in this system. This program will automatically use - * UTF-8 instead as the charset. - * - * In case of windows client, the locale string is not valid POSIX format, user needs to set the - * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is - * CP936, CP437 for English charset. - * - */ - - char sep = '.'; - char *locale = NULL; - - locale = setlocale(LC_CTYPE, ""); - if (locale == NULL) { - // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - tstrncpy(outLocale, "en_US.UTF-8", TD_LOCALE_LEN); - } else { - tstrncpy(outLocale, locale, TD_LOCALE_LEN); - // printf("locale not configured, set to system default:%s", outLocale); - } - - /* if user does not specify the charset, extract it from locale */ - char *str = strrchr(outLocale, sep); - if (str != NULL) { - str++; - - char *revisedCharset = taosCharsetReplace(str); - - if (NULL == revisedCharset) { - tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN); - } else { - tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN); - - taosMemoryFree(revisedCharset); - } - // printf("charset not configured, set to system default:%s", outCharset); - } else { - tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN); - // printf("can't get locale and charset from system, set it to UTF-8"); - } - #else /* * POSIX format locale string: diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 32b1023ed7..3051005789 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -150,7 +150,7 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void #endif } -const char *taosInetNtoa(struct in_addr ipInt, char *dstStr, int32_t len) { +const char *taosInetNtop(struct in_addr ipInt, char *dstStr, int32_t len) { const char *r = inet_ntop(AF_INET, &ipInt, dstStr, len); if (NULL == r) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -233,15 +233,19 @@ int32_t taosBlockSIGPIPE() { } int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { + int32_t code = 0; OS_PARAM_CHECK(fqdn); OS_PARAM_CHECK(ip); + int64_t limitMs = 1000; + int64_t st = taosGetTimestampMs(), cost = 0; #ifdef WINDOWS // Initialize Winsock WSADATA wsaData; int iResult; iResult = WSAStartup(MAKEWORD(2, 2), &wsaData); if (iResult != 0) { - return TAOS_SYSTEM_WINSOCKET_ERROR(WSAGetLastError()); + code = TAOS_SYSTEM_WINSOCKET_ERROR(WSAGetLastError()); + goto _err; } #endif @@ -260,12 +264,12 @@ int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { inRetry = true; continue; } else if (EAI_SYSTEM == ret) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; + code = TAOS_SYSTEM_ERROR(errno); + goto _err; } - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; + code = TAOS_SYSTEM_ERROR(errno); + goto _err; } struct sockaddr *sa = result->ai_addr; @@ -275,8 +279,7 @@ int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { *ip = ia.s_addr; freeaddrinfo(result); - - return 0; + goto _err; } #else struct addrinfo hints = {0}; @@ -292,7 +295,7 @@ int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { struct in_addr ia = si->sin_addr; *ip = ia.s_addr; freeaddrinfo(result); - return 0; + goto _err; } else { #ifdef EAI_SYSTEM if (ret == EAI_SYSTEM) { @@ -305,9 +308,16 @@ int32_t taosGetIpv4FromFqdn(const char *fqdn, uint32_t *ip) { #endif *ip = 0xFFFFFFFF; - return TSDB_CODE_RPC_FQDN_ERROR; + code = TSDB_CODE_RPC_FQDN_ERROR; + goto _err; } #endif +_err: + cost = taosGetTimestampMs() - st; + if (cost >= limitMs) { + uWarn("get ip from fqdn:%s, cost:%" PRId64 "ms", fqdn, cost); + } + return code; } int32_t taosGetFqdn(char *fqdn) { @@ -386,11 +396,19 @@ int32_t taosGetFqdn(char *fqdn) { return 0; } -void tinet_ntoa(char *ipstr, uint32_t ip) { +void taosInetNtoa(char *ipstr, uint32_t ip) { if (ipstr == NULL) { return; } - (void)snprintf(ipstr, TD_IP_LEN, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24); + unsigned char *bytes = (unsigned char *) &ip; + (void)snprintf(ipstr, TD_IP_LEN, "%d.%d.%d.%d", bytes[0], bytes[1], bytes[2], bytes[3]); +} + +uint32_t taosInetAddr(const char *ipstr){ + if (ipstr == NULL) { + return 0; + } + return inet_addr(ipstr); } int32_t taosIgnSIGPIPE() { diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 0ee4f1c496..147f0462eb 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -120,33 +120,20 @@ int32_t taosStr2int64(const char *str, int64_t *val) { if (str == NULL || val == NULL) { return TSDB_CODE_INVALID_PARA; } + errno = 0; char *endptr = NULL; int64_t ret = strtoll(str, &endptr, 10); - if (errno == ERANGE && (ret == LLONG_MAX || ret == LLONG_MIN)) { + if (errno != 0) { return TAOS_SYSTEM_ERROR(errno); - } else if (errno == EINVAL && ret == 0) { - return TSDB_CODE_INVALID_PARA; } else { + if (endptr == str) { + return TSDB_CODE_INVALID_PARA; + } *val = ret; return 0; } } -int32_t taosStr2int16(const char *str, int16_t *val) { - OS_PARAM_CHECK(str); - OS_PARAM_CHECK(val); - int64_t tmp = 0; - int32_t code = taosStr2int64(str, &tmp); - if (code) { - return code; - } else if (tmp > INT16_MAX || tmp < INT16_MIN) { - return TAOS_SYSTEM_ERROR(ERANGE); - } else { - *val = (int16_t)tmp; - return 0; - } -} - int32_t taosStr2int32(const char *str, int32_t *val) { OS_PARAM_CHECK(str); OS_PARAM_CHECK(val); @@ -161,6 +148,20 @@ int32_t taosStr2int32(const char *str, int32_t *val) { return 0; } } +int32_t taosStr2int16(const char *str, int16_t *val) { + OS_PARAM_CHECK(str); + OS_PARAM_CHECK(val); + int64_t tmp = 0; + int32_t code = taosStr2int64(str, &tmp); + if (code) { + return code; + } else if (tmp > INT16_MAX || tmp < INT16_MIN) { + return TAOS_SYSTEM_ERROR(ERANGE); + } else { + *val = (int16_t)tmp; + return 0; + } +} int32_t taosStr2int8(const char *str, int8_t *val) { OS_PARAM_CHECK(str); @@ -177,6 +178,70 @@ int32_t taosStr2int8(const char *str, int8_t *val) { } } +int32_t taosStr2Uint64(const char *str, uint64_t *val) { + if (str == NULL || val == NULL) { + return TSDB_CODE_INVALID_PARA; + } + char *endptr = NULL; + errno = 0; + uint64_t ret = strtoull(str, &endptr, 10); + + if (errno != 0) { + return TAOS_SYSTEM_ERROR(errno); + } else { + if (endptr == str) { + return TSDB_CODE_INVALID_PARA; + } + *val = ret; + return 0; + } +} + +int32_t taosStr2Uint32(const char *str, uint32_t *val) { + OS_PARAM_CHECK(str); + OS_PARAM_CHECK(val); + uint64_t tmp = 0; + int32_t code = taosStr2Uint64(str, &tmp); + if (code) { + return code; + } else if (tmp > UINT32_MAX) { + return TAOS_SYSTEM_ERROR(ERANGE); + } else { + *val = (int32_t)tmp; + return 0; + } +} + +int32_t taosStr2Uint16(const char *str, uint16_t *val) { + OS_PARAM_CHECK(str); + OS_PARAM_CHECK(val); + uint64_t tmp = 0; + int32_t code = taosStr2Uint64(str, &tmp); + if (code) { + return code; + } else if (tmp > UINT16_MAX) { + return TAOS_SYSTEM_ERROR(ERANGE); + } else { + *val = (int16_t)tmp; + return 0; + } +} + +int32_t taosStr2Uint8(const char *str, uint8_t *val) { + OS_PARAM_CHECK(str); + OS_PARAM_CHECK(val); + uint64_t tmp = 0; + int32_t code = taosStr2Uint64(str, &tmp); + if (code) { + return code; + } else if (tmp > UINT8_MAX) { + return TAOS_SYSTEM_ERROR(ERANGE); + } else { + *val = (int8_t)tmp; + return 0; + } +} + int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { if ((f1_ucs4 == NULL || f2_ucs4 == NULL)) { return TSDB_CODE_INVALID_PARA; @@ -223,86 +288,31 @@ int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) terrno = TSDB_CODE_INVALID_PARA; return terrno; } - + (void)memcpy(target_ucs4, source_ucs4, len_ucs4 * sizeof(TdUcs4)); return TSDB_CODE_SUCCESS; } -typedef struct { - iconv_t conv; - int8_t inUse; -} SConv; - -// 0: Mbs --> Ucs4 -// 1: Ucs4--> Mbs -SConv *gConv[2] = {NULL, NULL}; -int32_t convUsed[2] = {0, 0}; -int32_t gConvMaxNum[2] = {0, 0}; - -int32_t taosConvInit(void) { - int8_t M2C = 0; - gConvMaxNum[M2C] = 512; - gConvMaxNum[1 - M2C] = 512; - - gConv[M2C] = taosMemoryCalloc(gConvMaxNum[M2C], sizeof(SConv)); - if (gConv[M2C] == NULL) { - return terrno; - } - - gConv[1 - M2C] = taosMemoryCalloc(gConvMaxNum[1 - M2C], sizeof(SConv)); - if (gConv[1 - M2C] == NULL) { - taosMemoryFree(gConv[M2C]); - return terrno; - } - - for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) { - gConv[M2C][i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); - if ((iconv_t)-1 == gConv[M2C][i].conv) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - } - for (int32_t i = 0; i < gConvMaxNum[1 - M2C]; ++i) { - gConv[1 - M2C][i].conv = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); - if ((iconv_t)-1 == gConv[1 - M2C][i].conv) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - } - - return 0; -} - -void taosConvDestroy() { - int8_t M2C = 0; - for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) { - (void)iconv_close(gConv[M2C][i].conv); - } - for (int32_t i = 0; i < gConvMaxNum[1 - M2C]; ++i) { - (void)iconv_close(gConv[1 - M2C][i].conv); - } - taosMemoryFreeClear(gConv[M2C]); - taosMemoryFreeClear(gConv[1 - M2C]); - gConvMaxNum[M2C] = -1; - gConvMaxNum[1 - M2C] = -1; -} - -iconv_t taosAcquireConv(int32_t *idx, ConvType type) { +iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt) { if(idx == NULL) { terrno = TSDB_CODE_INVALID_PARA; return (iconv_t)-1; } - if (gConvMaxNum[type] <= 0) { + if (charsetCxt == NULL){ + charsetCxt = tsCharsetCxt; + } + SConvInfo *info = (SConvInfo *)charsetCxt; + if (info == NULL) { *idx = -1; if (type == M2C) { - iconv_t c = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); + iconv_t c = iconv_open(DEFAULT_UNICODE_ENCODEC, "UTF-8"); if ((iconv_t)-1 == c) { terrno = TAOS_SYSTEM_ERROR(errno); } return c; } else { - iconv_t c = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); + iconv_t c = iconv_open("UTF-8", DEFAULT_UNICODE_ENCODEC); if ((iconv_t)-1 == c) { terrno = TAOS_SYSTEM_ERROR(errno); } @@ -311,9 +321,9 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type) { } while (true) { - int32_t used = atomic_add_fetch_32(&convUsed[type], 1); - if (used > gConvMaxNum[type]) { - used = atomic_sub_fetch_32(&convUsed[type], 1); + int32_t used = atomic_add_fetch_32(&info->convUsed[type], 1); + if (used > info->gConvMaxNum[type]) { + (void)atomic_sub_fetch_32(&info->convUsed[type], 1); (void)sched_yield(); continue; } @@ -321,42 +331,47 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type) { break; } - int32_t startId = taosGetSelfPthreadId() % gConvMaxNum[type]; + int32_t startId = taosGetSelfPthreadId() % info->gConvMaxNum[type]; while (true) { - if (gConv[type][startId].inUse) { - startId = (startId + 1) % gConvMaxNum[type]; + if (info->gConv[type][startId].inUse) { + startId = (startId + 1) % info->gConvMaxNum[type]; continue; } - int8_t old = atomic_val_compare_exchange_8(&gConv[type][startId].inUse, 0, 1); + int8_t old = atomic_val_compare_exchange_8(&info->gConv[type][startId].inUse, 0, 1); if (0 == old) { break; } } *idx = startId; - if ((iconv_t)0 == gConv[type][startId].conv) { + if ((iconv_t)0 == info->gConv[type][startId].conv) { return (iconv_t)-1; } else { - return gConv[type][startId].conv; + return info->gConv[type][startId].conv; } } -void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) { +void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetCxt) { if (idx < 0) { (void)iconv_close(conv); return; } - atomic_store_8(&gConv[type][idx].inUse, 0); - (void)atomic_sub_fetch_32(&convUsed[type], 1); + if (charsetCxt == NULL){ + charsetCxt = tsCharsetCxt; + } + SConvInfo *info = (SConvInfo *)charsetCxt; + + atomic_store_8(&info->gConv[type][idx].inUse, 0); + (void)atomic_sub_fetch_32(&info->convUsed[type], 1); } -bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) { +bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt) { if (ucs4_max_len == 0) { return true; } - if(ucs4_max_len < 0 || mbs == NULL || ucs4 == NULL) { + if (ucs4_max_len < 0 || mbs == NULL || ucs4 == NULL) { terrno = TSDB_CODE_INVALID_PARA; return false; } @@ -368,20 +383,22 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4 (void)memset(ucs4, 0, ucs4_max_len); int32_t idx = -1; - iconv_t conv = taosAcquireConv(&idx, M2C); + iconv_t conv = taosAcquireConv(&idx, M2C, charsetCxt); if ((iconv_t)-1 == conv) { return false; } - - size_t ucs4_input_len = mbsLength; - size_t outLeft = ucs4_max_len; + + size_t ucs4_input_len = mbsLength; + size_t outLeft = ucs4_max_len; if (iconv(conv, (char **)&mbs, &ucs4_input_len, (char **)&ucs4, &outLeft) == -1) { + char buf[512] = {0}; + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s %d %d", strerror(terrno), errno, EILSEQ); terrno = TAOS_SYSTEM_ERROR(errno); - taosReleaseConv(idx, conv, M2C); + taosReleaseConv(idx, conv, M2C, charsetCxt); return false; } - taosReleaseConv(idx, conv, M2C); + taosReleaseConv(idx, conv, M2C, charsetCxt); if (len != NULL) { *len = (int32_t)(ucs4_max_len - outLeft); if (*len < 0) { @@ -397,11 +414,11 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4 // if success, return the number of bytes written to mbs ( >= 0) // otherwise return error code ( < 0) -int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { +int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt) { if (ucs4_max_len == 0) { return 0; } - if(ucs4_max_len < 0 || ucs4 == NULL || mbs == NULL) { + if (ucs4_max_len < 0 || ucs4 == NULL || mbs == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -413,22 +430,22 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { int32_t idx = -1; int32_t code = 0; - iconv_t conv = taosAcquireConv(&idx, C2M); + iconv_t conv = taosAcquireConv(&idx, C2M, charsetCxt); if ((iconv_t)-1 == conv) { return terrno; } - - size_t ucs4_input_len = ucs4_max_len; - size_t outLen = ucs4_max_len; + + size_t ucs4_input_len = ucs4_max_len; + size_t outLen = ucs4_max_len; if (iconv(conv, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) { code = TAOS_SYSTEM_ERROR(errno); - taosReleaseConv(idx, conv, C2M); - terrno = code; + taosReleaseConv(idx, conv, C2M, charsetCxt); + terrno = code; return code; } - - taosReleaseConv(idx, conv, C2M); - + + taosReleaseConv(idx, conv, C2M, charsetCxt); + return (int32_t)(ucs4_max_len - outLen); #endif } @@ -439,7 +456,7 @@ int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t c if (ucs4_max_len == 0) { return 0; } - if(ucs4_max_len < 0 || ucs4 == NULL || mbs == NULL) { + if (ucs4_max_len < 0 || ucs4 == NULL || mbs == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -455,13 +472,13 @@ int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t c terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - + return (int32_t)(ucs4_max_len - outLen); #endif } bool taosValidateEncodec(const char *encodec) { - if(encodec == NULL) { + if (encodec == NULL) { terrno = TSDB_CODE_INVALID_PARA; return false; } @@ -513,7 +530,7 @@ int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len, int32_t } int32_t taosHexDecode(const char *src, char *dst, int32_t len) { - if(!src || !dst || len <= 0) { + if (!src || !dst || len <= 0) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -556,9 +573,10 @@ int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size) { return mbstowcs(pWchars, pStrs, size); } -int32_t taosWcharToMb(char *pStr, TdWchar wchar) { +int32_t taosWcharToMb(char *pStr, TdWchar wchar) { OS_PARAM_CHECK(pStr); - return wctomb(pStr, wchar); } + return wctomb(pStr, wchar); +} char *taosStrCaseStr(const char *str, const char *pattern) { if (str == NULL) { @@ -580,7 +598,7 @@ char *taosStrCaseStr(const char *str, const char *pattern) { } int64_t taosStr2Int64(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -592,7 +610,7 @@ int64_t taosStr2Int64(const char *str, char **pEnd, int32_t radix) { } uint64_t taosStr2UInt64(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -604,7 +622,7 @@ uint64_t taosStr2UInt64(const char *str, char **pEnd, int32_t radix) { } int32_t taosStr2Int32(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -616,7 +634,7 @@ int32_t taosStr2Int32(const char *str, char **pEnd, int32_t radix) { } uint32_t taosStr2UInt32(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -628,7 +646,7 @@ uint32_t taosStr2UInt32(const char *str, char **pEnd, int32_t radix) { } int16_t taosStr2Int16(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -640,7 +658,7 @@ int16_t taosStr2Int16(const char *str, char **pEnd, int32_t radix) { } uint16_t taosStr2UInt16(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -652,7 +670,7 @@ uint16_t taosStr2UInt16(const char *str, char **pEnd, int32_t radix) { } int8_t taosStr2Int8(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -661,7 +679,7 @@ int8_t taosStr2Int8(const char *str, char **pEnd, int32_t radix) { } uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -673,7 +691,7 @@ uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix) { } double taosStr2Double(const char *str, char **pEnd) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -682,7 +700,7 @@ double taosStr2Double(const char *str, char **pEnd) { } float taosStr2Float(const char *str, char **pEnd) { - if(str == NULL) { + if (str == NULL) { terrno = TSDB_CODE_INVALID_PARA; return 0; } @@ -698,7 +716,7 @@ bool isHex(const char *z, uint32_t n) { } bool isValidateHex(const char *z, uint32_t n) { - if(!z) { + if (!z) { terrno = TSDB_CODE_INVALID_PARA; return false; } @@ -724,12 +742,12 @@ int32_t taosHex2Ascii(const char *z, uint32_t n, void **data, uint32_t *size) { } return 0; } - + uint8_t *tmp = (uint8_t *)taosMemoryCalloc(*size, 1); if (tmp == NULL) { return terrno; } - + int8_t num = 0; uint8_t *byte = tmp + *size - 1; @@ -749,7 +767,7 @@ int32_t taosHex2Ascii(const char *z, uint32_t n, void **data, uint32_t *size) { } } *data = tmp; - + return 0; } @@ -825,7 +843,7 @@ int32_t taosAscii2Hex(const char *z, uint32_t n, void **data, uint32_t *size) { if (tmp == NULL) { return terrno; } - + *data = tmp; *(tmp++) = '\\'; *(tmp++) = 'x'; @@ -834,7 +852,7 @@ int32_t taosAscii2Hex(const char *z, uint32_t n, void **data, uint32_t *size) { tmp[i * 2] = valueOf(val >> 4); tmp[i * 2 + 1] = valueOf(val & 0x0F); } - + return 0; } diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 60339fc646..29cbcaeb2c 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -345,6 +345,12 @@ char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { #endif } +size_t +taosStrfTime(char *s, size_t maxsize, char const *format, struct tm const *t){ + if (!s || !format || !t) return 0; + return strftime(s, maxsize, format, t); +} + int32_t taosGetTimeOfDay(struct timeval *tv) { if (tv == NULL) { return TSDB_CODE_INVALID_PARA; @@ -420,68 +426,53 @@ int64_t user_mktime64(const uint32_t year, const uint32_t mon, const uint32_t da return _res + time_zone; } -time_t taosMktime(struct tm *timep) { +time_t taosMktime(struct tm *timep, timezone_t tz) { #ifdef WINDOWS -#if 0 - struct tm tm1 = {0}; - LARGE_INTEGER t; - FILETIME f; - SYSTEMTIME s; - FILETIME ff; - SYSTEMTIME ss; - LARGE_INTEGER offset; - - time_t tt = 0; - localtime_s(&tm1, &tt); - ss.wYear = tm1.tm_year + 1900; - ss.wMonth = tm1.tm_mon + 1; - ss.wDay = tm1.tm_mday; - ss.wHour = tm1.tm_hour; - ss.wMinute = tm1.tm_min; - ss.wSecond = tm1.tm_sec; - ss.wMilliseconds = 0; - SystemTimeToFileTime(&ss, &ff); - offset.QuadPart = ff.dwHighDateTime; - offset.QuadPart <<= 32; - offset.QuadPart |= ff.dwLowDateTime; - - s.wYear = timep->tm_year + 1900; - s.wMonth = timep->tm_mon + 1; - s.wDay = timep->tm_mday; - s.wHour = timep->tm_hour; - s.wMinute = timep->tm_min; - s.wSecond = timep->tm_sec; - s.wMilliseconds = 0; - SystemTimeToFileTime(&s, &f); - t.QuadPart = f.dwHighDateTime; - t.QuadPart <<= 32; - t.QuadPart |= f.dwLowDateTime; - - t.QuadPart -= offset.QuadPart; - return (time_t)(t.QuadPart / 10000000); -#else time_t result = mktime(timep); if (result != -1) { return result; } -#ifdef _MSC_VER -#if _MSC_VER >= 1900 - int64_t tz = _timezone; -#endif -#endif + int64_t tzw = 0; + #ifdef _MSC_VER + #if _MSC_VER >= 1900 + tzw = _timezone; + #endif + #endif return user_mktime64(timep->tm_year + 1900, timep->tm_mon + 1, timep->tm_mday, timep->tm_hour, timep->tm_min, - timep->tm_sec, tz); -#endif + timep->tm_sec, tzw); #else - time_t r = mktime(timep); + time_t r = tz != NULL ? mktime_z(tz, timep) : mktime(timep); if (r == (time_t)-1) { terrno = TAOS_SYSTEM_ERROR(errno); } + timezone = -timep->tm_gmtoff; return r; #endif } -struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf, int32_t bufSize) { +struct tm *taosGmTimeR(const time_t *timep, struct tm *result){ + if (timep == NULL || result == NULL) { + return NULL; + } +#ifdef WINDOWS + return gmtime_s(result, timep); +#else + return gmtime_r(timep, result); +#endif +} + +time_t taosTimeGm(struct tm *tmp){ + if (tmp == NULL) { + return -1; + } +#ifdef WINDOWS + return _mkgmtime(tmp); +#else + return timegm(tmp); +#endif +} + +struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf, int32_t bufSize, timezone_t tz) { struct tm *res = NULL; if (timep == NULL || result == NULL) { return NULL; @@ -537,118 +528,15 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf, int3 return NULL; } } + return result; #else - res = localtime_r(timep, result); + res = tz != NULL ? localtime_rz(tz, timep, result): localtime_r(timep, result); if (res == NULL && buf != NULL) { (void)snprintf(buf, bufSize, "NaN"); } + timezone = -result->tm_gmtoff; + return res; #endif - return result; -} - -static int isLeapYear(time_t year) { - if (year % 4) - return 0; - else if (year % 100) - return 1; - else if (year % 400) - return 0; - else - return 1; -} - -struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst) { - if (result == NULL) { - return NULL; - } -#ifdef WINDOWS - if (*timep < 0) { - return NULL; - // TODO: bugs in following code - SYSTEMTIME ss, s; - FILETIME ff, f; - LARGE_INTEGER offset; - struct tm tm1; - time_t tt = 0; - if (localtime_s(&tm1, &tt) != 0) { - return NULL; - } - ss.wYear = tm1.tm_year + 1900; - ss.wMonth = tm1.tm_mon + 1; - ss.wDay = tm1.tm_mday; - ss.wHour = tm1.tm_hour; - ss.wMinute = tm1.tm_min; - ss.wSecond = tm1.tm_sec; - ss.wMilliseconds = 0; - SystemTimeToFileTime(&ss, &ff); - offset.QuadPart = ff.dwHighDateTime; - offset.QuadPart <<= 32; - offset.QuadPart |= ff.dwLowDateTime; - offset.QuadPart += *timep * 10000000; - f.dwLowDateTime = offset.QuadPart & 0xffffffff; - f.dwHighDateTime = (offset.QuadPart >> 32) & 0xffffffff; - FileTimeToSystemTime(&f, &s); - result->tm_sec = s.wSecond; - result->tm_min = s.wMinute; - result->tm_hour = s.wHour; - result->tm_mday = s.wDay; - result->tm_mon = s.wMonth - 1; - result->tm_year = s.wYear - 1900; - result->tm_wday = s.wDayOfWeek; - result->tm_yday = 0; - result->tm_isdst = 0; - } else { - if (localtime_s(result, timep) != 0) { - return NULL; - } - } -#elif defined(LINUX) - time_t secsMin = 60, secsHour = 3600, secsDay = 3600 * 24; - long tz = timezone; - - time_t t = *timep; - t -= tz; /* Adjust for timezone. */ - t += 3600 * dst; /* Adjust for daylight time. */ - time_t days = t / secsDay; /* Days passed since epoch. */ - time_t seconds = t % secsDay; /* Remaining seconds. */ - - result->tm_isdst = dst; - result->tm_hour = seconds / secsHour; - result->tm_min = (seconds % secsHour) / secsMin; - result->tm_sec = (seconds % secsHour) % secsMin; - - /* 1/1/1970 was a Thursday, that is, day 4 from the POV of the tm structure - * where sunday = 0, so to calculate the day of the week we have to add 4 - * and take the modulo by 7. */ - result->tm_wday = (days + 4) % 7; - - /* Calculate the current year. */ - result->tm_year = 1970; - while (1) { - /* Leap years have one day more. */ - time_t daysOfYear = 365 + isLeapYear(result->tm_year); - if (daysOfYear > days) break; - days -= daysOfYear; - result->tm_year++; - } - result->tm_yday = days; /* Number of day of the current year. */ - /* We need to calculate in which month and day of the month we are. To do - * so we need to skip days according to how many days there are in each - * month, and adjust for the leap year that has one more day in February. */ - int mdays[12] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; - mdays[1] += isLeapYear(result->tm_year); - result->tm_mon = 0; - while (days >= mdays[result->tm_mon]) { - days -= mdays[result->tm_mon]; - result->tm_mon++; - } - - result->tm_mday = days + 1; /* Add 1 since our 'days' is zero-based. */ - result->tm_year -= 1900; /* Surprisingly tm_year is year-1900. */ -#else - localtime_r(timep, result); -#endif - return result; } int32_t taosGetTimestampSec() { return (int32_t)time(NULL); } diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 89c7ce9e31..395c1cbb82 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -33,7 +33,8 @@ #include #pragma warning(pop) -char *win_tz[139][2] = {{"China Standard Time", "Asia/Shanghai"}, +#define W_TZ_NUM 139 +char *win_tz[W_TZ_NUM][2] = {{"China Standard Time", "Asia/Shanghai"}, {"AUS Central Standard Time", "Australia/Darwin"}, {"AUS Eastern Standard Time", "Australia/Sydney"}, {"Afghanistan Standard Time", "Asia/Kabul"}, @@ -172,7 +173,8 @@ char *win_tz[139][2] = {{"China Standard Time", "Asia/Shanghai"}, {"West Pacific Standard Time", "Pacific/Port_Moresby"}, {"Yakutsk Standard Time", "Asia/Yakutsk"}, {"Yukon Standard Time", "America/Whitehorse"}}; -char *tz_win[554][2] = {{"Asia/Shanghai", "China Standard Time"}, +#define W_TZ_CITY_NUM 554 +char *tz_win[W_TZ_CITY_NUM][2] = {{"Asia/Shanghai", "China Standard Time"}, {"Africa/Abidjan", "Greenwich Standard Time"}, {"Africa/Accra", "Greenwich Standard Time"}, {"Africa/Addis_Ababa", "E. Africa Standard Time"}, @@ -740,51 +742,19 @@ char *tz_win[554][2] = {{"Asia/Shanghai", "China Standard Time"}, #include #endif -static int isdst_now = 0; - -void parseTimeStr(char *p, char to[5]) { - for (int i = 0; i < 5; ++i) { - if (strlen(p) > i) { - to[i] = p[i]; - } else { - to[i] = '0'; - } - } - if (strlen(p) == 2) { - to[1] = '0'; - to[2] = p[1]; - } -} - -int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8_t *outDaylight, - enum TdTimezone *tsTimezone) { - if (inTimezoneStr == NULL || inTimezoneStr[0] == 0) { +int32_t taosSetGlobalTimezone(const char *tz) { + if (tz == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - int32_t code = TSDB_CODE_SUCCESS; - size_t len = strlen(inTimezoneStr); - if (len >= TD_TIMEZONE_LEN) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - char buf[TD_TIMEZONE_LEN] = {0}; - for (int32_t i = 0; i < len; i++) { - if (inTimezoneStr[i] == ' ' || inTimezoneStr[i] == '(') { - buf[i] = 0; - break; - } - buf[i] = inTimezoneStr[i]; - } - + uDebug("[tz]set timezone to %s", tz) #ifdef WINDOWS - char winStr[TD_LOCALE_LEN * 2]; - memset(winStr, 0, sizeof(winStr)); - for (size_t i = 0; i < 554; i++) { - if (strcmp(tz_win[i][0], buf) == 0) { - char keyPath[256]; - char keyValue[100]; + char winStr[TD_TIMEZONE_LEN * 2] = {0}; + for (size_t i = 0; i < W_TZ_CITY_NUM; i++) { + if (strcmp(tz_win[i][0], tz) == 0) { + char keyPath[256] = {0}; + char keyValue[100] = {0}; DWORD keyValueSize = sizeof(keyValue); snprintf(keyPath, sizeof(keyPath), "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\%s", tz_win[i][1]); RegGetValue(HKEY_LOCAL_MACHINE, keyPath, "Display", RRF_RT_ANY, NULL, (PVOID)&keyValue, &keyValueSize); @@ -792,90 +762,127 @@ int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, i keyValue[4] = (keyValue[4] == '+' ? '-' : '+'); keyValue[10] = 0; snprintf(winStr, sizeof(winStr), "TZ=%s:00", &(keyValue[1])); - *tsTimezone = -taosStr2Int32(&keyValue[4], NULL, 10); } break; } } - if (winStr[0] == 0) { - char *p = strchr(inTimezoneStr, '+'); - if (p == NULL) p = strchr(inTimezoneStr, '-'); - if (p != NULL) { - char *pp = strchr(inTimezoneStr, '('); - char *ppp = strchr(inTimezoneStr, ','); - int indexStr; - if (pp == NULL || ppp == NULL) { - indexStr = tsnprintf(winStr, sizeof(winStr), "TZ=UTC"); - } else { - memcpy(winStr, "TZ=", 3); - pp++; - memcpy(&winStr[3], pp, ppp - pp); - indexStr = ppp - pp + 3; - } - char to[5]; - parseTimeStr(p, to); - snprintf(&winStr[indexStr], sizeof(winStr) - indexStr, "%c%c%c:%c%c:00", (to[0] == '+' ? '+' : '-'), to[1], to[2], to[3], to[4]); - *tsTimezone = -taosStr2Int32(p, NULL, 10); - } else { - *tsTimezone = 0; - } - } + _putenv(winStr); _tzset(); - if (outTimezoneStr != inTimezoneStr) { - tstrncpy(outTimezoneStr, inTimezoneStr, TD_TIMEZONE_LEN); - } - *outDaylight = 0; - -#elif defined(_TD_DARWIN_64) - - code = setenv("TZ", buf, 1); - if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - tzset(); - int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); - *tsTimezone = tz; - tz += isdst_now; - - snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[isdst_now], tz >= 0 ? "+" : "-", abs(tz)); - *outDaylight = isdst_now; - #else - code = setenv("TZ", buf, 1); + code = setenv("TZ", tz, 1); if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } tzset(); - int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); - *tsTimezone = tz; - tz += isdst_now; - (void)snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[isdst_now], tz >= 0 ? "+" : "-", abs(tz)); - *outDaylight = isdst_now; - #endif - return code; + time_t tx1 = taosGetTimestampSec(); + return taosFormatTimezoneStr(tx1, tz, NULL, tsTimezoneStr); } -int32_t taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { - int32_t code = 0; +int32_t taosGetLocalTimezoneOffset() { + time_t tx1 = taosGetTimestampSec(); + struct tm tm1; + if (taosLocalTime(&tx1, &tm1, NULL, 0, NULL) == NULL) { + uError("%s failed to get local time: code:%d", __FUNCTION__, errno); + return TSDB_CODE_TIME_ERROR; + } #ifdef WINDOWS - char value[100]; - char keyPath[100]; + return -_timezone; +#else + return (int32_t)(tm1.tm_gmtoff); +#endif +} + +int32_t taosFormatTimezoneStr(time_t t, const char* tz, timezone_t sp, char *outTimezoneStr){ + struct tm tm1; + if (taosLocalTime(&t, &tm1, NULL, 0, sp) == NULL) { + uError("%s failed to get local time: code:%d", __FUNCTION__, errno); + return TSDB_CODE_TIME_ERROR; + } + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + * n/a (UTC, +0000) + */ + + char str1[TD_TIMEZONE_LEN] = {0}; + if (taosStrfTime(str1, sizeof(str1), "%Z", &tm1) == 0){ + uError("failed to get timezone name"); + return TSDB_CODE_TIME_ERROR; + } + + char str2[TD_TIMEZONE_LEN] = {0}; + if (taosStrfTime(str2, sizeof(str2), "%z", &tm1) == 0){ + uError("failed to get timezone offset"); + return TSDB_CODE_TIME_ERROR; + } + (void)snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s)", tz, str1, str2); + uDebug("[tz] system timezone:%s", outTimezoneStr); + return 0; +} + +#ifndef WINDOWS +void getTimezoneStr(char *tz) { + do { + int n = readlink("/etc/localtime", tz, TD_TIMEZONE_LEN - 1); + if (n < 0) { + uWarn("[tz] failed to readlink /etc/localtime, reason:%s", strerror(errno)); + break; + } + + char *zi = strstr(tz, "zoneinfo"); + if (zi == NULL) { + uWarn("[tz] failed to find zoneinfo in /etc/localtime"); + break; + } + zi += sizeof("zoneinfo"); + memmove(tz, zi, TD_TIMEZONE_LEN - (zi - tz)); + goto END; + } while (0); + + + TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); + if (pFile == NULL) { + uWarn("[tz] failed to open /etc/timezone, reason:%s", strerror(errno)); + goto END; + } + int len = taosReadFile(pFile, tz, TD_TIMEZONE_LEN - 1); + TAOS_UNUSED(taosCloseFile(&pFile)); + if (len <= 0) { + uWarn("[tz] failed to read /etc/timezone, len:%d", len); + goto END; + } + if (tz[len - 1] == '\n') { + tz[len - 1] = '\0'; + } + +END: + if (tz[0] == '\0') { + memcpy(tz, TZ_UNKNOWN, sizeof(TZ_UNKNOWN)); + } + uDebug("[tz] system timezone:%s", tz); +} +#endif + +int32_t taosGetSystemTimezone(char *outTimezoneStr) { +#ifdef WINDOWS + char value[100] = {0}; + char keyPath[100] = {0}; DWORD bufferSize = sizeof(value); LONG result = RegGetValue(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation", "TimeZoneKeyName", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); if (result != ERROR_SUCCESS) { return TAOS_SYSTEM_WINAPI_ERROR(result); } - tstrncpy(outTimezoneStr, "not configured", TD_TIMEZONE_LEN); - *tsTimezone = 0; if (bufferSize > 0) { - for (size_t i = 0; i < 139; i++) { + for (size_t i = 0; i < W_TZ_NUM; i++) { if (strcmp(win_tz[i][0], value) == 0) { tstrncpy(outTimezoneStr, win_tz[i][1], TD_TIMEZONE_LEN); bufferSize = sizeof(value); @@ -884,172 +891,19 @@ int32_t taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) if (result != ERROR_SUCCESS) { return TAOS_SYSTEM_WINAPI_ERROR(result); } - if (bufferSize > 0) { - // value[4] = (value[4] == '+' ? '-' : '+'); - snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (UTC, %c%c%c%c%c)", outTimezoneStr, value[4], value[5], value[6], value[8], - value[9]); - *tsTimezone = taosStr2Int32(&value[4], NULL, 10); + if (bufferSize > 0) { // value like (UTC+05:30) Chennai, Kolkata, Mumbai, New Delhi + snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (UTC, %c%c%c%c%c)", outTimezoneStr, + value[4], value[5], value[6], value[8], value[9]); } break; } } } return 0; -#elif defined(_TD_DARWIN_64) - char buf[4096] = {0}; - char *tz = NULL; - { - int n = readlink("/etc/localtime", buf, sizeof(buf)); - if (n < 0) { - return TSDB_CODE_TIME_ERROR; - } - buf[n] = '\0'; - - char *zi = strstr(buf, "zoneinfo"); - if (!zi) { - return TSDB_CODE_TIME_ERROR; - } - tz = zi + strlen("zoneinfo") + 1; - - code = setenv("TZ", tz, 1); - if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - tzset(); - } - - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ - time_t tx1 = taosGetTimestampSec(); - struct tm tm1; - if (taosLocalTime(&tx1, &tm1, NULL, 0) == NULL) { - return TSDB_CODE_TIME_ERROR; - } - daylight = tm1.tm_isdst; - isdst_now = tm1.tm_isdst; - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], - -timezone / 3600); - return 0; #else - - char buf[4096] = {0}; - char *tz = NULL; - { - int n = readlink("/etc/localtime", buf, sizeof(buf)-1); - if (n < 0) { - if (taosCheckExistFile("/etc/timezone")) { - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ - time_t tx1 = taosGetTimestampSec(); - struct tm tm1; - if(taosLocalTime(&tx1, &tm1, NULL, 0) == NULL) { - return TSDB_CODE_TIME_ERROR; - } - /* load time zone string from /etc/timezone */ - // FILE *f = fopen("/etc/timezone", "r"); - errno = 0; - TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); - char buf[68] = {0}; - if (pFile != NULL) { - int len = taosReadFile(pFile, buf, 64); - if (len < 0) { - TAOS_UNUSED(taosCloseFile(&pFile)); - return TSDB_CODE_TIME_ERROR; - } - - TAOS_UNUSED(taosCloseFile(&pFile)); - - buf[sizeof(buf) - 1] = 0; - char *lineEnd = strstr(buf, "\n"); - if (lineEnd != NULL) { - *lineEnd = 0; - } - - // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables - if (strlen(buf) > 0) { - code = setenv("TZ", buf, 1); - if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - } - } - // get and set default timezone - tzset(); - /* - * get CURRENT time zone. - * system current time zone is affected by daylight saving time(DST) - * - * e.g., the local time zone of London in DST is GMT+01:00, - * otherwise is GMT+00:00 - */ - int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; - *tsTimezone = tz; - tz += daylight; - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - (void)snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", - abs(tz)); - } else { - return TSDB_CODE_TIME_ERROR; - } - return 0; - } - buf[n] = '\0'; - - char *zi = strstr(buf, "zoneinfo"); - if (!zi) { - return TSDB_CODE_TIME_ERROR; - } - tz = zi + strlen("zoneinfo") + 1; - - code = setenv("TZ", tz, 1); - if (-1 == code) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - tzset(); - } - - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ + char tz[TD_TIMEZONE_LEN] = {0}; + getTimezoneStr(tz); time_t tx1 = taosGetTimestampSec(); - struct tm tm1; - if(taosLocalTime(&tx1, &tm1, NULL, 0) == NULL) { - return TSDB_CODE_TIME_ERROR; - } - isdst_now = tm1.tm_isdst; - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - (void)snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], - -timezone / 3600); - return 0; + return taosFormatTimezoneStr(tx1, tz, NULL, outTimezoneStr); #endif -} +} \ No newline at end of file diff --git a/source/os/test/CMakeLists.txt b/source/os/test/CMakeLists.txt index cc7110517f..13fea463f7 100644 --- a/source/os/test/CMakeLists.txt +++ b/source/os/test/CMakeLists.txt @@ -71,6 +71,9 @@ add_test( COMMAND osTimeTests ) + +if(TD_LINUX) + add_executable(osAtomicTests "osAtomicTests.cpp") target_link_libraries(osAtomicTests os util gtest_main) add_test( @@ -78,6 +81,22 @@ add_test( COMMAND osAtomicTests ) +add_executable(osDirTests "osDirTests.cpp") +target_link_libraries(osDirTests os util gtest_main) +add_test( + NAME osDirTests + COMMAND osDirTests +) + +add_executable(osEnvTests "osEnvTests.cpp") +target_link_libraries(osEnvTests os util gtest_main) +add_test( + NAME osEnvTests + COMMAND osEnvTests +) + +endif() + add_executable(osSemaphoreTests "osSemaphoreTests.cpp") target_link_libraries(osSemaphoreTests os util gtest_main) add_test( diff --git a/source/os/test/osAtomicTests.cpp b/source/os/test/osAtomicTests.cpp index eebc1574c3..170ba1cc7d 100644 --- a/source/os/test/osAtomicTests.cpp +++ b/source/os/test/osAtomicTests.cpp @@ -29,87 +29,434 @@ #include "os.h" #include "tlog.h" -TEST(osAtomicTests, Exchange16) { - int16_t value = 123; - int16_t new_value = 456; - int16_t result = atomic_exchange_16(&value, new_value); - EXPECT_EQ(result, 123); - EXPECT_EQ(value, 456); +TEST(osAtomicTests, atomic_load) { + int8_t result8 = 0, value8 = 8; + int16_t result16 = 0, value16 = 16; + int32_t result32 = 0, value32 = 32; + int64_t result64 = 0, value64 = 64; + void* resultp = &result64; + void* valuep = &value64; + + result8 = atomic_load_8(&value8); + result16 = atomic_load_16(&value16); + result32 = atomic_load_32(&value32); + result64 = atomic_load_64(&value64); + resultp = atomic_load_ptr(&valuep); + + EXPECT_EQ(result8, 8); + EXPECT_EQ(result16, 16); + EXPECT_EQ(result32, 32); + EXPECT_EQ(result64, 64); + EXPECT_EQ(resultp, &value64); + EXPECT_EQ(value8, 8); + EXPECT_EQ(value16, 16); + EXPECT_EQ(value32, 32); + EXPECT_EQ(value64, 64); + EXPECT_EQ(valuep, &value64); } -TEST(osAtomicTests, Exchange32) { - int32_t value = 123; - int32_t new_value = 456; - int32_t result = atomic_exchange_32(&value, new_value); - EXPECT_EQ(result, 123); - EXPECT_EQ(value, 456); +TEST(osAtomicTests, atomic_store) { + int8_t result8 = 0, value8 = 8; + int16_t result16 = 0, value16 = 16; + int32_t result32 = 0, value32 = 32; + int64_t result64 = 0, value64 = 64; + double resultd = 0, valued = 64; + void* resultp = &result64; + void* valuep = &value64; + + atomic_store_8(&result8, value8); + atomic_store_16(&result16, value16); + atomic_store_32(&result32, value32); + atomic_store_64(&result64, value64); + atomic_store_double(&resultd, valued); + atomic_store_ptr(&resultp, valuep); + + EXPECT_EQ(result8, 8); + EXPECT_EQ(result16, 16); + EXPECT_EQ(result32, 32); + EXPECT_EQ(result64, 64); + EXPECT_DOUBLE_EQ(resultd, 64); + EXPECT_EQ(resultp, &value64); } -TEST(osAtomicTests, Exchange64) { - int64_t value = 123; - int64_t new_value = 456; - int64_t result = atomic_exchange_64(&value, new_value); - EXPECT_EQ(result, 123); - EXPECT_EQ(value, 456); +TEST(osAtomicTests, atomic_exchange) { + int8_t result8 = 0, value8 = 8, newval8 = 18; + int16_t result16 = 0, value16 = 16, newval16 = 116; + int32_t result32 = 0, value32 = 32, newval32 = 132; + int64_t result64 = 0, value64 = 64, newval64 = 164; + double resultd = 0, valued = 64, newvald = 164; + void* valuep = &value64; + void* newvalp = &newval64; + void* resultp = &result64; + + result8 = atomic_exchange_8(&value8, newval8); + result16 = atomic_exchange_16(&value16, newval16); + result32 = atomic_exchange_32(&value32, newval32); + result64 = atomic_exchange_64(&value64, newval64); + resultd = atomic_exchange_double(&valued, newvald); + resultp = atomic_exchange_ptr(&valuep, newvalp); + + EXPECT_EQ(value8, 18); + EXPECT_EQ(value16, 116); + EXPECT_EQ(value32, 132); + EXPECT_EQ(value64, 164); + EXPECT_DOUBLE_EQ(valued, 164); + EXPECT_EQ(valuep, &newval64); + + EXPECT_EQ(result8, 8); + EXPECT_EQ(result16, 16); + EXPECT_EQ(result32, 32); + EXPECT_EQ(result64, 64); + EXPECT_DOUBLE_EQ(resultd, 64); + EXPECT_EQ(resultp, &value64); } -TEST(osAtomicTests, ExchangePtr) { - int value1 = 123; - int value2 = 456; - int* ptr = &value1; - int* result = (int*)atomic_exchange_ptr(&ptr, &value2); - EXPECT_EQ(result, &value1); - EXPECT_EQ(*ptr, 456); +TEST(osAtomicTests, atomic_val_compare_exchange) { + int8_t result8 = 0, value8 = 8, oldval8 = 8, newval8 = 18; + int16_t result16 = 0, value16 = 16, oldval16 = 16, newval16 = 116; + int32_t result32 = 0, value32 = 32, oldval32 = 32, newval32 = 132; + int64_t result64 = 0, value64 = 64, oldval64 = 64, newval64 = 164; + void* resultp = NULL; + void* valuep = &value64; + void* oldvalp = &value64; + void* newvalp = &newval64; + + result8 = atomic_val_compare_exchange_8(&value8, oldval8, newval8); + result16 = atomic_val_compare_exchange_16(&value16, oldval16, newval16); + result32 = atomic_val_compare_exchange_32(&value32, oldval32, newval32); + result64 = atomic_val_compare_exchange_64(&value64, oldval64, newval64); + resultp = atomic_val_compare_exchange_ptr(&valuep, oldvalp, newvalp); + + EXPECT_EQ(result8, 8); + EXPECT_EQ(value8, 18); + EXPECT_EQ(result16, 16); + EXPECT_EQ(value16, 116); + EXPECT_EQ(result32, 32); + EXPECT_EQ(value32, 132); + EXPECT_EQ(result64, 64); + EXPECT_EQ(value64, 164); + EXPECT_EQ(resultp, &value64); + EXPECT_EQ(valuep, &newval64); + + oldval8 = 9; + oldval16 = 99; + oldval32 = 999; + oldval64 = 9999; + oldvalp = NULL; + + result8 = atomic_val_compare_exchange_8(&value8, oldval8, newval8); + result16 = atomic_val_compare_exchange_16(&value16, oldval16, newval16); + result32 = atomic_val_compare_exchange_32(&value32, oldval32, newval32); + result64 = atomic_val_compare_exchange_64(&value64, oldval64, newval64); + resultp = atomic_val_compare_exchange_ptr(&valuep, oldvalp, newvalp); + + EXPECT_EQ(result8, 18); + EXPECT_EQ(value8, 18); + EXPECT_EQ(result16, 116); + EXPECT_EQ(value16, 116); + EXPECT_EQ(result32, 132); + EXPECT_EQ(value32, 132); + EXPECT_EQ(result64, 164); + EXPECT_EQ(value64, 164); + EXPECT_EQ(resultp, &newval64); + EXPECT_EQ(valuep, &newval64); } -TEST(osAtomicTests, ValCompareExchange8) { - int8_t value = 12; - int8_t oldval = 12; - int8_t newval = 45; - int8_t result = atomic_val_compare_exchange_8(&value, oldval, newval); - EXPECT_EQ(result, 12); - EXPECT_EQ(value, 45); +TEST(osAtomicTests, atomic_add_fetch) { + int8_t result8 = 0, value8 = 8; + int16_t result16 = 0, value16 = 16; + int32_t result32 = 0, value32 = 32; + int64_t result64 = 0, value64 = 64; - oldval = 78; - result = atomic_val_compare_exchange_8(&value, oldval, newval); - EXPECT_EQ(result, 45); - EXPECT_EQ(value, 45); + int64_t valuex = 128; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_add_fetch_8(&value8, 10); + result16 = atomic_add_fetch_16(&value16, 10); + result32 = atomic_add_fetch_32(&value32, 10); + result64 = atomic_add_fetch_64(&value64, 10); + resultx = (int64_t)atomic_add_fetch_ptr(valuep, 10); + + EXPECT_EQ(result8, 18); + EXPECT_EQ(value8, 18); + EXPECT_EQ(result16, 26); + EXPECT_EQ(value16, 26); + EXPECT_EQ(result32, 42); + EXPECT_EQ(value32, 42); + EXPECT_EQ(result64, 74); + EXPECT_EQ(value64, 74); + EXPECT_EQ(resultx, 138); + EXPECT_EQ(*valuep, 138); + EXPECT_EQ(valuex, 138); } -TEST(osAtomicTests, ValCompareExchange16) { - int16_t value = 123; - int16_t oldval = 123; - int16_t newval = 456; - int16_t result = atomic_val_compare_exchange_16(&value, oldval, newval); - EXPECT_EQ(result, 123); - EXPECT_EQ(value, 456); +TEST(osAtomicTests, atomic_fetch_add) { + int8_t result8 = 0, value8 = 8; + int16_t result16 = 0, value16 = 16; + int32_t result32 = 0, value32 = 32; + int64_t result64 = 0, value64 = 64; + double resultd = 0, valued = 64; - oldval = 789; - result = atomic_val_compare_exchange_16(&value, oldval, newval); - EXPECT_EQ(result, 456); - EXPECT_EQ(value, 456); + int64_t valuex = 128; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_fetch_add_8(&value8, 10); + result16 = atomic_fetch_add_16(&value16, 10); + result32 = atomic_fetch_add_32(&value32, 10); + result64 = atomic_fetch_add_64(&value64, 10); + resultd = atomic_fetch_add_double(&valued, 10); + resultx = (int64_t)atomic_fetch_add_ptr(valuep, 10); + + EXPECT_EQ(result8, 8); + EXPECT_EQ(value8, 18); + EXPECT_EQ(result16, 16); + EXPECT_EQ(value16, 26); + EXPECT_EQ(result32, 32); + EXPECT_EQ(value32, 42); + EXPECT_EQ(result64, 64); + EXPECT_EQ(value64, 74); + EXPECT_DOUBLE_EQ(resultd, 64); + EXPECT_DOUBLE_EQ(valued, 74); + EXPECT_EQ(resultx, 128); + EXPECT_EQ(*valuep, 138); + EXPECT_EQ(valuex, 138); } -TEST(osAtomicTests, TestAtomicExchange8) { - volatile int8_t value = 42; - int8_t new_value = 100; - int8_t old_value = atomic_exchange_8(&value, new_value); - EXPECT_EQ(old_value, 42); - EXPECT_EQ(value, new_value); +TEST(osAtomicTests, atomic_sub_fetch) { + int8_t result8 = 0, value8 = 8; + int16_t result16 = 0, value16 = 16; + int32_t result32 = 0, value32 = 32; + int64_t result64 = 0, value64 = 64; + + int64_t valuex = 128; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_sub_fetch_8(&value8, 10); + result16 = atomic_sub_fetch_16(&value16, 10); + result32 = atomic_sub_fetch_32(&value32, 10); + result64 = atomic_sub_fetch_64(&value64, 10); + resultx = (int64_t)atomic_sub_fetch_ptr(valuep, 10); + + EXPECT_EQ(result8, -2); + EXPECT_EQ(value8, -2); + EXPECT_EQ(result16, 6); + EXPECT_EQ(value16, 6); + EXPECT_EQ(result32, 22); + EXPECT_EQ(value32, 22); + EXPECT_EQ(result64, 54); + EXPECT_EQ(value64, 54); + EXPECT_EQ(resultx, 118); + EXPECT_EQ(*valuep, 118); + EXPECT_EQ(valuex, 118); } -TEST(osAtomicTests, TestAtomicAddFetch16) { - volatile int16_t value = 42; - int16_t increment = 10; - int16_t new_value = atomic_add_fetch_16(&value, increment); - EXPECT_EQ(new_value, 52); - EXPECT_EQ(value, 52); +TEST(osAtomicTests, atomic_fetch_sub) { + int8_t result8 = 0, value8 = 8; + int16_t result16 = 0, value16 = 16; + int32_t result32 = 0, value32 = 32; + int64_t result64 = 0, value64 = 64; + double resultd = 0, valued = 64; + + int64_t valuex = 128; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_fetch_sub_8(&value8, 10); + result16 = atomic_fetch_sub_16(&value16, 10); + result32 = atomic_fetch_sub_32(&value32, 10); + result64 = atomic_fetch_sub_64(&value64, 10); + resultd = atomic_fetch_sub_double(&valued, 10); + resultx = (int64_t)atomic_fetch_sub_ptr(valuep, 10); + + EXPECT_EQ(result8, 8); + EXPECT_EQ(value8, -2); + EXPECT_EQ(result16, 16); + EXPECT_EQ(value16, 6); + EXPECT_EQ(result32, 32); + EXPECT_EQ(value32, 22); + EXPECT_EQ(result64, 64); + EXPECT_EQ(value64, 54); + EXPECT_DOUBLE_EQ(resultd, 64); + EXPECT_DOUBLE_EQ(valued, 54); + EXPECT_EQ(resultx, 128); + EXPECT_EQ(*valuep, 118); + EXPECT_EQ(valuex, 118); } -//TEST(osAtomicTests, AddFetchPtr) { -// uintptr_t val = 0; -// uintptr_t* ptr = &val; -// uintptr_t ret = atomic_add_fetch_ptr(ptr, 10); -// EXPECT_EQ(ret, 10); -// EXPECT_EQ(val, 10); -//} +TEST(osAtomicTests, atomic_and_fetch) { + int8_t result8 = 0, value8 = 3; + int16_t result16 = 0, value16 = 3; + int32_t result32 = 0, value32 = 3; + int64_t result64 = 0, value64 = 3; + + int64_t valuex = 3; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_and_fetch_8(&value8, 5); + result16 = atomic_and_fetch_16(&value16, 5); + result32 = atomic_and_fetch_32(&value32, 5); + result64 = atomic_and_fetch_64(&value64, 5); + resultx = (int64_t)atomic_and_fetch_ptr(valuep, 5); + + EXPECT_EQ(result8, 1); + EXPECT_EQ(value8, 1); + EXPECT_EQ(result16, 1); + EXPECT_EQ(value16, 1); + EXPECT_EQ(result32, 1); + EXPECT_EQ(value32, 1); + EXPECT_EQ(result64, 1); + EXPECT_EQ(value64, 1); + EXPECT_EQ(resultx, 1); + EXPECT_EQ(*valuep, 1); + EXPECT_EQ(valuex, 1); +} + +TEST(osAtomicTests, atomic_fetch_and) { + int8_t result8 = 0, value8 = 3; + int16_t result16 = 0, value16 = 3; + int32_t result32 = 0, value32 = 3; + int64_t result64 = 0, value64 = 3; + + int64_t valuex = 3; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_fetch_and_8(&value8, 5); + result16 = atomic_fetch_and_16(&value16, 5); + result32 = atomic_fetch_and_32(&value32, 5); + result64 = atomic_fetch_and_64(&value64, 5); + resultx = (int64_t)atomic_fetch_and_ptr(valuep, 5); + + EXPECT_EQ(result8, 3); + EXPECT_EQ(value8, 1); + EXPECT_EQ(result16, 3); + EXPECT_EQ(value16, 1); + EXPECT_EQ(result32, 3); + EXPECT_EQ(value32, 1); + EXPECT_EQ(result64, 3); + EXPECT_EQ(value64, 1); + EXPECT_EQ(resultx, 3); + EXPECT_EQ(*valuep, 1); + EXPECT_EQ(valuex, 1); +} + +TEST(osAtomicTests, atomic_or_fetch) { + int8_t result8 = 0, value8 = 3; + int16_t result16 = 0, value16 = 3; + int32_t result32 = 0, value32 = 3; + int64_t result64 = 0, value64 = 3; + + int64_t valuex = 3; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_or_fetch_8(&value8, 5); + result16 = atomic_or_fetch_16(&value16, 5); + result32 = atomic_or_fetch_32(&value32, 5); + result64 = atomic_or_fetch_64(&value64, 5); + resultx = (int64_t)atomic_or_fetch_ptr(valuep, 5); + + EXPECT_EQ(result8, 7); + EXPECT_EQ(value8, 7); + EXPECT_EQ(result16, 7); + EXPECT_EQ(value16, 7); + EXPECT_EQ(result32, 7); + EXPECT_EQ(value32, 7); + EXPECT_EQ(result64, 7); + EXPECT_EQ(value64, 7); + EXPECT_EQ(resultx, 7); + EXPECT_EQ(*valuep, 7); + EXPECT_EQ(valuex, 7); +} + +TEST(osAtomicTests, atomic_fetch_or) { + int8_t result8 = 0, value8 = 3; + int16_t result16 = 0, value16 = 3; + int32_t result32 = 0, value32 = 3; + int64_t result64 = 0, value64 = 3; + + int64_t valuex = 3; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_fetch_or_8(&value8, 5); + result16 = atomic_fetch_or_16(&value16, 5); + result32 = atomic_fetch_or_32(&value32, 5); + result64 = atomic_fetch_or_64(&value64, 5); + resultx = (int64_t)atomic_fetch_or_ptr(valuep, 5); + + EXPECT_EQ(result8, 3); + EXPECT_EQ(value8, 7); + EXPECT_EQ(result16, 3); + EXPECT_EQ(value16, 7); + EXPECT_EQ(result32, 3); + EXPECT_EQ(value32, 7); + EXPECT_EQ(result64, 3); + EXPECT_EQ(value64, 7); + EXPECT_EQ(resultx, 3); + EXPECT_EQ(*valuep, 7); + EXPECT_EQ(valuex, 7); +} + + +TEST(osAtomicTests, atomic_xor_fetch) { + int8_t result8 = 0, value8 = 3; + int16_t result16 = 0, value16 = 3; + int32_t result32 = 0, value32 = 3; + int64_t result64 = 0, value64 = 3; + + int64_t valuex = 3; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_xor_fetch_8(&value8, 5); + result16 = atomic_xor_fetch_16(&value16, 5); + result32 = atomic_xor_fetch_32(&value32, 5); + result64 = atomic_xor_fetch_64(&value64, 5); + resultx = (int64_t)atomic_xor_fetch_ptr(valuep, 5); + + EXPECT_EQ(result8, 6); + EXPECT_EQ(value8, 6); + EXPECT_EQ(result16, 6); + EXPECT_EQ(value16, 6); + EXPECT_EQ(result32, 6); + EXPECT_EQ(value32, 6); + EXPECT_EQ(result64, 6); + EXPECT_EQ(value64, 6); + EXPECT_EQ(resultx, 6); + EXPECT_EQ(*valuep, 6); + EXPECT_EQ(valuex, 6); +} + +TEST(osAtomicTests, atomic_fetch_xor) { + int8_t result8 = 0, value8 = 3; + int16_t result16 = 0, value16 = 3; + int32_t result32 = 0, value32 = 3; + int64_t result64 = 0, value64 = 3; + + int64_t valuex = 3; + int64_t* valuep = &valuex; + int64_t resultx = 0; + + result8 = atomic_fetch_xor_8(&value8, 5); + result16 = atomic_fetch_xor_16(&value16, 5); + result32 = atomic_fetch_xor_32(&value32, 5); + result64 = atomic_fetch_xor_64(&value64, 5); + resultx = (int64_t)atomic_fetch_xor_ptr(valuep, 5); + + EXPECT_EQ(result8, 3); + EXPECT_EQ(value8, 6); + EXPECT_EQ(result16, 3); + EXPECT_EQ(value16, 6); + EXPECT_EQ(result32, 3); + EXPECT_EQ(value32, 6); + EXPECT_EQ(result64, 3); + EXPECT_EQ(value64, 6); + EXPECT_EQ(resultx, 3); + EXPECT_EQ(*valuep, 6); + EXPECT_EQ(valuex, 6); +} \ No newline at end of file diff --git a/source/os/test/osDirTests.cpp b/source/os/test/osDirTests.cpp new file mode 100644 index 0000000000..1dba7299be --- /dev/null +++ b/source/os/test/osDirTests.cpp @@ -0,0 +1,464 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "os.h" +#include "tlog.h" + +TEST(osDirTests, taosRemoveDir) { + int32_t ret = 0; + + const char* testDir = "/tmp/tdengine-test-dir"; + if (taosDirExist(testDir)) { + taosRemoveDir(testDir); + } + + taosRemoveDir(testDir); + + ret = taosMkDir(testDir); + EXPECT_EQ(ret, 0); + + const char* testFile = "/tmp/tdengine-test-dir/test-file"; + TdFilePtr testFilePtr = taosCreateFile(testFile, TD_FILE_CREATE); + EXPECT_NE(testFilePtr, nullptr); + ret = taosCloseFile(&testFilePtr); + EXPECT_EQ(ret, 0); + + const char* testDir2 = "/tmp/tdengine-test-dir/test-dir2"; + ret = taosMkDir(testDir2); + EXPECT_EQ(ret, 0); + + const char* testFile2 = "/tmp/tdengine-test-dir/test-dir2/test-file2"; + TdFilePtr testFilePtr2 = taosCreateFile(testFile2, TD_FILE_CREATE); + EXPECT_NE(testFilePtr2, nullptr); + ret = taosCloseFile(&testFilePtr2); + EXPECT_EQ(ret, 0); + + taosRemoveDir(testDir); + + bool exist = taosDirExist(testDir); + EXPECT_EQ(exist, false); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosDirExist) { + const char* dir1 = NULL; + bool exist = taosDirExist(dir1); + EXPECT_EQ(exist, false); + + char dir2[2048] = {0}; + for (int32_t i = 0; i < 2047; ++i) { + dir2[i] = 1; + } + exist = taosDirExist(dir2); + EXPECT_EQ(exist, false); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosMulMkDir) { + int32_t ret = 0; + + const char* dir1 = NULL; + ret = taosMulMkDir(dir1); + EXPECT_EQ(ret, -1); + + char dir2[2048] = {0}; + for (int32_t i = 0; i < 2047; ++i) { + dir2[i] = '1'; + } + ret = taosMulMkDir(dir2); + EXPECT_EQ(ret, -1); + + const char* dir3 = "/tmp/tdengine-test-dir/1/2/3/4"; + taosRemoveDir(dir3); + ret = taosMulMkDir(dir3); + EXPECT_EQ(ret, 0); + taosRemoveDir(dir3); + + const char* dir4 = "./tdengine-test-dir/1/2/3/4"; + taosRemoveDir(dir4); + ret = taosMulMkDir(dir4); + EXPECT_EQ(ret, 0); + taosRemoveDir(dir4); + + const char* dir5 = "tdengine-test-dir/1/2/3/4"; + taosRemoveDir(dir5); + ret = taosMulMkDir(dir5); + EXPECT_EQ(ret, 0); + ret = taosMulMkDir(dir5); + EXPECT_EQ(ret, 0); + taosRemoveDir(dir5); + + const char* testFile = "/tmp/tdengine-test-dir/test-file"; + TdFilePtr testFilePtr = taosCreateFile(testFile, TD_FILE_CREATE); + EXPECT_NE(testFilePtr, nullptr); + ret = taosCloseFile(&testFilePtr); + EXPECT_EQ(ret, 0); + char dir6[2048] = {0}; + strcpy(dir6, "/tmp/tdengine-test-dir/test-file/1/2/3/4"); + ret = taosMulMkDir(dir6); + EXPECT_NE(ret, 0); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosMulModeMkDir) { + int32_t ret = 0; + + const char* dir1 = NULL; + ret = taosMulModeMkDir(dir1, 777, true); + EXPECT_NE(ret, 0); + + char dir2[2048] = {0}; + for (int32_t i = 0; i < 2047; ++i) { + dir2[i] = '1'; + } + ret = taosMulModeMkDir(dir2, 777, true); + EXPECT_NE(ret, 0); + + const char* dir3 = "/tmp/tdengine-test-dir/1/2/3/4"; + taosRemoveDir(dir3); + ret = taosMulMkDir(dir3); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir3, 777, true); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir3, 777, false); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir3, 999, true); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir3, 999, false); + EXPECT_EQ(ret, 0); + taosRemoveDir(dir3); + + const char* dir4 = "./tdengine-test-dir/1/2/3/4"; + taosRemoveDir(dir4); + ret = taosMulModeMkDir(dir4, 777, true); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir4, 777, false); + EXPECT_EQ(ret, 0); + taosRemoveDir(dir4); + + const char* dir5 = "tdengine-test-dir/1/2/3/4"; + taosRemoveDir(dir5); + ret = taosMulModeMkDir(dir5, 777, true); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir5, 777, false); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir5, 777, true); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir5, 777, false); + EXPECT_EQ(ret, 0); + taosRemoveDir(dir5); + + const char* testFile = "/tmp/tdengine-test-dir/test-file"; + TdFilePtr testFilePtr = taosCreateFile(testFile, TD_FILE_CREATE); + EXPECT_NE(testFilePtr, nullptr); + ret = taosCloseFile(&testFilePtr); + EXPECT_EQ(ret, 0); + char dir6[2048] = {0}; + strcpy(dir6, "/tmp/tdengine-test-dir/test-file/1/2/3/4"); + ret = taosMulModeMkDir(dir6, 777, true); + EXPECT_NE(ret, 0); + + const char* dir7 = "tdengine-test-dir/1/2/3/5"; + taosRemoveDir(dir7); + ret = taosMulModeMkDir(dir7, 999, true); + EXPECT_EQ(ret, 0); + ret = taosMulModeMkDir(dir7, 999, false); + EXPECT_EQ(ret, 0); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosRemoveOldFiles) { + int32_t ret = 0; + const char* testDir = "/tmp/tdengine-test-dir"; + if (taosDirExist(testDir)) { + taosRemoveDir(testDir); + } + taosRemoveDir(testDir); + ret = taosMkDir(testDir); + EXPECT_EQ(ret, 0); + + const char* testFile = "/tmp/tdengine-test-dir/test-file"; + TdFilePtr testFilePtr = taosCreateFile(testFile, TD_FILE_CREATE); + EXPECT_NE(testFilePtr, nullptr); + ret = taosCloseFile(&testFilePtr); + EXPECT_EQ(ret, 0); + + taosRemoveOldFiles(testFile, 10); + + const char* testDir2 = "/tmp/tdengine-test-dir/test-dir2"; + ret = taosMkDir(testDir2); + EXPECT_EQ(ret, 0); + + const char* testFile3 = "/tmp/tdengine-test-dir/log.1433726073.gz"; + TdFilePtr testFilePtr3 = taosCreateFile(testFile3, TD_FILE_CREATE); + EXPECT_NE(testFilePtr3, nullptr); + ret = taosCloseFile(&testFilePtr3); + EXPECT_EQ(ret, 0); + + const char* testFile4 = "/tmp/tdengine-test-dir/log.80.gz"; + TdFilePtr testFilePtr4 = taosCreateFile(testFile4, TD_FILE_CREATE); + EXPECT_NE(testFilePtr4, nullptr); + ret = taosCloseFile(&testFilePtr4); + EXPECT_EQ(ret, 0); + + char testFile5[1024]; + snprintf(testFile5, 1024, "/tmp/tdengine-test-dir/log.%d.gz", taosGetTimestampSec()); + TdFilePtr testFilePtr5 = taosCreateFile(testFile5, TD_FILE_CREATE); + EXPECT_NE(testFilePtr5, nullptr); + ret = taosCloseFile(&testFilePtr5); + EXPECT_EQ(ret, 0); + + const char* testFile6 = "/tmp/tdengine-test-dir/log.1433726073.gz"; + TdFilePtr testFilePtr6 = taosCreateFile(testFile6, TD_FILE_CREATE); + EXPECT_NE(testFilePtr6, nullptr); + ret = taosCloseFile(&testFilePtr6); + EXPECT_EQ(ret, 0); + + taosRemoveOldFiles(testDir, 10); + + bool exist = taosDirExist(testDir); + EXPECT_EQ(exist, true); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosExpandDir) { + int32_t ret = 0; + const char* testDir = "/tmp/tdengine-test-dir"; + ret = taosMkDir(testDir); + EXPECT_EQ(ret, 0); + + char fullpath[1024] = {0}; + ret = taosExpandDir(testDir, NULL, 1024); + EXPECT_NE(ret, 0); + ret = taosExpandDir(NULL, fullpath, 1024); + EXPECT_NE(ret, 0); + ret = taosExpandDir(testDir, fullpath, 1024); + EXPECT_EQ(ret, 0); + + ret = taosExpandDir("/x123", fullpath, 1024); + EXPECT_EQ(ret, 0); + + ret = taosExpandDir("", fullpath, 1024); + EXPECT_EQ(ret, 0); + + char dir2[2048] = {0}; + for (int32_t i = 0; i < 2047; ++i) { + dir2[i] = '1'; + } + ret = taosExpandDir(dir2, fullpath, 1024); + EXPECT_EQ(ret, 0); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosRealPath) { + int32_t ret = 0; + char testDir[PATH_MAX * 2] = "/tmp/tdengine-test-dir"; + ret = taosMkDir(testDir); + EXPECT_EQ(ret, 0); + + char fullpath[PATH_MAX * 2] = {0}; + + ret = taosRealPath(testDir, NULL, PATH_MAX * 2); + EXPECT_EQ(ret, 0); + + ret = taosRealPath(NULL, fullpath, PATH_MAX * 2); + EXPECT_NE(ret, 0); + + ret = taosRealPath(testDir, fullpath, PATH_MAX * 2); + EXPECT_EQ(ret, 0); + + ret = taosRealPath(testDir, fullpath, 12); + EXPECT_NE(ret, 0); + + ret = taosRealPath("/c/d", fullpath, 1024); + EXPECT_NE(ret, 0); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosIsDir) { + bool ret = taosIsDir("/c/d"); + EXPECT_EQ(ret, false); +} + +TEST(osDirTests, taosDirName) { + char* ret = taosDirName(NULL); + EXPECT_EQ(ret, nullptr); + + char name1[24] = "xyz"; + ret = taosDirName(name1); + EXPECT_NE(ret, nullptr); + EXPECT_EQ(name1[0], 0); + + char name2[24] = "/root/xyz"; + ret = taosDirName(name2); + EXPECT_NE(ret, nullptr); + EXPECT_STREQ(ret, "/root"); +} + +TEST(osDirTests, taosDirEntryBaseName) { + char* ret = taosDirEntryBaseName(NULL); + EXPECT_EQ(ret, nullptr); + + char name1[12] = "/"; + ret = taosDirEntryBaseName(name1); + EXPECT_STREQ(ret, "/"); + + char name2[12] = "/root/"; + ret = taosDirEntryBaseName(name2); + EXPECT_STREQ(ret, "root"); + + char name3[12] = "/root"; + ret = taosDirEntryBaseName(name3); + EXPECT_STREQ(ret, "root"); + + char name4[12] = "root"; + ret = taosDirEntryBaseName(name4); + EXPECT_STREQ(ret, "root"); +} + +TEST(osDirTests, taosOpenDir) { + TdDirPtr ret = taosOpenDir(NULL); + EXPECT_EQ(ret, nullptr); +} + +TEST(osDirTests, taosReadDir) { + TdDirEntryPtr rddir = taosReadDir(NULL); + EXPECT_EQ(rddir, nullptr); + + int32_t ret = 0; + const char* testDir = "/tmp/tdengine-test-dir"; + ret = taosMkDir(testDir); + EXPECT_EQ(ret, 0); + + const char* testFile = "/tmp/tdengine-test-dir/test-file"; + TdFilePtr testFilePtr = taosCreateFile(testFile, TD_FILE_CREATE); + EXPECT_NE(testFilePtr, nullptr); + ret = taosCloseFile(&testFilePtr); + EXPECT_EQ(ret, 0); + + TdDirPtr dir = taosOpenDir(testFile); + EXPECT_EQ(dir, nullptr); + + const char* testDir2 = "/tmp/tdengine-test-dir/test-dir2"; + ret = taosMkDir(testDir2); + EXPECT_EQ(ret, 0); + + const char* testFile2 = "/tmp/tdengine-test-dir/test-dir2/test-file2"; + TdFilePtr testFilePtr2 = taosCreateFile(testFile2, TD_FILE_CREATE); + EXPECT_NE(testFilePtr2, nullptr); + ret = taosCloseFile(&testFilePtr2); + EXPECT_EQ(ret, 0); + + dir = taosOpenDir(testFile); + EXPECT_EQ(dir, nullptr); + + rddir = taosReadDir(dir); + EXPECT_EQ(rddir, nullptr); + + dir = taosOpenDir(testDir); + EXPECT_NE(dir, nullptr); + + rddir = taosReadDir(dir); + EXPECT_NE(rddir, nullptr); + + bool entry = taosDirEntryIsDir(NULL); + EXPECT_EQ(entry, false); + + char* entryname = taosGetDirEntryName(NULL); + EXPECT_EQ(entryname, nullptr); + + entryname = taosGetDirEntryName(rddir); + EXPECT_NE(entryname, nullptr); + + int32_t code = taosCloseDir(NULL); + EXPECT_NE(code, 0); + + code = taosCloseDir(&dir); + EXPECT_EQ(code, 0); + + taosRemoveDir("/tmp/tdengine-test-dir"); +} + +TEST(osDirTests, taosGetDirSize) { + TdDirEntryPtr rddir = taosReadDir(NULL); + EXPECT_EQ(rddir, nullptr); + + int32_t ret = 0; + const char* testDir = "/tmp/tdengine-test-dir"; + ret = taosMkDir(testDir); + EXPECT_EQ(ret, 0); + + const char* testFile = "/tmp/tdengine-test-dir/test-file"; + TdFilePtr testFilePtr = taosCreateFile(testFile, TD_FILE_CREATE); + EXPECT_NE(testFilePtr, nullptr); + ret = taosCloseFile(&testFilePtr); + EXPECT_EQ(ret, 0); + + TdDirPtr dir = taosOpenDir(testFile); + EXPECT_EQ(dir, nullptr); + + const char* testDir2 = "/tmp/tdengine-test-dir/test-dir2"; + ret = taosMkDir(testDir2); + EXPECT_EQ(ret, 0); + + const char* testFile2 = "/tmp/tdengine-test-dir/test-dir2/test-file2"; + TdFilePtr testFilePtr2 = taosCreateFile(testFile2, TD_FILE_CREATE); + EXPECT_NE(testFilePtr2, nullptr); + ret = taosCloseFile(&testFilePtr2); + EXPECT_EQ(ret, 0); + + int64_t size = -1; + ret = taosGetDirSize(testFile, &size); + EXPECT_NE(ret, 0); + + ret = taosGetDirSize(testDir, &size); + EXPECT_EQ(ret, 0); + + taosRemoveDir("/tmp/tdengine-test-dir"); + + taosRemoveDir("./tdengine-test-dir/1/2/3/5"); + taosRemoveDir("./tdengine-test-dir/1/2/3/4"); + taosRemoveDir("./tdengine-test-dir/1/2/3"); + taosRemoveDir("./tdengine-test-dir/1/2"); + taosRemoveDir("./tdengine-test-dir/1"); + taosRemoveDir("./tdengine-test-dir/"); + taosRemoveDir("tdengine-test-dir/1/2/3/5"); + taosRemoveDir("tdengine-test-dir/1/2/3/4"); + taosRemoveDir("tdengine-test-dir/1/2/3"); + taosRemoveDir("tdengine-test-dir/1/2"); + taosRemoveDir("tdengine-test-dir/1"); + taosRemoveDir("tdengine-test-dir/"); + taosRemoveDir("tdengine-test-dir/"); +} diff --git a/source/os/test/osEnvTests.cpp b/source/os/test/osEnvTests.cpp new file mode 100644 index 0000000000..6af6f84b68 --- /dev/null +++ b/source/os/test/osEnvTests.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "os.h" +#include "tlog.h" + +TEST(osEnvTests, osDefaultInit) { + int32_t ret = 0; + + strcpy(tsTimezoneStr, ""); + strcpy(tsTempDir, ""); + tsNumOfCores = 0; + ret = osDefaultInit(); + EXPECT_EQ(ret, 0); + + strcpy(tsTempDir, "/tmp"); + ret = osDefaultInit(); + EXPECT_EQ(ret, 0); + + osCleanup(); +} + +TEST(osEnvTests, osUpdate) { + int32_t ret = 0; + + strcpy(tsLogDir, ""); + strcpy(tsDataDir, ""); + strcpy(tsTempDir, ""); + + ret = osUpdate(); + EXPECT_EQ(ret, 0); +} + +TEST(osEnvTests, osSufficient) { + bool ret = 0; + + tsLogSpace.size.avail = 10000; + tsDataSpace.size.avail = 10000; + tsTempSpace.size.avail = 10000; + tsLogSpace.reserved = 2000; + tsDataSpace.reserved = 2000; + tsDataSpace.reserved = 2000; + + ret = osLogSpaceAvailable(); + EXPECT_EQ(ret, true); + + ret = osTempSpaceAvailable(); + EXPECT_EQ(ret, true); + + ret = osDataSpaceAvailable(); + EXPECT_EQ(ret, true); + + ret = osLogSpaceSufficient(); + EXPECT_EQ(ret, true); + + ret = osDataSpaceSufficient(); + EXPECT_EQ(ret, true); + + ret = osTempSpaceSufficient(); + EXPECT_EQ(ret, true); + + taosSetSystemLocale(NULL); + taosSetSystemLocale("1"); + + osSetProcPath(1, NULL); + osSetProcPath(0, (char **)&ret); +} \ No newline at end of file diff --git a/source/os/test/osStringTests.cpp b/source/os/test/osStringTests.cpp index de07d21959..36f5207346 100644 --- a/source/os/test/osStringTests.cpp +++ b/source/os/test/osStringTests.cpp @@ -154,3 +154,483 @@ TEST(osStringTests, ostsnprintfTests) { EXPECT_EQ(ret, 11); EXPECT_STREQ(buffer, "Float: 3.14"); } +TEST(osStringTests, osStr2Int64) { + int64_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2int64(NULL, &val); + assert(result == TSDB_CODE_INVALID_PARA); + + result = taosStr2int64("123", NULL); + ASSERT_NE(result, 0); + + // 测试无效输入 + result = taosStr2int64("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2int64("", &val); + ASSERT_NE(result, 0); + + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%lld", LLONG_MAX); + result = taosStr2int64(large_num, &val); + assert(result == 0); + assert(val == LLONG_MAX); + + snprintf(large_num, sizeof(large_num), "%lld", LLONG_MIN); + result = taosStr2int64(large_num, &val); + assert(result == 0); + assert(val == LLONG_MIN); + + result = taosStr2int64("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2int64("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2int64("12345", &val); + assert(result == 0); + assert(val == 12345); + + result = taosStr2int64("-12345", &val); + assert(result == 0); + assert(val == -12345); + + result = taosStr2int64("0", &val); + assert(result == 0); + assert(val == 0); + + // 测试带空格的字符串 + result = taosStr2int64(" 12345", &val); + assert(result == 0); + assert(val == 12345); + + result = taosStr2int64("12345 ", &val); + assert(result == 0); + assert(val == 12345); +} +TEST(osStringTests, osStr2int32) { + int32_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2int32(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + result = taosStr2int32("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + // 测试无效输入 + result = taosStr2int32("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2int32("", &val); + ASSERT_NE(result, 0); + + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%d", INT_MAX); + result = taosStr2int32(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT_MAX); + + snprintf(large_num, sizeof(large_num), "%d", INT_MIN); + result = taosStr2int32(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT_MIN); + + // 测试大于 INT32 范围的值 + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT_MAX + 1); + result = taosStr2int32(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT_MIN - 1); + result = taosStr2int32(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + + result = taosStr2int32("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2int32("abc123", &val); + ASSERT_NE(result, 0); + + // 测试有效的整数字符串 + result = taosStr2int32("12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2int32("-12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, -12345); + + result = taosStr2int32("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); + + // 测试带空格的字符串 + result = taosStr2int32(" 12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2int32("12345 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); +} + +TEST(osStringTests, taosStr2int16) { + int16_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2int16(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + result = taosStr2int16("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + // 测试无效输入 + result = taosStr2int16("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2int16("", &val); + ASSERT_NE(result, 0); + + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%d", INT16_MAX); + result = taosStr2int16(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT16_MAX); + + snprintf(large_num, sizeof(large_num), "%d", INT16_MIN); + result = taosStr2int16(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT16_MIN); + + // 测试大于 INT16 范围的值 + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT16_MAX + 1); + result = taosStr2int16(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT16_MIN - 1); + result = taosStr2int16(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + result = taosStr2int16("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2int16("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2int16("12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2int16("-12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, -12345); + + result = taosStr2int16("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); + + // 测试带空格的字符串 + result = taosStr2int16(" 12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2int16("12345 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); +} + + +TEST(osStringTests, taosStr2int8) { + int8_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2int8(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + result = taosStr2int8("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + // 测试无效输入 + result = taosStr2int8("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2int8("", &val); + ASSERT_NE(result, 0); + + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%d", INT8_MAX); + result = taosStr2int8(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT8_MAX); + + snprintf(large_num, sizeof(large_num), "%d", INT8_MIN); + result = taosStr2int8(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, INT8_MIN); + + // 测试大于 INT8 范围的值 + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT8_MAX + 1); + result = taosStr2int8(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + snprintf(large_num, sizeof(large_num), "%lld", (long long)INT8_MIN - 1); + result = taosStr2int8(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + result = taosStr2int8("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2int8("abc123", &val); + ASSERT_NE(result, 0); + + // 测试有效的整数字符串 + result = taosStr2int8("123", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2int8("-123", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, -123); + + result = taosStr2int8("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); + + // 测试带空格的字符串 + result = taosStr2int8(" 123", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2int8("123 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); +} + +TEST(osStringTests, osStr2Uint64) { + uint64_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2Uint64(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + result = taosStr2Uint64("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + // 测试无效输入 + result = taosStr2Uint64("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2Uint64("", &val); + ASSERT_NE(result, 0); + + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%llu", ULLONG_MAX); + result = taosStr2Uint64(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, ULLONG_MAX); + + result = taosStr2Uint64("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2Uint64("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2Uint64("12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2Uint64("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); + + // 测试带空格的字符串 + result = taosStr2Uint64(" 12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2Uint64("12345 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + +} + +TEST(osStringTests, taosStr2Uint32) { + uint32_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2Uint32(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + result = taosStr2Uint32("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + // 测试无效输入 + result = taosStr2Uint32("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2Uint32("", &val); + ASSERT_NE(result, 0); + + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%u", UINT32_MAX); + result = taosStr2Uint32(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, UINT32_MAX); + + // 测试大于 UINT32 范围的值 + snprintf(large_num, sizeof(large_num), "%llu", (unsigned long long)UINT32_MAX + 1); + result = taosStr2Uint32(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + result = taosStr2Uint32("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2Uint32("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2Uint32("12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2Uint32("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); + + // 测试带空格的字符串 + result = taosStr2Uint32(" 12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2Uint32("12345 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); +} + +TEST(osStringTests, taosStr2Uint16) { + uint16_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2Uint16(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + result = taosStr2Uint16("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + // 测试无效输入 + result = taosStr2Uint16("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2Uint16("", &val); + ASSERT_NE(result, 0); + + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%u", UINT16_MAX); + result = taosStr2Uint16(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, UINT16_MAX); + + // 测试大于 UINT16 范围的值 + snprintf(large_num, sizeof(large_num), "%llu", (unsigned long long)UINT16_MAX + 1); + result = taosStr2Uint16(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + result = taosStr2Uint16("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2Uint16("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2Uint16("12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2Uint16("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); + + // 测试带空格的字符串 + result = taosStr2Uint16(" 12345", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); + + result = taosStr2Uint16("12345 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 12345); +} + +TEST(osStringTests, taosStr2Uint8) { + uint8_t val; + int32_t result; + + // 测试空指针输入 + result = taosStr2Uint8(NULL, &val); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + result = taosStr2Uint8("123", NULL); + ASSERT_EQ(result, TSDB_CODE_INVALID_PARA); + + // 测试无效输入 + result = taosStr2Uint8("abc", &val); + ASSERT_NE(result, 0); + + result = taosStr2Uint8("", &val); + ASSERT_NE(result, 0); + + // 测试超出范围的值 + char large_num[50]; + snprintf(large_num, sizeof(large_num), "%u", UINT8_MAX); + result = taosStr2Uint8(large_num, &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, UINT8_MAX); + + // 测试大于 UINT8 范围的值 + snprintf(large_num, sizeof(large_num), "%llu", (unsigned long long)UINT8_MAX + 1); + result = taosStr2Uint8(large_num, &val); + ASSERT_EQ(result, TAOS_SYSTEM_ERROR(ERANGE)); + + result = taosStr2Uint8("123abc", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2Uint8("abc123", &val); + ASSERT_NE(result, 0); + // 测试有效的整数字符串 + result = taosStr2Uint8("123", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2Uint8("0", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 0); + + // 测试带空格的字符串 + result = taosStr2Uint8(" 123", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); + + result = taosStr2Uint8("123 ", &val); + ASSERT_EQ(result, 0); + ASSERT_EQ(val, 123); +} + diff --git a/source/os/test/osTimeTests.cpp b/source/os/test/osTimeTests.cpp index 42837cc537..5c8c837dca 100644 --- a/source/os/test/osTimeTests.cpp +++ b/source/os/test/osTimeTests.cpp @@ -29,30 +29,11 @@ #include "os.h" #include "tlog.h" -TEST(osTimeTests, taosLocalTimeNolock) { - time_t currentTime; - // Test when result is not NULL - struct tm expectedTime; - struct tm* result = taosLocalTimeNolock(&expectedTime, ¤tTime, 1); - if (result) { - EXPECT_EQ(expectedTime.tm_year, result->tm_year); - EXPECT_EQ(expectedTime.tm_mon, result->tm_mon); - EXPECT_EQ(expectedTime.tm_mday, result->tm_mday); - EXPECT_EQ(expectedTime.tm_hour, result->tm_hour); - EXPECT_EQ(expectedTime.tm_min, result->tm_min); - EXPECT_EQ(expectedTime.tm_sec, result->tm_sec); - EXPECT_EQ(expectedTime.tm_wday, result->tm_wday); - EXPECT_EQ(expectedTime.tm_yday, result->tm_yday); - EXPECT_EQ(expectedTime.tm_isdst, result->tm_isdst); - } -} - - TEST(osTimeTests, taosLocalTime) { // Test 1: Test when both timep and result are not NULL time_t timep = 1617531000; // 2021-04-04 18:10:00 struct tm result; - struct tm* local_time = taosLocalTime(&timep, &result, NULL, 0); + struct tm* local_time = taosLocalTime(&timep, &result, NULL, 0, NULL); ASSERT_NE(local_time, nullptr); ASSERT_EQ(local_time->tm_year, 121); ASSERT_EQ(local_time->tm_mon, 3); @@ -62,13 +43,13 @@ TEST(osTimeTests, taosLocalTime) { ASSERT_EQ(local_time->tm_sec, 00); // Test 2: Test when timep is NULL - local_time = taosLocalTime(NULL, &result, NULL, 0); + local_time = taosLocalTime(NULL, &result, NULL, 0, NULL); ASSERT_EQ(local_time, nullptr); // Test 4: Test when timep is negative on Windows #ifdef WINDOWS time_t pos_timep = 1609459200; // 2021-01-01 08:00:00 - local_time = taosLocalTime(&pos_timep, &result, NULL, 0); + local_time = taosLocalTime(&pos_timep, &result, NULL, 0, NULL); ASSERT_NE(local_time, nullptr); ASSERT_EQ(local_time->tm_year, 121); ASSERT_EQ(local_time->tm_mon, 0); @@ -78,7 +59,7 @@ TEST(osTimeTests, taosLocalTime) { ASSERT_EQ(local_time->tm_sec, 0); time_t neg_timep = -1617531000; // 1918-09-29 21:50:00 - local_time = taosLocalTime(&neg_timep, &result, NULL, 0); + local_time = taosLocalTime(&neg_timep, &result, NULL, 0, NULL); ASSERT_NE(local_time, nullptr); ASSERT_EQ(local_time->tm_year, 18); ASSERT_EQ(local_time->tm_mon, 8); @@ -88,7 +69,7 @@ TEST(osTimeTests, taosLocalTime) { ASSERT_EQ(local_time->tm_sec, 0); time_t neg_timep2 = -315619200; // 1960-01-01 08:00:00 - local_time = taosLocalTime(&neg_timep2, &result, NULL, 0); + local_time = taosLocalTime(&neg_timep2, &result, NULL, 0, NULL); ASSERT_NE(local_time, nullptr); ASSERT_EQ(local_time->tm_year, 60); ASSERT_EQ(local_time->tm_mon, 0); @@ -98,7 +79,7 @@ TEST(osTimeTests, taosLocalTime) { ASSERT_EQ(local_time->tm_sec, 0); time_t zero_timep = 0; // 1970-01-01 08:00:00 - local_time = taosLocalTime(&zero_timep, &result, NULL, 0); + local_time = taosLocalTime(&zero_timep, &result, NULL, 0, NULL); ASSERT_NE(local_time, nullptr); ASSERT_EQ(local_time->tm_year, 70); ASSERT_EQ(local_time->tm_mon, 0); @@ -108,7 +89,7 @@ TEST(osTimeTests, taosLocalTime) { ASSERT_EQ(local_time->tm_sec, 0); time_t neg_timep3 = -78115158887; - local_time = taosLocalTime(&neg_timep3, &result, NULL, 0); + local_time = taosLocalTime(&neg_timep3, &result, NULL, 0, NULL); ASSERT_EQ(local_time, nullptr); #endif } \ No newline at end of file diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index ebc379897f..c95030b06e 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1508,7 +1508,7 @@ int32_t comparewcsRegexMatch(const void *pString, const void *pPattern) { return 1; // terrno has been set } - int convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pPattern), len, pattern); + int convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pPattern), len, pattern, NULL); if (convertLen < 0) { taosMemoryFree(pattern); return 1; // terrno has been set @@ -1523,7 +1523,7 @@ int32_t comparewcsRegexMatch(const void *pString, const void *pPattern) { return 1; // terrno has been set } - convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pString), len, str); + convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pString), len, str, NULL); if (convertLen < 0) { taosMemoryFree(str); taosMemoryFree(pattern); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index d2a2b1fb9a..7ff1f18387 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -24,6 +24,7 @@ #include "tlog.h" #include "tunit.h" #include "tutil.h" +#include "tconv.h" #define CFG_NAME_PRINT_LEN 32 #define CFG_SRC_PRINT_LEN 12 @@ -151,11 +152,13 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { } static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - bool tmp = false; + int32_t code = 0; + bool tmp = false; if (strcasecmp(value, "true") == 0) { tmp = true; } - if (atoi(value) > 0) { + int32_t val = 0; + if ((code = taosStr2int32(value, &val)) == 0 && val > 0) { tmp = true; } @@ -245,19 +248,69 @@ static int32_t doSetConf(SConfigItem *pItem, const char *value, ECfgSrcType styp } static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - TAOS_CHECK_RETURN(doSetConf(pItem, value, stype)); - if (strlen(value) == 0) { - uError("cfg:%s, type:%s src:%s, value:%s, skip to set timezone", pItem->name, cfgDtypeStr(pItem->dtype), - cfgStypeStr(stype), value); - TAOS_RETURN(TSDB_CODE_SUCCESS); + if (stype == CFG_STYPE_ALTER_SERVER_CMD || (pItem->dynScope & CFG_DYN_CLIENT) == 0){ + uError("failed to config timezone, not support"); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); } + if (value == NULL) { + uError("cfg:%s, type:%s src:%s, value is null, skip to set timezone", pItem->name, cfgDtypeStr(pItem->dtype), + cfgStypeStr(stype)); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } TAOS_CHECK_RETURN(osSetTimezone(value)); + + TAOS_CHECK_RETURN(doSetConf(pItem, tsTimezoneStr, stype)); + + TAOS_RETURN(TSDB_CODE_SUCCESS); +} + +static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + if (stype == CFG_STYPE_ALTER_SERVER_CMD || stype == CFG_STYPE_ALTER_CLIENT_CMD){ + uError("failed to config charset, not support"); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } + + if (value == NULL || strlen(value) == 0) { + uError("cfg:%s, type:%s src:%s, value:%s, skip to set charset", pItem->name, cfgDtypeStr(pItem->dtype), + cfgStypeStr(stype), value); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } + + if (!taosValidateEncodec(value)) { + uError("invalid charset:%s", value); + TAOS_RETURN(terrno); + } + + if ((tsCharsetCxt = taosConvInit(value)) == NULL) { + TAOS_RETURN(terrno); + } + (void)memcpy(tsCharset, value, strlen(value) + 1); + TAOS_CHECK_RETURN(doSetConf(pItem, value, stype)); + + TAOS_RETURN(TSDB_CODE_SUCCESS); +} + +static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + if (stype == CFG_STYPE_ALTER_SERVER_CMD || (pItem->dynScope & CFG_DYN_CLIENT) == 0){ + uError("failed to config locale, not support"); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } + + if (value == NULL || strlen(value) == 0 || taosSetSystemLocale(value) != 0) { + uError("cfg:%s, type:%s src:%s, value:%s, skip to set locale", pItem->name, cfgDtypeStr(pItem->dtype), + cfgStypeStr(stype), value); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } + + TAOS_CHECK_RETURN(doSetConf(pItem, value, stype)); + TAOS_RETURN(TSDB_CODE_SUCCESS); } static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, const char *level, const char *primary, const char *disable, ECfgSrcType stype) { + int32_t code = 0; (void)taosThreadMutexLock(&pCfg->lock); SConfigItem *pItem = cfgGetItem(pCfg, name); @@ -278,20 +331,40 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, SDiskCfg cfg = {0}; tstrncpy(cfg.dir, pItem->str, sizeof(cfg.dir)); - cfg.level = level ? atoi(level) : 0; - cfg.primary = primary ? atoi(primary) : 1; - cfg.disable = disable ? atoi(disable) : 0; + + if (level == NULL || strlen(level) == 0) { + cfg.level = 0; + } else { + code = taosStr2int32(level, &cfg.level); + TAOS_CHECK_GOTO(code, NULL, _err); + } + + if (primary == NULL || strlen(primary) == 0) { + cfg.primary = 1; + } else { + code = taosStr2int32(primary, &cfg.primary); + TAOS_CHECK_GOTO(code, NULL, _err); + } + + if (disable == NULL || strlen(disable) == 0) { + cfg.disable = 0; + } else { + code = taosStr2int8(disable, &cfg.disable); + TAOS_CHECK_GOTO(code, NULL, _err); + } void *ret = taosArrayPush(pItem->array, &cfg); if (ret == NULL) { - (void)taosThreadMutexUnlock(&pCfg->lock); - - TAOS_RETURN(terrno); + code = terrno; + TAOS_CHECK_GOTO(code, NULL, _err); } pItem->stype = stype; (void)taosThreadMutexUnlock(&pCfg->lock); TAOS_RETURN(TSDB_CODE_SUCCESS); +_err: + (void)taosThreadMutexUnlock(&pCfg->lock); + TAOS_RETURN(code); } static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool resetArray) { @@ -315,7 +388,7 @@ static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool rese if (pDebugFlagItem == NULL) return -1; if (pDebugFlagItem->array != NULL) { SLogVar logVar = {0}; - (void)strncpy(logVar.name, name, TSDB_LOG_VAR_LEN - 1); + tstrncpy(logVar.name, name, TSDB_LOG_VAR_LEN); if (NULL == taosArrayPush(pDebugFlagItem->array, &logVar)) { TAOS_RETURN(terrno); } @@ -368,11 +441,11 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy break; } case CFG_DTYPE_CHARSET: { - code = doSetConf(pItem, value, stype); + code = cfgSetCharset(pItem, value, stype); break; } case CFG_DTYPE_LOCALE: { - code = doSetConf(pItem, value, stype); + code = cfgSetLocale(pItem, value, stype); break; } case CFG_DTYPE_NONE: @@ -412,6 +485,7 @@ void cfgLock(SConfig *pCfg) { void cfgUnLock(SConfig *pCfg) { (void)taosThreadMutexUnlock(&pCfg->lock); } int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer) { + int32_t code = 0; ECfgDynType dynType = isServer ? CFG_DYN_SERVER : CFG_DYN_CLIENT; cfgLock(pCfg); @@ -443,8 +517,9 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p } } break; case CFG_DTYPE_BOOL: { - int32_t ival = (int32_t)atoi(pVal); - if (ival != 0 && ival != 1) { + int32_t ival = 0; + code = taosStr2int32(pVal, &ival); + if (code != 0 || (ival != 0 && ival != 1)) { uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name, cfgDtypeStr(pItem->dtype), ival); cfgUnLock(pCfg); TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); @@ -634,6 +709,10 @@ const char *cfgStypeStr(ECfgSrcType type) { return "taos_options"; case CFG_STYPE_ENV_CMD: return "env_cmd"; + case CFG_STYPE_ALTER_CLIENT_CMD: + return "alter_client_cmd"; + case CFG_STYPE_ALTER_SERVER_CMD: + return "alter_server_cmd"; default: return "invalid"; } @@ -684,11 +763,19 @@ int32_t cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t case CFG_DTYPE_DOUBLE: len = tsnprintf(buf, bufSize, "%f", pItem->fval); break; + case CFG_DTYPE_TIMEZONE:{ +// char str1[TD_TIMEZONE_LEN] = {0}; +// time_t tx1 = taosGetTimestampSec(); +// if (taosFormatTimezoneStr(tx1, buf, NULL, str1) != 0) { +// tstrncpy(str1, "tz error", sizeof(str1)); +// } +// len = tsnprintf(buf, bufSize, "%s", str1); +// break; + } case CFG_DTYPE_STRING: case CFG_DTYPE_DIR: case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: - case CFG_DTYPE_TIMEZONE: case CFG_DTYPE_NONE: len = tsnprintf(buf, bufSize, "%s", pItem->str); break; @@ -887,9 +974,10 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { for (size_t j = 0; j < sz; ++j) { SDiskCfg *pCfg = taosArrayGet(pItem->array, j); if (dump) { - (void)printf("%s %s %s l:%d p:%d d:%"PRIi8"\n", src, name, pCfg->dir, pCfg->level, pCfg->primary, pCfg->disable); + (void)printf("%s %s %s l:%d p:%d d:%" PRIi8 "\n", src, name, pCfg->dir, pCfg->level, pCfg->primary, + pCfg->disable); } else { - uInfo("%s %s %s l:%d p:%d d:%"PRIi8, src, name, pCfg->dir, pCfg->level, pCfg->primary, pCfg->disable); + uInfo("%s %s %s l:%d p:%d d:%" PRIi8, src, name, pCfg->dir, pCfg->level, pCfg->primary, pCfg->disable); } } break; @@ -924,7 +1012,7 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { name = value = value2 = value3 = value4 = NULL; olen = vlen = vlen2 = vlen3 = vlen4 = 0; - strncpy(line, *pEnv, sizeof(line) - 1); + tstrncpy(line, *pEnv, sizeof(line)); pEnv++; if (taosEnvToCfg(line, line) < 0) { uTrace("failed to convert env to cfg:%s", line); @@ -969,7 +1057,7 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { int32_t index = 0; if (envCmd == NULL) TAOS_RETURN(TSDB_CODE_SUCCESS); while (envCmd[index] != NULL) { - strncpy(buf, envCmd[index], sizeof(buf) - 1); + tstrncpy(buf, envCmd[index], sizeof(buf)); buf[sizeof(buf) - 1] = 0; if (taosEnvToCfg(buf, buf) < 0) { uTrace("failed to convert env to cfg:%s", buf); @@ -1420,7 +1508,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl char **pEnv = environ; line[1023] = 0; while (*pEnv != NULL) { - strncpy(line, *pEnv, sizeof(line) - 1); + tstrncpy(line, *pEnv, sizeof(line)); pEnv++; if (strncmp(line, "TAOS_APOLLO_URL", 14) == 0) { char *p = strchr(line, '='); diff --git a/source/util/src/tconv.c b/source/util/src/tconv.c new file mode 100644 index 0000000000..d9932489f8 --- /dev/null +++ b/source/util/src/tconv.c @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC + +#include "tconv.h" +#include "thash.h" +#include "osString.h" + +#define CONV_MAX_NUM 32 +SHashObj *gConvInfo = NULL; + +// M2C: Mbs --> Ucs4 +// C2M: Ucs4--> Mbs + +static void taosConvDestroyInner(void *arg) { + SConvInfo *info = (SConvInfo *)arg; + if (info == NULL) { + return; + } + for (int32_t i = 0; i < info->gConvMaxNum[M2C]; ++i) { + (void)iconv_close(info->gConv[M2C][i].conv); + } + for (int32_t i = 0; i < info->gConvMaxNum[C2M]; ++i) { + (void)iconv_close(info->gConv[C2M][i].conv); + } + taosMemoryFreeClear(info->gConv[M2C]); + taosMemoryFreeClear(info->gConv[C2M]); + + info->gConvMaxNum[M2C] = -1; + info->gConvMaxNum[C2M] = -1; +} + +void* taosConvInit(const char* charset) { + if (charset == NULL){ + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + + void* conv = NULL; + static int32_t lock_c = 0; + + for (int i = 1; atomic_val_compare_exchange_32(&lock_c, 0, 1) != 0; ++i) { + if (i % 1000 == 0) { + uInfo("haven't acquire lock after spin %d times.", i); + (void)sched_yield(); + } + } + + if (gConvInfo == NULL){ + gConvInfo = taosHashInit(0, MurmurHash3_32, false, HASH_ENTRY_LOCK); + if (gConvInfo == NULL) { + atomic_store_32(&lock_c, 0); + goto END; + } + taosHashSetFreeFp(gConvInfo, taosConvDestroyInner); + } + + conv = taosHashGet(gConvInfo, charset, strlen(charset)); + if (conv != NULL){ + goto END; + } + + SConvInfo info = {0}; + info.gConvMaxNum[M2C] = CONV_MAX_NUM; + info.gConvMaxNum[C2M] = CONV_MAX_NUM; + tstrncpy(info.charset, charset, sizeof(info.charset)); + + info.gConv[M2C] = taosMemoryCalloc(info.gConvMaxNum[M2C], sizeof(SConv)); + if (info.gConv[M2C] == NULL) { + goto FAILED; + } + + info.gConv[C2M] = taosMemoryCalloc(info.gConvMaxNum[C2M], sizeof(SConv)); + if (info.gConv[C2M] == NULL) { + goto FAILED; + } + + for (int32_t i = 0; i < info.gConvMaxNum[M2C]; ++i) { + info.gConv[M2C][i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, charset); + if ((iconv_t)-1 == info.gConv[M2C][i].conv) { + terrno = TAOS_SYSTEM_ERROR(errno); + goto FAILED; + } + } + for (int32_t i = 0; i < info.gConvMaxNum[C2M]; ++i) { + info.gConv[C2M][i].conv = iconv_open(charset, DEFAULT_UNICODE_ENCODEC); + if ((iconv_t)-1 == info.gConv[C2M][i].conv) { + terrno = TAOS_SYSTEM_ERROR(errno); + goto FAILED; + } + } + + int32_t code = taosHashPut(gConvInfo, charset, strlen(charset), &info, sizeof(info)); + if (code != 0){ + goto FAILED; + } + conv = taosHashGet(gConvInfo, charset, strlen(charset)); + goto END; + +FAILED: + taosConvDestroyInner(&info); + +END: + atomic_store_32(&lock_c, 0); + return conv; +} + +void taosConvDestroy() { + taosHashCleanup(gConvInfo); +} diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 2df933dfa2..f5143779f2 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -58,12 +58,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MAX_SESSIONS, "rpc open too many ses TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_ERROR, "rpc network error") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_BUSY, "rpc network busy") TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_MODULE_QUIT, "http-report already quit") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MODULE_QUIT, "rpc module already quit") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ASYNC_MODULE_QUIT, "rpc async module already quit") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ASYNC_IN_PROCESS, "rpc async in process") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NO_STATE, "rpc no state") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_STATE_DROPED, "rpc state already dropped") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MSG_EXCCED_LIMIT, "rpc msg exceed limit") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MODULE_QUIT, "rpc module already quit") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ASYNC_MODULE_QUIT, "rpc async module already quit") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ASYNC_IN_PROCESS, "rpc async in process") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NO_STATE, "rpc no state") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_STATE_DROPED, "rpc state already dropped") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MSG_EXCCED_LIMIT, "rpc msg exceed limit") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED, "Client and server's time is not synchronized") @@ -167,7 +167,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_COMPRESS_PARAM_ERROR, "Invalid compress para TAOS_DEFINE_ERROR(TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR, "Invalid compress level param") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_FAIL_GENERATE_JSON, "failed to generate JSON") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR, "bind number out of range or not match") - +TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUPPORTTED_IN_WINDOWS, "Operation not supported in windows") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INTERNAL_ERROR, "Internal error") @@ -444,6 +444,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_ARB_NOT_SYNCED, "Vgroup peer is not sy TAOS_DEFINE_ERROR(TSDB_CODE_VND_WRITE_DISABLED, "Vnode write is disabled for snapshot") TAOS_DEFINE_ERROR(TSDB_CODE_VND_COLUMN_COMPRESS_ALREADY_EXIST,"Same with old param") TAOS_DEFINE_ERROR(TSDB_CODE_VND_TTL_FLUSH_INCOMPLETION, "Failed to flush all ttl modification to tdb") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_ALREADY_EXIST_BUT_NOT_MATCH, "Vnode already exist but Dbid not match") // tsdb @@ -875,6 +876,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY, "Queue out of memory TAOS_DEFINE_ERROR(TSDB_CODE_AUDIT_NOT_FORMAT_TO_JSON, "can't format to json") TAOS_DEFINE_ERROR(TSDB_CODE_AUDIT_FAIL_SEND_AUDIT_RECORD, "Failed to send out audit record") TAOS_DEFINE_ERROR(TSDB_CODE_AUDIT_FAIL_GENERATE_JSON, "Failed to generate json") + #ifdef TAOS_ERROR_C }; #endif diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 693935575a..7a5852f329 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -20,6 +20,7 @@ #include "tglobal.h" #include "tjson.h" #include "tutil.h" +#include "ttime.h" #define LOG_MAX_LINE_SIZE (10024) #define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3) @@ -171,30 +172,12 @@ static int32_t getDay(char *buf, int32_t bufSize) { return code; } struct tm tmInfo; - if (taosLocalTime(&t, &tmInfo, buf, bufSize) != NULL) { - TAOS_UNUSED(strftime(buf, bufSize, "%Y-%m-%d", &tmInfo)); + if (taosLocalTime(&t, &tmInfo, buf, bufSize, NULL) != NULL) { + TAOS_UNUSED(taosStrfTime(buf, bufSize, "%Y-%m-%d", &tmInfo)); } return 0; } -static int64_t getTimestampToday() { - time_t t; - int32_t code = taosTime(&t); - if (code != 0) { - uError("failed to get time, reason:%s", tstrerror(code)); - return 0; - } - struct tm tm; - if (taosLocalTime(&t, &tm, NULL, 0) == NULL) { - return 0; - } - tm.tm_hour = 0; - tm.tm_min = 0; - tm.tm_sec = 0; - - return (int64_t)taosMktime(&tm); -} - static void getFullPathName(char *fullName, const char *logName) { if (strlen(tsLogDir) != 0) { char lastC = tsLogDir[strlen(tsLogDir) - 1]; @@ -230,7 +213,7 @@ int32_t taosInitSlowLog() { } (void)snprintf(name, PATH_MAX + TD_TIME_STR_LEN, "%s.%s", tsLogObj.slowLogName, day); - tsLogObj.timestampToday = getTimestampToday(); + tsLogObj.timestampToday = taosGetTimestampToday(TSDB_TIME_PRECISION_SECONDS, NULL); tsLogObj.slowHandle = taosLogBuffNew(LOG_SLOW_BUF_SIZE); if (tsLogObj.slowHandle == NULL) return terrno; @@ -536,7 +519,7 @@ static void taosOpenNewSlowLogFile() { TdFilePtr pOldFile = tsLogObj.slowHandle->pFile; tsLogObj.slowHandle->pFile = pFile; (void)taosCloseFile(&pOldFile); - tsLogObj.timestampToday = getTimestampToday(); + tsLogObj.timestampToday = taosGetTimestampToday(TSDB_TIME_PRECISION_SECONDS, NULL); (void)taosThreadMutexUnlock(&tsLogObj.logMutex); } @@ -722,8 +705,11 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { TAOS_UNUSED(taosGetTimeOfDay(&timeSecs)); time_t curTime = timeSecs.tv_sec; - ptm = taosLocalTime(&curTime, &Tm, NULL, 0); - + ptm = taosLocalTime(&curTime, &Tm, NULL, 0, NULL); + if (ptm == NULL){ + uError("%s failed to get local time, code:%d", __FUNCTION__ , errno); + return 0; + } return sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " %s %s", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId(), LOG_EDITION_FLG, flags); diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index dd3c6ece40..bf1f150262 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -417,25 +417,6 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]) { return 0; } -char *taosIpStr(uint32_t ipInt) { - static char ipStrArray[3][30]; - static int32_t ipStrIndex = 0; - - char *ipStr = ipStrArray[(ipStrIndex++) % 3]; - // sprintf(ipStr, "0x%x:%u.%u.%u.%u", ipInt, ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt - // >> 24)); - sprintf(ipStr, "%u.%u.%u.%u", ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24)); - return ipStr; -} - -void taosIp2String(uint32_t ip, char *str) { - sprintf(str, "%u.%u.%u.%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24)); -} - -void taosIpPort2String(uint32_t ip, uint16_t port, char *str) { - sprintf(str, "%u.%u.%u.%u:%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24), port); -} - size_t tstrncspn(const char *str, size_t size, const char *reject, size_t rsize) { if (rsize == 0 || rsize == 1) { char *p = strnchr(str, reject[0], size, false); @@ -544,6 +525,64 @@ bool tIsValidFileName(const char *fileName, const char *pattern) { bool tIsValidFilePath(const char *filePath, const char *pattern) { const char *filePathPattern = "^[a-zA-Z0-9:/\\_.-]+$"; - return tIsValidFileName(filePath, pattern ? pattern : filePathPattern); +} + +bool taosIsBigChar(char c) { + if (c >= 'A' && c <= 'Z') { + return true; + } else { + return false; + } +} + +bool taosIsSmallChar(char c) { + if (c >= 'a' && c <= 'z') { + return true; + } else { + return false; + } +} + +bool taosIsNumberChar(char c) { + if (c >= '0' && c <= '9') { + return true; + } else { + return false; + } +} + +bool taosIsSpecialChar(char c) { + switch (c) { + case '!': + case '@': + case '#': + case '$': + case '%': + case '^': + case '&': + case '*': + case '(': + case ')': + case '-': + case '_': + case '+': + case '=': + case '[': + case ']': + case '{': + case '}': + case ':': + case ';': + case '>': + case '<': + case '?': + case '|': + case '~': + case ',': + case '.': + return true; + default: + return false; + } } \ No newline at end of file diff --git a/source/util/src/tversion.c b/source/util/src/tversion.c index d1a2ecf827..1a8d8f2d23 100644 --- a/source/util/src/tversion.c +++ b/source/util/src/tversion.c @@ -18,6 +18,7 @@ #include "taoserror.h" int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) { + int32_t code = 0; if (vstr == NULL) { return terrno = TSDB_CODE_INVALID_VERSION_STRING; } @@ -31,7 +32,10 @@ int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) { if (vstr[spos] != '.') { tmp[spos - tpos] = vstr[spos]; } else { - vnum[vpos] = atoi(tmp); + code = taosStr2int32(tmp, &vnum[vpos]); + if (code != 0) { + return code; + } memset(tmp, 0, sizeof(tmp)); vpos++; tpos = spos + 1; @@ -39,7 +43,10 @@ int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) { } if ('\0' != tmp[0] && vpos < 4) { - vnum[vpos] = atoi(tmp); + code = taosStr2int32(tmp, &vnum[vpos]); + if (code != 0) { + return code; + } } if (vnum[0] <= 0) { diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index e801008970..55da8c0929 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -33,6 +33,13 @@ ENDIF() INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc) INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/include/common) +add_executable(heapTest "heapTest.cpp") +target_link_libraries(heapTest os util gtest_main) +add_test( + NAME heapTest + COMMAND heapTest +) + # arrayTest add_executable(arrayTest "arrayTest.cpp") target_link_libraries(arrayTest os util gtest_main) diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index 974677d26c..074fee07ea 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -230,8 +230,8 @@ static int32_t tSStructA_v1_decode(SCoder *pCoder, SStructA_v1 *pSAV1) { if (tDecodeI32(pCoder, &pSAV1->A_a) < 0) return -1; if (tDecodeI64(pCoder, &pSAV1->A_b) < 0) return -1; - const char *tstr; - uint64_t len; + const char *tstr = NULL; + uint64_t len = 0; if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1; pSAV1->A_c = (char *)tCoderMalloc(pCoder, len + 1); memcpy(pSAV1->A_c, tstr, len + 1); @@ -269,8 +269,8 @@ static int32_t tSStructA_v2_decode(SCoder *pCoder, SStructA_v2 *pSAV2) { if (tDecodeI32(pCoder, &pSAV2->A_a) < 0) return -1; if (tDecodeI64(pCoder, &pSAV2->A_b) < 0) return -1; - const char *tstr; - uint64_t len; + const char *tstr = NULL; + uint64_t len = 0; if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1; pSAV2->A_c = (char *)tCoderMalloc(pCoder, len + 1); memcpy(pSAV2->A_c, tstr, len + 1); diff --git a/source/util/test/heapTest.cpp b/source/util/test/heapTest.cpp index 51eeb26ed3..fd61e5032b 100644 --- a/source/util/test/heapTest.cpp +++ b/source/util/test/heapTest.cpp @@ -20,7 +20,7 @@ int32_t heapCompare(const HeapNode* a, const HeapNode* b) { return 1; } -TEST(TD_UTIL_HEAP_TEST, heapTest) { +TEST(heapTest, heapTest) { Heap* heap = heapCreate(heapCompare); ASSERT_TRUE(heap != NULL); ASSERT_EQ(0, heapSize(heap)); diff --git a/tests/army/authorith/authBasic.py b/tests/army/authorith/authBasic.py index 9a453cf687..a682b08ba0 100644 --- a/tests/army/authorith/authBasic.py +++ b/tests/army/authorith/authBasic.py @@ -40,9 +40,9 @@ class TDTestCase(TBase): def test_common_user_privileges(self): self.prepare_data() # create user - self.create_user("test", "test") + self.create_user("test", "test12@#*") # check user 'test' privileges - testconn = taos.connect(user='test', password='test') + testconn = taos.connect(user='test', password='test12@#*') cursor = testconn.cursor() testSql = TDSql() testSql.init(cursor) @@ -87,9 +87,9 @@ class TDTestCase(TBase): def test_common_user_with_createdb_privileges(self): self.prepare_data() # create user - self.create_user("test", "test") + self.create_user("test", "test12@#*") # check user 'test' privileges - testconn = taos.connect(user='test', password='test') + testconn = taos.connect(user='test', password='test12@#*') cursor = testconn.cursor() testSql = TDSql() testSql.init(cursor) @@ -133,8 +133,8 @@ class TDTestCase(TBase): testSql.checkRows(2) # create another user 'test1' - self.create_user("test1", "test1") - test1conn = taos.connect(user='test1', password='test1') + self.create_user("test1", "test12@#*^%") + test1conn = taos.connect(user='test1', password='test12@#*^%') cursor1 = test1conn.cursor() test1Sql = TDSql() test1Sql.init(cursor1) diff --git a/tests/army/cluster/test_drop_table_by_uid.py b/tests/army/cluster/test_drop_table_by_uid.py index f09006b37b..3d10863fc2 100644 --- a/tests/army/cluster/test_drop_table_by_uid.py +++ b/tests/army/cluster/test_drop_table_by_uid.py @@ -305,9 +305,9 @@ class TDTestCase(TBase): """ try: # create new user and grant create database priviledge - tdSql.execute("create user test pass 'test';") + tdSql.execute("create user test pass 'ab45*&TC';") tdSql.execute("alter user test createdb 1;") - conn = taos.connect(user="test", password="test") + conn = taos.connect(user="test", password="ab45*&TC") cursor = conn.cursor() # create database and tables with new user tdLog.info("Prepare data for test case test_abnormal_drop_table_with_non_root_user") diff --git a/tests/army/whole/checkErrorCode.py b/tests/army/whole/checkErrorCode.py index 156cacb482..b24c2ae7b5 100644 --- a/tests/army/whole/checkErrorCode.py +++ b/tests/army/whole/checkErrorCode.py @@ -61,7 +61,7 @@ ignoreCodes = [ '0x80003107', '0x80003108', '0x80003109', '0x80003110', '0x80003111', '0x80003112', '0x80003250', '0x80004003', '0x80004004', '0x80004005', '0x80004006', '0x80004007', '0x80004008', '0x80004009', '0x80004010', '0x80004011', '0x80004012', '0x80004013', '0x80004014', '0x80004015', '0x80004016', '0x80004102', '0x80004103', '0x80004104', '0x80004105', '0x80004106', '0x80004107', '0x80004108', '0x80004109', '0x80005100', - '0x80005101', '0x80006000', '0x80006100', '0x80006101', '0x80006102', '0x80000019', '0x80002639', '0x80002666'] + '0x80005101', '0x80006000', '0x80006100', '0x80006101', '0x80006102', '0x80000019', '0x80002639', '0x80002666', '0x80000237'] class TDTestCase(TBase): diff --git a/tests/ci/container_build_newmachine.sh b/tests/ci/container_build_newmachine.sh index 3c87cf156f..369429b99f 100755 --- a/tests/ci/container_build_newmachine.sh +++ b/tests/ci/container_build_newmachine.sh @@ -80,7 +80,7 @@ else fi -mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan +rm -rf ${REP_REAL_PATH}/debug date ret=$? diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 10da451cba..cf959fb681 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1083,6 +1083,7 @@ ,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 ,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/stt.py -N 3 ,,n,system-test,python3 ./test.py -f eco-system/meta/database/keep_time_offset.py +,,y,system-test,./pytest.sh python3 ./test.py -f eco-system/manager/schema_change.py -N 3 -M 3 #tsim test ,,y,script,./test.sh -f tsim/query/timeline.sim diff --git a/tests/pytest/user/pass_len.py b/tests/pytest/user/pass_len.py index 346b8424fe..2324f5993f 100644 --- a/tests/pytest/user/pass_len.py +++ b/tests/pytest/user/pass_len.py @@ -26,10 +26,10 @@ class TDTestCase: def run(self): print("==============step1") try: - tdSql.execute("create user abc pass '123456'") + tdSql.execute("create user abc pass '123456rf@#'") except Exception as e: tdLog.exit(e) - print("create user abc pass '123456'") + print("create user abc pass '123456rf@#'") print("==============step2") try: diff --git a/tests/run_all_ci_cases.sh b/tests/run_all_ci_cases.sh index 486c47ff4c..959af66d19 100644 --- a/tests/run_all_ci_cases.sh +++ b/tests/run_all_ci_cases.sh @@ -27,7 +27,6 @@ function runCasesOneByOne () { || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_ALLCI_REPORT end_time=`date +%s` echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_ALLCI_REPORT - if $case failed elif [[ "$line" == *"$2"* ]]; then if [[ "$cmd" == *"pytest.sh"* ]]; then diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c index 259d3bec8e..ca8392771e 100644 --- a/tests/script/api/passwdTest.c +++ b/tests/script/api/passwdTest.c @@ -267,7 +267,7 @@ void createUsers(TAOS *taos, const char *host, char *qstr) { } // alter pass for users - sprintf(qstr, "alter user %s pass 'taos'", users[i]); + sprintf(qstr, "alter user %s pass 'taos@123'", users[i]); queryDB(taos, qstr); } } @@ -302,7 +302,7 @@ void passVerTestMulti(const char *host, char *qstr) { queryDB(taos[0], "create table if not exists demo2.stb (ts timestamp, c1 int) tags(t1 int)"); queryDB(taos[0], "create table if not exists demo3.stb (ts timestamp, c1 int) tags(t1 int)"); - strcpy(qstr, "alter user root pass 'taos'"); + strcpy(qstr, "alter user root pass 'taos@123'"); queryDB(taos[0], qstr); // calculate the nPassVerNotified for root and users @@ -345,7 +345,7 @@ void sysInfoTest(TAOS *taosRoot, const char *host, char *qstr) { char userName[USER_LEN] = "user0"; for (int i = 0; i < nRoot; ++i) { - taos[i] = taos_connect(host, "user0", "taos", NULL, 0); + taos[i] = taos_connect(host, "user0", "taos@123", NULL, 0); if (taos[i] == NULL) { fprintf(stderr, "failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/); exit(1); @@ -427,7 +427,7 @@ _loop: printf("\n\n%s:%d LOOP %d, nTestUsers:%d\n", __func__, __LINE__, nLoop, nTestUsers); for (int i = 0; i < nTestUsers; ++i) { // sprintf(users[i], "user%d", i); - taosu[i] = taos_connect(host, users[i], "taos", NULL, 0); + taosu[i] = taos_connect(host, users[i], "taos@123", NULL, 0); if (taosu[i] == NULL) { printf("failed to connect to server, user:%s, reason:%s\n", users[i], "null taos" /*taos_errstr(taos)*/); exit(1); @@ -476,7 +476,7 @@ _loop: nUserDropped = 0; for (int i = 0; i < nTestUsers; ++i) { sprintf(users[i], "user%d", i); - sprintf(qstr, "CREATE USER %s PASS 'taos'", users[i]); + sprintf(qstr, "CREATE USER %s PASS 'taos@123'", users[i]); fprintf(stderr, "%s:%d create user:%s\n", __func__, __LINE__, users[i]); queryDB(taos, qstr); } diff --git a/tests/script/api/stmt2-example.c b/tests/script/api/stmt2-example.c index 803e54c156..692ff90a06 100644 --- a/tests/script/api/stmt2-example.c +++ b/tests/script/api/stmt2-example.c @@ -33,30 +33,47 @@ void do_stmt(TAOS* taos) { char* tbs[2] = {"tb", "tb2"}; int t1_val[2] = {0, 1}; - int t2_len[2] = {3, 3}; - TAOS_STMT2_BIND tags[2][2] = {{{0, &t1_val[0], NULL, NULL, 0}, {0, "a1", &t2_len[0], NULL, 0}}, - {{0, &t1_val[1], NULL, NULL, 0}, {0, "a2", &t2_len[1], NULL, 0}}}; + int t2_len[2] = {10, 10}; + int t3_len[2] = {sizeof(int), sizeof(int)}; + TAOS_STMT2_BIND tags[2][2] = {{{0, &t1_val[0], &t3_len[0], NULL, 0}, {0, "after1", &t2_len[0], NULL, 0}}, + {{0, &t1_val[1], &t3_len[1], NULL, 0}, {0, "after2", &t2_len[1], NULL, 0}}}; TAOS_STMT2_BIND params[2][2] = { - {{TSDB_DATA_TYPE_TIMESTAMP, v.ts, NULL, is_null, 2}, {TSDB_DATA_TYPE_BINARY, v.b, b_len, is_null2, 2}}, - {{TSDB_DATA_TYPE_TIMESTAMP, v.ts, NULL, is_null, 2}, {TSDB_DATA_TYPE_BINARY, v.b, b_len, is_null2, 2}}}; + {{TSDB_DATA_TYPE_TIMESTAMP, v.ts, t64_len, is_null, 2}, {TSDB_DATA_TYPE_BINARY, v.b, b_len, NULL, 2}}, + {{TSDB_DATA_TYPE_TIMESTAMP, v.ts, t64_len, is_null, 2}, {TSDB_DATA_TYPE_BINARY, v.b, b_len, NULL, 2}}}; TAOS_STMT2_BIND* tagv[2] = {&tags[0][0], &tags[1][0]}; TAOS_STMT2_BIND* paramv[2] = {¶ms[0][0], ¶ms[1][0]}; TAOS_STMT2_BINDV bindv = {2, &tbs[0], &tagv[0], ¶mv[0]}; TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); - const char* sql = "insert into db.? using db.stb tags(?, ?) values(?,?)"; - int code = taos_stmt2_prepare(stmt, sql, 0); + + // Equivalent to : + // const char* sql = "insert into db.? using db.stb tags(?, ?) values(?,?)"; + const char* sql = "insert into db.stb(tbname,ts,b,t1,t2) values(?,?,?,?,?)"; + + int code = taos_stmt2_prepare(stmt, sql, 0); if (code != 0) { printf("failed to execute taos_stmt2_prepare. error:%s\n", taos_stmt2_error(stmt)); taos_stmt2_close(stmt); return; } + int fieldNum = 0; + TAOS_FIELD_STB* pFields = NULL; + code = taos_stmt2_get_stb_fields(stmt, &fieldNum, &pFields); + if (code != 0) { + printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt)); + } else { + printf("col nums:%d\n", fieldNum); + for (int i = 0; i < fieldNum; i++) { + printf("field[%d]: %s, data_type:%d, field_type:%d\n", i, pFields[i].name, pFields[i].type, + pFields[i].field_type); + } + } + int64_t ts = 1591060628000; for (int i = 0; i < 2; ++i) { - // v.ts[i] = ts++; - v.ts[i] = ts; - // t64_len[i] = sizeof(int64_t); + v.ts[i] = ts++; + t64_len[i] = sizeof(int64_t); } strcpy(v.b, "abcdefg"); b_len[0] = (int)strlen(v.b); diff --git a/tests/script/tsim/mnode/basic2.sim b/tests/script/tsim/mnode/basic2.sim index 5be29e88a6..71714a11b8 100644 --- a/tests/script/tsim/mnode/basic2.sim +++ b/tests/script/tsim/mnode/basic2.sim @@ -67,7 +67,7 @@ if $data(2)[2] != follower then endi print =============== create user -sql create user user1 PASS 'user1' +sql create user user1 PASS 'user1@#xy' sql select * from information_schema.ins_users if $rows != 2 then return -1 diff --git a/tests/script/tsim/mnode/basic3.sim b/tests/script/tsim/mnode/basic3.sim index ff7c44b67d..e19b4e9443 100644 --- a/tests/script/tsim/mnode/basic3.sim +++ b/tests/script/tsim/mnode/basic3.sim @@ -68,7 +68,7 @@ if $leaderExist != 1 then endi print =============== step3: create user -sql create user user1 PASS 'user1' +sql create user user1 PASS 'user121$*' sql select * from information_schema.ins_users if $rows != 2 then return -1 diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index fbf9d50c25..1bd55cbdfe 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -8,7 +8,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c udf -v 1 system sh/exec.sh -n dnode1 -s start sql connect -sql alter user root pass 'taosdata2' +sql alter user root pass '12s34(*&xx' system sh/exec.sh -n dnode1 -s stop system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/tsim/stream/streamTwaFwcInterval.sim b/tests/script/tsim/stream/streamTwaFwcInterval.sim index 8640650310..5151d7caff 100644 --- a/tests/script/tsim/stream/streamTwaFwcInterval.sim +++ b/tests/script/tsim/stream/streamTwaFwcInterval.sim @@ -289,6 +289,64 @@ if $data51 != $query1_data51 then goto loop3 endi +print ======step3 +sql create database test3 vgroups 1; +sql use test3; + +sql create stable st(ts timestamp, a int, b int , c int)tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams3 trigger force_window_close IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt3 as select _wstart, twa(a), ta from st partition by tbname,ta interval(10s); + +run tsim/stream/checkTaskStatus.sim + +sql insert into t1 values(now + 3000a,1,1,1); +sql flush database test; +sql insert into t1 values(now + 3001a,10,10,10); +sql insert into t1 values(now + 13s,50,50,50); + +sleep 1000 + +print sql select _wstart, twa(a), ta from st partition by tbname,ta interval(10s) order by 1; +sql select _wstart, twa(a), ta from st partition by tbname,ta interval(10s) order by 1; + +$query_data01 = $data01 + +print $data00 $data01 $data02 $data03 $data04 +print $data10 $data11 $data12 $data13 $data14 +print $data20 $data21 $data22 $data23 $data24 +print $data30 $data31 $data32 $data33 $data34 +print $data40 $data41 $data42 $data43 $data44 +print $data50 $data51 $data52 $data53 $data54 + + +$loop_count = 0 +loop4: + +sleep 2000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +print 2 sql select * from streamt3 order by 1; +sql select * from streamt3 order by 1; + +print $data00 $data01 $data02 $data03 $data04 +print $data10 $data11 $data12 $data13 $data14 +print $data20 $data21 $data22 $data23 $data24 +print $data30 $data31 $data32 $data33 $data34 +print $data40 $data41 $data42 $data43 $data44 +print $data50 $data51 $data52 $data53 $data54 + +if $data01 != $query_data01 then + print ======data01======$data01 + print ====query_data01=$query_data01 + goto loop4 +endi + print end system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/sync/mnodeLeaderTransfer.sim b/tests/script/tsim/sync/mnodeLeaderTransfer.sim index ed21ac19c3..a1d364fa7c 100644 --- a/tests/script/tsim/sync/mnodeLeaderTransfer.sim +++ b/tests/script/tsim/sync/mnodeLeaderTransfer.sim @@ -33,7 +33,7 @@ sql create mnode on dnode 3 sleep 3000 print =============== create user -sql create user user1 PASS 'user1' +sql create user user1 PASS 'usersdf1$*' sql select * from information_schema.ins_users if $rows != 2 then return -1 diff --git a/tests/script/tsim/trans/lossdata1.sim b/tests/script/tsim/trans/lossdata1.sim index 82e5923468..5b29d2b193 100644 --- a/tests/script/tsim/trans/lossdata1.sim +++ b/tests/script/tsim/trans/lossdata1.sim @@ -11,8 +11,8 @@ system sh/exec.sh -n dnode1 -s start sql connect print =============== create user1 -sql create user user1 PASS 'user1' -sql create user user2 PASS 'user2' +sql create user user1 PASS 'use@##r1$*' +sql create user user2 PASS 'use&*r2$*' sql select * from information_schema.ins_users if $rows != 3 then return -1 diff --git a/tests/script/tsim/user/password.sim b/tests/script/tsim/user/password.sim index d26b9dbc2e..729097e7e1 100644 --- a/tests/script/tsim/user/password.sim +++ b/tests/script/tsim/user/password.sim @@ -4,8 +4,8 @@ system sh/exec.sh -n dnode1 -s start sql connect print ============= step1 -sql create user u_read pass 'taosdata1' -sql create user u_write pass 'taosdata1' +sql create user u_read pass 'tbx12F132!' +sql create user u_write pass 'tbx12145&*' sql alter user u_read pass 'taosdata' sql alter user u_write pass 'taosdata' @@ -15,6 +15,164 @@ if $rows != 3 then return -1 endi +# invalid password format + +sql_error create user user_p1 pass 'taosdata1' +sql_error create user user_p1 pass 'taosdata2' +sql_error create user user_p1 pass '!@#$%^&3' +sql_error create user user_p1 pass '1234564' +sql_error create user user_p1 pass 'taosdataa' +sql_error create user user_p1 pass 'taosdatab' +sql_error create user user_p1 pass '!@#$%^&c' +sql_error create user user_p1 pass '123456d' +sql_error create user user_p1 pass 'taosdataE' +sql_error create user user_p1 pass 'taosdataF' +sql_error create user user_p1 pass '!@#$%^&G' +sql_error create user user_p1 pass '12333315H' +sql_error create user user_p1 pass 'aaaaaaaat1' +sql_error create user user_p1 pass 'TTTTTTTTT2' +sql_error create user user_p1 pass '!@#$%^&!3' +sql_error create user user_p1 pass '12345654' +sql_error create user user_p1 pass 'taosdatata' +sql_error create user user_p1 pass 'TAOSDATATb' +sql_error create user user_p1 pass '!@#$%^&!c' +sql_error create user user_p1 pass '1234565d' +sql_error create user user_p1 pass 'taosdatatE' +sql_error create user user_p1 pass 'TAOSDATATF' +sql_error create user user_p1 pass '!@#$$*!G' +sql_error create user user_p1 pass '1234565H' +sql_error create user user_p1 pass 'taosdataaosdata!' +sql_error create user user_p1 pass 'taosdataaosdata@' +sql_error create user user_p1 pass '!@#$%^&@*#' +sql_error create user user_p1 pass '!@#$%^&' +sql_error create user user_p1 pass '!@#$%^&@*#@' +sql_error create user user_p1 pass '!@#$%^&@*##' +sql_error create user user_p1 pass '!@#$%^&@*#$' +sql_error create user user_p1 pass '!@#$%^&@*#%' +sql_error create user user_p1 pass '!@#$%^&@*#^' +sql_error create user user_p1 pass '!@#$%^&@*#&' +sql_error create user user_p1 pass '!@#$%^&@*#*' +sql_error create user user_p1 pass '!@#$%^&@*#(' +sql_error create user user_p1 pass '!@#$%^&@*#)' +sql_error create user user_p1 pass '!@#$%^&@*#-' +sql_error create user user_p1 pass '!@#$%^&@*#_' +sql_error create user user_p1 pass '!@#$%^&@*#+' +sql_error create user user_p1 pass '!@#$%^&@*#=' +sql_error create user user_p1 pass '!@#$%^&@*#[' +sql_error create user user_p1 pass '!@#$%^&@*#]' +sql_error create user user_p1 pass '!@#$%^&@*#{' +sql_error create user user_p1 pass '!@#$%^&@*#}' +sql_error create user user_p1 pass '!@#$%^&@*#:' +sql_error create user user_p1 pass '!@#$%^&@*#;' +sql_error create user user_p1 pass '!@#$%^&@*#>' +sql_error create user user_p1 pass '!@#$%^&@*#<' +sql_error create user user_p1 pass '!@#$%^&@*#?' +sql_error create user user_p1 pass '!@#$%^&@*#|' +sql_error create user user_p1 pass '!@#$%^&@*#~' +sql_error create user user_p1 pass '!@#$%^&@*#,' +sql_error create user user_p1 pass '!@#$%^&@*#.' +sql_error create user user_p1 pass 'tbd1234TTT\' +sql_error create user user_p1 pass 'tbd1234TTT/' +sql_error create user user_p1 pass 'tbd1234TTT`' +sql_error create user user_p1 pass 'taosdatax' +sql_error create user user_p1 pass 'taosdatay' + +sql_error create user user_p1 pass 'abcd!@1' +sql create user user_p2 pass 'abcd!@12' +sql create user user_p3 pass 'abcd!@123' +sql create user user_p4 pass 'abcd!@1234' +sql create user user_p5 pass 'abcd!@12345' +sql create user user_p6 pass 'abcd!@123456' +sql create user user_p7 pass 'abcd!@1234567' +sql create user user_p8 pass 'abcd!@123456789' +sql create user user_p9 pass 'abcd!@1234567890' +sql_error create user user_p10 pass 'abcd!@1234567890T' +sql drop user user_p2 +sql drop user user_p3 +sql drop user user_p4 +sql drop user user_p5 +sql drop user user_p6 +sql drop user user_p7 +sql drop user user_p8 +sql drop user user_p9 + +sql create user user_p1 pass 'xt12!@cd' + +sql_error alter user user_p1 pass 'abcd!@1' +sql alter user user_p1 pass 'abcd!@12' +sql alter user user_p1 pass 'abcd!@123' +sql alter user user_p1 pass 'abcd!@1234' +sql alter user user_p1 pass 'abcd!@12345' +sql alter user user_p1 pass 'abcd!@123456' +sql alter user user_p1 pass 'abcd!@1234567' +sql alter user user_p1 pass 'abcd!@123456789' +sql alter user user_p1 pass 'abcd!@1234567890' +sql_error user user_p1 pass 'abcd!@1234567890T' +sql_error alter user user_p1 pass 'taosdata1' +sql_error alter user user_p1 pass 'taosdata2' +sql_error alter user user_p1 pass '!@#$%^&3' +sql_error alter user user_p1 pass '1234564' +sql_error alter user user_p1 pass 'taosdataa' +sql_error alter user user_p1 pass 'taosdatab' +sql_error alter user user_p1 pass '!@#$%^&c' +sql_error alter user user_p1 pass '123456d' +sql_error alter user user_p1 pass 'taosdataE' +sql_error alter user user_p1 pass 'taosdataF' +sql_error alter user user_p1 pass '!@#$%^&G' +sql_error alter user user_p1 pass '12334515H' +sql_error alter user user_p1 pass 'aasfdsft1' +sql_error alter user user_p1 pass 'TAOSDATAT2' +sql_error alter user user_p1 pass '!@#$%^&!3' +sql_error alter user user_p1 pass '12345654' +sql_error alter user user_p1 pass 'taosdatata' +sql_error alter user user_p1 pass 'TAOSDATATb' +sql_error alter user user_p1 pass '!@#$%^&!c' +sql_error alter user user_p1 pass '1234565d' +sql_error alter user user_p1 pass 'taosdatatE' +sql_error alter user user_p1 pass 'TAOSDATATF' +sql_error alter user user_p1 pass '*%^^%###!G' +sql_error alter user user_p1 pass '1234565H' +sql_error alter user user_p1 pass 'taosdataaosdata!' +sql_error alter user user_p1 pass 'taosdataaosdata@' +sql_error alter user user_p1 pass '!@#$%^&@*#' +sql_error alter user user_p1 pass '!@#$%^&' +sql_error alter user user_p1 pass '!@#$%^&@*#@' +sql_error alter user user_p1 pass '!@#$%^&@*##' +sql_error alter user user_p1 pass '!@#$%^&@*#$' +sql_error alter user user_p1 pass '!@#$%^&@*#%' +sql_error alter user user_p1 pass '!@#$%^&@*#^' +sql_error alter user user_p1 pass '!@#$%^&@*#&' +sql_error alter user user_p1 pass '!@#$%^&@*#*' +sql_error alter user user_p1 pass '!@#$%^&@*#(' +sql_error alter user user_p1 pass '!@#$%^&@*#)' +sql_error alter user user_p1 pass '!@#$%^&@*#-' +sql_error alter user user_p1 pass '!@#$%^&@*#_' +sql_error alter user user_p1 pass '!@#$%^&@*#+' +sql_error alter user user_p1 pass '!@#$%^&@*#=' +sql_error alter user user_p1 pass '!@#$%^&@*#[' +sql_error alter user user_p1 pass '!@#$%^&@*#]' +sql_error alter user user_p1 pass '!@#$%^&@*#{' +sql_error alter user user_p1 pass '!@#$%^&@*#}' +sql_error alter user user_p1 pass '!@#$%^&@*#:' +sql_error alter user user_p1 pass '!@#$%^&@*#;' +sql_error alter user user_p1 pass '!@#$%^&@*#>' +sql_error alter user user_p1 pass '!@#$%^&@*#<' +sql_error alter user user_p1 pass '!@#$%^&@*#?' +sql_error alter user user_p1 pass '!@#$%^&@*#|' +sql_error alter user user_p1 pass '!@#$%^&@*#~' +sql_error alter user user_p1 pass '!@#$%^&@*#,' +sql_error alter user user_p1 pass '!@#$%^&@*#.' +sql_error alter user user_p1 pass 'tbd1234TTT\' +sql_error alter user user_p1 pass 'tbd1234TTT/' +sql_error alter user user_p1 pass 'tbd1234TTT`' +sql_error alter user user_p1 pass 'taosdatax' +sql_error alter user user_p1 pass 'taosdatay' + +sql drop user user_p1 + +sql create user user_px pass 'taosdata' +sql drop user user_px + print ============= step2 print user u_read login sql close @@ -54,7 +212,7 @@ sql create user oroot pass 'taosdata' sql_error create user $user PASS 'abcd012345678901234567891234567890abcd012345678901234567891234567890abcd012345678901234567891234567890abcd012345678901234567891234567890123' sql_error create userabcd012345678901234567891234567890abcd01234567890123456789123456789 PASS 'taosdata' sql_error create user abcd0123456789012345678901234567890111 PASS '123' -sql create user abc01234567890123456789 PASS '123' +sql create user abc01234567890123456789 PASS '123xyzYDE' sql show users if $rows != 5 then @@ -84,4 +242,34 @@ sql_error REVOKE all ON *.* from o_root; sql_error GRANT read,write ON *.* to o_root; sql_error REVOKE read,write ON *.* from o_root; + +sql create user u01 pass 'taosdata1!' +sql create user u02 pass 'taosdata1@' +sql create user u03 pass 'taosdata1#' +# sql create user u04 pass 'taosdata1$' +sql create user u05 pass 'taosdata1%' +sql create user u06 pass 'taosdata1^' +sql create user u07 pass 'taosdata1&' +sql create user u08 pass 'taosdata1*' +sql create user u09 pass 'taosdata1(' +sql create user u10 pass 'taosdata1)' +sql create user u11 pass 'taosdata1-' +sql create user u12 pass 'taosdata1_' +sql create user u13 pass 'taosdata1+' +sql create user u14 pass 'taosdata1=' +sql create user u15 pass 'taosdata1[' +sql create user u16 pass 'taosdata1]' +sql create user u17 pass 'taosdata1{' +sql create user u18 pass 'taosdata1}' +sql create user u19 pass 'taosdata1:' +sql create user u20 pass 'taosdata1;' +sql create user u21 pass 'taosdata1>' +sql create user u22 pass 'taosdata1<' +sql create user u23 pass 'taosdata1?' +sql create user u24 pass 'taosdata1|' +sql create user u25 pass 'taosdata1~' +sql create user u26 pass 'taosdata1,' +sql create user u27 pass 'taosdata1.' + +return system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/user/privilege_db.sim b/tests/script/tsim/user/privilege_db.sim index 50eaa12108..fb0f9e4566 100644 --- a/tests/script/tsim/user/privilege_db.sim +++ b/tests/script/tsim/user/privilege_db.sim @@ -17,8 +17,8 @@ if $rows != 5 then endi print =============== create users -sql create user user1 PASS 'user1' -sql create user user2 PASS 'user2' +sql create user user1 PASS '123124(*&xx)' +sql create user user2 PASS '1234(*&xx' sql select * from information_schema.ins_users if $rows != 3 then return -1 diff --git a/tests/script/tsim/user/whitelist.sim b/tests/script/tsim/user/whitelist.sim index 4722c00efa..5f98b92bda 100644 --- a/tests/script/tsim/user/whitelist.sim +++ b/tests/script/tsim/user/whitelist.sim @@ -4,8 +4,8 @@ system sh/exec.sh -n dnode1 -s start sql connect print ============= step1 -sql create user u_read pass 'taosdata1' host '127.0.0.1/24','192.168.1.0/24' -sql create user u_write pass 'taosdata1' host '127.0.0.1','192.168.1.0' +sql create user u_read pass 'taosdata1xad@#' host '127.0.0.1/24','192.168.1.0/24' +sql create user u_write pass 'taosdata1TadBD' host '127.0.0.1','192.168.1.0' sql alter user u_read add host '3.3.3.4/24' sql_error alter user u_write drop host '4.4.4.5/25' @@ -16,8 +16,8 @@ if $rows != 3 then endi print ============= step2 -sql_error create user read1 pass 'taosdata1' host '127.0.0/24' -sql_error create user write1 pass 'taosdata1' host '4.4.4.4/33' +sql_error create user read1 pass 'taosdata1XR' host '127.0.0/24' +sql_error create user write1 pass 'TZtaosdata1' host '4.4.4.4/33' sql show users if $rows != 3 then diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 5967d2f6d3..81de6cdd4c 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -225,7 +225,7 @@ class TDTestCase: tdSql.checkEqual(True, len(tdSql.queryResult) in range(302, 303)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") - tdSql.checkEqual(56, len(tdSql.queryResult)) + tdSql.checkEqual(60, len(tdSql.queryResult)) def ins_dnodes_check(self): tdSql.execute('drop database if exists db2') diff --git a/tests/system-test/0-others/subscribe_stream_privilege.py b/tests/system-test/0-others/subscribe_stream_privilege.py index b477af9f57..a447e43e8b 100644 --- a/tests/system-test/0-others/subscribe_stream_privilege.py +++ b/tests/system-test/0-others/subscribe_stream_privilege.py @@ -114,7 +114,7 @@ class TDTestCase: consumer_dict = { "group.id": "g1", "td.connect.user": self.user_name, - "td.connect.pass": "test", + "td.connect.pass": "123456rf@#", "auto.offset.reset": "earliest" } consumer = Consumer(consumer_dict) @@ -167,7 +167,7 @@ class TDTestCase: def create_user(self): tdSql.execute(f'create topic {self.topic_name} as database {self.dbnames[0]}') - tdSql.execute(f'create user {self.user_name} pass "test"') + tdSql.execute(f'create user {self.user_name} pass "123456rf@#"') def run(self): self.prepare_data() diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py index 91e9f2fb89..b43723161f 100644 --- a/tests/system-test/0-others/taosShell.py +++ b/tests/system-test/0-others/taosShell.py @@ -158,7 +158,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() # time.sleep(2) - tdSql.query("create user testpy pass 'testpy'") + tdSql.query("create user testpy pass 'testpy243#@'") tdSql.query("alter user testpy createdb 1") #hostname = socket.gethostname() @@ -175,7 +175,7 @@ class TDTestCase: checkNetworkStatus = ['0: unavailable', '1: network ok', '2: service ok', '3: service degraded', '4: exiting'] netrole = ['client', 'server'] - keyDict = {'h':'', 'P':'6030', 'p':'testpy', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ + keyDict = {'h':'', 'P':'6030', 'p':'testpy243#@', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ 'k':'', 't':'', 'n':'', 'l':'1024', 'N':'100', 'V':'', 'd':'db', 'w':'30', '-help':'', '-usage':'', '?':''} keyDict['h'] = self.hostname diff --git a/tests/system-test/0-others/taosShellError.py b/tests/system-test/0-others/taosShellError.py index 5e6a590806..e7221b6409 100644 --- a/tests/system-test/0-others/taosShellError.py +++ b/tests/system-test/0-others/taosShellError.py @@ -134,7 +134,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() # time.sleep(2) - tdSql.query("create user testpy pass 'testpy'") + tdSql.query("create user testpy pass 'testpy243#@'") #hostname = socket.gethostname() #tdLog.info ("hostname: %s" % hostname) @@ -150,7 +150,7 @@ class TDTestCase: checkNetworkStatus = ['0: unavailable', '1: network ok', '2: service ok', '3: service degraded', '4: exiting'] netrole = ['client', 'server'] - keyDict = {'h':'', 'P':'6030', 'p':'testpy', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ + keyDict = {'h':'', 'P':'6030', 'p':'testpy243#@', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ 'k':'', 't':'', 'n':'', 'l':'1024', 'N':'100', 'V':'', 'd':'db', 'w':'30', '-help':'', '-usage':'', '?':''} keyDict['h'] = self.hostname diff --git a/tests/system-test/0-others/taosShellNetChk.py b/tests/system-test/0-others/taosShellNetChk.py index d2efa5d9fe..37d82b7e84 100644 --- a/tests/system-test/0-others/taosShellNetChk.py +++ b/tests/system-test/0-others/taosShellNetChk.py @@ -133,7 +133,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() - tdSql.query("create user testpy pass 'testpy'") + tdSql.query("create user testpy pass 'testpy243#@'") buildPath = self.getBuildPath() if (buildPath == ""): @@ -146,7 +146,7 @@ class TDTestCase: checkNetworkStatus = ['0: unavailable', '1: network ok', '2: service ok', '3: service degraded', '4: exiting'] netrole = ['client', 'server'] - keyDict = {'h':'', 'P':'6030', 'p':'testpy', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ + keyDict = {'h':'', 'P':'6030', 'p':'testpy243#@', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ 'k':'', 't':'', 'n':'', 'l':'1024', 'N':'100', 'V':'', 'd':'db', 'w':'30', '-help':'', '-usage':'', '?':''} keyDict['h'] = self.hostname diff --git a/tests/system-test/0-others/taosdShell.py b/tests/system-test/0-others/taosdShell.py index 9b0628ec12..8f30930957 100644 --- a/tests/system-test/0-others/taosdShell.py +++ b/tests/system-test/0-others/taosdShell.py @@ -132,7 +132,7 @@ class TDTestCase: def preData(self): # database\stb\tb\chiild-tb\rows\topics - tdSql.execute("create user testpy pass 'testpy'") + tdSql.execute("create user testpy pass 'testpy243#@'") tdSql.execute("drop database if exists db0;") tdSql.execute("create database db0 wal_retention_period 3600;") tdSql.execute("use db0;") diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index c4d24582e4..33b263c155 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -182,14 +182,14 @@ class TDTestCase: for i in range(self.users_count): user = User() user.name = f"user_test{i}" - user.passwd = f"taosdata{i}" + user.passwd = f"taosdata@1{i}" user.db_set = set() self.users.append(user) return self.users @property def __passwd_list(self): - return [f"taosdata{i}" for i in range(self.users_count) ] + return [f"taosdata@1{i}" for i in range(self.users_count) ] @property def __privilege(self): @@ -210,34 +210,34 @@ class TDTestCase: def create_user_err(self): sqls = [ - "create users u1 pass 'u1passwd' ", - "create user '' pass 'u1passwd' ", - "create user pass 'u1passwd' ", - "create user u1 pass u1passwd ", - "create user u1 password 'u1passwd' ", - "create user u1 pass u1passwd ", + "create users u1 pass 'u1Passwd' ", + "create user '' pass 'u1Passwd' ", + "create user pass 'u1Passwd' ", + "create user u1 pass u1Passwd ", + "create user u1 password 'u1Passwd' ", + "create user u1 pass u1Passwd ", "create user u1 pass '' ", "create user u1 pass ' ' ", "create user u1 pass ", - "create user u1 u2 pass 'u1passwd' 'u2passwd' ", - "create user u1 u2 pass 'u1passwd', 'u2passwd' ", - "create user u1, u2 pass 'u1passwd', 'u2passwd' ", - "create user u1, u2 pass 'u1passwd' 'u2passwd' ", + "create user u1 u2 pass 'u1Passwd' 'u2passwd' ", + "create user u1 u2 pass 'u1Passwd', 'u2passwd' ", + "create user u1, u2 pass 'u1Passwd', 'u2passwd' ", + "create user u1, u2 pass 'u1Passwd' 'u2passwd' ", # length of user_name must <= 23 - "create user u12345678901234567890123 pass 'u1passwd' " , + "create user u12345678901234567890123 pass 'u1Passwd' " , # length of passwd must <= 128 "create user u1 pass 'u12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678' " , # password must have not " ' ~ ` \ - "create user u1 pass 'u1passwd\\' " , - "create user u1 pass 'u1passwd~' " , - "create user u1 pass 'u1passwd\"' " , - "create user u1 pass 'u1passwd\'' " , - "create user u1 pass 'u1passwd`' " , + "create user u1 pass 'u1Passwd\\' " , + "create user u1 pass 'u1Passwd~' " , + "create user u1 pass 'u1Passwd\"' " , + "create user u1 pass 'u1Passwd\'' " , + "create user u1 pass 'u1Passwd`' " , # must after create a user named u1 - "create user u1 pass 'u1passwd' " , + "create user u1 pass 'u1Passwd' " , ] - tdSql.execute("create user u1 pass 'u1passwd' ") + tdSql.execute("create user u1 pass 'u1Passwd' ") for sql in sqls: tdSql.error(sql) @@ -258,12 +258,12 @@ class TDTestCase: def alter_pass_err(self): # sourcery skip: remove-redundant-fstring sqls = [ - f"alter users {self.__user_list[0]} pass 'newpass' " , + f"alter users {self.__user_list[0]} pass 'newpassT1' " , f"alter user {self.__user_list[0]} pass '' " , f"alter user {self.__user_list[0]} pass ' ' " , - f"alter user anyuser pass 'newpass' " , + f"alter user anyuser pass 'newpassT1' " , f"alter user {self.__user_list[0]} pass " , - f"alter user {self.__user_list[0]} password 'newpass' " , + f"alter user {self.__user_list[0]} password 'newpassT1' " , ] for sql in sqls: tdSql.error(sql) @@ -649,7 +649,7 @@ class TDTestCase: # user = conn # 不能创建用户 tdLog.printNoPrefix("==========step4.1: normal user can not create user") - user.error("create use utest1 pass 'utest1pass'") + user.error("create use utest1 pass 'utest1Pass'") # 可以查看用户 tdLog.printNoPrefix("==========step4.2: normal user can show user") user.query("show users") diff --git a/tests/system-test/0-others/user_manage.py b/tests/system-test/0-others/user_manage.py index 6f90a2873a..e48231b059 100644 --- a/tests/system-test/0-others/user_manage.py +++ b/tests/system-test/0-others/user_manage.py @@ -80,16 +80,16 @@ class TDTestCase: for user_name in ['jiacy1_all', 'jiacy1_read', 'jiacy1_write', 'jiacy1_none', 'jiacy0_all', 'jiacy0_read', 'jiacy0_write', 'jiacy0_none']: if 'jiacy1' in user_name.lower(): - tdSql.execute(f'create user {user_name} pass "123" sysinfo 1') + tdSql.execute(f'create user {user_name} pass "123abc!@#" sysinfo 1') elif 'jiacy0' in user_name.lower(): - tdSql.execute(f'create user {user_name} pass "123" sysinfo 0') + tdSql.execute(f'create user {user_name} pass "123abc!@#" sysinfo 0') for user_name in ['jiacy1_all', 'jiacy1_read', 'jiacy0_all', 'jiacy0_read']: tdSql.execute(f'grant read on db to {user_name}') for user_name in ['jiacy1_all', 'jiacy1_write', 'jiacy0_all', 'jiacy0_write']: tdSql.execute(f'grant write on db to {user_name}') def user_privilege_check(self): - jiacy1_read_conn = taos.connect(user='jiacy1_read', password='123') + jiacy1_read_conn = taos.connect(user='jiacy1_read', password='123abc!@#') sql = "create table ntb (ts timestamp,c0 int)" expectErrNotOccured = True try: @@ -107,14 +107,14 @@ class TDTestCase: pass def drop_topic(self): - jiacy1_all_conn = taos.connect(user='jiacy1_all', password='123') - jiacy1_read_conn = taos.connect(user='jiacy1_read', password='123') - jiacy1_write_conn = taos.connect(user='jiacy1_write', password='123') - jiacy1_none_conn = taos.connect(user='jiacy1_none', password='123') - jiacy0_all_conn = taos.connect(user='jiacy0_all', password='123') - jiacy0_read_conn = taos.connect(user='jiacy0_read', password='123') - jiacy0_write_conn = taos.connect(user='jiacy0_write', password='123') - jiacy0_none_conn = taos.connect(user='jiacy0_none', password='123') + jiacy1_all_conn = taos.connect(user='jiacy1_all', password='123abc!@#') + jiacy1_read_conn = taos.connect(user='jiacy1_read', password='123abc!@#') + jiacy1_write_conn = taos.connect(user='jiacy1_write', password='123abc!@#') + jiacy1_none_conn = taos.connect(user='jiacy1_none', password='123abc!@#') + jiacy0_all_conn = taos.connect(user='jiacy0_all', password='123abc!@#') + jiacy0_read_conn = taos.connect(user='jiacy0_read', password='123abc!@#') + jiacy0_write_conn = taos.connect(user='jiacy0_write', password='123abc!@#') + jiacy0_none_conn = taos.connect(user='jiacy0_none', password='123abc!@#') tdSql.execute('create topic root_db as select * from db.stb') for user in [jiacy1_all_conn, jiacy1_read_conn, jiacy0_all_conn, jiacy0_read_conn]: user.execute(f'create topic db_jiacy as select * from db.stb') @@ -149,7 +149,7 @@ class TDTestCase: tdSql.execute('create topic db_topic as select * from db.stb') tdSql.execute('grant subscribe on db_topic to jiacy1_all') print("build consumer") - tmq = Consumer({"group.id": "tg2", "td.connect.user": "jiacy1_all", "td.connect.pass": "123", + tmq = Consumer({"group.id": "tg2", "td.connect.user": "jiacy1_all", "td.connect.pass": "123abc!@#", "enable.auto.commit": "true"}) print("build topic list") tmq.subscribe(["db_topic"]) diff --git a/tests/system-test/0-others/user_privilege.py b/tests/system-test/0-others/user_privilege.py index a731e85ddb..809881589a 100644 --- a/tests/system-test/0-others/user_privilege.py +++ b/tests/system-test/0-others/user_privilege.py @@ -58,7 +58,7 @@ class TDTestCase: self.stbnum_grant = 200 def create_user(self): - tdSql.execute(f'create user {self.user_name} pass "test"') + tdSql.execute(f'create user {self.user_name} pass "test123@#$"') tdSql.execute(f'grant read on {self.dbnames[0]}.{self.stbname} with t2 = "Beijing" to {self.user_name}') tdSql.execute(f'grant write on {self.dbnames[1]}.{self.stbname} with t1 = 2 to {self.user_name}') @@ -75,7 +75,7 @@ class TDTestCase: tdSql.execute(f'create table {self.stbname}_grant_{i} (ts timestamp, c0 int) tags(t0 int)') def user_read_privilege_check(self, dbname): - testconn = taos.connect(user='test', password='test') + testconn = taos.connect(user='test', password='test123@#$') expectErrNotOccured = False try: @@ -94,7 +94,7 @@ class TDTestCase: pass def user_write_privilege_check(self, dbname): - testconn = taos.connect(user='test', password='test') + testconn = taos.connect(user='test', password='test123@#$') expectErrNotOccured = False try: @@ -110,7 +110,7 @@ class TDTestCase: pass def user_privilege_error_check(self): - testconn = taos.connect(user='test', password='test') + testconn = taos.connect(user='test', password='test123@#$') expectErrNotOccured = False sql_list = [f"alter talbe {self.dbnames[0]}.stb_1 set t2 = 'Wuhan'", diff --git a/tests/system-test/0-others/user_privilege_all.py b/tests/system-test/0-others/user_privilege_all.py index 846b76317e..76f952f572 100644 --- a/tests/system-test/0-others/user_privilege_all.py +++ b/tests/system-test/0-others/user_privilege_all.py @@ -21,7 +21,7 @@ class TDTestCase: self.setsql = TDSetSql() # user info self.username = 'test' - self.password = 'test' + self.password = 'test123@#$' # db info self.dbname = "user_privilege_all_db" self.stbname = 'stb' diff --git a/tests/system-test/0-others/user_privilege_multi_users.py b/tests/system-test/0-others/user_privilege_multi_users.py index 53ff136e63..69ad32b756 100644 --- a/tests/system-test/0-others/user_privilege_multi_users.py +++ b/tests/system-test/0-others/user_privilege_multi_users.py @@ -19,7 +19,7 @@ class TDTestCase: # user info self.userNum = 100 self.basic_username = "user" - self.password = "pwd" + self.password = "test123@#$" # db info self.dbname = "user_privilege_multi_users" diff --git a/tests/system-test/0-others/user_privilege_show.py b/tests/system-test/0-others/user_privilege_show.py index 9f49778ba8..41a920967d 100644 --- a/tests/system-test/0-others/user_privilege_show.py +++ b/tests/system-test/0-others/user_privilege_show.py @@ -20,7 +20,7 @@ class TDTestCase: self.setsql = TDSetSql() # user info self.username = 'test' - self.password = 'test' + self.password = 'test123@#$' # db info self.dbname = "user_privilege_show" self.stbname = 'stb' diff --git a/tests/system-test/0-others/view/non_marterial_view/test_view.py b/tests/system-test/0-others/view/non_marterial_view/test_view.py index 3b6f774788..7e062143d2 100644 --- a/tests/system-test/0-others/view/non_marterial_view/test_view.py +++ b/tests/system-test/0-others/view/non_marterial_view/test_view.py @@ -231,7 +231,7 @@ class TDTestCase: """ self.prepare_data() username = "view_test" - password = "test" + password = "test123@#$" self.create_user(username, password) # grant all db permission to user tdSql.execute("grant all on view_db.* to view_test;") @@ -271,7 +271,7 @@ class TDTestCase: """This test case is used to verify the view permission with db write and view all """ username = "view_test" - password = "test" + password = "test123@#$" self.create_user(username, password) conn = taos.connect(user=username, password=password) self.prepare_data(conn) @@ -302,7 +302,7 @@ class TDTestCase: """This test case is used to verify the view permission with db write and view read """ username = "view_test" - password = "test" + password = "test123@#$" self.create_user(username, password) conn = taos.connect(user=username, password=password) self.prepare_data() @@ -338,7 +338,7 @@ class TDTestCase: """This test case is used to verify the view permission with db write and view alter """ username = "view_test" - password = "test" + password = "test123@#$" self.create_user(username, password) conn = taos.connect(user=username, password=password) self.prepare_data() @@ -362,7 +362,7 @@ class TDTestCase: """This test case is used to verify the view permission with db read and view all """ username = "view_test" - password = "test" + password = "test123@#$" self.create_user(username, password) conn = taos.connect(user=username, password=password) self.prepare_data() @@ -388,7 +388,7 @@ class TDTestCase: """This test case is used to verify the view permission with db read and view alter """ username = "view_test" - password = "test" + password = "test123@#$" self.create_user(username, password) conn = taos.connect(user=username, password=password) self.prepare_data() @@ -413,7 +413,7 @@ class TDTestCase: """This test case is used to verify the view permission with db read and view read """ username = "view_test" - password = "test" + password = "test123@#$" self.create_user(username, password) conn = taos.connect(user=username, password=password) self.prepare_data() diff --git a/tests/system-test/0-others/walFileIdex.py b/tests/system-test/0-others/walFileIdex.py index f8309519cd..8ef0c7c181 100644 --- a/tests/system-test/0-others/walFileIdex.py +++ b/tests/system-test/0-others/walFileIdex.py @@ -40,7 +40,7 @@ class TDTestCase: def preData(self): # database\stb\tb\chiild-tb\rows\topics - tdSql.execute("create user testpy pass 'testpy'") + tdSql.execute("create user testpy pass 'test123@#$'") tdSql.execute("drop database if exists db0;") tdSql.execute("create database db0 WAL_RETENTION_PERIOD -1 WAL_RETENTION_SIZE -1 ;") tdSql.execute("use db0;") diff --git a/tests/system-test/1-insert/boundary.py b/tests/system-test/1-insert/boundary.py index 4476236ca6..25782fd0c3 100644 --- a/tests/system-test/1-insert/boundary.py +++ b/tests/system-test/1-insert/boundary.py @@ -33,7 +33,7 @@ class TDTestCase: self.colname_length_boundary = self.boundary.COL_KEY_MAX_LENGTH self.tagname_length_boundary = self.boundary.TAG_KEY_MAX_LENGTH self.username_length_boundary = 23 - self.password_length_boundary = 31 + self.password_length_boundary = 14 def dbname_length_check(self): dbname_length = randint(1,self.dbname_length_boundary-1) for dbname in [tdCom.get_long_name(self.dbname_length_boundary),tdCom.get_long_name(dbname_length)]: @@ -120,27 +120,31 @@ class TDTestCase: def username_length_check(self): username_length = randint(1,self.username_length_boundary-1) for username in [tdCom.get_long_name(username_length),tdCom.get_long_name(self.username_length_boundary)]: - tdSql.execute(f'create user {username} pass "123"') + tdSql.execute(f'create user {username} pass "test123@#$"') tdSql.query('show users') for user in tdSql.queryResult: if user[0].lower() != 'root': tdSql.checkEqual(user[0],username) tdSql.execute(f'drop user {username}') username = tdCom.get_long_name(self.username_length_boundary+1) - tdSql.error(f'create user {username} pass "123"') + tdSql.error(f'create user {username} pass "test123@#$"') if "Name or password too long" in tdSql.error_info: tdLog.info("error info is true!") else: tdLog.exit("error info is not true") def password_length_check(self): - password_length = randint(1,self.password_length_boundary-1) + password_length = randint(8,self.password_length_boundary-1) + index = 0 for password in [tdCom.get_long_name(password_length),tdCom.get_long_name(self.password_length_boundary)]: - username = tdCom.get_long_name(3) - tdSql.execute(f'create user {username} pass "{password}"') + index += 1 + username = tdCom.get_long_name(12) + str(index) + tdSql.execute(f'create user {username} pass "{password}@1"') + index += 1 + username = tdCom.get_long_name(12) + str(index) password = tdCom.get_long_name(self.password_length_boundary+1) - tdSql.error(f'create user {username} pass "{password}"') - if "Name or password too long" in tdSql.error_info: + tdSql.error(f'create user {username} pass "{password}@1"') + if "Invalid password format" in tdSql.error_info: tdLog.info("error info is true!") else: tdLog.exit("error info is not true") diff --git a/tests/system-test/2-query/and_or_for_byte.py b/tests/system-test/2-query/and_or_for_byte.py index ca9c1f2bef..22ec7ef29b 100644 --- a/tests/system-test/2-query/and_or_for_byte.py +++ b/tests/system-test/2-query/and_or_for_byte.py @@ -515,6 +515,18 @@ class TDTestCase: tdSql.query(f"select t1 from {dbname}.stb1 where abs(c1+t1)=1") tdSql.checkRows(1) tdSql.checkData(0,0,0) + tdSql.query(f"select * from {dbname}.stb1") + rows = tdSql.queryRows + tdSql.query(f"select t1 from {dbname}.stb1 where abs(c1+t1)=1 or (1<2)") + tdSql.checkRows(rows) + tdSql.query(f"select t1 from {dbname}.stb1 where abs(c1+t1)=1 and (1<2)") + tdSql.checkRows(1) + tdSql.checkData(0,0,0) + tdSql.query(f"select t1 from {dbname}.stb1 where abs(c1+t1)=1 and (1>2)") + tdSql.checkRows(0) + tdSql.query(f"select t1 from {dbname}.stb1 where abs(c1+t1)=1 or (1>2)") + tdSql.checkRows(1) + tdSql.checkData(0,0,0) tdSql.query( f"select abs(c1+t1)*t1 from {dbname}.stb1 where abs(c1)/floor(abs(ceil(t1))) ==1") diff --git a/tests/system-test/2-query/columnLenUpdated.py b/tests/system-test/2-query/columnLenUpdated.py index 4c92236fca..7470bd907c 100644 --- a/tests/system-test/2-query/columnLenUpdated.py +++ b/tests/system-test/2-query/columnLenUpdated.py @@ -104,7 +104,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() # time.sleep(2) - tdSql.query("create user testpy pass 'testpy'") + tdSql.query("create user testpy pass 'test123@#$'") buildPath = self.getBuildPath() if (buildPath == ""): @@ -117,7 +117,7 @@ class TDTestCase: checkNetworkStatus = ['0: unavailable', '1: network ok', '2: service ok', '3: service degraded', '4: exiting'] netrole = ['client', 'server'] - keyDict = {'h':'', 'P':'6030', 'p':'testpy', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ + keyDict = {'h':'', 'P':'6030', 'p':'test123@#$', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ 'k':'', 't':'', 'n':'', 'l':'1024', 'N':'100', 'V':'', 'd':'db', 'w':'30', '-help':'', '-usage':'', '?':''} keyDict['h'] = self.hostname diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 74f5e86267..f3c5bb171c 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -451,7 +451,74 @@ class TDTestCase: sql = "select avg(c1), concat(t9,t10) from db.stb group by concat(t9,t10), concat(t9,t10),tbname" tdSql.query(sql, queryTimes=1) tdSql.checkRows(5) + + def test_TS_5727(self): + tdSql.execute(f" use {self.dbname} ") + stableName = "test5727" + sql = f"CREATE STABLE {self.dbname}.{stableName} (`ts` TIMESTAMP, `WaterConsumption` FLOAT, \ + `ElecConsumption` INT, `Status` BOOL, `status2` BOOL, `online` BOOL) \ + TAGS (`ActivationTime` TIMESTAMP, `ProductId` INT, \ + `ProductMac` VARCHAR(24), `location` INT)" + tdSql.execute(sql) + + sql = f'CREATE TABLE {self.dbname}.`d00` USING {self.dbname}.{stableName} \ + (`ActivationTime`, `ProductId`, `ProductMac`, `location`) \ + TAGS (1733124710578, 1001, "00:11:22:33:44:55", 100000)' + tdSql.execute(sql) + sql = f'CREATE TABLE {self.dbname}.`d01` USING {self.dbname}.{stableName} \ + (`ActivationTime`, `ProductId`, `ProductMac`, `location`) \ + TAGS (1733124723572, 1002, "00:12:22:33:44:55", 200000)' + tdSql.execute(sql) + sql = f'CREATE TABLE {self.dbname}.`d02` USING {self.dbname}.{stableName} \ + (`ActivationTime`, `ProductId`, `ProductMac`, `location`) \ + TAGS (1733124730908, 1003, "00:11:23:33:44:55", 100000)' + tdSql.execute(sql) + + sql = f'insert into {self.dbname}.d00 values(now - 2s, 5, 5, true, true, false);' + tdSql.execute(sql) + sql = f'insert into {self.dbname}.d01 values(now - 1s, 6, 5, true, true, true);' + tdSql.execute(sql) + sql = f'insert into {self.dbname}.d02 values(now, 6, 7, true, true, true);' + tdSql.execute(sql) + + sql = f'select `location`, tbname from {self.dbname}.{stableName} where ts < now group by tbname order by tbname;' + tdSql.query(sql) + tdSql.checkRows(3) + tdSql.checkData(0, 0, 100000) + tdSql.checkData(1, 0, 200000) + tdSql.checkData(2, 0, 100000) + tdSql.checkData(0, 1, "d00") + tdSql.checkData(1, 1, "d01") + tdSql.checkData(2, 1, "d02") + + sql = f'select tbname,last(online) as online,location from {self.dbname}.{stableName} where ts < now group by tbname order by tbname;' + tdSql.query(sql) + tdSql.checkRows(3) + tdSql.checkData(0, 0, "d00") + tdSql.checkData(1, 0, "d01") + tdSql.checkData(2, 0, "d02") + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(0, 2, 100000) + tdSql.checkData(1, 2, 200000) + tdSql.checkData(2, 2, 100000) + + sql = f'select location,tbname,last_row(online) as online from {self.dbname}.{stableName} where ts < now group by tbname order by tbname;' + tdSql.query(sql) + tdSql.checkRows(3) + tdSql.checkData(0, 0, 100000) + tdSql.checkData(1, 0, 200000) + tdSql.checkData(2, 0, 100000) + tdSql.checkData(0, 1, "d00") + tdSql.checkData(1, 1, "d01") + tdSql.checkData(2, 1, "d02") + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, True) + tdSql.checkData(2, 2, True) + + def run(self): tdSql.prepare() self.prepare_db() @@ -493,6 +560,8 @@ class TDTestCase: # self.test_groupby('group', 5, 5) self.test_error() + + self.test_TS_5727() def stop(self): diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py index c4635dcf50..824488652f 100644 --- a/tests/system-test/2-query/max_partition.py +++ b/tests/system-test/2-query/max_partition.py @@ -64,6 +64,10 @@ class TDTestCase: tdSql.query(f"select tbname , max(c1) from {dbname}.sub_stb_1 where c1 is null group by c1 order by c1 desc ") tdSql.checkRows(1) tdSql.checkData(0,0,"sub_stb_1") + tdSql.query(f"select tbname , max(c1) from {dbname}.sub_stb_1 group by c1 order by c1 desc ") + rows = tdSql.queryRows + tdSql.query(f"select tbname , max(c1) from {dbname}.sub_stb_1 where c1 is null or (1<2) group by c1 order by c1 desc ") + tdSql.checkRows(rows) tdSql.query(f"select max(c1) ,c2 ,t2,tbname from {dbname}.stb group by abs(c1) order by abs(c1)") tdSql.checkRows(self.row_nums+1) diff --git a/tests/system-test/2-query/normal.py b/tests/system-test/2-query/normal.py index 161a9e610d..32357d2a37 100644 --- a/tests/system-test/2-query/normal.py +++ b/tests/system-test/2-query/normal.py @@ -75,6 +75,25 @@ class TDTestCase: tdSql.checkData(1, 1, 3) tdSql.checkData(2, 1, 9) + tdSql.query(f"select * from {dbname}.stb_1 order by ts") + rows = tdSql.queryRows + tdSql.query(f"select * from {dbname}.stb_1 where col1 in (1, 2) or (1<2) order by ts") + tdSql.checkRows(rows) + + tdSql.query(f"select * from (select * from {dbname}.stb_1 where col1 in (1, 9, 3) or (1<2) order by ts)") + tdSql.checkRows(rows) + + tdSql.query(f"select * from {dbname}.stb_1 where col1 in (1, 2) and (1<2) order by ts") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query(f"select * from {dbname}.stb_1 where col1 in (1, 9, 3) and (1<2) order by ts") + tdSql.checkRows(3) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 3) + tdSql.checkData(2, 1, 9) + tdSql.query(f"select * from {dbname}.stb_1 where col1 in (1, 9, 3, 'xy') order by ts") tdSql.checkRows(3) tdSql.checkData(0, 1, 1) @@ -160,7 +179,7 @@ class TDTestCase: tdSql.prepare() self.timeZoneTest() - # self.inAndNotinTest() + self.inAndNotinTest() def stop(self): diff --git a/tests/system-test/2-query/not.py b/tests/system-test/2-query/not.py index 1254226db3..9f4813bebc 100644 --- a/tests/system-test/2-query/not.py +++ b/tests/system-test/2-query/not.py @@ -28,6 +28,10 @@ class TDTestCase: for type_name in stype: tdsql.execute(f"drop table if exists {dbname}.{stbname}") tdsql.execute(f"create table if not exists {dbname}.{stbname} (ts timestamp, v1 {type_name}) tags(t1 {type_name})") + + tdsql.query(f"select t1, * from {dbname}.{stbname} where t1 not in (1, 2) or (1<2) order by t1") + tdsql.checkRows(0) + tdsql.execute(f"insert into {dbname}.sub_1 using {dbname}.{stbname} tags(1) values({self.ts}, 10)") tdsql.execute(f"insert into {dbname}.sub_2 using {dbname}.{stbname} tags(2) values({self.ts + 1000}, 20)") tdsql.execute(f"insert into {dbname}.sub_3 using {dbname}.{stbname} tags(3) values({self.ts + 2000}, 30)") @@ -36,6 +40,11 @@ class TDTestCase: tdsql.query(f"select t1, * from {dbname}.{stbname} where t1 not in (1, 2) order by t1") tdsql.checkRows(1) tdsql.checkData(0, 0, 3) + tdsql.query(f"select * from {dbname}.{stbname} where t1 not in (1, 2) or (1<2) order by t1") + tdsql.checkRows(3) + tdsql.checkData(0, 1, 10) + tdsql.checkData(1, 1, 20) + tdsql.checkData(2, 1, 30) # Test case 2: NOT BETWEEN tdsql.query(f"select * from {dbname}.{stbname} where v1 not between 10 and 20 order by t1") diff --git a/tests/system-test/2-query/select_null.py b/tests/system-test/2-query/select_null.py index bd92e4cf5c..4a6fb3a583 100755 --- a/tests/system-test/2-query/select_null.py +++ b/tests/system-test/2-query/select_null.py @@ -500,7 +500,7 @@ class TDTestCase: tdSql.execute('create stable sel_null.join_stable(`时间戳` timestamp, c1 int) tags(`标签1` int)', queryTimes=1) tdSql.query('select a.值 from sel_null.stable1 a join sel_null.join_stable b on a.ts = 时间戳;', queryTimes=1) tdSql.query('select a.值 from sel_null.stable1 a join sel_null.join_stable b on a.ts = b.时间戳;', queryTimes=1) - tdSql.execute('create user user1 pass "asd"', queryTimes=1) + tdSql.execute('create user user1 pass "asdxtz@#12"', queryTimes=1) tdSql.execute('grant write on sel_null.stable1 with 标签1 = 1 to user1',queryTimes=1) tdSql.execute('select count(*) from sel_null.stable1 state_window(值)', queryTimes=1) diff --git a/tests/system-test/2-query/timezone.py b/tests/system-test/2-query/timezone.py index 33bcb16595..dbb4d2187e 100644 --- a/tests/system-test/2-query/timezone.py +++ b/tests/system-test/2-query/timezone.py @@ -142,26 +142,69 @@ class TDTestCase: tdSql.query(f"select ts from {self.dbname}.d5") tdSql.checkData(0, 0, "2021-07-01 20:00:00.000") - tdSql.execute(f"insert into {self.dbname}.d6 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-115',1)") - tdSql.query(f"select ts from {self.dbname}.d6") - tdSql.checkData(0, 0, "2021-07-01 19:05:00.000") + tdSql.error(f"insert into {self.dbname}.d6 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-115',1)") tdSql.execute(f"insert into {self.dbname}.d7 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-1105',1)") tdSql.query(f"select ts from {self.dbname}.d7") tdSql.checkData(0, 0, "2021-07-01 19:05:00.000") - tdSql.error(f"insert into {self.dbname}.d21 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+12:10',1)") - tdSql.error(f"insert into {self.dbname}.d22 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-24:10',1)") - tdSql.error(f"insert into {self.dbname}.d23 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+12:10',1)") - tdSql.error(f"insert into {self.dbname}.d24 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-24:10',1)") - tdSql.error(f"insert into {self.dbname}.d24 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-24100',1)") - tdSql.error(f"insert into {self.dbname}.d24 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-1210',1)") + # tdSql.error(f"insert into {self.dbname}.d21 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+12:10',1)") + # tdSql.error(f"insert into {self.dbname}.d22 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-24:10',1)") + # tdSql.error(f"insert into {self.dbname}.d23 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+12:10',1)") + # tdSql.error(f"insert into {self.dbname}.d24 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-24:10',1)") + # tdSql.error(f"insert into {self.dbname}.d24 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-24100',1)") + # tdSql.error(f"insert into {self.dbname}.d24 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-1210',1)") tdSql.execute(f'drop database {self.dbname}') + # def timezone_check(self, cursor, timezone): + # cursor.execute("show local variables") + # res = cursor.fetchall() + # for i in range(cursor.rowcount): + # if res[i][0] == "timezone" : + # if res[i][1].find(timezone) == -1: + # tdLog.exit("show timezone:%s != %s"%(res[i][1],timezone)) + # + # def timezone_check_set_options(self): + # buildPath = tdCom.getBuildPath() + # cmdStr = '%s/build/bin/timezone_test'%(buildPath) + # print("cmdStr:", cmdStr) + # tdLog.info(cmdStr) + # ret = os.system(cmdStr) + # if ret != 0: + # tdLog.exit("timezone_test error") + # + # def timezone_check_conf(self, timezone): + # updateCfgDict = ({"clientCfg" : {'timezone': 'UTC'} },) + # tdDnodes.sim.deploy(updateCfgDict) + # conn = taos.connect(config=tdDnodes.getSimCfgPath()) + # cursor = conn.cursor() + # self.timezone_check(cursor, 'UTC') + # cursor.close() + def timezone_check(self, sql, timezone): + tdSql.query(sql) + rows = tdSql.getRows() + for i in range(rows): + if tdSql.getData(i, 0) == "timezone" : + if tdSql.getData(i, 1).find(timezone) == -1: + tdLog.exit("show timezone:%s != %s"%(tdSql.getData(i, 1), timezone)) + + def charset_check(self, sql, charset): + tdSql.query(sql) + rows = tdSql.getRows() + for i in range(rows): + if tdSql.getData(i, 0) == "charset" : + if tdSql.getData(i, 1).find(charset) == -1: + tdLog.exit("show charset:%s != %s"%(tdSql.getData(i, 1), charset)) def run(self): # sourcery skip: extract-duplicate-method timezone = self.get_system_timezone() + # timezone = "Asia/Shanghai" + self.charset_check("show local variables", "UTF-8") + self.timezone_check("show dnode 1 variables", "UTF-8") + self.timezone_check("show local variables", timezone) + self.timezone_check("show dnode 1 variables", timezone) + self.timezone_check_ntb(timezone) self.timezone_check_stb(timezone) self.timezone_format_test() diff --git a/tests/system-test/2-query/timezone_conf.py b/tests/system-test/2-query/timezone_conf.py new file mode 100644 index 0000000000..da45e2c5bd --- /dev/null +++ b/tests/system-test/2-query/timezone_conf.py @@ -0,0 +1,44 @@ + +from util.log import * +from util.sql import * +from util.cases import * +from util.sqlset import * +from util.cluster import * + +class TDTestCase: + updatecfgDict = {"clientCfg" : {'timezone': 'UTC', 'charset': 'gbk'}, 'timezone': 'UTC-8', 'charset': 'gbk'} + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def timezone_check(self, timezone, sql): + tdSql.query(sql) + rows = tdSql.getRows() + for i in range(rows): + if tdSql.getData(i, 0) == "timezone" : + if tdSql.getData(i, 1).find(timezone) == -1: + tdLog.exit("show timezone:%s != %s"%(tdSql.getData(i, 1), timezone)) + + def charset_check(self, charset, sql): + tdSql.query(sql) + rows = tdSql.getRows() + for i in range(rows): + if tdSql.getData(i, 0) == "charset" : + if tdSql.getData(i, 1).find(charset) == -1: + tdLog.exit("show charset:%s != %s"%(tdSql.getData(i, 1), charset)) + + def run(self): + self.charset_check('gbk', "show local variables") + self.charset_check('gbk', "show dnode 1 variables") + self.timezone_check('UTC', "show local variables") + # time.sleep(50) + self.timezone_check('UTC-8', "show dnode 1 variables") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/union.py b/tests/system-test/2-query/union.py index 5104489592..24ccac5d90 100644 --- a/tests/system-test/2-query/union.py +++ b/tests/system-test/2-query/union.py @@ -426,6 +426,15 @@ class TDTestCase: tdLog.printNoPrefix("==========step4:after wal, all check again ") self.all_test() + self.test_TD_33137() + + def test_TD_33137(self): + sql = "select 'asd' union all select 'asdasd'" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(2) + sql = "select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, stable_name `TABLE_NAME`, 'TABLE' `TABLE_TYPE`, table_comment `REMARKS` from information_schema.ins_stables union all select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, table_name `TABLE_NAME`, case when `type`='SYSTEM_TABLE' then 'TABLE' when `type`='NORMAL_TABLE' then 'TABLE' when `type`='CHILD_TABLE' then 'TABLE' else 'UNKNOWN' end `TABLE_TYPE`, table_comment `REMARKS` from information_schema.ins_tables union all select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, view_name `TABLE_NAME`, 'VIEW' `TABLE_TYPE`, NULL `REMARKS` from information_schema.ins_views" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(48) def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py b/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py index 2941a643fd..e3e5d25e5d 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py @@ -111,7 +111,7 @@ class TDTestCase: "batchNum": 5000 } username="user1" - passwd="123" + passwd="test123@#$" dnodeNumbers=int(dnodeNumbers) mnodeNums=int(mnodeNums) diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py index 1d2644c65f..ca1bc44fbb 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py @@ -163,15 +163,15 @@ class TDTestCase: threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) for i in range(5): - clusterComCreate.createUser(newTdSql,f"user{i}",f"pass{i}") - userTdSql=tdCom.newTdSql(user=f"user{i}",password=f"pass{i}") - clusterComCreate.alterUser(userTdSql,f"user{i}",f"pass{i+1}") + clusterComCreate.createUser(newTdSql,f"user{i}",f"passwd@{i}") + userTdSql=tdCom.newTdSql(user=f"user{i}",password=f"passwd@{i}") + clusterComCreate.alterUser(userTdSql,f"user{i}",f"passwd@{i+1}") clusterComCreate.deleteUser(newTdSql,f"user{i}") for j in range(5): i=100 - clusterComCreate.createUser(newTdSql,f"user{i}",f"pass{i}") - userTdSql=tdCom.newTdSql(user=f"user{i}",password=f"pass{i}") - clusterComCreate.alterUser(userTdSql,f"user{i}",f"pass{i+1}") + clusterComCreate.createUser(newTdSql,f"user{i}",f"passwd@{i}") + userTdSql=tdCom.newTdSql(user=f"user{i}",password=f"passwd@{i}") + clusterComCreate.alterUser(userTdSql,f"user{i}",f"passwd@{i+1}") clusterComCreate.deleteUser(newTdSql,f"user{i}") for tr in threads: diff --git a/tests/system-test/99-TDcase/TS-5130.py b/tests/system-test/99-TDcase/TS-5130.py index 504500fa0e..994053ac13 100644 --- a/tests/system-test/99-TDcase/TS-5130.py +++ b/tests/system-test/99-TDcase/TS-5130.py @@ -13,9 +13,9 @@ class TDTestCase: self.conn = conn tdSql.init(conn.cursor(), False) self.passwd = {'root':'taosdata', - 'test':'test'} + 'test':'test123@#$'} def prepare_user(self): - tdSql.execute(f"create user test pass 'test' sysinfo 1") + tdSql.execute(f"create user test pass 'test123@#$' sysinfo 1") def test_connect_user(self, uname): try: diff --git a/tests/system-test/eco-system/manager/schema_change.py b/tests/system-test/eco-system/manager/schema_change.py index 400d2b100b..8eef773d48 100644 --- a/tests/system-test/eco-system/manager/schema_change.py +++ b/tests/system-test/eco-system/manager/schema_change.py @@ -133,8 +133,17 @@ class TDTestCase: if len(tags) < 10: return - sel_cols = random.sample(columns, random.randint(2,int(len(columns)/10))) - sel_tags = random.sample(tags, random.randint(1,int(len(tags)/10))) + # cmax + cmax = int(len(columns)/10) + if cmax <=2 : + cmax = 3 + sel_cols = random.sample(columns, random.randint(2, cmax)) + + # tmax + tmax = int(len(tags)/10) + if tmax <=1 : + tmax = 2 + sel_tags = random.sample(tags, random.randint(1, tmax)) field_cols = ",".join(sel_cols) field_tags = ",".join(sel_tags) @@ -217,18 +226,43 @@ class TDTestCase: #time.sleep(0.3) + # create db and stb + def create(self, db, stb, cols, tags): + # create db + sql = f"create database {db} ;" + tdSql.execute(sql) + + # switch db + tdSql.execute(f"use {db};") + + # cols + sql1 = "" + for k, v in cols.items(): + sql1 += f",{k} {v}" + # tags + sql2 = "" + for k, v in tags.items(): + if sql2 == "": + sql2 = f"{k} {v}" + else: + sql2 += f",{k} {v}" + + # create stb + sql = f"create table {db}.{stb}(ts timestamp {sql1}) tags({sql2})" + tdSql.execute(sql) + # run def run(self): # seed random.seed(int(time.time())) self.dbname = "schema_change" - # switch db - tdSql.execute(f"use {self.dbname};") + # create db + self.create(self.dbname, "meters", self.column_dict, self.tag_dict) + # change meters - self.change_schema(1000000) - + self.change_schema(1000) def stop(self): diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 6fcebd667d..6c1a972f7e 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -337,7 +337,7 @@ char *shellFormatTimestamp(char *buf, int32_t bufSize, int64_t val, int32_t prec } struct tm ptm = {0}; - if (taosLocalTime(&tt, &ptm, buf, bufSize) == NULL) { + if (taosLocalTime(&tt, &ptm, buf, bufSize, NULL) == NULL) { return buf; } size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index 7054eb218f..cb0410e9bf 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -15,6 +15,7 @@ add_executable(tmq_multi_thread_test tmq_multi_thread_test.c) add_executable(tmq_offset_test tmq_offset_test.c) add_executable(varbinary_test varbinary_test.c) add_executable(replay_test replay_test.c) +add_executable(timezone_test timezone_test.c) if(${TD_LINUX}) add_executable(tsz_test tsz_test.c) @@ -142,6 +143,14 @@ target_link_libraries( PUBLIC os ) +target_link_libraries( + timezone_test + PUBLIC taos + PUBLIC util + PUBLIC common + PUBLIC os +) + if(${TD_LINUX}) target_link_libraries( tsz_test diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index bf04352232..d922a6454e 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1436,7 +1436,7 @@ int sml_td22900_Test() { int sml_td24070_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "CREATE user test_db pass 'test'"); + TAOS_RES *pRes = taos_query(taos, "CREATE user test_db pass 'test@123'"); ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); @@ -1460,7 +1460,7 @@ int sml_td24070_Test() { // test db privilege - taos = taos_connect("localhost", "test_db", "test", NULL, 0); + taos = taos_connect("localhost", "test_db", "test@123", NULL, 0); const char* sql[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299372000"}; pRes = taos_query(taos, "use td24070_read"); @@ -1491,11 +1491,11 @@ int sml_td24070_Test() { // test stable privilege taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - pRes = taos_query(taos, "CREATE user test_stb_read pass 'test'"); + pRes = taos_query(taos, "CREATE user test_stb_read pass 'test@123'"); ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); - pRes = taos_query(taos, "CREATE user test_stb_write pass 'test'"); + pRes = taos_query(taos, "CREATE user test_stb_write pass 'test@123'"); ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); @@ -1508,7 +1508,7 @@ int sml_td24070_Test() { taos_free_result(pRes); taos_close(taos); - taos = taos_connect("localhost", "test_stb_read", "test", "td24070_write", 0); + taos = taos_connect("localhost", "test_stb_read", "test@123", "td24070_write", 0); const char* sql1[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299373000"}; pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, @@ -1520,7 +1520,7 @@ int sml_td24070_Test() { taos_free_result(pRes); taos_close(taos); - taos = taos_connect("localhost", "test_stb_write", "test", "td24070_write", 0); + taos = taos_connect("localhost", "test_stb_write", "test@123", "td24070_write", 0); const char* sql2[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299373000"}; pRes = taos_schemaless_insert(taos, (char **)sql2, sizeof(sql2) / sizeof(sql2[0]), TSDB_SML_LINE_PROTOCOL, @@ -1536,11 +1536,11 @@ int sml_td24070_Test() { // test table privilege taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - pRes = taos_query(taos, "CREATE user test_tb_read pass 'test'"); + pRes = taos_query(taos, "CREATE user test_tb_read pass 'test@123'"); ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); - pRes = taos_query(taos, "CREATE user test_tb_write pass 'test'"); + pRes = taos_query(taos, "CREATE user test_tb_write pass 'test@123'"); ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); @@ -1553,7 +1553,7 @@ int sml_td24070_Test() { taos_free_result(pRes); taos_close(taos); - taos = taos_connect("localhost", "test_tb_read", "test", "td24070_write", 0); + taos = taos_connect("localhost", "test_tb_read", "test@123", "td24070_write", 0); const char* sql3[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299374000"}; @@ -1566,7 +1566,7 @@ int sml_td24070_Test() { taos_free_result(pRes); taos_close(taos); - taos = taos_connect("localhost", "test_tb_write", "test", "td24070_write", 0); + taos = taos_connect("localhost", "test_tb_write", "test@123", "td24070_write", 0); const char* sql4[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299374000"}; pRes = taos_schemaless_insert(taos, (char **)sql4, sizeof(sql4) / sizeof(sql4[0]), TSDB_SML_LINE_PROTOCOL, diff --git a/utils/test/c/timezone_test.c b/utils/test/c/timezone_test.c new file mode 100644 index 0000000000..5a3fa87976 --- /dev/null +++ b/utils/test/c/timezone_test.c @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "taos.h" +#include "types.h" +#include "tlog.h" + +char* timezone_name = "America/New_York"; +void check_timezone(const char* tzName){ + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT(pConn != NULL); + TAOS_RES *pRes = taos_query(pConn, "show local variables"); + ASSERT(taos_errno(pRes) == 0); + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(pRes)) != NULL) { + if (strcmp(row[0], "timezone") == 0){ + ASSERT(strstr(row[1], tzName) != NULL); + } + } + taos_free_result(pRes); + taos_close(pConn); +} + +void timezone_set_options_test(){ + taos_options(TSDB_OPTION_TIMEZONE, timezone_name); + check_timezone(timezone_name); +} + +void timezone_insert_test(){ + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "drop database if exists tz_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "create database if not exists tz_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use tz_db"); + int code = taos_errno(pRes); + taos_free_result(pRes); + ASSERT(code == 0); + + pRes = taos_query(taos, "create stable stb (ts timestamp, c1 int) tags (t1 timestamp, t2 binary(32))"); + taos_free_result(pRes); + + pRes = taos_query(taos, "create table ntb (ts timestamp, c1 int)"); + taos_free_result(pRes); + + pRes = taos_query(taos, "insert into tb1 using stb tags (1, 'test') values (1, 2)"); + taos_free_result(pRes); + + pRes = taos_query(taos, "insert into tb1 using stb tags ('2013-04-12T10:52:01', 'test') values ('2013-04-12T10:52:01', 2)"); + taos_free_result(pRes); + + pRes = taos_query(taos, "insert into ntb values ('2013-04-12T10:52:01', 2)"); + taos_free_result(pRes); + + pRes = taos_query(taos, "insert into ntb values (1, 2)"); + taos_free_result(pRes); + + + /* + select today(); + tsConver.sim + "COMPACT DATABASE test START WITH '2023-03-07 14:01:23' END WITH '2023-03-08 14:01:23'" + TK_TIMEZONE + + ts > c1('2013-04-12T10:52:01') + in ('2013-04-12T10:52:01') + + offsetFromTz hash join + */ +} + +int main(int argc, char *argv[]) { + int ret = 0; + timezone_set_options_test(); + return ret; +} diff --git a/utils/test/c/tmqDemo.c b/utils/test/c/tmqDemo.c index 40ed98132c..550613ee9a 100644 --- a/utils/test/c/tmqDemo.c +++ b/utils/test/c/tmqDemo.c @@ -597,7 +597,7 @@ void printParaIntoFile() { time_t tTime = taosGetTimestampSec(); struct tm tm; - taosLocalTime(&tTime, &tm, NULL, 0); + taosLocalTime(&tTime, &tm, NULL, 0, NULL); taosFprintfFile(pFile, "###################################################################\n"); taosFprintfFile(pFile, "# configDir: %s\n", configDir); diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index c045629d1f..1ed9af00f2 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -166,7 +166,7 @@ static void printHelp() { char* getCurrentTimeString(char* timeString) { time_t tTime = taosGetTimestampSec(); struct tm tm; - taosLocalTime(&tTime, &tm, NULL, 0); + taosLocalTime(&tTime, &tm, NULL, 0, NULL); sprintf(timeString, "%d-%02d-%02d %02d:%02d:%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); @@ -472,10 +472,10 @@ static char* shellFormatTimestamp(char* buf, int32_t bufSize, int64_t val, int32 } struct tm ptm; - if (taosLocalTime(&tt, &ptm, buf, bufSize) == NULL) { + if (taosLocalTime(&tt, &ptm, buf, bufSize, NULL) == NULL) { return buf; } - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); + size_t pos = taosStrfTime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", ms); diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c index c7cec64dec..a9772addbb 100644 --- a/utils/tsim/src/simExe.c +++ b/utils/tsim/src/simExe.c @@ -797,10 +797,10 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { tt = (*(int64_t *)row[i]) / 1000000000; } - if (taosLocalTime(&tt, &tp, timeStr, sizeof(timeStr)) == NULL) { + if (taosLocalTime(&tt, &tp, timeStr, sizeof(timeStr), NULL) == NULL) { break; } - strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", &tp); + taosStrfTime(timeStr, 64, "%y-%m-%d %H:%M:%S", &tp); if (precision == TSDB_TIME_PRECISION_MILLI) { sprintf(value, "%s.%03d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000)); } else if (precision == TSDB_TIME_PRECISION_MICRO) {