diff --git a/cmake/cmake.define b/cmake/cmake.define index 10f2172541..5b65738c70 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -121,7 +121,7 @@ ELSE () MESSAGE(STATUS "Compile with Address Sanitizer!") ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-reserved-user-defined-literal -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") ENDIF () # disable all assert diff --git a/cmake/cmake.version b/cmake/cmake.version index de85025a8c..29f560148e 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.3.1") + SET(TD_VER_NUMBER "3.0.3.2") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 1c401ae80e..b2f335e1f7 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG d8059ff + GIT_TAG cb1e89c SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 897ccdd158..3f27cb2517 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 04296a5 + GIT_TAG ddd654a SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/13-operation/03-tolerance.md b/docs/en/13-operation/03-tolerance.md index 4f33748e68..c8d2c3f3f6 100644 --- a/docs/en/13-operation/03-tolerance.md +++ b/docs/en/13-operation/03-tolerance.md @@ -18,14 +18,8 @@ To achieve absolutely no data loss, set wal_level to 2 and wal_fsync_period to 0 ## Disaster Recovery -TDengine uses replication to provide high availability. +TDengine provides disaster recovery by using taosX to replicate data between two TDengine clusters which are deployed in two distant data centers. Assume there are two TDengine clusters, A and B, A is the source and B is the target, and A takes the workload of writing and querying. You can deploy `taosX` in the data center where cluster A resides in, `taosX` consumes the data written into cluster A and writes into cluster B. If the data center of cluster A is disrupted because of disaster, you can switch to cluster B to take the workload of data writing and querying, and deploy a `taosX` in the data center of cluster B to replicate data from cluster B to cluster A if cluster A has been recovered, or another cluster C if cluster A has not been recovered. -A TDengine cluster is managed by mnodes. You can configure up to three mnodes to ensure high availability. The data replication between mnode replicas is performed in a synchronous way to guarantee metadata consistency. +You can use the data replication feature of `taosX` to build more complicated disaster recovery solution. -The number of replicas for time series data in TDengine is associated with each database. There can be many databases in a cluster and each database can be configured with a different number of replicas. When creating a database, the parameter `replica` is used to specify the number of replicas. To achieve high availability, set `replica` to 3. - -The number of dnodes in a TDengine cluster must NOT be lower than the number of replicas for any database, otherwise it would fail when trying to create a table. - -As long as the dnodes of a TDengine cluster are deployed on different physical machines and the replica number is higher than 1, high availability can be achieved without any other assistance. For disaster recovery, dnodes of a TDengine cluster should be deployed in geographically different data centers. - -Alternatively, you can use taosX to synchronize the data from one TDengine cluster to another cluster in a remote location. However, taosX is only available in TDengine enterprise version, for more information please contact tdengine.com. +taosX is only provided in TDengine enterprise edition, for more details please contact business@tdengine.com. diff --git a/docs/en/14-reference/03-connector/07-python.mdx b/docs/en/14-reference/03-connector/07-python.mdx index 69be15f9e8..bfbdd929c2 100644 --- a/docs/en/14-reference/03-connector/07-python.mdx +++ b/docs/en/14-reference/03-connector/07-python.mdx @@ -353,6 +353,86 @@ For a more detailed description of the `sql()` method, please refer to [RestClie +### Usage with req_id + +By using the optional req_id parameter, you can specify a request ID that can be used for tracing. + + + + +##### TaosConnection class + +The `TaosConnection` class contains both an implementation of the PEP249 Connection interface (e.g., the `cursor()` method and the `close()` method) and many extensions (e.g., the `execute()`, `query()`, `schemaless_insert()`, and `subscribe()` methods). + +```python title="execute method" +{{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:insert}} +``` + +```python title="query method" +{{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:query}} +``` + +:::tip +The queried results can only be fetched once. For example, only one of `fetch_all()` and `fetch_all_into_dict()` can be used in the example above. Repeated fetches will result in an empty list. +::: + +##### Use of TaosResult class + +In the above example of using the `TaosConnection` class, we have shown two ways to get the result of a query: `fetch_all()` and `fetch_all_into_dict()`. In addition, `TaosResult` also provides methods to iterate through the result set by rows (`rows_iter`) or by data blocks (`blocks_iter`). Using these two methods will be more efficient in scenarios where the query has a large amount of data. + +```python title="blocks_iter method" +{{#include docs/examples/python/result_set_with_req_id_examples.py}} +``` +##### Use of the TaosCursor class + +The `TaosConnection` class and the `TaosResult` class already implement all the functionality of the native interface. If you are familiar with the interfaces in the PEP249 specification, you can also use the methods provided by the `TaosCursor` class. + +```python title="Use of TaosCursor" +{{#include docs/examples/python/cursor_usage_native_reference_with_req_id.py}} +``` + +:::note +The TaosCursor class uses native connections for write and query operations. In a client-side multi-threaded scenario, this cursor instance must remain thread exclusive and cannot be shared across threads for use, otherwise, it will result in errors in the returned results. + +::: + + + + +##### Use of TaosRestCursor class + +The `TaosRestCursor` class is an implementation of the PEP249 Cursor interface. + +```python title="Use of TaosRestCursor" +{{#include docs/examples/python/connect_rest_with_req_id_examples.py:basic}} +``` +- `cursor.execute`: Used to execute arbitrary SQL statements. +- `cursor.rowcount` : For write operations, returns the number of successful rows written. For query operations, returns the number of rows in the result set. +- `cursor.description` : Returns the description of the field. Please refer to [TaosRestCursor](https://docs.taosdata.com/api/taospy/taosrest/cursor.html) for the specific format of the description information. + +##### Use of the RestClient class + +The `RestClient` class is a direct wrapper for the [REST API](/reference/rest-api). It contains only a `sql()` method for executing arbitrary SQL statements and returning the result. + +```python title="Use of RestClient" +{{#include docs/examples/python/rest_client_with_req_id_example.py}} +``` + +For a more detailed description of the `sql()` method, please refer to [RestClient](https://docs.taosdata.com/api/taospy/taosrest/restclient.html). + + + + +```python +{{#include docs/examples/python/connect_websocket_with_req_id_examples.py:basic}} +``` + +- `conn.execute`: can use to execute arbitrary SQL statements, and return the number of rows affected. +- `conn.query`: can use to execute query SQL statements, and return the query results. + + + + ### Used with pandas diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index a76074f507..430487a3af 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -99,6 +99,9 @@ The parameters described in this document by the effect that they have on the sy ## Monitoring Parameters +:::note +Please note the `taoskeeper` needs to be installed and running to create the `log` database and receiving metrics sent by `taosd` as the full monitoring solution. + ### monitor | Attribute | Description | diff --git a/docs/en/14-reference/14-taosKeeper.md b/docs/en/14-reference/14-taosKeeper.md index 8176b70e3d..895bd82e19 100644 --- a/docs/en/14-reference/14-taosKeeper.md +++ b/docs/en/14-reference/14-taosKeeper.md @@ -13,14 +13,12 @@ taosKeeper is a tool for TDengine that exports monitoring metrics. With taosKeep ## Installation - +There are two ways to install taosKeeper: Methods of installing taosKeeper: - - -- You can compile taosKeeper separately and install it. Please refer to the [taosKeeper](https://github.com/taosdata/taoskeeper) repository for details. --> -You can compile taosKeeper separately and install it. Please refer to the [taosKeeper](https://github.com/taosdata/taoskeeper) repository for details. +- Installing the official TDengine installer will automatically install taosKeeper. Please refer to [TDengine installation](/operation/pkg-install) for details. +- You can compile taosKeeper separately and install it. Please refer to the [taosKeeper](https://github.com/taosdata/taoskeeper) repository for details. ## Configuration and Launch ### Configuration @@ -110,7 +108,7 @@ The following `launchctl` commands can help you manage taoskeeper service: #### Launch With Configuration File -You can quickly launch taosKeeper with the following commands. If you do not specify a configuration file, `/etc/taos/keeper.toml` is used by default. If this file does not specify configurations, the default values are used. +You can quickly launch taosKeeper with the following commands. If you do not specify a configuration file, `/etc/taos/keeper.toml` is used by default. If this file does not specify configurations, the default values are used. ```shell $ taoskeeper -c @@ -188,19 +186,36 @@ $ curl http://127.0.0.1:6043/metrics Sample result set (excerpt): ```shell -# HELP taos_cluster_info_connections_total +# HELP taos_cluster_info_connections_total # TYPE taos_cluster_info_connections_total counter taos_cluster_info_connections_total{cluster_id="5981392874047724755"} 16 -# HELP taos_cluster_info_dbs_total +# HELP taos_cluster_info_dbs_total # TYPE taos_cluster_info_dbs_total counter taos_cluster_info_dbs_total{cluster_id="5981392874047724755"} 2 -# HELP taos_cluster_info_dnodes_alive +# HELP taos_cluster_info_dnodes_alive # TYPE taos_cluster_info_dnodes_alive counter taos_cluster_info_dnodes_alive{cluster_id="5981392874047724755"} 1 -# HELP taos_cluster_info_dnodes_total +# HELP taos_cluster_info_dnodes_total # TYPE taos_cluster_info_dnodes_total counter taos_cluster_info_dnodes_total{cluster_id="5981392874047724755"} 1 -# HELP taos_cluster_info_first_ep +# HELP taos_cluster_info_first_ep # TYPE taos_cluster_info_first_ep gauge taos_cluster_info_first_ep{cluster_id="5981392874047724755",value="hlb:6030"} 1 -``` \ No newline at end of file +``` + +### check_health + +``` +$ curl -i http://127.0.0.1:6043/check_health +``` + +Response: + +``` +HTTP/1.1 200 OK +Content-Type: application/json; charset=utf-8 +Date: Mon, 03 Apr 2023 07:20:38 GMT +Content-Length: 19 + +{"version":"1.0.0"} +``` diff --git a/docs/en/20-third-party/01-grafana.mdx b/docs/en/20-third-party/01-grafana.mdx index 5a2942b144..b0e9dea707 100644 --- a/docs/en/20-third-party/01-grafana.mdx +++ b/docs/en/20-third-party/01-grafana.mdx @@ -77,7 +77,7 @@ sudo -u grafana grafana-cli plugins install tdengine-datasource You can also download zip files from [GitHub](https://github.com/taosdata/grafanaplugin/releases/tag/latest) or [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) and install manually. The commands are as follows: ```bash -GF_VERSION=3.2.7 +GF_VERSION=3.3.1 # from GitHub wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/tdengine-datasource-$GF_VERSION.zip # from Grafana diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index 9910a3d89a..b160058d02 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,14 @@ For TDengine 2.x installation packages by version, please visit [here](https://w import Release from "/components/ReleaseV3"; +## 3.0.3.2 + + + +## 3.0.3.1 + + + ## 3.0.3.1 diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index a060d2dd77..17581b780a 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -10,6 +10,10 @@ For other historical version installers, please visit [here](https://www.taosdat import Release from "/components/ReleaseV3"; +## 2.4.11 + + + ## 2.4.10 diff --git a/docs/examples/c/tmq_example.c b/docs/examples/c/tmq_example.c index a3bade308a..d958428b8f 100644 --- a/docs/examples/c/tmq_example.c +++ b/docs/examples/c/tmq_example.c @@ -70,7 +70,7 @@ static int32_t init_env() { taos_free_result(pRes); // create database - pRes = taos_query(pConn, "create database tmqdb"); + pRes = taos_query(pConn, "create database tmqdb wal_retention_period 3600"); if (taos_errno(pRes) != 0) { printf("error in create tmqdb, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/docs/examples/csharp/influxdbLine/Program.cs b/docs/examples/csharp/influxdbLine/Program.cs index fa3cb21fe0..a620c01609 100644 --- a/docs/examples/csharp/influxdbLine/Program.cs +++ b/docs/examples/csharp/influxdbLine/Program.cs @@ -48,7 +48,7 @@ namespace TDengineExample static void PrepareDatabase(IntPtr conn) { - IntPtr res = TDengine.Query(conn, "CREATE DATABASE test"); + IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600"); if (TDengine.ErrorNo(res) != 0) { throw new Exception("failed to create database, reason: " + TDengine.Error(res)); diff --git a/docs/examples/csharp/optsJSON/Program.cs b/docs/examples/csharp/optsJSON/Program.cs index b67b5af62b..8dcc1dce92 100644 --- a/docs/examples/csharp/optsJSON/Program.cs +++ b/docs/examples/csharp/optsJSON/Program.cs @@ -54,7 +54,7 @@ namespace TDengineExample static void PrepareDatabase(IntPtr conn) { - IntPtr res = TDengine.Query(conn, "CREATE DATABASE test"); + IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600"); if (TDengine.ErrorNo(res) != 0) { throw new Exception("failed to create database, reason: " + TDengine.Error(res)); diff --git a/docs/examples/csharp/optsTelnet/Program.cs b/docs/examples/csharp/optsTelnet/Program.cs index e73ceb041a..ccd29d0cfc 100644 --- a/docs/examples/csharp/optsTelnet/Program.cs +++ b/docs/examples/csharp/optsTelnet/Program.cs @@ -58,7 +58,7 @@ namespace TDengineExample static void PrepareDatabase(IntPtr conn) { - IntPtr res = TDengine.Query(conn, "CREATE DATABASE test"); + IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600"); if (TDengine.ErrorNo(res) != 0) { throw new Exception("failed to create database, reason: " + TDengine.Error(res)); diff --git a/docs/examples/csharp/sqlInsert/Program.cs b/docs/examples/csharp/sqlInsert/Program.cs index f23a6e1663..25a945a459 100644 --- a/docs/examples/csharp/sqlInsert/Program.cs +++ b/docs/examples/csharp/sqlInsert/Program.cs @@ -11,7 +11,7 @@ namespace TDengineExample IntPtr conn = GetConnection(); try { - IntPtr res = TDengine.Query(conn, "CREATE DATABASE power"); + IntPtr res = TDengine.Query(conn, "CREATE DATABASE power WAL_RETENTION_PERIOD 3600"); CheckRes(conn, res, "failed to create database"); res = TDengine.Query(conn, "USE power"); CheckRes(conn, res, "failed to change database"); diff --git a/docs/examples/csharp/stmtInsert/Program.cs b/docs/examples/csharp/stmtInsert/Program.cs index 80cadb2ff8..2e856a49bb 100644 --- a/docs/examples/csharp/stmtInsert/Program.cs +++ b/docs/examples/csharp/stmtInsert/Program.cs @@ -76,7 +76,7 @@ namespace TDengineExample static void PrepareSTable() { - IntPtr res = TDengine.Query(conn, "CREATE DATABASE power"); + IntPtr res = TDengine.Query(conn, "CREATE DATABASE power WAL_RETENTION_PERIOD 3600"); CheckResPtr(res, "failed to create database"); res = TDengine.Query(conn, "USE power"); CheckResPtr(res, "failed to change database"); diff --git a/docs/examples/go/sub/main.go b/docs/examples/go/sub/main.go index 1f7218936f..01bf5e6421 100644 --- a/docs/examples/go/sub/main.go +++ b/docs/examples/go/sub/main.go @@ -15,7 +15,7 @@ func main() { panic(err) } defer db.Close() - _, err = db.Exec("create database if not exists example_tmq") + _, err = db.Exec("create database if not exists example_tmq wal_retention_period 3600") if err != nil { panic(err) } diff --git a/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java b/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java index e9af5e9ce0..8da6f77bae 100644 --- a/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java @@ -35,7 +35,7 @@ public class SubscribeDemo { try (Statement statement = connection.createStatement()) { statement.executeUpdate("drop topic if exists " + TOPIC); statement.executeUpdate("drop database if exists " + DB_NAME); - statement.executeUpdate("create database " + DB_NAME); + statement.executeUpdate("create database " + DB_NAME + " wal_retention_period 3600"); statement.executeUpdate("use " + DB_NAME); statement.executeUpdate( "CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT) TAGS (`groupid` INT, `location` BINARY(24))"); diff --git a/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java b/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java index d953a73641..83cb04f552 100644 --- a/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java @@ -35,7 +35,7 @@ public class WebsocketSubscribeDemo { Statement statement = connection.createStatement()) { statement.executeUpdate("drop topic if exists " + TOPIC); statement.executeUpdate("drop database if exists " + DB_NAME); - statement.executeUpdate("create database " + DB_NAME); + statement.executeUpdate("create database " + DB_NAME + " wal_retention_period 3600"); statement.executeUpdate("use " + DB_NAME); statement.executeUpdate( "CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT) TAGS (`groupid` INT, `location` BINARY(24))"); diff --git a/docs/examples/python/conn_websocket_pandas.py b/docs/examples/python/conn_websocket_pandas.py index eac386732c..5cad5384b2 100644 --- a/docs/examples/python/conn_websocket_pandas.py +++ b/docs/examples/python/conn_websocket_pandas.py @@ -4,7 +4,7 @@ import taos taos_conn = taos.connect() taos_conn.execute('drop database if exists power') -taos_conn.execute('create database if not exists power') +taos_conn.execute('create database if not exists power wal_retention_period 3600') taos_conn.execute("use power") taos_conn.execute( "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") diff --git a/docs/examples/python/connect_rest_with_req_id_examples.py b/docs/examples/python/connect_rest_with_req_id_examples.py new file mode 100644 index 0000000000..3feb574fa6 --- /dev/null +++ b/docs/examples/python/connect_rest_with_req_id_examples.py @@ -0,0 +1,44 @@ +# ANCHOR: connect +from taosrest import connect, TaosRestConnection, TaosRestCursor + +conn = connect(url="http://localhost:6041", + user="root", + password="taosdata", + timeout=30) + +# ANCHOR_END: connect +# ANCHOR: basic +# create STable +cursor = conn.cursor() +cursor.execute("DROP DATABASE IF EXISTS power", req_id=1) +cursor.execute("CREATE DATABASE power", req_id=2) +cursor.execute( + "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)", req_id=3) + +# insert data +cursor.execute("""INSERT INTO power.d1001 USING power.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) + power.d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) + power.d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) + power.d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)""", req_id=4) +print("inserted row count:", cursor.rowcount) + +# query data +cursor.execute("SELECT * FROM power.meters LIMIT 3", req_id=5) +# get total rows +print("queried row count:", cursor.rowcount) +# get column names from cursor +column_names = [meta[0] for meta in cursor.description] +# get rows +data = cursor.fetchall() +print(column_names) +for row in data: + print(row) + +# output: +# inserted row count: 8 +# queried row count: 3 +# ['ts', 'current', 'voltage', 'phase', 'location', 'groupid'] +# [datetime.datetime(2018, 10, 3, 14, 38, 5, 500000, tzinfo=datetime.timezone(datetime.timedelta(seconds=28800), '+08:00')), 11.8, 221, 0.28, 'california.losangeles', 2] +# [datetime.datetime(2018, 10, 3, 14, 38, 16, 600000, tzinfo=datetime.timezone(datetime.timedelta(seconds=28800), '+08:00')), 13.4, 223, 0.29, 'california.losangeles', 2] +# [datetime.datetime(2018, 10, 3, 14, 38, 5, tzinfo=datetime.timezone(datetime.timedelta(seconds=28800), '+08:00')), 10.8, 223, 0.29, 'california.losangeles', 3] +# ANCHOR_END: basic diff --git a/docs/examples/python/connect_websocket_examples.py b/docs/examples/python/connect_websocket_examples.py index c50976efbf..29452bbf9d 100644 --- a/docs/examples/python/connect_websocket_examples.py +++ b/docs/examples/python/connect_websocket_examples.py @@ -6,7 +6,7 @@ conn = taosws.connect("taosws://root:taosdata@localhost:6041") # ANCHOR: basic conn.execute("drop database if exists connwspy") -conn.execute("create database if not exists connwspy") +conn.execute("create database if not exists connwspy wal_retention_period 3600") conn.execute("use connwspy") conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)") conn.execute("create table if not exists tb1 using stb tags (1)") diff --git a/docs/examples/python/connect_websocket_with_req_id_examples.py b/docs/examples/python/connect_websocket_with_req_id_examples.py new file mode 100644 index 0000000000..f5f76c8446 --- /dev/null +++ b/docs/examples/python/connect_websocket_with_req_id_examples.py @@ -0,0 +1,29 @@ +# ANCHOR: connect +import taosws + +conn = taosws.connect("taosws://root:taosdata@localhost:6041") +# ANCHOR_END: connect + +# ANCHOR: basic +conn.execute("drop database if exists connwspy", req_id=1) +conn.execute("create database if not exists connwspy", req_id=2) +conn.execute("use connwspy", req_id=3) +conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)", req_id=4) +conn.execute("create table if not exists tb1 using stb tags (1)", req_id=5) +conn.execute("insert into tb1 values (now, 1)", req_id=6) +conn.execute("insert into tb1 values (now, 2)", req_id=7) +conn.execute("insert into tb1 values (now, 3)", req_id=8) + +r = conn.execute("select * from stb", req_id=9) +result = conn.query("select * from stb", req_id=10) +num_of_fields = result.field_count +print(num_of_fields) + +for row in result: + print(row) + +# output: +# 3 +# ('2023-02-28 15:56:13.329 +08:00', 1, 1) +# ('2023-02-28 15:56:13.333 +08:00', 2, 1) +# ('2023-02-28 15:56:13.337 +08:00', 3, 1) diff --git a/docs/examples/python/connection_usage_native_reference_with_req_id.py b/docs/examples/python/connection_usage_native_reference_with_req_id.py new file mode 100644 index 0000000000..24d0914ad5 --- /dev/null +++ b/docs/examples/python/connection_usage_native_reference_with_req_id.py @@ -0,0 +1,45 @@ +import taos + +# ANCHOR: insert +conn = taos.connect() +# Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement. +conn.execute("DROP DATABASE IF EXISTS test", req_id=1) +conn.execute("CREATE DATABASE test", req_id=2) +# change database. same as execute "USE db" +conn.select_db("test") +conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3) +affected_row = conn.execute("INSERT INTO t1 USING weather TAGS(1) VALUES (now, 23.5) (now+1m, 23.5) (now+2m, 24.4)", req_id=4) +print("affected_row", affected_row) +# output: +# affected_row 3 +# ANCHOR_END: insert + +# ANCHOR: query +# Execute a sql and get its result set. It's useful for SELECT statement +result = conn.query("SELECT * from weather", req_id=5) + +# Get fields from result +fields = result.fields +for field in fields: + print(field) # {name: ts, type: 9, bytes: 8} + +# output: +# {name: ts, type: 9, bytes: 8} +# {name: temperature, type: 6, bytes: 4} +# {name: location, type: 4, bytes: 4} + +# Get data from result as list of tuple +data = result.fetch_all() +print(data) +# output: +# [(datetime.datetime(2022, 4, 27, 9, 4, 25, 367000), 23.5, 1), (datetime.datetime(2022, 4, 27, 9, 5, 25, 367000), 23.5, 1), (datetime.datetime(2022, 4, 27, 9, 6, 25, 367000), 24.399999618530273, 1)] + +# Or get data from result as a list of dict +# map_data = result.fetch_all_into_dict() +# print(map_data) +# output: +# [{'ts': datetime.datetime(2022, 4, 27, 9, 1, 15, 343000), 'temperature': 23.5, 'location': 1}, {'ts': datetime.datetime(2022, 4, 27, 9, 2, 15, 343000), 'temperature': 23.5, 'location': 1}, {'ts': datetime.datetime(2022, 4, 27, 9, 3, 15, 343000), 'temperature': 24.399999618530273, 'location': 1}] +# ANCHOR_END: query + + +conn.close() \ No newline at end of file diff --git a/docs/examples/python/cursor_usage_native_reference_with_req_id.py b/docs/examples/python/cursor_usage_native_reference_with_req_id.py new file mode 100644 index 0000000000..15207ee6bc --- /dev/null +++ b/docs/examples/python/cursor_usage_native_reference_with_req_id.py @@ -0,0 +1,32 @@ +import taos + +conn = taos.connect() +cursor = conn.cursor() + +cursor.execute("DROP DATABASE IF EXISTS test", req_id=1) +cursor.execute("CREATE DATABASE test", req_id=2) +cursor.execute("USE test", req_id=3) +cursor.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=4) + +for i in range(1000): + location = str(i % 10) + tb = "t" + location + cursor.execute(f"INSERT INTO {tb} USING weather TAGS({location}) VALUES (now+{i}a, 23.5) (now+{i + 1}a, 23.5)", req_id=5+i) + +cursor.execute("SELECT count(*) FROM weather", req_id=1005) +data = cursor.fetchall() +print("count:", data[0][0]) +cursor.execute("SELECT tbname, * FROM weather LIMIT 2", req_id=1006) +col_names = [meta[0] for meta in cursor.description] +print(col_names) +rows = cursor.fetchall() +print(rows) + +cursor.close() +conn.close() + +# output: +# count: 2000 +# ['tbname', 'ts', 'temperature', 'location'] +# row_count: -1 +# [('t0', datetime.datetime(2022, 4, 27, 14, 54, 24, 392000), 23.5, 0), ('t0', datetime.datetime(2022, 4, 27, 14, 54, 24, 393000), 23.5, 0)] diff --git a/docs/examples/python/kafka_example_common.py b/docs/examples/python/kafka_example_common.py index 566748c94e..1c735abfc0 100644 --- a/docs/examples/python/kafka_example_common.py +++ b/docs/examples/python/kafka_example_common.py @@ -5,7 +5,7 @@ LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanD 'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale', 'California.SantaClara', 'California.Cupertino'] -CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1' +CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1 wal_retention_period 3600' USE_DATABASE_SQL = 'use {}' DROP_TABLE_SQL = 'drop table if exists meters' DROP_DATABASE_SQL = 'drop database if exists {}' diff --git a/docs/examples/python/rest_client_with_req_id_example.py b/docs/examples/python/rest_client_with_req_id_example.py new file mode 100644 index 0000000000..918398e51e --- /dev/null +++ b/docs/examples/python/rest_client_with_req_id_example.py @@ -0,0 +1,9 @@ +from taosrest import RestClient + +client = RestClient("http://localhost:6041", user="root", password="taosdata") +res: dict = client.sql("SELECT ts, current FROM power.meters LIMIT 1", req_id=1) +print(res) + +# output: +# {'status': 'succ', 'head': ['ts', 'current'], 'column_meta': [['ts', 9, 8], ['current', 6, 4]], 'data': [[datetime.datetime(2018, 10, 3, 14, 38, 5, tzinfo=datetime.timezone(datetime.timedelta(seconds=28800), '+08:00')), 10.3]], 'rows': 1} + diff --git a/docs/examples/python/result_set_with_req_id_examples.py b/docs/examples/python/result_set_with_req_id_examples.py new file mode 100644 index 0000000000..90ae2f4f26 --- /dev/null +++ b/docs/examples/python/result_set_with_req_id_examples.py @@ -0,0 +1,33 @@ +import taos + +conn = taos.connect() +conn.execute("DROP DATABASE IF EXISTS test", req_id=1) +conn.execute("CREATE DATABASE test", req_id=2) +conn.select_db("test") +conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3) +# prepare data +for i in range(2000): + location = str(i % 10) + tb = "t" + location + conn.execute(f"INSERT INTO {tb} USING weather TAGS({location}) VALUES (now+{i}a, 23.5) (now+{i + 1}a, 23.5)", req_id=4+i) + +result: taos.TaosResult = conn.query("SELECT * FROM weather", req_id=2004) + +block_index = 0 +blocks: taos.TaosBlocks = result.blocks_iter() +for rows, length in blocks: + print("block ", block_index, " length", length) + print("first row in this block:", rows[0]) + block_index += 1 + +conn.close() + +# possible output: +# block 0 length 1200 +# first row in this block: (datetime.datetime(2022, 4, 27, 15, 14, 52, 46000), 23.5, 0) +# block 1 length 1200 +# first row in this block: (datetime.datetime(2022, 4, 27, 15, 14, 52, 76000), 23.5, 3) +# block 2 length 1200 +# first row in this block: (datetime.datetime(2022, 4, 27, 15, 14, 52, 99000), 23.5, 6) +# block 3 length 400 +# first row in this block: (datetime.datetime(2022, 4, 27, 15, 14, 52, 122000), 23.5, 9) diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py index 6f7fb87c89..5b462fa153 100644 --- a/docs/examples/python/tmq_example.py +++ b/docs/examples/python/tmq_example.py @@ -6,7 +6,7 @@ def init_tmq_env(db, topic): conn = taos.connect() conn.execute("drop topic if exists {}".format(topic)) conn.execute("drop database if exists {}".format(db)) - conn.execute("create database if not exists {}".format(db)) + conn.execute("create database if not exists {} wal_retention_period 3600".format(db)) conn.select_db(db) conn.execute( "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))") diff --git a/docs/zh/05-get-started/index.md b/docs/zh/05-get-started/index.md index 62eeb2f1c2..16172277b5 100644 --- a/docs/zh/05-get-started/index.md +++ b/docs/zh/05-get-started/index.md @@ -4,7 +4,7 @@ description: '快速设置 TDengine 环境并体验其高效写入和查询' --- import xiaot from './xiaot.webp' -import xiaot_new from './xiaot-new.webp' +import xiaot_new from './xiaot-03.webp' import channel from './channel.webp' import official_account from './official-account.webp' diff --git a/docs/zh/05-get-started/xiaot-03.webp b/docs/zh/05-get-started/xiaot-03.webp new file mode 100644 index 0000000000..c115346e9e Binary files /dev/null and b/docs/zh/05-get-started/xiaot-03.webp differ diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 0a9bd56bfb..d1c1258365 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -17,7 +17,7 @@ import TabItem from '@theme/TabItem'; - JDBC 原生连接：Java 应用在物理节点 1（pnode1）上使用 TSDBDriver 直接调用客户端驱动（libtaos.so 或 taos.dll）的 API 将写入和查询请求发送到位于物理节点 2（pnode2）上的 taosd 实例。 - JDBC REST 连接：Java 应用通过 RestfulDriver 将 SQL 封装成一个 REST 请求，发送给物理节点 2 的 REST 服务器（taosAdapter），通过 REST 服务器请求 taosd 并返回结果。 -使用 REST 连接，不依赖 TDengine 客户端驱动，可以跨平台，更加方便灵活，但性能比原生连接器低约 30%。 +使用 REST 连接，不依赖 TDengine 客户端驱动，可以跨平台，更加方便灵活。 :::info TDengine 的 JDBC 驱动实现尽可能与关系型数据库驱动保持一致，但 TDengine 与关系对象型数据库的使用场景和技术特征存在差异，所以`taos-jdbcdriver` 与传统的 JDBC driver 也存在一定差异。在使用时需要注意以下几点： diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index fdfb141e11..5395610df3 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -353,6 +353,85 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 +### 与 req_id 一起使用 + +使用可选的 req_id 参数，指定请求 id，可以用于 tracing + + + + +##### TaosConnection 类的使用 + +`TaosConnection` 类既包含对 PEP249 Connection 接口的实现(如：`cursor`方法和 `close` 方法)，也包含很多扩展功能（如： `execute`、 `query`、`schemaless_insert` 和 `subscribe` 方法。 + +```python title="execute 方法" +{{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:insert}} +``` + +```python title="query 方法" +{{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:query}} +``` + +:::tip +查询结果只能获取一次。比如上面的示例中 `fetch_all()` 和 `fetch_all_into_dict()` 只能用一个。重复获取得到的结果为空列表。 +::: + +##### TaosResult 类的使用 + +上面 `TaosConnection` 类的使用示例中，我们已经展示了两种获取查询结果的方法： `fetch_all()` 和 `fetch_all_into_dict()`。除此之外 `TaosResult` 还提供了按行迭代(`rows_iter`)或按数据块迭代(`blocks_iter`)结果集的方法。在查询数据量较大的场景，使用这两个方法会更高效。 + +```python title="blocks_iter 方法" +{{#include docs/examples/python/result_set_with_req_id_examples.py}} +``` +##### TaosCursor 类的使用 + +`TaosConnection` 类和 `TaosResult` 类已经实现了原生接口的所有功能。如果你对 PEP249 规范中的接口比较熟悉也可以使用 `TaosCursor` 类提供的方法。 + +```python title="TaosCursor 的使用" +{{#include docs/examples/python/cursor_usage_native_reference_with_req_id.py}} +``` + +:::note +TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线程的场景下，这个游标实例必须保持线程独享，不能跨线程共享使用，否则会导致返回结果出现错误。 + +::: + + + + +##### TaosRestCursor 类的使用 + +`TaosRestCursor` 类是对 PEP249 Cursor 接口的实现。 + +```python title="TaosRestCursor 的使用" +{{#include docs/examples/python/connect_rest_with_req_id_examples.py:basic}} +``` +- `cursor.execute` ： 用来执行任意 SQL 语句。 +- `cursor.rowcount`： 对于写入操作返回写入成功记录数。对于查询操作，返回结果集行数。 +- `cursor.description` ： 返回字段的描述信息。关于描述信息的具体格式请参考[TaosRestCursor](https://docs.taosdata.com/api/taospy/taosrest/cursor.html)。 + +##### RestClient 类的使用 + +`RestClient` 类是对于 [REST API](../rest-api) 的直接封装。它只包含一个 `sql()` 方法用于执行任意 SQL 语句， 并返回执行结果。 + +```python title="RestClient 的使用" +{{#include docs/examples/python/rest_client_with_req_id_example.py}} +``` + +对于 `sql()` 方法更详细的介绍， 请参考 [RestClient](https://docs.taosdata.com/api/taospy/taosrest/restclient.html)。 + + + +```python +{{#include docs/examples/python/connect_websocket_with_req_id_examples.py:basic}} +``` + +- `conn.execute`: 用来执行任意 SQL 语句，返回影响的行数 +- `conn.query`: 用来执行查询 SQL 语句，返回查询结果 + + + + ### 与 pandas 一起使用 diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md index 8fd704ef55..d416febd55 100644 --- a/docs/zh/12-taos-sql/20-keywords.md +++ b/docs/zh/12-taos-sql/20-keywords.md @@ -269,7 +269,7 @@ description: TDengine 保留关键字的详细列表 - SPLIT - STABLE - STABLES -- STAR +- START - STATE - STATE_WINDOW - STATEMENT diff --git a/docs/zh/14-reference/07-tdinsight/index.mdx b/docs/zh/14-reference/07-tdinsight/index.mdx index 8783d47f00..b4eda61ebd 100644 --- a/docs/zh/14-reference/07-tdinsight/index.mdx +++ b/docs/zh/14-reference/07-tdinsight/index.mdx @@ -43,8 +43,6 @@ sudo apt-get update sudo apt-get install grafana ``` -### 在 CentOS / RHEL 上安装 Grafana - @@ -79,7 +77,37 @@ sudo yum install \ - +### 安装 TDengine 数据源插件 + + + + +从 GitHub 安装 TDengine 最新版数据源插件。 + +```bash +get_latest_release() { + curl --silent "https://api.github.com/repos/taosdata/grafanaplugin/releases/latest" | + grep '"tag_name":' | + sed -E 's/.*"v([^"]+)".*/\1/' +} +TDENGINE_PLUGIN_VERSION=$(get_latest_release) +sudo grafana-cli \ + --pluginUrl https://github.com/taosdata/grafanaplugin/releases/download/v$TDENGINE_PLUGIN_VERSION/tdengine-datasource-$TDENGINE_PLUGIN_VERSION.zip \ + plugins install tdengine-datasource +``` + +:::note +3.1.6 和更早版本插件需要在配置文件 `/etc/grafana/grafana.ini` 中添加如下设置，以启用未签名插件。 + +```ini +[plugins] +allow_loading_unsigned_plugins = tdengine-datasource +``` + +::: + + + 我们提供了一个自动化安装脚本 [`TDinsight.sh`](https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh) 脚本以便用户快速进行安装配置。 @@ -175,33 +203,7 @@ sudo ./TDengine.sh -n TDengine-Env1 -a http://another:6041 -u root -p taosdata - 特别地，当您使用 Grafana Cloud 或其他组织时，`-O` 可用于设置组织 ID。 `-G` 可指定 Grafana 插件安装目录。 `-e` 参数将仪表盘设置为可编辑。 - - -### 安装 TDengine 数据源插件 - -从 GitHub 安装 TDengine 最新版数据源插件。 - -```bash -get_latest_release() { - curl --silent "https://api.github.com/repos/taosdata/grafanaplugin/releases/latest" | - grep '"tag_name":' | - sed -E 's/.*"v([^"]+)".*/\1/' -} -TDENGINE_PLUGIN_VERSION=$(get_latest_release) -sudo grafana-cli \ - --pluginUrl https://github.com/taosdata/grafanaplugin/releases/download/v$TDENGINE_PLUGIN_VERSION/tdengine-datasource-$TDENGINE_PLUGIN_VERSION.zip \ - plugins install tdengine-datasource -``` - -:::note -3.1.6 和更早版本插件需要在配置文件 `/etc/grafana/grafana.ini` 中添加如下设置，以启用未签名插件。 - -```ini -[plugins] -allow_loading_unsigned_plugins = tdengine-datasource -``` - -::: + ### 启动 Grafana 服务 @@ -233,8 +235,7 @@ sudo systemctl enable grafana-server ![TDengine Database TDinsight 数据源测试](./assets/howto-add-datasource-test.webp) - - + ### 导入仪表盘 diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 6eeb577ab5..e5efd77f80 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -99,6 +99,9 @@ taos --dump-config ## 监控相关 +:::note +请注意，完整的监控功能需要安装并运行 `taoskeeper` 服务。taoskeeper 负责接收监控指标数据并创建 `log` 库。 + ### monitor | 属性 | 说明 | diff --git a/docs/zh/14-reference/14-taosKeeper.md b/docs/zh/14-reference/14-taosKeeper.md index 66c21dc1a1..12b609584a 100644 --- a/docs/zh/14-reference/14-taosKeeper.md +++ b/docs/zh/14-reference/14-taosKeeper.md @@ -13,12 +13,11 @@ taosKeeper 是 TDengine 3.0 版本监控指标的导出工具，通过简单的 ## 安装 - +taosKeeper 有两种安装方式： taosKeeper 安装方式： - +- 安装 TDengine 官方安装包的同时会自动安装 taosKeeper, 详情请参考[ TDengine 安装](/operation/pkg-install)。 - - 单独编译 taosKeeper 并安装，详情请参考 [taosKeeper](https://github.com/taosdata/taoskeeper) 仓库。 ## 配置和运行方式 @@ -112,7 +111,7 @@ Active: inactive (dead) #### 配置文件启动 -执行以下命令即可快速体验 taosKeeper。当不指定 taosKeeper 配置文件时，优先使用 `/etc/taos/keeper.toml` 配置，否则将使用默认配置。 +执行以下命令即可快速体验 taosKeeper。当不指定 taosKeeper 配置文件时，优先使用 `/etc/taos/keeper.toml` 配置，否则将使用默认配置。 ```shell $ taoskeeper -c @@ -190,19 +189,36 @@ $ curl http://127.0.0.1:6043/metrics 部分结果集： ```shell -# HELP taos_cluster_info_connections_total +# HELP taos_cluster_info_connections_total # TYPE taos_cluster_info_connections_total counter taos_cluster_info_connections_total{cluster_id="5981392874047724755"} 16 -# HELP taos_cluster_info_dbs_total +# HELP taos_cluster_info_dbs_total # TYPE taos_cluster_info_dbs_total counter taos_cluster_info_dbs_total{cluster_id="5981392874047724755"} 2 -# HELP taos_cluster_info_dnodes_alive +# HELP taos_cluster_info_dnodes_alive # TYPE taos_cluster_info_dnodes_alive counter taos_cluster_info_dnodes_alive{cluster_id="5981392874047724755"} 1 -# HELP taos_cluster_info_dnodes_total +# HELP taos_cluster_info_dnodes_total # TYPE taos_cluster_info_dnodes_total counter taos_cluster_info_dnodes_total{cluster_id="5981392874047724755"} 1 -# HELP taos_cluster_info_first_ep +# HELP taos_cluster_info_first_ep # TYPE taos_cluster_info_first_ep gauge taos_cluster_info_first_ep{cluster_id="5981392874047724755",value="hlb:6030"} 1 ``` + +### check_health + +``` +$ curl -i http://127.0.0.1:6043/check_health +``` + +返回结果： + +``` +HTTP/1.1 200 OK +Content-Type: application/json; charset=utf-8 +Date: Mon, 03 Apr 2023 07:20:38 GMT +Content-Length: 19 + +{"version":"1.0.0"} +``` diff --git a/docs/zh/17-operation/03-tolerance.md b/docs/zh/17-operation/03-tolerance.md index 79cf10c39a..bf2a651c1a 100644 --- a/docs/zh/17-operation/03-tolerance.md +++ b/docs/zh/17-operation/03-tolerance.md @@ -19,12 +19,8 @@ TDengine 接收到应用的请求数据包时，先将请求的原始数据包 ## 灾备 -TDengine 的集群通过多个副本的机制，来提供系统的高可用性，同时具备一定的灾备能力。 +TDengine 灾备是通过在异地的两个数据中心中设置两个 TDengine 集群并利用 taosX 的数据复制能力来实现的。假定两个集群为集群 A 和集群 B，其中集群 A 为源集群，承担写入请求并提供查询服务。则在集群 A 所在数据中心中可以配置 taosX 利用 TDengine 提供的数据订阅能力，实时消费集群 A 中新写入的数据，并同步到集群 B。如果发生了灾难导致集群 A 所在数据中心不可用，则可以启用集群 B 作为数据写入和查询的主节点，并在集群 B 所处数据中心中配置 taosX 将数据复制到已经恢复的集群 A 或者新建的集群 C。 -TDengine 集群是由 mnode 负责管理的，为保证 mnode 的高可靠，可以配置 三个 mnode 副本。为保证元数据的强一致性，mnode 副本之间通过同步方式进行数据复制，保证了元数据的强一致性。 +利用 taosX 的数据复制能力也可以构造出更复杂的灾备方案。 -TDengine 集群中的时序数据的副本数是与数据库关联的，一个集群里可以有多个数据库，每个数据库可以配置不同的副本数。创建数据库时，通过参数 replica 指定副本数。为了支持高可靠，需要设置副本数为 3。 - -TDengine 集群的节点数必须大于等于副本数，否则创建表时将报错。 - -当 TDengine 集群中的节点部署在不同的物理机上，并设置多个副本数时，就实现了系统的高可靠性，无需再使用其他软件或工具。TDengine 企业版还可以将副本部署在不同机房，从而实现异地容灾。 +taosX 只在 TDengine 企业版中提供，关于其具体细节，请联系 business@taosdata.com diff --git a/docs/zh/20-third-party/01-grafana.mdx b/docs/zh/20-third-party/01-grafana.mdx index 5927dc4fca..7980736a1c 100644 --- a/docs/zh/20-third-party/01-grafana.mdx +++ b/docs/zh/20-third-party/01-grafana.mdx @@ -77,7 +77,7 @@ sudo -u grafana grafana-cli plugins install tdengine-datasource 或者从 [GitHub](https://github.com/taosdata/grafanaplugin/releases/tag/latest) 或 [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) 下载 .zip 文件到本地并解压到 Grafana 插件目录。命令行下载示例如下： ```bash -GF_VERSION=3.2.9 +GF_VERSION=3.3.1 # from GitHub wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/tdengine-datasource-$GF_VERSION.zip # from Grafana diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 62d5c5f592..b4441ad078 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.0.3.2 + + + ## 3.0.3.1 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index 0e4eda6001..cce6834f12 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -10,6 +10,10 @@ taosTools 各版本安装包下载链接如下： import Release from "/components/ReleaseV3"; +## 2.4.11 + + + ## 2.4.10 diff --git a/examples/lua/OpenResty/rest/tdpool/init.lua b/examples/lua/OpenResty/rest/tdpool/init.lua index ebf8e91756..c0c6d56590 100644 --- a/examples/lua/OpenResty/rest/tdpool/init.lua +++ b/examples/lua/OpenResty/rest/tdpool/init.lua @@ -1,16 +1,15 @@ local _M = {} local driver = require "luaconnector51" -local water_mark = 0 -local occupied = 0 -local connection_pool = {} +td_pool_watermark = 0 +td_pool_occupied = 0 +td_connection_pool = {} -function _M.new(o,config) +function _M.new(o, config) o = o or {} - o.connection_pool = connection_pool - o.water_mark = water_mark - o.occupied = occupied - if #connection_pool == 0 then - + o.connection_pool = td_connection_pool + o.watermark = td_pool_watermark + o.occupied = td_pool_occupied + if #td_connection_pool == 0 then for i = 1, config.connection_pool_size do local res = driver.connect(config) if res.code ~= 0 then @@ -18,8 +17,8 @@ function _M.new(o,config) return nil else local object = {obj = res.conn, state = 0} - table.insert(o.connection_pool,i, object) - ngx.log(ngx.INFO, "add connection, now pool size:"..#(o.connection_pool)) + table.insert(td_connection_pool, i, object) + ngx.log(ngx.INFO, "add connection, now pool size:"..#(td_connection_pool)) end end @@ -32,13 +31,13 @@ function _M:get_connection() local connection_obj - for i = 1, #connection_pool do - connection_obj = connection_pool[i] + for i = 1, #td_connection_pool do + connection_obj = td_connection_pool[i] if connection_obj.state == 0 then connection_obj.state = 1 - occupied = occupied +1 - if occupied > water_mark then - water_mark = occupied + td_pool_occupied = td_pool_occupied + 1 + if td_pool_occupied > td_pool_watermark then + td_pool_watermark = td_pool_occupied end return connection_obj["obj"] end @@ -49,21 +48,27 @@ function _M:get_connection() return nil end -function _M:get_water_mark() +function _M:get_watermark() - return water_mark + return td_pool_watermark +end + + +function _M:get_current_load() + + return td_pool_occupied end function _M:release_connection(conn) local connection_obj - for i = 1, #connection_pool do - connection_obj = connection_pool[i] + for i = 1, #td_connection_pool do + connection_obj = td_connection_pool[i] if connection_obj["obj"] == conn then connection_obj["state"] = 0 - occupied = occupied -1 + td_pool_occupied = td_pool_occupied -1 return end end diff --git a/examples/lua/OpenResty/rest/test.lua b/examples/lua/OpenResty/rest/test.lua index 48aeef3fb4..cb4a1479f3 100644 --- a/examples/lua/OpenResty/rest/test.lua +++ b/examples/lua/OpenResty/rest/test.lua @@ -4,8 +4,21 @@ local Pool = require "tdpool" local config = require "config" ngx.say("start time:"..os.time()) -local pool = Pool.new(Pool,config) -local conn = pool:get_connection() +local pool = Pool.new(Pool, config) +local another_pool = Pool.new(Pool, config) +local conn, conn1, conn2 +conn = pool:get_connection() +conn1 = pool:get_connection() +conn2 = pool:get_connection() +local temp_conn = another_pool:get_connection() +ngx.say("pool size:"..config.connection_pool_size) +ngx.say("pool watermark:"..pool:get_watermark()) +ngx.say("pool current load:"..pool:get_current_load()) +pool:release_connection(conn1) +pool:release_connection(conn2) +another_pool:release_connection(temp_conn) +ngx.say("pool watermark:"..pool:get_watermark()) +ngx.say("pool current load:"..pool:get_current_load()) local res = driver.query(conn,"drop database if exists nginx") if res.code ~=0 then @@ -31,7 +44,6 @@ end res = driver.query(conn,"create table m1 (ts timestamp, speed int,owner binary(20))") if res.code ~=0 then ngx.say("create table---failed: "..res.error) - else ngx.say("create table--- pass.") end @@ -83,8 +95,5 @@ while not flag do -- ngx.say("i am here once...") ngx.sleep(0.001) -- time unit is second end - -ngx.say("pool water_mark:"..pool:get_water_mark()) - pool:release_connection(conn) ngx.say("end time:"..os.time()) diff --git a/include/common/ttime.h b/include/common/ttime.h index 4a7c47d172..f189959f22 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -64,7 +64,7 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { : 1000000000; time_t t = taosTime(NULL); struct tm tm; - taosLocalTime(&t, &tm); + taosLocalTime(&t, &tm, NULL); tm.tm_hour = 0; tm.tm_min = 0; tm.tm_sec = 0; diff --git a/include/os/osDir.h b/include/os/osDir.h index 73871602c5..55c7a15764 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -31,21 +31,49 @@ extern "C" { #endif +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + #ifdef WINDOWS + #define TD_TMP_DIR_PATH "C:\\Windows\\Temp\\" +#ifdef CUS_NAME +#define TD_CFG_DIR_PATH "C:\\"CUS_NAME"\\cfg\\" +#define TD_DATA_DIR_PATH "C:\\"CUS_NAME"\\data\\" +#define TD_LOG_DIR_PATH "C:\\"CUS_NAME"\\log\\" +#else #define TD_CFG_DIR_PATH "C:\\TDengine\\cfg\\" #define TD_DATA_DIR_PATH "C:\\TDengine\\data\\" #define TD_LOG_DIR_PATH "C:\\TDengine\\log\\" +#endif // CUS_NAME + #elif defined(_TD_DARWIN_64) + +#ifdef CUS_PROMPT +#define TD_TMP_DIR_PATH "/tmp/"CUS_PROMPT"d/" +#define TD_CFG_DIR_PATH "/etc/"CUS_PROMPT"/" +#define TD_DATA_DIR_PATH "/var/lib/"CUS_PROMPT"/" +#define TD_LOG_DIR_PATH "/var/log/"CUS_PROMPT"/" +#else #define TD_TMP_DIR_PATH "/tmp/taosd/" #define TD_CFG_DIR_PATH "/etc/taos/" #define TD_DATA_DIR_PATH "/var/lib/taos/" #define TD_LOG_DIR_PATH "/var/log/taos/" +#endif // CUS_PROMPT + #else + #define TD_TMP_DIR_PATH "/tmp/" +#ifdef CUS_PROMPT +#define TD_CFG_DIR_PATH "/etc/"CUS_PROMPT"/" +#define TD_DATA_DIR_PATH "/var/lib/"CUS_PROMPT"/" +#define TD_LOG_DIR_PATH "/var/log/"CUS_PROMPT"/" +#else #define TD_CFG_DIR_PATH "/etc/taos/" #define TD_DATA_DIR_PATH "/var/lib/taos/" #define TD_LOG_DIR_PATH "/var/log/taos/" +#endif // CUS_PROMPT #endif typedef struct TdDir *TdDirPtr; diff --git a/include/os/osTime.h b/include/os/osTime.h index 0a0a54119b..51a285a139 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -91,7 +91,7 @@ static FORCE_INLINE int64_t taosGetMonoTimestampMs() { } char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm); -struct tm *taosLocalTime(const time_t *timep, struct tm *result); +struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf); struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst); time_t taosTime(time_t *t); time_t taosMktime(struct tm *timep); diff --git a/include/util/cus_name.h b/include/util/cus_name.h new file mode 100644 index 0000000000..16f677f855 --- /dev/null +++ b/include/util/cus_name.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _CUS_NAME_H_ +#define _CUS_NAME_H_ + +#ifndef CUS_NAME +#define CUS_NAME "TDengine" +#endif + +#ifndef CUS_PROMPT +#define CUS_PROMPT "taos" +#endif + +#ifndef CUS_EMAIL +#define CUS_EMAIL "" +#endif + +#endif // _CUS_NAME_H_ diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 65d506cca6..80e17ccfd6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -289,6 +289,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_DB_ACCT TAOS_DEF_ERROR_CODE(0, 0x0389) // internal #define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) // #define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x038B) +#define TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO TAOS_DEF_ERROR_CODE(0, 0x038C) // #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) // 2.x // #define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) // 2.x // #define TSDB_CODE_MND_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x0392) // 2.x diff --git a/include/util/tjson.h b/include/util/tjson.h index af0b163986..6922930c13 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -25,7 +25,7 @@ extern "C" { #define tjsonGetNumberValue(pJson, pName, val, code) \ do { \ - int64_t _tmp = 0; \ + uint64_t _tmp = 0; \ code = tjsonGetBigIntValue(pJson, pName, &_tmp); \ val = _tmp; \ } while (0) diff --git a/include/util/tutil.h b/include/util/tutil.h index c8f1b29e10..e0801e5295 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -29,7 +29,7 @@ extern "C" { int32_t strdequote(char *src); size_t strtrim(char *src); char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote); -TdUcs4 *wcsnchr(const TdUcs4 *haystack, TdUcs4 needle, size_t len); +TdUcs4* wcsnchr(const TdUcs4* haystack, TdUcs4 needle, size_t len); char **strsplit(char *src, const char *delim, int32_t *num); char *strtolower(char *dst, const char *src); @@ -37,11 +37,11 @@ char *strntolower(char *dst, const char *src, int32_t n); char *strntolower_s(char *dst, const char *src, int32_t n); int64_t strnatoi(char *num, int32_t len); -size_t tstrncspn(const char *str, size_t ssize, const char *reject, size_t rsize); -size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize); +size_t tstrncspn(const char *str, size_t ssize, const char *reject, size_t rsize); +size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize); -char *strbetween(char *string, char *begin, char *end); -char *paGetToken(char *src, char **token, int32_t *tokenLen); +char *strbetween(char *string, char *begin, char *end); +char *paGetToken(char *src, char **token, int32_t *tokenLen); int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]); int32_t taosHexStrToByteArray(char hexstr[], char bytes[]); @@ -92,26 +92,12 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, } } -#define TSDB_CHECK(condition, CODE, LINO, LABEL, ERRNO) \ - if (!(condition)) { \ - (CODE) = (ERRNO); \ - (LINO) = __LINE__; \ - goto LABEL; \ - } - #define TSDB_CHECK_CODE(CODE, LINO, LABEL) \ - if ((CODE)) { \ - (LINO) = __LINE__; \ + if (CODE) { \ + LINO = __LINE__; \ goto LABEL; \ } -#define TSDB_CHECK_NULL(ptr, CODE, LINO, LABEL, ERRNO) \ - if ((ptr) == NULL) { \ - (CODE) = (ERRNO); \ - (LINO) = __LINE__; \ - goto LABEL; \ - } - #ifdef __cplusplus } #endif diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index a98dc5a236..2159899aa2 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -1,7 +1,6 @@ ######################################################## # # # Configuration # -# Any questions, please email support@taosdata.com # # # ######################################################## @@ -13,7 +12,7 @@ ############### 1. Cluster End point ############################ -# The end point of the first dnode in the cluster to be connected to when this dnode or a CLI `taos` is started +# The end point of the first dnode in the cluster to be connected to when this dnode or the CLI utility is started # firstEp hostname:6030 # The end point of the second dnode to be connected to if the firstEp is not available @@ -25,7 +24,7 @@ # The FQDN of the host on which this dnode will be started. It can be IP address # fqdn hostname -# The port for external access after this dnode is started +# The port for external access after this dnode is started # serverPort 6030 # The maximum number of connections a dnode can accept @@ -96,7 +95,7 @@ # if free disk space is less than this value, this dnode will fail to start # minimalDataDirGB 2.0 -# enable/disable system monitor +# enable/disable system monitor # monitor 1 # The following parameter is used to limit the maximum number of lines in log files. @@ -114,8 +113,8 @@ # The following parameters are used for debug purpose only by this dnode. # debugFlag is a 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR -# Available debug levels are: -# 131: output warning and error +# Available debug levels are: +# 131: output warning and error # 135: output debug, warning and error # 143: output trace, debug, warning and error to log # 199: output debug, warning and error to both screen and file @@ -130,7 +129,7 @@ # debug flag for util # uDebugFlag 131 -# debug flag for rpc +# debug flag for rpc # rpcDebugFlag 131 # debug flag for jni @@ -139,7 +138,7 @@ # debug flag for query # qDebugFlag 131 -# debug flag for taosc driver +# debug flag for client driver # cDebugFlag 131 # debug flag for dnode messages diff --git a/packaging/cfg/taosd.service b/packaging/cfg/taosd.service index fff4b74e62..52c4b1d1e2 100644 --- a/packaging/cfg/taosd.service +++ b/packaging/cfg/taosd.service @@ -1,5 +1,5 @@ [Unit] -Description=TDengine server service +Description=server service After=network-online.target Wants=network-online.target diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index a3f8b53d33..858a6ac668 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -4,7 +4,7 @@ # is required to use systemd to manage services at boot set -e -#set -x +# set -x verMode=edge pagMode=full @@ -34,21 +34,25 @@ benchmarkName="taosBenchmark" dumpName="taosdump" demoName="taosdemo" xname="taosx" -explorerName="${clientName}-explorer" clientName2="taos" -serverName2="taosd" +serverName2="${clientName2}d" +configFile2="${clientName2}.cfg" productName2="TDengine" emailName2="taosdata.com" +xname2="${clientName2}x" +adapterName2="${clientName2}adapter" +explorerName="${clientName2}-explorer" benchmarkName2="${clientName2}Benchmark" +demoName2="${clientName2}demo" dumpName2="${clientName2}dump" uninstallScript2="rm${clientName2}" historyFile="${clientName2}_history" logDir="/var/log/${clientName2}" configDir="/etc/${clientName2}" -installDir="/usr/local/${clientName}" +installDir="/usr/local/${clientName2}" data_dir=${dataDir} log_dir=${logDir} @@ -206,15 +210,15 @@ function install_main_path() { function install_bin() { # Remove links - ${csudo}rm -f ${bin_link_dir}/${clientName} || : - ${csudo}rm -f ${bin_link_dir}/${serverName} || : + ${csudo}rm -f ${bin_link_dir}/${clientName2} || : + ${csudo}rm -f ${bin_link_dir}/${serverName2} || : ${csudo}rm -f ${bin_link_dir}/${udfdName} || : ${csudo}rm -f ${bin_link_dir}/${adapterName} || : - ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : - ${csudo}rm -f ${bin_link_dir}/${demoName} || : - ${csudo}rm -f ${bin_link_dir}/${benchmarkName} || : - ${csudo}rm -f ${bin_link_dir}/${dumpName} || : - ${csudo}rm -f ${bin_link_dir}/${xname} || : + ${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : + ${csudo}rm -f ${bin_link_dir}/${xname2} || : ${csudo}rm -f ${bin_link_dir}/${explorerName} || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : @@ -222,24 +226,23 @@ function install_bin() { ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/* #Make link - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : - [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : + [ -x ${install_main_dir}/bin/${clientName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${clientName2} ${bin_link_dir}/${clientName2} || : + [ -x ${install_main_dir}/bin/${serverName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${serverName2} ${bin_link_dir}/${serverName2} || : [ -x ${install_main_dir}/bin/${udfdName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${udfdName} ${bin_link_dir}/${udfdName} || : - [ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || : - [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || : - [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : - [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : - [ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -sf ${install_main_dir}/bin/${xname} ${bin_link_dir}/${xname} || : + [ -x ${install_main_dir}/bin/${adapterName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${adapterName2} ${bin_link_dir}/${adapterName2} || : + [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${demoName2} || : + [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2} || : + [ -x ${install_main_dir}/bin/${dumpName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName2} ${bin_link_dir}/${dumpName2} || : + [ -x ${install_main_dir}/bin/${xname2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${xname2} ${bin_link_dir}/${xname2} || : [ -x ${install_main_dir}/bin/${explorerName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${explorerName} ${bin_link_dir}/${explorerName} || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : - [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : + if [ "$clientName2" == "${clientName}" ]; then + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : + fi + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName2} || : - [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName2} || : - [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName2} || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript2} || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript2} || : fi } @@ -399,7 +402,7 @@ function set_hostname() { ${csudo}sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network || : fi - ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile2} serverFqdn=$newHostname if [[ -e /etc/hosts ]]; then @@ -433,7 +436,7 @@ function set_ipAsFqdn() { echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" localFqdn="127.0.0.1" # Write the local FQDN to configuration file - ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile2} serverFqdn=$localFqdn echo return @@ -455,7 +458,7 @@ function set_ipAsFqdn() { read -p "Please choose an IP from local IP list:" localFqdn else # Write the local FQDN to configuration file - ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile2} serverFqdn=$localFqdn break fi @@ -519,15 +522,15 @@ function install_adapter_config() { function install_config() { - if [ ! -f "${cfg_install_dir}/${configFile}" ]; then + if [ ! -f "${cfg_install_dir}/${configFile2}" ]; then ${csudo}mkdir -p ${cfg_install_dir} - [ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir} + [ -f ${script_dir}/cfg/${configFile2} ] && ${csudo}cp ${script_dir}/cfg/${configFile2} ${cfg_install_dir} ${csudo}chmod 644 ${cfg_install_dir}/* else - ${csudo}cp -f ${script_dir}/cfg/${configFile} ${cfg_install_dir}/${configFile}.new + ${csudo}cp -f ${script_dir}/cfg/${configFile2} ${cfg_install_dir}/${configFile2}.new fi - ${csudo}ln -sf ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg + ${csudo}ln -sf ${cfg_install_dir}/${configFile2} ${install_main_dir}/cfg [ ! -z $1 ] && return 0 || : # only install client @@ -548,7 +551,7 @@ function install_config() { read firstEp while true; do if [ ! -z "$firstEp" ]; then - ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile2} break else break @@ -600,8 +603,8 @@ function install_web() { function clean_service_on_sysvinit() { - if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then - ${csudo}service ${serverName} stop || : + if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then + ${csudo}service ${serverName2} stop || : fi if ps aux | grep -v grep | grep tarbitrator &>/dev/null; then @@ -609,30 +612,30 @@ function clean_service_on_sysvinit() { fi if ((${initd_mod} == 1)); then - if [ -e ${service_config_dir}/${serverName} ]; then - ${csudo}chkconfig --del ${serverName} || : + if [ -e ${service_config_dir}/${serverName2} ]; then + ${csudo}chkconfig --del ${serverName2} || : fi if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo}chkconfig --del tarbitratord || : fi elif ((${initd_mod} == 2)); then - if [ -e ${service_config_dir}/${serverName} ]; then - ${csudo}insserv -r ${serverName} || : + if [ -e ${service_config_dir}/${serverName2} ]; then + ${csudo}insserv -r ${serverName2} || : fi if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo}insserv -r tarbitratord || : fi elif ((${initd_mod} == 3)); then - if [ -e ${service_config_dir}/${serverName} ]; then - ${csudo}update-rc.d -f ${serverName} remove || : + if [ -e ${service_config_dir}/${serverName2} ]; then + ${csudo}update-rc.d -f ${serverName2} remove || : fi if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo}update-rc.d -f tarbitratord remove || : fi fi - ${csudo}rm -f ${service_config_dir}/${serverName} || : + ${csudo}rm -f ${service_config_dir}/${serverName2} || : ${csudo}rm -f ${service_config_dir}/tarbitratord || : if $(which init &>/dev/null); then @@ -653,24 +656,24 @@ function install_service_on_sysvinit() { fi if ((${initd_mod} == 1)); then - ${csudo}chkconfig --add ${serverName} || : - ${csudo}chkconfig --level 2345 ${serverName} on || : + ${csudo}chkconfig --add ${serverName2} || : + ${csudo}chkconfig --level 2345 ${serverName2} on || : elif ((${initd_mod} == 2)); then - ${csudo}insserv ${serverName} || : - ${csudo}insserv -d ${serverName} || : + ${csudo}insserv ${serverName2} || : + ${csudo}insserv -d ${serverName2} || : elif ((${initd_mod} == 3)); then - ${csudo}update-rc.d ${serverName} defaults || : + ${csudo}update-rc.d ${serverName2} defaults || : fi } function clean_service_on_systemd() { - taosd_service_config="${service_config_dir}/${serverName}.service" - if systemctl is-active --quiet ${serverName}; then + service_config="${service_config_dir}/${serverName2}.service" + if systemctl is-active --quiet ${serverName2}; then echo "${productName} is running, stopping it..." - ${csudo}systemctl stop ${serverName} &>/dev/null || echo &>/dev/null + ${csudo}systemctl stop ${serverName2} &>/dev/null || echo &>/dev/null fi - ${csudo}systemctl disable ${serverName} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${taosd_service_config} + ${csudo}systemctl disable ${serverName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${service_config} tarbitratord_service_config="${service_config_dir}/tarbitratord.service" if systemctl is-active --quiet tarbitratord; then @@ -687,19 +690,19 @@ function clean_service_on_systemd() { function install_service_on_systemd() { clean_service_on_systemd - [ -f ${script_dir}/cfg/${serverName}.service ] && - ${csudo}cp ${script_dir}/cfg/${serverName}.service \ + [ -f ${script_dir}/cfg/${serverName2}.service ] && + ${csudo}cp ${script_dir}/cfg/${serverName2}.service \ ${service_config_dir}/ || : # if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then - # [ -f ${script_dir}/cfg/${serverName}.service ] && - # ${csudo}cp ${script_dir}/cfg/${serverName}.service \ + # [ -f ${script_dir}/cfg/${serverName2}.service ] && + # ${csudo}cp ${script_dir}/cfg/${serverName2}.service \ # ${service_config_dir}/${serverName2}.service || : # fi ${csudo}systemctl daemon-reload - ${csudo}systemctl enable ${serverName} + ${csudo}systemctl enable ${serverName2} ${csudo}systemctl daemon-reload } @@ -719,7 +722,7 @@ function install_service() { elif ((${service_mod} == 1)); then install_service_on_sysvinit else - kill_process ${serverName} + kill_process ${serverName2} fi } @@ -756,10 +759,10 @@ function is_version_compatible() { if [ -f ${script_dir}/driver/vercomp.txt ]; then min_compatible_version=$(cat ${script_dir}/driver/vercomp.txt) else - min_compatible_version=$(${script_dir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 5) + min_compatible_version=$(${script_dir}/bin/${serverName2} -V | head -1 | cut -d ' ' -f 5) fi - exist_version=$(${installDir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 3) + exist_version=$(${installDir}/bin/${serverName2} -V | head -1 | cut -d ' ' -f 3) vercomp $exist_version "3.0.0.0" case $? in 2) @@ -829,13 +832,13 @@ function updateProduct() { echo -e "${GREEN}Start to update ${productName2}...${NC}" # Stop the service if running - if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then + if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then if ((${service_mod} == 0)); then - ${csudo}systemctl stop ${serverName} || : + ${csudo}systemctl stop ${serverName2} || : elif ((${service_mod} == 1)); then - ${csudo}service ${serverName} stop || : + ${csudo}service ${serverName2} stop || : else - kill_process ${serverName} + kill_process ${serverName2} fi sleep 1 fi @@ -862,21 +865,21 @@ function updateProduct() { openresty_work=false echo - echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile}" - [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/taosadapter.toml" + echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}" + [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml" if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName}${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName2}${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}" elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName} start${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service taosadapter start${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName2} start${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ./${serverName}${NC}" - [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: taosadapter &${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ./${serverName2}${NC}" + [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${clientName2}adapter &${NC}" fi if [ ${openresty_work} = 'true' ]; then @@ -887,7 +890,7 @@ function updateProduct() { if ((${prompt_force} == 1)); then echo "" - echo -e "${RED}Please run '${serverName} --force-keep-file' at first time for the exist ${productName2} $exist_version!${NC}" + echo -e "${RED}Please run '${serverName2} --force-keep-file' at first time for the exist ${productName2} $exist_version!${NC}" fi echo echo -e "\033[44;32;1m${productName2} is updated successfully!${NC}" @@ -899,7 +902,7 @@ function updateProduct() { echo -e "\033[44;32;1m${productName2} client is updated successfully!${NC}" fi - rm -rf $(tar -tf ${tarName} | grep -v "^\./$") + rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") } function installProduct() { @@ -944,21 +947,21 @@ function installProduct() { # Ask if to start the service echo - echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile}" - [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/taosadapter.toml" + echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}" + [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml" if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName}${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName2}${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}" elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName} start${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service taosadapter start${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName2} start${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${serverName}${NC}" - [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: taosadapter &${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${serverName2}${NC}" + [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${clientName2}adapter &${NC}" fi if [ ! -z "$firstEp" ]; then @@ -991,7 +994,7 @@ function installProduct() { fi touch ~/.${historyFile} - rm -rf $(tar -tf ${tarName} | grep -v "^\./$") + rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") } ## ==============================Main program starts from here============================ @@ -1002,7 +1005,7 @@ if [ "$verType" == "server" ]; then echo -e "\033[44;31;5mThe default data directory ${data_dir} contains old data of ${productName2} 2.x, please clear it before installing!\033[0m" else # Install server and client - if [ -x ${bin_dir}/${serverName} ]; then + if [ -x ${bin_dir}/${serverName2} ]; then update_flag=1 updateProduct else @@ -1012,7 +1015,7 @@ if [ "$verType" == "server" ]; then elif [ "$verType" == "client" ]; then interactiveFqdn=no # Only install client - if [ -x ${bin_dir}/${clientName} ]; then + if [ -x ${bin_dir}/${clientName2} ]; then update_flag=1 updateProduct client else diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index d941fbc0cb..53b9c80f10 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -95,7 +95,7 @@ function install_main_path() { ${csudo}mkdir -p ${install_main_dir}/cfg ${csudo}mkdir -p ${install_main_dir}/bin ${csudo}mkdir -p ${install_main_dir}/driver - if [ $productName == "TDengine" ]; then + if [ "$productName2" == "TDengine" ]; then ${csudo}mkdir -p ${install_main_dir}/examples fi ${csudo}mkdir -p ${install_main_dir}/include @@ -118,18 +118,19 @@ function install_bin() { #Make link [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : if [ "$osType" != "Darwin" ]; then - [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${install_main_dir}/bin/${demoName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${demoName2} ${bin_link_dir}/${demoName2} || : fi [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then #Make link - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName2} || : + [ -x ${install_main_dir}/bin/${clientName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName2} ${bin_link_dir}/${clientName2} || : if [ "$osType" != "Darwin" ]; then - [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/${demoName2} || : + [ -x ${install_main_dir}/bin/${demoName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${demoName2} ${bin_link_dir}/${demoName2} || : + [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2} || : fi - [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript2} || : + [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript2} || : fi } @@ -305,7 +306,7 @@ function update_TDengine() { echo echo -e "\033[44;32;1m${productName2} client is updated successfully!${NC}" - rm -rf $(tar -tf ${tarName}) + rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") } function install_TDengine() { @@ -332,7 +333,7 @@ function install_TDengine() { echo echo -e "\033[44;32;1m${productName2} client is installed successfully!${NC}" - rm -rf $(tar -tf ${tarName}) + rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") } diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index f46de0f94b..b473f3b527 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -2,7 +2,7 @@ # # Generate tar.gz package for linux client in all os system set -e -# set -x +set -x curr_dir=$(pwd) compile_dir=$1 @@ -23,9 +23,12 @@ clientName2="${12}" productName="TDengine" clientName="taos" +benchmarkName="taosBenchmark" configFile="taos.cfg" tarName="package.tar.gz" +benchmarkName2="${clientName2}Benchmark" + if [ "$osType" != "Darwin" ]; then script_dir="$(dirname $(readlink -f $0))" top_dir="$(readlink -f ${script_dir}/../..)" @@ -53,11 +56,12 @@ fi # Directories and files. -if [ "$verMode" == "cluster" ]; then - sed -i 's/verMode=edge/verMode=cluster/g' ${script_dir}/remove_client.sh - sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh - sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" ${script_dir}/remove_client.sh -fi +#if [ "$verMode" == "cluster" ]; then +# sed -i 's/verMode=edge/verMode=cluster/g' ${script_dir}/remove_client.sh +# sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh +# sed -i "s/configFile2=\"taos\"/configFile2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh +# sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" ${script_dir}/remove_client.sh +#fi if [ "$osType" != "Darwin" ]; then if [ "$pagMode" == "lite" ]; then @@ -66,6 +70,7 @@ if [ "$osType" != "Darwin" ]; then ${script_dir}/remove_client.sh" else bin_files="${build_dir}/bin/${clientName} \ + ${build_dir}/bin/${benchmarkName} \ ${script_dir}/remove_client.sh \ ${script_dir}/set_core.sh \ ${script_dir}/get_client.sh" @@ -153,6 +158,7 @@ if [ "$verMode" == "cluster" ]; then sed -i 's/verMode=edge/verMode=cluster/g' install_client_temp.sh sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" install_client_temp.sh sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" install_client_temp.sh + sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" install_client_temp.sh sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" install_client_temp.sh sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" install_client_temp.sh diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 29160238ce..10093bdf49 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -96,7 +96,7 @@ else ${taostools_bin_files} \ ${taosx_bin} \ ${explorer_bin_files} \ - ${build_dir}/bin/taosadapter \ + ${build_dir}/bin/${clientName}adapter \ ${build_dir}/bin/udfd \ ${script_dir}/remove.sh \ ${script_dir}/set_core.sh \ @@ -135,12 +135,12 @@ mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} -if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then - cp ${compile_dir}/test/cfg/taosadapter.toml ${install_dir}/cfg || : +if [ -f "${compile_dir}/test/cfg/${clientName}adapter.toml" ]; then + cp ${compile_dir}/test/cfg/${clientName}adapter.toml ${install_dir}/cfg || : fi -if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then - cp ${compile_dir}/test/cfg/taosadapter.service ${install_dir}/cfg || : +if [ -f "${compile_dir}/test/cfg/${clientName}adapter.service" ]; then + cp ${compile_dir}/test/cfg/${clientName}adapter.service ${install_dir}/cfg || : fi if [ -f "${cfg_dir}/${serverName}.service" ]; then @@ -152,16 +152,16 @@ mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${se mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm if [ $adapterName != "taosadapter" ]; then - mv ${install_dir}/cfg/taosadapter.toml ${install_dir}/cfg/$adapterName.toml + mv ${install_dir}/cfg/${clientName2}adapter.toml ${install_dir}/cfg/$adapterName.toml sed -i "s/path = \"\/var\/log\/taos\"/path = \"\/var\/log\/${productName}\"/g" ${install_dir}/cfg/$adapterName.toml sed -i "s/password = \"taosdata\"/password = \"${defaultPasswd}\"/g" ${install_dir}/cfg/$adapterName.toml - mv ${install_dir}/cfg/taosadapter.service ${install_dir}/cfg/$adapterName.service + mv ${install_dir}/cfg/${clientName2}adapter.service ${install_dir}/cfg/$adapterName.service sed -i "s/TDengine/${productName}/g" ${install_dir}/cfg/$adapterName.service sed -i "s/taosAdapter/${adapterName}/g" ${install_dir}/cfg/$adapterName.service sed -i "s/taosadapter/${adapterName}/g" ${install_dir}/cfg/$adapterName.service - mv ${install_dir}/bin/taosadapter ${install_dir}/bin/${adapterName} + mv ${install_dir}/bin/${clientName2}adapter ${install_dir}/bin/${adapterName} mv ${install_dir}/bin/taosd-dump-cfg.gdb ${install_dir}/bin/${serverName}-dump-cfg.gdb fi @@ -233,8 +233,10 @@ if [ "$verMode" == "cluster" ]; then sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" remove_temp.sh sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" remove_temp.sh + sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" remove_temp.sh sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" remove_temp.sh - sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" remove_temp.sh + cusDomain=`echo "${cusEmail2}" | sed 's/^[^@]*@//'` + sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusDomain}\"/g" remove_temp.sh mv remove_temp.sh ${install_dir}/bin/remove.sh fi if [ "$verMode" == "cloud" ]; then @@ -262,8 +264,10 @@ if [ "$verMode" == "cluster" ]; then sed -i 's/verMode=edge/verMode=cluster/g' install_temp.sh sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" install_temp.sh sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" install_temp.sh + sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" install_temp.sh sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" install_temp.sh - sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" install_temp.sh + cusDomain=`echo "${cusEmail2}" | sed 's/^[^@]*@//'` + sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusDomain}\"/g" install_temp.sh mv install_temp.sh ${install_dir}/install.sh fi if [ "$verMode" == "cloud" ]; then diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 2479e48670..8ed3bd74b9 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -40,11 +40,16 @@ serverName2="taosd" clientName2="taos" productName2="TDengine" +adapterName2="${clientName2}adapter" +demoName2="${clientName2}demo" benchmarkName2="${clientName2}Benchmark" dumpName2="${clientName2}dump" +keeperName2="${clientName2}keeper" +xName2="${clientName2}x" +explorerName2="${clientName2}-explorer" uninstallScript2="rm${clientName2}" -installDir="/usr/local/${clientName}" +installDir="/usr/local/${clientName2}" #install main path install_main_dir=${installDir} @@ -55,8 +60,8 @@ local_bin_link_dir="/usr/local/bin" service_config_dir="/etc/systemd/system" -taos_service_name=${serverName} -taosadapter_service_name="taosadapter" +taos_service_name=${serverName2} +taosadapter_service_name="${clientName2}adapter" tarbitrator_service_name="tarbitratord" csudo="" if command -v sudo >/dev/null; then @@ -84,14 +89,14 @@ else fi function kill_taosadapter() { - pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}') + pid=$(ps -ef | grep "${adapterName2}" | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : fi } function kill_taosd() { - pid=$(ps -ef | grep ${serverName} | grep -v "grep" | awk '{print $2}') + pid=$(ps -ef | grep ${serverName2} | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : fi @@ -109,17 +114,17 @@ function clean_bin() { ${csudo}rm -f ${bin_link_dir}/${clientName} || : ${csudo}rm -f ${bin_link_dir}/${serverName} || : ${csudo}rm -f ${bin_link_dir}/udfd || : - ${csudo}rm -f ${bin_link_dir}/taosadapter || : - ${csudo}rm -f ${bin_link_dir}/taosBenchmark || : - ${csudo}rm -f ${bin_link_dir}/taosdemo || : - ${csudo}rm -f ${bin_link_dir}/taosdump || : - ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : + ${csudo}rm -f ${bin_link_dir}/${adapterName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : + ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : ${csudo}rm -f ${bin_link_dir}/tarbitrator || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : - ${csudo}rm -f ${bin_link_dir}/taoskeeper || : - ${csudo}rm -f ${bin_link_dir}/taosx || : - ${csudo}rm -f ${bin_link_dir}/taos-explorer || : + ${csudo}rm -f ${bin_link_dir}/${keeperName2} || : + ${csudo}rm -f ${bin_link_dir}/${xName2} || : + ${csudo}rm -f ${bin_link_dir}/${explorerName2} || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then ${csudo}rm -f ${bin_link_dir}/${clientName2} || : @@ -130,8 +135,8 @@ function clean_bin() { } function clean_local_bin() { - ${csudo}rm -f ${local_bin_link_dir}/taosBenchmark || : - ${csudo}rm -f ${local_bin_link_dir}/taosdemo || : + ${csudo}rm -f ${local_bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${local_bin_link_dir}/${demoName2} || : } function clean_lib() { @@ -173,7 +178,7 @@ function clean_service_on_systemd() { ${csudo}systemctl disable ${taos_service_name} &>/dev/null || echo &>/dev/null ${csudo}rm -f ${taosd_service_config} - taosadapter_service_config="${service_config_dir}/taosadapter.service" + taosadapter_service_config="${service_config_dir}/${clientName2}adapter.service" if systemctl is-active --quiet ${taosadapter_service_name}; then echo "${productName2} ${clientName2}Adapter is running, stopping it..." ${csudo}systemctl stop ${taosadapter_service_name} &>/dev/null || echo &>/dev/null @@ -235,8 +240,8 @@ function clean_service_on_sysvinit() { function clean_service_on_launchctl() { ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : - ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : - ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : + ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : + ${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : } function clean_service() { diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index 10a0fb5e02..2bdb56fac2 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -15,11 +15,12 @@ uninstallScript="rmtaos" clientName2="taos" productName2="TDengine" -benchmarkName2="${clientName}Benchmark" -dumpName2="${clientName}dump" -uninstallScript2="rm${clientName}" +benchmarkName2="${clientName2}Benchmark" +demoName2="${clientName2}demo" +dumpName2="${clientName2}dump" +uninstallScript2="rm${clientName2}" -installDir="/usr/local/${clientName}" +installDir="/usr/local/${clientName2}" #install main path install_main_dir=${installDir} @@ -44,14 +45,17 @@ function kill_client() { function clean_bin() { # Remove link - ${csudo}rm -f ${bin_link_dir}/${clientName} || : - ${csudo}rm -f ${bin_link_dir}/taosdemo || : - ${csudo}rm -f ${bin_link_dir}/taosdump || : + ${csudo}rm -f ${bin_link_dir}/${clientName2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : ${csudo}rm -f ${bin_link_dir}/set_core || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then ${csudo}rm -f ${bin_link_dir}/${clientName2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : ${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || : fi diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index de08ba66cc..874ac12f5c 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -30,6 +30,10 @@ #include "tsched.h" #include "ttime.h" +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -541,9 +545,15 @@ void taos_init_imp(void) { deltaToUtcInitOnce(); - if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { + char logDirName[64] = {0}; +#ifdef CUS_PROMPT + snprintf(logDirName, 64, "%slog", CUS_PROMPT); +#else + snprintf(logDirName, 64, "taoslog"); +#endif + if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { // ignore create log failed, only print - printf(" WARING: Create taoslog failed:%s. configDir=%s\n", strerror(errno), configDir); + printf(" WARING: Create %s failed:%s. configDir=%s\n", logDirName, strerror(errno), configDir); } if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) { diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 919a09962b..2c15980167 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -114,6 +114,8 @@ static const SSysDbTableSchema userFuncSchema[] = { {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "code_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "bufsize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "func_language", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "func_body", .bytes = TSDB_MAX_BINARY_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysDbTableSchema userIdxSchema[] = { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 3c8d394b43..03102b11a3 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1864,7 +1864,9 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { } } struct tm ptm = {0}; - taosLocalTime(&tt, &ptm); + if (taosLocalTime(&tt, &ptm, buf) == NULL) { + return buf; + } size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index aeeec1d61c..1c2d533977 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -228,7 +228,11 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); if (taosIsDir(cfgDir)) { +#ifdef CUS_PROMPT + snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "%s.cfg", CUS_PROMPT, cfgDir); +#else snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); +#endif } else { tstrncpy(cfgFile, cfgDir, sizeof(cfgDir)); } diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 7996498d45..dcd539bd91 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -727,7 +727,7 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { struct tm tm; time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision)); - taosLocalTime(&tt, &tm); + taosLocalTime(&tt, &tm, NULL); int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)numOfMonth; tm.tm_year = mon / 12; tm.tm_mon = mon % 12; @@ -750,11 +750,11 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char struct tm tm; time_t t = (time_t)skey; - taosLocalTime(&t, &tm); + taosLocalTime(&t, &tm, NULL); int32_t smon = tm.tm_year * 12 + tm.tm_mon; t = (time_t)ekey; - taosLocalTime(&t, &tm); + taosLocalTime(&t, &tm, NULL); int32_t emon = tm.tm_year * 12 + tm.tm_mon; if (unit == 'y') { @@ -774,7 +774,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio start /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; time_t tt = (time_t)start; - taosLocalTime(&tt, &tm); + taosLocalTime(&tt, &tm, NULL); tm.tm_sec = 0; tm.tm_min = 0; tm.tm_hour = 0; @@ -867,13 +867,17 @@ const char* fmtts(int64_t ts) { if (ts > -62135625943 && ts < 32503651200) { time_t t = (time_t)ts; - taosLocalTime(&t, &tm); + if (taosLocalTime(&t, &tm, buf) == NULL) { + return buf; + } pos += strftime(buf + pos, sizeof(buf), "s=%Y-%m-%d %H:%M:%S", &tm); } if (ts > -62135625943000 && ts < 32503651200000) { time_t t = (time_t)(ts / 1000); - taosLocalTime(&t, &tm); + if (taosLocalTime(&t, &tm, buf) == NULL) { + return buf; + } if (pos > 0) { buf[pos++] = ' '; buf[pos++] = '|'; @@ -885,7 +889,9 @@ const char* fmtts(int64_t ts) { { time_t t = (time_t)(ts / 1000000); - taosLocalTime(&t, &tm); + if (taosLocalTime(&t, &tm, buf) == NULL) { + return buf; + } if (pos > 0) { buf[pos++] = ' '; buf[pos++] = '|'; @@ -937,7 +943,9 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) ASSERT(false); } - taosLocalTime(", &ptm); + if (taosLocalTime(", &ptm, buf) == NULL) { + return; + } int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm); length += snprintf(ts + length, fractionLen, format, mod); length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm); diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index d3cffaa185..892b4a9a40 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -19,6 +19,21 @@ #include "tconfig.h" #include "tglobal.h" +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#else +#ifndef CUS_NAME + #define CUS_NAME "TDengine" +#endif + +#ifndef CUS_PROMPT + #define CUS_PROMPT "taos" +#endif + +#ifndef CUS_EMAIL + #define CUS_EMAIL "" +#endif +#endif // clang-format off #define DM_APOLLO_URL "The apollo string to use when configuring the server, such as: -a 'jsonFile:./tests/cfg.json', cfg.json text can be '{\"fqdn\":\"td1\"}'." #define DM_CFG_DIR "Configuration directory." @@ -228,7 +243,7 @@ static void dmDumpCfg() { } static int32_t dmInitLog() { - return taosCreateLog("taosdlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); + return taosCreateLog(CUS_PROMPT"dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); } static void taosCleanupArgs() { diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index 8ed7fc6a11..b2b86775ab 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -32,6 +32,7 @@ bool mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb); const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]); int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic); +int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9c94a908a1..df22034068 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -846,6 +846,18 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { goto _OVER; } + int32_t numOfTopics = 0; + if (mndGetNumOfTopics(pMnode, pDb->name, &numOfTopics) != 0) { + goto _OVER; + } + + if (numOfTopics != 0 && alterReq.walRetentionPeriod == 0) { + terrno = TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO; + mError("db:%s, not allowed to set WAL_RETENTION_PERIOD 0 when there are topics defined. numOfTopics:%d", pDb->name, + numOfTopics); + goto _OVER; + } + memcpy(&dbObj, pDb, sizeof(SDbObj)); if (dbObj.cfg.pRetensions != NULL) { dbObj.cfg.pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index a0ae622985..e707af2694 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -543,6 +543,7 @@ static int32_t mndRetrieveFuncs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)b2, false); + taosMemoryFree(b2); } else { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, NULL, true); @@ -569,6 +570,26 @@ static int32_t mndRetrieveFuncs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&pFunc->bufSize, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + char* language = ""; + if (pFunc->scriptType == TSDB_FUNC_SCRIPT_BIN_LIB) { + language = "C"; + } else if (pFunc->scriptType == TSDB_FUNC_SCRIPT_PYTHON) { + language = "Python"; + } + char varLang[TSDB_TYPE_STR_MAX_LEN + 1] = {0}; + varDataSetLen(varLang, strlen(language)); + strcpy(varDataVal(varLang), language); + colDataSetVal(pColInfo, numOfRows, (const char *)varLang, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + int32_t varCodeLen = (pFunc->codeSize + VARSTR_HEADER_SIZE) > TSDB_MAX_BINARY_LEN ? TSDB_MAX_BINARY_LEN : pFunc->codeSize + VARSTR_HEADER_SIZE; + char *b4 = taosMemoryMalloc(varCodeLen); + memcpy(varDataVal(b4), pFunc->pCode, varCodeLen - VARSTR_HEADER_SIZE); + varDataSetLen(b4, varCodeLen - VARSTR_HEADER_SIZE); + colDataSetVal(pColInfo, numOfRows, (const char*)b4, false); + taosMemoryFree(b4); + numOfRows++; sdbRelease(pSdb, pFunc); } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index ef94eb536d..5c20887cf5 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -653,7 +653,7 @@ _OVER: pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER) { mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, - pMnode->stopped, state.restored, syncStr(state.restored)); + pMnode->stopped, state.restored, syncStr(state.state)); return -1; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 991f1099a6..f6da370916 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -605,6 +605,12 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { goto _OVER; } + if (pDb->cfg.walRetentionPeriod == 0) { + terrno = TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO; + mError("db:%s, not allowed to create topic when WAL_RETENTION_PERIOD is zero", pDb->name); + goto _OVER; + } + code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb, pReq->info.conn.user); if (code == 0) { code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -793,7 +799,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } -static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { +int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { *pNumOfTopics = 0; SSdb *pSdb = pMnode->pSdb; @@ -943,4 +949,4 @@ int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return code; } -#endif \ No newline at end of file +#endif diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index b85d8d7746..5519d43012 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -88,98 +88,6 @@ _exit: return code; } -extern int32_t tsdbDelFileToJson(const SDelFile *pDelFile, cJSON *pJson); -extern int32_t tsdbJsonToDelFile(const cJSON *pJson, SDelFile *pDelFile); -extern int32_t tsdbDFileSetToJson(const SDFileSet *pSet, cJSON *pJson); -extern int32_t tsdbJsonToDFileSet(const cJSON *pJson, SDFileSet *pDelFile); - -static int32_t tsdbFSToJsonStr(STsdbFS *pFS, char **ppStr) { - int32_t code = 0; - int32_t lino = 0; - cJSON *pJson; - - ppStr[0] = NULL; - - pJson = cJSON_CreateObject(); - TSDB_CHECK_NULL(pJson, code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - - // format version - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "format", 1), code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - - // SDelFile - if (pFS->pDelFile) { - code = tsdbDelFileToJson(pFS->pDelFile, cJSON_AddObjectToObject(pJson, "del")); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // aDFileSet - cJSON *aSetJson = cJSON_AddArrayToObject(pJson, "file set"); - TSDB_CHECK_NULL(aSetJson, code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - for (int32_t iSet = 0; iSet < taosArrayGetSize(pFS->aDFileSet); iSet++) { - cJSON *pSetJson = cJSON_CreateObject(); - TSDB_CHECK_NULL(pSetJson, code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - - cJSON_AddItemToArray(aSetJson, pSetJson); - - code = tsdbDFileSetToJson(taosArrayGet(pFS->aDFileSet, iSet), pSetJson); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // print - ppStr[0] = cJSON_Print(pJson); - TSDB_CHECK_NULL(ppStr[0], code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - -_exit: - cJSON_Delete(pJson); - if (code) tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); - return code; -} - -static int32_t tsdbJsonStrToFS(const char *pStr, STsdbFS *pFS) { - int32_t code = 0; - int32_t lino; - - cJSON *pJson = cJSON_Parse(pStr); - TSDB_CHECK(pJson, code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - - const cJSON *pItem; - - // format version - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "format")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - - // SDelFile - if (cJSON_IsObject(pItem = cJSON_GetObjectItem(pJson, "del"))) { - pFS->pDelFile = (SDelFile *)taosMemoryCalloc(1, sizeof(SDelFile)); - TSDB_CHECK_NULL(pFS->pDelFile, code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - - code = tsdbJsonToDelFile(pItem, pFS->pDelFile); - TSDB_CHECK_CODE(code, lino, _exit); - - pFS->pDelFile->nRef = 1; - } else { - pFS->pDelFile = NULL; - } - - // aDFileSet - taosArrayClear(pFS->aDFileSet); - - const cJSON *pSetJson; - TSDB_CHECK(cJSON_IsArray(pItem = cJSON_GetObjectItem(pJson, "file set")), code, lino, _exit, - TSDB_CODE_FILE_CORRUPTED); - cJSON_ArrayForEach(pSetJson, pItem) { - SDFileSet *pSet = (SDFileSet *)taosArrayReserve(pFS->aDFileSet, 1); - TSDB_CHECK_NULL(pSet, code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - - code = tsdbJsonToDFileSet(pSetJson, pSet); - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - cJSON_Delete(pJson); - if (code) tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); - return code; -} - static int32_t tsdbSaveFSToFile(STsdbFS *pFS, const char *fname) { int32_t code = 0; int32_t lino = 0; @@ -224,84 +132,6 @@ _exit: return code; } -static int32_t tsdbSaveFSToJsonFile(STsdbFS *pFS, const char *fname) { - int32_t code; - int32_t lino; - char *pData; - - code = tsdbFSToJsonStr(pFS, &pData); - if (code) return code; - - TdFilePtr pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t n = taosWriteFile(pFD, pData, strlen(pData) + 1); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (taosFsyncFile(pFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - taosCloseFile(&pFD); - -_exit: - taosMemoryFree(pData); - if (code) { - tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); - } - - return code; -} - -static int32_t tsdbLoadFSFromJsonFile(const char *fname, STsdbFS *pFS) { - int32_t code = 0; - int32_t lino = 0; - char *pData = NULL; - - TdFilePtr pFD = taosOpenFile(fname, TD_FILE_READ); - if (pFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t size; - if (taosFStatFile(pFD, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if ((pData = taosMemoryMalloc(size)) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (taosReadFile(pFD, pData, size) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - taosCloseFile(&pFD); - - TSDB_CHECK_CODE(code = tsdbJsonStrToFS(pData, pFS), lino, _exit); - -_exit: - if (pData) taosMemoryFree(pData); - if (code) tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); - return code; -} - int32_t tsdbFSCreate(STsdbFS *pFS) { int32_t code = 0; @@ -439,8 +269,7 @@ int32_t tDFileSetCmprFn(const void *p1, const void *p2) { return 0; } -static void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t, char *current_json, - char *current_json_t) { +static void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t) { SVnode *pVnode = pTsdb->pVnode; if (pVnode->pTfs) { if (current) { @@ -451,14 +280,6 @@ static void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t, ch snprintf(current_t, TSDB_FILENAME_LEN - 1, "%s%s%s%sCURRENT.t", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), TD_DIRSEP, pTsdb->path, TD_DIRSEP); } - if (current_json) { - snprintf(current_json, TSDB_FILENAME_LEN - 1, "%s%s%s%scurrent.json", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), - TD_DIRSEP, pTsdb->path, TD_DIRSEP); - } - if (current_json_t) { - snprintf(current_json_t, TSDB_FILENAME_LEN - 1, "%s%s%s%scurrent.json.t", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), - TD_DIRSEP, pTsdb->path, TD_DIRSEP); - } } else { if (current) { snprintf(current, TSDB_FILENAME_LEN - 1, "%s%sCURRENT", pTsdb->path, TD_DIRSEP); @@ -466,12 +287,6 @@ static void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t, ch if (current_t) { snprintf(current_t, TSDB_FILENAME_LEN - 1, "%s%sCURRENT.t", pTsdb->path, TD_DIRSEP); } - if (current_json) { - snprintf(current_json, TSDB_FILENAME_LEN - 1, "%s%scurrent.json", pTsdb->path, TD_DIRSEP); - } - if (current_json_t) { - snprintf(current_json_t, TSDB_FILENAME_LEN - 1, "%s%scurrent.json.t", pTsdb->path, TD_DIRSEP); - } } } @@ -887,15 +702,20 @@ _exit: return code; } -static int32_t tsdbFSCommitImpl(STsdb *pTsdb, const char *fname, const char *tfname, bool isJson) { +// EXPOSED APIS ==================================================================================== +int32_t tsdbFSCommit(STsdb *pTsdb) { int32_t code = 0; int32_t lino = 0; STsdbFS fs = {0}; - if (!taosCheckExistFile(tfname)) goto _exit; + char current[TSDB_FILENAME_LEN] = {0}; + char current_t[TSDB_FILENAME_LEN] = {0}; + tsdbGetCurrentFName(pTsdb, current, current_t); + + if (!taosCheckExistFile(current_t)) goto _exit; // rename the file - if (taosRenameFile(tfname, fname) < 0) { + if (taosRenameFile(current_t, current) < 0) { code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code, lino, _exit); } @@ -904,11 +724,7 @@ static int32_t tsdbFSCommitImpl(STsdb *pTsdb, const char *fname, const char *tfn code = tsdbFSCreate(&fs); TSDB_CHECK_CODE(code, lino, _exit); - if (isJson) { - code = tsdbLoadFSFromJsonFile(fname, &fs); - } else { - code = tsdbLoadFSFromFile(fname, &fs); - } + code = tsdbLoadFSFromFile(current, &fs); TSDB_CHECK_CODE(code, lino, _exit); // apply file change @@ -923,19 +739,18 @@ _exit: return code; } -// EXPOSED APIS ==================================================================================== -int32_t tsdbFSCommit(STsdb *pTsdb) { - char current_json[TSDB_FILENAME_LEN] = {0}; - char current_json_t[TSDB_FILENAME_LEN] = {0}; - tsdbGetCurrentFName(pTsdb, NULL, NULL, current_json, current_json_t); - return tsdbFSCommitImpl(pTsdb, current_json, current_json_t, true); -} - int32_t tsdbFSRollback(STsdb *pTsdb) { int32_t code = 0; - char current_json_t[TSDB_FILENAME_LEN] = {0}; - tsdbGetCurrentFName(pTsdb, NULL, NULL, NULL, current_json_t); - (void)taosRemoveFile(current_json_t); + int32_t lino = 0; + + char current_t[TSDB_FILENAME_LEN] = {0}; + tsdbGetCurrentFName(pTsdb, NULL, current_t); + (void)taosRemoveFile(current_t); + +_exit: + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(errno)); + } return code; } @@ -951,33 +766,13 @@ int32_t tsdbFSOpen(STsdb *pTsdb, int8_t rollback) { // open impl char current[TSDB_FILENAME_LEN] = {0}; char current_t[TSDB_FILENAME_LEN] = {0}; - char current_json[TSDB_FILENAME_LEN] = {0}; - char current_json_t[TSDB_FILENAME_LEN] = {0}; - tsdbGetCurrentFName(pTsdb, current, current_t, current_json, current_json_t); + tsdbGetCurrentFName(pTsdb, current, current_t); if (taosCheckExistFile(current)) { - // CURRENT file exists code = tsdbLoadFSFromFile(current, &pTsdb->fs); TSDB_CHECK_CODE(code, lino, _exit); if (taosCheckExistFile(current_t)) { - if (rollback) { - (void)taosRemoveFile(current_t); - } else { - code = tsdbFSCommitImpl(pTsdb, current, current_t, false); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - code = tsdbSaveFSToJsonFile(&pTsdb->fs, current_json); - TSDB_CHECK_CODE(code, lino, _exit); - (void)taosRemoveFile(current); - } else if (taosCheckExistFile(current_json)) { - // current.json exists - code = tsdbLoadFSFromJsonFile(current_json, &pTsdb->fs); - TSDB_CHECK_CODE(code, lino, _exit); - - if (taosCheckExistFile(current_json_t)) { if (rollback) { code = tsdbFSRollback(pTsdb); TSDB_CHECK_CODE(code, lino, _exit); @@ -987,10 +782,11 @@ int32_t tsdbFSOpen(STsdb *pTsdb, int8_t rollback) { } } } else { - // empty TSDB - ASSERT(!rollback); - code = tsdbSaveFSToJsonFile(&pTsdb->fs, current_json); + // empty one + code = tsdbSaveFSToFile(&pTsdb->fs, current); TSDB_CHECK_CODE(code, lino, _exit); + + ASSERT(!rollback); } // scan and fix FS @@ -1228,12 +1024,12 @@ _exit: int32_t tsdbFSPrepareCommit(STsdb *pTsdb, STsdbFS *pFSNew) { int32_t code = 0; int32_t lino = 0; - char current_json_t[TSDB_FILENAME_LEN]; + char tfname[TSDB_FILENAME_LEN]; - tsdbGetCurrentFName(pTsdb, NULL, NULL, NULL, current_json_t); + tsdbGetCurrentFName(pTsdb, NULL, tfname); - // generate current.json - code = tsdbSaveFSToJsonFile(pFSNew, current_json_t); + // gnrt CURRENT.t + code = tsdbSaveFSToFile(pFSNew, tfname); TSDB_CHECK_CODE(code, lino, _exit); _exit: diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 8b2b2caec7..d91475376b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -92,11 +92,11 @@ static int32_t tGetSmaFile(uint8_t *p, SSmaFile *pSmaFile) { } // EXPOSED APIS ================================================== -static char *getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t commitId, char fname[]) { - const char *p1 = tfsGetDiskPath(pTsdb->pVnode->pTfs, did); - int32_t len = strlen(p1); +static char* getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t commitId, char fname[]) { + const char* p1 = tfsGetDiskPath(pTsdb->pVnode->pTfs, did); + int32_t len = strlen(p1); - char *p = memcpy(fname, p1, len); + char* p = memcpy(fname, p1, len); p += len; *(p++) = TD_DIRSEP[0]; @@ -121,25 +121,25 @@ static char *getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t } void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]) { - char *p = getFileNamePrefix(pTsdb, did, fid, pHeadF->commitID, fname); + char* p = getFileNamePrefix(pTsdb, did, fid, pHeadF->commitID, fname); memcpy(p, ".head", 5); p[5] = 0; } void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]) { - char *p = getFileNamePrefix(pTsdb, did, fid, pDataF->commitID, fname); + char* p = getFileNamePrefix(pTsdb, did, fid, pDataF->commitID, fname); memcpy(p, ".data", 5); p[5] = 0; } void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]) { - char *p = getFileNamePrefix(pTsdb, did, fid, pSttF->commitID, fname); + char* p = getFileNamePrefix(pTsdb, did, fid, pSttF->commitID, fname); memcpy(p, ".stt", 4); p[4] = 0; } void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) { - char *p = getFileNamePrefix(pTsdb, did, fid, pSmaF->commitID, fname); + char* p = getFileNamePrefix(pTsdb, did, fid, pSmaF->commitID, fname); memcpy(p, ".sma", 4); p[4] = 0; } @@ -280,272 +280,6 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { return n; } -static int32_t tDiskIdToJson(const SDiskID *pDiskId, cJSON *pJson) { - int32_t code = 0; - int32_t lino; - - if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; - - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "level", pDiskId->level), code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "id", pDiskId->id), code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - -_exit: - return code; -} -static int32_t tJsonToDiskId(const cJSON *pJson, SDiskID *pDiskId) { - int32_t code = 0; - int32_t lino; - - const cJSON *pItem; - - // level - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "level")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - pDiskId->level = (int32_t)pItem->valuedouble; - - // id - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "id")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - pDiskId->id = (int32_t)pItem->valuedouble; - -_exit: - return code; -} - -static int32_t tHeadFileToJson(const SHeadFile *pHeadF, cJSON *pJson) { - int32_t code = 0; - int32_t lino; - - if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; - - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "commit id", pHeadF->commitID), code, lino, _exit, - TSDB_CODE_OUT_OF_MEMORY); - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "size", pHeadF->size), code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "offset", pHeadF->offset), code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - -_exit: - return code; -} - -static int32_t tJsonToHeadFile(const cJSON *pJson, SHeadFile *pHeadF) { - int32_t code = 0; - int32_t lino; - - const cJSON *pItem; - - // commit id - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "commit id")), code, lino, _exit, - TSDB_CODE_FILE_CORRUPTED); - pHeadF->commitID = (int64_t)pItem->valuedouble; - - // size - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "size")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - pHeadF->size = (int64_t)pItem->valuedouble; - - // offset - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "offset")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - pHeadF->offset = (int64_t)pItem->valuedouble; - -_exit: - return code; -} - -static int32_t tDataFileToJson(const SDataFile *pDataF, cJSON *pJson) { - int32_t code = 0; - int32_t lino; - - if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; - - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "commit id", pDataF->commitID), code, lino, _exit, - TSDB_CODE_OUT_OF_MEMORY); - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "size", pDataF->size), code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - -_exit: - return code; -} - -static int32_t tJsonToDataFile(const cJSON *pJson, SDataFile *pDataF) { - int32_t code = 0; - int32_t lino; - - const cJSON *pItem; - - // commit id - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "commit id")), code, lino, _exit, - TSDB_CODE_FILE_CORRUPTED); - pDataF->commitID = (int64_t)pItem->valuedouble; - - // size - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "size")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - pDataF->size = (int64_t)pItem->valuedouble; - -_exit: - return code; -} - -static int32_t tSmaFileToJson(const SSmaFile *pSmaF, cJSON *pJson) { - int32_t code = 0; - int32_t lino; - - if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; - - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "commit id", pSmaF->commitID), code, lino, _exit, - TSDB_CODE_OUT_OF_MEMORY); - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "size", pSmaF->size), code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - -_exit: - return code; -} - -static int32_t tJsonToSmaFile(const cJSON *pJson, SSmaFile *pSmaF) { - int32_t code = 0; - int32_t lino; - - // commit id - const cJSON *pItem; - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "commit id")), code, lino, _exit, - TSDB_CODE_FILE_CORRUPTED); - pSmaF->commitID = (int64_t)pItem->valuedouble; - - // size - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "size")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - pSmaF->size = (int64_t)pItem->valuedouble; - -_exit: - return code; -} - -static int32_t tSttFileToJson(const SSttFile *pSttF, cJSON *pJson) { - int32_t code = 0; - int32_t lino; - - if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; - - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "commit id", pSttF->commitID), code, lino, _exit, - TSDB_CODE_OUT_OF_MEMORY); - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "size", pSttF->size), code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "offset", pSttF->offset), code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - -_exit: - return code; -} - -static int32_t tJsonToSttFile(const cJSON *pJson, SSttFile *pSttF) { - int32_t code = 0; - int32_t lino; - - const cJSON *pItem; - - // commit id - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "commit id")), code, lino, _exit, - TSDB_CODE_FILE_CORRUPTED); - pSttF->commitID = (int64_t)pItem->valuedouble; - - // size - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "size")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - pSttF->size = (int64_t)pItem->valuedouble; - - // offset - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "offset")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - pSttF->offset = (int64_t)pItem->valuedouble; - -_exit: - return code; -} - -int32_t tsdbDFileSetToJson(const SDFileSet *pSet, cJSON *pJson) { - int32_t code = 0; - int32_t lino; - - if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; - - code = tDiskIdToJson(&pSet->diskId, cJSON_AddObjectToObject(pJson, "disk id")); - TSDB_CHECK_CODE(code, lino, _exit); - - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "fid", pSet->fid), code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - - // head - code = tHeadFileToJson(pSet->pHeadF, cJSON_AddObjectToObject(pJson, "head")); - TSDB_CHECK_CODE(code, lino, _exit); - - // data - code = tDataFileToJson(pSet->pDataF, cJSON_AddObjectToObject(pJson, "data")); - TSDB_CHECK_CODE(code, lino, _exit); - - // sma - code = tSmaFileToJson(pSet->pSmaF, cJSON_AddObjectToObject(pJson, "sma")); - TSDB_CHECK_CODE(code, lino, _exit); - - // stt array - cJSON *aSttJson = cJSON_AddArrayToObject(pJson, "stt"); - TSDB_CHECK_NULL(aSttJson, code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { - cJSON *pSttJson = cJSON_CreateObject(); - TSDB_CHECK_NULL(pSttJson, code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - - cJSON_AddItemToArray(aSttJson, pSttJson); - - code = tSttFileToJson(pSet->aSttF[iStt], pSttJson); - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - return code; -} - -int32_t tsdbJsonToDFileSet(const cJSON *pJson, SDFileSet *pSet) { - int32_t code = 0; - int32_t lino; - - const cJSON *pItem; - // disk id - TSDB_CHECK(cJSON_IsObject(pItem = cJSON_GetObjectItem(pJson, "disk id")), code, lino, _exit, - TSDB_CODE_FILE_CORRUPTED); - code = tJsonToDiskId(pItem, &pSet->diskId); - TSDB_CHECK_CODE(code, lino, _exit); - - // fid - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "fid")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - pSet->fid = (int32_t)pItem->valuedouble; - - // head - TSDB_CHECK(cJSON_IsObject(pItem = cJSON_GetObjectItem(pJson, "head")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - TSDB_CHECK_NULL(pSet->pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)), code, lino, _exit, - TSDB_CODE_OUT_OF_MEMORY); - TSDB_CHECK_CODE(code = tJsonToHeadFile(pItem, pSet->pHeadF), lino, _exit); - pSet->pHeadF->nRef = 1; - - // data - TSDB_CHECK(cJSON_IsObject(pItem = cJSON_GetObjectItem(pJson, "data")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - TSDB_CHECK_NULL(pSet->pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile)), code, lino, _exit, - TSDB_CODE_OUT_OF_MEMORY); - TSDB_CHECK_CODE(code = tJsonToDataFile(pItem, pSet->pDataF), lino, _exit); - pSet->pDataF->nRef = 1; - - // sma - TSDB_CHECK(cJSON_IsObject(pItem = cJSON_GetObjectItem(pJson, "sma")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - TSDB_CHECK_NULL(pSet->pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)), code, lino, _exit, - TSDB_CODE_OUT_OF_MEMORY); - TSDB_CHECK_CODE(code = tJsonToSmaFile(pItem, pSet->pSmaF), lino, _exit); - pSet->pSmaF->nRef = 1; - - // stt array - const cJSON *element; - pSet->nSttF = 0; - TSDB_CHECK(cJSON_IsArray(pItem = cJSON_GetObjectItem(pJson, "stt")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - cJSON_ArrayForEach(element, pItem) { - TSDB_CHECK(cJSON_IsObject(element), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - - pSet->aSttF[pSet->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - TSDB_CHECK_NULL(pSet->aSttF[pSet->nSttF], code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - TSDB_CHECK_CODE(code = tJsonToSttFile(element, pSet->aSttF[pSet->nSttF]), lino, _exit); - pSet->aSttF[pSet->nSttF]->nRef = 1; - pSet->nSttF++; - } - -_exit: - if (code) tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); - return code; -} - // SDelFile =============================================== void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]) { snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%dver%" PRId64 "%s", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), @@ -571,42 +305,3 @@ int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile) { return n; } - -int32_t tsdbDelFileToJson(const SDelFile *pDelFile, cJSON *pJson) { - if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; - - int32_t code = 0; - int32_t lino; - - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "commit id", pDelFile->commitID), code, lino, _exit, - TSDB_CODE_OUT_OF_MEMORY); - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "size", pDelFile->size), code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY); - TSDB_CHECK_NULL(cJSON_AddNumberToObject(pJson, "offset", pDelFile->offset), code, lino, _exit, - TSDB_CODE_OUT_OF_MEMORY); - -_exit: - return code; -} - -int32_t tsdbJsonToDelFile(const cJSON *pJson, SDelFile *pDelFile) { - int32_t code = 0; - int32_t lino; - - const cJSON *pItem; - - // commit id - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "commit id")), code, lino, _exit, - TSDB_CODE_FILE_CORRUPTED); - pDelFile->commitID = cJSON_GetNumberValue(pItem); - - // size - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "size")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - pDelFile->size = cJSON_GetNumberValue(pItem); - - // offset - TSDB_CHECK(cJSON_IsNumber(pItem = cJSON_GetObjectItem(pJson, "offset")), code, lino, _exit, TSDB_CODE_FILE_CORRUPTED); - pDelFile->offset = cJSON_GetNumberValue(pItem); - -_exit: - return code; -} \ No newline at end of file diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ce647014ae..be87dcd6ff 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -88,7 +88,7 @@ static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t ord struct tm tm; time_t t = (time_t)key; - taosLocalTime(&t, &tm); + taosLocalTime(&t, &tm, NULL); int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); tm.tm_year = mon / 12; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index fef588a503..0f2eb4e0d7 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -281,7 +281,7 @@ static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t o struct tm tm; time_t t = (time_t)key; - taosLocalTime(&t, &tm); + taosLocalTime(&t, &tm, NULL); int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); tm.tm_year = mon / 12; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 0257b3d5e6..e2b68a8b05 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -213,8 +213,9 @@ static int32_t addTimezoneParam(SNodeList* pList) { char buf[6] = {0}; time_t t = taosTime(NULL); struct tm tmInfo; - taosLocalTime(&t, &tmInfo); - strftime(buf, sizeof(buf), "%z", &tmInfo); + if (taosLocalTime(&t, &tmInfo, buf) != NULL) { + strftime(buf, sizeof(buf), "%z", &tmInfo); + } int32_t len = (int32_t)strlen(buf); SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 611344063b..fad118297e 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1070,8 +1070,15 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, if (code != 0) { return code; } + SUdfcUvSession *session = handle; code = doCallUdfScalarFunc(handle, input, numOfCols, output); + if (code != TSDB_CODE_SUCCESS) { + fnError("udfc scalar function execution failure"); + releaseUdfFuncHandle(udfName); + return code; + } + if (output->columnData == NULL) { fnError("udfc scalar function calculate error. no column data"); code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 195a08525c..88362201b4 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1067,9 +1067,15 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * } struct tm tmInfo; - taosLocalTime((const time_t *)&timeVal, &tmInfo); + int32_t len = 0; + + if (taosLocalTime((const time_t *)&timeVal, &tmInfo, buf) == NULL) { + len = (int32_t)strlen(buf); + goto _end; + } + strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo); - int32_t len = (int32_t)strlen(buf); + len = (int32_t)strlen(buf); // add timezone string if (tzLen > 0) { @@ -1103,6 +1109,7 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * len += fracLen; } +_end: memmove(buf + VARSTR_HEADER_SIZE, buf, len); varDataSetLen(buf, len); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index a8f9362757..848de4f36d 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -290,14 +290,22 @@ int32_t walEndSnapshot(SWal *pWal) { int ts = taosGetTimestampSec(); ver = TMAX(ver - pWal->vers.logRetention, pWal->vers.firstVer - 1); + + bool hasTopic = false; + int64_t refVer = ver; void *pIter = NULL; while (1) { pIter = taosHashIterate(pWal->pRefHash, pIter); if (pIter == NULL) break; SWalRef *pRef = *(SWalRef **)pIter; if (pRef->refVer == -1) continue; - ver = TMIN(ver, pRef->refVer - 1); + refVer = TMIN(refVer, pRef->refVer - 1); wDebug("vgId:%d, wal found ref %" PRId64 ", refId %" PRId64, pWal->cfg.vgId, pRef->refVer, pRef->refId); + hasTopic = true; + } + // compatible mode + if (pWal->cfg.retentionPeriod == 0 && hasTopic) { + ver = TMIN(ver, refVer); } int deleteCnt = 0; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 52309a7b35..84004ed3c1 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -17,6 +17,10 @@ #include "os.h" #include "taoserror.h" +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + #define PROCESS_ITEM 12 #define UUIDLEN37 37 @@ -252,7 +256,11 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { #ifdef WINDOWS // ASSERT(0); #elif defined(_TD_DARWIN_64) +#ifdef CUS_PROMPT + const char *filepath = "/usr/local/"CUS_PROMPT"/email"; +#else const char *filepath = "/usr/local/taos/email"; +#endif // CUS_PROMPT TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); if (pFile == NULL) return false; @@ -264,8 +272,12 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { taosCloseFile(&pFile); return 0; +#else +#ifdef CUS_PROMPT + const char *filepath = "/usr/local/"CUS_PROMPT"/email"; #else const char *filepath = "/usr/local/taos/email"; +#endif // CUS_PROMPT TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); if (pFile == NULL) return false; diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 5d5bff8c48..f120b6650a 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -407,12 +407,21 @@ time_t taosMktime(struct tm *timep) { #endif } -struct tm *taosLocalTime(const time_t *timep, struct tm *result) { +struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) { + struct tm *res = NULL; + if (result == NULL) { - return localtime(timep); + res = localtime(timep); + if (res == NULL && buf != NULL) { + sprintf(buf, "NaN"); + } + return res; } #ifdef WINDOWS if (*timep < 0) { + if (buf != NULL) { + sprintf(buf, "NaN"); + } return NULL; // TODO: bugs in following code SYSTEMTIME ss, s; @@ -421,6 +430,9 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result) { struct tm tm1; time_t tt = 0; if (localtime_s(&tm1, &tt) != 0 ) { + if (buf != NULL) { + sprintf(buf, "NaN"); + } return NULL; } ss.wYear = tm1.tm_year + 1900; @@ -449,11 +461,17 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result) { result->tm_isdst = 0; } else { if (localtime_s(result, timep) != 0) { + if (buf != NULL) { + sprintf(buf, "NaN"); + } return NULL; } } #else - localtime_r(timep, result); + res = localtime_r(timep, result); + if (res == NULL && buf != NULL) { + sprintf(buf, "NaN"); + } #endif return result; } diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index ad223bff27..cd6ad7cdb5 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -893,7 +893,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - taosLocalTime(&tx1, &tm1); + taosLocalTime(&tx1, &tm1, NULL); daylight = tm1.tm_isdst; /* @@ -921,7 +921,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - taosLocalTime(&tx1, &tm1); + taosLocalTime(&tx1, &tm1, NULL); /* load time zone string from /etc/timezone */ // FILE *f = fopen("/etc/timezone", "r"); errno = 0; @@ -1008,7 +1008,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - taosLocalTime(&tx1, &tm1); + taosLocalTime(&tx1, &tm1, NULL); /* * format example: diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3148cf65c0..2ea43e2cde 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -224,6 +224,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, "Invalid database name TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, "Too many databases for account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, "Database in dropping status") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_EXIST, "Database not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO, "WAL retention period is zero") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_ACCT, "Invalid database account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_OPTION_UNCHANGED, "Database options not changed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_INDEX_NOT_EXIST, "Index not exist") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index bd9ea058b4..a3d3c399ab 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -121,7 +121,7 @@ static FORCE_INLINE void taosUpdateDaylight() { struct timeval timeSecs; taosGetTimeOfDay(&timeSecs); time_t curTime = timeSecs.tv_sec; - ptm = taosLocalTime(&curTime, &Tm); + ptm = taosLocalTime(&curTime, &Tm, NULL); tsDaylightActive = ptm->tm_isdst; } static FORCE_INLINE int32_t taosGetDaylight() { return tsDaylightActive; } @@ -437,7 +437,7 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { taosGetTimeOfDay(&timeSecs); time_t curTime = timeSecs.tv_sec; - ptm = taosLocalTime(&curTime, &Tm); + ptm = taosLocalTime(&curTime, &Tm, NULL); return sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " %s", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId(), flags); diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_telnet_tcp.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_telnet_tcp.json index 9e1241397f..e609fcfebd 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_telnet_tcp.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_telnet_tcp.json @@ -16,7 +16,7 @@ "num_of_records_per_req": 10, "databases": [{ "dbinfo": { - "name": "db", + "name": "opentsdb_telnet", "drop": "yes" }, "super_tables": [{ diff --git a/tests/docs-examples-test/python.sh b/tests/docs-examples-test/python.sh index a7501b54ed..2a44ee7552 100644 --- a/tests/docs-examples-test/python.sh +++ b/tests/docs-examples-test/python.sh @@ -52,7 +52,7 @@ python3 conn_rest_pandas.py taos -s "drop database if exists power" # 11 -taos -s "create database if not exists test" +taos -s "create database if not exists test wal_retention_period 3600" python3 connect_native_reference.py # 12 diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 6e0b180ed8..c0a899bbe4 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -120,6 +120,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py ,,n,system-test,python3 ./test.py -f 0-others/compatibility.py ,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py +,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py @@ -1102,9 +1103,9 @@ ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py -#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R -#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R #docs-examples test ,,n,docs-examples-test,bash python.sh diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 4830d2f8b0..536b8f30d3 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -17,6 +17,7 @@ import time import datetime import inspect import importlib +import traceback from util.log import * @@ -75,6 +76,7 @@ class TDCases: case.run() except Exception as e: tdLog.notice(repr(e)) + traceback.print_exc() tdLog.exit("%s failed" % (fileName)) case.stop() runNum += 1 diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 414b8b2099..783ee476cb 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -251,7 +251,7 @@ class TDSql: if self.queryResult[row][col] != data: if self.cursor.istype(col, "TIMESTAMP"): - # suppose user want to check nanosecond timestamp if a longer data passed`` + # suppose user want to check nanosecond timestamp if a longer data passed`` if isinstance(data,str) : if (len(data) >= 28): if self.queryResult[row][col] == _parse_ns_timestamp(data): @@ -260,7 +260,7 @@ class TDSql: else: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) - tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) else: if self.queryResult[row][col].astimezone(datetime.timezone.utc) == _parse_datetime(data).astimezone(datetime.timezone.utc): # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") @@ -270,12 +270,12 @@ class TDSql: args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) return - elif isinstance(data,int) : - if len(str(data)) == 16 : + elif isinstance(data,int): + if len(str(data)) == 16: precision = 'us' - elif len(str(data)) == 13 : + elif len(str(data)) == 13: precision = 'ms' - elif len(str(data)) == 19 : + elif len(str(data)) == 19: precision = 'ns' else: caller = inspect.getframeinfo(inspect.stack()[1][0]) @@ -303,11 +303,21 @@ class TDSql: args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) return + elif isinstance(data,datetime.datetime): + dt_obj = self.queryResult[row][col] + delt_data = data-datetime.datetime.fromtimestamp(0,data.tzinfo) + delt_result = self.queryResult[row][col] - datetime.datetime.fromtimestamp(0,self.queryResult[row][col].tzinfo) + if delt_data == delt_result: + tdLog.info("check successfully") + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + return else: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) - if str(self.queryResult[row][col]) == str(data): # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") diff --git a/tests/script/tsim/query/udfpy.sim b/tests/script/tsim/query/udfpy.sim index 2340235daa..025df7984b 100644 --- a/tests/script/tsim/query/udfpy.sim +++ b/tests/script/tsim/query/udfpy.sim @@ -42,6 +42,25 @@ sql show functions; if $rows != 4 then return -1 endi + +sql select func_language, func_body,name from information_schema.ins_functions order by name +if $rows != 4 then + return -1 +endi + +if $data00 != @C@ then + return -1 +endi +if $data10 != @C@ then + return -1 +endi +if $data20 != @Python@ then + return -1 +endi +if $data30 != @Python@ then + return -1 +endi + sql select bit_and(f, f) from t; if $rows != 2 then return -1 diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index 6880f290f5..b296290214 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -83,6 +86,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -155,6 +161,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -226,6 +235,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) diff --git a/tests/script/tsim/tmq/basic1Of2Cons.sim b/tests/script/tsim/tmq/basic1Of2Cons.sim index 11b645c4d1..4c966c370e 100644 --- a/tests/script/tsim/tmq/basic1Of2Cons.sim +++ b/tests/script/tsim/tmq/basic1Of2Cons.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -83,6 +86,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for stb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -186,6 +192,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -288,6 +297,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) diff --git a/tests/script/tsim/tmq/basic2.sim b/tests/script/tsim/tmq/basic2.sim index dce73be592..6d49b46c85 100644 --- a/tests/script/tsim/tmq/basic2.sim +++ b/tests/script/tsim/tmq/basic2.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -118,6 +121,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -175,6 +181,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) diff --git a/tests/script/tsim/tmq/basic2Of2Cons.sim b/tests/script/tsim/tmq/basic2Of2Cons.sim index 87559305ba..db660a0c93 100644 --- a/tests/script/tsim/tmq/basic2Of2Cons.sim +++ b/tests/script/tsim/tmq/basic2Of2Cons.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -147,6 +150,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -234,6 +240,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) diff --git a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim index dda5e0059e..54e10126f1 100644 --- a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim +++ b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -168,6 +171,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -259,6 +265,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) diff --git a/tests/script/tsim/tmq/basic3.sim b/tests/script/tsim/tmq/basic3.sim index 8d677766d7..1e95fa90a5 100644 --- a/tests/script/tsim/tmq/basic3.sim +++ b/tests/script/tsim/tmq/basic3.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -83,6 +86,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -154,6 +160,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -225,6 +234,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) diff --git a/tests/script/tsim/tmq/basic3Of2Cons.sim b/tests/script/tsim/tmq/basic3Of2Cons.sim index 4921c86c45..be0292c57b 100644 --- a/tests/script/tsim/tmq/basic3Of2Cons.sim +++ b/tests/script/tsim/tmq/basic3Of2Cons.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -82,6 +85,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -197,6 +203,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -299,6 +308,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) diff --git a/tests/script/tsim/tmq/basic4.sim b/tests/script/tsim/tmq/basic4.sim index 9b418f12f2..33a66628d0 100644 --- a/tests/script/tsim/tmq/basic4.sim +++ b/tests/script/tsim/tmq/basic4.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -115,6 +118,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -172,6 +178,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) diff --git a/tests/script/tsim/tmq/basic4Of2Cons.sim b/tests/script/tsim/tmq/basic4Of2Cons.sim index f1755f732b..fdee3f633e 100644 --- a/tests/script/tsim/tmq/basic4Of2Cons.sim +++ b/tests/script/tsim/tmq/basic4Of2Cons.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -156,6 +159,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -244,6 +250,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) diff --git a/tests/script/tsim/tmq/snapshot.sim b/tests/script/tsim/tmq/snapshot.sim index de0468e6f2..81fff35224 100644 --- a/tests/script/tsim/tmq/snapshot.sim +++ b/tests/script/tsim/tmq/snapshot.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -83,6 +86,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -152,6 +158,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -223,6 +232,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) diff --git a/tests/script/tsim/tmq/snapshot1.sim b/tests/script/tsim/tmq/snapshot1.sim index e586719db2..c79892ae1d 100644 --- a/tests/script/tsim/tmq/snapshot1.sim +++ b/tests/script/tsim/tmq/snapshot1.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -147,6 +150,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -224,6 +230,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) diff --git a/tests/script/tsim/tmq/topic.sim b/tests/script/tsim/tmq/topic.sim index cb1e74798e..0bf0873e9f 100644 --- a/tests/script/tsim/tmq/topic.sim +++ b/tests/script/tsim/tmq/topic.sim @@ -39,6 +39,8 @@ endi sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 print == create super table sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) diff --git a/tests/script/tsim/user/privilege_db.sim b/tests/script/tsim/user/privilege_db.sim index b708fdab64..50eaa12108 100644 --- a/tests/script/tsim/user/privilege_db.sim +++ b/tests/script/tsim/user/privilege_db.sim @@ -4,13 +4,13 @@ system sh/exec.sh -n dnode1 -s start sql connect print =============== create db -sql create database d1 vgroups 1; +sql create database d1 vgroups 1 wal_retention_period 3600; sql use d1 sql create table d1_stb (ts timestamp, i int) tags (j int) sql create topic d1_topic_1 as select ts, i from d1_stb -sql create database d2 vgroups 1; -sql create database d3 vgroups 1; +sql create database d2 vgroups 1 wal_retention_period 3600; +sql create database d3 vgroups 1 wal_retention_period 3600; sql select * from information_schema.ins_databases if $rows != 5 then return -1 diff --git a/tests/script/tsim/user/privilege_topic.sim b/tests/script/tsim/user/privilege_topic.sim index 9ce5bebec3..9d096a9780 100644 --- a/tests/script/tsim/user/privilege_topic.sim +++ b/tests/script/tsim/user/privilege_topic.sim @@ -4,9 +4,9 @@ system sh/exec.sh -n dnode1 -s start sql connect print =============== create db -sql create database root_d1 vgroups 1; -sql create database root_d2 vgroups 1; -sql create database root_d3 vgroups 1; +sql create database root_d1 vgroups 1 wal_retention_period 3600; +sql create database root_d2 vgroups 1 wal_retention_period 3600; +sql create database root_d3 vgroups 1 wal_retention_period 3600; sql show user privileges if $rows != 1 then diff --git a/tests/system-test/0-others/backquote_check.py b/tests/system-test/0-others/backquote_check.py index 3357723253..be8590f913 100644 --- a/tests/system-test/0-others/backquote_check.py +++ b/tests/system-test/0-others/backquote_check.py @@ -29,7 +29,7 @@ class TDTestCase: self.streamname = 'stm' self.streamtb = 'stm_stb' def topic_name_check(self): - tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'create database if not exists {self.dbname} wal_retention_period 3600') tdSql.execute(f'use {self.dbname}') tdSql.execute(f'create stable {self.stbname} (ts timestamp,c0 int) tags(t0 int)') for name in [self.dbname,self.stbname]: @@ -56,12 +56,12 @@ class TDTestCase: tdSql.execute(f'drop topic `{name}`') def db_name_check(self): - tdSql.execute(f'create database if not exists `{self.dbname}`') + tdSql.execute(f'create database if not exists `{self.dbname}` wal_retention_period 3600') tdSql.execute(f'use `{self.dbname}`') tdSql.execute(f'drop database {self.dbname}') def stream_name_check(self): - tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'create database if not exists {self.dbname} wal_retention_period 3600') tdSql.execute(f'use {self.dbname}') tdSql.execute(f'create stable {self.stbname} (ts timestamp,c0 int) tags(t0 int)') tdSql.execute(f'create stream `{self.streamname}` into `{self.streamtb}` as select count(*) from {self.stbname} interval(10s);') @@ -84,4 +84,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 4684b7191b..1922ebfb2f 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -187,6 +187,7 @@ class TDTestCase: tdsql.execute("drop database if exists db") tdsql.execute("create database db") tdsql.execute("use db") + tdsql.execute("alter database db wal_retention_period 3600") tdsql.execute("create stable db.stb1 (ts timestamp, c1 int) tags (t1 int);") tdsql.execute("insert into db.ct1 using db.stb1 TAGS(1) values(now(),11);") tdsql.error(" insert into `db.ct2` using db.stb1 TAGS(9) values(now(),11);") diff --git a/tests/system-test/0-others/performance_schema.py b/tests/system-test/0-others/performance_schema.py index 70e86009a6..9d2a362254 100755 --- a/tests/system-test/0-others/performance_schema.py +++ b/tests/system-test/0-others/performance_schema.py @@ -75,7 +75,7 @@ class TDTestCase: def prepare_data(self): tdSql.execute(f"create database if not exists {self.dbname} vgroups 2") #1 query tdSql.execute(f'use {self.dbname}') #1 query - + tdsql.execute(f"alter database {self.dbname} wal_retention_period 3600") tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) #1 query for i in range(self.tbnum): #self.tbnum query @@ -92,12 +92,12 @@ class TDTestCase: def run(self): tdSqlTran = TDSql() tdSqlTran.init(self.obj.conn.cursor()) - tdSqlTran.execute(f"create database if not exists %s vgroups 20"%(self.obj.transTestDBName)) + tdSqlTran.execute(f"create database if not exists %s vgroups 20 wal_retention_period 3600"%(self.obj.transTestDBName)) tdSqlTran.execute(f"DROP DATABASE %s"%(self.obj.transTestDBName)) def init_tmq_env(self, db, topic): self.conn.execute("drop topic if exists {}".format(topic)) - self.conn.execute("create database if not exists {}".format(db)) + self.conn.execute("create database if not exists {} wal_retention_period 3600".format(db)) self.conn.select_db(db) self.conn.execute( "create stable if not exists stb_sub (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))") @@ -211,4 +211,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/show.py b/tests/system-test/0-others/show.py index 3e176fe251..b284605a0e 100644 --- a/tests/system-test/0-others/show.py +++ b/tests/system-test/0-others/show.py @@ -28,7 +28,7 @@ class TDTestCase: self.perf_param = ['apps','connections','consumers','queries','transactions'] self.perf_param_list = ['apps','connections','consumers','queries','trans'] self.dbname = "db" - self.vgroups = 10 + self.vgroups = 4 self.stbname = f'`{tdCom.getLongName(5)}`' self.tbname = f'`{tdCom.getLongName(3)}`' self.db_param = { diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index 944ff52d5b..8094c4e0f5 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -292,7 +292,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() # time.sleep(2) - vgroups = "30" + vgroups = "4" sql = "create database db3 vgroups " + vgroups tdSql.query(sql) sql = "create table db3.stb (ts timestamp, f int) tags (t int)" diff --git a/tests/system-test/0-others/taosdShell.py b/tests/system-test/0-others/taosdShell.py index ce049b8515..3b9eb66705 100644 --- a/tests/system-test/0-others/taosdShell.py +++ b/tests/system-test/0-others/taosdShell.py @@ -129,7 +129,7 @@ class TDTestCase: # database\stb\tb\chiild-tb\rows\topics tdSql.execute("create user testpy pass 'testpy'") tdSql.execute("drop database if exists db0;") - tdSql.execute("create database db0;") + tdSql.execute("create database db0 wal_retention_period 3600;") tdSql.execute("use db0;") tdSql.execute("create table if not exists db0.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned);") tdSql.execute("create table db0.ct1 using db0.stb tags(1000);") @@ -145,7 +145,7 @@ class TDTestCase: #stream tdSql.execute("drop database if exists source_db;") - tdSql.query("create database source_db vgroups 3;") + tdSql.query("create database source_db vgroups 3 wal_retention_period 3600;") tdSql.query("use source_db") tdSql.query("create table if not exists source_db.stb (ts timestamp, k int) tags (a int);") tdSql.query("create table source_db.ct1 using source_db.stb tags(1000);create table source_db.ct2 using source_db.stb tags(2000);create table source_db.ct3 using source_db.stb tags(3000);") diff --git a/tests/system-test/0-others/telemetry.py b/tests/system-test/0-others/telemetry.py index bc5d276faa..3b6cb10509 100644 --- a/tests/system-test/0-others/telemetry.py +++ b/tests/system-test/0-others/telemetry.py @@ -181,7 +181,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() # time.sleep(2) - vgroups = "30" + vgroups = "4" sql = "create database db3 vgroups " + vgroups tdSql.query(sql) diff --git a/tests/system-test/0-others/testRoll.py b/tests/system-test/0-others/testRoll.py index 56e5b3630a..c5489146dc 100644 --- a/tests/system-test/0-others/testRoll.py +++ b/tests/system-test/0-others/testRoll.py @@ -13,7 +13,7 @@ def init_tmq_env(db, topic): conn.execute("drop topic if exists {}".format(topic)) conn.execute("drop database if exists {}".format(db)) - conn.execute("create database if not exists {} replica 1 ".format(db)) + conn.execute("create database if not exists {} replica 1 wal_retention_period 3600".format(db)) conn.select_db(db) conn.execute( "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))") @@ -37,7 +37,7 @@ def init_tmq_rest_env(db, topic): conn.execute("drop topic if exists {}".format(topic)) conn.execute("drop database if exists {}".format(db)) - conn.execute("create database if not exists {} replica 3 ".format(db)) + conn.execute("create database if not exists {} replica 3 wal_retention_period 3600".format(db)) conn.select_db(db) conn.execute( "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))") @@ -80,4 +80,4 @@ if __name__ == '__main__': val = res.value() for block in val: - print(block.fetchall()) \ No newline at end of file + print(block.fetchall()) diff --git a/tests/system-test/0-others/udfpy/af_count.py b/tests/system-test/0-others/udfpy/af_count.py new file mode 100644 index 0000000000..ce29abca13 --- /dev/null +++ b/tests/system-test/0-others/udfpy/af_count.py @@ -0,0 +1,23 @@ +import pickle + +def init(): + pass + +def destroy(): + pass + +def start(): + return pickle.dumps(0) + +def finish(buf): + count = pickle.loads(buf) + return count + +def reduce(datablock, buf): + (rows, cols) = datablock.shape() + count = pickle.loads(buf) + for i in range(rows): + val = datablock.data(i, 0) + if val is not None: + count += 1 + return pickle.dumps(count) \ No newline at end of file diff --git a/tests/system-test/0-others/udfpy/af_min.py b/tests/system-test/0-others/udfpy/af_min.py new file mode 100644 index 0000000000..9f1aadf414 --- /dev/null +++ b/tests/system-test/0-others/udfpy/af_min.py @@ -0,0 +1,30 @@ +import pickle + +def init(): + pass + +def destroy(): + pass + +def start(): + return pickle.dumps([]) + +def finish(buf): + mins = pickle.loads(buf) + min_val = None + for min in mins: + if min_val is None or (min is not None and min < min_val): + min_val = min + return min_val + +def reduce(datablock, buf): + (rows, cols) = datablock.shape() + mins = pickle.loads(buf) + min = None + for i in range(rows): + val = datablock.data(i, 0) + if min is None or (val is not None and val < min) : + min = val + if min is not None: + mins.append(min) + return pickle.dumps(mins) diff --git a/tests/system-test/0-others/udfpy/af_null.py b/tests/system-test/0-others/udfpy/af_null.py new file mode 100644 index 0000000000..230eac6888 --- /dev/null +++ b/tests/system-test/0-others/udfpy/af_null.py @@ -0,0 +1,19 @@ +import pickle + +def init(): + pass + +def destroy(): + pass + +def start(): + return pickle.dumps([]) + +def finish(buf): + return None + +def reduce(datablock, buf): + (rows, cols) = datablock.shape() + mins = pickle.loads(buf) + mins.append(None) + return pickle.dumps(mins) diff --git a/tests/system-test/0-others/udfpy/af_sum.py b/tests/system-test/0-others/udfpy/af_sum.py new file mode 100644 index 0000000000..8b88aba56c --- /dev/null +++ b/tests/system-test/0-others/udfpy/af_sum.py @@ -0,0 +1,26 @@ +import pickle + +def init(): + pass + +def destroy(): + pass + +def start(): + return pickle.dumps(None) + +def finish(buf): + sum = pickle.loads(buf) + return sum + +def reduce(datablock, buf): + (rows, cols) = datablock.shape() + sum = pickle.loads(buf) + for i in range(rows): + val = datablock.data(i, 0) + if val is not None: + if sum is None: + sum = val + else: + sum += val + return pickle.dumps(sum) diff --git a/tests/system-test/0-others/udfpy/sf_concat_nch.py b/tests/system-test/0-others/udfpy/sf_concat_nch.py new file mode 100644 index 0000000000..84d8eb2c96 --- /dev/null +++ b/tests/system-test/0-others/udfpy/sf_concat_nch.py @@ -0,0 +1,27 @@ +# init +def init(): + pass + +# destroy +def destroy(): + pass + +def process(block): + (nrows, ncols) = block.shape() + results = [] + for i in range(nrows): + row = [] + for j in range(ncols): + val = block.data(i, j) + if val is None: + row = None + break + row.append(val.decode('utf_32_le')) + if row is None: + results.append(None) + else: + row_str = ''.join(row) + results.append(row_str.encode('utf_32_le')) + return results + + diff --git a/tests/system-test/0-others/udfpy/sf_concat_var.py b/tests/system-test/0-others/udfpy/sf_concat_var.py new file mode 100644 index 0000000000..fc8292c718 --- /dev/null +++ b/tests/system-test/0-others/udfpy/sf_concat_var.py @@ -0,0 +1,26 @@ +# init +def init(): + pass + +# destroy +def destroy(): + pass + +def process(block): + (nrows, ncols) = block.shape() + results = [] + for i in range(nrows): + row = [] + for j in range(ncols): + val = block.data(i, j) + if val is None: + row = None + break + row.append(val.decode('utf-8')) + if row is None: + results.append(None) + else: + results.append(''.join(row)) + return results + + diff --git a/tests/system-test/0-others/udfpy/sf_multi_args.py b/tests/system-test/0-others/udfpy/sf_multi_args.py new file mode 100644 index 0000000000..1026661d8d --- /dev/null +++ b/tests/system-test/0-others/udfpy/sf_multi_args.py @@ -0,0 +1,23 @@ +# init +def init(): + pass + +# destroy +def destroy(): + pass + +# return origin column one value +def process(block): + (nrows, ncols) = block.shape() + results = [] + for i in range(nrows): + rows = [] + for j in range(ncols): + val = block.data(i, j) + if type(val) is bytes: + rows.append(val.decode('utf-8')) + else: + rows.append(repr(val)) + results.append(','.join(rows)) + return results + diff --git a/tests/system-test/0-others/udfpy/sf_null.py b/tests/system-test/0-others/udfpy/sf_null.py new file mode 100644 index 0000000000..c22ca95b19 --- /dev/null +++ b/tests/system-test/0-others/udfpy/sf_null.py @@ -0,0 +1,16 @@ + +# init +def init(): + pass + +# destroy +def destroy(): + pass + +# return origin column one value +def process(block): + (rows, cols) = block.shape() + results = [] + for i in range(rows): + results.append(None) + return results \ No newline at end of file diff --git a/tests/system-test/0-others/udfpy/sf_origin.py b/tests/system-test/0-others/udfpy/sf_origin.py new file mode 100644 index 0000000000..9158e044d2 --- /dev/null +++ b/tests/system-test/0-others/udfpy/sf_origin.py @@ -0,0 +1,15 @@ +# init +def init(): + pass + +# destroy +def destroy(): + pass + +# return origin column one value +def process(block): + (rows, cols) = block.shape() + results = [] + for i in range(rows): + results.append(block.data(i,0)) + return results diff --git a/tests/system-test/0-others/udfpy_main.py b/tests/system-test/0-others/udfpy_main.py new file mode 100644 index 0000000000..916b032edb --- /dev/null +++ b/tests/system-test/0-others/udfpy_main.py @@ -0,0 +1,464 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +import random +import os +import subprocess + + +class PerfDB: + def __init__(self): + self.sqls = [] + self.spends = [] + + # execute + def execute(self, sql): + print(f" perfdb execute {sql}") + stime = time.time() + ret = tdSql.execute(sql, 1) + spend = time.time() - stime + + self.sqls.append(sql) + self.spends.append(spend) + return ret + + # query + def query(self, sql): + print(f" perfdb query {sql}") + start = time.time() + ret = tdSql.query(sql, None, 1) + spend = time.time() - start + self.sqls.append(sql) + self.spends.append(spend) + return ret + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + + # udf path + self.udf_path = os.path.dirname(os.path.realpath(__file__)) + "/udfpy" + + + self.column_dict = { + 'ts': 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': 'varchar(120)', + 'col13': 'nchar(100)', + } + self.tag_dict = { + 't1': 'tinyint', + 't2': 'smallint', + 't3': 'int', + 't4': 'bigint', + 't5': 'tinyint unsigned', + 't6': 'smallint unsigned', + 't7': 'int unsigned', + 't8': 'bigint unsigned', + 't9': 'float', + 't10': 'double', + 't11': 'bool', + 't12': 'varchar(120)', + 't13': 'nchar(100)', + } + + def set_stb_sql(self,stbname,column_dict,tag_dict): + column_sql = '' + tag_sql = '' + for k,v in column_dict.items(): + column_sql += f"{k} {v}, " + for k,v in tag_dict.items(): + tag_sql += f"{k} {v}, " + create_stb_sql = f'create stable {stbname} ({column_sql[:-2]}) tags ({tag_sql[:-2]})' + return create_stb_sql + + # create stable and child tables + def create_table(self, stbname, tbname, count): + tdSql.execute("create database db wal_retention_period 4") + tdSql.execute('use db') + self.child_count = count + self.stbname = stbname + self.tbname = tbname + + # create stable + create_table_sql = self.set_stb_sql(stbname, self.column_dict, self.tag_dict) + tdSql.execute(create_table_sql) + + batch_size = 1000 + # create child table + for i in range(count): + ti = i % 128 + tags = f'{ti},{ti},{i},{i},{ti},{ti},{i},{i},{i}.000{i},{i}.000{i},true,"var{i}","nch{i}"' + sql = f'create table {tbname}{i} using {stbname} tags({tags});' + tdSql.execute(sql) + if i % batch_size == 0: + tdLog.info(f" create child table {i} ...") + + tdLog.info(f" create {count} child tables ok.") + + # create with dicts + def create_sf_dicts(self, dicts, filename): + for fun_name, out_type in dicts.items(): + sql = f' create function {fun_name} as "{self.udf_path}/{filename}" outputtype {out_type} language "Python" ' + tdSql.execute(sql) + tdLog.info(sql) + + # create_udfpy_function + def create_scalar_udfpy(self): + # scalar funciton + self.scalar_funs = { + 'sf0': 'timestamp', + 'sf1': 'tinyint', + 'sf2': 'smallint', + 'sf3': 'int', + 'sf4': 'bigint', + 'sf5': 'tinyint unsigned', + 'sf6': 'smallint unsigned', + 'sf7': 'int unsigned', + 'sf8': 'bigint unsigned', + 'sf9': 'float', + 'sf10': 'double', + 'sf11': 'bool', + 'sf12': 'varchar(120)', + 'sf13': 'nchar(100)' + } + # agg function + self.agg_funs = { + 'af1': 'tinyint', + 'af2': 'smallint', + 'af3': 'int', + 'af4': 'bigint', + 'af5': 'tinyint unsigned', + 'af6': 'smallint unsigned', + 'af7': 'int unsigned', + 'af8': 'bigint unsigned', + 'af9': 'float', + 'af10': 'double', + 'af11': 'bool', + 'af12': 'varchar(120)', + 'af13': 'nchar(100)', + 'af14': 'timestamp' + } + + # multi_args + self.create_sf_dicts(self.scalar_funs, "sf_origin.py") + fun_name = "sf_multi_args" + self.create_udf_sf(fun_name, f'{fun_name}.py', "binary(1024)") + + # all type check null + for col_name, col_type in self.column_dict.items(): + self.create_udf_sf(f"sf_null_{col_name}", "sf_null.py", col_type) + + # concat + fun_name = "sf_concat_var" + self.create_udf_sf(fun_name, f'{fun_name}.py', "varchar(1024)") + fun_name = "sf_concat_nch" + self.create_udf_sf(fun_name, f'{fun_name}.py', "nchar(1024)") + + + # fun_name == fun_name.py + def create_udf_sf(self, fun_name, file_name, out_type): + sql = f'create function {fun_name} as "{self.udf_path}/{file_name}" outputtype {out_type} language "Python" ' + tdSql.execute(sql) + tdLog.info(sql) + + def create_udf_af(self, fun_name, file_name, out_type, bufsize): + sql = f'create aggregate function {fun_name} as "{self.udf_path}/{file_name}" outputtype {out_type} bufsize {bufsize} language "Python" ' + tdSql.execute(sql) + tdLog.info(sql) + + + # sql1 query result eual with sql2 + def verify_same_result(self, sql1, sql2): + # query + result1 = tdSql.getResult(sql1) + tdSql.query(sql2) + + for i, row in enumerate(result1): + for j , val in enumerate(row): + tdSql.checkData(i, j, result1[i][j]) + + # same value like select col1, udf_fun1(col1) from st + def verify_same_value(self, sql, col=0): + tdSql.query(sql) + nrows = tdSql.getRows() + for i in range(nrows): + val = tdSql.getData(i, col) + tdSql.checkData(i, col + 1, val) + + # verify multi values + def verify_same_multi_values(self, sql): + tdSql.query(sql) + nrows = tdSql.getRows() + for i in range(nrows): + udf_val = tdSql.getData(i, 0) + vals = udf_val.split(',') + for j,val in enumerate(vals, 1): + tdSql.checkData(i, j, val) + + # query multi-args + def query_multi_args(self): + cols = list(self.column_dict.keys()) + list(self.tag_dict.keys()) + cols.remove("col13") + cols.remove("t13") + cols.remove("ts") + ncols = len(cols) + print(cols) + for i in range(2, ncols): + sample = random.sample(cols, i) + print(sample) + cols_name = ','.join(sample) + sql = f'select sf_multi_args({cols_name}),{cols_name} from {self.stbname} limit 10' + self.verify_same_multi_values(sql) + tdLog.info(sql) + + + # query_udfpy + def query_scalar_udfpy(self): + # col + for col_name, col_type in self.column_dict.items(): + for fun_name, out_type in self.scalar_funs.items(): + if col_type == out_type : + sql = f'select {col_name}, {fun_name}({col_name}) from {self.stbname} limit 10' + tdLog.info(sql) + self.verify_same_value(sql) + sql = f'select * from (select {col_name} as a, {fun_name}({col_name}) as b from {self.stbname} limit 100) order by b,a desc' + tdLog.info(sql) + self.verify_same_value(sql) + + # multi-args + self.query_multi_args() + + # all type check null + for col_name, col_type in self.column_dict.items(): + fun_name = f"sf_null_{col_name}" + sql = f'select {fun_name}({col_name}) from {self.stbname}' + tdSql.query(sql) + if col_type != "timestamp": + tdSql.checkData(0, 0, "None") + else: + val = tdSql.getData(0, 0) + if val is not None: + tdLog.exit(f" check {sql} not expect None.") + + # concat + sql = f'select sf_concat_var(col12, t12), concat(col12, t12) from {self.stbname} limit 1000' + self.verify_same_value(sql) + sql = f'select sf_concat_nch(col13, t13), concat(col13, t13) from {self.stbname} limit 1000' + self.verify_same_value(sql) + + # create aggregate + def create_aggr_udfpy(self): + + bufsize = 200 * 1024 + # all type check null + for col_name, col_type in self.column_dict.items(): + self.create_udf_af(f"af_null_{col_name}", "af_null.py", col_type, bufsize) + + # min + file_name = "af_min.py" + fun_name = "af_min_float" + self.create_udf_af(fun_name, file_name, f"float", bufsize) + fun_name = "af_min_int" + self.create_udf_af(fun_name, file_name, f"int", bufsize) + + # sum + file_name = "af_sum.py" + fun_name = "af_sum_float" + self.create_udf_af(fun_name, file_name, f"float", bufsize) + fun_name = "af_sum_int" + self.create_udf_af(fun_name, file_name, f"int", bufsize) + fun_name = "af_sum_bigint" + self.create_udf_af(fun_name, file_name, f"bigint", bufsize) + + # count + file_name = "af_count.py" + fun_name = "af_count_float" + self.create_udf_af(fun_name, file_name, f"float", bufsize) + fun_name = "af_count_int" + self.create_udf_af(fun_name, file_name, f"int", bufsize) + fun_name = "af_count_bigint" + self.create_udf_af(fun_name, file_name, f"bigint", bufsize) + + + # query aggregate + def query_aggr_udfpy(self) : + # all type check null + for col_name, col_type in self.column_dict.items(): + fun_name = f"af_null_{col_name}" + sql = f'select {fun_name}({col_name}) from {self.stbname}' + tdSql.query(sql) + if col_type != "timestamp": + tdSql.checkData(0, 0, "None") + else: + val = tdSql.getData(0, 0) + if val is not None: + tdLog.exit(f" check {sql} not expect None.") + + # min + sql = f'select min(col3), af_min_int(col3) from {self.stbname}' + self.verify_same_value(sql) + sql = f'select min(col7), af_min_int(col7) from {self.stbname}' + self.verify_same_value(sql) + sql = f'select min(col9), af_min_float(col9) from {self.stbname}' + self.verify_same_value(sql) + + # sum + sql = f'select sum(col1), af_sum_int(col1) from d0' + self.verify_same_value(sql) + sql = f'select sum(col3), af_sum_bigint(col3) from {self.stbname}' + self.verify_same_value(sql) + sql = f'select sum(col9), af_sum_float(col9) from {self.stbname}' + self.verify_same_value(sql) + + # count + sql = f'select count(col1), af_count_int(col1) from {self.stbname}' + self.verify_same_value(sql) + sql = f'select count(col7), af_count_bigint(col7) from {self.stbname}' + self.verify_same_value(sql) + sql = f'select count(col8), af_count_float(col8) from {self.stbname}' + self.verify_same_value(sql) + + # nest + sql = f'select a+1000,b+1000 from (select count(col8) as a, af_count_float(col8) as b from {self.stbname})' + self.verify_same_value(sql) + # group by + sql = f'select a+1000,b+1000 from (select count(col8) as a, af_count_float(col8) as b from {self.stbname} group by tbname)' + self.verify_same_value(sql) + # two filed expr + sql = f'select sum(col1+col2),af_sum_float(col1+col2) from {self.stbname};' + self.verify_same_value(sql) + # interval + sql = f'select af_sum_float(col2+col3),sum(col3+col2) from {self.stbname} interval(1s)' + self.verify_same_value(sql) + + + # insert to child table d1 data + def insert_data(self, tbname, rows): + ts = 1670000000000 + values = "" + batch_size = 500 + child_name = "" + for i in range(self.child_count): + for j in range(rows): + tj = j % 128 + cols = f'{tj},{tj},{j},{j},{tj},{tj},{j},{j},{j}.000{j},{j}.000{j},true,"var{j}","nch{j}涛思数据codepage is utf_32_le"' + value = f'({ts+j},{cols})' + if values == "": + values = value + else: + values += f",{value}" + if j % batch_size == 0 or j + 1 == rows: + sql = f'insert into {tbname}{i} values {values};' + tdSql.execute(sql) + tdLog.info(f" child table={i} rows={j} insert data.") + values = "" + + # partial columns upate + sql = f'insert into {tbname}0(ts, col1, col9, col11) values(now, 100, 200, 0)' + tdSql.execute(sql) + sql = f'insert into {tbname}0(ts, col2, col5, col8) values(now, 100, 200, 300)' + tdSql.execute(sql) + sql = f'insert into {tbname}0(ts, col3, col7, col13) values(now, null, null, null)' + tdSql.execute(sql) + sql = f'insert into {tbname}0(ts) values(now)' + tdSql.execute(sql) + tdLog.info(f" insert {rows} to child table {self.child_count} .") + + + # create stream + def create_stream(self): + sql = f"create stream ma into sta subtable(concat('sta_',tbname)) \ + as select _wstart,count(col1),af_count_bigint(col1) from {self.stbname} partition by tbname interval(1s);" + tdSql.execute(sql) + tdLog.info(sql) + + # query stream + def verify_stream(self): + sql = f"select * from sta limit 10" + self.verify_same_value(sql, 1) + + # create tmq + def create_tmq(self): + sql = f"create topic topa as select concat(col12,t12),sf_concat_var(col12,t12) from {self.stbname};" + tdSql.execute(sql) + tdLog.info(sql) + + def install_taospy(self): + tdLog.info("install taospyudf...") + packs = ["taospyudf"] + for pack in packs: + subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-i', 'https://pypi.org/simple', '-U', pack]) + tdLog.info("call ldconfig...") + os.system("ldconfig") + tdLog.info("install taospyudf successfully.") + + # run + def run(self): + self.install_taospy() + + # var + stable = "meters" + tbname = "d" + count = 10 + rows = 5000 + # do + self.create_table(stable, tbname, count) + + # create + self.create_scalar_udfpy() + self.create_aggr_udfpy() + + # create stream + self.create_stream() + + # create tmq + self.create_tmq() + + # insert data + self.insert_data(tbname, rows) + + # query + self.query_scalar_udfpy() + self.query_aggr_udfpy() + + # show performance + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/user_manage.py b/tests/system-test/0-others/user_manage.py index 6f90a2873a..1e33d4bb1c 100644 --- a/tests/system-test/0-others/user_manage.py +++ b/tests/system-test/0-others/user_manage.py @@ -115,6 +115,7 @@ class TDTestCase: jiacy0_read_conn = taos.connect(user='jiacy0_read', password='123') jiacy0_write_conn = taos.connect(user='jiacy0_write', password='123') jiacy0_none_conn = taos.connect(user='jiacy0_none', password='123') + tdSql.execute('alter database db wal_retention_period 3600') tdSql.execute('create topic root_db as select * from db.stb') for user in [jiacy1_all_conn, jiacy1_read_conn, jiacy0_all_conn, jiacy0_read_conn]: user.execute(f'create topic db_jiacy as select * from db.stb') diff --git a/tests/system-test/0-others/walFileIdex.py b/tests/system-test/0-others/walFileIdex.py index cd34c7e5e3..f8309519cd 100644 --- a/tests/system-test/0-others/walFileIdex.py +++ b/tests/system-test/0-others/walFileIdex.py @@ -58,7 +58,7 @@ class TDTestCase: #stream tdSql.execute("drop database if exists source_db;") - tdSql.query("create database source_db vgroups 3;") + tdSql.query("create database source_db vgroups 3 wal_retention_period 3600;") tdSql.query("use source_db") tdSql.query("create table if not exists source_db.stb (ts timestamp, k int) tags (a int);") tdSql.query("create table source_db.ct1 using source_db.stb tags(1000);create table source_db.ct2 using source_db.stb tags(2000);create table source_db.ct3 using source_db.stb tags(3000);") diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py index a8bfea2741..9954b3557e 100644 --- a/tests/system-test/1-insert/drop.py +++ b/tests/system-test/1-insert/drop.py @@ -54,7 +54,7 @@ class TDTestCase: insert_list = [] self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) def drop_ntb_check(self): - tdSql.execute(f'create database if not exists {self.dbname} replica {self.replicaVar}') + tdSql.execute(f'create database if not exists {self.dbname} replica {self.replicaVar} wal_retention_period 3600') tdSql.execute(f'use {self.dbname}') tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) self.insert_data(self.column_dict,self.ntbname,self.rowNum) @@ -80,7 +80,7 @@ class TDTestCase: tag_values = [ f'1' ] - tdSql.execute(f"create database if not exists {self.dbname} replica {self.replicaVar}") + tdSql.execute(f"create database if not exists {self.dbname} replica {self.replicaVar} wal_retention_period 3600") tdSql.execute(f'use {self.dbname}') tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict)) for i in range(self.tbnum): @@ -116,7 +116,7 @@ class TDTestCase: tdSql.checkRows(self.tbnum) tdSql.execute(f'drop database {self.dbname}') def drop_topic_check(self): - tdSql.execute(f'create database {self.dbname} replica {self.replicaVar}') + tdSql.execute(f'create database {self.dbname} replica {self.replicaVar} wal_retention_period 3600') tdSql.execute(f'use {self.dbname}') stbname = tdCom.getLongName(5,"letters") topic_name = tdCom.getLongName(5,"letters") @@ -132,7 +132,7 @@ class TDTestCase: tdSql.execute(f'drop database {self.dbname}') def drop_stream_check(self): - tdSql.execute(f'create database {self.dbname} replica 1') + tdSql.execute(f'create database {self.dbname} replica 1 wal_retention_period 3600') tdSql.execute(f'use {self.dbname}') stbname = tdCom.getLongName(5,"letters") stream_name = tdCom.getLongName(5,"letters") diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py index de434eed7d..ede1f28324 100644 --- a/tests/system-test/2-query/cast.py +++ b/tests/system-test/2-query/cast.py @@ -38,11 +38,8 @@ class TDTestCase: if data_tb_col[i] is None: tdSql.checkData( i, 0 , None ) if col_name not in ["c2", "double"] or tbname != f"{self.dbname}.t1" or i != 10: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_tb_col[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = datetime.datetime.fromtimestamp(data_tb_col[i]/1000) + tdSql.checkData( i, 0, date_init_stamp) def __range_to_timestamp(self, cols, tables): for col in cols: @@ -60,7 +57,7 @@ class TDTestCase: self.__range_to_timestamp(cols=__col_list, tables=__table_list) def all_test(self): - + _datetime_epoch = datetime.datetime.fromtimestamp(0) tdSql.query(f"select c1 from {self.dbname}.ct4") data_ct4_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] tdSql.query(f"select c1 from {self.dbname}.t1") @@ -99,22 +96,16 @@ class TDTestCase: if data_ct4_c1[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c1[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_ct4_c1[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select cast(c1 as timestamp) as b from {self.dbname}.t1") for i in range(len(data_t1_c1)): if data_t1_c1[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c1[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_t1_c1[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdLog.printNoPrefix("==========step8: cast bigint to bigint, expect no changes") @@ -156,11 +147,8 @@ class TDTestCase: if data_ct4_c2[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c2[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_ct4_c2[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select cast(c2 as timestamp) as b from {self.dbname}.t1") @@ -170,11 +158,8 @@ class TDTestCase: elif i == 10: continue else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c2[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_t1_c2[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdLog.printNoPrefix("==========step12: cast smallint to bigint, expect no changes") @@ -216,22 +201,16 @@ class TDTestCase: if data_ct4_c3[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c3[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_ct4_c3[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select cast(c3 as timestamp) as b from {self.dbname}.t1") for i in range(len(data_t1_c3)): if data_t1_c3[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c3[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_t1_c3[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdLog.printNoPrefix("==========step16: cast tinyint to bigint, expect no changes") @@ -273,22 +252,16 @@ class TDTestCase: if data_ct4_c4[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c4[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_ct4_c4[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select cast(c4 as timestamp) as b from {self.dbname}.t1") for i in range(len(data_t1_c4)): if data_t1_c4[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c4[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_t1_c4[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdLog.printNoPrefix("==========step20: cast float to bigint, expect no changes") @@ -326,21 +299,15 @@ class TDTestCase: if data_ct4_c5[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c5[i])/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_ct4_c5[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select cast(c5 as timestamp) as b from {self.dbname}.t1") for i in range(len(data_t1_c5)): if data_t1_c5[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c5[i])/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_t1_c5[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdLog.printNoPrefix("==========step24: cast double to bigint, expect no changes") tdSql.query(f"select c6 from {self.dbname}.ct4") @@ -382,11 +349,8 @@ class TDTestCase: if data_ct4_c6[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c6[i])/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_ct4_c6[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select cast(c6 as timestamp) as b from {self.dbname}.t1") for i in range(len(data_t1_c6)): @@ -395,11 +359,8 @@ class TDTestCase: elif i == 10: continue else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c6[i])/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_t1_c6[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdLog.printNoPrefix("==========step28: cast bool to bigint, expect no changes") tdSql.query(f"select c7 from {self.dbname}.ct4") @@ -436,21 +397,15 @@ class TDTestCase: if data_ct4_c7[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c7[i])/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_ct4_c7[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select cast(c7 as timestamp) as b from {self.dbname}.t1") for i in range(len(data_t1_c7)): if data_t1_c7[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c7[i])/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_t1_c7[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select c8 from {self.dbname}.ct4") @@ -694,7 +649,7 @@ class TDTestCase: tdSql.query(f"select cast('123' as tinyint unsigned) as b from {self.dbname}.stb1 partition by tbname") # uion with cast and common cols - + tdSql.query(f"select cast(c2 as int) as b from {self.dbname}.stb1 union all select c1 from {self.dbname}.stb1 ") tdSql.query(f"select cast(c3 as bool) as b from {self.dbname}.stb1 union all select c7 from {self.dbname}.ct1 ") tdSql.query(f"select cast(c4 as tinyint) as b from {self.dbname}.stb1 union all select c4 from {self.dbname}.stb1") diff --git a/tests/system-test/2-query/odbc.py b/tests/system-test/2-query/odbc.py index 9bbff4af21..f9232dddf8 100644 --- a/tests/system-test/2-query/odbc.py +++ b/tests/system-test/2-query/odbc.py @@ -22,7 +22,7 @@ class TDTestCase: tdSql.execute("insert into db.ctb using db.stb tags(1) (ts, c1) values (now, 1)") tdSql.query("select count(*) from information_schema.ins_columns") - tdSql.checkData(0, 0, 272) + tdSql.checkData(0, 0, 274) tdSql.query("select * from information_schema.ins_columns where table_name = 'ntb'") tdSql.checkRows(14) diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index 69cf378da3..080b431ffe 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -57,7 +57,7 @@ class TDTestCase: return cur def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -149,6 +149,7 @@ class TDTestCase: topicFromStb = 'topic_stb_column' topicFromCtb = 'topic_ctb_column' + tdSql.execute("alter database %s wal_retention_period 3600" % (parameterDict['dbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName'])) diff --git a/tests/system-test/7-tmq/create_wrong_topic.py b/tests/system-test/7-tmq/create_wrong_topic.py index 39d519fec1..77d43149cd 100644 --- a/tests/system-test/7-tmq/create_wrong_topic.py +++ b/tests/system-test/7-tmq/create_wrong_topic.py @@ -44,6 +44,7 @@ class TDTestCase: def wrong_topic(self): tdSql.prepare() tdSql.execute('use db') + tdSql.execute('alter database db wal_retention_period 3600') stbname = f'db.{tdCom.getLongName(5, "letters")}' tag_dict = { 't0':'int' @@ -75,4 +76,4 @@ class TDTestCase: tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py index 808a4935e3..44f58279be 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py @@ -67,6 +67,7 @@ class TDTestCase: tdLog.info("flush db to let data falls into the disk") tdSql.query("flush database %s"%(paraDict['dbName'])) + tdSql.execute("alter database %s wal_retention_period 3600"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal.py b/tests/system-test/7-tmq/dataFromTsdbNWal.py index 8386c22cd0..0f4f1694c1 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal.py @@ -67,6 +67,7 @@ class TDTestCase: tdLog.info("flush db to let data falls into the disk") tdSql.query("flush database %s"%(paraDict['dbName'])) + tdSql.execute("alter database %s wal_retention_period 3600"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/db.py b/tests/system-test/7-tmq/db.py index 04f5aac559..e0d1e2c5b6 100644 --- a/tests/system-test/7-tmq/db.py +++ b/tests/system-test/7-tmq/db.py @@ -60,7 +60,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") tdSql.query("drop database if exists %s "%(cdbName)) - tdSql.query("create database %s vgroups 1"%(cdbName)) + tdSql.query("create database %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) diff --git a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py index 4371a909c2..fa22cad726 100644 --- a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py +++ b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py @@ -134,6 +134,7 @@ class TDTestCase: paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl + tdSql.execute("alter database dbt wal_retention_period 3600") tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha' "%(paraDict['dbName'], paraDict['stbName']) diff --git a/tests/system-test/7-tmq/schema.py b/tests/system-test/7-tmq/schema.py index 04224fbc7e..95c1839964 100644 --- a/tests/system-test/7-tmq/schema.py +++ b/tests/system-test/7-tmq/schema.py @@ -60,7 +60,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") tdSql.query("drop database if exists %s "%(cdbName)) - tdSql.query("create database %s vgroups 1"%(cdbName)) + tdSql.query("create database %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -115,7 +115,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/7-tmq/stbFilter.py b/tests/system-test/7-tmq/stbFilter.py index 6b48a6d570..3f862ae047 100644 --- a/tests/system-test/7-tmq/stbFilter.py +++ b/tests/system-test/7-tmq/stbFilter.py @@ -45,6 +45,7 @@ class TDTestCase: tmqCom.initConsumerTable() tmqCom.create_database(tsql=tdSql, dbName=paraDict["dbName"],dropFlag=paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=paraDict['replica']) + tdSql.execute("alter database %s wal_retention_period 3600"%(paraDict["dbName"])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/stbTagFilter-1ctb.py b/tests/system-test/7-tmq/stbTagFilter-1ctb.py index c4a7c8cae5..7ee5fce5a8 100644 --- a/tests/system-test/7-tmq/stbTagFilter-1ctb.py +++ b/tests/system-test/7-tmq/stbTagFilter-1ctb.py @@ -106,6 +106,7 @@ class TDTestCase: # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + tdSql.execute("alter database dbt wal_retention_period 3600") tdLog.info("create topics from stb1") topicFromStb1 = 'topic_UpperCase_stb1' # queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) diff --git a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py index c380d201b2..71b7fdef5d 100644 --- a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py +++ b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py index 0fa9bcfbd4..9f01f20470 100644 --- a/tests/system-test/7-tmq/subscribeDb.py +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -52,7 +52,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -99,7 +99,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -180,7 +180,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -278,7 +278,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -345,7 +345,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) tdSql.execute("create table if not exists %s.%s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%(parameterDict['dbName'], parameterDict['stbName'])) tdLog.info("create topics from db") @@ -415,7 +415,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/7-tmq/subscribeDb0.py b/tests/system-test/7-tmq/subscribeDb0.py index 50ef52cb15..ed13fcbe06 100644 --- a/tests/system-test/7-tmq/subscribeDb0.py +++ b/tests/system-test/7-tmq/subscribeDb0.py @@ -52,7 +52,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -99,7 +99,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -180,7 +180,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -262,7 +262,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/7-tmq/subscribeDb1.py b/tests/system-test/7-tmq/subscribeDb1.py index 89bf337a20..605e862950 100644 --- a/tests/system-test/7-tmq/subscribeDb1.py +++ b/tests/system-test/7-tmq/subscribeDb1.py @@ -52,7 +52,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -99,7 +99,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -180,8 +180,8 @@ class TDTestCase: parameterDict['cfg'] = cfgPath self.initConsumerTable() - tdLog.info("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdLog.info("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -197,7 +197,7 @@ class TDTestCase: 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict2['dbName'], parameterDict2['vgroups'], parameterDict2['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict2['dbName'], parameterDict2['vgroups'], parameterDict2['replica'])) prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) prepareEnvThread2.start() @@ -268,7 +268,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -284,7 +284,7 @@ class TDTestCase: 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict2['dbName'], parameterDict2['vgroups'], parameterDict2['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict2['dbName'], parameterDict2['vgroups'], parameterDict2['replica'])) prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) prepareEnvThread2.start() diff --git a/tests/system-test/7-tmq/subscribeDb2.py b/tests/system-test/7-tmq/subscribeDb2.py index d045842e45..60cde8d7f0 100644 --- a/tests/system-test/7-tmq/subscribeDb2.py +++ b/tests/system-test/7-tmq/subscribeDb2.py @@ -53,7 +53,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -100,7 +100,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -185,7 +185,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -263,7 +263,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/7-tmq/subscribeDb3.py b/tests/system-test/7-tmq/subscribeDb3.py index 747ea3ba86..f82f89f1b9 100644 --- a/tests/system-test/7-tmq/subscribeDb3.py +++ b/tests/system-test/7-tmq/subscribeDb3.py @@ -52,7 +52,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) tdSql.query("drop table if exists %s.notifyinfo "%(cdbName)) @@ -122,7 +122,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -203,7 +203,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -280,7 +280,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/7-tmq/subscribeDb4.py b/tests/system-test/7-tmq/subscribeDb4.py index c14d3b27b1..764362c708 100644 --- a/tests/system-test/7-tmq/subscribeDb4.py +++ b/tests/system-test/7-tmq/subscribeDb4.py @@ -65,6 +65,7 @@ class TDTestCase: tmqCom.initConsumerTable(self.cdbName) tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"]) + tdSql.execute("alter database %s wal_retention_period 3600" % (self.paraDict['dbName'])) self.paraDict["stbName"] = 'stb1' tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) diff --git a/tests/system-test/7-tmq/subscribeStb.py b/tests/system-test/7-tmq/subscribeStb.py index 3ff0b25ff6..c8b66adfa2 100644 --- a/tests/system-test/7-tmq/subscribeStb.py +++ b/tests/system-test/7-tmq/subscribeStb.py @@ -59,7 +59,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -114,7 +114,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/7-tmq/subscribeStb0.py b/tests/system-test/7-tmq/subscribeStb0.py index 1463cad627..717cf05bdc 100644 --- a/tests/system-test/7-tmq/subscribeStb0.py +++ b/tests/system-test/7-tmq/subscribeStb0.py @@ -59,7 +59,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -114,7 +114,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/7-tmq/subscribeStb1.py b/tests/system-test/7-tmq/subscribeStb1.py index edbe1bc3c6..67dee363b3 100644 --- a/tests/system-test/7-tmq/subscribeStb1.py +++ b/tests/system-test/7-tmq/subscribeStb1.py @@ -59,7 +59,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -114,7 +114,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/7-tmq/subscribeStb2.py b/tests/system-test/7-tmq/subscribeStb2.py index 6c3e122902..422cb23ffd 100644 --- a/tests/system-test/7-tmq/subscribeStb2.py +++ b/tests/system-test/7-tmq/subscribeStb2.py @@ -59,7 +59,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -114,7 +114,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/7-tmq/subscribeStb3.py b/tests/system-test/7-tmq/subscribeStb3.py index 025f403282..7205e84620 100644 --- a/tests/system-test/7-tmq/subscribeStb3.py +++ b/tests/system-test/7-tmq/subscribeStb3.py @@ -59,7 +59,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -114,7 +114,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/7-tmq/subscribeStb4.py b/tests/system-test/7-tmq/subscribeStb4.py index 6aa3da66a4..bb8afcf14e 100644 --- a/tests/system-test/7-tmq/subscribeStb4.py +++ b/tests/system-test/7-tmq/subscribeStb4.py @@ -59,7 +59,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -114,7 +114,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py index cdcdadbcbb..6f556382bb 100644 --- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -200,6 +200,7 @@ class TDTestCase: tdLog.info("async insert data") pThread = tmqCom.asyncInsertData(paraDict) + tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create topics from stb with filter") # queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) diff --git a/tests/system-test/7-tmq/tmqAlterSchema.py b/tests/system-test/7-tmq/tmqAlterSchema.py index a70678219f..1a8b0693b8 100644 --- a/tests/system-test/7-tmq/tmqAlterSchema.py +++ b/tests/system-test/7-tmq/tmqAlterSchema.py @@ -65,6 +65,7 @@ class TDTestCase: queryStringList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") @@ -175,6 +176,7 @@ class TDTestCase: queryStringList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ntb") diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index 41073d83ae..5d0af636b2 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1, wal_retention_size=-1,wal_retention_period=-1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) # tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqCheckData.py b/tests/system-test/7-tmq/tmqCheckData.py index 04d0744ab5..a9671241a9 100644 --- a/tests/system-test/7-tmq/tmqCheckData.py +++ b/tests/system-test/7-tmq/tmqCheckData.py @@ -80,6 +80,7 @@ class TDTestCase: tdLog.info("insert data") tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create topics from stb with filter") queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as %s" %(topicNameList[0], queryString) diff --git a/tests/system-test/7-tmq/tmqCheckData1.py b/tests/system-test/7-tmq/tmqCheckData1.py index b9dac62833..e06c29c5a2 100644 --- a/tests/system-test/7-tmq/tmqCheckData1.py +++ b/tests/system-test/7-tmq/tmqCheckData1.py @@ -73,6 +73,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py index f372a2b742..3ad1d097e1 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py index c7f95f6f41..fdd683d08d 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py index 26dacf514d..f05f0abeff 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py index d6f100041b..75b49a34fc 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index 87832ac0ef..2862bcb09b 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index 8ed4a6df97..cca29c178d 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py index 6a03f0f751..00d2491c97 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py index c11159c6e5..3b1dbae443 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py index 37946d0c22..a799fa5719 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py index 439845aa54..f0bedbb187 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py index 53ff020b08..a63927dd8b 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1.py b/tests/system-test/7-tmq/tmqConsFromTsdb1.py index 4bb6cf463f..8fcc991d4e 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsumerGroup.py b/tests/system-test/7-tmq/tmqConsumerGroup.py index 02093a2d88..b1aef9d762 100644 --- a/tests/system-test/7-tmq/tmqConsumerGroup.py +++ b/tests/system-test/7-tmq/tmqConsumerGroup.py @@ -73,6 +73,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqDelete-1ctb.py b/tests/system-test/7-tmq/tmqDelete-1ctb.py index b09efdd1e6..6a62247541 100644 --- a/tests/system-test/7-tmq/tmqDelete-1ctb.py +++ b/tests/system-test/7-tmq/tmqDelete-1ctb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1,wal_retention_size=-1, wal_retention_period=-1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqDelete-multiCtb.py b/tests/system-test/7-tmq/tmqDelete-multiCtb.py index e59e3d6ecd..7a47cd6025 100644 --- a/tests/system-test/7-tmq/tmqDelete-multiCtb.py +++ b/tests/system-test/7-tmq/tmqDelete-multiCtb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar,wal_retention_size=-1, wal_retention_period=-1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqDnode.py b/tests/system-test/7-tmq/tmqDnode.py index e1d6d91e2d..b96ca24ac5 100644 --- a/tests/system-test/7-tmq/tmqDnode.py +++ b/tests/system-test/7-tmq/tmqDnode.py @@ -110,7 +110,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index a44ff916e5..3ad7d7692d 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -55,6 +55,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1,wal_retention_size=-1, wal_retention_period=-1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -145,7 +146,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - tdSql.query(queryString) + tdSql.query(queryString, None, 50) totalRowsFromQury = tdSql.getRows() tdLog.info("act consume rows: %d, act query rows: %d"%(totalConsumeRows, totalRowsFromQury)) @@ -186,6 +187,7 @@ class TDTestCase: tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) # tdLog.info("create ctb") @@ -234,7 +236,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - tdSql.query(queryString) + tdSql.query(queryString, None, 50) totalRowsFromQuery = tdSql.getRows() tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, totalRowsFromQuery)) diff --git a/tests/system-test/7-tmq/tmqDnodeRestart1.py b/tests/system-test/7-tmq/tmqDnodeRestart1.py index 982cc0a631..cff55a1239 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart1.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart1.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1,wal_retention_size=-1, wal_retention_period=-1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py index 6c49fae299..198a5ed6df 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py @@ -57,6 +57,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") @@ -143,6 +144,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py index 3fc5a2fdc7..0b9cb7e66a 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py @@ -57,6 +57,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") @@ -143,6 +144,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") diff --git a/tests/system-test/7-tmq/tmqDropStb.py b/tests/system-test/7-tmq/tmqDropStb.py index 3789632984..a94747e574 100644 --- a/tests/system-test/7-tmq/tmqDropStb.py +++ b/tests/system-test/7-tmq/tmqDropStb.py @@ -64,6 +64,7 @@ class TDTestCase: tmqCom.initConsumerTable(self.cdbName) tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"]) + tdSql.execute("alter database %s wal_retention_period 3600" % (self.paraDict['dbName'])) self.paraDict["stbName"] = 'stb1' tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) diff --git a/tests/system-test/7-tmq/tmqDropStbCtb.py b/tests/system-test/7-tmq/tmqDropStbCtb.py index c9e34136cc..587baf12aa 100644 --- a/tests/system-test/7-tmq/tmqDropStbCtb.py +++ b/tests/system-test/7-tmq/tmqDropStbCtb.py @@ -54,6 +54,7 @@ class TDTestCase: # tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqError.py b/tests/system-test/7-tmq/tmqError.py index a39bac8dd1..b2038f1644 100644 --- a/tests/system-test/7-tmq/tmqError.py +++ b/tests/system-test/7-tmq/tmqError.py @@ -116,7 +116,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/7-tmq/tmqModule.py b/tests/system-test/7-tmq/tmqModule.py index d6b4aff938..187152c9ce 100644 --- a/tests/system-test/7-tmq/tmqModule.py +++ b/tests/system-test/7-tmq/tmqModule.py @@ -110,7 +110,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py index 31ddc1b0f8..e9234f6c7a 100644 --- a/tests/system-test/7-tmq/tmqShow.py +++ b/tests/system-test/7-tmq/tmqShow.py @@ -51,6 +51,7 @@ class TDTestCase: consumerIdList = [0, 1, 2, 3] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py index 85222a941b..7f322dc258 100644 --- a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py +++ b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py @@ -94,6 +94,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replica) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py index 5b17d3f583..a923232706 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py @@ -116,6 +116,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -163,6 +164,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") @@ -265,6 +267,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py index 0a7bf5e349..bee174376d 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py @@ -116,6 +116,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -163,6 +164,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") @@ -265,6 +267,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUdf.py b/tests/system-test/7-tmq/tmqUdf.py index 5a0a0481d7..5bb8e3034c 100644 --- a/tests/system-test/7-tmq/tmqUdf.py +++ b/tests/system-test/7-tmq/tmqUdf.py @@ -116,6 +116,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -163,6 +164,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") @@ -266,6 +268,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUpdate-1ctb.py b/tests/system-test/7-tmq/tmqUpdate-1ctb.py index 2fa6600fb9..8fdf7748a3 100644 --- a/tests/system-test/7-tmq/tmqUpdate-1ctb.py +++ b/tests/system-test/7-tmq/tmqUpdate-1ctb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -207,7 +208,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl consumerId = 1 if self.snapshot == 0: - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1/2)) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1+ 1/2 + 1/2)) elif self.snapshot == 1: expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1)) diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py index e2ba47c3fd..8b67f6f825 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py @@ -55,6 +55,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -213,7 +214,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl consumerId = 1 if self.snapshot == 0: - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1/2) * (1/2*3)) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] *(2 + 1/2*1/2*2 + 1/2 + 1/4)) elif self.snapshot == 1: expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/2)) diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py index 6b8c10de27..5a35c4f5ee 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py @@ -55,6 +55,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py index 3975013e74..84617efae4 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py @@ -55,6 +55,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUpdateWithConsume.py b/tests/system-test/7-tmq/tmqUpdateWithConsume.py index 4f595788da..6a9bb0ae92 100644 --- a/tests/system-test/7-tmq/tmqUpdateWithConsume.py +++ b/tests/system-test/7-tmq/tmqUpdateWithConsume.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar, wal_retention_size=-1, wal_retention_period=-1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/99-TDcase/TD-15517.py b/tests/system-test/99-TDcase/TD-15517.py index db06700284..aeb28063e4 100644 --- a/tests/system-test/99-TDcase/TD-15517.py +++ b/tests/system-test/99-TDcase/TD-15517.py @@ -51,7 +51,7 @@ class TDTestCase: return cur def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" diff --git a/tests/system-test/99-TDcase/TD-15554.py b/tests/system-test/99-TDcase/TD-15554.py index 51934ccd44..c8ddeca3c3 100644 --- a/tests/system-test/99-TDcase/TD-15554.py +++ b/tests/system-test/99-TDcase/TD-15554.py @@ -50,7 +50,7 @@ class TDTestCase: return cur def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" diff --git a/tests/system-test/99-TDcase/TD-15557.py b/tests/system-test/99-TDcase/TD-15557.py index 884c028a65..f9e676f4f8 100644 --- a/tests/system-test/99-TDcase/TD-15557.py +++ b/tests/system-test/99-TDcase/TD-15557.py @@ -64,7 +64,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -141,7 +141,7 @@ class TDTestCase: 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -214,7 +214,7 @@ class TDTestCase: 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -298,7 +298,7 @@ class TDTestCase: 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/99-TDcase/TD-15563.py b/tests/system-test/99-TDcase/TD-15563.py index 5ea652b4fb..cc4c3f3b39 100644 --- a/tests/system-test/99-TDcase/TD-15563.py +++ b/tests/system-test/99-TDcase/TD-15563.py @@ -52,7 +52,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -95,7 +95,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -176,7 +176,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -238,7 +238,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -304,7 +304,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/99-TDcase/TD-16025.py b/tests/system-test/99-TDcase/TD-16025.py index 8c9fa9319f..3e25b8c833 100644 --- a/tests/system-test/99-TDcase/TD-16025.py +++ b/tests/system-test/99-TDcase/TD-16025.py @@ -59,7 +59,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -110,7 +110,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/99-TDcase/TD-16821.py b/tests/system-test/99-TDcase/TD-16821.py index f57fae752f..78ac172f30 100644 --- a/tests/system-test/99-TDcase/TD-16821.py +++ b/tests/system-test/99-TDcase/TD-16821.py @@ -73,6 +73,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") diff --git a/tests/system-test/99-TDcase/TD-17255.py b/tests/system-test/99-TDcase/TD-17255.py index 0f83468754..5f68a5b738 100644 --- a/tests/system-test/99-TDcase/TD-17255.py +++ b/tests/system-test/99-TDcase/TD-17255.py @@ -53,6 +53,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -97,6 +98,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -181,6 +183,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -267,6 +270,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("insert data by auto create ctb") diff --git a/tests/system-test/99-TDcase/TD-17699.py b/tests/system-test/99-TDcase/TD-17699.py index 2862f4a78d..6956e88aec 100644 --- a/tests/system-test/99-TDcase/TD-17699.py +++ b/tests/system-test/99-TDcase/TD-17699.py @@ -65,6 +65,7 @@ class TDTestCase: tmqCom.initConsumerTable(self.cdbName) tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"]) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) self.paraDict["stbName"] = 'stb1' tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 600e33feab..7b1dc3a541 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -26,10 +26,6 @@ ELSE () SET(LINK_WEBSOCKET "") ENDIF () -IF (CUS_NAME OR CUS_PROMPT OR CUS_EMAIL) - ADD_DEFINITIONS(-I${CMAKE_CURRENT_SOURCE_DIR}/../../../enterprise/packaging) -ENDIF (CUS_NAME OR CUS_PROMPT OR CUS_EMAIL) - IF (TD_LINUX AND TD_ALPINE) SET(LINK_ARGP "/usr/lib/libargp.a") ELSE () diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 2b7d829652..1eb4edb28a 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -19,18 +19,6 @@ #include "shellInt.h" -#ifndef CUS_NAME - char cusName[] = "TDengine"; -#endif - -#ifndef CUS_PROMPT - char cusPrompt[] = "taos"; -#endif - -#ifndef CUS_EMAIL - char cusEmail[] = ""; -#endif - #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" #endif @@ -58,7 +46,7 @@ #define SHELL_VERSION "Print program version." #ifdef WEBSOCKET -#define SHELL_DSN "Use dsn to connect to the TDengine cloud server or to a remote server which provides WebSocket connection." +#define SHELL_DSN "Use dsn to connect to the cloud server or to a remote server which provides WebSocket connection." #define SHELL_REST "Use RESTful mode when connecting." #define SHELL_TIMEOUT "Set the timeout for websocket query in seconds, default is 30." #endif @@ -92,7 +80,11 @@ void shellPrintHelp() { #endif printf("%s%s%s%s\r\n", indent, "-w,", indent, SHELL_WIDTH); printf("%s%s%s%s\r\n", indent, "-V,", indent, SHELL_VERSION); - printf("\r\n\r\nReport bugs to %s.\r\n", cusEmail); +#ifdef CUS_EMAIL + printf("\r\n\r\nReport bugs to %s.\r\n", CUS_EMAIL); +#else + printf("\r\n\r\nReport bugs to %s.\r\n", "support@taosdata.com"); +#endif } #ifdef LINUX @@ -104,7 +96,11 @@ void shellPrintHelp() { #endif const char *argp_program_version = version; -const char *argp_program_bug_address = cusEmail; +#ifdef CUS_EMAIL +const char *argp_program_bug_address = CUS_EMAIL; +#else +const char *argp_program_bug_address = "support@taosdata.com"; +#endif static struct argp_option shellOptions[] = { {"host", 'h', "HOST", 0, SHELL_HOST}, @@ -414,10 +410,19 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) { shell.info.clientVersion = "Welcome to the %s Command Line Interface, Client Version:%s\r\n" "Copyright (c) 2022 by %s, all rights reserved.\r\n\r\n"; - strcpy(shell.info.cusName, cusName); - sprintf(shell.info.promptHeader, "%s> ", cusPrompt); +#ifdef CUS_NAME + strcpy(shell.info.cusName, CUS_NAME); +#else + strcpy(shell.info.cusName, "TDengine"); +#endif char promptContinueFormat[32] = {0}; - sprintf(promptContinueFormat, "%%%zus> ", strlen(cusPrompt)); +#ifdef CUS_PROMPT + sprintf(shell.info.promptHeader, "%s> ", CUS_PROMPT); + sprintf(promptContinueFormat, "%%%zus> ", strlen(CUS_PROMPT)); +#else + sprintf(shell.info.promptHeader, "taos> "); + sprintf(promptContinueFormat, "%%%zus> ", strlen("taos")); +#endif sprintf(shell.info.promptContinue, promptContinueFormat, " "); shell.info.promptSize = strlen(shell.info.promptHeader); snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index a8986351b7..140720af81 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -84,15 +84,15 @@ SWords shellCommands[] = { {"create table using tags(", 0, 0, NULL}, {"create database " " " - " ;", - 0, 0, NULL}, + " ;", 0, 0, NULL}, {"create dnode ", 0, 0, NULL}, {"create index on ()", 0, 0, NULL}, {"create mnode on dnode ;", 0, 0, NULL}, {"create qnode on dnode ;", 0, 0, NULL}, {"create stream into as select", 0, 0, NULL}, // 26 append sub sql {"create topic as select", 0, 0, NULL}, // 27 append sub sql - {"create function ", 0, 0, NULL}, + {"create function as outputtype language ", 0, 0, NULL}, + {"create aggregate function as outputtype bufsize language ", 0, 0, NULL}, {"create user pass sysinfo 0;", 0, 0, NULL}, {"create user pass sysinfo 1;", 0, 0, NULL}, {"describe ", 0, 0, NULL}, @@ -105,7 +105,7 @@ SWords shellCommands[] = { {"drop qnode on dnode ;", 0, 0, NULL}, {"drop user ;", 0, 0, NULL}, // 40 - {"drop function", 0, 0, NULL}, + {"drop function ;", 0, 0, NULL}, {"drop consumer group on ", 0, 0, NULL}, {"drop topic ;", 0, 0, NULL}, {"drop stream ;", 0, 0, NULL}, @@ -272,6 +272,8 @@ char* key_systable[] = { "ins_subscriptions", "ins_streams", "ins_stream_tasks", "ins_vnodes", "ins_user_privileges", "perf_connections", "perf_queries", "perf_consumers", "perf_trans", "perf_apps"}; +char* udf_language[] = {"\'Python\'", "\'C\'"}; + // // ------- global variant define --------- // @@ -291,25 +293,29 @@ bool waitAutoFill = false; #define WT_VAR_USERNAME 4 #define WT_VAR_TOPIC 5 #define WT_VAR_STREAM 6 -#define WT_VAR_ALLTABLE 7 -#define WT_VAR_FUNC 8 -#define WT_VAR_KEYWORD 9 -#define WT_VAR_TBACTION 10 -#define WT_VAR_DBOPTION 11 -#define WT_VAR_ALTER_DBOPTION 12 -#define WT_VAR_DATATYPE 13 -#define WT_VAR_KEYTAGS 14 -#define WT_VAR_ANYWORD 15 -#define WT_VAR_TBOPTION 16 -#define WT_VAR_USERACTION 17 -#define WT_VAR_KEYSELECT 18 -#define WT_VAR_SYSTABLE 19 +#define WT_VAR_UDFNAME 7 -#define WT_VAR_CNT 20 - -#define WT_FROM_DB_MAX 6 // max get content from db +#define WT_FROM_DB_MAX 7 // max get content from db #define WT_FROM_DB_CNT (WT_FROM_DB_MAX + 1) +#define WT_VAR_ALLTABLE 8 +#define WT_VAR_FUNC 9 +#define WT_VAR_KEYWORD 10 +#define WT_VAR_TBACTION 11 +#define WT_VAR_DBOPTION 12 +#define WT_VAR_ALTER_DBOPTION 13 +#define WT_VAR_DATATYPE 14 +#define WT_VAR_KEYTAGS 15 +#define WT_VAR_ANYWORD 16 +#define WT_VAR_TBOPTION 17 +#define WT_VAR_USERACTION 18 +#define WT_VAR_KEYSELECT 19 +#define WT_VAR_SYSTABLE 20 +#define WT_VAR_LANGUAGE 21 + +#define WT_VAR_CNT 22 + + #define WT_TEXT 0xFF char dbName[256] = ""; // save use database name; @@ -319,13 +325,13 @@ TdThreadMutex tiresMutex; // save thread handle obtain var name from db server TdThread* threads[WT_FROM_DB_CNT]; // obtain var name with sql from server -char varTypes[WT_VAR_CNT][64] = {"", "", "", "", "", - "", "", "", "", "", - "", "", "", "", "", - "", "", "", ""}; +char varTypes[WT_VAR_CNT][64] = { + "", "", "", "", "", "", "", + "", "", "", "", "", "", "", + "", "", "", "", "", "", "", ""}; char varSqls[WT_FROM_DB_CNT][64] = {"show databases;", "show stables;", "show tables;", "show dnodes;", - "show users;", "show topics;", "show streams;"}; + "show users;", "show topics;", "show streams;", "show functions;"}; // var words current cursor, if user press any one key except tab, cursorVar can be reset to -1 int cursorVar = -1; @@ -390,7 +396,8 @@ void showHelp() { create qnode on dnode ;\n\ create stream into as select ...\n\ create topic as select ...\n\ - create function ...\n\ + create function as outputtype language \'C\' | \'Python\' ;\n\ + create aggregate function as outputtype bufsize language \'C\' | \'Python\';\n\ create user pass ...\n\ ----- D ----- \n\ describe \n\ @@ -401,7 +408,7 @@ void showHelp() { drop mnode on dnode ;\n\ drop qnode on dnode ;\n\ drop user ;\n\ - drop function ;\n\ + drop function ;\n\ drop consumer group ... \n\ drop topic ;\n\ drop stream ;\n\ @@ -643,6 +650,7 @@ bool shellAutoInit() { GenerateVarType(WT_VAR_USERACTION, user_actions, sizeof(user_actions) / sizeof(char*)); GenerateVarType(WT_VAR_KEYSELECT, key_select, sizeof(key_select) / sizeof(char*)); GenerateVarType(WT_VAR_SYSTABLE, key_systable, sizeof(key_systable) / sizeof(char*)); + GenerateVarType(WT_VAR_LANGUAGE, udf_language, sizeof(udf_language) / sizeof(char*)); return true; } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index a87ba16267..616540a54a 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -291,8 +291,7 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) { } struct tm ptm = {0}; - if (taosLocalTime(&tt, &ptm) == NULL) { - sprintf(buf, "NaN"); + if (taosLocalTime(&tt, &ptm, buf) == NULL) { return buf; } size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); diff --git a/utils/test/c/tmqDemo.c b/utils/test/c/tmqDemo.c index d105b50579..ce069c2b05 100644 --- a/utils/test/c/tmqDemo.c +++ b/utils/test/c/tmqDemo.c @@ -597,7 +597,7 @@ void printParaIntoFile() { time_t tTime = taosGetTimestampSec(); struct tm tm; - taosLocalTime(&tTime, &tm); + taosLocalTime(&tTime, &tm, NULL); taosFprintfFile(pFile, "###################################################################\n"); taosFprintfFile(pFile, "# configDir: %s\n", configDir); diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index 69debe7ab5..f2de219f4e 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -166,7 +166,7 @@ static void printHelp() { char* getCurrentTimeString(char* timeString) { time_t tTime = taosGetTimestampSec(); struct tm tm; - taosLocalTime(&tTime, &tm); + taosLocalTime(&tTime, &tm, NULL); sprintf(timeString, "%d-%02d-%02d %02d:%02d:%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); @@ -472,7 +472,9 @@ static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) { } struct tm ptm; - taosLocalTime(&tt, &ptm); + if (taosLocalTime(&tt, &ptm, buf) == NULL) { + return buf; + } size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 1f25eae366..1cc2a48469 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -441,7 +441,7 @@ int32_t init_env() { taos_free_result(pRes); char sql[128] = {0}; - snprintf(sql, 128, "create database if not exists db_taosx vgroups %d", g_conf.dstVgroups); + snprintf(sql, 128, "create database if not exists db_taosx vgroups %d wal_retention_period 3600", g_conf.dstVgroups); pRes = taos_query(pConn, sql); if (taos_errno(pRes) != 0) { printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); @@ -470,7 +470,7 @@ int32_t init_env() { } taos_free_result(pRes); - snprintf(sql, 128, "create database if not exists abc1 vgroups %d", g_conf.srcVgroups); + snprintf(sql, 128, "create database if not exists abc1 vgroups %d wal_retention_period 3600", g_conf.srcVgroups); pRes = taos_query(pConn, sql); if (taos_errno(pRes) != 0) { printf("error in create db, reason:%s\n", taos_errstr(pRes)); diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c index 9dd63d14a2..1be28635ef 100644 --- a/utils/tsim/src/simExe.c +++ b/utils/tsim/src/simExe.c @@ -772,7 +772,9 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { tt = (*(int64_t *)row[i]) / 1000000000; } - taosLocalTime(&tt, &tp); + if (taosLocalTime(&tt, &tp, timeStr) == NULL) { + break; + } strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", &tp); if (precision == TSDB_TIME_PRECISION_MILLI) { sprintf(value, "%s.%03d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000));