diff --git a/Jenkinsfile2 b/Jenkinsfile2 index d7df07f06a..b95b3ff86b 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -218,12 +218,12 @@ def pre_test_win(){ if (env.CHANGE_URL =~ /\/TDengine\//) { bat ''' cd %WIN_INTERNAL_ROOT% - git pull + git pull origin ''' + env.CHANGE_TARGET + ''' ''' bat ''' cd %WIN_COMMUNITY_ROOT% git remote prune origin - git pull + git pull origin ''' + env.CHANGE_TARGET + ''' ''' bat ''' cd %WIN_COMMUNITY_ROOT% @@ -236,7 +236,7 @@ def pre_test_win(){ } else if (env.CHANGE_URL =~ /\/TDinternal\//) { bat ''' cd %WIN_INTERNAL_ROOT% - git pull + git pull origin ''' + env.CHANGE_TARGET + ''' ''' bat ''' cd %WIN_INTERNAL_ROOT% diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 1a2f5d396f..16444c07f2 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 05fb2ff + GIT_TAG be729ab SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index b75d4607b8..52e2c0944a 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 285b5e0 + GIT_TAG 8207c74 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index 627cd53c09..ca8fff8da5 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG 76bc64d + GIT_TAG 1bdfca3 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/07-develop/01-connect/_connect_cs.mdx b/docs/en/07-develop/01-connect/_connect_cs.mdx index f8d8e519fd..b81f49b2f0 100644 --- a/docs/en/07-develop/01-connect/_connect_cs.mdx +++ b/docs/en/07-develop/01-connect/_connect_cs.mdx @@ -1,8 +1,7 @@ ```csharp title="Native Connection" -{{#include docs/examples/csharp/ConnectExample.cs}} +{{#include docs/examples/csharp/connect/Program.cs}} ``` -:::info -C# connector supports only native connection for now. - -::: +```csharp title="WebSocket Connection" +{{#include docs/examples/csharp/wsConnect/Program.cs}} +``` diff --git a/docs/en/07-develop/03-insert-data/_cs_line.mdx b/docs/en/07-develop/03-insert-data/_cs_line.mdx index 71f46c62be..ae49901c3a 100644 --- a/docs/en/07-develop/03-insert-data/_cs_line.mdx +++ b/docs/en/07-develop/03-insert-data/_cs_line.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/InfluxDBLineExample.cs}} +{{#include docs/examples/csharp/influxdbLine/Program.cs}} ``` diff --git a/docs/en/07-develop/03-insert-data/_cs_opts_json.mdx b/docs/en/07-develop/03-insert-data/_cs_opts_json.mdx index 8d80d042c9..2627648616 100644 --- a/docs/en/07-develop/03-insert-data/_cs_opts_json.mdx +++ b/docs/en/07-develop/03-insert-data/_cs_opts_json.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/OptsJsonExample.cs}} +{{#include docs/examples/csharp/optsJSON/Program.cs}} ``` diff --git a/docs/en/07-develop/03-insert-data/_cs_opts_telnet.mdx b/docs/en/07-develop/03-insert-data/_cs_opts_telnet.mdx index cff32abf1f..660db13fd1 100644 --- a/docs/en/07-develop/03-insert-data/_cs_opts_telnet.mdx +++ b/docs/en/07-develop/03-insert-data/_cs_opts_telnet.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/OptsTelnetExample.cs}} +{{#include docs/examples/csharp/optsTelnet/Program.cs}} ``` diff --git a/docs/en/07-develop/03-insert-data/_cs_sql.mdx b/docs/en/07-develop/03-insert-data/_cs_sql.mdx index 1dc7bb3d13..42a6bc4315 100644 --- a/docs/en/07-develop/03-insert-data/_cs_sql.mdx +++ b/docs/en/07-develop/03-insert-data/_cs_sql.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/SQLInsertExample.cs}} +{{#include docs/examples/csharp/sqlInsert/Program.cs}} ``` diff --git a/docs/en/07-develop/03-insert-data/_cs_stmt.mdx b/docs/en/07-develop/03-insert-data/_cs_stmt.mdx index 229c874ab9..d8d73ca15e 100644 --- a/docs/en/07-develop/03-insert-data/_cs_stmt.mdx +++ b/docs/en/07-develop/03-insert-data/_cs_stmt.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/StmtInsertExample.cs}} +{{#include docs/examples/csharp/stmtInsert/Program.cs}} ``` diff --git a/docs/en/07-develop/04-query-data/_cs.mdx b/docs/en/07-develop/04-query-data/_cs.mdx index 4bb582ecbf..745ab36811 100644 --- a/docs/en/07-develop/04-query-data/_cs.mdx +++ b/docs/en/07-develop/04-query-data/_cs.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/QueryExample.cs}} +{{#include docs/examples/csharp/query/Program.cs}} ``` diff --git a/docs/en/07-develop/04-query-data/_cs_async.mdx b/docs/en/07-develop/04-query-data/_cs_async.mdx index 3ecf635fd3..19c8e58f32 100644 --- a/docs/en/07-develop/04-query-data/_cs_async.mdx +++ b/docs/en/07-develop/04-query-data/_cs_async.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/AsyncQueryExample.cs}} +{{#include docs/examples/csharp/asyncQuery/Program.cs}} ``` diff --git a/docs/en/07-develop/_sub_cs.mdx b/docs/en/07-develop/_sub_cs.mdx index a435ea0273..093b617e9b 100644 --- a/docs/en/07-develop/_sub_cs.mdx +++ b/docs/en/07-develop/_sub_cs.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/SubscribeDemo.cs}} +{{#include docs/examples/csharp/subscribe/Program.cs}} ``` \ No newline at end of file diff --git a/docs/en/14-reference/03-connector/09-csharp.mdx b/docs/en/14-reference/03-connector/09-csharp.mdx index bc16cd086b..45bf7fdf82 100644 --- a/docs/en/14-reference/03-connector/09-csharp.mdx +++ b/docs/en/14-reference/03-connector/09-csharp.mdx @@ -17,7 +17,7 @@ import CSAsyncQuery from "../../07-develop/04-query-data/_cs_async.mdx" `TDengine.Connector` is a C# language connector provided by TDengine that allows C# developers to develop C# applications that access TDengine cluster data. -The `TDengine.Connector` connector supports connect to TDengine instances via the TDengine client driver (taosc), providing data writing, querying, subscription, schemaless writing, bind interface, etc. The `TDengine.Connector` currently does not provide a REST connection interface. Developers can write their RESTful application by referring to the [REST API](/reference/rest-api/) documentation. +The `TDengine.Connector` connector supports connect to TDengine instances via the TDengine client driver (taosc), providing data writing, querying, subscription, schemaless writing, bind interface, etc.The `TDengine.Connector` also supports WebSocket and developers can build connection through DSN, which supports data writing, querying, and parameter binding, etc. This article describes how to install `TDengine.Connector` in a Linux or Windows environment and connect to TDengine clusters via `TDengine.Connector` to perform basic operations such as data writing and querying. @@ -35,6 +35,10 @@ Please refer to [version support list](/reference/connector#version-support) ## Supported features + + + + 1. Connection Management 2. General Query 3. Continuous Query @@ -42,6 +46,18 @@ Please refer to [version support list](/reference/connector#version-support) 5. Subscription 6. Schemaless + + + + +1. Connection Management +2. General Query +3. Continuous Query +4. Parameter Binding + + + + ## Installation Steps ### Pre-installation preparation @@ -74,12 +90,18 @@ cp -r src/ myProject cd myProject dotnet add exmaple.csproj reference src/TDengine.csproj ``` + ## Establish a Connection -``` C# + + + + + +``` csharp using TDengineDriver; namespace TDengineExample @@ -112,14 +134,62 @@ namespace TDengineExample ``` + + + + +The structure of the DSN description string is as follows: + +```text +[]://[[:@]:][/][?=[&=]] +|------------|---|-----------|-----------|------|------|------------|-----------------------| +| protocol | | username | password | host | port | database | params | +``` + +The parameters are described as follows: + +* **protocol**: Specify which connection method to use (support http/ws). For example, `ws://localhost:6041` uses Websocket to establish connections. +* **username/password**: Username and password used to create connections. +* **host/port**: Specifies the server and port to establish a connection. Websocket connections default to `localhost:6041`. +* **database**: Specify the default database to connect to. +* **params**：Optional parameters. + +A sample DSN description string is as follows: + +```text +ws://localhost:6041/test +``` + +``` csharp +{{#include docs/examples/csharp/wsConnect/Program.cs}} +``` + + + + ## Usage examples ### Write data #### SQL Write + + + + + + + + +```csharp +{{#include docs/examples/csharp/wsInsert/Program.cs}} +``` + + + + #### InfluxDB line protocol write @@ -132,12 +202,48 @@ namespace TDengineExample +#### Parameter Binding + + + + + +``` csharp +{{#include docs/examples/csharp/stmtInsert/Program.cs}} +``` + + + + + +```csharp +{{#include docs/examples/csharp/wsStmt/Program.cs}} +``` + + + + ### Query data #### Synchronous Query + + + + + + + + +```csharp +{{#include docs/examples/csharp/wsQuery/Program.cs}} +``` + + + + #### Asynchronous query @@ -145,18 +251,21 @@ namespace TDengineExample ### More sample programs |Sample program |Sample program description | -|--------------------------------------------------------------------------------------------------------------------|------------ --------------------------------| +|--------------------------------------------------------------------------------------------------------------------|--------------------------------------------| | [CURD](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/Query/Query.cs) | Table creation, data insertion, and query examples with TDengine.Connector | | [JSON Tag](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/JSONTag) | Writing and querying JSON tag data with TDengine Connector | | [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/Stmt) | Parameter binding with TDengine Connector | | [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/schemaless) | Schemaless writes with TDengine Connector | | [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/AsyncQuery/QueryAsync.cs) | Asynchronous queries with TDengine Connector | -| [TMQ](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/TMQ/TMQ.cs) | Data subscription with TDengine Connector | +| [Subscription](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/TMQ/TMQ.cs) | Subscription example with TDengine Connector | +| [Basic WebSocket Usage](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSample.cs) | WebSocket basic data in and out with TDengine connector | +| [WebSocket Parameter Binding](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSTMT.cs) | WebSocket parameter binding example | ## Important update records | TDengine.Connector | Description | |--------------------|--------------------------------| +| 3.0.1 | Support WebSocket and Cloud，With function query, insert, and parameter binding| | 3.0.0 | Supports TDengine 3.0.0.0. TDengine 2.x is not supported. Added `TDengine.Impl.GetData()` interface to deserialize query results. | | 1.0.7 | Fixed TDengine.Query() memory leak. | | 1.0.6 | Fix schemaless bug in 1.0.4 and 1.0.5. | diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 02921c3f6a..5ab6f59454 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -164,7 +164,7 @@ The parameters described in this document by the effect that they have on the sy | Attribute | Description | | -------- | -------------------- | | Applicable | Client only | -| 含义 | SMA index optimization policy | +| Meaning | SMA index optimization policy | | Unit | None | | Default Value | 0 | | Notes | @@ -325,7 +325,7 @@ The charset that takes effect is UTF-8. | Applicable | Server Only | | Meaning | Maximum number of vnodes per dnode | | Value Range | 0-4096 | -| Default Value | 256 | +| Default Value | 2x the CPU cores | ## Time Parameters @@ -697,152 +697,154 @@ To prevent system resource from being exhausted by multiple concurrent streams, | 15 | telemetryPort | No | Yes | | 16 | queryPolicy | No | Yes | | 17 | querySmaOptimize | No | Yes | -| 18 | queryBufferSize | Yes | Yes | -| 19 | maxNumOfDistinctRes | Yes | Yes | -| 20 | minSlidingTime | Yes | Yes | -| 21 | minIntervalTime | Yes | Yes | -| 22 | countAlwaysReturnValue | Yes | Yes | -| 23 | dataDir | Yes | Yes | -| 24 | minimalDataDirGB | Yes | Yes | -| 25 | supportVnodes | No | Yes | -| 26 | tempDir | Yes | Yes | -| 27 | minimalTmpDirGB | Yes | Yes | -| 28 | compressMsgSize | Yes | Yes | -| 29 | compressColData | Yes | Yes | -| 30 | smlChildTableName | Yes | Yes | -| 31 | smlTagName | Yes | Yes | -| 32 | smlDataFormat | No | Yes | -| 33 | statusInterval | Yes | Yes | -| 34 | shellActivityTimer | Yes | Yes | -| 35 | transPullupInterval | No | Yes | -| 36 | mqRebalanceInterval | No | Yes | -| 37 | ttlUnit | No | Yes | -| 38 | ttlPushInterval | No | Yes | -| 39 | numOfTaskQueueThreads | No | Yes | -| 40 | numOfRpcThreads | No | Yes | -| 41 | numOfCommitThreads | Yes | Yes | -| 42 | numOfMnodeReadThreads | No | Yes | -| 43 | numOfVnodeQueryThreads | No | Yes | -| 44 | numOfVnodeStreamThreads | No | Yes | -| 45 | numOfVnodeFetchThreads | No | Yes | -| 46 | numOfVnodeWriteThreads | No | Yes | -| 47 | numOfVnodeSyncThreads | No | Yes | -| 48 | numOfQnodeQueryThreads | No | Yes | -| 49 | numOfQnodeFetchThreads | No | Yes | -| 50 | numOfSnodeSharedThreads | No | Yes | -| 51 | numOfSnodeUniqueThreads | No | Yes | -| 52 | rpcQueueMemoryAllowed | No | Yes | -| 53 | logDir | Yes | Yes | -| 54 | minimalLogDirGB | Yes | Yes | -| 55 | numOfLogLines | Yes | Yes | -| 56 | asyncLog | Yes | Yes | -| 57 | logKeepDays | Yes | Yes | -| 58 | debugFlag | Yes | Yes | -| 59 | tmrDebugFlag | Yes | Yes | -| 60 | uDebugFlag | Yes | Yes | -| 61 | rpcDebugFlag | Yes | Yes | -| 62 | jniDebugFlag | Yes | Yes | -| 63 | qDebugFlag | Yes | Yes | -| 64 | cDebugFlag | Yes | Yes | -| 65 | dDebugFlag | Yes | Yes | -| 66 | vDebugFlag | Yes | Yes | -| 67 | mDebugFlag | Yes | Yes | -| 68 | wDebugFlag | Yes | Yes | -| 69 | sDebugFlag | Yes | Yes | -| 70 | tsdbDebugFlag | Yes | Yes | -| 71 | tqDebugFlag | No | Yes | -| 72 | fsDebugFlag | Yes | Yes | -| 73 | udfDebugFlag | No | Yes | -| 74 | smaDebugFlag | No | Yes | -| 75 | idxDebugFlag | No | Yes | -| 76 | tdbDebugFlag | No | Yes | -| 77 | metaDebugFlag | No | Yes | -| 78 | timezone | Yes | Yes | -| 79 | locale | Yes | Yes | -| 80 | charset | Yes | Yes | -| 81 | udf | Yes | Yes | -| 82 | enableCoreFile | Yes | Yes | -| 83 | arbitrator | Yes | No | -| 84 | numOfThreadsPerCore | Yes | No | -| 85 | numOfMnodes | Yes | No | -| 86 | vnodeBak | Yes | No | -| 87 | balance | Yes | No | -| 88 | balanceInterval | Yes | No | -| 89 | offlineThreshold | Yes | No | -| 90 | role | Yes | No | -| 91 | dnodeNopLoop | Yes | No | -| 92 | keepTimeOffset | Yes | No | -| 93 | rpcTimer | Yes | No | -| 94 | rpcMaxTime | Yes | No | -| 95 | rpcForceTcp | Yes | No | -| 96 | tcpConnTimeout | Yes | No | -| 97 | syncCheckInterval | Yes | No | -| 98 | maxTmrCtrl | Yes | No | -| 99 | monitorReplica | Yes | No | -| 100 | smlTagNullName | Yes | No | -| 101 | keepColumnName | Yes | No | -| 102 | ratioOfQueryCores | Yes | No | -| 103 | maxStreamCompDelay | Yes | No | -| 104 | maxFirstStreamCompDelay | Yes | No | -| 105 | retryStreamCompDelay | Yes | No | -| 106 | streamCompDelayRatio | Yes | No | -| 107 | maxVgroupsPerDb | Yes | No | -| 108 | maxTablesPerVnode | Yes | No | -| 109 | minTablesPerVnode | Yes | No | -| 110 | tableIncStepPerVnode | Yes | No | -| 111 | cache | Yes | No | -| 112 | blocks | Yes | No | -| 113 | days | Yes | No | -| 114 | keep | Yes | No | -| 115 | minRows | Yes | No | -| 116 | maxRows | Yes | No | -| 117 | quorum | Yes | No | -| 118 | comp | Yes | No | -| 119 | walLevel | Yes | No | -| 120 | fsync | Yes | No | -| 121 | replica | Yes | No | -| 122 | partitions | Yes | No | -| 123 | quorum | Yes | No | -| 124 | update | Yes | No | -| 125 | cachelast | Yes | No | -| 126 | maxSQLLength | Yes | No | -| 127 | maxWildCardsLength | Yes | No | -| 128 | maxRegexStringLen | Yes | No | -| 129 | maxNumOfOrderedRes | Yes | No | -| 130 | maxConnections | Yes | No | -| 131 | mnodeEqualVnodeNum | Yes | No | -| 132 | http | Yes | No | -| 133 | httpEnableRecordSql | Yes | No | -| 134 | httpMaxThreads | Yes | No | -| 135 | restfulRowLimit | Yes | No | -| 136 | httpDbNameMandatory | Yes | No | -| 137 | httpKeepAlive | Yes | No | -| 138 | enableRecordSql | Yes | No | -| 139 | maxBinaryDisplayWidth | Yes | No | -| 140 | stream | Yes | No | -| 141 | retrieveBlockingModel | Yes | No | -| 142 | tsdbMetaCompactRatio | Yes | No | -| 143 | defaultJSONStrType | Yes | No | -| 144 | walFlushSize | Yes | No | -| 145 | keepTimeOffset | Yes | No | -| 146 | flowctrl | Yes | No | -| 147 | slaveQuery | Yes | No | -| 148 | adjustMaster | Yes | No | -| 149 | topicBinaryLen | Yes | No | -| 150 | telegrafUseFieldNum | Yes | No | -| 151 | deadLockKillQuery | Yes | No | -| 152 | clientMerge | Yes | No | -| 153 | sdbDebugFlag | Yes | No | -| 154 | odbcDebugFlag | Yes | No | -| 155 | httpDebugFlag | Yes | No | -| 156 | monDebugFlag | Yes | No | -| 157 | cqDebugFlag | Yes | No | -| 158 | shortcutFlag | Yes | No | -| 159 | probeSeconds | Yes | No | -| 160 | probeKillSeconds | Yes | No | -| 161 | probeInterval | Yes | No | -| 162 | lossyColumns | Yes | No | -| 163 | fPrecision | Yes | No | -| 164 | dPrecision | Yes | No | -| 165 | maxRange | Yes | No | -| 166 | range | Yes | No | +| 18 | queryRsmaTolerance | No | Yes | +| 19 | queryBufferSize | Yes | Yes | +| 20 | maxNumOfDistinctRes | Yes | Yes | +| 21 | minSlidingTime | Yes | Yes | +| 22 | minIntervalTime | Yes | Yes | +| 23 | countAlwaysReturnValue | Yes | Yes | +| 24 | dataDir | Yes | Yes | +| 25 | minimalDataDirGB | Yes | Yes | +| 26 | supportVnodes | No | Yes | +| 27 | tempDir | Yes | Yes | +| 28 | minimalTmpDirGB | Yes | Yes | +| 29 | compressMsgSize | Yes | Yes | +| 30 | compressColData | Yes | Yes | +| 31 | smlChildTableName | Yes | Yes | +| 32 | smlTagName | Yes | Yes | +| 33 | smlDataFormat | No | Yes | +| 34 | statusInterval | Yes | Yes | +| 35 | shellActivityTimer | Yes | Yes | +| 36 | transPullupInterval | No | Yes | +| 37 | mqRebalanceInterval | No | Yes | +| 38 | ttlUnit | No | Yes | +| 39 | ttlPushInterval | No | Yes | +| 40 | numOfTaskQueueThreads | No | Yes | +| 41 | numOfRpcThreads | No | Yes | +| 42 | numOfCommitThreads | Yes | Yes | +| 43 | numOfMnodeReadThreads | No | Yes | +| 44 | numOfVnodeQueryThreads | No | Yes | +| 45 | numOfVnodeStreamThreads | No | Yes | +| 46 | numOfVnodeFetchThreads | No | Yes | +| 47 | numOfVnodeWriteThreads | No | Yes | +| 48 | numOfVnodeSyncThreads | No | Yes | +| 49 | numOfVnodeRsmaThreads | No | Yes | +| 50 | numOfQnodeQueryThreads | No | Yes | +| 51 | numOfQnodeFetchThreads | No | Yes | +| 52 | numOfSnodeSharedThreads | No | Yes | +| 53 | numOfSnodeUniqueThreads | No | Yes | +| 54 | rpcQueueMemoryAllowed | No | Yes | +| 55 | logDir | Yes | Yes | +| 56 | minimalLogDirGB | Yes | Yes | +| 57 | numOfLogLines | Yes | Yes | +| 58 | asyncLog | Yes | Yes | +| 59 | logKeepDays | Yes | Yes | +| 60 | debugFlag | Yes | Yes | +| 61 | tmrDebugFlag | Yes | Yes | +| 62 | uDebugFlag | Yes | Yes | +| 63 | rpcDebugFlag | Yes | Yes | +| 64 | jniDebugFlag | Yes | Yes | +| 65 | qDebugFlag | Yes | Yes | +| 66 | cDebugFlag | Yes | Yes | +| 67 | dDebugFlag | Yes | Yes | +| 68 | vDebugFlag | Yes | Yes | +| 69 | mDebugFlag | Yes | Yes | +| 70 | wDebugFlag | Yes | Yes | +| 71 | sDebugFlag | Yes | Yes | +| 72 | tsdbDebugFlag | Yes | Yes | +| 73 | tqDebugFlag | No | Yes | +| 74 | fsDebugFlag | Yes | Yes | +| 75 | udfDebugFlag | No | Yes | +| 76 | smaDebugFlag | No | Yes | +| 77 | idxDebugFlag | No | Yes | +| 78 | tdbDebugFlag | No | Yes | +| 79 | metaDebugFlag | No | Yes | +| 80 | timezone | Yes | Yes | +| 81 | locale | Yes | Yes | +| 82 | charset | Yes | Yes | +| 83 | udf | Yes | Yes | +| 84 | enableCoreFile | Yes | Yes | +| 85 | arbitrator | Yes | No | +| 86 | numOfThreadsPerCore | Yes | No | +| 87 | numOfMnodes | Yes | No | +| 88 | vnodeBak | Yes | No | +| 89 | balance | Yes | No | +| 90 | balanceInterval | Yes | No | +| 91 | offlineThreshold | Yes | No | +| 92 | role | Yes | No | +| 93 | dnodeNopLoop | Yes | No | +| 94 | keepTimeOffset | Yes | No | +| 95 | rpcTimer | Yes | No | +| 96 | rpcMaxTime | Yes | No | +| 97 | rpcForceTcp | Yes | No | +| 98 | tcpConnTimeout | Yes | No | +| 99 | syncCheckInterval | Yes | No | +| 100 | maxTmrCtrl | Yes | No | +| 101 | monitorReplica | Yes | No | +| 102 | smlTagNullName | Yes | No | +| 103 | keepColumnName | Yes | No | +| 104 | ratioOfQueryCores | Yes | No | +| 105 | maxStreamCompDelay | Yes | No | +| 106 | maxFirstStreamCompDelay | Yes | No | +| 107 | retryStreamCompDelay | Yes | No | +| 108 | streamCompDelayRatio | Yes | No | +| 109 | maxVgroupsPerDb | Yes | No | +| 110 | maxTablesPerVnode | Yes | No | +| 111 | minTablesPerVnode | Yes | No | +| 112 | tableIncStepPerVnode | Yes | No | +| 113 | cache | Yes | No | +| 114 | blocks | Yes | No | +| 115 | days | Yes | No | +| 116 | keep | Yes | No | +| 117 | minRows | Yes | No | +| 118 | maxRows | Yes | No | +| 119 | quorum | Yes | No | +| 120 | comp | Yes | No | +| 121 | walLevel | Yes | No | +| 122 | fsync | Yes | No | +| 123 | replica | Yes | No | +| 124 | partitions | Yes | No | +| 125 | quorum | Yes | No | +| 126 | update | Yes | No | +| 127 | cachelast | Yes | No | +| 128 | maxSQLLength | Yes | No | +| 129 | maxWildCardsLength | Yes | No | +| 130 | maxRegexStringLen | Yes | No | +| 131 | maxNumOfOrderedRes | Yes | No | +| 132 | maxConnections | Yes | No | +| 133 | mnodeEqualVnodeNum | Yes | No | +| 134 | http | Yes | No | +| 135 | httpEnableRecordSql | Yes | No | +| 136 | httpMaxThreads | Yes | No | +| 137 | restfulRowLimit | Yes | No | +| 138 | httpDbNameMandatory | Yes | No | +| 139 | httpKeepAlive | Yes | No | +| 140 | enableRecordSql | Yes | No | +| 141 | maxBinaryDisplayWidth | Yes | No | +| 142 | stream | Yes | No | +| 143 | retrieveBlockingModel | Yes | No | +| 144 | tsdbMetaCompactRatio | Yes | No | +| 145 | defaultJSONStrType | Yes | No | +| 146 | walFlushSize | Yes | No | +| 147 | keepTimeOffset | Yes | No | +| 148 | flowctrl | Yes | No | +| 149 | slaveQuery | Yes | No | +| 150 | adjustMaster | Yes | No | +| 151 | topicBinaryLen | Yes | No | +| 152 | telegrafUseFieldNum | Yes | No | +| 153 | deadLockKillQuery | Yes | No | +| 154 | clientMerge | Yes | No | +| 155 | sdbDebugFlag | Yes | No | +| 156 | odbcDebugFlag | Yes | No | +| 157 | httpDebugFlag | Yes | No | +| 158 | monDebugFlag | Yes | No | +| 159 | cqDebugFlag | Yes | No | +| 160 | shortcutFlag | Yes | No | +| 161 | probeSeconds | Yes | No | +| 162 | probeKillSeconds | Yes | No | +| 163 | probeInterval | Yes | No | +| 164 | lossyColumns | Yes | No | +| 165 | fPrecision | Yes | No | +| 166 | dPrecision | Yes | No | +| 167 | maxRange | Yes | No | +| 168 | range | Yes | No | diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index e3901114d3..414986d107 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -6,6 +6,9 @@ description: TDengine release history, Release Notes and download links. import Release from "/components/ReleaseV3"; +## 3.0.1.2 + + ## 3.0.1.1 diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index 228990df3b..086d3adea2 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -6,6 +6,10 @@ description: taosTools release history, Release Notes, download links. import Release from "/components/ReleaseV3"; +## 2.2.2 + + + ## 2.2.0 diff --git a/docs/examples/csharp/.gitignore b/docs/examples/csharp/.gitignore index 694da603b9..c228f1be2a 100644 --- a/docs/examples/csharp/.gitignore +++ b/docs/examples/csharp/.gitignore @@ -1,12 +1,27 @@ -bin -obj .vs -*.sln -wsConnect/obj -wsInsert/obj -wsQuery/obj -wsStmt/obj +asyncQuery/bin +connect/bin +influxdbLine/bin +optsJSON/bin +optsTelnet/bin +query/bin +sqlInsert/bin +stmtInsert/bin +subscribe/bin wsConnect/bin wsInsert/bin wsQuery/bin -wsStmt/bin \ No newline at end of file +wsStmt/bin +asyncQuery/obj +connect/obj +influxdbLine/obj +optsJSON/obj +optsTelnet/obj +query/obj +sqlInsert/obj +stmtInsert/obj +subscribe/obj +wsConnect/obj +wsInsert/obj +wsQuery/obj +wsStmt/obj \ No newline at end of file diff --git a/docs/examples/csharp/QueryExample.cs b/docs/examples/csharp/QueryExample.cs deleted file mode 100644 index d75bb8d661..0000000000 --- a/docs/examples/csharp/QueryExample.cs +++ /dev/null @@ -1,82 +0,0 @@ -using TDengineDriver; -using TDengineDriver.Impl; -using System.Runtime.InteropServices; - -namespace TDengineExample -{ - internal class QueryExample - { - static void Main() - { - IntPtr conn = GetConnection(); - // run query - IntPtr res = TDengine.Query(conn, "SELECT * FROM meters LIMIT 2"); - if (TDengine.ErrorNo(res) != 0) - { - Console.WriteLine("Failed to query since: " + TDengine.Error(res)); - TDengine.Close(conn); - TDengine.Cleanup(); - return; - } - - // get filed count - int fieldCount = TDengine.FieldCount(res); - Console.WriteLine("fieldCount=" + fieldCount); - - // print column names - List metas = LibTaos.GetMeta(res); - for (int i = 0; i < metas.Count; i++) - { - Console.Write(metas[i].name + "\t"); - } - Console.WriteLine(); - - // print values - List resData = LibTaos.GetData(res); - for (int i = 0; i < resData.Count; i++) - { - Console.Write($"|{resData[i].ToString()} \t"); - if (((i + 1) % metas.Count == 0)) - { - Console.WriteLine(""); - } - } - Console.WriteLine(); - - if (TDengine.ErrorNo(res) != 0) - { - Console.WriteLine($"Query is not complete, Error {TDengine.ErrorNo(res)} {TDengine.Error(res)}"); - } - // exit - TDengine.FreeResult(res); - TDengine.Close(conn); - TDengine.Cleanup(); - } - static IntPtr GetConnection() - { - string host = "localhost"; - short port = 6030; - string username = "root"; - string password = "taosdata"; - string dbname = "power"; - var conn = TDengine.Connect(host, username, password, dbname, port); - if (conn == IntPtr.Zero) - { - Console.WriteLine("Connect to TDengine failed"); - System.Environment.Exit(0); - } - else - { - Console.WriteLine("Connect to TDengine success"); - } - return conn; - } - } -} - -// output: -// Connect to TDengine success -// fieldCount=6 -// ts current voltage phase location groupid -// 1648432611249 10.3 219 0.31 California.SanFrancisco 2 -// 1648432611749 12.6 218 0.33 California.SanFrancisco 2 \ No newline at end of file diff --git a/docs/examples/csharp/SQLInsertExample.cs b/docs/examples/csharp/SQLInsertExample.cs deleted file mode 100644 index 3ce70fe914..0000000000 --- a/docs/examples/csharp/SQLInsertExample.cs +++ /dev/null @@ -1,70 +0,0 @@ -using TDengineDriver; - - -namespace TDengineExample -{ - internal class SQLInsertExample - { - - static void Main() - { - IntPtr conn = GetConnection(); - IntPtr res = TDengine.Query(conn, "CREATE DATABASE power"); - CheckRes(conn, res, "failed to create database"); - res = TDengine.Query(conn, "USE power"); - CheckRes(conn, res, "failed to change database"); - res = TDengine.Query(conn, "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)"); - CheckRes(conn, res, "failed to create stable"); - var sql = "INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) " + - "d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) " + - "d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " + - "d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)"; - res = TDengine.Query(conn, sql); - CheckRes(conn, res, "failed to insert data"); - int affectedRows = TDengine.AffectRows(res); - Console.WriteLine("affectedRows " + affectedRows); - TDengine.FreeResult(res); - ExitProgram(conn, 0); - } - - static IntPtr GetConnection() - { - string host = "localhost"; - short port = 6030; - string username = "root"; - string password = "taosdata"; - string dbname = ""; - var conn = TDengine.Connect(host, username, password, dbname, port); - if (conn == IntPtr.Zero) - { - Console.WriteLine("Connect to TDengine failed"); - Environment.Exit(0); - } - else - { - Console.WriteLine("Connect to TDengine success"); - } - return conn; - } - - static void CheckRes(IntPtr conn, IntPtr res, String errorMsg) - { - if (TDengine.ErrorNo(res) != 0) - { - Console.Write(errorMsg + " since: " + TDengine.Error(res)); - ExitProgram(conn, 1); - } - } - - static void ExitProgram(IntPtr conn, int exitCode) - { - TDengine.Close(conn); - TDengine.Cleanup(); - Environment.Exit(exitCode); - } - } -} - -// output: -// Connect to TDengine success -// affectedRows 8 diff --git a/docs/examples/csharp/AsyncQueryExample.cs b/docs/examples/csharp/asyncQuery/Program.cs similarity index 81% rename from docs/examples/csharp/AsyncQueryExample.cs rename to docs/examples/csharp/asyncQuery/Program.cs index 0d47325932..864f06a15e 100644 --- a/docs/examples/csharp/AsyncQueryExample.cs +++ b/docs/examples/csharp/asyncQuery/Program.cs @@ -11,11 +11,17 @@ namespace TDengineExample static void Main() { IntPtr conn = GetConnection(); - QueryAsyncCallback queryAsyncCallback = new QueryAsyncCallback(QueryCallback); - TDengine.QueryAsync(conn, "select * from meters", queryAsyncCallback, IntPtr.Zero); - Thread.Sleep(2000); - TDengine.Close(conn); - TDengine.Cleanup(); + try + { + QueryAsyncCallback queryAsyncCallback = new QueryAsyncCallback(QueryCallback); + TDengine.QueryAsync(conn, "select * from meters", queryAsyncCallback, IntPtr.Zero); + Thread.Sleep(2000); + } + finally + { + TDengine.Close(conn); + } + } static void QueryCallback(IntPtr param, IntPtr taosRes, int code) @@ -27,11 +33,11 @@ namespace TDengineExample } else { - Console.WriteLine($"async query data failed, failed code {code}"); + throw new Exception($"async query data failed,code:{code},reason:{TDengine.Error(taosRes)}"); } } - // Iteratively call this interface until "numOfRows" is no greater than 0. + // Iteratively call this interface until "numOfRows" is no greater than 0. static void FetchRawBlockCallback(IntPtr param, IntPtr taosRes, int numOfRows) { if (numOfRows > 0) @@ -43,7 +49,7 @@ namespace TDengineExample for (int i = 0; i < dataList.Count; i++) { - if (i != 0 && (i+1) % metaList.Count == 0) + if (i != 0 && (i + 1) % metaList.Count == 0) { Console.WriteLine("{0}\t|", dataList[i]); } @@ -63,7 +69,7 @@ namespace TDengineExample } else { - Console.WriteLine($"FetchRawBlockCallback callback error, error code {numOfRows}"); + throw new Exception($"FetchRawBlockCallback callback error, error code {numOfRows}"); } TDengine.FreeResult(taosRes); } @@ -79,8 +85,7 @@ namespace TDengineExample var conn = TDengine.Connect(host, username, password, dbname, port); if (conn == IntPtr.Zero) { - Console.WriteLine("Connect to TDengine failed"); - Environment.Exit(0); + throw new Exception("Connect to TDengine failed"); } else { diff --git a/docs/examples/csharp/asyncquery.csproj b/docs/examples/csharp/asyncQuery/asyncquery.csproj similarity index 98% rename from docs/examples/csharp/asyncquery.csproj rename to docs/examples/csharp/asyncQuery/asyncquery.csproj index 045969edd7..23e590cd25 100644 --- a/docs/examples/csharp/asyncquery.csproj +++ b/docs/examples/csharp/asyncQuery/asyncquery.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/ConnectExample.cs b/docs/examples/csharp/connect/Program.cs similarity index 90% rename from docs/examples/csharp/ConnectExample.cs rename to docs/examples/csharp/connect/Program.cs index f3548ee65d..955db40c7c 100644 --- a/docs/examples/csharp/ConnectExample.cs +++ b/docs/examples/csharp/connect/Program.cs @@ -16,7 +16,7 @@ namespace TDengineExample var conn = TDengine.Connect(host, username, password, dbname, port); if (conn == IntPtr.Zero) { - Console.WriteLine("Connect to TDengine failed"); + throw new Exception("Connect to TDengine failed"); } else { diff --git a/docs/examples/csharp/connect.csproj b/docs/examples/csharp/connect/connect.csproj similarity index 100% rename from docs/examples/csharp/connect.csproj rename to docs/examples/csharp/connect/connect.csproj diff --git a/docs/examples/csharp/csharp.sln b/docs/examples/csharp/csharp.sln new file mode 100644 index 0000000000..560dde55cb --- /dev/null +++ b/docs/examples/csharp/csharp.sln @@ -0,0 +1,94 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30114.105 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "asyncquery", "asyncQuery\asyncquery.csproj", "{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "connect", "connect\connect.csproj", "{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "influxdbline", "influxdbLine\influxdbline.csproj", "{6A24FB80-1E3C-4E2D-A5AB-914FA583874D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "optsJSON", "optsJSON\optsJSON.csproj", "{6725A961-0C66-4196-AC98-8D3F3D757D6C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "optstelnet", "optsTelnet\optstelnet.csproj", "{B3B50D25-688B-44D4-8683-482ABC52FFCA}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "query", "query\query.csproj", "{F2B7D13B-FE04-4C5C-BB6D-C12E0A9D9970}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "stmtinsert", "stmtInsert\stmtinsert.csproj", "{B40D6BED-BE3C-4B44-9B12-28BE441311BA}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "subscribe", "subscribe\subscribe.csproj", "{C3D45A8E-AFC0-4547-9F3C-467B0B583DED}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "wsConnect", "wsConnect\wsConnect.csproj", "{51E19494-845E-49ED-97C7-749AE63111BD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "wsInsert", "wsInsert\wsInsert.csproj", "{13E2233B-4AFF-40D9-AF42-AB3F01617540}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "wsQuery", "wsQuery\wsQuery.csproj", "{0F394169-C456-442C-929D-C2D43A0EEC7B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "wsStmt", "wsStmt\wsStmt.csproj", "{27B9C9AB-9055-4BF2-8A14-4E59F09D5985}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "sqlinsert", "sqlInsert\sqlinsert.csproj", "{CD24BD12-8550-4627-A11D-707B446F48C3}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Release|Any CPU.Build.0 = Release|Any CPU + {CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Release|Any CPU.Build.0 = Release|Any CPU + {6A24FB80-1E3C-4E2D-A5AB-914FA583874D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6A24FB80-1E3C-4E2D-A5AB-914FA583874D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6A24FB80-1E3C-4E2D-A5AB-914FA583874D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6A24FB80-1E3C-4E2D-A5AB-914FA583874D}.Release|Any CPU.Build.0 = Release|Any CPU + {6725A961-0C66-4196-AC98-8D3F3D757D6C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6725A961-0C66-4196-AC98-8D3F3D757D6C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6725A961-0C66-4196-AC98-8D3F3D757D6C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6725A961-0C66-4196-AC98-8D3F3D757D6C}.Release|Any CPU.Build.0 = Release|Any CPU + {B3B50D25-688B-44D4-8683-482ABC52FFCA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B3B50D25-688B-44D4-8683-482ABC52FFCA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B3B50D25-688B-44D4-8683-482ABC52FFCA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B3B50D25-688B-44D4-8683-482ABC52FFCA}.Release|Any CPU.Build.0 = Release|Any CPU + {F2B7D13B-FE04-4C5C-BB6D-C12E0A9D9970}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F2B7D13B-FE04-4C5C-BB6D-C12E0A9D9970}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F2B7D13B-FE04-4C5C-BB6D-C12E0A9D9970}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F2B7D13B-FE04-4C5C-BB6D-C12E0A9D9970}.Release|Any CPU.Build.0 = Release|Any CPU + {B40D6BED-BE3C-4B44-9B12-28BE441311BA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B40D6BED-BE3C-4B44-9B12-28BE441311BA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B40D6BED-BE3C-4B44-9B12-28BE441311BA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B40D6BED-BE3C-4B44-9B12-28BE441311BA}.Release|Any CPU.Build.0 = Release|Any CPU + {C3D45A8E-AFC0-4547-9F3C-467B0B583DED}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C3D45A8E-AFC0-4547-9F3C-467B0B583DED}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C3D45A8E-AFC0-4547-9F3C-467B0B583DED}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C3D45A8E-AFC0-4547-9F3C-467B0B583DED}.Release|Any CPU.Build.0 = Release|Any CPU + {51E19494-845E-49ED-97C7-749AE63111BD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {51E19494-845E-49ED-97C7-749AE63111BD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {51E19494-845E-49ED-97C7-749AE63111BD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {51E19494-845E-49ED-97C7-749AE63111BD}.Release|Any CPU.Build.0 = Release|Any CPU + {13E2233B-4AFF-40D9-AF42-AB3F01617540}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {13E2233B-4AFF-40D9-AF42-AB3F01617540}.Debug|Any CPU.Build.0 = Debug|Any CPU + {13E2233B-4AFF-40D9-AF42-AB3F01617540}.Release|Any CPU.ActiveCfg = Release|Any CPU + {13E2233B-4AFF-40D9-AF42-AB3F01617540}.Release|Any CPU.Build.0 = Release|Any CPU + {0F394169-C456-442C-929D-C2D43A0EEC7B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0F394169-C456-442C-929D-C2D43A0EEC7B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0F394169-C456-442C-929D-C2D43A0EEC7B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0F394169-C456-442C-929D-C2D43A0EEC7B}.Release|Any CPU.Build.0 = Release|Any CPU + {27B9C9AB-9055-4BF2-8A14-4E59F09D5985}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {27B9C9AB-9055-4BF2-8A14-4E59F09D5985}.Debug|Any CPU.Build.0 = Debug|Any CPU + {27B9C9AB-9055-4BF2-8A14-4E59F09D5985}.Release|Any CPU.ActiveCfg = Release|Any CPU + {27B9C9AB-9055-4BF2-8A14-4E59F09D5985}.Release|Any CPU.Build.0 = Release|Any CPU + {CD24BD12-8550-4627-A11D-707B446F48C3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CD24BD12-8550-4627-A11D-707B446F48C3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CD24BD12-8550-4627-A11D-707B446F48C3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CD24BD12-8550-4627-A11D-707B446F48C3}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/docs/examples/csharp/InfluxDBLineExample.cs b/docs/examples/csharp/influxdbLine/Program.cs similarity index 73% rename from docs/examples/csharp/InfluxDBLineExample.cs rename to docs/examples/csharp/influxdbLine/Program.cs index 7b4453f4ac..fa3cb21fe0 100644 --- a/docs/examples/csharp/InfluxDBLineExample.cs +++ b/docs/examples/csharp/influxdbLine/Program.cs @@ -17,8 +17,7 @@ namespace TDengineExample IntPtr res = TDengine.SchemalessInsert(conn, lines, lines.Length, (int)TDengineSchemalessProtocol.TSDB_SML_LINE_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS); if (TDengine.ErrorNo(res) != 0) { - Console.WriteLine("SchemalessInsert failed since " + TDengine.Error(res)); - ExitProgram(conn, 1); + throw new Exception("SchemalessInsert failed since " + TDengine.Error(res)); } else { @@ -26,7 +25,6 @@ namespace TDengineExample Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows"); } TDengine.FreeResult(res); - ExitProgram(conn, 0); } static IntPtr GetConnection() @@ -39,9 +37,7 @@ namespace TDengineExample var conn = TDengine.Connect(host, username, password, dbname, port); if (conn == IntPtr.Zero) { - Console.WriteLine("Connect to TDengine failed"); - TDengine.Cleanup(); - Environment.Exit(1); + throw new Exception("Connect to TDengine failed"); } else { @@ -55,23 +51,15 @@ namespace TDengineExample IntPtr res = TDengine.Query(conn, "CREATE DATABASE test"); if (TDengine.ErrorNo(res) != 0) { - Console.WriteLine("failed to create database, reason: " + TDengine.Error(res)); - ExitProgram(conn, 1); + throw new Exception("failed to create database, reason: " + TDengine.Error(res)); } res = TDengine.Query(conn, "USE test"); if (TDengine.ErrorNo(res) != 0) { - Console.WriteLine("failed to change database, reason: " + TDengine.Error(res)); - ExitProgram(conn, 1); + throw new Exception("failed to change database, reason: " + TDengine.Error(res)); } } - static void ExitProgram(IntPtr conn, int exitCode) - { - TDengine.Close(conn); - TDengine.Cleanup(); - Environment.Exit(exitCode); - } } } diff --git a/docs/examples/csharp/influxdbline.csproj b/docs/examples/csharp/influxdbLine/influxdbline.csproj similarity index 100% rename from docs/examples/csharp/influxdbline.csproj rename to docs/examples/csharp/influxdbLine/influxdbline.csproj diff --git a/docs/examples/csharp/OptsJsonExample.cs b/docs/examples/csharp/optsJSON/Program.cs similarity index 53% rename from docs/examples/csharp/OptsJsonExample.cs rename to docs/examples/csharp/optsJSON/Program.cs index 2c41acc5c9..b67b5af62b 100644 --- a/docs/examples/csharp/OptsJsonExample.cs +++ b/docs/examples/csharp/optsJSON/Program.cs @@ -7,27 +7,31 @@ namespace TDengineExample static void Main() { IntPtr conn = GetConnection(); - PrepareDatabase(conn); - string[] lines = { "[{\"metric\": \"meters.current\", \"timestamp\": 1648432611249, \"value\": 10.3, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," + + try + { + PrepareDatabase(conn); + string[] lines = { "[{\"metric\": \"meters.current\", \"timestamp\": 1648432611249, \"value\": 10.3, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," + " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611249, \"value\": 219, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}, " + "{\"metric\": \"meters.current\", \"timestamp\": 1648432611250, \"value\": 12.6, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," + " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611250, \"value\": 221, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}]" }; - IntPtr res = TDengine.SchemalessInsert(conn, lines, 1, (int)TDengineSchemalessProtocol.TSDB_SML_JSON_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED); - if (TDengine.ErrorNo(res) != 0) - { - Console.WriteLine("SchemalessInsert failed since " + TDengine.Error(res)); - ExitProgram(conn, 1); + IntPtr res = TDengine.SchemalessInsert(conn, lines, 1, (int)TDengineSchemalessProtocol.TSDB_SML_JSON_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + if (TDengine.ErrorNo(res) != 0) + { + throw new Exception("SchemalessInsert failed since " + TDengine.Error(res)); + } + else + { + int affectedRows = TDengine.AffectRows(res); + Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows"); + } + TDengine.FreeResult(res); } - else + finally { - int affectedRows = TDengine.AffectRows(res); - Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows"); + TDengine.Close(conn); } - TDengine.FreeResult(res); - ExitProgram(conn, 0); - } static IntPtr GetConnection() { @@ -39,9 +43,7 @@ namespace TDengineExample var conn = TDengine.Connect(host, username, password, dbname, port); if (conn == IntPtr.Zero) { - Console.WriteLine("Connect to TDengine failed"); - TDengine.Cleanup(); - Environment.Exit(1); + throw new Exception("Connect to TDengine failed"); } else { @@ -55,22 +57,13 @@ namespace TDengineExample IntPtr res = TDengine.Query(conn, "CREATE DATABASE test"); if (TDengine.ErrorNo(res) != 0) { - Console.WriteLine("failed to create database, reason: " + TDengine.Error(res)); - ExitProgram(conn, 1); + throw new Exception("failed to create database, reason: " + TDengine.Error(res)); } res = TDengine.Query(conn, "USE test"); if (TDengine.ErrorNo(res) != 0) { - Console.WriteLine("failed to change database, reason: " + TDengine.Error(res)); - ExitProgram(conn, 1); + throw new Exception("failed to change database, reason: " + TDengine.Error(res)); } } - - static void ExitProgram(IntPtr conn, int exitCode) - { - TDengine.Close(conn); - TDengine.Cleanup(); - Environment.Exit(exitCode); - } } } diff --git a/docs/examples/csharp/optsjson.csproj b/docs/examples/csharp/optsJSON/optsJSON.csproj similarity index 100% rename from docs/examples/csharp/optsjson.csproj rename to docs/examples/csharp/optsJSON/optsJSON.csproj diff --git a/docs/examples/csharp/OptsTelnetExample.cs b/docs/examples/csharp/optsTelnet/Program.cs similarity index 59% rename from docs/examples/csharp/OptsTelnetExample.cs rename to docs/examples/csharp/optsTelnet/Program.cs index bb752db1af..e73ceb041a 100644 --- a/docs/examples/csharp/OptsTelnetExample.cs +++ b/docs/examples/csharp/optsTelnet/Program.cs @@ -7,8 +7,10 @@ namespace TDengineExample static void Main() { IntPtr conn = GetConnection(); - PrepareDatabase(conn); - string[] lines = { + try + { + PrepareDatabase(conn); + string[] lines = { "meters.current 1648432611249 10.3 location=California.SanFrancisco groupid=2", "meters.current 1648432611250 12.6 location=California.SanFrancisco groupid=2", "meters.current 1648432611249 10.8 location=California.LosAngeles groupid=3", @@ -18,20 +20,22 @@ namespace TDengineExample "meters.voltage 1648432611249 221 location=California.LosAngeles groupid=3", "meters.voltage 1648432611250 217 location=California.LosAngeles groupid=3", }; - IntPtr res = TDengine.SchemalessInsert(conn, lines, lines.Length, (int)TDengineSchemalessProtocol.TSDB_SML_TELNET_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED); - if (TDengine.ErrorNo(res) != 0) - { - Console.WriteLine("SchemalessInsert failed since " + TDengine.Error(res)); - ExitProgram(conn, 1); + IntPtr res = TDengine.SchemalessInsert(conn, lines, lines.Length, (int)TDengineSchemalessProtocol.TSDB_SML_TELNET_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + if (TDengine.ErrorNo(res) != 0) + { + throw new Exception("SchemalessInsert failed since " + TDengine.Error(res)); + } + else + { + int affectedRows = TDengine.AffectRows(res); + Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows"); + } + TDengine.FreeResult(res); } - else + catch { - int affectedRows = TDengine.AffectRows(res); - Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows"); + TDengine.Close(conn); } - TDengine.FreeResult(res); - ExitProgram(conn, 0); - } static IntPtr GetConnection() { @@ -43,9 +47,7 @@ namespace TDengineExample var conn = TDengine.Connect(host, username, password, dbname, port); if (conn == IntPtr.Zero) { - Console.WriteLine("Connect to TDengine failed"); - TDengine.Cleanup(); - Environment.Exit(1); + throw new Exception("Connect to TDengine failed"); } else { @@ -59,22 +61,13 @@ namespace TDengineExample IntPtr res = TDengine.Query(conn, "CREATE DATABASE test"); if (TDengine.ErrorNo(res) != 0) { - Console.WriteLine("failed to create database, reason: " + TDengine.Error(res)); - ExitProgram(conn, 1); + throw new Exception("failed to create database, reason: " + TDengine.Error(res)); } res = TDengine.Query(conn, "USE test"); if (TDengine.ErrorNo(res) != 0) { - Console.WriteLine("failed to change database, reason: " + TDengine.Error(res)); - ExitProgram(conn, 1); + throw new Exception("failed to change database, reason: " + TDengine.Error(res)); } } - - static void ExitProgram(IntPtr conn, int exitCode) - { - TDengine.Close(conn); - TDengine.Cleanup(); - Environment.Exit(exitCode); - } } } diff --git a/docs/examples/csharp/optstelnet.csproj b/docs/examples/csharp/optsTelnet/optstelnet.csproj similarity index 100% rename from docs/examples/csharp/optstelnet.csproj rename to docs/examples/csharp/optsTelnet/optstelnet.csproj diff --git a/docs/examples/csharp/query/Program.cs b/docs/examples/csharp/query/Program.cs new file mode 100644 index 0000000000..84c7f9db1f --- /dev/null +++ b/docs/examples/csharp/query/Program.cs @@ -0,0 +1,80 @@ +using TDengineDriver; +using TDengineDriver.Impl; +using System.Runtime.InteropServices; + +namespace TDengineExample +{ + internal class QueryExample + { + static void Main() + { + IntPtr conn = GetConnection(); + try + { + // run query + IntPtr res = TDengine.Query(conn, "SELECT * FROM meters LIMIT 2"); + if (TDengine.ErrorNo(res) != 0) + { + throw new Exception("Failed to query since: " + TDengine.Error(res)); + } + + // get filed count + int fieldCount = TDengine.FieldCount(res); + Console.WriteLine("fieldCount=" + fieldCount); + + // print column names + List metas = LibTaos.GetMeta(res); + for (int i = 0; i < metas.Count; i++) + { + Console.Write(metas[i].name + "\t"); + } + Console.WriteLine(); + + // print values + List resData = LibTaos.GetData(res); + for (int i = 0; i < resData.Count; i++) + { + Console.Write($"|{resData[i].ToString()} \t"); + if (((i + 1) % metas.Count == 0)) + { + Console.WriteLine(""); + } + } + Console.WriteLine(); + + // Free result after use + TDengine.FreeResult(res); + } + finally + { + TDengine.Close(conn); + } + + } + static IntPtr GetConnection() + { + string host = "localhost"; + short port = 6030; + string username = "root"; + string password = "taosdata"; + string dbname = "power"; + var conn = TDengine.Connect(host, username, password, dbname, port); + if (conn == IntPtr.Zero) + { + throw new Exception("Connect to TDengine failed"); + } + else + { + Console.WriteLine("Connect to TDengine success"); + } + return conn; + } + } +} + +// output: +// Connect to TDengine success +// fieldCount=6 +// ts current voltage phase location groupid +// 1648432611249 10.3 219 0.31 California.SanFrancisco 2 +// 1648432611749 12.6 218 0.33 California.SanFrancisco 2 \ No newline at end of file diff --git a/docs/examples/csharp/query.csproj b/docs/examples/csharp/query/query.csproj similarity index 98% rename from docs/examples/csharp/query.csproj rename to docs/examples/csharp/query/query.csproj index 39fc135d5a..c97dbd3051 100644 --- a/docs/examples/csharp/query.csproj +++ b/docs/examples/csharp/query/query.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/sqlInsert/Program.cs b/docs/examples/csharp/sqlInsert/Program.cs new file mode 100644 index 0000000000..f23a6e1663 --- /dev/null +++ b/docs/examples/csharp/sqlInsert/Program.cs @@ -0,0 +1,69 @@ +using TDengineDriver; + + +namespace TDengineExample +{ + internal class SQLInsertExample + { + + static void Main() + { + IntPtr conn = GetConnection(); + try + { + IntPtr res = TDengine.Query(conn, "CREATE DATABASE power"); + CheckRes(conn, res, "failed to create database"); + res = TDengine.Query(conn, "USE power"); + CheckRes(conn, res, "failed to change database"); + res = TDengine.Query(conn, "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)"); + CheckRes(conn, res, "failed to create stable"); + var sql = "INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) " + + "d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) " + + "d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " + + "d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)"; + res = TDengine.Query(conn, sql); + CheckRes(conn, res, "failed to insert data"); + int affectedRows = TDengine.AffectRows(res); + Console.WriteLine("affectedRows " + affectedRows); + TDengine.FreeResult(res); + } + finally + { + TDengine.Close(conn); + } + + } + + static IntPtr GetConnection() + { + string host = "localhost"; + short port = 6030; + string username = "root"; + string password = "taosdata"; + string dbname = ""; + var conn = TDengine.Connect(host, username, password, dbname, port); + if (conn == IntPtr.Zero) + { + throw new Exception("Connect to TDengine failed"); + } + else + { + Console.WriteLine("Connect to TDengine success"); + } + return conn; + } + + static void CheckRes(IntPtr conn, IntPtr res, String errorMsg) + { + if (TDengine.ErrorNo(res) != 0) + { + throw new Exception($"{errorMsg} since: {TDengine.Error(res)}"); + } + } + + } +} + +// output: +// Connect to TDengine success +// affectedRows 8 diff --git a/docs/examples/csharp/sqlinsert.csproj b/docs/examples/csharp/sqlInsert/sqlinsert.csproj similarity index 100% rename from docs/examples/csharp/sqlinsert.csproj rename to docs/examples/csharp/sqlInsert/sqlinsert.csproj diff --git a/docs/examples/csharp/StmtInsertExample.cs b/docs/examples/csharp/stmtInsert/Program.cs similarity index 52% rename from docs/examples/csharp/StmtInsertExample.cs rename to docs/examples/csharp/stmtInsert/Program.cs index 0a4098091f..87e1971feb 100644 --- a/docs/examples/csharp/StmtInsertExample.cs +++ b/docs/examples/csharp/stmtInsert/Program.cs @@ -9,45 +9,50 @@ namespace TDengineExample static void Main() { conn = GetConnection(); - PrepareSTable(); - // 1. init and prepare - stmt = TDengine.StmtInit(conn); - if (stmt == IntPtr.Zero) + try { - Console.WriteLine("failed to init stmt, " + TDengine.Error(stmt)); - ExitProgram(); - } - int res = TDengine.StmtPrepare(stmt, "INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)"); - CheckStmtRes(res, "failed to prepare stmt"); + PrepareSTable(); + // 1. init and prepare + stmt = TDengine.StmtInit(conn); + if (stmt == IntPtr.Zero) + { + throw new Exception("failed to init stmt."); + } + int res = TDengine.StmtPrepare(stmt, "INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)"); + CheckStmtRes(res, "failed to prepare stmt"); - // 2. bind table name and tags - TAOS_MULTI_BIND[] tags = new TAOS_MULTI_BIND[2] { TaosMultiBind.MultiBindBinary(new string[]{"California.SanFrancisco"}), TaosMultiBind.MultiBindInt(new int?[] {2}) }; - res = TDengine.StmtSetTbnameTags(stmt, "d1001", tags); - CheckStmtRes(res, "failed to bind table name and tags"); + // 2. bind table name and tags + TAOS_MULTI_BIND[] tags = new TAOS_MULTI_BIND[2] { TaosMultiBind.MultiBindBinary(new string[] { "California.SanFrancisco" }), TaosMultiBind.MultiBindInt(new int?[] { 2 }) }; + res = TDengine.StmtSetTbnameTags(stmt, "d1001", tags); + CheckStmtRes(res, "failed to bind table name and tags"); - // 3. bind values - TAOS_MULTI_BIND[] values = new TAOS_MULTI_BIND[4] { + // 3. bind values + TAOS_MULTI_BIND[] values = new TAOS_MULTI_BIND[4] { TaosMultiBind.MultiBindTimestamp(new long[2] { 1648432611249, 1648432611749}), TaosMultiBind.MultiBindFloat(new float?[2] { 10.3f, 12.6f}), TaosMultiBind.MultiBindInt(new int?[2] { 219, 218}), TaosMultiBind.MultiBindFloat(new float?[2]{ 0.31f, 0.33f}) }; - res = TDengine.StmtBindParamBatch(stmt, values); - CheckStmtRes(res, "failed to bind params"); + res = TDengine.StmtBindParamBatch(stmt, values); + CheckStmtRes(res, "failed to bind params"); - // 4. add batch - res = TDengine.StmtAddBatch(stmt); - CheckStmtRes(res, "failed to add batch"); + // 4. add batch + res = TDengine.StmtAddBatch(stmt); + CheckStmtRes(res, "failed to add batch"); - // 5. execute - res = TDengine.StmtExecute(stmt); - CheckStmtRes(res, "faild to execute"); + // 5. execute + res = TDengine.StmtExecute(stmt); + CheckStmtRes(res, "faild to execute"); + + // 6. free + TaosMultiBind.FreeTaosBind(tags); + TaosMultiBind.FreeTaosBind(values); + } + finally + { + TDengine.Close(conn); + } - // 6. free - TaosMultiBind.FreeTaosBind(tags); - TaosMultiBind.FreeTaosBind(values); - TDengine.Close(conn); - TDengine.Cleanup(); } static IntPtr GetConnection() @@ -60,8 +65,7 @@ namespace TDengineExample var conn = TDengine.Connect(host, username, password, dbname, port); if (conn == IntPtr.Zero) { - Console.WriteLine("Connect to TDengine failed"); - Environment.Exit(0); + throw new Exception("Connect to TDengine failed"); } else { @@ -70,8 +74,6 @@ namespace TDengineExample return conn; } - - static void PrepareSTable() { IntPtr res = TDengine.Query(conn, "CREATE DATABASE power"); @@ -90,9 +92,8 @@ namespace TDengineExample int code = TDengine.StmtClose(stmt); if (code != 0) { - Console.WriteLine($"falied to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} "); + throw new Exception($"falied to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} "); } - ExitProgram(); } } @@ -100,16 +101,9 @@ namespace TDengineExample { if (TDengine.ErrorNo(res) != 0) { - Console.WriteLine(errorMsg + " since:" + TDengine.Error(res)); - ExitProgram(); + throw new Exception(errorMsg + " since:" + TDengine.Error(res)); } } - static void ExitProgram() - { - TDengine.Close(conn); - TDengine.Cleanup(); - Environment.Exit(1); - } } } diff --git a/docs/examples/csharp/stmtinsert.csproj b/docs/examples/csharp/stmtInsert/stmtinsert.csproj similarity index 100% rename from docs/examples/csharp/stmtinsert.csproj rename to docs/examples/csharp/stmtInsert/stmtinsert.csproj diff --git a/docs/examples/csharp/SubscribeDemo.cs b/docs/examples/csharp/subscribe/Program.cs similarity index 90% rename from docs/examples/csharp/SubscribeDemo.cs rename to docs/examples/csharp/subscribe/Program.cs index b62ff12e5e..1fba209f22 100644 --- a/docs/examples/csharp/SubscribeDemo.cs +++ b/docs/examples/csharp/subscribe/Program.cs @@ -11,11 +11,10 @@ namespace TMQExample { IntPtr conn = GetConnection(); string topic = "topic_example"; - Console.WriteLine($"create topic if not exist {topic} as select * from meters"); //create topic IntPtr res = TDengine.Query(conn, $"create topic if not exists {topic} as select * from meters"); - - if (res == IntPtr.Zero) + + if (TDengine.ErrorNo(res) != 0 ) { throw new Exception($"create topic failed, reason:{TDengine.Error(res)}"); } @@ -26,7 +25,7 @@ namespace TMQExample TDConnectUser = "root", TDConnectPasswd = "taosdata", MsgWithTableName = "true", - TDConnectIp = "127.0.0.1", + TDConnectIp = "127.0.0.1", }; // create consumer @@ -65,7 +64,6 @@ namespace TMQExample List topics = consumer.Subscription(); topics.ForEach(t => Console.WriteLine("topic name:{0}", t)); - // unsubscribe consumer.Unsubscribe(); @@ -73,7 +71,6 @@ namespace TMQExample consumer.Close(); TDengine.Close(conn); - } static IntPtr GetConnection() @@ -86,8 +83,7 @@ namespace TMQExample var conn = TDengine.Connect(host, username, password, dbname, port); if (conn == IntPtr.Zero) { - Console.WriteLine("Connect to TDengine failed"); - System.Environment.Exit(0); + throw new Exception("Connect to TDengine failed"); } else { diff --git a/docs/examples/csharp/subscribe.csproj b/docs/examples/csharp/subscribe/subscribe.csproj similarity index 98% rename from docs/examples/csharp/subscribe.csproj rename to docs/examples/csharp/subscribe/subscribe.csproj index eff29b3bf4..8ae1cf6bc6 100644 --- a/docs/examples/csharp/subscribe.csproj +++ b/docs/examples/csharp/subscribe/subscribe.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/wsInsert/Program.cs b/docs/examples/csharp/wsInsert/Program.cs index 4ff830b437..4cd812cda9 100644 --- a/docs/examples/csharp/wsInsert/Program.cs +++ b/docs/examples/csharp/wsInsert/Program.cs @@ -47,7 +47,7 @@ namespace Examples } else { - Console.WriteLine("{0} success affect {2} rows, cost {1} nanoseconds", desc, LibTaosWS.WSTakeTiming(wsRes),LibTaosWS.WSAffectRows(wsRes)); + Console.WriteLine("{0} success affect {2} rows, cost {1} nanoseconds", desc, LibTaosWS.WSTakeTiming(wsRes), LibTaosWS.WSAffectRows(wsRes)); } } } diff --git a/docs/examples/csharp/wsQuery/Program.cs b/docs/examples/csharp/wsQuery/Program.cs index bf3cf2bbe2..de5591aa53 100644 --- a/docs/examples/csharp/wsQuery/Program.cs +++ b/docs/examples/csharp/wsQuery/Program.cs @@ -55,7 +55,7 @@ namespace Examples // Free result after use. LibTaosWS.WSFreeResult(wsRes); - + // close connection. LibTaosWS.WSClose(wsConn); } diff --git a/docs/zh/07-develop/01-connect/_connect_cs.mdx b/docs/zh/07-develop/01-connect/_connect_cs.mdx index 9d0755fc64..169bf37a63 100644 --- a/docs/zh/07-develop/01-connect/_connect_cs.mdx +++ b/docs/zh/07-develop/01-connect/_connect_cs.mdx @@ -1,5 +1,5 @@ ```csharp title="原生连接" -{{#include docs/examples/csharp/ConnectExample.cs}} +{{#include docs/examples/csharp/connect/Program.cs}} ``` ```csharp title="WebSocket 连接" diff --git a/docs/zh/07-develop/03-insert-data/_cs_line.mdx b/docs/zh/07-develop/03-insert-data/_cs_line.mdx index 71f46c62be..ae49901c3a 100644 --- a/docs/zh/07-develop/03-insert-data/_cs_line.mdx +++ b/docs/zh/07-develop/03-insert-data/_cs_line.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/InfluxDBLineExample.cs}} +{{#include docs/examples/csharp/influxdbLine/Program.cs}} ``` diff --git a/docs/zh/07-develop/03-insert-data/_cs_opts_json.mdx b/docs/zh/07-develop/03-insert-data/_cs_opts_json.mdx index 8d80d042c9..2627648616 100644 --- a/docs/zh/07-develop/03-insert-data/_cs_opts_json.mdx +++ b/docs/zh/07-develop/03-insert-data/_cs_opts_json.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/OptsJsonExample.cs}} +{{#include docs/examples/csharp/optsJSON/Program.cs}} ``` diff --git a/docs/zh/07-develop/03-insert-data/_cs_opts_telnet.mdx b/docs/zh/07-develop/03-insert-data/_cs_opts_telnet.mdx index cff32abf1f..660db13fd1 100644 --- a/docs/zh/07-develop/03-insert-data/_cs_opts_telnet.mdx +++ b/docs/zh/07-develop/03-insert-data/_cs_opts_telnet.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/OptsTelnetExample.cs}} +{{#include docs/examples/csharp/optsTelnet/Program.cs}} ``` diff --git a/docs/zh/07-develop/03-insert-data/_cs_sql.mdx b/docs/zh/07-develop/03-insert-data/_cs_sql.mdx index 1dc7bb3d13..42a6bc4315 100644 --- a/docs/zh/07-develop/03-insert-data/_cs_sql.mdx +++ b/docs/zh/07-develop/03-insert-data/_cs_sql.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/SQLInsertExample.cs}} +{{#include docs/examples/csharp/sqlInsert/Program.cs}} ``` diff --git a/docs/zh/07-develop/03-insert-data/_cs_stmt.mdx b/docs/zh/07-develop/03-insert-data/_cs_stmt.mdx index 229c874ab9..d8d73ca15e 100644 --- a/docs/zh/07-develop/03-insert-data/_cs_stmt.mdx +++ b/docs/zh/07-develop/03-insert-data/_cs_stmt.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/StmtInsertExample.cs}} +{{#include docs/examples/csharp/stmtInsert/Program.cs}} ``` diff --git a/docs/zh/07-develop/04-query-data/_cs.mdx b/docs/zh/07-develop/04-query-data/_cs.mdx index 4bb582ecbf..745ab36811 100644 --- a/docs/zh/07-develop/04-query-data/_cs.mdx +++ b/docs/zh/07-develop/04-query-data/_cs.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/QueryExample.cs}} +{{#include docs/examples/csharp/query/Program.cs}} ``` diff --git a/docs/zh/07-develop/04-query-data/_cs_async.mdx b/docs/zh/07-develop/04-query-data/_cs_async.mdx index 3ecf635fd3..19c8e58f32 100644 --- a/docs/zh/07-develop/04-query-data/_cs_async.mdx +++ b/docs/zh/07-develop/04-query-data/_cs_async.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/AsyncQueryExample.cs}} +{{#include docs/examples/csharp/asyncQuery/Program.cs}} ``` diff --git a/docs/zh/07-develop/_sub_cs.mdx b/docs/zh/07-develop/_sub_cs.mdx index a435ea0273..093b617e9b 100644 --- a/docs/zh/07-develop/_sub_cs.mdx +++ b/docs/zh/07-develop/_sub_cs.mdx @@ -1,3 +1,3 @@ ```csharp -{{#include docs/examples/csharp/SubscribeDemo.cs}} +{{#include docs/examples/csharp/subscribe/Program.cs}} ``` \ No newline at end of file diff --git a/docs/zh/08-connector/40-csharp.mdx b/docs/zh/08-connector/40-csharp.mdx index 9ba8be2c22..f78fd9e3d9 100644 --- a/docs/zh/08-connector/40-csharp.mdx +++ b/docs/zh/08-connector/40-csharp.mdx @@ -208,7 +208,7 @@ namespace TDengineExample ``` csharp -{{#include docs/examples/csharp/StmtInsertExample.cs}} +{{#include docs/examples/csharp/stmtInsert/Program.cs}} ``` diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 7b31e10572..179a3c6df0 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -325,7 +325,7 @@ charset 的有效值是 UTF-8。 | 适用范围 | 仅服务端适用 | | 含义 | dnode 支持的最大 vnode 数目 | | 取值范围 | 0-4096 | -| 缺省值 | 256 | +| 缺省值 | CPU 核数的 2 倍 | ## 时间相关 @@ -668,153 +668,154 @@ charset 的有效值是 UTF-8。 | 15 | telemetryPort | 否 | 是 | | 16 | queryPolicy | 否 | 是 | | 17 | querySmaOptimize | 否 | 是 | -| 18 | queryBufferSize | 是 | 是 | -| 19 | maxNumOfDistinctRes | 是 | 是 | -| 20 | minSlidingTime | 是 | 是 | -| 21 | minIntervalTime | 是 | 是 | -| 22 | countAlwaysReturnValue | 是 | 是 | -| 23 | dataDir | 是 | 是 | -| 24 | minimalDataDirGB | 是 | 是 | -| 25 | supportVnodes | 否 | 是 | -| 26 | tempDir | 是 | 是 | -| 27 | minimalTmpDirGB | 是 | 是 | -| 28 | compressMsgSize | 是 | 是 | -| 29 | compressColData | 是 | 是 | -| 30 | smlChildTableName | 是 | 是 | -| 31 | smlTagName | 是 | 是 | -| 32 | smlDataFormat | 否 | 是 | -| 33 | statusInterval | 是 | 是 | -| 34 | shellActivityTimer | 是 | 是 | -| 35 | transPullupInterval | 否 | 是 | -| 36 | mqRebalanceInterval | 否 | 是 | -| 37 | ttlUnit | 否 | 是 | -| 38 | ttlPushInterval | 否 | 是 | -| 39 | numOfTaskQueueThreads | 否 | 是 | -| 40 | numOfRpcThreads | 否 | 是 | -| 41 | numOfCommitThreads | 是 | 是 | -| 42 | numOfMnodeReadThreads | 否 | 是 | -| 43 | numOfVnodeQueryThreads | 否 | 是 | -| 44 | numOfVnodeStreamThreads | 否 | 是 | -| 45 | numOfVnodeFetchThreads | 否 | 是 | -| 46 | numOfVnodeWriteThreads | 否 | 是 | -| 47 | numOfVnodeSyncThreads | 否 | 是 | -| 48 | numOfVnodeRsmaThreads | 否 | 是 | -| 49 | numOfQnodeQueryThreads | 否 | 是 | -| 50 | numOfQnodeFetchThreads | 否 | 是 | -| 51 | numOfSnodeSharedThreads | 否 | 是 | -| 52 | numOfSnodeUniqueThreads | 否 | 是 | -| 53 | rpcQueueMemoryAllowed | 否 | 是 | -| 54 | logDir | 是 | 是 | -| 55 | minimalLogDirGB | 是 | 是 | -| 56 | numOfLogLines | 是 | 是 | -| 57 | asyncLog | 是 | 是 | -| 58 | logKeepDays | 是 | 是 | -| 59 | debugFlag | 是 | 是 | -| 60 | tmrDebugFlag | 是 | 是 | -| 61 | uDebugFlag | 是 | 是 | -| 62 | rpcDebugFlag | 是 | 是 | -| 63 | jniDebugFlag | 是 | 是 | -| 64 | qDebugFlag | 是 | 是 | -| 65 | cDebugFlag | 是 | 是 | -| 66 | dDebugFlag | 是 | 是 | -| 67 | vDebugFlag | 是 | 是 | -| 68 | mDebugFlag | 是 | 是 | -| 69 | wDebugFlag | 是 | 是 | -| 70 | sDebugFlag | 是 | 是 | -| 71 | tsdbDebugFlag | 是 | 是 | -| 72 | tqDebugFlag | 否 | 是 | -| 73 | fsDebugFlag | 是 | 是 | -| 74 | udfDebugFlag | 否 | 是 | -| 75 | smaDebugFlag | 否 | 是 | -| 76 | idxDebugFlag | 否 | 是 | -| 77 | tdbDebugFlag | 否 | 是 | -| 78 | metaDebugFlag | 否 | 是 | -| 79 | timezone | 是 | 是 | -| 80 | locale | 是 | 是 | -| 81 | charset | 是 | 是 | -| 82 | udf | 是 | 是 | -| 83 | enableCoreFile | 是 | 是 | -| 84 | arbitrator | 是 | 否 | -| 85 | numOfThreadsPerCore | 是 | 否 | -| 86 | numOfMnodes | 是 | 否 | -| 87 | vnodeBak | 是 | 否 | -| 88 | balance | 是 | 否 | -| 89 | balanceInterval | 是 | 否 | -| 90 | offlineThreshold | 是 | 否 | -| 91 | role | 是 | 否 | -| 92 | dnodeNopLoop | 是 | 否 | -| 93 | keepTimeOffset | 是 | 否 | -| 94 | rpcTimer | 是 | 否 | -| 95 | rpcMaxTime | 是 | 否 | -| 96 | rpcForceTcp | 是 | 否 | -| 97 | tcpConnTimeout | 是 | 否 | -| 98 | syncCheckInterval | 是 | 否 | -| 99 | maxTmrCtrl | 是 | 否 | -| 100 | monitorReplica | 是 | 否 | -| 101 | smlTagNullName | 是 | 否 | -| 102 | keepColumnName | 是 | 否 | -| 103 | ratioOfQueryCores | 是 | 否 | -| 104 | maxStreamCompDelay | 是 | 否 | -| 105 | maxFirstStreamCompDelay | 是 | 否 | -| 106 | retryStreamCompDelay | 是 | 否 | -| 107 | streamCompDelayRatio | 是 | 否 | -| 108 | maxVgroupsPerDb | 是 | 否 | -| 109 | maxTablesPerVnode | 是 | 否 | -| 110 | minTablesPerVnode | 是 | 否 | -| 111 | tableIncStepPerVnode | 是 | 否 | -| 112 | cache | 是 | 否 | -| 113 | blocks | 是 | 否 | -| 114 | days | 是 | 否 | -| 115 | keep | 是 | 否 | -| 116 | minRows | 是 | 否 | -| 117 | maxRows | 是 | 否 | -| 118 | quorum | 是 | 否 | -| 119 | comp | 是 | 否 | -| 120 | walLevel | 是 | 否 | -| 121 | fsync | 是 | 否 | -| 122 | replica | 是 | 否 | -| 123 | partitions | 是 | 否 | -| 124 | quorum | 是 | 否 | -| 125 | update | 是 | 否 | -| 126 | cachelast | 是 | 否 | -| 127 | maxSQLLength | 是 | 否 | -| 128 | maxWildCardsLength | 是 | 否 | -| 129 | maxRegexStringLen | 是 | 否 | -| 130 | maxNumOfOrderedRes | 是 | 否 | -| 131 | maxConnections | 是 | 否 | -| 132 | mnodeEqualVnodeNum | 是 | 否 | -| 133 | http | 是 | 否 | -| 134 | httpEnableRecordSql | 是 | 否 | -| 135 | httpMaxThreads | 是 | 否 | -| 136 | restfulRowLimit | 是 | 否 | -| 137 | httpDbNameMandatory | 是 | 否 | -| 138 | httpKeepAlive | 是 | 否 | -| 139 | enableRecordSql | 是 | 否 | -| 140 | maxBinaryDisplayWidth | 是 | 否 | -| 141 | stream | 是 | 否 | -| 142 | retrieveBlockingModel | 是 | 否 | -| 143 | tsdbMetaCompactRatio | 是 | 否 | -| 144 | defaultJSONStrType | 是 | 否 | -| 145 | walFlushSize | 是 | 否 | -| 146 | keepTimeOffset | 是 | 否 | -| 147 | flowctrl | 是 | 否 | -| 148 | slaveQuery | 是 | 否 | -| 149 | adjustMaster | 是 | 否 | -| 150 | topicBinaryLen | 是 | 否 | -| 151 | telegrafUseFieldNum | 是 | 否 | -| 152 | deadLockKillQuery | 是 | 否 | -| 153 | clientMerge | 是 | 否 | -| 154 | sdbDebugFlag | 是 | 否 | -| 155 | odbcDebugFlag | 是 | 否 | -| 156 | httpDebugFlag | 是 | 否 | -| 157 | monDebugFlag | 是 | 否 | -| 158 | cqDebugFlag | 是 | 否 | -| 159 | shortcutFlag | 是 | 否 | -| 160 | probeSeconds | 是 | 否 | -| 161 | probeKillSeconds | 是 | 否 | -| 162 | probeInterval | 是 | 否 | -| 163 | lossyColumns | 是 | 否 | -| 164 | fPrecision | 是 | 否 | -| 165 | dPrecision | 是 | 否 | -| 166 | maxRange | 是 | 否 | -| 167 | range | 是 | 否 | +| 18 | queryRsmaTolerance | 否 | 是 | +| 19 | queryBufferSize | 是 | 是 | +| 20 | maxNumOfDistinctRes | 是 | 是 | +| 21 | minSlidingTime | 是 | 是 | +| 22 | minIntervalTime | 是 | 是 | +| 23 | countAlwaysReturnValue | 是 | 是 | +| 24 | dataDir | 是 | 是 | +| 25 | minimalDataDirGB | 是 | 是 | +| 26 | supportVnodes | 否 | 是 | +| 27 | tempDir | 是 | 是 | +| 28 | minimalTmpDirGB | 是 | 是 | +| 29 | compressMsgSize | 是 | 是 | +| 30 | compressColData | 是 | 是 | +| 31 | smlChildTableName | 是 | 是 | +| 32 | smlTagName | 是 | 是 | +| 33 | smlDataFormat | 否 | 是 | +| 34 | statusInterval | 是 | 是 | +| 35 | shellActivityTimer | 是 | 是 | +| 36 | transPullupInterval | 否 | 是 | +| 37 | mqRebalanceInterval | 否 | 是 | +| 38 | ttlUnit | 否 | 是 | +| 39 | ttlPushInterval | 否 | 是 | +| 40 | numOfTaskQueueThreads | 否 | 是 | +| 41 | numOfRpcThreads | 否 | 是 | +| 42 | numOfCommitThreads | 是 | 是 | +| 43 | numOfMnodeReadThreads | 否 | 是 | +| 44 | numOfVnodeQueryThreads | 否 | 是 | +| 45 | numOfVnodeStreamThreads | 否 | 是 | +| 46 | numOfVnodeFetchThreads | 否 | 是 | +| 47 | numOfVnodeWriteThreads | 否 | 是 | +| 48 | numOfVnodeSyncThreads | 否 | 是 | +| 49 | numOfVnodeRsmaThreads | 否 | 是 | +| 50 | numOfQnodeQueryThreads | 否 | 是 | +| 51 | numOfQnodeFetchThreads | 否 | 是 | +| 52 | numOfSnodeSharedThreads | 否 | 是 | +| 53 | numOfSnodeUniqueThreads | 否 | 是 | +| 54 | rpcQueueMemoryAllowed | 否 | 是 | +| 55 | logDir | 是 | 是 | +| 56 | minimalLogDirGB | 是 | 是 | +| 57 | numOfLogLines | 是 | 是 | +| 58 | asyncLog | 是 | 是 | +| 59 | logKeepDays | 是 | 是 | +| 60 | debugFlag | 是 | 是 | +| 61 | tmrDebugFlag | 是 | 是 | +| 62 | uDebugFlag | 是 | 是 | +| 63 | rpcDebugFlag | 是 | 是 | +| 64 | jniDebugFlag | 是 | 是 | +| 65 | qDebugFlag | 是 | 是 | +| 66 | cDebugFlag | 是 | 是 | +| 67 | dDebugFlag | 是 | 是 | +| 68 | vDebugFlag | 是 | 是 | +| 69 | mDebugFlag | 是 | 是 | +| 70 | wDebugFlag | 是 | 是 | +| 71 | sDebugFlag | 是 | 是 | +| 72 | tsdbDebugFlag | 是 | 是 | +| 73 | tqDebugFlag | 否 | 是 | +| 74 | fsDebugFlag | 是 | 是 | +| 75 | udfDebugFlag | 否 | 是 | +| 76 | smaDebugFlag | 否 | 是 | +| 77 | idxDebugFlag | 否 | 是 | +| 78 | tdbDebugFlag | 否 | 是 | +| 79 | metaDebugFlag | 否 | 是 | +| 80 | timezone | 是 | 是 | +| 81 | locale | 是 | 是 | +| 82 | charset | 是 | 是 | +| 83 | udf | 是 | 是 | +| 84 | enableCoreFile | 是 | 是 | +| 85 | arbitrator | 是 | 否 | +| 86 | numOfThreadsPerCore | 是 | 否 | +| 87 | numOfMnodes | 是 | 否 | +| 88 | vnodeBak | 是 | 否 | +| 89 | balance | 是 | 否 | +| 90 | balanceInterval | 是 | 否 | +| 91 | offlineThreshold | 是 | 否 | +| 92 | role | 是 | 否 | +| 93 | dnodeNopLoop | 是 | 否 | +| 94 | keepTimeOffset | 是 | 否 | +| 95 | rpcTimer | 是 | 否 | +| 96 | rpcMaxTime | 是 | 否 | +| 97 | rpcForceTcp | 是 | 否 | +| 98 | tcpConnTimeout | 是 | 否 | +| 99 | syncCheckInterval | 是 | 否 | +| 100 | maxTmrCtrl | 是 | 否 | +| 101 | monitorReplica | 是 | 否 | +| 102 | smlTagNullName | 是 | 否 | +| 103 | keepColumnName | 是 | 否 | +| 104 | ratioOfQueryCores | 是 | 否 | +| 105 | maxStreamCompDelay | 是 | 否 | +| 106 | maxFirstStreamCompDelay | 是 | 否 | +| 107 | retryStreamCompDelay | 是 | 否 | +| 108 | streamCompDelayRatio | 是 | 否 | +| 109 | maxVgroupsPerDb | 是 | 否 | +| 110 | maxTablesPerVnode | 是 | 否 | +| 111 | minTablesPerVnode | 是 | 否 | +| 112 | tableIncStepPerVnode | 是 | 否 | +| 113 | cache | 是 | 否 | +| 114 | blocks | 是 | 否 | +| 115 | days | 是 | 否 | +| 116 | keep | 是 | 否 | +| 117 | minRows | 是 | 否 | +| 118 | maxRows | 是 | 否 | +| 119 | quorum | 是 | 否 | +| 120 | comp | 是 | 否 | +| 121 | walLevel | 是 | 否 | +| 122 | fsync | 是 | 否 | +| 123 | replica | 是 | 否 | +| 124 | partitions | 是 | 否 | +| 125 | quorum | 是 | 否 | +| 126 | update | 是 | 否 | +| 127 | cachelast | 是 | 否 | +| 128 | maxSQLLength | 是 | 否 | +| 129 | maxWildCardsLength | 是 | 否 | +| 130 | maxRegexStringLen | 是 | 否 | +| 131 | maxNumOfOrderedRes | 是 | 否 | +| 132 | maxConnections | 是 | 否 | +| 133 | mnodeEqualVnodeNum | 是 | 否 | +| 134 | http | 是 | 否 | +| 135 | httpEnableRecordSql | 是 | 否 | +| 136 | httpMaxThreads | 是 | 否 | +| 137 | restfulRowLimit | 是 | 否 | +| 138 | httpDbNameMandatory | 是 | 否 | +| 139 | httpKeepAlive | 是 | 否 | +| 140 | enableRecordSql | 是 | 否 | +| 141 | maxBinaryDisplayWidth | 是 | 否 | +| 142 | stream | 是 | 否 | +| 143 | retrieveBlockingModel | 是 | 否 | +| 144 | tsdbMetaCompactRatio | 是 | 否 | +| 145 | defaultJSONStrType | 是 | 否 | +| 146 | walFlushSize | 是 | 否 | +| 147 | keepTimeOffset | 是 | 否 | +| 148 | flowctrl | 是 | 否 | +| 149 | slaveQuery | 是 | 否 | +| 150 | adjustMaster | 是 | 否 | +| 151 | topicBinaryLen | 是 | 否 | +| 152 | telegrafUseFieldNum | 是 | 否 | +| 153 | deadLockKillQuery | 是 | 否 | +| 154 | clientMerge | 是 | 否 | +| 155 | sdbDebugFlag | 是 | 否 | +| 156 | odbcDebugFlag | 是 | 否 | +| 157 | httpDebugFlag | 是 | 否 | +| 158 | monDebugFlag | 是 | 否 | +| 159 | cqDebugFlag | 是 | 否 | +| 160 | shortcutFlag | 是 | 否 | +| 161 | probeSeconds | 是 | 否 | +| 162 | probeKillSeconds | 是 | 否 | +| 163 | probeInterval | 是 | 否 | +| 164 | lossyColumns | 是 | 否 | +| 165 | fPrecision | 是 | 否 | +| 166 | dPrecision | 是 | 否 | +| 167 | maxRange | 是 | 否 | +| 168 | range | 是 | 否 | diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 59a241b6c8..b05cf7a942 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -6,6 +6,9 @@ description: TDengine 发布历史、Release Notes 及下载链接 import Release from "/components/ReleaseV3"; +## 3.0.1.2 + + ## 3.0.1.1 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index 9ff8f42224..f793981d06 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -6,6 +6,10 @@ description: taosTools 的发布历史、Release Notes 和下载链接 import Release from "/components/ReleaseV3"; +## 2.2.2 + + + ## 2.2.0 diff --git a/examples/lua/OpenResty/so/luaconnector51.so b/examples/lua/OpenResty/so/luaconnector51.so index 442de6e39f..168d3a9d24 100755 Binary files a/examples/lua/OpenResty/so/luaconnector51.so and b/examples/lua/OpenResty/so/luaconnector51.so differ diff --git a/examples/lua/lua51/lua_connector51.c b/examples/lua/lua51/lua_connector51.c index 578622bf1f..4c702b2aae 100644 --- a/examples/lua/lua51/lua_connector51.c +++ b/examples/lua/lua51/lua_connector51.c @@ -2,7 +2,7 @@ #include #include #include -#include "../../../../include/client/taos.h" +#include "taos.h" #include "lauxlib.h" #include "lua.h" #include "lualib.h" @@ -35,7 +35,7 @@ static int l_connect(lua_State *L){ } lua_getfield(L, 1, "port"); - if (lua_isnumber(L,-1)){ + if (lua_isnumber(L, -1)){ port = lua_tonumber(L, -1); //printf("port = %d\n", port); } @@ -60,7 +60,6 @@ static int l_connect(lua_State *L){ lua_settop(L,0); - taos_init(); lua_newtable(L); int table_index = lua_gettop(L); @@ -102,7 +101,7 @@ static int l_query(lua_State *L){ printf("failed, reason:%s\n", taos_errstr(result)); lua_pushinteger(L, -1); lua_setfield(L, table_index, "code"); - lua_pushstring(L, taos_errstr(taos)); + lua_pushstring(L, taos_errstr(result)); lua_setfield(L, table_index, "error"); return 1; @@ -113,7 +112,6 @@ static int l_query(lua_State *L){ int rows = 0; int num_fields = taos_field_count(result); const TAOS_FIELD *fields = taos_fetch_fields(result); - //char temp[256]; const int affectRows = taos_affected_rows(result); // printf(" affect rows:%d\r\n", affectRows); @@ -122,7 +120,7 @@ static int l_query(lua_State *L){ lua_pushinteger(L, affectRows); lua_setfield(L, table_index, "affected"); lua_newtable(L); - + while ((row = taos_fetch_row(result))) { //printf("row index:%d\n",rows); rows++; @@ -136,7 +134,7 @@ static int l_query(lua_State *L){ } lua_pushstring(L,fields[i].name); - + int32_t* length = taos_fetch_lengths(result); switch (fields[i].type) { case TSDB_DATA_TYPE_TINYINT: lua_pushinteger(L,*((char *)row[i])); @@ -158,7 +156,8 @@ static int l_query(lua_State *L){ break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - lua_pushstring(L,(char *)row[i]); + //printf("type:%d, max len:%d, current len:%d\n",fields[i].type, fields[i].bytes, length[i]); + lua_pushlstring(L,(char *)row[i], length[i]); break; case TSDB_DATA_TYPE_TIMESTAMP: lua_pushinteger(L,*((int64_t *)row[i])); @@ -166,6 +165,7 @@ static int l_query(lua_State *L){ case TSDB_DATA_TYPE_BOOL: lua_pushinteger(L,*((char *)row[i])); break; + case TSDB_DATA_TYPE_NULL: default: lua_pushnil(L); break; @@ -235,112 +235,6 @@ static int l_async_query(lua_State *L){ return 1; } -void stream_cb(void *param, TAOS_RES *result, TAOS_ROW row){ - struct cb_param* p = (struct cb_param*) param; - TAOS_FIELD *fields = taos_fetch_fields(result); - int numFields = taos_num_fields(result); - - // printf("\nnumfields:%d\n", numFields); - //printf("\n\r-----------------------------------------------------------------------------------\n"); - - lua_State *L = p->state; - lua_rawgeti(L, LUA_REGISTRYINDEX, p->callback); - - lua_newtable(L); - - for (int i = 0; i < numFields; ++i) { - if (row[i] == NULL) { - continue; - } - - lua_pushstring(L,fields[i].name); - - switch (fields[i].type) { - case TSDB_DATA_TYPE_TINYINT: - lua_pushinteger(L,*((char *)row[i])); - break; - case TSDB_DATA_TYPE_SMALLINT: - lua_pushinteger(L,*((short *)row[i])); - break; - case TSDB_DATA_TYPE_INT: - lua_pushinteger(L,*((int *)row[i])); - break; - case TSDB_DATA_TYPE_BIGINT: - lua_pushinteger(L,*((int64_t *)row[i])); - break; - case TSDB_DATA_TYPE_FLOAT: - lua_pushnumber(L,*((float *)row[i])); - break; - case TSDB_DATA_TYPE_DOUBLE: - lua_pushnumber(L,*((double *)row[i])); - break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - lua_pushstring(L,(char *)row[i]); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - lua_pushinteger(L,*((int64_t *)row[i])); - break; - case TSDB_DATA_TYPE_BOOL: - lua_pushinteger(L,*((char *)row[i])); - break; - default: - lua_pushnil(L); - break; - } - - lua_settable(L, -3); - } - - lua_call(L, 1, 0); - - // printf("-----------------------------------------------------------------------------------\n\r"); -} - -static int l_open_stream(lua_State *L){ - int r = luaL_ref(L, LUA_REGISTRYINDEX); - TAOS * taos = (TAOS*)lua_topointer(L,1); - const char * sqlstr = lua_tostring(L,2); - int stime = luaL_checknumber(L,3); - - lua_newtable(L); - int table_index = lua_gettop(L); - - struct cb_param *p = malloc(sizeof(struct cb_param)); - p->state = L; - p->callback=r; - // printf("r:%d, L:%d\n",r,L); - void * s = taos_open_stream(taos,sqlstr,stream_cb,stime,p,NULL); - if (s == NULL) { - printf("failed to open stream, reason:%s\n", taos_errstr(taos)); - free(p); - lua_pushnumber(L, -1); - lua_setfield(L, table_index, "code"); - lua_pushstring(L, taos_errstr(taos)); - lua_setfield(L, table_index, "error"); - lua_pushlightuserdata(L,NULL); - lua_setfield(L, table_index, "stream"); - }else{ - // printf("success to open stream\n"); - lua_pushnumber(L, 0); - lua_setfield(L, table_index, "code"); - lua_pushstring(L, taos_errstr(taos)); - lua_setfield(L, table_index, "error"); - p->stream = s; - lua_pushlightuserdata(L,p); - lua_setfield(L, table_index, "stream");//stream has different content in lua and c. - } - - return 1; -} - -static int l_close_stream(lua_State *L){ - //TODO:get stream and free cb_param - struct cb_param *p = lua_touserdata(L,1); - taos_close_stream(p->stream); - free(p); - return 0; -} static int l_close(lua_State *L){ TAOS *taos= (TAOS*)lua_topointer(L,1); @@ -367,8 +261,6 @@ static const struct luaL_Reg lib[] = { {"query", l_query}, {"query_a",l_async_query}, {"close", l_close}, - {"open_stream", l_open_stream}, - {"close_stream", l_close_stream}, {NULL, NULL} }; diff --git a/examples/lua/lua_connector.c b/examples/lua/lua_connector.c index 3c13b196b9..ce13ab3829 100644 --- a/examples/lua/lua_connector.c +++ b/examples/lua/lua_connector.c @@ -5,7 +5,7 @@ #include #include #include -#include "taos.h" +#include struct cb_param{ lua_State* state; @@ -60,6 +60,8 @@ static int l_connect(lua_State *L){ lua_settop(L,0); + taos_init(); + lua_newtable(L); int table_index = lua_gettop(L); diff --git a/examples/lua/test.lua b/examples/lua/test.lua index 89c0904c6a..3d725cc6a3 100644 --- a/examples/lua/test.lua +++ b/examples/lua/test.lua @@ -9,6 +9,50 @@ local config = { max_packet_size = 1024 * 1024 } +function dump(obj) + local getIndent, quoteStr, wrapKey, wrapVal, dumpObj + getIndent = function(level) + return string.rep("\t", level) + end + quoteStr = function(str) + return '"' .. string.gsub(str, '"', '\\"') .. '"' + end + wrapKey = function(val) + if type(val) == "number" then + return "[" .. val .. "]" + elseif type(val) == "string" then + return "[" .. quoteStr(val) .. "]" + else + return "[" .. tostring(val) .. "]" + end + end + wrapVal = function(val, level) + if type(val) == "table" then + return dumpObj(val, level) + elseif type(val) == "number" then + return val + elseif type(val) == "string" then + return quoteStr(val) + else + return tostring(val) + end + end + dumpObj = function(obj, level) + if type(obj) ~= "table" then + return wrapVal(obj) + end + level = level + 1 + local tokens = {} + tokens[#tokens + 1] = "{" + for k, v in pairs(obj) do + tokens[#tokens + 1] = getIndent(level) .. wrapKey(k) .. " = " .. wrapVal(v, level) .. "," + end + tokens[#tokens + 1] = getIndent(level - 1) .. "}" + return table.concat(tokens, "\n") + end + return dumpObj(obj, 0) +end + local conn local res = driver.connect(config) if res.code ~=0 then @@ -37,7 +81,7 @@ else print("select db--- pass.") end -res = driver.query(conn,"create table m1 (ts timestamp, speed int,owner binary(20))") +res = driver.query(conn,"create table m1 (ts timestamp, speed int, owner binary(20), mark nchar(30))") if res.code ~=0 then print("create table---failed: "..res.error) return @@ -45,7 +89,7 @@ else print("create table--- pass.") end -res = driver.query(conn,"insert into m1 values ('2019-09-01 00:00:00.001',0,'robotspace'), ('2019-09-01 00:00:00.002',1,'Hilink'),('2019-09-01 00:00:00.003',2,'Harmony')") +res = driver.query(conn,"insert into m1 values ('2019-09-01 00:00:00.001', 0, 'robotspace', '世界人民大团结万岁'), ('2019-09-01 00:00:00.002', 1, 'Hilink', '⾾⾿⿀⿁⿂⿃⿄⿅⿆⿇⿈⿉⿊⿋⿌⿍⿎⿏⿐⿑⿒⿓⿔⿕'),('2019-09-01 00:00:00.003', 2, 'Harmony', '₠₡₢₣₤₥₦₧₨₩₪₫€₭₮₯₰₱₲₳₴₵')") if res.code ~=0 then print("insert records failed: "..res.error) return @@ -64,21 +108,25 @@ if res.code ~=0 then return else if (#(res.item) == 3) then - print("select--- pass") + print("select--- pass") + print(res.item[1].mark) + print(res.item[2].mark) + print(res.item[3].mark) + else print("select--- failed: expect 3 affected records, actually received "..#(res.item)) end end -res = driver.query(conn,"CREATE TABLE thermometer (ts timestamp, degree double) TAGS(location binary(20), type int)") +res = driver.query(conn,"create table thermometer (ts timestamp, degree double) tags(location binary(20), type int)") if res.code ~=0 then print(res.error) return else print("create super table--- pass") end -res = driver.query(conn,"CREATE TABLE therm1 USING thermometer TAGS ('beijing', 1)") +res = driver.query(conn,"create table therm1 using thermometer tags ('beijing', 1)") if res.code ~=0 then print(res.error) return @@ -86,7 +134,7 @@ else print("create table--- pass") end -res = driver.query(conn,"INSERT INTO therm1 VALUES ('2019-09-01 00:00:00.001', 20),('2019-09-01 00:00:00.002', 21)") +res = driver.query(conn,"insert into therm1 values ('2019-09-01 00:00:00.001', 20),('2019-09-01 00:00:00.002', 21)") if res.code ~=0 then print(res.error) @@ -99,14 +147,14 @@ else end end -res = driver.query(conn,"SELECT COUNT(*) count, AVG(degree) AS av, MAX(degree), MIN(degree) FROM thermometer WHERE location='beijing' or location='tianjin' GROUP BY location, type") +res = driver.query(conn,"select count(*) as cnt, avg(degree) as av, max(degree), min(degree) from thermometer where location='beijing' or location='tianjin' group by location, type") if res.code ~=0 then print("select from super table--- failed:"..res.error) return else print("select from super table--- pass") for i = 1, #(res.item) do - print("res:"..res.item[i].count) + print("res:"..res.item[i].cnt) end end @@ -127,30 +175,13 @@ end driver.query_a(conn,"INSERT INTO therm1 VALUES ('2019-09-01 00:00:00.005', 100),('2019-09-01 00:00:00.006', 101),('2019-09-01 00:00:00.007', 102)", async_query_callback) +res = driver.query(conn, "create stream stream_avg_degree into avg_degree as select avg(degree) from thermometer interval(5s) sliding(1s)") -function stream_callback(t) - print("------------------------") - print("continuous query result:") - for key, value in pairs(t) do - print("key:"..key..", value:"..value) - end -end - -local stream -res = driver.open_stream(conn,"SELECT COUNT(*) as count, AVG(degree) as avg, MAX(degree) as max, MIN(degree) as min FROM thermometer interval(2s) sliding(2s);)",0, stream_callback) -if res.code ~=0 then - print("open stream--- failed:"..res.error) - return -else - print("open stream--- pass") - stream = res.stream -end - -print("From now on we start continous insert in an definite (infinite if you want) loop.") +print("From now on we start continous insert in an definite loop, pls wait for about 10 seconds and check stream table for result.") local loop_index = 0 -while loop_index < 30 do +while loop_index < 10 do local t = os.time()*1000 - local v = loop_index + local v = math.random(20) res = driver.query(conn,string.format("INSERT INTO therm1 VALUES (%d, %d)",t,v)) if res.code ~=0 then @@ -162,7 +193,5 @@ while loop_index < 30 do os.execute("sleep " .. 1) loop_index = loop_index + 1 end - -driver.close_stream(stream) - +driver.query(conn,"DROP STREAM IF EXISTS avg_therm_s") driver.close(conn) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 66bae5ad3b..9b69bec5b3 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -94,7 +94,10 @@ extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in // query client extern int32_t tsQueryPolicy; extern int32_t tsQuerySmaOptimize; +extern int32_t tsQueryRsmaTolerance; extern bool tsQueryPlannerTrace; +extern int32_t tsQueryNodeChunkSize; +extern bool tsQueryUseNodeAllocator; // client extern int32_t tsMinSlidingTime; diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 65ddc180d6..60c7b18367 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -78,7 +78,6 @@ enum { MAIN_SCAN = 0x0u, REVERSE_SCAN = 0x1u, // todo remove it REPEAT_SCAN = 0x2u, // repeat scan belongs to the master scan - MERGE_STAGE = 0x20u, }; typedef struct SPoint1 { @@ -156,11 +155,6 @@ typedef struct SqlFunctionCtx { char udfName[TSDB_FUNC_NAME_LEN]; } SqlFunctionCtx; -enum { - TEXPR_BINARYEXPR_NODE = 0x1, - TEXPR_UNARYEXPR_NODE = 0x2, -}; - typedef struct tExprNode { int32_t nodeType; union { @@ -182,8 +176,9 @@ struct SScalarParam { SColumnInfoData *columnData; SHashObj *pHashFilter; int32_t hashValueType; - void *param; // other parameter, such as meta handle from vnode, to extract table name/tag value + void *param; // other parameter, such as meta handle from vnode, to extract table name/tag value int32_t numOfRows; + int32_t numOfQualified; // number of qualified elements in the final results }; void cleanupResultRowEntry(struct SResultRowEntryInfo *pCell); @@ -201,8 +196,6 @@ int32_t taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // udf api -struct SUdfInfo; - /** * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf * @return error code @@ -226,6 +219,7 @@ int32_t udfStartUdfd(int32_t startDnodeId); * @return */ int32_t udfStopUdfd(); + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 6500d3d183..634dae9ec5 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -275,6 +275,17 @@ typedef struct SNodeList { SListCell* pTail; } SNodeList; +typedef struct SNodeAllocator SNodeAllocator; + +int32_t nodesInitAllocatorSet(); +void nodesDestroyAllocatorSet(); +int32_t nodesCreateAllocator(int64_t queryId, int32_t chunkSize, int64_t* pAllocatorId); +int32_t nodesAcquireAllocator(int64_t allocatorId); +int32_t nodesReleaseAllocator(int64_t allocatorId); +int64_t nodesMakeAllocatorWeakRef(int64_t allocatorId); +int64_t nodesReleaseAllocatorWeakRef(int64_t allocatorId); +void nodesDestroyAllocator(int64_t allocatorId); + SNode* nodesMakeNode(ENodeType type); void nodesDestroyNode(SNode* pNode); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 95bde85864..b1a937910d 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -56,6 +56,7 @@ typedef struct SParseContext { bool nodeOffline; SArray* pTableMetaPos; // sql table pos => catalog data pos SArray* pTableVgroupPos; // sql table pos => catalog data pos + int64_t allocatorId; } SParseContext; int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery); diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index e03ac3811a..e52fe39527 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -39,6 +39,7 @@ typedef struct SPlanContext { int32_t msgLen; const char* pUser; bool sysInfo; + int64_t allocatorId; } SPlanContext; // Create the physical plan for the query, according to the AST. diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index 1f1d9dea93..e02b4a6172 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -31,13 +31,17 @@ enum { FLT_OPTION_NEED_UNIQE = 4, }; +#define FILTER_RESULT_ALL_QUALIFIED 0x1 +#define FILTER_RESULT_NONE_QUALIFIED 0x2 +#define FILTER_RESULT_PARTIAL_QUALIFIED 0x3 + typedef struct SFilterColumnParam { int32_t numOfCols; SArray *pDataBlock; } SFilterColumnParam; extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options); -extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t **p, SColumnDataAgg *statis, int16_t numOfCols); +extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData** p, SColumnDataAgg *statis, int16_t numOfCols, int32_t* pFilterResStatus); extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param); extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param); extern int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict); diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index e6973cd390..738d057e6a 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -67,6 +67,7 @@ typedef struct SSchedulerReq { SRequestConnInfo *pConn; SArray *pNodeList; SQueryPlan *pDag; + int64_t allocatorRefId; const char *sql; int64_t startTs; schedulerExecFp execFp; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 840e7309fe..837d0c6303 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -552,7 +552,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_VALUE_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x2653) #define TSDB_CODE_PAR_INVALID_DELETE_WHERE TAOS_DEF_ERROR_CODE(0, 0x2655) #define TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG TAOS_DEF_ERROR_CODE(0, 0x2656) - #define TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x2657) #define TSDB_CODE_PAR_INVALID_WINDOW_PC TAOS_DEF_ERROR_CODE(0, 0x2658) #define TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x2659) @@ -565,6 +564,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_SELECTED_EXPR TAOS_DEF_ERROR_CODE(0, 0x2661) #define TSDB_CODE_PAR_GET_META_ERROR TAOS_DEF_ERROR_CODE(0, 0x2662) #define TSDB_CODE_PAR_NOT_UNIQUE_TABLE_ALIAS TAOS_DEF_ERROR_CODE(0, 0x2663) +#define TSDB_CODE_PAR_NOT_SUPPORT_JOIN TAOS_DEF_ERROR_CODE(0, 0x2664) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/include/util/tdef.h b/include/util/tdef.h index 840a2671fa..0c7c2ea3a3 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -225,7 +225,8 @@ typedef enum ELogicConditionType { #define TSDB_APP_NAME_LEN TSDB_UNI_LEN #define TSDB_TB_COMMENT_LEN 1025 -#define TSDB_QUERY_ID_LEN 26 +#define TSDB_QUERY_ID_LEN 26 +#define TSDB_TRANS_OPER_LEN 16 /** * In some scenarios uint16_t (0~65535) is used to store the row len. diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index 794b3968fe..20f93ecaec 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -202,8 +202,8 @@ elif [[ ${packgeName} =~ "tar" ]];then cd ${oriInstallPath}/${originTdpPath} && tar xf ${subFile} fi - cd ${oriInstallPath}/${originTdpPath} && tree > ${installPath}/base_${originversion}_checkfile - cd ${installPath}/${tdPath} && tree > ${installPath}/now_${version}_checkfile + cd ${oriInstallPath}/${originTdpPath} && tree -I "driver" > ${installPath}/base_${originversion}_checkfile + cd ${installPath}/${tdPath} && tree -I "driver" > ${installPath}/now_${version}_checkfile cd ${installPath} diff ${installPath}/base_${originversion}_checkfile ${installPath}/now_${version}_checkfile > ${installPath}/diffFile.log @@ -215,6 +215,7 @@ elif [[ ${packgeName} =~ "tar" ]];then exit -1 else echoColor G "The number and names of files are the same as previous installation packages" + rm -rf ${installPath}/diffFile.log fi echoColor YD "===== install Package of tar =====" cd ${installPath}/${tdPath} @@ -251,6 +252,9 @@ if [[ ${packgeName} =~ "server" ]] ;then systemctl restart taosd fi +rm -rf ${installPath}/${packgeName} +rm -rf ${installPath}/${tdPath}/ + # if ([[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "tar" ]]) || [[ ${packgeName} =~ "client" ]] ;then # echoColor G "===== install taos-tools when package is lite or client =====" # cd ${installPath} diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 574d8188fe..535a436c6c 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -59,9 +59,9 @@ enum { #define SHOW_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) -#define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY) -#define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ) -#define TD_RES_TMQ_META(res) (*(int8_t*)res == RES_TYPE__TMQ_META) +#define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY) +#define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ) +#define TD_RES_TMQ_META(res) (*(int8_t*)res == RES_TYPE__TMQ_META) #define TD_RES_TMQ_METADATA(res) (*(int8_t*)res == RES_TYPE__TMQ_METADATA) typedef struct SAppInstInfo SAppInstInfo; @@ -250,6 +250,8 @@ typedef struct SRequestObj { bool inRetry; uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog uint32_t retry; + int64_t allocatorRefId; + SQuery* pQuery; } SRequestObj; typedef struct SSyncQueryParam { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index b739aedca0..624abe6da5 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -288,6 +288,7 @@ void *createRequest(uint64_t connId, int32_t type) { pRequest->body.resInfo.convertUcs4 = true; // convert ucs4 by default pRequest->type = type; + pRequest->allocatorRefId = -1; pRequest->pDb = getDbOfConnection(pTscObj); pRequest->pTscObj = pTscObj; @@ -349,6 +350,8 @@ void doDestroyRequest(void *p) { taosArrayDestroy(pRequest->tableList); taosArrayDestroy(pRequest->dbList); taosArrayDestroy(pRequest->targetTableList); + qDestroyQuery(pRequest->pQuery); + nodesDestroyAllocator(pRequest->allocatorRefId); destroyQueryExecRes(&pRequest->body.resInfo.execRes); @@ -411,6 +414,7 @@ void taos_init_imp(void) { initTaskQueue(); fmFuncMgtInit(); + nodesInitAllocatorSet(); clientConnRefPool = taosOpenRef(200, destroyTscObj); clientReqRefPool = taosOpenRef(40960, doDestroyRequest); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 75ccd44977..acf87f2aca 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -274,13 +274,13 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { SClientHbBatchRsp pRsp = {0}; if (TSDB_CODE_SUCCESS == code) { tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp); - } - int32_t now = taosGetTimestampSec(); - int32_t delta = abs(now - pRsp.svrTimestamp); - if (delta > timestampDeltaLimit) { - code = TSDB_CODE_TIME_UNSYNCED; - tscError("time diff: %ds is too big", delta); + int32_t now = taosGetTimestampSec(); + int32_t delta = abs(now - pRsp.svrTimestamp); + if (delta > timestampDeltaLimit) { + code = TSDB_CODE_TIME_UNSYNCED; + tscError("time diff: %ds is too big", delta); + } } int32_t rspNum = taosArrayGetSize(pRsp.rsps); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 5128695359..034f78e5f6 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -195,6 +195,19 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, return TSDB_CODE_TSC_OUT_OF_MEMORY; } + (*pRequest)->allocatorRefId = -1; + if (tsQueryUseNodeAllocator && !qIsInsertValuesSql((*pRequest)->sqlstr, (*pRequest)->sqlLen)) { + if (TSDB_CODE_SUCCESS != + nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) { + tscError("%d failed to create node allocator, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self, + (*pRequest)->requestId, pTscObj->id, sql); + + destroyRequest(*pRequest); + *pRequest = NULL; + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + tscDebugL("0x%" PRIx64 " SQL: %s, reqId:0x%" PRIx64, (*pRequest)->self, (*pRequest)->sqlstr, (*pRequest)->requestId); return TSDB_CODE_SUCCESS; } @@ -682,6 +695,8 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList }; int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob); + + destroyQueryExecRes(&pRequest->body.resInfo.execRes); memcpy(&pRequest->body.resInfo.execRes, &res, sizeof(res)); if (code != TSDB_CODE_SUCCESS) { @@ -1023,7 +1038,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pUser = pRequest->pTscObj->user, - .sysInfo = pRequest->pTscObj->sysInfo}; + .sysInfo = pRequest->pTscObj->sysInfo, + .allocatorId = pRequest->allocatorRefId}; SAppInstInfo* pAppInfo = getAppInfo(pRequest); SQueryPlan* pDag = NULL; @@ -1048,6 +1064,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM .pConn = &conn, .pNodeList = pNodeList, .pDag = pDag, + .allocatorRefId = pRequest->allocatorRefId, .sql = pRequest->sqlstr, .startTs = pRequest->metric.start, .execFp = schedulerExecCb, diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 13cbaa0e22..831212bdee 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -65,6 +65,7 @@ void taos_cleanup(void) { fmFuncMgtDestroy(); qCleanupKeywordsTable(); + nodesDestroyAllocatorSet(); id = clientConnRefPool; clientConnRefPool = -1; @@ -669,7 +670,6 @@ typedef struct SqlParseWrapper { SParseContext *pCtx; SCatalogReq catalogReq; SRequestObj *pRequest; - SQuery *pQuery; } SqlParseWrapper; static void destoryTablesReq(void *p) { @@ -695,8 +695,8 @@ static void destorySqlParseWrapper(SqlParseWrapper *pWrapper) { void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { SqlParseWrapper *pWrapper = (SqlParseWrapper *)param; - SQuery *pQuery = pWrapper->pQuery; SRequestObj *pRequest = pWrapper->pRequest; + SQuery *pQuery = pRequest->pQuery; pRequest->metric.ctgEnd = taosGetTimestampUs(); @@ -725,10 +725,10 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { tscDebug("0x%" PRIx64 " analysis semantics completed, start async query, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId); launchAsyncQuery(pRequest, pQuery, pResultMeta); - qDestroyQuery(pQuery); } else { destorySqlParseWrapper(pWrapper); - qDestroyQuery(pQuery); + qDestroyQuery(pRequest->pQuery); + pRequest->pQuery = NULL; if (NEED_CLIENT_HANDLE_ERROR(code)) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); @@ -775,7 +775,8 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { .enableSysInfo = pTscObj->sysInfo, .async = true, .svrVer = pTscObj->sVer, - .nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes)}; + .nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes), + .allocatorId = pRequest->allocatorRefId}; return TSDB_CODE_SUCCESS; } @@ -800,12 +801,10 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { goto _error; } - SQuery *pQuery = NULL; - pRequest->metric.syntaxStart = taosGetTimestampUs(); SCatalogReq catalogReq = {.forceUpdate = updateMetaForce, .qNodeRequired = qnodeRequired(pRequest)}; - code = qParseSqlSyntax(pCxt, &pQuery, &catalogReq); + code = qParseSqlSyntax(pCxt, &pRequest->pQuery, &catalogReq); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -815,9 +814,9 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { if (!updateMetaForce) { STscObj *pTscObj = pRequest->pTscObj; SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; - if (NULL == pQuery->pRoot) { + if (NULL == pRequest->pQuery->pRoot) { atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); - } else if (QUERY_NODE_SELECT_STMT == pQuery->pRoot->type) { + } else if (QUERY_NODE_SELECT_STMT == pRequest->pQuery->pRoot->type) { atomic_add_fetch_64((int64_t *)&pActivity->numOfQueryReq, 1); } } @@ -829,7 +828,6 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { } pWrapper->pCtx = pCxt; - pWrapper->pQuery = pQuery; pWrapper->pRequest = pRequest; pWrapper->catalogReq = catalogReq; diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 25abd65b40..a45f7b2913 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -227,6 +227,7 @@ static const SSysDbTableSchema transSchema[] = { {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "stage", .bytes = TSDB_TRANS_STAGE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "oper", .bytes = TSDB_TRANS_OPER_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "stable", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "failed_times", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 8a8e6a83a5..23cc868658 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1278,7 +1278,9 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { colDataAssign(pDst, pSrc, src->info.rows, &src->info); } + uint32_t cap = dst->info.capacity; dst->info = src->info; + dst->info.capacity = cap; return 0; } @@ -1302,8 +1304,9 @@ int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) { colDataAssign(pDst, pSrc, src->info.rows, &src->info); } - + uint32_t cap = dst->info.capacity; dst->info = src->info; + dst->info.capacity = cap; return TSDB_CODE_SUCCESS; } @@ -1333,6 +1336,8 @@ SSDataBlock* createSpecialDataBlock(EStreamType type) { // group id taosArrayPush(pBlock->pDataBlock, &infoData); + infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP; + infoData.info.bytes = sizeof(TSKEY); // calculate start ts taosArrayPush(pBlock->pDataBlock, &infoData); // calculate end ts @@ -2280,4 +2285,3 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { ASSERT(pStart - pData == dataLen); return pStart; } - diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 80be8c6a35..09f5d4b4b2 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -91,7 +91,10 @@ bool tsSmlDataFormat = // query int32_t tsQueryPolicy = 1; int32_t tsQuerySmaOptimize = 0; +int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data. bool tsQueryPlannerTrace = false; +int32_t tsQueryNodeChunkSize = 32 * 1024; +bool tsQueryUseNodeAllocator = true; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -285,6 +288,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1; if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, 1) != 0) return -1; if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, true) != 0) return -1; + if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, true) != 0) return -1; + if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, true) != 0) return -1; if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1; if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; @@ -419,6 +424,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, 1) != 0) return -1; if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 100000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, 0) != 0) return -1; if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1; if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, 0) != 0) return -1; @@ -644,6 +650,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32; tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval; + tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32; + tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval; return 0; } @@ -716,6 +724,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32; tsTtlPushInterval = cfgGetItem(pCfg, "ttlPushInterval")->i32; tsUptimeInterval = cfgGetItem(pCfg, "uptimeInterval")->i32; + tsQueryRsmaTolerance = cfgGetItem(pCfg, "queryRsmaTolerance")->i32; tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; tstrncpy(tsUdfdResFuncs, cfgGetItem(pCfg, "udfdResFuncs")->str, sizeof(tsUdfdResFuncs)); @@ -978,6 +987,12 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; } else if (strcasecmp("queryPlannerTrace", name) == 0) { tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval; + } else if (strcasecmp("queryNodeChunkSize", name) == 0) { + tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32; + } else if (strcasecmp("queryUseNodeAllocator", name) == 0) { + tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval; + } else if (strcasecmp("queryRsmaTolerance", name) == 0) { + tsQueryRsmaTolerance = cfgGetItem(pCfg, "queryRsmaTolerance")->i32; } break; } diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index ebbb9fa5d4..30f5483198 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -74,6 +74,7 @@ typedef struct { TdThread thread; SVnodeMgmt *pMgmt; SWrapperCfg *pCfgs; + SVnodeObj **ppVnodes; } SVnodeThread; // vmInt.c diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 1f981cc9e0..f0c43d8b36 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -218,14 +218,14 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) { dInfo("start to close all vnodes"); int32_t numOfVnodes = 0; - SVnodeObj **pVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); + SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); for (int32_t i = 0; i < numOfVnodes; ++i) { - vmCloseVnode(pMgmt, pVnodes[i]); + vmCloseVnode(pMgmt, ppVnodes[i]); } - if (pVnodes != NULL) { - taosMemoryFree(pVnodes); + if (ppVnodes != NULL) { + taosMemoryFree(ppVnodes); } if (pMgmt->hash != NULL) { @@ -331,22 +331,92 @@ static int32_t vmRequire(const SMgmtInputOpt *pInput, bool *required) { return 0; } -static int32_t vmStart(SVnodeMgmt *pMgmt) { - int32_t numOfVnodes = 0; - SVnodeObj **pVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); +static void *vmRestoreVnodeInThread(void *param) { + SVnodeThread *pThread = param; + SVnodeMgmt *pMgmt = pThread->pMgmt; - for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = pVnodes[i]; - vnodeStart(pVnode->pImpl); + dInfo("thread:%d, start to restore %d vnodes", pThread->threadIndex, pThread->vnodeNum); + setThreadName("restore-vnodes"); + + for (int32_t v = 0; v < pThread->vnodeNum; ++v) { + SVnodeObj *pVnode = pThread->ppVnodes[v]; + + char stepDesc[TSDB_STEP_DESC_LEN] = {0}; + snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been restored", pVnode->vgId, + pMgmt->state.openVnodes, pMgmt->state.totalVnodes); + tmsgReportStartup("vnode-restore", stepDesc); + + int32_t code = vnodeStart(pVnode->pImpl); + if (code != 0) { + dError("vgId:%d, failed to restore vnode by thread:%d", pVnode->vgId, pThread->threadIndex); + pThread->failed++; + } else { + dDebug("vgId:%d, is restored by thread:%d", pVnode->vgId, pThread->threadIndex); + pThread->opened++; + atomic_add_fetch_32(&pMgmt->state.openVnodes, 1); + } } + dInfo("thread:%d, numOfVnodes:%d, restored:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, + pThread->failed); + return NULL; +} + +static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) { + int32_t numOfVnodes = 0; + SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); + + int32_t threadNum = tsNumOfCores / 2; + if (threadNum < 1) threadNum = 1; + int32_t vnodesPerThread = numOfVnodes / threadNum + 1; + + SVnodeThread *threads = taosMemoryCalloc(threadNum, sizeof(SVnodeThread)); + for (int32_t t = 0; t < threadNum; ++t) { + threads[t].threadIndex = t; + threads[t].pMgmt = pMgmt; + threads[t].ppVnodes = taosMemoryCalloc(vnodesPerThread, sizeof(SVnode *)); + } + + for (int32_t v = 0; v < numOfVnodes; ++v) { + int32_t t = v % threadNum; + SVnodeThread *pThread = &threads[t]; + pThread->ppVnodes[pThread->vnodeNum++] = ppVnodes[v]; + } + + pMgmt->state.openVnodes = 0; + dInfo("restore %d vnodes with %d threads", numOfVnodes, threadNum); + + for (int32_t t = 0; t < threadNum; ++t) { + SVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum == 0) continue; + + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&pThread->thread, &thAttr, vmRestoreVnodeInThread, pThread) != 0) { + dError("thread:%d, failed to create thread to restore vnode since %s", pThread->threadIndex, strerror(errno)); + } + + taosThreadAttrDestroy(&thAttr); + } + + for (int32_t t = 0; t < threadNum; ++t) { + SVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { + taosThreadJoin(pThread->thread, NULL); + taosThreadClear(&pThread->thread); + } + taosMemoryFree(pThread->ppVnodes); + } + taosMemoryFree(threads); + for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = pVnodes[i]; + SVnodeObj *pVnode = ppVnodes[i]; vmReleaseVnode(pMgmt, pVnode); } - if (pVnodes != NULL) { - taosMemoryFree(pVnodes); + if (ppVnodes != NULL) { + taosMemoryFree(ppVnodes); } return 0; @@ -360,7 +430,7 @@ SMgmtFunc vmGetMgmtFunc() { SMgmtFunc mgmtFunc = {0}; mgmtFunc.openFp = vmInit; mgmtFunc.closeFp = (NodeCloseFp)vmCleanup; - mgmtFunc.startFp = (NodeStartFp)vmStart; + mgmtFunc.startFp = (NodeStartFp)vmStartVnodes; mgmtFunc.stopFp = (NodeStopFp)vmStop; mgmtFunc.requiredFp = vmRequire; mgmtFunc.getHandlesFp = vmGetMsgHandles; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 537c9ca834..c3d03a6c5e 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -171,6 +171,7 @@ typedef struct { int32_t stopFunc; int32_t paramLen; void* param; + char opername[TSDB_TRANS_OPER_LEN]; SArray* pRpcArray; } STrans; diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 36d056a941..2372fa30e5 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -61,7 +61,8 @@ void mndCleanupTrans(SMnode *pMnode); STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId); void mndReleaseTrans(SMnode *pMnode, STrans *pTrans); -STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, const SRpcMsg *pReq); +STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, const SRpcMsg *pReq, + const char *opername); void mndTransDrop(STrans *pTrans); int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw); diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 33f0bb7a34..e0713e1570 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -81,7 +81,7 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) { mDebug("acct:%s, will be created when deploying, raw:%p", acctObj.acct, pRaw); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "create-acct"); if (pTrans == NULL) { mError("acct:%s, failed to create since %s", acctObj.acct, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index aafcd19992..6cabf6d57a 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -246,7 +246,7 @@ static int32_t mndCreateBnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, bnodeObj.createdTime = taosGetTimestampMs(); bnodeObj.updateTime = bnodeObj.createdTime; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-bnode"); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create bnode:%d", pTrans->id, pCreate->dnodeId); @@ -354,7 +354,7 @@ static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBn static int32_t mndDropBnode(SMnode *pMnode, SRpcMsg *pReq, SBnodeObj *pObj) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-bnode"); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop bnode:%d", pTrans->id, pObj->id); diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 7d633f90bd..5ba39b4f1a 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -235,7 +235,7 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { mDebug("cluster:%" PRId64 ", will be created when deploying, raw:%p", clusterObj.id, pRaw); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "create-cluster"); if (pTrans == NULL) { mError("cluster:%" PRId64 ", failed to create since %s", clusterObj.id, terrstr()); return -1; @@ -316,7 +316,7 @@ static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) { } mTrace("update cluster uptime to %" PRId64, clusterObj.upTime); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "update-uptime"); if (pTrans == NULL) return -1; SSdbRaw *pCommitRaw = mndClusterActionEncode(&clusterObj); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index cdfa208665..e0dbc26122 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -109,7 +109,7 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) { mndReleaseConsumer(pMnode, pConsumer); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "lost-csm"); if (pTrans == NULL) goto FAIL; if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL; if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL; @@ -142,7 +142,7 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { mndReleaseConsumer(pMnode, pConsumer); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "recover-csm"); if (pTrans == NULL) goto FAIL; if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL; if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL; @@ -465,7 +465,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { int32_t newTopicNum = taosArrayGetSize(newSub); // check topic existance - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe"); if (pTrans == NULL) goto SUBSCRIBE_OVER; for (int32_t i = 0; i < newTopicNum; i++) { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index d42ba1f7ff..39ec3484db 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -541,7 +541,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, } int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "create-db"); if (pTrans == NULL) goto _OVER; // mndTransSetSerial(pTrans); mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); @@ -773,7 +773,7 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * } static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *pNew) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "alter-db"); if (pTrans == NULL) return -1; mDebug("trans:%d, used to alter db:%s", pTrans->id, pOld->name); @@ -1027,7 +1027,7 @@ static int32_t mndBuildDropDbRsp(SDbObj *pDb, int32_t *pRspLen, void **ppRsp, bo static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-db"); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 26b4080d14..c6fdf23023 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -104,7 +104,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { memcpy(&dnodeObj.fqdn, tsLocalFqdn, TSDB_FQDN_LEN); snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port); - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-dnode"); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create dnode:%s on first deploy", pTrans->id, dnodeObj.ep); @@ -488,7 +488,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC memcpy(dnodeObj.fqdn, pCreate->fqdn, TSDB_FQDN_LEN); snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port); - pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq); + pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq, "create-dnode"); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep); @@ -667,7 +667,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM SSdbRaw *pRaw = NULL; STrans *pTrans = NULL; - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "drop-dnode"); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index e6f4b48524..429d840202 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -219,7 +219,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre } memcpy(func.pCode, pCreate->pCode, func.codeSize); - pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); + pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-func"); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create func:%s", pTrans->id, pCreate->name); @@ -249,7 +249,7 @@ _OVER: static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "drop-func"); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 71bda4d4f3..a41f958c0f 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -91,7 +91,7 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) { mInfo("mnode:%d, will be created when deploying, raw:%p", mnodeObj.id, pRaw); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-mnode"); if (pTrans == NULL) { mError("mnode:%d, failed to create since %s", mnodeObj.id, terrstr()); return -1; @@ -362,7 +362,7 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, mnodeObj.createdTime = taosGetTimestampMs(); mnodeObj.updateTime = mnodeObj.createdTime; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "create-mnode"); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); @@ -571,7 +571,7 @@ static int32_t mndDropMnode(SMnode *pMnode, SRpcMsg *pReq, SMnodeObj *pObj) { int32_t code = -1; STrans *pTrans = NULL; - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "drop-mnode"); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id); diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index 037a46345f..797aa88670 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -181,7 +181,7 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) { tDecodeSMqCMCommitOffsetReq(&decoder, &commitOffsetReq); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "commit-offset"); for (int32_t i = 0; i < commitOffsetReq.num; i++) { SMqOffset *pOffset = &commitOffsetReq.offsets[i]; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index f057f6190d..cd6e91b154 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -248,7 +248,7 @@ static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, qnodeObj.createdTime = taosGetTimestampMs(); qnodeObj.updateTime = qnodeObj.createdTime; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-qnode"); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId); @@ -364,7 +364,7 @@ int32_t mndSetDropQnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SQnodeObj *pO static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-qnode"); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 8638cc5118..3fdd3995f9 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -587,7 +587,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea nodesDestroyNode((SNode *)pPlan); int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "create-sma"); if (pTrans == NULL) goto _OVER; mndTransSetDbName(pTrans, pDb->name, NULL); mndTransSetSerial(pTrans); @@ -799,7 +799,7 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p pStb = mndAcquireStb(pMnode, pSma->stb); if (pStb == NULL) goto _OVER; - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-sma"); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop sma:%s", pTrans->id, pSma->name); diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index d18a233d29..5f4231aa6a 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -253,7 +253,7 @@ static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, snodeObj.createdTime = taosGetTimestampMs(); snodeObj.updateTime = snodeObj.createdTime; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode"); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); @@ -375,7 +375,7 @@ int32_t mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pO static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode"); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8719f9f8f0..ecb24becba 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -800,7 +800,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea SStbObj stbObj = {0}; int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "create-stb"); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); @@ -1843,7 +1843,7 @@ _OVER: static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp, void *alterOriData, int32_t alterOriDataLen) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "alter-stb"); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to alter stb:%s", pTrans->id, pStb->name); @@ -2042,7 +2042,7 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "drop-stb"); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 515d5987b9..a039b54b80 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -666,7 +666,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "create-stream"); if (pTrans == NULL) { mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); goto _OVER; @@ -759,7 +759,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { return -1; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-stream"); if (pTrans == NULL) { mError("stream:%s, failed to drop since %s", dropReq.name, terrstr()); sdbRelease(pMnode->pSdb, pStream); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 3a3bf36594..43eae3f60c 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -441,7 +441,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "persist-reb"); mndTransSetDbName(pTrans, pOutput->pSub->dbName, NULL); if (pTrans == NULL) return -1; @@ -674,7 +674,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pReq) { return -1; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "drop-cgroup"); if (pTrans == NULL) { mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); mndReleaseSubscribe(pMnode, pSub); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index b24d7067bc..8fe2d1af15 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -440,7 +440,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * /*topicObj.withTbName = 1;*/ /*topicObj.withSchema = 1;*/ - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-topic"); if (pTrans == NULL) { mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); taosMemoryFreeClear(topicObj.ast); @@ -663,7 +663,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { return -1; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "drop-topic"); mndTransSetDbName(pTrans, pTopic->db, NULL); if (pTrans == NULL) { mError("topic:%s, failed to drop since %s", pTopic->name, terrstr()); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 8d42cdcfd6..9d918e2abd 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -25,7 +25,7 @@ #define TRANS_VER_NUMBER 1 #define TRANS_ARRAY_SIZE 8 -#define TRANS_RESERVE_SIZE 64 +#define TRANS_RESERVE_SIZE 48 static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); @@ -223,6 +223,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, _OVER) } + SDB_SET_BINARY(pRaw, dataPos, pTrans->opername, TSDB_TRANS_OPER_LEN, _OVER) SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -305,6 +306,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { if (pTrans->commitActions == NULL) goto _OVER; for (int32_t i = 0; i < redoActionNum; ++i) { + memset(&action, 0, sizeof(action)); SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) @@ -340,6 +342,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { } for (int32_t i = 0; i < undoActionNum; ++i) { + memset(&action, 0, sizeof(action)); SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) @@ -375,6 +378,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { } for (int32_t i = 0; i < commitActionNum; ++i) { + memset(&action, 0, sizeof(action)); SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) @@ -417,6 +421,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, _OVER); } + SDB_GET_BINARY(pRaw, dataPos, pTrans->opername, TSDB_TRANS_OPER_LEN, _OVER); SDB_GET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) terrno = 0; @@ -455,6 +460,20 @@ static const char *mndTransStr(ETrnStage stage) { } } +static void mndSetTransLastAction(STrans *pTrans, STransAction *pAction) { + if (pAction != NULL) { + pTrans->lastAction = pAction->id; + pTrans->lastMsgType = pAction->msgType; + pTrans->lastEpset = pAction->epSet; + pTrans->lastErrorNo = pAction->errCode; + } else { + pTrans->lastAction = 0; + pTrans->lastMsgType = 0; + memset(&pTrans->lastEpset, 0, sizeof(pTrans->lastEpset)); + pTrans->lastErrorNo = 0; + } +} + static void mndTransTestStartFunc(SMnode *pMnode, void *param, int32_t paramLen) { mInfo("test trans start, param:%s, len:%d", (char *)param, paramLen); } @@ -582,7 +601,8 @@ void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { sdbRelease(pSdb, pTrans); } -STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, const SRpcMsg *pReq) { +STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, const SRpcMsg *pReq, + const char *opername) { STrans *pTrans = taosMemoryCalloc(1, sizeof(STrans)); if (pTrans == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -590,6 +610,10 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, return NULL; } + if (opername != NULL) { + tstrncpy(pTrans->opername, opername, TSDB_TRANS_OPER_LEN); + } + pTrans->id = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; @@ -1037,18 +1061,12 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi mInfo("trans:%d, %s:%d write to sdb, type:%s status:%s", pTrans->id, mndTransStr(pAction->stage), pAction->id, sdbTableName(pAction->pRaw->type), sdbStatusName(pAction->pRaw->status)); - pTrans->lastAction = pAction->id; - pTrans->lastMsgType = pAction->msgType; - pTrans->lastEpset = pAction->epSet; - pTrans->lastErrorNo = 0; + mndSetTransLastAction(pTrans, pAction); } else { pAction->errCode = (terrno != 0) ? terrno : code; mError("trans:%d, %s:%d failed to write sdb since %s, type:%s status:%s", pTrans->id, mndTransStr(pAction->stage), pAction->id, terrstr(), sdbTableName(pAction->pRaw->type), sdbStatusName(pAction->pRaw->status)); - pTrans->lastAction = pAction->id; - pTrans->lastMsgType = pAction->msgType; - pTrans->lastEpset = pAction->epSet; - pTrans->lastErrorNo = pAction->errCode; + mndSetTransLastAction(pTrans, pAction); } return code; @@ -1082,15 +1100,10 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio if (code == 0) { pAction->msgSent = 1; pAction->msgReceived = 0; - pAction->errCode = 0; + pAction->errCode = TSDB_CODE_ACTION_IN_PROGRESS; mInfo("trans:%d, %s:%d is sent, %s", pTrans->id, mndTransStr(pAction->stage), pAction->id, detail); - pTrans->lastAction = pAction->id; - pTrans->lastMsgType = pAction->msgType; - pTrans->lastEpset = pAction->epSet; - if (pTrans->lastErrorNo == 0) { - pTrans->lastErrorNo = TSDB_CODE_ACTION_IN_PROGRESS; - } + mndSetTransLastAction(pTrans, pAction); } else { pAction->msgSent = 0; pAction->msgReceived = 0; @@ -1098,10 +1111,7 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio mError("trans:%d, %s:%d not send since %s, %s", pTrans->id, mndTransStr(pAction->stage), pAction->id, terrstr(), detail); - pTrans->lastAction = pAction->id; - pTrans->lastMsgType = pAction->msgType; - pTrans->lastEpset = pAction->epSet; - pTrans->lastErrorNo = pAction->errCode; + mndSetTransLastAction(pTrans, pAction); } return code; @@ -1112,10 +1122,7 @@ static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction pAction->errCode = 0; mInfo("trans:%d, %s:%d confirm action executed", pTrans->id, mndTransStr(pAction->stage), pAction->id); - pTrans->lastAction = pAction->id; - pTrans->lastMsgType = pAction->msgType; - pTrans->lastEpset = pAction->epSet; - pTrans->lastErrorNo = 0; + mndSetTransLastAction(pTrans, pAction); return 0; } @@ -1161,25 +1168,19 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA errCode = pAction->errCode; pErrAction = pAction; } + } else { + pErrAction = pAction; } } + mndSetTransLastAction(pTrans, pErrAction); + if (numOfExecuted == numOfActions) { if (errCode == 0) { - pTrans->lastAction = 0; - pTrans->lastMsgType = 0; - memset(&pTrans->lastEpset, 0, sizeof(pTrans->lastEpset)); - pTrans->lastErrorNo = 0; mInfo("trans:%d, all %d actions execute successfully", pTrans->id, numOfActions); return 0; } else { mError("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errCode & 0XFFFF); - if (pErrAction != NULL) { - pTrans->lastAction = pErrAction->id; - pTrans->lastMsgType = pErrAction->msgType; - pTrans->lastEpset = pErrAction->epSet; - pTrans->lastErrorNo = pErrAction->errCode; - } mndTransResetActions(pMnode, pTrans, pArray); terrno = errCode; return errCode; @@ -1220,6 +1221,8 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) if (numOfActions == 0) return code; if (pTrans->redoActionPos >= numOfActions) return code; + mInfo("trans:%d, execute %d actions serial", pTrans->id, numOfActions); + for (int32_t action = pTrans->redoActionPos; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pTrans->redoActions, pTrans->redoActionPos); @@ -1248,16 +1251,8 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) if (code == 0) { pTrans->failedTimes = 0; - pTrans->lastAction = action; - pTrans->lastMsgType = 0; - pTrans->lastErrorNo = 0; - memset(&pTrans->lastEpset, 0, sizeof(pTrans->lastEpset)); - } else { - pTrans->lastAction = action; - pTrans->lastMsgType = pAction->msgType; - pTrans->lastErrorNo = code; - pTrans->lastEpset = pAction->epSet; } + mndSetTransLastAction(pTrans, pAction); if (mndCannotExecuteTransAction(pMnode)) break; @@ -1599,6 +1594,11 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)stage, false); + char opername[TSDB_TRANS_OPER_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(opername, pTrans->opername, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)opername, false); + char dbname[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(dbname, mndGetDbStr(pTrans->dbname), pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5da119bb30..c4adb03e21 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -81,7 +81,7 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char mDebug("user:%s, will be created when deploying, raw:%p", userObj.user, pRaw); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "create-user"); if (pTrans == NULL) { mError("user:%s, failed to create since %s", userObj.user, terrstr()); return -1; @@ -299,7 +299,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate userObj.sysInfo = pCreate->sysInfo; userObj.enable = pCreate->enable; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-user"); if (pTrans == NULL) { mError("user:%s, failed to create since %s", pCreate->user, terrstr()); return -1; @@ -383,7 +383,7 @@ _OVER: } static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SRpcMsg *pReq) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "alter-user"); if (pTrans == NULL) { mError("user:%s, failed to alter since %s", pOld->user, terrstr()); return -1; @@ -598,7 +598,7 @@ _OVER: } static int32_t mndDropUser(SMnode *pMnode, SRpcMsg *pReq, SUserObj *pUser) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "drop-user"); if (pTrans == NULL) { mError("user:%s, failed to drop since %s", pUser->user, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index a95bdef323..cc9ed5f9ba 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1223,7 +1223,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSdbRaw *pRaw = NULL; STrans *pTrans = NULL; - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "red-vgroup"); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mDebug("trans:%d, used to redistribute vgroup, vgId:%d", pTrans->id, pVgroup->vgId); @@ -1606,7 +1606,7 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj STrans *pTrans = NULL; SArray *pArray = mndBuildDnodesArray(pMnode, 0); - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "split-vgroup"); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mDebug("trans:%d, used to split vgroup, vgId:%d", pTrans->id, pVgroup->vgId); @@ -1774,7 +1774,7 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { pBalancedVgroups = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (pBalancedVgroups == NULL) goto _OVER; - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "balance-vgroup"); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mDebug("trans:%d, used to balance vgroup", pTrans->id); diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index aee8aa2748..60be7cfbc0 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -115,7 +115,7 @@ class MndTestTrans2 : public ::testing::Test { userObj.superUser = 1; SRpcMsg rpcMsg = {0}; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, conflict, &rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, conflict, &rpcMsg, ""); SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); mndTransAppendRedolog(pTrans, pRedoRaw); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); @@ -148,7 +148,7 @@ class MndTestTrans2 : public ::testing::Test { userObj.superUser = 1; SRpcMsg rpcMsg = {0}; - STrans *pTrans = mndTransCreate(pMnode, policy, conflict, &rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, policy, conflict, &rpcMsg, ""); SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); mndTransAppendRedolog(pTrans, pRedoRaw); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); @@ -220,7 +220,7 @@ class MndTestTrans2 : public ::testing::Test { userObj.superUser = 1; SRpcMsg rpcMsg = {0}; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, &rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, &rpcMsg, ""); SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); mndTransAppendRedolog(pTrans, pRedoRaw); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 07ec7d0694..2b44cdcef1 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -295,12 +295,17 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { } /** - * @brief step 3: consume the SubmitReq in buffer + * @brief step 3: commit should wait for all SubmitReq in buffer be consumed * 1) This is high cost task and should not put in asyncPreCommit originally. * 2) But, if put in asyncCommit, would trigger taskInfo cloning frequently. */ - if (tdRSmaProcessExecImpl(pSma, RSMA_EXEC_COMMIT) < 0) { - return TSDB_CODE_FAILED; + nLoops = 0; + while (atomic_load_64(&pRSmaStat->nBufItems) > 0) { + ++nLoops; + if (nLoops > 1000) { + sched_yield(); + nLoops = 0; + } } smaInfo("vgId:%d, rsma commit, wait for all items to be consumed, TID:%p", SMA_VID(pSma), diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index b870ea1b62..ccf4ebb39f 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -278,7 +278,6 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { smaDebug("vgId:%d, destroy rsma stat %p", SMA_VID(pSma), pRSmaStat); // step 1: set rsma trigger stat cancelled atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED); - tsem_destroy(&(pStat->notEmpty)); // step 2: destroy the rsma info and associated fetch tasks taosHashCleanup(RSMA_INFO_HASH(pStat)); @@ -306,6 +305,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { tdRSmaFSClose(RSMA_FS(pStat)); // step 6: free pStat + tsem_destroy(&(pStat->notEmpty)); taosMemoryFreeClear(pStat); } } diff --git a/source/dnode/vnode/src/sma/smaFS.c b/source/dnode/vnode/src/sma/smaFS.c index 8e8611f0e8..a5f4e8d2e8 100644 --- a/source/dnode/vnode/src/sma/smaFS.c +++ b/source/dnode/vnode/src/sma/smaFS.c @@ -150,6 +150,8 @@ static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **outpu regex_t regex; int code = 0; + terrno = TSDB_CODE_SUCCESS; + tdGetVndDirName(TD_VID(pVnode), tfsGetPrimaryPath(pVnode->pTfs), VNODE_RSMA_DIR, true, dir); if (!taosCheckExistFile(dir)) { diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 412def646f..c6ec31cc21 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -1906,7 +1906,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { while (true) { // step 1: rsma exec - consume data in buffer queue for all suids - if (type == RSMA_EXEC_OVERFLOW || type == RSMA_EXEC_COMMIT) { + if (type == RSMA_EXEC_OVERFLOW) { void *pIter = NULL; while ((pIter = taosHashIterate(infoHash, pIter))) { SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; @@ -1962,42 +1962,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { atomic_val_compare_exchange_8(&pInfo->assigned, 1, 0); } } - if (type == RSMA_EXEC_COMMIT) { - if (atomic_load_64(&pRSmaStat->nBufItems) <= 0) { - break; - } else { - // commit should wait for all items be consumed - continue; - } - } - } -#if 0 - else if (type == RSMA_EXEC_COMMIT) { - while (pIter) { - SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; - if (taosQueueItemSize(pInfo->iQueue)) { - if (atomic_val_compare_exchange_8(&pInfo->assigned, 0, 1) == 0) { - taosReadAllQitems(pInfo->iQueue, pInfo->iQall); // queue has mutex lock - int32_t qallItemSize = taosQallItemSize(pInfo->iQall); - if (qallItemSize > 0) { - atomic_fetch_sub_64(&pRSmaStat->nBufItems, qallItemSize); - nIdle = 0; - - // batch exec - tdRSmaBatchExec(pSma, pInfo, pInfo->qall, pSubmitArr, type); - } - - // tdRSmaFetchAllResult(pSma, pInfo, pSubmitArr); - atomic_val_compare_exchange_8(&pInfo->assigned, 1, 0); - } - } - ASSERT(taosQueueItemSize(pInfo->iQueue) == 0); - pIter = taosHashIterate(infoHash, pIter); - } - break; - } -#endif - else { + } else { ASSERT(0); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 14dbfbd63e..ca6f6346a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -193,6 +193,8 @@ static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); static int32_t doBuildDataBlock(STsdbReader* pReader); static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); +static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo); +static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } @@ -893,7 +895,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn if (pData->cid < pColData->info.colId) { colIndex += 1; } else if (pData->cid == pColData->info.colId) { - if (pData->flag == HAS_NONE || pData->flag == HAS_NULL) { + if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL|HAS_NONE)) { colDataAppendNNULL(pColData, 0, remain); } else { if (IS_NUMERIC_TYPE(pColData->info.type) && asc) { @@ -1537,7 +1539,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* minKey = k.ts; } - if (minKey > key && pBlockData->nRow > 0) { + if (minKey > key && hasDataInFileBlock(pBlockData, pDumpInfo)) { minKey = key; } } else { @@ -1550,7 +1552,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* minKey = k.ts; } - if (minKey < key && pBlockData->nRow > 0) { + if (minKey < key && hasDataInFileBlock(pBlockData, pDumpInfo)) { minKey = key; } } @@ -1688,7 +1690,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - if (pBlockData->nRow > 0) { + if (hasDataInFileBlock(pBlockData, pDumpInfo)) { // no last block available, only data block exists if (!hasDataInLastBlock(pLastBlockReader)) { return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); @@ -1753,7 +1755,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsLast = getCurrentKeyInLastBlock(pLastBlockReader); } - int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + int64_t key = hasDataInFileBlock(pBlockData, pDumpInfo)? pBlockData->aTSKEY[pDumpInfo->rowIndex]:INT64_MIN; TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY ik = TSDBROW_KEY(piRow); @@ -1769,7 +1771,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* minKey = ik.ts; } - if (minKey > key && pBlockData->nRow > 0) { + if (minKey > key && hasDataInFileBlock(pBlockData, pDumpInfo)) { minKey = key; } @@ -1786,7 +1788,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* minKey = ik.ts; } - if (minKey < key && pBlockData->nRow > 0) { + if (minKey < key && hasDataInFileBlock(pBlockData, pDumpInfo)) { minKey = key; } @@ -2021,6 +2023,13 @@ static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { } static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } +bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo) { + if (pBlockData->nRow > 0) { + ASSERT(pBlockData->nRow == pDumpInfo->totalRows); + } + + return pBlockData->nRow > 0 && (!pDumpInfo->allDumped); +} int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader) { @@ -2052,7 +2061,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - int64_t key = (pBlockData->nRow > 0) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN; + int64_t key = (pBlockData->nRow > 0 && (!pDumpInfo->allDumped)) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN; if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal) { return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); } else { @@ -2664,8 +2673,11 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret int8_t* pLevel) { if (VND_IS_RSMA(pVnode)) { int8_t level = 0; - int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); - int64_t offset = TSDB_TICK_PER_SECOND(pVnode->config.tsdbCfg.precision); + int8_t precision = pVnode->config.tsdbCfg.precision; + int64_t now = taosGetTimestamp(precision); + int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1 + : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000 + : 1000000); for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) { SRetention* pRetention = retentions + level; @@ -3290,9 +3302,31 @@ void* tsdbGetIvtIdx(SMeta* pMeta) { uint64_t getReaderMaxVersion(STsdbReader* pReader) { return pReader->verRange.maxVer; } + +static int32_t doOpenReaderImpl(STsdbReader* pReader) { + SDataBlockIter* pBlockIter = &pReader->status.blockIter; + + initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader); + resetDataBlockIterator(&pReader->status.blockIter, pReader->order); + + // no data in files, let's try buffer in memory + if (pReader->status.fileIter.numOfFiles == 0) { + pReader->status.loadFromFile = false; + return TSDB_CODE_SUCCESS; + } else { + return initForFirstBlockInFile(pReader, pBlockIter); + } +} + // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTableList, STsdbReader** ppReader, const char* idstr) { + STimeWindow window = pCond->twindows; + if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) { + pCond->twindows.skey += 1; + pCond->twindows.ekey -= 1; + } + int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, 4096, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; @@ -3300,21 +3334,20 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl // check for query time window STsdbReader* pReader = *ppReader; - if (isEmptyQueryTimeWindow(&pReader->window)) { + if (isEmptyQueryTimeWindow(&pReader->window) && pCond->type == TIMEWINDOW_RANGE_CONTAINED) { tsdbDebug("%p query window not overlaps with the data set, no result returned, %s", pReader, pReader->idStr); return TSDB_CODE_SUCCESS; } if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) { // update the SQueryTableDataCond to create inner reader - STimeWindow w = pCond->twindows; - int32_t order = pCond->order; + int32_t order = pCond->order; if (order == TSDB_ORDER_ASC) { - pCond->twindows.ekey = pCond->twindows.skey; + pCond->twindows.ekey = window.skey; pCond->twindows.skey = INT64_MIN; pCond->order = TSDB_ORDER_DESC; } else { - pCond->twindows.skey = pCond->twindows.ekey; + pCond->twindows.skey = window.ekey; pCond->twindows.ekey = INT64_MAX; pCond->order = TSDB_ORDER_ASC; } @@ -3326,12 +3359,14 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl } if (order == TSDB_ORDER_ASC) { - pCond->twindows.skey = w.ekey; + pCond->twindows.skey = window.ekey; pCond->twindows.ekey = INT64_MAX; } else { pCond->twindows.skey = INT64_MIN; - pCond->twindows.ekey = w.ekey; + pCond->twindows.ekey = window.ekey; } + pCond->order = order; + code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; @@ -3340,20 +3375,22 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl // NOTE: the endVersion in pCond is the data version not schema version, so pCond->endVersion is not correct here. if (pCond->suid != 0) { - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, /*pCond->endVersion*/ -1); + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, -1); if (pReader->pSchema == NULL) { tsdbError("failed to get table schema, suid:%"PRIu64", ver:%"PRId64" , %s", pReader->suid, -1, pReader->idStr); } } else if (taosArrayGetSize(pTableList) > 0) { STableKeyInfo* pKey = taosArrayGet(pTableList, 0); - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, /*pCond->endVersion*/ -1); + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1); if (pReader->pSchema == NULL) { tsdbError("failed to get table schema, uid:%"PRIu64", ver:%"PRId64" , %s", pKey->uid, -1, pReader->idStr); } } + STsdbReader* p = pReader->innerReader[0] != NULL? pReader->innerReader[0]:pReader; + int32_t numOfTables = taosArrayGetSize(pTableList); - pReader->status.pTableMap = createDataBlockScanInfo(pReader, pTableList->pData, numOfTables); + pReader->status.pTableMap = createDataBlockScanInfo(p, pTableList->pData, numOfTables); if (pReader->status.pTableMap == NULL) { tsdbReaderClose(pReader); *ppReader = NULL; @@ -3368,40 +3405,36 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl } if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { - SDataBlockIter* pBlockIter = &pReader->status.blockIter; - - initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader); - resetDataBlockIterator(&pReader->status.blockIter, pReader->order); - - // no data in files, let's try buffer in memory - if (pReader->status.fileIter.numOfFiles == 0) { - pReader->status.loadFromFile = false; - } else { - code = initForFirstBlockInFile(pReader, pBlockIter); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = doOpenReaderImpl(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; } } else { - STsdbReader* pPrevReader = pReader->innerReader[0]; - SDataBlockIter* pBlockIter = &pPrevReader->status.blockIter; + STsdbReader* pPrevReader = pReader->innerReader[0]; + STsdbReader* pNextReader = pReader->innerReader[1]; - code = tsdbTakeReadSnap(pPrevReader->pTsdb, &pPrevReader->pReadSnap, pReader->idStr); + // we need only one row + pPrevReader->capacity = 1; + pPrevReader->status.pTableMap = pReader->status.pTableMap; + pPrevReader->pReadSnap = pReader->pReadSnap; + + pNextReader->capacity = 1; + pNextReader->status.pTableMap = pReader->status.pTableMap; + pNextReader->pReadSnap = pReader->pReadSnap; + + code = doOpenReaderImpl(pPrevReader); if (code != TSDB_CODE_SUCCESS) { - goto _err; + return code; } - initFilesetIterator(&pPrevReader->status.fileIter, pPrevReader->pReadSnap->fs.aDFileSet, pPrevReader); - resetDataBlockIterator(&pPrevReader->status.blockIter, pPrevReader->order); + code = doOpenReaderImpl(pNextReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - // no data in files, let's try buffer in memory - if (pPrevReader->status.fileIter.numOfFiles == 0) { - pPrevReader->status.loadFromFile = false; - } else { - code = initForFirstBlockInFile(pPrevReader, pBlockIter); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = doOpenReaderImpl(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; } } @@ -3418,6 +3451,19 @@ void tsdbReaderClose(STsdbReader* pReader) { return; } + { + if (pReader->innerReader[0] != NULL) { + pReader->innerReader[0]->status.pTableMap = NULL; + pReader->innerReader[0]->pReadSnap = NULL; + + pReader->innerReader[1]->status.pTableMap = NULL; + pReader->innerReader[1]->pReadSnap = NULL; + + tsdbReaderClose(pReader->innerReader[0]); + tsdbReaderClose(pReader->innerReader[1]); + } + } + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; taosMemoryFreeClear(pSupInfo->plist); @@ -3508,32 +3554,32 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { return false; } - if (pReader->innerReader[0] != NULL) { + if (pReader->innerReader[0] != NULL && pReader->step == 0) { bool ret = doTsdbNextDataBlock(pReader->innerReader[0]); + resetDataBlockScanInfo(pReader->innerReader[0]->status.pTableMap, pReader->innerReader[0]->window.ekey); + pReader->step = EXTERNAL_ROWS_PREV; + if (ret) { - pReader->step = EXTERNAL_ROWS_PREV; return ret; } - - tsdbReaderClose(pReader->innerReader[0]); - pReader->innerReader[0] = NULL; } - pReader->step = EXTERNAL_ROWS_MAIN; + if (pReader->step == EXTERNAL_ROWS_PREV) { + pReader->step = EXTERNAL_ROWS_MAIN; + } + bool ret = doTsdbNextDataBlock(pReader); if (ret) { return ret; } - if (pReader->innerReader[1] != NULL) { + if (pReader->innerReader[1] != NULL && pReader->step == EXTERNAL_ROWS_MAIN) { + resetDataBlockScanInfo(pReader->innerReader[1]->status.pTableMap, pReader->window.ekey); bool ret1 = doTsdbNextDataBlock(pReader->innerReader[1]); + pReader->step = EXTERNAL_ROWS_NEXT; if (ret1) { - pReader->step = EXTERNAL_ROWS_NEXT; return ret1; } - - tsdbReaderClose(pReader->innerReader[1]); - pReader->innerReader[1] = NULL; } return false; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a1ff79bd03..31a7b0facf 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -81,11 +81,6 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { int32_t getMaximumIdleDurationSec() { return tsShellActivityTimer * 2; } -static int32_t getExprFunctionId(SExprInfo* pExprInfo) { - assert(pExprInfo != NULL && pExprInfo->pExpr != NULL && pExprInfo->pExpr->nodeType == TEXPR_UNARYEXPR_NODE); - return 0; -} - static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock); static void releaseQueryBuf(size_t numOfTables); @@ -1115,7 +1110,7 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO } } -static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep); +static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep, int32_t status); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo) { if (pFilterNode == NULL || pBlock->info.rows == 0) { @@ -1126,18 +1121,17 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColM // todo move to the initialization function int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); - - size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - SFilterColumnParam param1 = {.numOfCols = numOfCols, .pDataBlock = pBlock->pDataBlock}; + SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock}; code = filterSetDataFromSlotId(filter, ¶m1); - int8_t* rowRes = NULL; + SColumnInfoData* p = NULL; + int32_t status = 0; // todo the keep seems never to be True?? - bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols); + bool keep = filterExecute(filter, pBlock, &p, NULL, param1.numOfCols, &status); filterFreeInfo(filter); - extractQualifiedTupleByFilterResult(pBlock, rowRes, keep); + extractQualifiedTupleByFilterResult(pBlock, p, keep, status); if (pColMatchInfo != NULL) { for (int32_t i = 0; i < taosArrayGetSize(pColMatchInfo); ++i) { @@ -1152,16 +1146,22 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColM } } - taosMemoryFree(rowRes); + colDataDestroy(p); + taosMemoryFree(p); } -void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep) { +void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep, int32_t status) { if (keep) { return; } - if (rowRes != NULL) { - int32_t totalRows = pBlock->info.rows; + int32_t totalRows = pBlock->info.rows; + + if (status == FILTER_RESULT_ALL_QUALIFIED) { + // here nothing needs to be done + } else if (status == FILTER_RESULT_NONE_QUALIFIED) { + pBlock->info.rows = 0; + } else { SSDataBlock* px = createOneDataBlock(pBlock, true); size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); @@ -1177,7 +1177,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR int32_t numOfRows = 0; for (int32_t j = 0; j < totalRows; ++j) { - if (rowRes[j] == 0) { + if (((int8_t*)p->pData)[j] == 0) { continue; } @@ -1189,6 +1189,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR numOfRows += 1; } + // todo this value can be assigned directly if (pBlock->info.rows == totalRows) { pBlock->info.rows = numOfRows; } else { @@ -1197,13 +1198,10 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR } blockDataDestroy(px); // fix memory leak - } else { - // do nothing - pBlock->info.rows = 0; } } -void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) { + void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) { // for simple group by query without interval, all the tables belong to one group result. SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SAggOperatorInfo* pAggInfo = pOperator->info; @@ -4246,10 +4244,10 @@ int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); if (pCtx[j].fpSet.finalize) { - int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); - if (TAOS_FAILED(code)) { - qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - T_LONG_JMP(pTaskInfo->env, code); + int32_t code1 = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + if (TAOS_FAILED(code1)) { + qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code1)); + T_LONG_JMP(pTaskInfo->env, code1); } } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { // do nothing, todo refactor diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index dc9e0e8d45..56fbb21309 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1315,10 +1315,14 @@ void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)pStartTs, false); colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)pEndTs, false); colDataAppend(pUidCol, pBlock->info.rows, (const char*)pUid, false); colDataAppend(pGpCol, pBlock->info.rows, (const char*)pGp, false); + colDataAppendNULL(pCalStartCol, pBlock->info.rows); + colDataAppendNULL(pCalEndCol, pBlock->info.rows); pBlock->info.rows++; } @@ -1622,6 +1626,7 @@ FETCH_NEXT_BLOCK: } else { pDelBlock = pBlock; } + printDataBlock(pBlock, "stream scan delete recv filtered"); if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) { generateDeleteResultBlock(pInfo, pDelBlock, pInfo->pDeleteDataRes); pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT; @@ -1709,9 +1714,11 @@ FETCH_NEXT_BLOCK: int32_t totBlockNum = taosArrayGetSize(pInfo->pBlockLists); + NEXT_SUBMIT_BLK: while (1) { if (pInfo->tqReader->pMsg == NULL) { if (pInfo->validBlockIndex >= totBlockNum) { + updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); doClearBufferedBlocks(pInfo); return NULL; } @@ -1784,7 +1791,12 @@ FETCH_NEXT_BLOCK: } qDebug("scan rows: %d", pBlockInfo->rows); - return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; + if (pBlockInfo->rows > 0) { + return pInfo->pRes; + } else { + goto NEXT_SUBMIT_BLK; + } + /*return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes;*/ } else { ASSERT(0); return NULL; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index f10e5e33f1..9e51f3638e 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1115,7 +1115,7 @@ static bool compareVal(const char* v, const SStateKeys* pKey) { if (varDataLen(v) != varDataLen(pKey->pData)) { return false; } else { - return strncmp(varDataVal(v), varDataVal(pKey->pData), varDataLen(v)) == 0; + return memcmp(varDataVal(v), varDataVal(pKey->pData), varDataLen(v)) == 0; } } else { return memcmp(pKey->pData, v, pKey->bytes) == 0; @@ -1695,6 +1695,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { } nodesDestroyNode((SNode*)pInfo->pPhyNode); colDataDestroy(&pInfo->twAggSup.timeWindowData); + cleanupGroupResInfo(&pInfo->groupResInfo); taosMemoryFreeClear(param); } @@ -2074,7 +2075,7 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo } } - pSliceInfo->fillLastPoint = isLastRow ? true : false; + pSliceInfo->fillLastPoint = isLastRow; } static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) { @@ -2293,15 +2294,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { SSDataBlock* pResBlock = pSliceInfo->pRes; SExprSupp* pSup = &pOperator->exprSupp; - // if (pOperator->status == OP_RES_TO_RETURN) { - // // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - // if (pResBlock->info.rows == 0 || !hasRemainResults(&pSliceInfo->groupResInfo)) { - // doSetOperatorCompleted(pOperator); - // } - // - // return pResBlock; - // } - int32_t order = TSDB_ORDER_ASC; SInterval* pInterval = &pSliceInfo->interval; SOperatorInfo* downstream = pOperator->pDownstream[0]; @@ -2431,6 +2423,9 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { break; } } + } else { + // store ts value as start, and calculate interp value when processing next block + doKeepLinearInfo(pSliceInfo, pBlock, i, true); } } else { // non-linear interpolation if (i < pBlock->info.rows - 1) { @@ -2509,6 +2504,9 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { break; } } + } else { // it is the last row of current block + // store ts value as start, and calculate interp value when processing next block + doKeepLinearInfo(pSliceInfo, pBlock, i, true); } } else { // non-linear interpolation pSliceInfo->current = @@ -2614,6 +2612,10 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode pInfo->interval.interval = pInterpPhyNode->interval; pInfo->current = pInfo->win.skey; + STableScanInfo* pScanInfo = (STableScanInfo*)downstream->info; + pScanInfo->cond.twindows = pInfo->win; + pScanInfo->cond.type = TIMEWINDOW_RANGE_EXTERNAL; + pOperator->name = "TimeSliceOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC; pOperator->blocking = false; @@ -3073,6 +3075,7 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) { taosArrayRemove(chArray, index); if (taosArrayGetSize(chArray) == 0) { // pull data is over + taosArrayDestroy(chArray); taosHashRemove(pMap, &winRes, sizeof(SWinKey)); } } @@ -3109,9 +3112,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); TSKEY maxTs = INT64_MIN; TSKEY minTs = INT64_MAX; @@ -3175,6 +3175,9 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } } + SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -5754,8 +5757,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); - SStreamState* pState = pTaskInfo->streamInfo.pState; - while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -5804,36 +5805,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); - -#if 0 - if (pState) { - printf(">>>>>>>> stream read backend\n"); - SWinKey key = { - .ts = 1, - .groupId = 2, - }; - char* val = NULL; - int32_t sz; - if (streamStateGet(pState, &key, (void**)&val, &sz) < 0) { - ASSERT(0); - } - printf("stream read %s %d\n", val, sz); - streamFreeVal(val); - - SStreamStateCur* pCur = streamStateGetCur(pState, &key); - ASSERT(pCur); - while (streamStateCurNext(pState, pCur) == 0) { - SWinKey key1; - const void* val1; - if (streamStateGetKVByCur(pCur, &key1, &val1, &sz) < 0) { - break; - } - printf("stream iter key groupId:%d ts:%d, value %s %d\n", key1.groupId, key1.ts, val1, sz); - } - streamStateFreeCur(pCur); - } -#endif - pOperator->status = OP_RES_TO_RETURN; closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap, pOperator); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index f3d3393ac3..0677bd7d63 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -207,7 +207,6 @@ static int32_t countTrailingSpaces(const SValueNode* pVal, bool isLtrim) { } return numOfSpaces; - } void static addTimezoneParam(SNodeList* pList) { @@ -322,7 +321,7 @@ static int32_t translateTrimStr(SFunctionNode* pFunc, char* pErrBuf, int32_t len } int32_t numOfSpaces = 0; - SNode* pParamNode1 = nodesListGetNode(pFunc->pParameterList, 0); + SNode* pParamNode1 = nodesListGetNode(pFunc->pParameterList, 0); // for select trim functions with constant value from table, // need to set the proper result result schema bytes to avoid // trailing garbage characters @@ -331,7 +330,6 @@ static int32_t translateTrimStr(SFunctionNode* pFunc, char* pErrBuf, int32_t len numOfSpaces = countTrailingSpaces(pValue, isLtrim); } - int32_t resBytes = pPara1->resType.bytes - numOfSpaces; pFunc->node.resType = (SDataType){.bytes = resBytes, .type = pPara1->resType.type}; return TSDB_CODE_SUCCESS; @@ -2141,7 +2139,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "avg", .type = FUNCTION_TYPE_AVG, - .classification = FUNC_MGT_AGG_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, .translateFunc = translateInNumOutDou, .dataRequiredFunc = statisDataRequired, .getEnvFunc = getAvgFuncEnv, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index a23f58a732..9177d15fae 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -47,6 +47,7 @@ typedef struct SSumRes { uint64_t usum; double dsum; }; + int16_t type; } SSumRes; typedef struct SAvgRes { @@ -73,6 +74,7 @@ typedef struct SMinmaxResInfo { STuplePos nullTuplePos; bool nullTupleSaved; + int16_t type; } SMinmaxResInfo; typedef struct STopBotResItem { @@ -485,8 +487,7 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SFirstLastRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); - int32_t type = pDestCtx->input.pData[0]->info.type; - int32_t bytes = pDestCtx->input.pData[0]->info.bytes; + int32_t bytes = pDBuf->bytes; SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SFirstLastRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); @@ -617,6 +618,7 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { int32_t type = pInput->pData[0]->info.type; SSumRes* pSumRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + pSumRes->type = type; if (IS_NULL_TYPE(type)) { numOfElem = 0; @@ -740,10 +742,10 @@ int32_t sumInvertFunction(SqlFunctionCtx* pCtx) { int32_t sumCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SSumRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); - int32_t type = pDestCtx->input.pData[0]->info.type; SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SSumRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); + int16_t type = pDBuf->type == TSDB_DATA_TYPE_NULL ? pSBuf->type : pDBuf->type; if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) { pDBuf->isum += pSBuf->isum; @@ -1072,10 +1074,10 @@ int32_t avgInvertFunction(SqlFunctionCtx* pCtx) { int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SAvgRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); - int32_t type = pDestCtx->input.pData[0]->info.type; SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SAvgRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); + int16_t type = pDBuf->type == TSDB_DATA_TYPE_NULL ? pSBuf->type : pDBuf->type; if (IS_SIGNED_NUMERIC_TYPE(type)) { pDBuf->sum.isum += pSBuf->sum.isum; @@ -1181,6 +1183,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SMinmaxResInfo* pBuf = GET_ROWCELL_INTERBUF(pResInfo); + pBuf->type = type; if (IS_NULL_TYPE(type)) { numOfElems = 0; @@ -1729,10 +1732,10 @@ void replaceTupleData(STuplePos* pDestPos, STuplePos* pSourcePos) { int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t isMinFunc) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SMinmaxResInfo* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); - int32_t type = pDestCtx->input.pData[0]->info.type; SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SMinmaxResInfo* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); + int16_t type = pDBuf->type == TSDB_DATA_TYPE_NULL ? pSBuf->type : pDBuf->type; if (IS_FLOAT_TYPE(type)) { if (pSBuf->assign && ((((*(double*)&pDBuf->v) < (*(double*)&pSBuf->v)) ^ isMinFunc) || !pDBuf->assign)) { *(double*)&pDBuf->v = *(double*)&pSBuf->v; @@ -2105,10 +2108,10 @@ int32_t stddevPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SStddevRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); - int32_t type = pDestCtx->input.pData[0]->info.type; SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SStddevRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); + int16_t type = pDBuf->type == TSDB_DATA_TYPE_NULL ? pSBuf->type : pDBuf->type; if (IS_SIGNED_NUMERIC_TYPE(type)) { pDBuf->isum += pSBuf->isum; @@ -3069,8 +3072,7 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SFirstLastRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); - int32_t type = pDestCtx->input.pData[0]->info.type; - int32_t bytes = pDestCtx->input.pData[0]->info.bytes; + int32_t bytes = pDBuf->bytes; SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SFirstLastRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); @@ -3746,9 +3748,9 @@ void addResult(SqlFunctionCtx* pCtx, STopBotResItem* pSourceItem, int16_t type, } int32_t topCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { - int32_t type = pDestCtx->input.pData[0]->info.type; SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); STopBotRes* pSBuf = getTopBotOutputInfo(pSourceCtx); + int16_t type = pSBuf->type; for (int32_t i = 0; i < pSResInfo->numOfRes; i++) { addResult(pDestCtx, pSBuf->pItems + i, type, true); } @@ -3756,9 +3758,9 @@ int32_t topCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { } int32_t bottomCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { - int32_t type = pDestCtx->input.pData[0]->info.type; SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); STopBotRes* pSBuf = getTopBotOutputInfo(pSourceCtx); + int16_t type = pSBuf->type; for (int32_t i = 0; i < pSResInfo->numOfRes; i++) { addResult(pDestCtx, pSBuf->pItems + i, type, false); } @@ -5414,8 +5416,8 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { int32_t i = pInput->startRowIndex; if (pCtx->start.key != INT64_MIN) { - //ASSERT((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || - // (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); + // ASSERT((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || + // (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); ASSERT(last->key == INT64_MIN); for (; i < pInput->numOfRows + pInput->startRowIndex; ++i) { diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 26735fa263..bc10ce71ae 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -26,11 +26,6 @@ typedef struct SFuncMgtService { SHashObj* pFuncNameHashTable; } SFuncMgtService; -typedef struct SUdfInfo { - SDataType outputDt; - int8_t funcType; -} SUdfInfo; - static SFuncMgtService gFunMgtService; static TdThreadOnce functionHashTableInit = PTHREAD_ONCE_INIT; static int32_t initFunctionCode = 0; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index afd05eff1d..c5851b7875 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -559,6 +559,8 @@ static const char* jkScanLogicPlanStableId = "StableId"; static const char* jkScanLogicPlanScanType = "ScanType"; static const char* jkScanLogicPlanScanCount = "ScanCount"; static const char* jkScanLogicPlanReverseScanCount = "ReverseScanCount"; +static const char* jkScanLogicPlanDynamicScanFuncs = "DynamicScanFuncs"; +static const char* jkScanLogicPlanDataRequired = "DataRequired"; static const char* jkScanLogicPlanTagCond = "TagCond"; static const char* jkScanLogicPlanGroupTags = "GroupTags"; @@ -590,6 +592,12 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanReverseScanCount, pNode->scanSeq[1]); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkScanLogicPlanDynamicScanFuncs, nodeToJson, pNode->pDynamicScanFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanDataRequired, pNode->dataRequired); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkScanLogicPlanTagCond, nodeToJson, pNode->pTagCond); } @@ -629,6 +637,12 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetUTinyIntValue(pJson, jkScanLogicPlanReverseScanCount, &pNode->scanSeq[1]); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkScanLogicPlanDynamicScanFuncs, &pNode->pDynamicScanFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkScanLogicPlanDataRequired, &pNode->dataRequired); + } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkScanLogicPlanTagCond, &pNode->pTagCond); } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 805ddb9e42..2e5a0d935b 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -21,9 +21,209 @@ #include "taoserror.h" #include "tdatablock.h" #include "thash.h" +#include "tref.h" -static SNode* makeNode(ENodeType type, size_t size) { - SNode* p = taosMemoryCalloc(1, size); +typedef struct SNodeMemChunk { + int32_t availableSize; + int32_t usedSize; + char* pBuf; + struct SNodeMemChunk* pNext; +} SNodeMemChunk; + +typedef struct SNodeAllocator { + int64_t self; + int64_t queryId; + int32_t chunkSize; + int32_t chunkNum; + SNodeMemChunk* pCurrChunk; + SNodeMemChunk* pChunks; + TdThreadMutex mutex; +} SNodeAllocator; + +static threadlocal SNodeAllocator* g_pNodeAllocator; +static int32_t g_allocatorReqRefPool = -1; + +static SNodeMemChunk* callocNodeChunk(SNodeAllocator* pAllocator) { + SNodeMemChunk* pNewChunk = taosMemoryCalloc(1, sizeof(SNodeMemChunk) + pAllocator->chunkSize); + if (NULL == pNewChunk) { + return NULL; + } + pNewChunk->pBuf = (char*)(pNewChunk + 1); + pNewChunk->availableSize = pAllocator->chunkSize; + pNewChunk->usedSize = 0; + pNewChunk->pNext = NULL; + if (NULL != pAllocator->pCurrChunk) { + pAllocator->pCurrChunk->pNext = pNewChunk; + } + pAllocator->pCurrChunk = pNewChunk; + if (NULL == pAllocator->pChunks) { + pAllocator->pChunks = pNewChunk; + } + ++(pAllocator->chunkNum); + return pNewChunk; +} + +static void* nodesCallocImpl(int32_t size) { + if (NULL == g_pNodeAllocator) { + return taosMemoryCalloc(1, size); + } + + if (g_pNodeAllocator->pCurrChunk->usedSize + size > g_pNodeAllocator->pCurrChunk->availableSize) { + if (NULL == callocNodeChunk(g_pNodeAllocator)) { + return NULL; + } + } + void* p = g_pNodeAllocator->pCurrChunk->pBuf + g_pNodeAllocator->pCurrChunk->usedSize; + g_pNodeAllocator->pCurrChunk->usedSize += size; + return p; +} + +static void* nodesCalloc(int32_t num, int32_t size) { + void* p = nodesCallocImpl(num * size + 1); + if (NULL == p) { + return NULL; + } + *(char*)p = (NULL != g_pNodeAllocator) ? 1 : 0; + return (char*)p + 1; +} + +static void nodesFree(void* p) { + char* ptr = (char*)p - 1; + if (0 == *ptr) { + taosMemoryFree(ptr); + } + return; +} + +static int32_t createNodeAllocator(int32_t chunkSize, SNodeAllocator** pAllocator) { + *pAllocator = taosMemoryCalloc(1, sizeof(SNodeAllocator)); + if (NULL == *pAllocator) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (*pAllocator)->chunkSize = chunkSize; + if (NULL == callocNodeChunk(*pAllocator)) { + taosMemoryFreeClear(*pAllocator); + return TSDB_CODE_OUT_OF_MEMORY; + } + taosThreadMutexInit(&(*pAllocator)->mutex, NULL); + return TSDB_CODE_SUCCESS; +} + +static void destroyNodeAllocator(void* p) { + if (NULL == p) { + return; + } + + SNodeAllocator* pAllocator = p; + + nodesDebug("query id %" PRIx64 " allocator id %" PRIx64 " alloc chunkNum: %d, chunkTotakSize: %d", + pAllocator->queryId, pAllocator->self, pAllocator->chunkNum, pAllocator->chunkNum * pAllocator->chunkSize); + + SNodeMemChunk* pChunk = pAllocator->pChunks; + while (NULL != pChunk) { + SNodeMemChunk* pTemp = pChunk->pNext; + taosMemoryFree(pChunk); + pChunk = pTemp; + } + taosThreadMutexDestroy(&pAllocator->mutex); + taosMemoryFree(pAllocator); +} + +int32_t nodesInitAllocatorSet() { + if (g_allocatorReqRefPool >= 0) { + nodesWarn("nodes already initialized"); + return TSDB_CODE_SUCCESS; + } + + g_allocatorReqRefPool = taosOpenRef(1024, destroyNodeAllocator); + if (g_allocatorReqRefPool < 0) { + nodesError("init nodes failed"); + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + +void nodesDestroyAllocatorSet() { + if (g_allocatorReqRefPool >= 0) { + SNodeAllocator* pAllocator = taosIterateRef(g_allocatorReqRefPool, 0); + int64_t refId = 0; + while (NULL != pAllocator) { + refId = pAllocator->self; + taosRemoveRef(g_allocatorReqRefPool, refId); + pAllocator = taosIterateRef(g_allocatorReqRefPool, refId); + } + taosCloseRef(g_allocatorReqRefPool); + } +} + +int32_t nodesCreateAllocator(int64_t queryId, int32_t chunkSize, int64_t* pAllocatorId) { + SNodeAllocator* pAllocator = NULL; + int32_t code = createNodeAllocator(chunkSize, &pAllocator); + if (TSDB_CODE_SUCCESS == code) { + pAllocator->self = taosAddRef(g_allocatorReqRefPool, pAllocator); + if (pAllocator->self <= 0) { + return terrno; + } + pAllocator->queryId = queryId; + *pAllocatorId = pAllocator->self; + } + return code; +} + +int32_t nodesAcquireAllocator(int64_t allocatorId) { + if (allocatorId <= 0) { + return TSDB_CODE_SUCCESS; + } + + SNodeAllocator* pAllocator = taosAcquireRef(g_allocatorReqRefPool, allocatorId); + if (NULL == pAllocator) { + return terrno; + } + taosThreadMutexLock(&pAllocator->mutex); + g_pNodeAllocator = pAllocator; + return TSDB_CODE_SUCCESS; +} + +int32_t nodesReleaseAllocator(int64_t allocatorId) { + if (allocatorId <= 0) { + return TSDB_CODE_SUCCESS; + } + + if (NULL == g_pNodeAllocator) { + nodesError("allocator id %" PRIx64 + " release failed: The nodesReleaseAllocator function needs to be called after the nodesAcquireAllocator " + "function is called!", + allocatorId); + return TSDB_CODE_FAILED; + } + SNodeAllocator* pAllocator = g_pNodeAllocator; + g_pNodeAllocator = NULL; + taosThreadMutexUnlock(&pAllocator->mutex); + return taosReleaseRef(g_allocatorReqRefPool, allocatorId); +} + +int64_t nodesMakeAllocatorWeakRef(int64_t allocatorId) { + if (allocatorId <= 0) { + return 0; + } + + SNodeAllocator* pAllocator = taosAcquireRef(g_allocatorReqRefPool, allocatorId); + return pAllocator->self; +} + +int64_t nodesReleaseAllocatorWeakRef(int64_t allocatorId) { return taosReleaseRef(g_allocatorReqRefPool, allocatorId); } + +void nodesDestroyAllocator(int64_t allocatorId) { + if (allocatorId <= 0) { + return; + } + + taosRemoveRef(g_allocatorReqRefPool, allocatorId); +} + +static SNode* makeNode(ENodeType type, int32_t size) { + SNode* p = nodesCalloc(1, size); if (NULL == p) { return NULL; } @@ -824,6 +1024,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pLogicNode->pWStartTs); nodesDestroyNode(pLogicNode->pValues); nodesDestroyList(pLogicNode->pFillExprs); + nodesDestroyList(pLogicNode->pNotFillExprs); break; } case QUERY_NODE_LOGIC_PLAN_SORT: { @@ -1021,12 +1222,12 @@ void nodesDestroyNode(SNode* pNode) { default: break; } - taosMemoryFreeClear(pNode); + nodesFree(pNode); return; } SNodeList* nodesMakeList() { - SNodeList* p = taosMemoryCalloc(1, sizeof(SNodeList)); + SNodeList* p = nodesCalloc(1, sizeof(SNodeList)); if (NULL == p) { return NULL; } @@ -1037,7 +1238,7 @@ int32_t nodesListAppend(SNodeList* pList, SNode* pNode) { if (NULL == pList || NULL == pNode) { return TSDB_CODE_FAILED; } - SListCell* p = taosMemoryCalloc(1, sizeof(SListCell)); + SListCell* p = nodesCalloc(1, sizeof(SListCell)); if (NULL == p) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY; @@ -1104,7 +1305,7 @@ int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) { } pTarget->pTail = pSrc->pTail; pTarget->length += pSrc->length; - taosMemoryFreeClear(pSrc); + nodesFree(pSrc); return TSDB_CODE_SUCCESS; } @@ -1124,7 +1325,7 @@ int32_t nodesListPushFront(SNodeList* pList, SNode* pNode) { if (NULL == pList || NULL == pNode) { return TSDB_CODE_FAILED; } - SListCell* p = taosMemoryCalloc(1, sizeof(SListCell)); + SListCell* p = nodesCalloc(1, sizeof(SListCell)); if (NULL == p) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY; @@ -1152,7 +1353,7 @@ SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { } SListCell* pNext = pCell->pNext; nodesDestroyNode(pCell->pNode); - taosMemoryFreeClear(pCell); + nodesFree(pCell); --(pList->length); return pNext; } @@ -1172,7 +1373,7 @@ void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc) { pPos->pPrev = pSrc->pTail; pTarget->length += pSrc->length; - taosMemoryFreeClear(pSrc); + nodesFree(pSrc); } SNode* nodesListGetNode(SNodeList* pList, int32_t index) { @@ -1204,7 +1405,7 @@ void nodesDestroyList(SNodeList* pList) { while (NULL != pNext) { pNext = nodesListErase(pList, pNext); } - taosMemoryFreeClear(pList); + nodesFree(pList); } void nodesClearList(SNodeList* pList) { @@ -1216,9 +1417,9 @@ void nodesClearList(SNodeList* pList) { while (NULL != pNext) { SListCell* tmp = pNext; pNext = pNext->pNext; - taosMemoryFreeClear(tmp); + nodesFree(tmp); } - taosMemoryFreeClear(pList); + nodesFree(pList); } void* nodesGetValueFromNode(SValueNode* pNode) { diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 6f11c653a4..379bd975b4 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -247,7 +247,8 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { pExpr->userAlias[len] = '\0'; } } - taosMemoryFreeClear(pNode); + pRawExpr->pNode = NULL; + nodesDestroyNode(pNode); return pRealizedExpr; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index ca9bb16c05..4fb55ed373 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1537,6 +1537,9 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { autoCreateTbl = true; } else if (!existedUsing) { CHECK_CODE(getTableMeta(pCxt, tbNum, &name, dbFName)); + if (TSDB_SUPER_TABLE == pCxt->pTableMeta->tableType) { + return buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported"); + } } STableDataBlocks* dataBuf = NULL; @@ -2534,7 +2537,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols if (p) kv = *p; } - if (kv){ + if (kv) { int32_t colLen = kv->length; if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { // uError("SML:data before:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); @@ -2547,7 +2550,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols } else { MemRowAppend(&pBuf, &(kv->value), colLen, ¶m); } - }else{ + } else { pBuilder->hasNone = true; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f0d5e5d67e..88dd5afe57 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -264,6 +264,8 @@ static bool beforeHaving(ESqlClause clause) { return clause < SQL_CLAUSE_HAVING; static bool afterHaving(ESqlClause clause) { return clause > SQL_CLAUSE_HAVING; } +static bool beforeWindow(ESqlClause clause) { return clause < SQL_CLAUSE_WINDOW; } + static bool hasSameTableAlias(SArray* pTables) { if (taosArrayGetSize(pTables) < 2) { return false; @@ -1476,6 +1478,10 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SFunctio if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC); } + if (beforeWindow(pCxt->currClause)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC, "There mustn't be %s", + pFunc->functionName); + } return TSDB_CODE_SUCCESS; } @@ -2213,6 +2219,17 @@ static int32_t setTableCacheLastMode(STranslateContext* pCxt, SSelectStmt* pSele return code; } +static int32_t checkJoinTable(STranslateContext* pCxt, SJoinTableNode* pJoinTable) { + if ((QUERY_NODE_TEMP_TABLE == nodeType(pJoinTable->pLeft) && + !isTimeLineQuery(((STempTableNode*)pJoinTable->pLeft)->pSubquery)) || + (QUERY_NODE_TEMP_TABLE == nodeType(pJoinTable->pRight) && + !isTimeLineQuery(((STempTableNode*)pJoinTable->pRight)->pSubquery))) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SUPPORT_JOIN, + "Join requires valid time series input"); + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pTable)) { @@ -2259,6 +2276,9 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { if (TSDB_CODE_SUCCESS == code) { code = translateTable(pCxt, pJoinTable->pRight); } + if (TSDB_CODE_SUCCESS == code) { + code = checkJoinTable(pCxt, pJoinTable); + } if (TSDB_CODE_SUCCESS == code) { pJoinTable->table.precision = calcJoinTablePrecision(pJoinTable); pJoinTable->table.singleTable = joinTableIsSingleTable(pJoinTable); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 7ee6a5b223..2fe6ebfb79 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -177,15 +177,18 @@ int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) { int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) { SParseMetaCache metaCache = {0}; - int32_t code = TSDB_CODE_SUCCESS; - if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) { - code = parseInsertSyntax(pCxt, pQuery, &metaCache); - } else { - code = parseSqlSyntax(pCxt, pQuery, &metaCache); + int32_t code = nodesAcquireAllocator(pCxt->allocatorId); + if (TSDB_CODE_SUCCESS == code) { + if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) { + code = parseInsertSyntax(pCxt, pQuery, &metaCache); + } else { + code = parseSqlSyntax(pCxt, pQuery, &metaCache); + } } if (TSDB_CODE_SUCCESS == code) { code = buildCatalogReq(pCxt, &metaCache, pCatalogReq); } + nodesReleaseAllocator(pCxt->allocatorId); destoryParseMetaCache(&metaCache, true); terrno = code; return code; @@ -194,7 +197,10 @@ int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCatalogReq, const struct SMetaData* pMetaData, SQuery* pQuery) { SParseMetaCache metaCache = {0}; - int32_t code = putMetaDataToCache(pCatalogReq, pMetaData, &metaCache, NULL == pQuery->pRoot); + int32_t code = nodesAcquireAllocator(pCxt->allocatorId); + if (TSDB_CODE_SUCCESS == code) { + code = putMetaDataToCache(pCatalogReq, pMetaData, &metaCache, NULL == pQuery->pRoot); + } if (TSDB_CODE_SUCCESS == code) { if (NULL == pQuery->pRoot) { code = parseInsertSql(pCxt, &pQuery, &metaCache); @@ -202,6 +208,7 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata code = analyseSemantic(pCxt, pQuery, &metaCache); } } + nodesReleaseAllocator(pCxt->allocatorId); destoryParseMetaCache(&metaCache, false); terrno = code; return code; diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 716dd7ffc0..be0ed72e23 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -445,4 +445,11 @@ TEST_F(ParserSelectTest, withoutFromSemanticCheck) { run("SELECT TBNAME", TSDB_CODE_PAR_INVALID_TBNAME); } +TEST_F(ParserSelectTest, joinSemanticCheck) { + useDb("root", "test"); + + run("SELECT * FROM (SELECT tag1, SUM(c1) s FROM st1 GROUP BY tag1) t1, st1 t2 where t1.tag1 = t2.tag1", + TSDB_CODE_PAR_NOT_SUPPORT_JOIN); +} + } // namespace ParserTest diff --git a/source/libs/parser/test/parTestUtil.cpp b/source/libs/parser/test/parTestUtil.cpp index 360b904c17..14c991917b 100644 --- a/source/libs/parser/test/parTestUtil.cpp +++ b/source/libs/parser/test/parTestUtil.cpp @@ -119,12 +119,18 @@ class ParserTestBaseImpl { TEST_INTERFACE_ASYNC_API }; - static void _destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { + static void destoryParseContext(SParseContext* pCxt) { + taosArrayDestroy(pCxt->pTableMetaPos); + taosArrayDestroy(pCxt->pTableVgroupPos); + delete pCxt; + } + + static void destoryParseMetaCacheWarpper(SParseMetaCache* pMetaCache, bool request) { destoryParseMetaCache(pMetaCache, request); delete pMetaCache; } - static void _destroyQuery(SQuery** pQuery) { + static void destroyQuery(SQuery** pQuery) { if (nullptr == pQuery) { return; } @@ -303,10 +309,10 @@ class ParserTestBaseImpl { setParseContext(sql, &cxt); if (qIsInsertValuesSql(cxt.pSql, cxt.sqlLen)) { - unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), _destroyQuery); + unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); doParseInsertSql(&cxt, query.get(), nullptr); } else { - unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), _destroyQuery); + unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); doParse(&cxt, query.get()); SQuery* pQuery = *(query.get()); @@ -335,7 +341,7 @@ class ParserTestBaseImpl { SParseContext cxt = {0}; setParseContext(sql, &cxt); - unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), _destroyQuery); + unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); doParseSql(&cxt, query.get()); SQuery* pQuery = *(query.get()); @@ -354,26 +360,26 @@ class ParserTestBaseImpl { void runAsyncInternalFuncs(const string& sql, int32_t expect, ParserStage checkStage) { reset(expect, checkStage, TEST_INTERFACE_ASYNC_INTERNAL); try { - SParseContext cxt = {0}; - setParseContext(sql, &cxt, true); + unique_ptr > cxt(new SParseContext(), destoryParseContext); + setParseContext(sql, cxt.get(), true); - unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), _destroyQuery); + unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); bool request = true; unique_ptr > metaCache( - new SParseMetaCache(), bind(_destoryParseMetaCache, _1, cref(request))); - bool isInsertValues = qIsInsertValuesSql(cxt.pSql, cxt.sqlLen); + new SParseMetaCache(), bind(destoryParseMetaCacheWarpper, _1, cref(request))); + bool isInsertValues = qIsInsertValuesSql(cxt->pSql, cxt->sqlLen); if (isInsertValues) { - doParseInsertSyntax(&cxt, query.get(), metaCache.get()); + doParseInsertSyntax(cxt.get(), query.get(), metaCache.get()); } else { - doParse(&cxt, query.get()); - doCollectMetaKey(&cxt, *(query.get()), metaCache.get()); + doParse(cxt.get(), query.get()); + doCollectMetaKey(cxt.get(), *(query.get()), metaCache.get()); } SQuery* pQuery = *(query.get()); unique_ptr catalogReq(new SCatalogReq(), MockCatalogService::destoryCatalogReq); - doBuildCatalogReq(&cxt, metaCache.get(), catalogReq.get()); + doBuildCatalogReq(cxt.get(), metaCache.get(), catalogReq.get()); string err; thread t1([&]() { @@ -386,13 +392,13 @@ class ParserTestBaseImpl { doPutMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get(), isInsertValues); if (isInsertValues) { - doParseInsertSql(&cxt, query.get(), metaCache.get()); + doParseInsertSql(cxt.get(), query.get(), metaCache.get()); } else { - doAuthenticate(&cxt, pQuery, metaCache.get()); + doAuthenticate(cxt.get(), pQuery, metaCache.get()); - doTranslate(&cxt, pQuery, metaCache.get()); + doTranslate(cxt.get(), pQuery, metaCache.get()); - doCalculateConstant(&cxt, pQuery); + doCalculateConstant(cxt.get(), pQuery); } } catch (const TerminateFlag& e) { // success and terminate @@ -423,13 +429,13 @@ class ParserTestBaseImpl { void runAsyncApis(const string& sql, int32_t expect, ParserStage checkStage) { reset(expect, checkStage, TEST_INTERFACE_ASYNC_API); try { - SParseContext cxt = {0}; - setParseContext(sql, &cxt); + unique_ptr > cxt(new SParseContext(), destoryParseContext); + setParseContext(sql, cxt.get()); unique_ptr catalogReq(new SCatalogReq(), MockCatalogService::destoryCatalogReq); - unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), _destroyQuery); - doParseSqlSyntax(&cxt, query.get(), catalogReq.get()); + unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); + doParseSqlSyntax(cxt.get(), query.get(), catalogReq.get()); SQuery* pQuery = *(query.get()); string err; @@ -438,7 +444,7 @@ class ParserTestBaseImpl { unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); doGetAllMeta(catalogReq.get(), metaData.get()); - doAnalyseSqlSemantic(&cxt, catalogReq.get(), metaData.get(), pQuery); + doAnalyseSqlSemantic(cxt.get(), catalogReq.get(), metaData.get(), pQuery); } catch (const TerminateFlag& e) { // success and terminate } catch (const runtime_error& e) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index b160f45479..a700a21301 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -617,7 +617,7 @@ static bool pushDownCondOptIsPriKey(SNode* pNode, SNodeList* pTableCols) { return false; } SColumnNode* pCol = (SColumnNode*)pNode; - if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) { + if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId || TSDB_SYSTEM_TABLE == pCol->tableType) { return false; } return pushDownCondOptBelongThisTable(pNode, pTableCols); diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index a3cb0c2654..106e674152 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -1007,6 +1007,7 @@ static int32_t stbSplSplitMergeScanNode(SSplitContext* pCxt, SLogicSubplan* pSub code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, pMergeScan, groupSort); } if (TSDB_CODE_SUCCESS == code) { + nodesDestroyNode((SNode*)pScan); code = nodesListMakeStrictAppend(&pSubplan->pChildren, (SNode*)splCreateScanSubplan(pCxt, pMergeScan, SPLIT_FLAG_STABLE_SPLIT)); } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 35903d45b1..e4f02f12e6 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -33,7 +33,10 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo SLogicSubplan* pLogicSubplan = NULL; SQueryLogicPlan* pLogicPlan = NULL; - int32_t code = createLogicPlan(pCxt, &pLogicSubplan); + int32_t code = nodesAcquireAllocator(pCxt->allocatorId); + if (TSDB_CODE_SUCCESS == code) { + code = createLogicPlan(pCxt, &pLogicSubplan); + } if (TSDB_CODE_SUCCESS == code) { code = optimizeLogicPlan(pCxt, pLogicSubplan); } @@ -49,6 +52,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo if (TSDB_CODE_SUCCESS == code) { dumpQueryPlan(*pPlan); } + nodesReleaseAllocator(pCxt->allocatorId); nodesDestroyNode((SNode*)pLogicSubplan); nodesDestroyNode((SNode*)pLogicPlan); diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp index a553d3addc..8b8f92bd4f 100644 --- a/source/libs/planner/test/planGroupByTest.cpp +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -40,6 +40,8 @@ TEST_F(PlanGroupByTest, basic) { run("SELECT COUNT(*) FROM st1 GROUP BY c1"); run("SELECT SUM(c1) FROM st1 GROUP BY c2 HAVING SUM(c1) IS NOT NULL"); + + run("SELECT AVG(c1) FROM st1"); } TEST_F(PlanGroupByTest, withPartitionBy) { diff --git a/source/libs/planner/test/planJoinTest.cpp b/source/libs/planner/test/planJoinTest.cpp index 66ef4d3f19..535bb0b416 100644 --- a/source/libs/planner/test/planJoinTest.cpp +++ b/source/libs/planner/test/planJoinTest.cpp @@ -28,6 +28,8 @@ TEST_F(PlanJoinTest, basic) { run("SELECT t1.*, t2.* FROM st1s1 t1, st1s2 t2 WHERE t1.ts = t2.ts"); run("SELECT t1.c1, t2.c1 FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts"); + + run("SELECT t1.c1, t2.c1 FROM st1 t1 JOIN st2 t2 ON t1.ts = t2.ts"); } TEST_F(PlanJoinTest, complex) { @@ -56,9 +58,3 @@ TEST_F(PlanJoinTest, multiJoin) { run("SELECT t1.c1, t2.c1 FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts JOIN st1s3 t3 ON t1.ts = t3.ts"); } - -TEST_F(PlanJoinTest, stable) { - useDb("root", "test"); - - run("SELECT t1.c1, t2.c1 FROM st1 t1 JOIN st2 t2 ON t1.ts = t2.ts "); -} diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index 8f6fc832a2..df6e72ce46 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -22,6 +22,7 @@ #include "mockCatalog.h" #include "parser.h" #include "planTestUtil.h" +#include "tglobal.h" class PlannerEnv : public testing::Environment { public: @@ -30,6 +31,8 @@ class PlannerEnv : public testing::Environment { initMetaDataEnv(); generateMetaData(); initLog(TD_TMP_DIR_PATH "td"); + initCfg(); + nodesInitAllocatorSet(); } virtual void TearDown() { @@ -37,6 +40,7 @@ class PlannerEnv : public testing::Environment { qCleanupKeywordsTable(); fmFuncMgtDestroy(); taosCloseLog(); + nodesDestroyAllocatorSet(); } PlannerEnv() {} @@ -67,6 +71,8 @@ class PlannerEnv : public testing::Environment { std::cout << "failed to init log file" << std::endl; } } + + void initCfg() { tsQueryPlannerTrace = true; } }; static void parseArg(int argc, char* argv[]) { @@ -79,6 +85,7 @@ static void parseArg(int argc, char* argv[]) { {"limitSql", required_argument, NULL, 'i'}, {"log", required_argument, NULL, 'l'}, {"queryPolicy", required_argument, NULL, 'q'}, + {"useNodeAllocator", required_argument, NULL, 'a'}, {0, 0, 0, 0} }; // clang-format on @@ -99,6 +106,9 @@ static void parseArg(int argc, char* argv[]) { case 'q': setQueryPolicy(optarg); break; + case 'a': + setUseNodeAllocator(optarg); + break; default: break; } diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 2b8e3d9864..73d695195c 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -41,6 +41,7 @@ using namespace testing; enum DumpModule { DUMP_MODULE_NOTHING = 1, + DUMP_MODULE_SQL, DUMP_MODULE_PARSER, DUMP_MODULE_LOGIC, DUMP_MODULE_OPTIMIZED, @@ -56,10 +57,13 @@ int32_t g_skipSql = 0; int32_t g_limitSql = 0; int32_t g_logLevel = 131; int32_t g_queryPolicy = QUERY_POLICY_VNODE; +bool g_useNodeAllocator = false; void setDumpModule(const char* pModule) { if (NULL == pModule) { g_dumpModule = DUMP_MODULE_ALL; + } else if (0 == strncasecmp(pModule, "sql", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_SQL; } else if (0 == strncasecmp(pModule, "parser", strlen(pModule))) { g_dumpModule = DUMP_MODULE_PARSER; } else if (0 == strncasecmp(pModule, "logic", strlen(pModule))) { @@ -79,10 +83,11 @@ void setDumpModule(const char* pModule) { } } -void setSkipSqlNum(const char* pNum) { g_skipSql = stoi(pNum); } -void setLimitSqlNum(const char* pNum) { g_limitSql = stoi(pNum); } -void setLogLevel(const char* pLogLevel) { g_logLevel = stoi(pLogLevel); } -void setQueryPolicy(const char* pQueryPolicy) { g_queryPolicy = stoi(pQueryPolicy); } +void setSkipSqlNum(const char* pArg) { g_skipSql = stoi(pArg); } +void setLimitSqlNum(const char* pArg) { g_limitSql = stoi(pArg); } +void setLogLevel(const char* pArg) { g_logLevel = stoi(pArg); } +void setQueryPolicy(const char* pArg) { g_queryPolicy = stoi(pArg); } +void setUseNodeAllocator(const char* pArg) { g_useNodeAllocator = stoi(pArg); } int32_t getLogLevel() { return g_logLevel; } @@ -124,6 +129,12 @@ class PlannerTestBaseImpl { } void runImpl(const string& sql, int32_t queryPolicy) { + int64_t allocatorId = 0; + if (g_useNodeAllocator) { + nodesCreateAllocator(sqlNo_, 32 * 1024, &allocatorId); + nodesAcquireAllocator(allocatorId); + } + reset(); tsQueryPolicy = queryPolicy; try { @@ -155,8 +166,13 @@ class PlannerTestBaseImpl { dump(g_dumpModule); } catch (...) { dump(DUMP_MODULE_ALL); + nodesReleaseAllocator(allocatorId); + nodesDestroyAllocator(allocatorId); throw; } + + nodesReleaseAllocator(allocatorId); + nodesDestroyAllocator(allocatorId); } void prepare(const string& sql) { @@ -216,6 +232,8 @@ class PlannerTestBaseImpl { doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan); unique_ptr plan(pPlan, (void (*)(SQueryPlan*))nodesDestroyNode); + checkPlanMsg((SNode*)pPlan); + dump(g_dumpModule); } catch (...) { dump(DUMP_MODULE_ALL); @@ -252,7 +270,6 @@ class PlannerTestBaseImpl { string splitLogicPlan_; string scaledLogicPlan_; string physiPlan_; - string physiPlanMsg_; vector physiSubplans_; }; @@ -276,17 +293,16 @@ class PlannerTestBaseImpl { res_.splitLogicPlan_.clear(); res_.scaledLogicPlan_.clear(); res_.physiPlan_.clear(); - res_.physiPlanMsg_.clear(); res_.physiSubplans_.clear(); } void dump(DumpModule module) { - cout << "========================================== " << sqlNo_ << " sql : [" << stmtEnv_.sql_ << "]" << endl; - if (DUMP_MODULE_NOTHING == module) { return; } + cout << "========================================== " << sqlNo_ << " sql : [" << stmtEnv_.sql_ << "]" << endl; + if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) { if (res_.prepareAst_.empty()) { cout << "+++++++++++++++++++++syntax tree : " << endl; @@ -411,8 +427,6 @@ class PlannerTestBaseImpl { SNode* pSubplan; FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { res_.physiSubplans_.push_back(toString(pSubplan)); } } - res_.physiPlanMsg_ = toMsg((SNode*)(*pPlan)); - cout << "json len: " << res_.physiPlan_.length() << ", msg len: " << res_.physiPlanMsg_.length() << endl; } void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { @@ -451,27 +465,16 @@ class PlannerTestBaseImpl { string toString(const SNode* pRoot) { char* pStr = NULL; int32_t len = 0; - - auto start = chrono::steady_clock::now(); DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len) - if (QUERY_NODE_PHYSICAL_PLAN == nodeType(pRoot)) { - cout << "nodesNodeToString: " - << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl; - } - string str(pStr); taosMemoryFreeClear(pStr); return str; } - string toMsg(const SNode* pRoot) { + void checkPlanMsg(const SNode* pRoot) { char* pStr = NULL; int32_t len = 0; - - auto start = chrono::steady_clock::now(); DO_WITH_THROW(nodesNodeToMsg, pRoot, &pStr, &len) - cout << "nodesNodeToMsg: " - << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl; string copyStr(pStr, len); SNode* pNode = NULL; @@ -491,9 +494,7 @@ class PlannerTestBaseImpl { nodesDestroyNode(pNode); taosMemoryFreeClear(pNewStr); - string str(pStr, len); taosMemoryFreeClear(pStr); - return str; } caseEnv caseEnv_; diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h index b0ddd726a6..be8b51f769 100644 --- a/source/libs/planner/test/planTestUtil.h +++ b/source/libs/planner/test/planTestUtil.h @@ -41,11 +41,12 @@ class PlannerTestBase : public testing::Test { std::unique_ptr impl_; }; -extern void setDumpModule(const char* pModule); -extern void setSkipSqlNum(const char* pNum); -extern void setLimitSqlNum(const char* pNum); -extern void setLogLevel(const char* pLogLevel); -extern void setQueryPolicy(const char* pQueryPolicy); +extern void setDumpModule(const char* pArg); +extern void setSkipSqlNum(const char* pArg); +extern void setLimitSqlNum(const char* pArg); +extern void setLogLevel(const char* pArg); +extern void setQueryPolicy(const char* pArg); +extern void setUseNodeAllocator(const char* pArg); extern int32_t getLogLevel(); #endif // PLAN_TEST_UTIL_H diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index e7695b2f04..5d243d86e8 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -99,7 +99,7 @@ typedef struct SFilterRange { typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t); typedef int32_t(*filter_desc_compare_func)(const void *, const void *); -typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t); +typedef bool(*filter_exec_func)(void *info, int32_t numOfRows, SColumnInfoData* p, SColumnDataAgg *statis, int16_t numOfCols); typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char*, void **); typedef struct SFilterRangeCompare { diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 9e67635437..dc6b505bb9 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -2976,14 +2976,12 @@ _return: return TSDB_CODE_SUCCESS; } -bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { +bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, SColumnInfoData* pRes, SColumnDataAgg *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; uint32_t *unitIdx = NULL; - if (*p == NULL) { - *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); - } + int8_t* p = (int8_t*)pRes->pData; for (int32_t i = 0; i < numOfRows; ++i) { //FILTER_UNIT_CLR_F(info); @@ -3002,35 +3000,35 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, uint8_t optr = cunit->optr; if (colDataIsNull((SColumnInfoData *)(cunit->colData), 0, i, NULL)) { - (*p)[i] = optr == OP_TYPE_IS_NULL ? true : false; + p[i] = (optr == OP_TYPE_IS_NULL) ? true : false; } else { if (optr == OP_TYPE_IS_NOT_NULL) { - (*p)[i] = 1; + p[i] = 1; } else if (optr == OP_TYPE_IS_NULL) { - (*p)[i] = 0; + p[i] = 0; } else if (cunit->rfunc >= 0) { - (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + p[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); } else { - (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + p[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); } //FILTER_UNIT_SET_R(info, uidx, p[i]); //FILTER_UNIT_SET_F(info, uidx); } - if ((*p)[i] == 0) { + if (p[i] == 0) { break; } } - if ((*p)[i]) { + if (p[i]) { break; } unitIdx += unitNum; } - if ((*p)[i] == 0) { + if (p[i] == 0) { all = false; } } @@ -3040,7 +3038,7 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, -int32_t filterExecuteBasedOnStatis(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols, bool* all) { +int32_t filterExecuteBasedOnStatis(SFilterInfo *info, int32_t numOfRows, SColumnInfoData* p, SColumnDataAgg *statis, int16_t numOfCols, bool* all) { if (statis && numOfRows >= FILTER_RM_UNIT_MIN_ROWS) { info->blkFlag = 0; @@ -3058,7 +3056,6 @@ int32_t filterExecuteBasedOnStatis(SFilterInfo *info, int32_t numOfRows, int8_t* assert(info->unitNum > 1); *all = filterExecuteBasedOnStatisImpl(info, numOfRows, p, statis, numOfCols); - goto _return; } } @@ -3067,59 +3064,55 @@ int32_t filterExecuteBasedOnStatis(SFilterInfo *info, int32_t numOfRows, int8_t* _return: info->blkFlag = 0; - return TSDB_CODE_SUCCESS; } - -static FORCE_INLINE bool filterExecuteImplAll(void *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { +static FORCE_INLINE bool filterExecuteImplAll(void *info, int32_t numOfRows, SColumnInfoData* p, SColumnDataAgg *statis, int16_t numOfCols) { return true; } -static FORCE_INLINE bool filterExecuteImplEmpty(void *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + +static FORCE_INLINE bool filterExecuteImplEmpty(void *info, int32_t numOfRows, SColumnInfoData* p, SColumnDataAgg *statis, int16_t numOfCols) { return false; } -static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + +static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, SColumnInfoData* pRes, SColumnDataAgg *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; - if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { - return all; - } + int8_t* p = (int8_t*)pRes->pData; - if (*p == NULL) { - *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); + if (filterExecuteBasedOnStatis(info, numOfRows, pRes, statis, numOfCols, &all) == 0) { + return all; } for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); - (*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); + p[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); - if ((*p)[i] == 0) { + if (p[i] == 0) { all = false; } } return all; } -static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { +static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows, SColumnInfoData* pRes, SColumnDataAgg *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; - if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + if (filterExecuteBasedOnStatis(info, numOfRows, pRes, statis, numOfCols, &all) == 0) { return all; } - if (*p == NULL) { - *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); - } + int8_t* p = (int8_t*)pRes->pData; for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); - (*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); - if ((*p)[i] == 0) { + p[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); + if (p[i] == 0) { all = false; } } @@ -3127,7 +3120,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows return all; } -bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { +bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, SColumnInfoData* pRes, SColumnDataAgg *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; uint16_t dataSize = info->cunits[0].dataSize; @@ -3136,13 +3129,11 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD void *valData2 = info->cunits[0].valData2; __compar_fn_t func = gDataCompare[info->cunits[0].func]; - if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + if (filterExecuteBasedOnStatis(info, numOfRows, pRes, statis, numOfCols, &all) == 0) { return all; } - if (*p == NULL) { - *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); - } + int8_t* p = (int8_t*) pRes->pData; for (int32_t i = 0; i < numOfRows; ++i) { void *colData = colDataGetData((SColumnInfoData *)info->cunits[0].colData, i); @@ -3152,9 +3143,9 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD continue; } - (*p)[i] = (*rfunc)(colData, colData, valData, valData2, func); + p[i] = (*rfunc)(colData, colData, valData, valData2, func); - if ((*p)[i] == 0) { + if (p[i] == 0) { all = false; } } @@ -3162,23 +3153,21 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD return all; } -bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { +bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, SColumnInfoData* pRes, SColumnDataAgg *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; - if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + if (filterExecuteBasedOnStatis(info, numOfRows, pRes, statis, numOfCols, &all) == 0) { return all; } - if (*p == NULL) { - *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); - } + int8_t* p = (int8_t*) pRes->pData; for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); if (colData == NULL || colDataIsNull_s((SColumnInfoData *)info->cunits[uidx].colData, i)) { - (*p)[i] = 0; + p[i] = 0; all = false; continue; } @@ -3191,14 +3180,14 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa qError("castConvert1 taosUcs4ToMbs error"); }else{ varDataSetLen(newColData, len); - (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData); + p[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData); } taosMemoryFreeClear(newColData); }else{ - (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); + p[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); } - if ((*p)[i] == 0) { + if (p[i] == 0) { all = false; } } @@ -3207,17 +3196,15 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa } -bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { +bool filterExecuteImpl(void *pinfo, int32_t numOfRows, SColumnInfoData* pRes, SColumnDataAgg *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; - if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + if (filterExecuteBasedOnStatis(info, numOfRows, pRes, statis, numOfCols, &all) == 0) { return all; } - if (*p == NULL) { - *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); - } + int8_t* p = (int8_t*) pRes->pData; for (int32_t i = 0; i < numOfRows; ++i) { //FILTER_UNIT_CLR_F(info); @@ -3235,14 +3222,14 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg uint8_t optr = cunit->optr; if (colData == NULL || colDataIsNull((SColumnInfoData *)(cunit->colData), 0, i, NULL)) { - (*p)[i] = optr == OP_TYPE_IS_NULL ? true : false; + p[i] = optr == OP_TYPE_IS_NULL ? true : false; } else { if (optr == OP_TYPE_IS_NOT_NULL) { - (*p)[i] = 1; + p[i] = 1; } else if (optr == OP_TYPE_IS_NULL) { - (*p)[i] = 0; + p[i] = 0; } else if (cunit->rfunc >= 0) { - (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + p[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); } else { if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == OP_TYPE_MATCH || cunit->optr == OP_TYPE_NMATCH)){ char *newColData = taosMemoryCalloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); @@ -3251,11 +3238,11 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg qError("castConvert1 taosUcs4ToMbs error"); }else{ varDataSetLen(newColData, len); - (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData); + p[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData); } taosMemoryFreeClear(newColData); }else{ - (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + p[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); } } @@ -3263,17 +3250,17 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg //FILTER_UNIT_SET_F(info, uidx); } - if ((*p)[i] == 0) { + if (p[i] == 0) { break; } } - if ((*p)[i]) { + if (p[i]) { break; } } - if ((*p)[i] == 0) { + if (p[i] == 0) { all = false; } } @@ -4026,38 +4013,63 @@ _return: FLT_RET(code); } -bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { +bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData** p, SColumnDataAgg *statis, int16_t numOfCols, int32_t *pResultStatus) { if (NULL == info) { + *pResultStatus = FILTER_RESULT_ALL_QUALIFIED; + return false; + } + + SScalarParam output = {0}; + SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + + int32_t code = sclCreateColumnInfoData(&type, pSrc->info.rows, &output); + if (code != TSDB_CODE_SUCCESS) { return false; } if (info->scalarMode) { - SScalarParam output = {0}; - - SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; - int32_t code = sclCreateColumnInfoData(&type, pSrc->info.rows, &output); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - SArray *pList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(pList, &pSrc); FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pList, &output)); - *p = taosMemoryMalloc(output.numOfRows * sizeof(bool)); - - memcpy(*p, output.columnData->pData, output.numOfRows); - colDataDestroy(output.columnData); - taosMemoryFree(output.columnData); + *p = output.columnData; taosArrayDestroy(pList); + + if (output.numOfQualified == output.numOfRows) { + *pResultStatus = FILTER_RESULT_ALL_QUALIFIED; + } else if (output.numOfQualified == 0) { + *pResultStatus = FILTER_RESULT_NONE_QUALIFIED; + } else { + *pResultStatus = FILTER_RESULT_PARTIAL_QUALIFIED; + } return false; + } else { + *p = output.columnData; + output.numOfRows = pSrc->info.rows; + + bool keep = (*info->func)(info, pSrc->info.rows, *p, statis, numOfCols); + + // todo this should be return during filter procedure + int32_t num = 0; + for(int32_t i = 0; i < output.numOfRows; ++i) { + if (((int8_t*)((*p)->pData))[i] == 1) { + ++num; + } + } + + if (num == output.numOfRows) { + *pResultStatus = FILTER_RESULT_ALL_QUALIFIED; + } else if (num == 0) { + *pResultStatus = FILTER_RESULT_NONE_QUALIFIED; + } else { + *pResultStatus = FILTER_RESULT_PARTIAL_QUALIFIED; + } + + return keep; } - - return (*info->func)(info, pSrc->info.rows, p, statis, numOfCols); } - typedef struct SClassifyConditionCxt { bool hasPrimaryKey; bool hasTagIndexCol; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index cd1f6624bd..9cba94d85a 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -606,6 +606,8 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o SCL_ERR_JRET(code); } + int32_t numOfQualified = 0; + bool value = false; bool complete = true; for (int32_t i = 0; i < rowNum; ++i) { @@ -631,6 +633,9 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o if (complete) { colDataAppend(output->columnData, i, (char*) &value, false); + if (value) { + numOfQualified++; + } } } @@ -639,8 +644,9 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o output->numOfRows = 0; } -_return: + output->numOfQualified = numOfQualified; +_return: sclFreeParamList(params, paramNum); SCL_RET(code); } @@ -847,7 +853,7 @@ EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { memcpy(res->datum.p, output.columnData->pData, len); } else if (IS_VAR_DATA_TYPE(type)) { //res->datum.p = taosMemoryCalloc(res->node.resType.bytes + VARSTR_HEADER_SIZE + 1, 1); - res->datum.p = taosMemoryCalloc(varDataTLen(output.columnData->pData), 1); + res->datum.p = taosMemoryCalloc(varDataTLen(output.columnData->pData) + 1, 1); res->node.resType.bytes = varDataTLen(output.columnData->pData); memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData)); } else { @@ -1242,6 +1248,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows); colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL); pDst->numOfRows = res->numOfRows; + pDst->numOfQualified = res->numOfQualified; } sclFreeParam(res); @@ -1249,7 +1256,6 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { } _return: - //nodesDestroyNode(pNode); sclFreeRes(ctx.pRes); return code; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index a003315fca..fe2a970aaa 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1475,19 +1475,19 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO void vectorAssign(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { SColumnInfoData *pOutputCol = pOut->columnData; - pOut->numOfRows = pLeft->numOfRows; -// if (IS_HELPER_NULL(pRight->columnData, 0)) { if(colDataIsNull_s(pRight->columnData, 0)){ - for (int32_t i = 0; i < pOut->numOfRows; ++i) { - colDataAppend(pOutputCol, i, NULL, true); - } + colDataAppendNNULL(pOutputCol, 0, pOut->numOfRows); } else { + char* d = colDataGetData(pRight->columnData, 0); for (int32_t i = 0; i < pOut->numOfRows; ++i) { - colDataAppend(pOutputCol, i, colDataGetData(pRight->columnData, 0), false); + colDataAppend(pOutputCol, i, d, false); } } + + ASSERT(pRight->numOfQualified == 1 || pRight->numOfQualified == 0); + pOut->numOfQualified = pRight->numOfQualified * pOut->numOfRows; } void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { @@ -1646,38 +1646,60 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, doReleaseVec(pRightCol, rightConvert); } -#define VEC_COM_INNER(pCol, index1, index2) \ - for (; i < pCol->numOfRows && i >= 0; i += step) {\ - if (IS_HELPER_NULL(pLeft->columnData, index1) || IS_HELPER_NULL(pRight->columnData, index2)) {\ - bool res = false;\ - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);\ - continue;\ - }\ - char *pLeftData = colDataGetData(pLeft->columnData, index1);\ - char *pRightData = colDataGetData(pRight->columnData, index2);\ - int64_t leftOut = 0;\ - int64_t rightOut = 0;\ - bool freeLeft = false;\ - bool freeRight = false;\ - bool isJsonnull = false;\ - bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight),\ - &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull, &freeLeft, &freeRight);\ - if(isJsonnull){\ - ASSERT(0);\ - }\ - if(!pLeftData || !pRightData){\ - result = false;\ - }\ - if(!result){\ - colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);\ - }else{\ - bool res = filterDoCompare(fp, optr, pLeftData, pRightData);\ - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);\ - }\ - if(freeLeft) taosMemoryFreeClear(pLeftData);\ - if(freeRight) taosMemoryFreeClear(pRightData);\ +int32_t doVectorCompareImpl(int32_t numOfRows, SScalarParam *pOut, int32_t startIndex, int32_t step, __compar_fn_t fp, + SScalarParam *pLeft, SScalarParam *pRight, int32_t optr) { + int32_t num = 0; + + for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) { + int32_t leftIndex = (i >= pLeft->numOfRows)? 0:i; + int32_t rightIndex = (i >= pRight->numOfRows)? 0:i; + + if (IS_HELPER_NULL(pLeft->columnData, leftIndex) || IS_HELPER_NULL(pRight->columnData, rightIndex)) { + bool res = false; + colDataAppendInt8(pOut->columnData, i, (int8_t *)&res); + continue; + } + + char * pLeftData = colDataGetData(pLeft->columnData, leftIndex); + char * pRightData = colDataGetData(pRight->columnData, rightIndex); + int64_t leftOut = 0; + int64_t rightOut = 0; + bool freeLeft = false; + bool freeRight = false; + bool isJsonnull = false; + + bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, + &leftOut, &rightOut, &isJsonnull, &freeLeft, &freeRight); + if (isJsonnull) { + ASSERT(0); + } + + if (!pLeftData || !pRightData) { + result = false; + } + + if (!result) { + colDataAppendInt8(pOut->columnData, i, (int8_t *)&result); + } else { + bool res = filterDoCompare(fp, optr, pLeftData, pRightData); + colDataAppendInt8(pOut->columnData, i, (int8_t *)&res); + if (res) { + ++num; + } + } + + if (freeLeft) { + taosMemoryFreeClear(pLeftData); + } + + if (freeRight) { + taosMemoryFreeClear(pRightData); + } } + return num; +} + void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -1704,16 +1726,12 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * char *pLeftData = colDataGetData(pLeft->columnData, i); bool res = filterDoCompare(fp, optr, pLeftData, pRight->pHashFilter); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); + if (res) { + pOut->numOfQualified++; + } } - return; - } - - if (pLeft->numOfRows == pRight->numOfRows) { - VEC_COM_INNER(pLeft, i, i) - } else if (pRight->numOfRows == 1) { - VEC_COM_INNER(pLeft, i, 0) - } else if (pLeft->numOfRows == 1) { - VEC_COM_INNER(pRight, 0, i) + } else { // normal compare + pOut->numOfQualified = doVectorCompareImpl(pOut->numOfRows, pOut, i, step, fp, pLeft, pRight, optr); } } diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 7fea286732..a62531a875 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -254,7 +254,8 @@ typedef struct SSchJob { SRequestConnInfo conn; SArray *nodeList; // qnode/vnode list, SArray SArray *levels; // starting from 0. SArray - SQueryPlan *pDag; + SQueryPlan *pDag; + int64_t allocatorRefId; SArray *dataSrcTasks; // SArray int32_t levelIdx; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 98501427ab..9880490594 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -673,6 +673,7 @@ void schFreeJobImpl(void *job) { destroyQueryExecRes(&pJob->execRes); qDestroyQueryPlan(pJob->pDag); + nodesReleaseAllocatorWeakRef(pJob->allocatorRefId); taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->fetchRes); @@ -724,6 +725,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->sql = strdup(pReq->sql); } pJob->pDag = pReq->pDag; + pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId); pJob->chkKillFp = pReq->chkKillFp; pJob->chkKillParam = pReq->chkKillParam; pJob->userRes.execFp = pReq->execFp; diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 8b3a2016b8..2cb227d84b 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -20,6 +20,7 @@ #include "tmsg.h" #include "tref.h" #include "trpc.h" +#include "tglobal.h" void schFreeTask(SSchJob *pJob, SSchTask *pTask) { schDeregisterTaskHb(pJob, pTask); @@ -870,8 +871,12 @@ int32_t schLaunchTaskImpl(void *param) { SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg, pTask->msgLen); SCH_ERR_JRET(code); - } else { - SCH_TASK_DLOGL("physical plan len:%d, %s", pTask->msgLen, pTask->msg); + } else if (tsQueryPlannerTrace) { + char *msg = NULL; + int32_t msgLen = 0; + qSubPlanToString(plan, &msg, &msgLen); + SCH_TASK_DLOGL("physical plan len:%d, %s", msgLen, msg); + taosMemoryFree(msg); } } diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index 1e2eefabf4..f4878ea861 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -80,6 +80,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) ASSERT(xFree); for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) { + tdbDebug("tdbPage/destroy/free ovfl cell: %p/%p", pPage->apOvfl[iOvfl], pPage); tdbOsFree(pPage->apOvfl[iOvfl]); } @@ -152,7 +153,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl pNewCell = (SCell *)tdbOsMalloc(szCell); memcpy(pNewCell, pCell, szCell); - tdbDebug("tdbPage/new ovfl cell: %p", pNewCell); + tdbDebug("tdbPage/insert/new ovfl cell: %p/%p", pNewCell, pPage); pPage->apOvfl[iOvfl] = pNewCell; pPage->aiOvfl[iOvfl] = idx; @@ -202,7 +203,7 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { if (pPage->aiOvfl[iOvfl] == idx) { // remove the over flow cell tdbOsFree(pPage->apOvfl[iOvfl]); - tdbDebug("tdbPage/free ovfl cell: %p", pPage->apOvfl[iOvfl]); + tdbDebug("tdbPage/drop/free ovfl cell: %p", pPage->apOvfl[iOvfl]); for (; (++iOvfl) < pPage->nOverflow;) { pPage->aiOvfl[iOvfl - 1] = pPage->aiOvfl[iOvfl] - 1; pPage->apOvfl[iOvfl - 1] = pPage->apOvfl[iOvfl]; @@ -255,6 +256,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL); pNewCell = (SCell *)tdbOsMalloc(szCell); memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell); + tdbDebug("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage); } pToPage->apOvfl[iOvfl] = pNewCell; diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 5284aeff77..c69046f707 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -116,7 +116,6 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { } #endif } - // TODO truncate file if (found == NULL) { // file corrupted, no complete log @@ -125,8 +124,20 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } + + // truncate file SWalCkHead* lastEntry = (SWalCkHead*)found; int64_t retVer = lastEntry->head.version; + int64_t lastEntryBeginOffset = offset + (int64_t)((char*)found - (char*)buf); + int64_t lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen; + if (lastEntryEndOffset != fileSize) { + wWarn("vgId:%d repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset, + fileSize); + taosFtruncateFile(pFile, lastEntryEndOffset); + ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = lastEntryEndOffset; + pWal->totSize -= (fileSize - lastEntryEndOffset); + } + taosCloseFile(&pFile); taosMemoryFree(buf); @@ -226,16 +237,92 @@ int walCheckAndRepairMeta(SWal* pWal) { } } - // TODO: set fileSize and lastVer if necessary - return 0; } int walCheckAndRepairIdx(SWal* pWal) { - // TODO: iterate all log files - // if idx not found, scan log and write idx - // if found, check complete by first and last entry of each idx file - // if idx incomplete, binary search last valid entry, and then build other part + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + for (int32_t i = 0; i < sz; i++) { + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); + + char fnameStr[WAL_FILE_LEN]; + walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); + int64_t fsize; + TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE); + if (pIdxFile == NULL) { + ASSERT(0); + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fnameStr, terrstr()); + return -1; + } + + taosFStatFile(pIdxFile, &fsize, NULL); + if (fsize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { + taosCloseFile(&pIdxFile); + continue; + } + + int32_t left = fsize % sizeof(SWalIdxEntry); + int64_t offset = taosLSeekFile(pIdxFile, -left, SEEK_END); + if (left != 0) { + taosFtruncateFile(pIdxFile, offset); + wWarn("vgId:%d wal truncate file %s to offset %ld since size invalid, file size %ld", pWal->cfg.vgId, fnameStr, + offset, fsize); + } + offset -= sizeof(SWalIdxEntry); + + SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer}; + while (1) { + if (offset < 0) { + taosLSeekFile(pIdxFile, 0, SEEK_SET); + taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); + break; + } + taosLSeekFile(pIdxFile, offset, SEEK_SET); + int64_t contLen = taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); + if (contLen < 0 || contLen != sizeof(SWalIdxEntry)) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if ((idxEntry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry) != offset) { + taosFtruncateFile(pIdxFile, offset); + wWarn("vgId:%d wal truncate file %s to offset %ld since entry invalid, entry ver %ld, entry offset %ld", + pWal->cfg.vgId, fnameStr, offset, idxEntry.ver, idxEntry.offset); + offset -= sizeof(SWalIdxEntry); + } else { + break; + } + } + + if (idxEntry.ver < pFileInfo->lastVer) { + char fLogNameStr[WAL_FILE_LEN]; + walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr); + TdFilePtr pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ); + if (pLogFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr()); + return -1; + } + while (idxEntry.ver < pFileInfo->lastVer) { + taosLSeekFile(pLogFile, idxEntry.offset, SEEK_SET); + SWalCkHead ckHead; + taosReadFile(pLogFile, &ckHead, sizeof(SWalCkHead)); + if (idxEntry.ver != ckHead.head.version) { + // todo truncate this idx also + taosCloseFile(&pLogFile); + wError("vgId:%d, invalid repair case, log seek to %ld to find ver %ld, actual ver %ld", pWal->cfg.vgId, + idxEntry.offset, idxEntry.ver, ckHead.head.version); + return -1; + } + idxEntry.ver = ckHead.head.version + 1; + idxEntry.offset = idxEntry.offset + sizeof(SWalCkHead) + ckHead.head.bodyLen; + wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset); + taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); + } + taosCloseFile(&pLogFile); + } + taosCloseFile(&pIdxFile); + } return 0; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index c939c8c436..a55f00d277 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -149,15 +149,21 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { walLoadMeta(pWal); if (walCheckAndRepairMeta(pWal) < 0) { + wError("vgId:%d cannot open wal since repair meta file failed", pWal->cfg.vgId); taosHashCleanup(pWal->pRefHash); taosRemoveRef(tsWal.refSetId, pWal->refId); taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); - taosMemoryFree(pWal); return NULL; } if (walCheckAndRepairIdx(pWal) < 0) { + wError("vgId:%d cannot open wal since repair idx file failed", pWal->cfg.vgId); + taosHashCleanup(pWal->pRefHash); + taosRemoveRef(tsWal.refSetId, pWal->refId); + taosThreadMutexDestroy(&pWal->mutex); + taosArrayDestroy(pWal->fileInfoSet); + return NULL; } wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, diff --git a/tests/docs-examples-test/csharp.sh b/tests/docs-examples-test/csharp.sh index d7f2670478..8d1031ab8f 100644 --- a/tests/docs-examples-test/csharp.sh +++ b/tests/docs-examples-test/csharp.sh @@ -6,24 +6,32 @@ pgrep taosd || taosd >> /dev/null 2>&1 & pgrep taosadapter || taosadapter >> /dev/null 2>&1 & cd ../../docs/examples/csharp -#dotnet run --project connect.csproj +dotnet run --project connect/connect.csproj -#taos -s "drop database if exists power" -#dotnet run --project sqlinsert.csproj -#dotnet run --project query.csproj -#dotnet run --project asyncquery.csproj -#dotnet run --project subscribe.csproj +taos -s "drop database if exists power" +dotnet run --project sqlInsert/sqlinsert.csproj +dotnet run --project query/query.csproj +dotnet run --project asyncQuery/asyncquery.csproj +dotnet run --project subscribe/subscribe.csproj -#taos -s "drop topic if exists topic_example" -#taos -s "drop database if exists power" -#dotnet run --project stmtinsert.csproj +taos -s "drop topic if exists topic_example" +taos -s "drop database if exists power" +dotnet run --project stmtInsert/stmtinsert.csproj -#taos -s "drop database if exists test" -#dotnet run --project influxdbline.csproj +taos -s "drop database if exists test" +dotnet run --project influxdbLine/influxdbline.csproj -#taos -s "drop database if exists test" -#dotnet run --project optstelnet.csproj +taos -s "drop database if exists test" +dotnet run --project optsTelnet/optstelnet.csproj -#taos -s "drop database if exists test" -#dotnet run --project optsjson.csproji -echo "uncomment temporily" +taos -s "drop database if exists test" +dotnet run --project optsJSON/optsJSON.csproj + +taos -s "create database if exists test" +dotnet run --project wsConnect/wsConnect.csproj +dotnet run --project wsInsert/wsInsert.csproj +dotnet run --project wsStmt/wsStmt.csproj +dotnet run --project wsQuery/wsQuery.csproj + +taos -s "drop database if exists test" +taos -s "drop database if exists power" \ No newline at end of file diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index f0ee9be46f..bb57f238f0 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -79,9 +79,11 @@ fi ulimit -c unlimited TMP_DIR=$WORKDIR/tmp - +SOURCEDIR=$WORKDIR/src MOUNT_DIR="" +packageName="TDengine-server-3.0.1.0-Linux-x64.tar.gz" rm -rf ${TMP_DIR}/thread_volume/$thread_no/sim +mkdir -p $SOURCEDIR mkdir -p ${TMP_DIR}/thread_volume/$thread_no/sim/tsim mkdir -p ${TMP_DIR}/thread_volume/$thread_no/coredump rm -rf ${TMP_DIR}/thread_volume/$thread_no/coredump/* @@ -90,6 +92,11 @@ if [ ! -d "${TMP_DIR}/thread_volume/$thread_no/$exec_dir" ]; then echo "cp -rf ${REPDIR}/tests/$subdir ${TMP_DIR}/thread_volume/$thread_no/" cp -rf ${REPDIR}/tests/$subdir ${TMP_DIR}/thread_volume/$thread_no/ fi + +if [ ! -f "${SOURCEDIR}/${packageName}" ]; then + wget -P ${SOURCEDIR} https://taosdata.com/assets-download/3.0/${packageName} +fi + MOUNT_DIR="$TMP_DIR/thread_volume/$thread_no/$exec_dir:$CONTAINER_TESTDIR/tests/$exec_dir" echo "$thread_no -> ${exec_dir}:$cmd" coredump_dir=`cat /proc/sys/kernel/core_pattern | xargs dirname` @@ -97,6 +104,7 @@ coredump_dir=`cat /proc/sys/kernel/core_pattern | xargs dirname` docker run \ -v $REP_MOUNT_PARAM \ -v $MOUNT_DIR \ + -v ${SOURCEDIR}:/usr/local/src/ \ -v "$TMP_DIR/thread_volume/$thread_no/sim:${SIM_DIR}" \ -v ${TMP_DIR}/thread_volume/$thread_no/coredump:$coredump_dir \ -v $WORKDIR/taos-connector-python/taos:/usr/local/lib/python3.8/site-packages/taos:ro \ diff --git a/tests/script/tsim/insert/basic.sim b/tests/script/tsim/insert/basic.sim index c4ef3e39da..c926cbc8b0 100644 --- a/tests/script/tsim/insert/basic.sim +++ b/tests/script/tsim/insert/basic.sim @@ -12,7 +12,7 @@ $tb = $tbPrefix . $i print =============== step1 sql drop database -x step1 step1: -sql create database $db +sql create database $db vgroups 2 sql use $db sql create table $tb (ts timestamp, speed int) diff --git a/tests/script/tsim/trans/create_db.sim b/tests/script/tsim/trans/create_db.sim index 8543bec144..f04e505dd5 100644 --- a/tests/script/tsim/trans/create_db.sim +++ b/tests/script/tsim/trans/create_db.sim @@ -56,7 +56,7 @@ if $data[0][2] != redoAction then return -1 endi -if $data[0][3] != d1 then +if $data[0][4] != d1 then return -1 endi @@ -120,7 +120,7 @@ if $system_content != Windows_NT then return -1 endi - if $data[0][3] != d2 then + if $data[0][4] != d2 then return -1 endi diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py new file mode 100644 index 0000000000..25b023bb76 --- /dev/null +++ b/tests/system-test/0-others/compatibility.py @@ -0,0 +1,157 @@ +from urllib.parse import uses_relative +import taos +import sys +import os +import time + + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + + +class TDTestCase: + def caseDescription(self): + ''' + 3.0 data compatibility test + case1: basedata version is 3.0.1.0 + ''' + return + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def getCfgPath(self): + buildPath = self.getBuildPath() + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + cfgPath = buildPath + "/../sim/dnode1/cfg/" + else: + cfgPath = buildPath + "/../sim/dnode1/cfg/" + + return cfgPath + + def installTaosd(self,bPath,cPath): + # os.system(f"rmtaos && mkdir -p {self.getBuildPath()}/build/lib/temp && mv {self.getBuildPath()}/build/lib/libtaos.so* {self.getBuildPath()}/build/lib/temp/ ") + # os.system(f" mv {bPath}/build {bPath}/build_bak ") + # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so {self.getBuildPath()}/build/lib/libtaos.so_bak ") + # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so.1 {self.getBuildPath()}/build/lib/libtaos.so.1_bak ") + + packagePath="/usr/local/src/" + packageName="TDengine-server-3.0.1.0-Linux-x64.tar.gz" + os.system(f"cd {packagePath} && tar xvf TDengine-server-3.0.1.0-Linux-x64.tar.gz && cd TDengine-server-3.0.1.0 && ./install.sh -e no " ) + tdDnodes.stop(1) + print(f"start taosd: nohup taosd -c {cPath} & ") + os.system(f" nohup taosd -c {cPath} & " ) + sleep(1) + + + + def buildTaosd(self,bPath): + # os.system(f"mv {bPath}/build_bak {bPath}/build ") + os.system(f" cd {bPath} && make install ") + + + def run(self): + bPath=self.getBuildPath() + cPath=self.getCfgPath() + dbname = "test" + stb = f"{dbname}.meters" + self.installTaosd(bPath,cPath) + tableNumbers=100 + recordNumbers1=100 + recordNumbers2=1000 + tdsqlF=tdCom.newTdSql() + print(tdsqlF) + tdsqlF.query(f"SELECT SERVER_VERSION();") + print(tdsqlF.query(f"SELECT SERVER_VERSION();")) + oldServerVersion=tdsqlF.queryResult[0][0] + tdLog.info(f"Base server version is {oldServerVersion}") + tdsqlF.query(f"SELECT CLIENT_VERSION();") + # the oldClientVersion can't be updated in the same python process,so the version is new compiled verison + oldClientVersion=tdsqlF.queryResult[0][0] + tdLog.info(f"Base client version is {oldClientVersion}") + + tdLog.printNoPrefix(f"==========step1:prepare and check data in old version-{oldServerVersion}") + tdLog.info(f"taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") + os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") + sleep(3) + + # tdsqlF.query(f"select count(*) from {stb}") + # tdsqlF.checkData(0,0,tableNumbers*recordNumbers1) + os.system("pkill taosd") + sleep(1) + + tdLog.printNoPrefix("==========step2:update new version ") + self.buildTaosd(bPath) + tdDnodes.start(1) + sleep(1) + tdsql=tdCom.newTdSql() + print(tdsql) + + + tdsql.query(f"SELECT SERVER_VERSION();") + nowServerVersion=tdsql.queryResult[0][0] + tdLog.info(f"New server version is {nowServerVersion}") + tdsql.query(f"SELECT CLIENT_VERSION();") + nowClientVersion=tdsql.queryResult[0][0] + tdLog.info(f"New client version is {nowClientVersion}") + + tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}") + tdsql.query(f"select count(*) from {stb}") + tdsql.checkData(0,0,tableNumbers*recordNumbers1) + os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ") + tdsql.query(f"select count(*) from {stb}") + tdsql.checkData(0,0,tableNumbers*recordNumbers2) + + tdsql=tdCom.newTdSql() + tdLog.printNoPrefix(f"==========step4:verify backticks in taos Sql-TD18542") + tdsql.execute("drop database if exists db") + tdsql.execute("create database db") + tdsql.execute("use db") + tdsql.execute("create stable db.stb1 (ts timestamp, c1 int) tags (t1 int);") + tdsql.execute("insert into db.ct1 using db.stb1 TAGS(1) values(now(),11);") + tdsql.error(" insert into `db.ct2` using db.stb1 TAGS(9) values(now(),11);") + tdsql.error(" insert into db.`db.ct2` using db.stb1 TAGS(9) values(now(),11);") + tdsql.execute("insert into `db`.ct3 using db.stb1 TAGS(3) values(now(),13);") + tdsql.query("select * from db.ct3") + tdsql.checkData(0,1,13) + tdsql.execute("insert into db.`ct4` using db.stb1 TAGS(4) values(now(),14);") + tdsql.query("select * from db.ct4") + tdsql.checkData(0,1,14) + tdsql.query("describe information_schema.ins_databases;") + qRows=tdsql.queryRows + for i in range(qRows) : + if tdsql.queryResult[i][0]=="retentions" : + return True + else: + return False + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 5550519e05..7bf0191ec1 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -595,11 +595,11 @@ class TDTestCase: tdSql.checkData(2, i, 15) tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(linear)") - tdSql.checkRows(1) + tdSql.checkRows(3) tdSql.checkCols(4) for i in range (tdSql.queryCols): - tdSql.checkData(0, i, 15) + tdSql.checkData(0, i, 13) tdLog.printNoPrefix("==========step10:test error cases") diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 847bc300b3..684d6a16e2 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -17,6 +17,7 @@ python3 ./test.py -f 0-others/udf_cfg2.py python3 ./test.py -f 0-others/sysinfo.py python3 ./test.py -f 0-others/user_control.py python3 ./test.py -f 0-others/fsync.py +python3 ./test.py -f 0-others/compatibility.py python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py diff --git a/tools/shell/src/shellUtil.c b/tools/shell/src/shellUtil.c index 0430428c38..8c47d16555 100644 --- a/tools/shell/src/shellUtil.c +++ b/tools/shell/src/shellUtil.c @@ -143,7 +143,7 @@ void shellCheckConnectMode() { shell.args.port = 6041; } shell.args.dsn = taosMemoryCalloc(1, 1024); - snprintf(shell.args.dsn, 1024, "ws://%s:%d/rest/ws", + snprintf(shell.args.dsn, 1024, "ws://%s:%d", shell.args.host, shell.args.port); } shell.args.cloud = false; diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index b8b8392b96..94bb909e29 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -206,26 +206,31 @@ void shellRunSingleCommandWebsocketImp(char *command) { printMode = true; // When output to a file, the switch does not work. } - if (!shell.ws_conn && shell_conn_ws_server(0)) { - return; - } - shell.stop_query = false; - st = taosGetTimestampUs(); + WS_RES* res; - WS_RES* res = ws_query_timeout(shell.ws_conn, command, shell.args.timeout); - int code = ws_errno(res); - if (code != 0) { - et = taosGetTimestampUs(); - fprintf(stderr, "\nDB: error: %s (%.6fs)\n", ws_errstr(res), (et - st)/1E6); - if (code == TSDB_CODE_WS_SEND_TIMEOUT || code == TSDB_CODE_WS_RECV_TIMEOUT) { - fprintf(stderr, "Hint: use -t to increase the timeout in seconds\n"); - } else if (code == TSDB_CODE_WS_INTERNAL_ERRO || code == TSDB_CODE_WS_CLOSED) { - fprintf(stderr, "TDengine server is down, will try to reconnect\n"); - shell.ws_conn = NULL; + for (int reconnectNum = 0; reconnectNum < 2; reconnectNum++) { + if (!shell.ws_conn && shell_conn_ws_server(0)) { + return; } - ws_free_result(res); - return; + st = taosGetTimestampUs(); + + res = ws_query_timeout(shell.ws_conn, command, shell.args.timeout); + int code = ws_errno(res); + if (code != 0 && !shell.stop_query) { + et = taosGetTimestampUs(); + fprintf(stderr, "\nDB: error: %s (%.6fs)\n", ws_errstr(res), (et - st)/1E6); + if (code == TSDB_CODE_WS_SEND_TIMEOUT || code == TSDB_CODE_WS_RECV_TIMEOUT) { + fprintf(stderr, "Hint: use -t to increase the timeout in seconds\n"); + } else if (code == TSDB_CODE_WS_INTERNAL_ERRO || code == TSDB_CODE_WS_CLOSED) { + fprintf(stderr, "TDengine server is down, will try to reconnect\n"); + shell.ws_conn = NULL; + } + ws_free_result(res); + if (reconnectNum == 0) continue; + return; + } + break; } double execute_time = ws_take_timing(res)/1E6;