Merge branch '3.0' of https://github.com/taosdata/TDengine into feat/TS-5776
This commit is contained in:
commit
f824a75f4d
|
@ -9,7 +9,7 @@ import imgStream from './assets/stream-processing-01.png';
|
|||
## Creating Stream Computing
|
||||
|
||||
```sql
|
||||
CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name[(field1_name, field2_name [PRIMARY KEY], ...)] [TAGS (create_definition [, create_definition] ...)] SUBTABLE(expression) AS subquery
|
||||
CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name[(field1_name, field2_name [PRIMARY KEY], ...)] [TAGS (create_definition [, create_definition] ...)] SUBTABLE(expression) AS subquery [notification_definition]
|
||||
stream_options: {
|
||||
TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time | FORCE_WINDOW_CLOSE]
|
||||
WATERMARK time
|
||||
|
@ -85,6 +85,8 @@ CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS
|
|||
SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10);
|
||||
```
|
||||
|
||||
notification_definition clause specifies the addresses to which notifications should be sent when designated events occur during window computations, such as window opening or closing. For more details, see [Stream Computing Event Notifications](#stream-computing-event-notifications).
|
||||
|
||||
## Stream Computation Partitioning
|
||||
|
||||
You can use `PARTITION BY TBNAME`, tags, regular columns, or expressions to partition a stream for multi-partition computation. Each partition's timeline and window are independent, aggregating separately, and writing into different subtables of the target table.
|
||||
|
@ -305,3 +307,223 @@ CREATE SNODE ON DNODE [id]
|
|||
|
||||
The id is the serial number of the dnode in the cluster. Please be mindful of the selected dnode, as the intermediate state of stream computing will automatically be backed up on it.
|
||||
Starting from version 3.3.4.0, in a multi-replica environment, creating a stream will perform an **existence check** of snode, requiring the snode to be created first. If the snode does not exist, the stream cannot be created.
|
||||
|
||||
## Stream Computing Event Notifications
|
||||
|
||||
### User Guide
|
||||
|
||||
Stream computing supports sending event notifications to external systems when windows open or close. Users can specify the events to be notified and the target addresses for receiving notification messages using the notification_definition clause.
|
||||
|
||||
```sql
|
||||
notification_definition:
|
||||
NOTIFY (url [, url] ...) ON (event_type [, event_type] ...) [notification_options]
|
||||
|
||||
event_type:
|
||||
'WINDOW_OPEN'
|
||||
| 'WINDOW_CLOSE'
|
||||
|
||||
notification_options: {
|
||||
NOTIFY_HISTORY [0|1]
|
||||
ON_FAILURE [DROP|PAUSE]
|
||||
}
|
||||
```
|
||||
|
||||
The rules for the syntax above are as follows:
|
||||
1. `url`: Specifies the target address for the notification. It must include the protocol, IP or domain name, port, and may include a path and parameters. Currently, only the websocket protocol is supported. For example: 'ws://localhost:8080', 'ws://localhost:8080/notify', 'wss://localhost:8080/notify?key=foo'.
|
||||
2. `event_type`: Defines the events that trigger notifications. Supported event types include:
|
||||
1. 'WINDOW_OPEN': Window open event; triggered when any type of window opens.
|
||||
2. 'WINDOW_CLOSE': Window close event; triggered when any type of window closes.
|
||||
3. `NOTIFY_HISTORY`: Controls whether to trigger notifications during the computation of historical data. The default value is 0, which means no notifications are sent.
|
||||
4. `ON_FAILURE`: Determines whether to allow dropping some events if sending notifications fails (e.g., in poor network conditions). The default value is `PAUSE`:
|
||||
1. PAUSE means that the stream computing task is paused if sending a notification fails. taosd will retry until the notification is successfully delivered and the task resumes.
|
||||
2. DROP means that if sending a notification fails, the event information is discarded, and the stream computing task continues running unaffected.
|
||||
|
||||
For example, the following creates a stream that computes the per-minute average current from electric meters and sends notifications to two target addresses when the window opens and closes. It does not send notifications for historical data and does not allow dropping notifications on failure:
|
||||
|
||||
```sql
|
||||
CREATE STREAM avg_current_stream FILL_HISTORY 1
|
||||
AS SELECT _wstart, _wend, AVG(current) FROM meters
|
||||
INTERVAL (1m)
|
||||
NOTIFY ('ws://localhost:8080/notify', 'wss://192.168.1.1:8080/notify?key=foo')
|
||||
ON ('WINDOW_OPEN', 'WINDOW_CLOSE');
|
||||
NOTIFY_HISTORY 0
|
||||
ON_FAILURE PAUSE;
|
||||
```
|
||||
|
||||
When the specified events are triggered, taosd will send a POST request to the given URL(s) with a JSON message body. A single request may contain events from several streams, and the event types may differ.
|
||||
|
||||
The details of the event information depend on the type of window:
|
||||
|
||||
1. Time Window: At the opening, the start time is sent; at the closing, the start time, end time, and computation result are sent.
|
||||
2. State Window: At the opening, the start time, previous window's state, and current window's state are sent; at closing, the start time, end time, computation result, current window state, and next window state are sent.
|
||||
3. Session Window: At the opening, the start time is sent; at the closing, the start time, end time, and computation result are sent.
|
||||
4. Event Window: At the opening, the start time along with the data values and corresponding condition index that triggered the window opening are sent; at the closing, the start time, end time, computation result, and the triggering data value and condition index for window closure are sent.
|
||||
5. Count Window: At the opening, the start time is sent; at the closing, the start time, end time, and computation result are sent.
|
||||
|
||||
An example structure for the notification message is shown below:
|
||||
|
||||
```json
|
||||
{
|
||||
"messageId": "unique-message-id-12345",
|
||||
"timestamp": 1733284887203,
|
||||
"streams": [
|
||||
{
|
||||
"streamName": "avg_current_stream",
|
||||
"events": [
|
||||
{
|
||||
"tableName": "t_a667a16127d3b5a18988e32f3e76cd30",
|
||||
"eventType": "WINDOW_OPEN",
|
||||
"eventTime": 1733284887097,
|
||||
"windowId": "window-id-67890",
|
||||
"windowType": "Time",
|
||||
"windowStart": 1733284800000
|
||||
},
|
||||
{
|
||||
"tableName": "t_a667a16127d3b5a18988e32f3e76cd30",
|
||||
"eventType": "WINDOW_CLOSE",
|
||||
"eventTime": 1733284887197,
|
||||
"windowId": "window-id-67890",
|
||||
"windowType": "Time",
|
||||
"windowStart": 1733284800000,
|
||||
"windowEnd": 1733284860000,
|
||||
"result": {
|
||||
"_wstart": 1733284800000,
|
||||
"avg(current)": 1.3
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"streamName": "max_voltage_stream",
|
||||
"events": [
|
||||
{
|
||||
"tableName": "t_96f62b752f36e9b16dc969fe45363748",
|
||||
"eventType": "WINDOW_OPEN",
|
||||
"eventTime": 1733284887231,
|
||||
"windowId": "window-id-13579",
|
||||
"windowType": "Event",
|
||||
"windowStart": 1733284800000,
|
||||
"triggerCondition": {
|
||||
"conditionIndex": 0,
|
||||
"fieldValue": {
|
||||
"c1": 10,
|
||||
"c2": 15
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"tableName": "t_96f62b752f36e9b16dc969fe45363748",
|
||||
"eventType": "WINDOW_CLOSE",
|
||||
"eventTime": 1733284887231,
|
||||
"windowId": "window-id-13579",
|
||||
"windowType": "Event",
|
||||
"windowStart": 1733284800000,
|
||||
"windowEnd": 1733284810000,
|
||||
"triggerCondition": {
|
||||
"conditionIndex": 1,
|
||||
"fieldValue": {
|
||||
"c1": 20
|
||||
"c2": 3
|
||||
}
|
||||
},
|
||||
"result": {
|
||||
"_wstart": 1733284800000,
|
||||
"max(voltage)": 220
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The following sections explain the fields in the notification message.
|
||||
|
||||
### Root-Level Field Descriptions
|
||||
|
||||
1. "messageId": A string that uniquely identifies the notification message. It ensures that the entire message can be tracked and de-duplicated.
|
||||
2. "timestamp": A long integer timestamp representing the time when the notification message was generated, accurate to the millisecond (i.e., the number of milliseconds since '00:00, Jan 1 1970 UTC').
|
||||
3. "streams": An array containing the event information for multiple stream tasks. (See the following sections for details.)
|
||||
|
||||
### "stream" Object Field Descriptions
|
||||
|
||||
1. "streamName": A string representing the name of the stream task, used to identify which stream the events belong to.
|
||||
2. "events": An array containing the list of event objects for the stream task. Each event object includes detailed information. (See the next sections for details.)
|
||||
|
||||
### "event" Object Field Descriptions
|
||||
|
||||
#### Common Fields
|
||||
|
||||
These fields are common to all event objects.
|
||||
1. "tableName": A string indicating the name of the target subtable.
|
||||
2. "eventType": A string representing the event type ("WINDOW_OPEN", "WINDOW_CLOSE", or "WINDOW_INVALIDATION").
|
||||
3. "eventTime": A long integer timestamp that indicates when the event was generated, accurate to the millisecond (i.e., the number of milliseconds since '00:00, Jan 1 1970 UTC').
|
||||
4. "windowId": A string representing the unique identifier for the window. This ID ensures that the open and close events for the same window can be correlated. In the case that taosd restarts due to a fault, some events may be sent repeatedly, but the windowId remains constant for the same window.
|
||||
5. "windowType": A string that indicates the window type ("Time", "State", "Session", "Event", or "Count").
|
||||
|
||||
#### Fields for Time Windows
|
||||
|
||||
These fields are present only when "windowType" is "Time".
|
||||
1. When "eventType" is "WINDOW_OPEN", the following field is included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window, matching the time precision of the result table.
|
||||
2. When "eventType" is "WINDOW_CLOSE", the following fields are included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window.
|
||||
1. "windowEnd": A long integer timestamp representing the end time of the window.
|
||||
1. "result": An object containing key-value pairs of the computed result columns and their corresponding values.
|
||||
|
||||
#### Fields for State Windows
|
||||
|
||||
These fields are present only when "windowType" is "State".
|
||||
1. When "eventType" is "WINDOW_OPEN", the following fields are included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window.
|
||||
1. "prevState": A value of the same type as the state column, representing the state of the previous window. If there is no previous window (i.e., this is the first window), it will be NULL.
|
||||
1. "curState": A value of the same type as the state column, representing the current window's state.
|
||||
2. When "eventType" is "WINDOW_CLOSE", the following fields are included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window.
|
||||
1. "windowEnd": A long integer timestamp representing the end time of the window.
|
||||
1. "curState": The current window's state.
|
||||
1. "nextState": The state for the next window.
|
||||
1. "result": An object containing key-value pairs of the computed result columns and their corresponding values.
|
||||
|
||||
#### Fields for Session Windows
|
||||
|
||||
These fields are present only when "windowType" is "Session".
|
||||
1. When "eventType" is "WINDOW_OPEN", the following field is included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window.
|
||||
2. When "eventType" is "WINDOW_CLOSE", the following fields are included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window.
|
||||
1. "windowEnd": A long integer timestamp representing the end time of the window.
|
||||
1. "result": An object containing key-value pairs of the computed result columns and their corresponding values.
|
||||
|
||||
#### Fields for Event Windows
|
||||
|
||||
These fields are present only when "windowType" is "Event".
|
||||
1. When "eventType" is "WINDOW_OPEN", the following fields are included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window.
|
||||
1. "triggerCondition": An object that provides information about the condition that triggered the window to open. It includes:
|
||||
1. "conditionIndex": An integer representing the index of the condition that triggered the window, starting from 0.
|
||||
1. "fieldValue": An object containing key-value pairs of the column names related to the condition and their respective values.
|
||||
2. When "eventType" is "WINDOW_CLOSE", the following fields are included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window.
|
||||
1. "windowEnd": A long integer timestamp representing the end time of the window.
|
||||
1. "triggerCondition": An object that provides information about the condition that triggered the window to close. It includes:
|
||||
1. "conditionIndex": An integer representing the index of the condition that triggered the closure, starting from 0.
|
||||
1. "fieldValue": An object containing key-value pairs of the related column names and their respective values.
|
||||
1. "result": An object containing key-value pairs of the computed result columns and their corresponding values.
|
||||
|
||||
#### Fields for Count Windows
|
||||
|
||||
These fields are present only when "windowType" is "Count".
|
||||
1. When "eventType" is "WINDOW_OPEN", the following field is included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window.
|
||||
2. When "eventType" is "WINDOW_CLOSE", the following fields are included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window.
|
||||
1. "windowEnd": A long integer timestamp representing the end time of the window.
|
||||
1. "result": An object containing key-value pairs of the computed result columns and their corresponding values.
|
||||
|
||||
#### Fields for Window Invalidation
|
||||
|
||||
Due to scenarios such as data disorder, updates, or deletions during stream computing, windows that have already been generated might be removed or their results need to be recalculated. In such cases, a notification with the eventType "WINDOW_INVALIDATION" is sent to inform which windows have been invalidated.
|
||||
For events with "eventType" as "WINDOW_INVALIDATION", the following fields are included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window.
|
||||
1. "windowEnd": A long integer timestamp representing the end time of the window.
|
||||
|
|
|
@ -138,7 +138,7 @@ The table below explains how the ODBC connector maps server data types to defaul
|
|||
| BIGINT | SQL_BIGINT | SQL_C_SBIGINT |
|
||||
| BIGINT UNSIGNED | SQL_BIGINT | SQL_C_UBIGINT |
|
||||
| FLOAT | SQL_REAL | SQL_C_FLOAT |
|
||||
| DOUBLE | SQL_REAL | SQL_C_DOUBLE |
|
||||
| DOUBLE | SQL_DOUBLE | SQL_C_DOUBLE |
|
||||
| BINARY | SQL_BINARY | SQL_C_BINARY |
|
||||
| SMALLINT | SQL_SMALLINT | SQL_C_SSHORT |
|
||||
| SMALLINT UNSIGNED | SQL_SMALLINT | SQL_C_USHORT |
|
||||
|
@ -146,33 +146,145 @@ The table below explains how the ODBC connector maps server data types to defaul
|
|||
| TINYINT UNSIGNED | SQL_TINYINT | SQL_C_UTINYINT |
|
||||
| BOOL | SQL_BIT | SQL_C_BIT |
|
||||
| NCHAR | SQL_VARCHAR | SQL_C_CHAR |
|
||||
| JSON | SQL_VARCHAR | SQL_C_CHAR |
|
||||
| VARCHAR | SQL_VARCHAR | SQL_C_CHAR |
|
||||
| JSON | SQL_WVARCHAR | SQL_C_WCHAR |
|
||||
| GEOMETRY | SQL_VARBINARY | SQL_C_BINARY |
|
||||
| VARBINARY | SQL_VARBINARY | SQL_C_BINARY |
|
||||
|
||||
## API Reference
|
||||
|
||||
This section summarizes the ODBC API by functionality. For a complete ODBC API reference, please visit the [ODBC Programmer's Reference page](http://msdn.microsoft.com/en-us/library/ms714177.aspx).
|
||||
### API List
|
||||
|
||||
### Data Source and Driver Management
|
||||
- **Currently exported ODBC functions are**:
|
||||
|
||||
| ODBC/Setup API | Linux | macOS | Windows | Note |
|
||||
| :----- | :---- | :---- | :---- | :---- |
|
||||
| ConfigDSN | ❌ | ❌ | ✅ | |
|
||||
| ConfigDriver | ❌ | ❌ | ✅ | |
|
||||
| ConfigTranslator | ❌ | ❌ | ❌ | |
|
||||
| SQLAllocHandle | ✅ | ✅ | ✅ | |
|
||||
| SQLBindCol | ✅ | ✅ | ✅ | Column-Wise Binding only |
|
||||
| SQLBindParameter | ✅ | ✅ | ✅ | Column-Wise Binding only |
|
||||
| SQLBrowseConnect | ❌ | ❌ | ❌ | |
|
||||
| SQLBulkOperations | ❌ | ❌ | ❌ | TDengine has no counterpart |
|
||||
| SQLCloseCursor | ✅ | ✅ | ✅ | |
|
||||
| SQLColAttribute | ✅ | ✅ | ✅ | |
|
||||
| SQLColumnPrivileges | ❌ | ❌ | ❌ | TDengine has no strict counterpart |
|
||||
| SQLColumns | ✅ | ✅ | ✅ | |
|
||||
| SQLCompleteAsync | ❌ | ❌ | ❌ | |
|
||||
| SQLConnect | ✅ | ✅ | ✅ | |
|
||||
| SQLCopyDesc | ❌ | ❌ | ❌ | |
|
||||
| SQLDescribeCol | ✅ | ✅ | ✅ | |
|
||||
| SQLDescribeParam | ✅ | ✅ | ✅ | |
|
||||
| SQLDisconnect | ✅ | ✅ | ✅ | |
|
||||
| SQLDriverConnect | ✅ | ✅ | ✅ | |
|
||||
| SQLEndTran | ✅ | ✅ | ✅ | TDengine is non-transactional, thus this is at most simulating |
|
||||
| SQLExecDirect | ✅ | ✅ | ✅ | |
|
||||
| SQLExecute | ✅ | ✅ | ✅ | |
|
||||
| SQLExtendedFetch | ❌ | ❌ | ❌ | |
|
||||
| SQLFetch | ✅ | ✅ | ✅ | |
|
||||
| SQLFetchScroll | ✅ | ✅ | ✅ | TDengine has no counterpart, just implement SQL_FETCH_NEXT |
|
||||
| SQLForeignKeys | ❌ | ❌ | ❌ | TDengine has no counterpart |
|
||||
| SQLFreeHandle | ✅ | ✅ | ✅ | |
|
||||
| SQLFreeStmt | ✅ | ✅ | ✅ | |
|
||||
| SQLGetConnectAttr | ✅ | ✅ | ✅ | Supports partial attributes; unsupported attributes are listed below. |
|
||||
| SQLGetCursorName | ❌ | ❌ | ❌ | TDengine has no counterpart |
|
||||
| SQLGetData | ✅ | ✅ | ✅ | |
|
||||
| SQLGetDescField | ❌ | ❌ | ❌ | |
|
||||
| SQLGetDescRec | ❌ | ❌ | ❌ | |
|
||||
| SQLGetDiagField | ✅ | ✅ | ✅ | |
|
||||
| SQLGetDiagRec | ✅ | ✅ | ✅ | |
|
||||
| SQLGetEnvAttr | ✅ | ✅ | ✅ | |
|
||||
| SQLGetInfo | ✅ | ✅ | ✅ | |
|
||||
| SQLGetStmtAttr | ✅ | ✅ | ✅ | Supports partial attributes; unsupported attributes are listed below. |
|
||||
| SQLGetTypeInfo | ✅ | ✅ | ✅ | |
|
||||
| SQLMoreResults | ✅ | ✅ | ✅ | |
|
||||
| SQLNativeSql | ❌ | ❌ | ❌ | TDengine has no counterpart |
|
||||
| SQLNumParams | ✅ | ✅ | ✅ | |
|
||||
| SQLNumResultCols | ✅ | ✅ | ✅ | |
|
||||
| SQLParamData | ❌ | ❌ | ❌ | TDengine has no counterpart |
|
||||
| SQLPrepare | ✅ | ✅ | ✅ | |
|
||||
| SQLPrimaryKeys | ✅ | ✅ | ✅ | |
|
||||
| SQLProcedureColumns | ❌ | ❌ | ❌ | TDengine has no counterpart |
|
||||
| SQLProcedures | ❌ | ❌ | ❌ | TDengine has no counterpart |
|
||||
| SQLPutData | ❌ | ❌ | ❌ | TDengine has no counterpart |
|
||||
| SQLRowCount | ✅ | ✅ | ✅ | |
|
||||
| SQLSetConnectAttr | ✅ | ✅ | ✅ | Supports partial attributes; unsupported attributes are listed below. |
|
||||
| SQLSetCursorName | ❌ | ❌ | ❌ | TDengine has no counterpart |
|
||||
| SQLSetDescField | ❌ | ❌ | ❌ | |
|
||||
| SQLSetDescRec | ❌ | ❌ | ❌ | |
|
||||
| SQLSetEnvAttr | ✅ | ✅ | ✅ | |
|
||||
| SQLSetPos | ❌ | ❌ | ❌ | TDengine has no counterpart |
|
||||
| SQLSetStmtAttr | ✅ | ✅ | ✅ | Supports partial attributes; unsupported attributes are listed below. |
|
||||
| SQLSpecialColumns | ❌ | ❌ | ❌ | TDengine has no counterpart |
|
||||
| SQLStatistics | ❌ | ❌ | ❌ | TDengine has no counterpart |
|
||||
| SQLTablePrivileges | ❌ | ❌ | ❌ | TDengine has no strict counterpart |
|
||||
| SQLTables | ✅ | ✅ | ✅ | |
|
||||
|
||||
- **Non-supported-statement-attributes (SQLSetStmtAttr)**
|
||||
|
||||
| Attribute | Note |
|
||||
| :----- | :---- |
|
||||
| SQL_ATTR_CONCURRENCY | TDengine has no updatable-CURSOR machanism |
|
||||
| SQL_ATTR_FETCH_BOOKMARK_PTR | TDengine has no BOOKMARK machanism |
|
||||
| SQL_ATTR_IMP_PARAM_DESC | |
|
||||
| SQL_ATTR_IMP_ROW_DESC | |
|
||||
| SQL_ATTR_KEYSET_SIZE | |
|
||||
| SQL_ATTR_PARAM_BIND_OFFSET_PTR | |
|
||||
| SQL_ATTR_PARAM_OPERATION_PTR | |
|
||||
| SQL_ATTR_ROW_NUMBER | Readonly attribute |
|
||||
| SQL_ATTR_ROW_OPERATION_PTR | |
|
||||
| SQL_ATTR_SIMULATE_CURSOR | |
|
||||
|
||||
- **Non-supported-connection-attributes (SQLSetConnectAttr)**
|
||||
|
||||
| Attribute | Note |
|
||||
| :----- | :---- |
|
||||
| SQL_ATTR_AUTO_IPD | Readonly attribute |
|
||||
| SQL_ATTR_CONNECTION_DEAD | Readonly attribute |
|
||||
| SQL_ATTR_ENLIST_IN_DTC | |
|
||||
| SQL_ATTR_PACKET_SIZE | |
|
||||
| SQL_ATTR_TRACE | |
|
||||
| SQL_ATTR_TRACEFILE | |
|
||||
| SQL_ATTR_TRANSLATE_LIB | |
|
||||
| SQL_ATTR_TRANSLATE_OPTION | |
|
||||
|
||||
- **Enable any programming language with ODBC-bindings/ODBC-plugings to communicate with TDengine:**
|
||||
|
||||
| programming language | ODBC-API or bindings/plugins |
|
||||
| :----- | :---- |
|
||||
| C/C++ | ODBC-API |
|
||||
| CSharp | System.Data.Odbc |
|
||||
| Erlang | odbc module |
|
||||
| Go | [odbc](https://github.com/alexbrainman/odbc), database/sql |
|
||||
| Haskell | HDBC, HDBC-odbc |
|
||||
| Common Lisp | plain-odbc |
|
||||
| Nodejs | odbc |
|
||||
| Python3 | pyodbc |
|
||||
| Rust | odbc |
|
||||
|
||||
### API Functional Categories
|
||||
|
||||
This section summarizes the ODBC API by functionality. For a complete ODBC API reference, please visit the [Microsoft Open Database Connectivity (ODBC)](https://learn.microsoft.com/en-us/sql/odbc/microsoft-open-database-connectivity-odbc).
|
||||
|
||||
#### Data Source and Driver Management
|
||||
|
||||
- API: ConfigDSN
|
||||
- **Supported**: Yes
|
||||
- **Supported**: Yes (Windows only)
|
||||
- **Standard**: ODBC
|
||||
- **Function**: Configures data sources
|
||||
|
||||
- API: ConfigDriver
|
||||
- **Supported**: Yes
|
||||
- **Supported**: Yes (Windows only)
|
||||
- **Standard**: ODBC
|
||||
- **Function**: Used to perform installation and configuration tasks related to a specific driver
|
||||
|
||||
- API: ConfigTranslator
|
||||
- **Supported**: Yes
|
||||
- **Supported**: No
|
||||
- **Standard**: ODBC
|
||||
- **Function**: Used to parse the DSN configuration, translating or converting between DSN configuration and actual database driver configuration
|
||||
|
||||
### Connecting to Data Sources
|
||||
#### Connecting to Data Sources
|
||||
|
||||
- API: SQLAllocHandle
|
||||
- **Supported**: Yes
|
||||
|
@ -204,7 +316,7 @@ This section summarizes the ODBC API by functionality. For a complete ODBC API r
|
|||
- **Standard**: Deprecated
|
||||
- **Function**: In ODBC 3.x, the ODBC 2.x function SQLAllocConnect has been replaced by SQLAllocHandle
|
||||
|
||||
### Retrieving Information about Drivers and Data Sources
|
||||
#### Retrieving Information about Drivers and Data Sources
|
||||
|
||||
- API: SQLDataSources
|
||||
- **Supported**: No
|
||||
|
@ -231,7 +343,7 @@ This section summarizes the ODBC API by functionality. For a complete ODBC API r
|
|||
- **Standard**: ISO 92
|
||||
- **Function**: Returns information about supported data types
|
||||
|
||||
### Setting and Retrieving Driver Properties
|
||||
#### Setting and Retrieving Driver Properties
|
||||
|
||||
- API: SQLSetConnectAttr
|
||||
- **Supported**: Yes
|
||||
|
@ -283,7 +395,7 @@ This section summarizes the ODBC API by functionality. For a complete ODBC API r
|
|||
- **Standard**: Deprecated
|
||||
- **Purpose**: In ODBC 3.x, the ODBC 2.0 function SQLSetStmtOption has been replaced by SQLGetStmtAttr
|
||||
|
||||
### Preparing SQL Requests
|
||||
#### Preparing SQL Requests
|
||||
|
||||
- API: SQLAllocStmt
|
||||
- **Supported**: Not supported
|
||||
|
@ -320,7 +432,7 @@ This section summarizes the ODBC API by functionality. For a complete ODBC API r
|
|||
- **Standard**: ODBC
|
||||
- **Purpose**: Sets options that control cursor behavior
|
||||
|
||||
### Submitting Requests
|
||||
#### Submitting Requests
|
||||
|
||||
- API: SQLExecute
|
||||
- **Supported**: Supported
|
||||
|
@ -357,7 +469,7 @@ This section summarizes the ODBC API by functionality. For a complete ODBC API r
|
|||
- **Standard**: ISO 92
|
||||
- **Function**: When using stream input mode, it can be used to send data blocks to output parameters
|
||||
|
||||
### Retrieving Results and Information About Results
|
||||
#### Retrieving Results and Information About Results
|
||||
|
||||
- API: SQLRowCount
|
||||
- **Support**: Supported
|
||||
|
@ -419,7 +531,7 @@ This section summarizes the ODBC API by functionality. For a complete ODBC API r
|
|||
- **Standard**: ODBC
|
||||
- **Function**: Performs bulk insert and bulk bookmark operations, including updates, deletions, and fetching by bookmark
|
||||
|
||||
### Retrieving Error or Diagnostic Information
|
||||
#### Retrieving Error or Diagnostic Information
|
||||
|
||||
- API: SQLError
|
||||
- **Support**: Not supported
|
||||
|
@ -436,7 +548,7 @@ This section summarizes the ODBC API by functionality. For a complete ODBC API r
|
|||
- **Standard**: ISO 92
|
||||
- **Function**: Returns additional diagnostic information (multiple diagnostic results)
|
||||
|
||||
### Retrieving Information About System Table Entries Related to the Data Source
|
||||
#### Retrieving Information About System Table Entries Related to the Data Source
|
||||
|
||||
- API: SQLColumnPrivileges
|
||||
- **Support**: Not supported
|
||||
|
@ -488,7 +600,7 @@ This section summarizes the ODBC API by functionality. For a complete ODBC API r
|
|||
- **Standard**: ODBC
|
||||
- **Function**: Returns column information for stored procedures, including details of input and output parameters
|
||||
|
||||
### Transaction Execution
|
||||
#### Transaction Execution
|
||||
|
||||
- API: SQLTransact
|
||||
- **Support**: Not supported
|
||||
|
@ -498,9 +610,9 @@ This section summarizes the ODBC API by functionality. For a complete ODBC API r
|
|||
- API: SQLEndTran
|
||||
- **Support**: Supported
|
||||
- **Standard**: ISO 92
|
||||
- **Function**: Used to commit or rollback transactions, TDengine does not support transactions, therefore rollback operation is not supported
|
||||
- **Function**: Used to commit or rollback transactions. TDengine is non-transactional, so this function can at most simulate commit or rollback operations. If there are any outstanding connections or statements, neither commit nor rollback will succeed
|
||||
|
||||
### Connection Termination
|
||||
#### Connection Termination
|
||||
|
||||
- API: SQLDisconnect
|
||||
- **Support**: Supported
|
||||
|
|
|
@ -3,6 +3,9 @@ title: 预测算法
|
|||
description: 预测算法
|
||||
---
|
||||
|
||||
import fc_result from '../pic/fc.png';
|
||||
import fc_result_figure from '../pic/fc-result.png';
|
||||
|
||||
时序数据预测处理以持续一个时间段的时序数据作为输入,预测接下来一个连续时间区间内时间序列数据趋势。用户可以指定输出的(预测)时间序列数据点的数量,因此其输出的结果行数不确定。为此,TDengine 使用新 SQL 函数 `FORECAST` 提供时序数据预测服务。基础数据(用于预测的历史时间序列数据)是该函数的输入,预测结果是该函数的输出。用户可以通过 `FORECAST` 函数调用 Anode 提供的预测算法提供的服务。
|
||||
|
||||
在后续章节中,使用时序数据表`foo`作为示例,介绍预测和异常检测算法的使用方式,`foo` 表的模式如下:
|
||||
|
@ -106,3 +109,37 @@ taos> select _flow, _fhigh, _frowts, forecast(i32) from foo;
|
|||
- PatchTST (Patch Time Series Transformer)
|
||||
- Temporal Fusion Transformer
|
||||
- TimesNet
|
||||
|
||||
## 算法有效性评估工具
|
||||
TDgpt 提供预测分析算法有效性评估工具 `analytics_compare`,调用该工具并设置合适的参数,能够使用 TDengine 中的数据作为回测依据,评估不同预测算法或相同的预测算法在不同的参数或训练模型的下的预测有效性。预测有效性的评估使用 `MSE` 和 `MAE` 指标作为依据,后续还将增加 `MAPE`指标。
|
||||
|
||||
```ini
|
||||
[forecast]
|
||||
# 训练数据的周期,每个周期包含多少个输入点
|
||||
period = 10
|
||||
|
||||
# 使用范围内最后 10 条记录作为预测结果
|
||||
rows = 10
|
||||
|
||||
# 训练数据开始时间
|
||||
start_time = 1949-01-01T00:00:00
|
||||
|
||||
# 训练数据结束时间
|
||||
end_time = 1960-12-01T00:00:00
|
||||
|
||||
# 输出结果的起始时间
|
||||
res_start_time = 1730000000000
|
||||
|
||||
# 是否绘制预测结果图
|
||||
gen_figure = true
|
||||
```
|
||||
|
||||
算法对比分析运行完成以后,生成 fc-results.xlsx 文件,其中包含了调用算法的预测分析误差、执行时间、调用参数等信息。如下图所示:
|
||||
|
||||
<img src={fc_result} width="760" alt="预测对比结果" />
|
||||
|
||||
|
||||
如果设置了 `gen_figure` 为 true,分析结果中还会有绘制的分析预测结果图(如下图所示)。
|
||||
|
||||
<img src={fc_result_figure} width="760" alt="预测对比结果" />
|
||||
|
||||
|
|
|
@ -4,6 +4,8 @@ description: 异常检测算法
|
|||
---
|
||||
|
||||
import ad from '../pic/anomaly-detection.png';
|
||||
import ad_result from '../pic/ad-result.png';
|
||||
import ad_result_figure from '../pic/ad-result-figure.png';
|
||||
|
||||
TDengine 中定义了异常(状态)窗口来提供异常检测服务。异常窗口可以视为一种特殊的**事件窗口(Event Window)**,即异常检测算法确定的连续异常时间序列数据所在的时间窗口。与普通事件窗口区别在于——时间窗口的起始时间和结束时间均是分析算法识别确定,不是用户给定的表达式进行判定。因此,在 `WHERE` 子句中使用 `ANOMALY_WINDOW` 关键词即可调用时序数据异常检测服务,同时窗口伪列(`_WSTART`, `_WEND`, `_WDURATION`)也能够像其他时间窗口一样用于描述异常窗口的起始时间(`_WSTART`)、结束时间(`_WEND`)、持续时间(`_WDURATION`)。例如:
|
||||
|
||||
|
@ -67,3 +69,38 @@ Query OK, 1 row(s) in set (0.028946s)
|
|||
### 内置异常检测算法
|
||||
分析平台内置了6个异常检查模型,分为3个类别,分别是[基于统计学的算法](./02-statistics-approach.md)、[基于数据密度的算法](./03-data-density.md)、以及[基于机器学习的算法](./04-machine-learning.md)。在不指定异常检测使用的方法的情况下,默认调用 IQR 进行异常检测。
|
||||
|
||||
### 异常检测算法有效性比较工具
|
||||
TDgpt 提供自动化的工具对比不同数据集的不同算法监测有效性,针对异常检测算法提供查全率(recall)和查准率(precision)两个指标衡量不同算法的有效性。
|
||||
通过在配置文件中(analysis.ini)设置以下的选项可以调用需要使用的异常检测算法,异常检测算法测试用数据的时间范围、是否生成标注结果的图片、调用的异常检测算法以及相应的参数。
|
||||
调用异常检测算法比较之前,需要人工手动标注异常监测数据集的结果,即设置[anno_res]选项的数值,第几个数值是异常点,需要标注在数组中,如下测试集中,第 9 个点是异常点,我们就标注异常结果为 [9].
|
||||
|
||||
```bash
|
||||
[ad]
|
||||
# training data start time
|
||||
start_time = 2021-01-01T01:01:01
|
||||
|
||||
# training data end time
|
||||
end_time = 2021-01-01T01:01:11
|
||||
|
||||
# draw the results or not
|
||||
gen_figure = true
|
||||
|
||||
# annotate the anomaly_detection result
|
||||
anno_res = [9]
|
||||
|
||||
# algorithms list that is involved in the comparion
|
||||
[ad.algos]
|
||||
ksigma={"k": 2}
|
||||
iqr={}
|
||||
grubbs={}
|
||||
lof={"algo":"auto", "n_neighbor": 3}
|
||||
```
|
||||
|
||||
对比程序执行完成以后,会自动生成名称为`ad_result.xlsx` 的文件,第一个卡片是算法运行结果(如下图所示),分别包含了算法名称、执行调用参数、查全率、查准率、执行时间 5 个指标。
|
||||
|
||||
<img src={ad_result} width="760" alt="异常检测对比结果" />
|
||||
|
||||
如果设置了 `gen_figure` 为 `true`,比较程序会自动将每个参与比较的算法分析结果采用图片方式呈现出来(如下图所示为 ksigma 的异常检测结果标注)。
|
||||
|
||||
<img src={ad_result_figure} width="760" alt="异常检测标注图" />
|
||||
|
||||
|
|
Binary file not shown.
After Width: | Height: | Size: 41 KiB |
Binary file not shown.
After Width: | Height: | Size: 20 KiB |
Binary file not shown.
After Width: | Height: | Size: 61 KiB |
Binary file not shown.
After Width: | Height: | Size: 13 KiB |
|
@ -8,7 +8,7 @@ description: 流式计算的相关 SQL 的详细语法
|
|||
## 创建流式计算
|
||||
|
||||
```sql
|
||||
CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name[(field1_name, field2_name [PRIMARY KEY], ...)] [TAGS (create_definition [, create_definition] ...)] SUBTABLE(expression) AS subquery
|
||||
CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name[(field1_name, field2_name [PRIMARY KEY], ...)] [TAGS (create_definition [, create_definition] ...)] SUBTABLE(expression) AS subquery [notification_definition]
|
||||
stream_options: {
|
||||
TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time | FORCE_WINDOW_CLOSE]
|
||||
WATERMARK time
|
||||
|
@ -83,6 +83,8 @@ CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS
|
|||
SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10);
|
||||
```
|
||||
|
||||
notification_definition 子句定义了窗口计算过程中,在窗口打开/关闭等指定事件发生时,需要向哪些地址发送通知。详见 [流式计算的事件通知](#流失计算的事件通知)
|
||||
|
||||
## 流式计算的 partition
|
||||
|
||||
可以使用 PARTITION BY TBNAME,tag,普通列或者表达式,对一个流进行多分区的计算,每个分区的时间线与时间窗口是独立的,会各自聚合,并写入到目的表中的不同子表。
|
||||
|
@ -297,3 +299,222 @@ CREATE SNODE ON DNODE [id]
|
|||
```
|
||||
其中的 id 是集群中的 dnode 的序号。请注意选择的dnode,流计算的中间状态将自动在其上进行备份。
|
||||
从 3.3.4.0 版本开始,在多副本环境中创建流会进行 snode 的**存在性检查**,要求首先创建 snode。如果 snode 不存在,无法创建流。
|
||||
|
||||
## 流式计算的事件通知
|
||||
|
||||
### 使用说明
|
||||
|
||||
流式计算支持在窗口打开/关闭时,向外部系统发送相关的事件通知。用户通过 notification_definition 来指定需要通知的事件,以及用于接收通知消息的目标地址。
|
||||
|
||||
```sql
|
||||
notification_definition:
|
||||
NOTIFY (url [, url] ...) ON (event_type [, event_type] ...) [notification_options]
|
||||
|
||||
event_type:
|
||||
'WINDOW_OPEN'
|
||||
| 'WINDOW_CLOSE'
|
||||
|
||||
notification_options: {
|
||||
NOTIFY_HISTORY [0|1]
|
||||
ON_FAILURE [DROP|PAUSE]
|
||||
}
|
||||
```
|
||||
|
||||
上述语法中的相关规则含义如下:
|
||||
1. `url`: 指定通知的目标地址,必须包括协议、IP 或域名、端口号,并允许包含路径、参数。目前仅支持 websocket 协议。例如:'ws://localhost:8080','ws://localhost:8080/notify','wss://localhost:8080/notify?key=foo'。
|
||||
1. `event_type`: 定义需要通知的事件,支持的事件类型有:
|
||||
1. 'WINDOW_OPEN':窗口打开事件,所有类型的窗口打开时都会触发
|
||||
1. 'WINDOW_CLOSE':窗口关闭事件,所有类型的窗口关闭时都会触发
|
||||
1. `NOTIFY_HISTORY`: 控制是否在计算历史数据时触发通知,默认值为0,即不触发
|
||||
1. `ON_FAILURE`: 向通知地址发送通知失败时(比如网络不佳场景)是否允许丢弃部分事件,默认值为 `PAUSE`
|
||||
1. PAUSE 表示发送通知失败时暂停流计算任务。taosd 会重试发送通知,直到发送成功后,任务自动恢复运行。
|
||||
1. DROP 表示发送通知失败时直接丢弃事件信息,流计算任务继续运行,不受影响
|
||||
|
||||
比如,以下示例创建一个流,计算电表电流的每分钟平均值,并在窗口打开、关闭时向两个通知地址发送通知,计算历史数据时不发送通知,不允许在通知发送失败时丢弃通知:
|
||||
|
||||
```sql
|
||||
CREATE STREAM avg_current_stream FILL_HISTORY 1
|
||||
AS SELECT _wstart, _wend, AVG(current) FROM meters
|
||||
INTERVAL (1m)
|
||||
NOTIFY ('ws://localhost:8080/notify', 'wss://192.168.1.1:8080/notify?key=foo')
|
||||
ON ('WINDOW_OPEN', 'WINDOW_CLOSE');
|
||||
NOTIFY_HISTORY 0
|
||||
ON_FAILURE PAUSE;
|
||||
```
|
||||
|
||||
当触发指定的事件时,taosd 会向指定的 URL 发送 POST 请求,消息体为 JSON 格式。一个请求可能包含若干个流的若干个事件,且事件类型不一定相同。
|
||||
事件信息视窗口类型而定:
|
||||
|
||||
1. 时间窗口:开始时发送起始时间;结束时发送起始时间、结束时间、计算结果。
|
||||
1. 状态窗口:开始时发送起始时间、前一个窗口的状态值、当前窗口的状态值;结束时发送起始时间、结束时间、计算结果、当前窗口的状态值、下一个窗口的状态值。
|
||||
1. 会话窗口:开始时发送起始时间;结束时发送起始时间、结束时间、计算结果。
|
||||
1. 事件窗口:开始时发送起始时间,触发窗口打开的数据值和对应条件编号;结束时发送起始时间、结束时间、计算结果、触发窗口关闭的数据值和对应条件编号。
|
||||
1. 计数窗口:开始时发送起始时间;结束时发送起始时间、结束时间、计算结果。
|
||||
|
||||
通知消息的结构示例如下:
|
||||
|
||||
```json
|
||||
{
|
||||
"messageId": "unique-message-id-12345",
|
||||
"timestamp": 1733284887203,
|
||||
"streams": [
|
||||
{
|
||||
"streamName": "avg_current_stream",
|
||||
"events": [
|
||||
{
|
||||
"tableName": "t_a667a16127d3b5a18988e32f3e76cd30",
|
||||
"eventType": "WINDOW_OPEN",
|
||||
"eventTime": 1733284887097,
|
||||
"windowId": "window-id-67890",
|
||||
"windowType": "Time",
|
||||
"windowStart": 1733284800000
|
||||
},
|
||||
{
|
||||
"tableName": "t_a667a16127d3b5a18988e32f3e76cd30",
|
||||
"eventType": "WINDOW_CLOSE",
|
||||
"eventTime": 1733284887197,
|
||||
"windowId": "window-id-67890",
|
||||
"windowType": "Time",
|
||||
"windowStart": 1733284800000,
|
||||
"windowEnd": 1733284860000,
|
||||
"result": {
|
||||
"_wstart": 1733284800000,
|
||||
"avg(current)": 1.3
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"streamName": "max_voltage_stream",
|
||||
"events": [
|
||||
{
|
||||
"tableName": "t_96f62b752f36e9b16dc969fe45363748",
|
||||
"eventType": "WINDOW_OPEN",
|
||||
"eventTime": 1733284887231,
|
||||
"windowId": "window-id-13579",
|
||||
"windowType": "Event",
|
||||
"windowStart": 1733284800000,
|
||||
"triggerCondition": {
|
||||
"conditionIndex": 0,
|
||||
"fieldValue": {
|
||||
"c1": 10,
|
||||
"c2": 15
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"tableName": "t_96f62b752f36e9b16dc969fe45363748",
|
||||
"eventType": "WINDOW_CLOSE",
|
||||
"eventTime": 1733284887231,
|
||||
"windowId": "window-id-13579",
|
||||
"windowType": "Event",
|
||||
"windowStart": 1733284800000,
|
||||
"windowEnd": 1733284810000,
|
||||
"triggerCondition": {
|
||||
"conditionIndex": 1,
|
||||
"fieldValue": {
|
||||
"c1": 20
|
||||
"c2": 3
|
||||
}
|
||||
},
|
||||
"result": {
|
||||
"_wstart": 1733284800000,
|
||||
"max(voltage)": 220
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
后续小节是通知消息中各个字段的说明。
|
||||
|
||||
### 根级字段说明
|
||||
|
||||
1. "messageId": 字符串类型,是通知消息的唯一标识符,确保整条消息可以被追踪和去重。
|
||||
1. "timestamp": 长整型时间戳,表示通知消息生成的时间,精确到毫秒,即: '00:00, Jan 1 1970 UTC' 以来的毫秒数。
|
||||
1. "streams": 对象数组,包含多个流任务的事件信息。(详细信息见下节)
|
||||
|
||||
### stream 对象的字段说明
|
||||
|
||||
1. "streamName": 字符串类型,流任务的名称,用于标识事件所属的流。
|
||||
1. "events": 对象数组,该流任务下的事件列表,包含一个或多个事件对象。(详细信息见下节)
|
||||
|
||||
### event 对象的字段说明
|
||||
|
||||
#### 通用字段
|
||||
|
||||
这部分是所有 event 对象所共有的字段。
|
||||
1. "tableName": 字符串类型,是对应目标子表的表名。
|
||||
1. "eventType": 字符串类型,表示事件类型 ("WINDOW_OPEN", "WINDOW_CLOSE" 或 "WINDOW_INVALIDATION")。
|
||||
1. "eventTime": 长整型时间戳,表示事件生成时间,精确到毫秒,即: '00:00, Jan 1 1970 UTC' 以来的毫秒数。
|
||||
1. "windowId": 字符串类型,窗口的唯一标识符,确保打开和关闭事件的 ID 一致,便于外部系统将两者关联。如果 taosd 发生故障重启,部分事件可能会重复发送,会保证同一窗口的 windowId 保持不变。
|
||||
1. "windowType": 字符串类型,表示窗口类型 ("Time", "State", "Session", "Event", "Count")。
|
||||
|
||||
#### 时间窗口相关字段
|
||||
|
||||
这部分是 "windowType" 为"Time" 时 event 对象才有的字段。
|
||||
1. 如果 "eventType" 为 "WINDOW_OPEN",则包含如下字段:
|
||||
1. "windowStart": 长整型时间戳,表示窗口的开始时间,精度与结果表的时间精度一致。
|
||||
1. 如果 "eventType" 为 "WINDOW_CLOSE",则包含如下字段:
|
||||
1. "windowStart": 长整型时间戳,表示窗口的开始时间,精度与结果表的时间精度一致。
|
||||
1. "windowEnd": 长整型时间戳,表示窗口的结束时间,精度与结果表的时间精度一致。
|
||||
1. "result": 计算结果,为键值对形式,包含窗口计算的结果列列名及其对应的值。
|
||||
|
||||
#### 状态窗口相关字段
|
||||
|
||||
这部分是 "windowType" 为"State" 时 event 对象才有的字段。
|
||||
1. 如果 "eventType" 为 "WINDOW_OPEN",则包含如下字段:
|
||||
1. "windowStart": 长整型时间戳,表示窗口的开始时间,精度与结果表的时间精度一致。
|
||||
1. "prevState": 与状态列的类型相同,表示上一个窗口的状态值。如果没有上一个窗口(即: 现在是第一个窗口),则为 NULL。
|
||||
1. "curState": 与状态列的类型相同,表示当前窗口的状态值。
|
||||
1. 如果 "eventType" 为 "WINDOW_CLOSE",则包含如下字段:
|
||||
1. "windowStart": 长整型时间戳,表示窗口的开始时间,精度与结果表的时间精度一致。
|
||||
1. "windowEnd": 长整型时间戳,表示窗口的结束时间,精度与结果表的时间精度一致。
|
||||
1. "curState": 与状态列的类型相同,表示当前窗口的状态值。
|
||||
1. "nextState": 与状态列的类型相同,表示下一个窗口的状态值。
|
||||
1. "result": 计算结果,为键值对形式,包含窗口计算的结果列列名及其对应的值。
|
||||
|
||||
#### 会话窗口相关字段
|
||||
|
||||
这部分是 "windowType" 为"Session" 时 event 对象才有的字段。
|
||||
1. 如果 "eventType" 为 "WINDOW_OPEN",则包含如下字段:
|
||||
1. "windowStart": 长整型时间戳,表示窗口的开始时间,精度与结果表的时间精度一致。
|
||||
1. 如果 "eventType" 为 "WINDOW_CLOSE",则包含如下字段:
|
||||
1. "windowStart": 长整型时间戳,表示窗口的开始时间,精度与结果表的时间精度一致。
|
||||
1. "windowEnd": 长整型时间戳,表示窗口的结束时间,精度与结果表的时间精度一致。
|
||||
1. "result": 计算结果,为键值对形式,包含窗口计算的结果列列名及其对应的值。
|
||||
|
||||
#### 事件窗口相关字段
|
||||
|
||||
这部分是 "windowType" 为"Event" 时 event 对象才有的字段。
|
||||
1. 如果 "eventType" 为 "WINDOW_OPEN",则包含如下字段:
|
||||
1. "windowStart": 长整型时间戳,表示窗口的开始时间,精度与结果表的时间精度一致。
|
||||
1. "triggerCondition": 触发窗口开始的条件信息,包括以下字段:
|
||||
1. "conditionIndex": 整型,表示满足的触发窗口开始的条件的索引,从0开始编号。
|
||||
1. "fieldValue": 键值对形式,包含条件列列名及其对应的值。
|
||||
1. 如果 "eventType" 为 "WINDOW_CLOSE",则包含如下字段:
|
||||
1. "windowStart": 长整型时间戳,表示窗口的开始时间,精度与结果表的时间精度一致。
|
||||
1. "windowEnd": 长整型时间戳,表示窗口的结束时间,精度与结果表的时间精度一致。
|
||||
1. "triggerCondition": 触发窗口关闭的条件信息,包括以下字段:
|
||||
1. "conditionIndex": 整型,表示满足的触发窗口关闭的条件的索引,从0开始编号。
|
||||
1. "fieldValue": 键值对形式,包含条件列列名及其对应的值。
|
||||
1. "result": 计算结果,为键值对形式,包含窗口计算的结果列列名及其对应的值。
|
||||
|
||||
#### 计数窗口相关字段
|
||||
|
||||
这部分是 "windowType" 为"Count" 时 event 对象才有的字段。
|
||||
1. 如果 "eventType" 为 "WINDOW_OPEN",则包含如下字段:
|
||||
1. "windowStart": 长整型时间戳,表示窗口的开始时间,精度与结果表的时间精度一致。
|
||||
1. 如果 "eventType" 为 "WINDOW_CLOSE",则包含如下字段:
|
||||
1. "windowStart": 长整型时间戳,表示窗口的开始时间,精度与结果表的时间精度一致。
|
||||
1. "windowEnd": 长整型时间戳,表示窗口的结束时间,精度与结果表的时间精度一致。
|
||||
1. "result": 计算结果,为键值对形式,包含窗口计算的结果列列名及其对应的值。
|
||||
|
||||
#### 窗口失效相关字段
|
||||
|
||||
因为流计算过程中会遇到数据乱序、更新、删除等情况,可能造成已生成的窗口被删除,或者结果需要重新计算。此时会向通知地址发送一条 "WINDOW_INVALIDATION" 的通知,说明哪些窗口已经被删除。
|
||||
这部分是 "eventType" 为 "WINDOW_INVALIDATION" 时,event 对象才有的字段。
|
||||
1. "windowStart": 长整型时间戳,表示窗口的开始时间,精度与结果表的时间精度一致。
|
||||
1. "windowEnd": 长整型时间戳,表示窗口的结束时间,精度与结果表的时间精度一致。
|
||||
|
|
|
@ -133,7 +133,7 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式:WebSocket 连接与
|
|||
| BIGINT | SQL_BIGINT | SQL_C_SBIGINT |
|
||||
| BIGINT UNSIGNED | SQL_BIGINT | SQL_C_UBIGINT |
|
||||
| FLOAT | SQL_REAL | SQL_C_FLOAT |
|
||||
| DOUBLE | SQL_REAL | SQL_C_DOUBLE |
|
||||
| DOUBLE | SQL_DOUBLE | SQL_C_DOUBLE |
|
||||
| BINARY | SQL_BINARY | SQL_C_BINARY |
|
||||
| SMALLINT | SQL_SMALLINT | SQL_C_SSHORT |
|
||||
| SMALLINT UNSIGNED | SQL_SMALLINT | SQL_C_USHORT |
|
||||
|
@ -141,35 +141,149 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式:WebSocket 连接与
|
|||
| TINYINT UNSIGNED | SQL_TINYINT | SQL_C_UTINYINT |
|
||||
| BOOL | SQL_BIT | SQL_C_BIT |
|
||||
| NCHAR | SQL_VARCHAR | SQL_C_CHAR |
|
||||
| JSON | SQL_VARCHAR | SQL_C_CHAR |
|
||||
| VARCHAR | SQL_VARCHAR | SQL_C_CHAR |
|
||||
| JSON | SQL_WVARCHAR | SQL_C_WCHAR |
|
||||
| GEOMETRY | SQL_VARBINARY | SQL_C_BINARY |
|
||||
| VARBINARY | SQL_VARBINARY | SQL_C_BINARY |
|
||||
|
||||
|
||||
## API 参考
|
||||
|
||||
本节按功能分类汇总了 ODBC API,关于完整的 ODBC API 参考,请访问 http://msdn.microsoft.com/en-us/library/ms714177.aspx 的 ODBC 程序员参考页面。
|
||||
### API 列表
|
||||
|
||||
### 数据源和驱动程序管理
|
||||
- **目前导出的ODBC函数有**:
|
||||
|
||||
| ODBC/Setup API | Linux | macOS | Windows | Note |
|
||||
| :----- | :---- | :---- | :---- | :---- |
|
||||
| ConfigDSN | ❌ | ❌ | ✅ | |
|
||||
| ConfigDriver | ❌ | ❌ | ✅ | |
|
||||
| ConfigTranslator | ❌ | ❌ | ❌ | |
|
||||
| SQLAllocHandle | ✅ | ✅ | ✅ | |
|
||||
| SQLBindCol | ✅ | ✅ | ✅ | 只能按列绑定 |
|
||||
| SQLBindParameter | ✅ | ✅ | ✅ | 只能按列绑定 |
|
||||
| SQLBrowseConnect | ❌ | ❌ | ❌ | |
|
||||
| SQLBulkOperations | ❌ | ❌ | ❌ | TDengine 没有对应的 |
|
||||
| SQLCloseCursor | ✅ | ✅ | ✅ | |
|
||||
| SQLColAttribute | ✅ | ✅ | ✅ | |
|
||||
| SQLColumnPrivileges | ❌ | ❌ | ❌ | TDengine 没有严格对应的 |
|
||||
| SQLColumns | ✅ | ✅ | ✅ | |
|
||||
| SQLCompleteAsync | ❌ | ❌ | ❌ | |
|
||||
| SQLConnect | ✅ | ✅ | ✅ | |
|
||||
| SQLCopyDesc | ❌ | ❌ | ❌ | |
|
||||
| SQLDescribeCol | ✅ | ✅ | ✅ | |
|
||||
| SQLDescribeParam | ✅ | ✅ | ✅ | |
|
||||
| SQLDisconnect | ✅ | ✅ | ✅ | |
|
||||
| SQLDriverConnect | ✅ | ✅ | ✅ | |
|
||||
| SQLEndTran | ✅ | ✅ | ✅ | TDengine是非事务性的,因此这最多是模拟 |
|
||||
| SQLExecDirect | ✅ | ✅ | ✅ | |
|
||||
| SQLExecute | ✅ | ✅ | ✅ | |
|
||||
| SQLExtendedFetch | ❌ | ❌ | ❌ | |
|
||||
| SQLFetch | ✅ | ✅ | ✅ | |
|
||||
| SQLFetchScroll | ✅ | ✅ | ✅ | TDengine 没有对应的, 仅仅实现 SQL_FETCH_NEXT |
|
||||
| SQLForeignKeys | ❌ | ❌ | ❌ | TDengine 没有对应的 |
|
||||
| SQLFreeHandle | ✅ | ✅ | ✅ | |
|
||||
| SQLFreeStmt | ✅ | ✅ | ✅ | |
|
||||
| SQLGetConnectAttr | ✅ | ✅ | ✅ | 支持部分属性,下面列出了不支持的属性 |
|
||||
| SQLGetCursorName | ❌ | ❌ | ❌ | TDengine 没有对应的 |
|
||||
| SQLGetData | ✅ | ✅ | ✅ | |
|
||||
| SQLGetDescField | ❌ | ❌ | ❌ | |
|
||||
| SQLGetDescRec | ❌ | ❌ | ❌ | |
|
||||
| SQLGetDiagField | ✅ | ✅ | ✅ | |
|
||||
| SQLGetDiagRec | ✅ | ✅ | ✅ | |
|
||||
| SQLGetEnvAttr | ✅ | ✅ | ✅ | |
|
||||
| SQLGetInfo | ✅ | ✅ | ✅ | |
|
||||
| SQLGetStmtAttr | ✅ | ✅ | ✅ | 支持部分属性,下面列出了不支持的属性 |
|
||||
| SQLGetTypeInfo | ✅ | ✅ | ✅ | |
|
||||
| SQLMoreResults | ✅ | ✅ | ✅ | |
|
||||
| SQLNativeSql | ❌ | ❌ | ❌ | TDengine 没有对应的 |
|
||||
| SQLNumParams | ✅ | ✅ | ✅ | |
|
||||
| SQLNumResultCols | ✅ | ✅ | ✅ | |
|
||||
| SQLParamData | ❌ | ❌ | ❌ | TDengine 没有对应的 |
|
||||
| SQLPrepare | ✅ | ✅ | ✅ | |
|
||||
| SQLPrimaryKeys | ✅ | ✅ | ✅ | |
|
||||
| SQLProcedureColumns | ❌ | ❌ | ❌ | TDengine 没有对应的 |
|
||||
| SQLProcedures | ❌ | ❌ | ❌ | TDengine 没有对应的 |
|
||||
| SQLPutData | ❌ | ❌ | ❌ | TDengine 没有对应的 |
|
||||
| SQLRowCount | ✅ | ✅ | ✅ | |
|
||||
| SQLSetConnectAttr | ✅ | ✅ | ✅ | 支持部分属性,下面列出了不支持的属性 |
|
||||
| SQLSetCursorName | ❌ | ❌ | ❌ | TDengine 没有对应的 |
|
||||
| SQLSetDescField | ❌ | ❌ | ❌ | |
|
||||
| SQLSetDescRec | ❌ | ❌ | ❌ | |
|
||||
| SQLSetEnvAttr | ✅ | ✅ | ✅ | |
|
||||
| SQLSetPos | ❌ | ❌ | ❌ | TDengine 没有对应的 |
|
||||
| SQLSetStmtAttr | ✅ | ✅ | ✅ | 支持部分属性,下面列出了不支持的属性 |
|
||||
| SQLSpecialColumns | ❌ | ❌ | ❌ | TDengine 没有对应的 |
|
||||
| SQLStatistics | ❌ | ❌ | ❌ | TDengine 没有对应的 |
|
||||
| SQLTablePrivileges | ❌ | ❌ | ❌ | TDengine 没有严格对应的 |
|
||||
| SQLTables | ✅ | ✅ | ✅ | |
|
||||
|
||||
- **不支持的语句属性 (SQLSetStmtAttr)**
|
||||
|
||||
| Attribute | Note |
|
||||
| :----- | :---- |
|
||||
| SQL_ATTR_CONCURRENCY | TDengine 没有 updatable-CURSOR 机制 |
|
||||
| SQL_ATTR_FETCH_BOOKMARK_PTR | TDengine 没有 BOOKMARK 机制 |
|
||||
| SQL_ATTR_IMP_PARAM_DESC | |
|
||||
| SQL_ATTR_IMP_ROW_DESC | |
|
||||
| SQL_ATTR_KEYSET_SIZE | |
|
||||
| SQL_ATTR_PARAM_BIND_OFFSET_PTR | |
|
||||
| SQL_ATTR_PARAM_OPERATION_PTR | |
|
||||
| SQL_ATTR_ROW_NUMBER | 只读属性 |
|
||||
| SQL_ATTR_ROW_OPERATION_PTR | |
|
||||
| SQL_ATTR_SIMULATE_CURSOR | |
|
||||
|
||||
- **不支持的连接属性 (SQLSetConnectAttr)**
|
||||
|
||||
| Attribute | Note |
|
||||
| :----- | :---- |
|
||||
| SQL_ATTR_AUTO_IPD | 只读属性 |
|
||||
| SQL_ATTR_CONNECTION_DEAD | 只读属性 |
|
||||
| SQL_ATTR_ENLIST_IN_DTC | |
|
||||
| SQL_ATTR_PACKET_SIZE | |
|
||||
| SQL_ATTR_TRACE | |
|
||||
| SQL_ATTR_TRACEFILE | |
|
||||
| SQL_ATTR_TRANSLATE_LIB | |
|
||||
| SQL_ATTR_TRANSLATE_OPTION | |
|
||||
|
||||
- **允许任何带有 ODBC 库的编程语言与 TDengine 通信:**
|
||||
|
||||
| programming language | ODBC-API or bindings/plugins |
|
||||
| :----- | :---- |
|
||||
| C/C++ | ODBC-API |
|
||||
| CSharp | System.Data.Odbc |
|
||||
| Erlang | odbc module |
|
||||
| Go | [odbc](https://github.com/alexbrainman/odbc), database/sql |
|
||||
| Haskell | HDBC, HDBC-odbc |
|
||||
| Common Lisp | plain-odbc |
|
||||
| Nodejs | odbc |
|
||||
| Python3 | pyodbc |
|
||||
| Rust | odbc |
|
||||
|
||||
|
||||
|
||||
### API 功能分类
|
||||
|
||||
本节按功能分类汇总了 ODBC API,关于完整的 ODBC API 参考,请访问 [Microsoft Open Database Connectivity (ODBC)](https://learn.microsoft.com/en-us/sql/odbc/microsoft-open-database-connectivity-odbc)。
|
||||
|
||||
#### 数据源和驱动程序管理
|
||||
|
||||
- API: ConfigDSN
|
||||
- **是否支持**: 支持
|
||||
- **是否支持**: 支持(仅 Windows)
|
||||
- **标准**: ODBC
|
||||
- **作用**: 配置数据源
|
||||
|
||||
- API: ConfigDriver
|
||||
- **是否支持**: 支持
|
||||
- **是否支持**: 支持(仅 Windows)
|
||||
- **标准**: ODBC
|
||||
- **作用**: 用于执行与特定驱动程序相关的安装和配置任务
|
||||
|
||||
- API: ConfigTranslator
|
||||
- **是否支持**: 支持
|
||||
- **是否支持**: 不支持
|
||||
- **标准**: ODBC
|
||||
- **作用**: 用于解析DSN的配置,在DSN配置和实际数据库驱动程序配置之间进行翻译或转换
|
||||
|
||||
|
||||
### 连接到数据源
|
||||
#### 连接到数据源
|
||||
|
||||
- API: SQLAllocHandle
|
||||
- **是否支持**: 支持
|
||||
|
@ -202,7 +316,7 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式:WebSocket 连接与
|
|||
- **作用**: 在 ODBC 3.x 中,ODBC 2.x 函数 SQLAllocConnect 已替换为 SQLAllocHandle
|
||||
|
||||
|
||||
### 获取有关驱动程序和数据源的信息
|
||||
#### 获取有关驱动程序和数据源的信息
|
||||
|
||||
- API: SQLDataSources
|
||||
- **是否支持**: 不支持
|
||||
|
@ -230,7 +344,7 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式:WebSocket 连接与
|
|||
- **作用**: 返回有关支持的数据类型的信息
|
||||
|
||||
|
||||
### 设置和检索驱动程序属性
|
||||
#### 设置和检索驱动程序属性
|
||||
|
||||
- API: SQLSetConnectAttr
|
||||
- **是否支持**: 支持
|
||||
|
@ -283,7 +397,7 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式:WebSocket 连接与
|
|||
- **作用**: 在 ODBC 3.x 中,ODBC 2.0 函数 SQLSetStmtOption 已替换为 SQLGetStmtAttr
|
||||
|
||||
|
||||
### 准备SQL请求
|
||||
#### 准备SQL请求
|
||||
|
||||
- API: SQLAllocStmt
|
||||
- **是否支持**: 不支持
|
||||
|
@ -321,7 +435,7 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式:WebSocket 连接与
|
|||
- **作用**: 设置控制光标行为的选项
|
||||
|
||||
|
||||
### 提交请求
|
||||
#### 提交请求
|
||||
|
||||
- API: SQLExecute
|
||||
- **是否支持**: 支持
|
||||
|
@ -359,7 +473,7 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式:WebSocket 连接与
|
|||
- **作用**: 当使用流输入方式时,可以用于向输出参数发送数据块
|
||||
|
||||
|
||||
### 检索结果和关于结果的信息
|
||||
#### 检索结果和关于结果的信息
|
||||
|
||||
- API: SQLRowCount
|
||||
- **是否支持**: 支持
|
||||
|
@ -422,7 +536,7 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式:WebSocket 连接与
|
|||
- **作用**: 执行批量插入和批量书签操作,包括更新、删除和按书签提取
|
||||
|
||||
|
||||
### 检索错误或诊断信息
|
||||
#### 检索错误或诊断信息
|
||||
|
||||
- API: SQLError
|
||||
- **是否支持**: 不支持
|
||||
|
@ -440,7 +554,7 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式:WebSocket 连接与
|
|||
- **作用**: 返回附加诊断信息(多条诊断结果)
|
||||
|
||||
|
||||
### 获取有关数据源的系统表项的信息
|
||||
#### 获取有关数据源的系统表项的信息
|
||||
|
||||
- API: SQLColumnPrivileges
|
||||
- **是否支持**: 不支持
|
||||
|
@ -493,7 +607,7 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式:WebSocket 连接与
|
|||
- **作用**: 返回存储过程的列信息,包括输入输出参数的详细信息
|
||||
|
||||
|
||||
### 执行事务
|
||||
#### 执行事务
|
||||
|
||||
- API: SQLTransact
|
||||
- **是否支持**: 不支持
|
||||
|
@ -503,10 +617,10 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式:WebSocket 连接与
|
|||
- API: SQLEndTran
|
||||
- **是否支持**: 支持
|
||||
- **标准**: ISO 92
|
||||
- **作用**: 用于提交或回滚事务,TDengine 不支持事务,因此不支持回滚操作
|
||||
- **作用**: 用于提交或回滚事务。TDengine 是非事务性的,因此 SQLEndTran 函数最多只能模拟提交或回滚操作。如果有任何未完成的连接或语句,无论是提交(commit)还是回滚(rollback)都不会成功
|
||||
|
||||
|
||||
### 终止连接
|
||||
#### 终止连接
|
||||
|
||||
- API: SQLDisconnect
|
||||
- **是否支持**: 支持
|
||||
|
|
|
@ -195,6 +195,8 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const
|
|||
// For the VAR_DATA_TYPE type, if a row already has data before inserting it (judged by offset != -1),
|
||||
// it will be inserted at the original position and the old data will be overwritten.
|
||||
int32_t colDataSetValOrCover(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull);
|
||||
int32_t varColSetVarData(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pVarData, int32_t varDataLen,
|
||||
bool isNull);
|
||||
int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData);
|
||||
int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows,
|
||||
bool trimValue);
|
||||
|
|
|
@ -2970,6 +2970,22 @@ typedef struct {
|
|||
int8_t notifyHistory;
|
||||
} SCMCreateStreamReq;
|
||||
|
||||
typedef struct STaskNotifyEventStat {
|
||||
int64_t notifyEventAddTimes; // call times of add function
|
||||
int64_t notifyEventAddElems; // elements added by add function
|
||||
double notifyEventAddCostSec; // time cost of add function
|
||||
int64_t notifyEventPushTimes; // call times of push function
|
||||
int64_t notifyEventPushElems; // elements pushed by push function
|
||||
double notifyEventPushCostSec; // time cost of push function
|
||||
int64_t notifyEventPackTimes; // call times of pack function
|
||||
int64_t notifyEventPackElems; // elements packed by pack function
|
||||
double notifyEventPackCostSec; // time cost of pack function
|
||||
int64_t notifyEventSendTimes; // call times of send function
|
||||
int64_t notifyEventSendElems; // elements sent by send function
|
||||
double notifyEventSendCostSec; // time cost of send function
|
||||
int64_t notifyEventHoldElems; // elements hold due to watermark
|
||||
} STaskNotifyEventStat;
|
||||
|
||||
typedef struct {
|
||||
int64_t streamId;
|
||||
} SCMCreateStreamRsp;
|
||||
|
|
|
@ -99,7 +99,7 @@ int32_t qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId);
|
|||
int32_t qSetStreamOpOpen(qTaskInfo_t tinfo);
|
||||
|
||||
int32_t qSetStreamNotifyInfo(qTaskInfo_t tinfo, int32_t eventTypes, const SSchemaWrapper* pSchemaWrapper,
|
||||
const char* stbFullName, bool newSubTableRule);
|
||||
const char* stbFullName, bool newSubTableRule, STaskNotifyEventStat* pNotifyEventStat);
|
||||
|
||||
/**
|
||||
* Set multiple input data blocks for the stream scan.
|
||||
|
|
|
@ -431,6 +431,7 @@ typedef struct SStateStore {
|
|||
int32_t (*updateInfoSerialize)(SEncoder* pEncoder, const SUpdateInfo* pInfo);
|
||||
int32_t (*updateInfoDeserialize)(SDecoder* pDeCoder, SUpdateInfo* pInfo);
|
||||
|
||||
SStreamStateCur* (*streamStateSessionSeekKeyPrev)(SStreamState* pState, const SSessionKey* key);
|
||||
SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key);
|
||||
SStreamStateCur* (*streamStateCountSeekKeyPrev)(SStreamState* pState, const SSessionKey* pKey, COUNT_TYPE count);
|
||||
SStreamStateCur* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key);
|
||||
|
|
|
@ -573,6 +573,7 @@ typedef enum EStreamNotifyOptionSetFlag {
|
|||
} EStreamNotifyOptionSetFlag;
|
||||
|
||||
typedef enum EStreamNotifyEventType {
|
||||
SNOTIFY_EVENT_WINDOW_INVALIDATION = 0,
|
||||
SNOTIFY_EVENT_WINDOW_OPEN = BIT_FLAG_MASK(0),
|
||||
SNOTIFY_EVENT_WINDOW_CLOSE = BIT_FLAG_MASK(1),
|
||||
} EStreamNotifyEventType;
|
||||
|
|
|
@ -65,6 +65,7 @@ int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* r
|
|||
int32_t streamStateSessionAllocWinBuffByNextPosition(SStreamState* pState, SStreamStateCur* pCur,
|
||||
const SSessionKey* pKey, void** pVal, int32_t* pVLen);
|
||||
|
||||
SStreamStateCur *streamStateSessionSeekKeyPrev(SStreamState *pState, const SSessionKey *key);
|
||||
SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key);
|
||||
SStreamStateCur* streamStateCountSeekKeyPrev(SStreamState* pState, const SSessionKey* pKey, COUNT_TYPE count);
|
||||
SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key);
|
||||
|
@ -162,4 +163,4 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2);
|
|||
}
|
||||
#endif
|
||||
|
||||
#endif /* ifndef _STREAM_STATE_H_ */
|
||||
#endif /* ifndef _STREAM_STATE_H_ */
|
||||
|
|
|
@ -463,6 +463,7 @@ struct SStreamTask {
|
|||
SUpstreamInfo upstreamInfo;
|
||||
STaskCheckInfo taskCheckInfo;
|
||||
SNotifyInfo notifyInfo;
|
||||
STaskNotifyEventStat notifyEventStat;
|
||||
|
||||
// the followings attributes don't be serialized
|
||||
SScanhistorySchedInfo schedHistoryInfo;
|
||||
|
@ -632,6 +633,7 @@ typedef struct STaskStatusEntry {
|
|||
int64_t startCheckpointVer;
|
||||
int64_t hTaskId;
|
||||
STaskCkptInfo checkpointInfo;
|
||||
STaskNotifyEventStat notifyEventStat;
|
||||
} STaskStatusEntry;
|
||||
|
||||
//typedef struct SNodeUpdateInfo {
|
||||
|
|
|
@ -100,10 +100,12 @@ void sessionWinStateCleanup(void* pBuff);
|
|||
SStreamStateCur* createStateCursor(SStreamFileState* pFileState);
|
||||
SStreamStateCur* sessionWinStateSeekKeyCurrentPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey);
|
||||
SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
|
||||
SStreamStateCur* sessionWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey);
|
||||
SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
|
||||
SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey, COUNT_TYPE count);
|
||||
int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
|
||||
void sessionWinStateMoveToNext(SStreamStateCur* pCur);
|
||||
void sessionWinStateMoveToPrev(SStreamStateCur* pCur);
|
||||
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey,
|
||||
range_cmpr_fn cmpFn);
|
||||
|
||||
|
|
|
@ -239,6 +239,7 @@ typedef enum ELogicConditionType {
|
|||
#define TSDB_USER_CGROUP_LEN (TSDB_USER_LEN + TSDB_CGROUP_LEN) // it is a null-terminated string
|
||||
#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string
|
||||
#define TSDB_STREAM_NOTIFY_URL_LEN 128 // it includes the terminating '\0'
|
||||
#define TSDB_STREAM_NOTIFY_STAT_LEN 350 // it includes the terminating '\0'
|
||||
#define TSDB_DB_NAME_LEN 65
|
||||
#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
|
||||
#define TSDB_PRIVILEDGE_CONDITION_LEN 48 * 1024
|
||||
|
|
|
@ -56,6 +56,7 @@ typedef struct STaskStatusEntry {
|
|||
int64_t startCheckpointVer;
|
||||
int64_t hTaskId;
|
||||
STaskCkptInfo checkpointInfo;
|
||||
STaskNotifyEventStat notifyEventStat;
|
||||
} STaskStatusEntry;
|
||||
|
||||
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo) {
|
||||
|
@ -523,6 +524,19 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) {
|
|||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->startCheckpointId));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->startCheckpointVer));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->hTaskId));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventAddTimes));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventAddElems));
|
||||
TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventAddCostSec));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPushTimes));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPushElems));
|
||||
TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventPushCostSec));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPackTimes));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPackElems));
|
||||
TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventPackCostSec));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventSendTimes));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventSendElems));
|
||||
TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventSendCostSec));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventHoldElems));
|
||||
}
|
||||
|
||||
int32_t numOfVgs = taosArrayGetSize(pReq->pUpdateNodes);
|
||||
|
@ -596,6 +610,20 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) {
|
|||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.startCheckpointVer));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.hTaskId));
|
||||
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventAddTimes));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventAddElems));
|
||||
TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventAddCostSec));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPushTimes));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPushElems));
|
||||
TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventPushCostSec));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPackTimes));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPackElems));
|
||||
TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventPackCostSec));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventSendTimes));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventSendElems));
|
||||
TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventSendCostSec));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventHoldElems));
|
||||
|
||||
entry.id.taskId = taskId;
|
||||
if (taosArrayPush(pReq->pTaskStatus, &entry) == NULL) {
|
||||
TAOS_CHECK_EXIT(terrno);
|
||||
|
@ -837,4 +865,4 @@ int32_t tDecodeStreamTaskRunReq(SDecoder* pDecoder, SStreamTaskRunReq* pReq) {
|
|||
|
||||
_exit:
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -212,6 +212,7 @@ static const SSysDbTableSchema streamTaskSchema[] = {
|
|||
{.name = "extra_info", .bytes = 25 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
|
||||
{.name = "history_task_id", .bytes = 16 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
|
||||
{.name = "history_task_status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
|
||||
{.name = "notify_event_stat", .bytes = TSDB_STREAM_NOTIFY_STAT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
|
||||
};
|
||||
|
||||
static const SSysDbTableSchema userTblsSchema[] = {
|
||||
|
|
|
@ -165,6 +165,55 @@ int32_t colDataSetValOrCover(SColumnInfoData* pColumnInfoData, uint32_t rowIndex
|
|||
return colDataSetValHelp(pColumnInfoData, rowIndex, pData, isNull);
|
||||
}
|
||||
|
||||
int32_t varColSetVarData(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pVarData, int32_t varDataLen,
|
||||
bool isNull) {
|
||||
if (!IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
}
|
||||
|
||||
if (isNull || pVarData == NULL) {
|
||||
pColumnInfoData->varmeta.offset[rowIndex] = -1; // it is a null value of VAR type.
|
||||
pColumnInfoData->hasNull = true;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t dataLen = VARSTR_HEADER_SIZE + varDataLen;
|
||||
if (pColumnInfoData->varmeta.offset[rowIndex] > 0) {
|
||||
pColumnInfoData->varmeta.length = pColumnInfoData->varmeta.offset[rowIndex];
|
||||
}
|
||||
|
||||
SVarColAttr* pAttr = &pColumnInfoData->varmeta;
|
||||
if (pAttr->allocLen < pAttr->length + dataLen) {
|
||||
uint32_t newSize = pAttr->allocLen;
|
||||
if (newSize <= 1) {
|
||||
newSize = 8;
|
||||
}
|
||||
|
||||
while (newSize < pAttr->length + dataLen) {
|
||||
newSize = newSize * 1.5;
|
||||
if (newSize > UINT32_MAX) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
}
|
||||
|
||||
char* buf = taosMemoryRealloc(pColumnInfoData->pData, newSize);
|
||||
if (buf == NULL) {
|
||||
return terrno;
|
||||
}
|
||||
|
||||
pColumnInfoData->pData = buf;
|
||||
pAttr->allocLen = newSize;
|
||||
}
|
||||
|
||||
uint32_t len = pColumnInfoData->varmeta.length;
|
||||
pColumnInfoData->varmeta.offset[rowIndex] = len;
|
||||
|
||||
(void)memmove(varDataVal(pColumnInfoData->pData + len), pVarData, varDataLen);
|
||||
varDataSetLen(pColumnInfoData->pData + len, varDataLen);
|
||||
pColumnInfoData->varmeta.length += dataLen;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx,
|
||||
const char* pData) {
|
||||
int32_t type = pColumnInfoData->info.type;
|
||||
|
|
|
@ -765,7 +765,7 @@ static int32_t addStreamTaskNotifyInfo(const SCMCreateStreamReq *createReq, cons
|
|||
TSDB_CHECK_NULL(pTask->notifyInfo.pNotifyAddrUrls, code, lino, _end, terrno);
|
||||
pTask->notifyInfo.notifyEventTypes = createReq->notifyEventTypes;
|
||||
pTask->notifyInfo.notifyErrorHandle = createReq->notifyErrorHandle;
|
||||
pTask->notifyInfo.streamName = taosStrdup(createReq->name);
|
||||
pTask->notifyInfo.streamName = taosStrdup(mndGetDbStr(createReq->name));
|
||||
TSDB_CHECK_NULL(pTask->notifyInfo.streamName, code, lino, _end, terrno);
|
||||
pTask->notifyInfo.stbFullName = taosStrdup(createReq->targetStbFullName);
|
||||
TSDB_CHECK_NULL(pTask->notifyInfo.stbFullName, code, lino, _end, terrno);
|
||||
|
|
|
@ -1309,8 +1309,8 @@ int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDataBlo
|
|||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
|
||||
// input queue
|
||||
char vbuf[40] = {0};
|
||||
char buf[38] = {0};
|
||||
char vbuf[TSDB_STREAM_NOTIFY_STAT_LEN + 2] = {0};
|
||||
char buf[TSDB_STREAM_NOTIFY_STAT_LEN] = {0};
|
||||
const char *queueInfoStr = "%4.2f MiB (%6.2f%)";
|
||||
snprintf(buf, tListLen(buf), queueInfoStr, pe->inputQUsed, pe->inputRate);
|
||||
STR_TO_VARSTR(vbuf, buf);
|
||||
|
@ -1503,6 +1503,47 @@ int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDataBlo
|
|||
code = colDataSetVal(pColInfo, numOfRows, 0, true);
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
|
||||
// notify_event_stat
|
||||
int32_t offset =0;
|
||||
if (pe->notifyEventStat.notifyEventAddTimes > 0) {
|
||||
offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Add %" PRId64 "x, %" PRId64 " elems in %lfs; ",
|
||||
pe->notifyEventStat.notifyEventAddTimes, pe->notifyEventStat.notifyEventAddElems,
|
||||
pe->notifyEventStat.notifyEventAddCostSec);
|
||||
}
|
||||
if (pe->notifyEventStat.notifyEventPushTimes > 0) {
|
||||
offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Push %" PRId64 "x, %" PRId64 " elems in %lfs; ",
|
||||
pe->notifyEventStat.notifyEventPushTimes, pe->notifyEventStat.notifyEventPushElems,
|
||||
pe->notifyEventStat.notifyEventPushCostSec);
|
||||
}
|
||||
if (pe->notifyEventStat.notifyEventPackTimes > 0) {
|
||||
offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Pack %" PRId64 "x, %" PRId64 " elems in %lfs; ",
|
||||
pe->notifyEventStat.notifyEventPackTimes, pe->notifyEventStat.notifyEventPackElems,
|
||||
pe->notifyEventStat.notifyEventPackCostSec);
|
||||
}
|
||||
if (pe->notifyEventStat.notifyEventSendTimes > 0) {
|
||||
offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Send %" PRId64 "x, %" PRId64 " elems in %lfs; ",
|
||||
pe->notifyEventStat.notifyEventSendTimes, pe->notifyEventStat.notifyEventSendElems,
|
||||
pe->notifyEventStat.notifyEventSendCostSec);
|
||||
}
|
||||
if (pe->notifyEventStat.notifyEventHoldElems > 0) {
|
||||
offset += tsnprintf(buf + offset, sizeof(buf) - offset, "[Hold %" PRId64 " elems] ",
|
||||
pe->notifyEventStat.notifyEventHoldElems);
|
||||
}
|
||||
TSDB_CHECK_CONDITION(offset < sizeof(buf), code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
|
||||
buf[offset] = '\0';
|
||||
|
||||
STR_TO_VARSTR(vbuf, buf);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
TSDB_CHECK_NULL(pColInfo, code, lino, _end, terrno);
|
||||
|
||||
if (offset == 0) {
|
||||
colDataSetNULL(pColInfo, numOfRows);
|
||||
} else {
|
||||
code = colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code) {
|
||||
mError("error happens during build task attr result blocks, lino:%d, code:%s", lino, tstrerror(code));
|
||||
|
@ -1689,4 +1730,4 @@ int32_t mndCheckForSnode(SMnode *pMnode, SDbObj *pSrcDb) {
|
|||
mError("snode not existed when trying to create stream in db with multiple replica");
|
||||
return TSDB_CODE_SNODE_NOT_DEPLOYED;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -97,6 +97,7 @@ void initStateStoreAPI(SStateStore* pStore) {
|
|||
pStore->updateInfoSerialize = updateInfoSerialize;
|
||||
pStore->updateInfoDeserialize = updateInfoDeserialize;
|
||||
|
||||
pStore->streamStateSessionSeekKeyPrev = streamStateSessionSeekKeyPrev;
|
||||
pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext;
|
||||
pStore->streamStateCountSeekKeyPrev = streamStateCountSeekKeyPrev;
|
||||
pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev;
|
||||
|
@ -123,4 +124,4 @@ void initStateStoreAPI(SStateStore* pStore) {
|
|||
void initFunctionStateStore(SFunctionStateStore* pStore) {
|
||||
pStore->streamStateFuncPut = streamStateFuncPut;
|
||||
pStore->streamStateFuncGet = streamStateFuncGet;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -20,14 +20,16 @@
|
|||
#include "curl/curl.h"
|
||||
#endif
|
||||
|
||||
#define STREAM_EVENT_NOTIFY_RETRY_MS 50 // 50ms
|
||||
#define STREAM_EVENT_NOTIFY_RETRY_MS 50 // 50 ms
|
||||
#define STREAM_EVENT_NOTIFY_MESSAAGE_SIZE_KB 8 * 1024 // 8 MB
|
||||
#define STREAM_EVENT_NOTIFY_FRAME_SIZE 256 * 1024 // 256 KB
|
||||
|
||||
typedef struct SStreamNotifyHandle {
|
||||
TdThreadMutex mutex;
|
||||
#ifndef WINDOWS
|
||||
CURL* curl;
|
||||
CURL* curl;
|
||||
#endif
|
||||
char* url;
|
||||
char* url;
|
||||
} SStreamNotifyHandle;
|
||||
|
||||
struct SStreamNotifyHandleMap {
|
||||
|
@ -49,6 +51,7 @@ static void stopStreamNotifyConn(SStreamNotifyHandle* pHandle) {
|
|||
}
|
||||
// TODO: add wait mechanism for peer connection close response
|
||||
curl_easy_cleanup(pHandle->curl);
|
||||
pHandle->curl = NULL;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
@ -258,7 +261,8 @@ _end:
|
|||
}
|
||||
|
||||
static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBlocks, char** pMsg,
|
||||
int32_t* nNotifyEvents) {
|
||||
int32_t* nNotifyEvents, STaskNotifyEventStat* pNotifyEventStat,
|
||||
int32_t* pBlockIdx) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
int32_t numOfBlocks = 0;
|
||||
|
@ -268,15 +272,20 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
|
|||
char* msgHeader = NULL;
|
||||
const char* msgTail = "]}]}";
|
||||
char* msg = NULL;
|
||||
int64_t startTime = 0;
|
||||
int64_t endTime = 0;
|
||||
int32_t nBlocks = 0;
|
||||
|
||||
TSDB_CHECK_NULL(pMsg, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
TSDB_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
*pMsg = NULL;
|
||||
numOfBlocks = taosArrayGetSize(pBlocks);
|
||||
*nNotifyEvents = 0;
|
||||
|
||||
for (int32_t i = 0; i < numOfBlocks; ++i) {
|
||||
for (int32_t i = *pBlockIdx; i < numOfBlocks; ++i) {
|
||||
SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
|
||||
nBlocks++;
|
||||
if (pDataBlock == NULL || pDataBlock->info.type != STREAM_NOTIFY_EVENT) {
|
||||
continue;
|
||||
}
|
||||
|
@ -287,13 +296,19 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
|
|||
msgLen += varDataLen(val) + 1;
|
||||
}
|
||||
*nNotifyEvents += pDataBlock->info.rows;
|
||||
if (msgLen >= STREAM_EVENT_NOTIFY_MESSAAGE_SIZE_KB * 1024) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
*pBlockIdx += nBlocks;
|
||||
|
||||
if (msgLen == 0) {
|
||||
// skip since no notification events found
|
||||
goto _end;
|
||||
}
|
||||
|
||||
startTime = taosGetMonoTimestampMs();
|
||||
code = getStreamNotifyEventHeader(streamName, &msgHeader);
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
msgHeaderLen = strlen(msgHeader);
|
||||
|
@ -306,7 +321,7 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
|
|||
TAOS_STRNCPY(p, msgHeader, msgHeaderLen);
|
||||
p += msgHeaderLen - msgTailLen;
|
||||
|
||||
for (int32_t i = 0; i < numOfBlocks; ++i) {
|
||||
for (int32_t i = *pBlockIdx - nBlocks; i < *pBlockIdx; ++i) {
|
||||
SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
|
||||
if (pDataBlock == NULL || pDataBlock->info.type != STREAM_NOTIFY_EVENT) {
|
||||
continue;
|
||||
|
@ -328,6 +343,11 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
|
|||
*pMsg = msg;
|
||||
msg = NULL;
|
||||
|
||||
endTime = taosGetMonoTimestampMs();
|
||||
pNotifyEventStat->notifyEventPackTimes++;
|
||||
pNotifyEventStat->notifyEventPackElems += *nNotifyEvents;
|
||||
pNotifyEventStat->notifyEventPackCostSec += (endTime - startTime) / 1000.0;
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tqError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
|
@ -354,9 +374,21 @@ static int32_t sendSingleStreamNotify(SStreamNotifyHandle* pHandle, char* msg) {
|
|||
TSDB_CHECK_NULL(pHandle->curl, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
totalLen = strlen(msg);
|
||||
while (sentLen < totalLen) {
|
||||
res = curl_ws_send(pHandle->curl, msg + sentLen, totalLen - sentLen, &nbytes, 0, CURLWS_TEXT);
|
||||
if (totalLen > 0) {
|
||||
// send PING frame to check if the connection is still alive
|
||||
res = curl_ws_send(pHandle->curl, "", 0, (size_t*)&sentLen, 0, CURLWS_PING);
|
||||
TSDB_CHECK_CONDITION(res == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
|
||||
}
|
||||
sentLen = 0;
|
||||
while (sentLen < totalLen) {
|
||||
size_t chunkSize = TMIN(totalLen - sentLen, STREAM_EVENT_NOTIFY_FRAME_SIZE);
|
||||
if (sentLen == 0) {
|
||||
res = curl_ws_send(pHandle->curl, msg, chunkSize, &nbytes, totalLen, CURLWS_TEXT | CURLWS_OFFSET);
|
||||
TSDB_CHECK_CONDITION(res == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
|
||||
} else {
|
||||
res = curl_ws_send(pHandle->curl, msg + sentLen, chunkSize, &nbytes, 0, CURLWS_TEXT | CURLWS_OFFSET);
|
||||
TSDB_CHECK_CONDITION(res == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
|
||||
}
|
||||
sentLen += nbytes;
|
||||
}
|
||||
|
||||
|
@ -379,6 +411,9 @@ int32_t tqSendAllNotifyEvents(const SArray* pBlocks, SStreamTask* pTask, SVnode*
|
|||
int32_t nNotifyAddr = 0;
|
||||
int32_t nNotifyEvents = 0;
|
||||
SStreamNotifyHandle* pHandle = NULL;
|
||||
int64_t startTime = 0;
|
||||
int64_t endTime = 0;
|
||||
int32_t blockIdx = 0;
|
||||
|
||||
TSDB_CHECK_NULL(pTask, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
TSDB_CHECK_NULL(pVnode, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
@ -388,50 +423,61 @@ int32_t tqSendAllNotifyEvents(const SArray* pBlocks, SStreamTask* pTask, SVnode*
|
|||
goto _end;
|
||||
}
|
||||
|
||||
code = packupStreamNotifyEvent(pTask->notifyInfo.streamName, pBlocks, &msg, &nNotifyEvents);
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
if (msg == NULL) {
|
||||
goto _end;
|
||||
}
|
||||
while (blockIdx < taosArrayGetSize(pBlocks)) {
|
||||
code = packupStreamNotifyEvent(pTask->notifyInfo.streamName, pBlocks, &msg, &nNotifyEvents, &pTask->notifyEventStat,
|
||||
&blockIdx);
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
if (msg == NULL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
tqDebug("stream task %s prepare to send %d notify events, total msg length: %" PRIu64, pTask->notifyInfo.streamName,
|
||||
nNotifyEvents, (uint64_t)strlen(msg));
|
||||
tqDebug("stream task %s prepare to send %d notify events, total msg length: %" PRIu64, pTask->notifyInfo.streamName,
|
||||
nNotifyEvents, (uint64_t)strlen(msg));
|
||||
|
||||
for (int32_t i = 0; i < nNotifyAddr; ++i) {
|
||||
if (streamTaskShouldStop(pTask)) {
|
||||
break;
|
||||
}
|
||||
const char* url = taosArrayGetP(pTask->notifyInfo.pNotifyAddrUrls, i);
|
||||
code = acquireStreamNotifyHandle(pVnode->pNotifyHandleMap, url, &pHandle);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tqError("failed to get stream notify handle of %s", url);
|
||||
if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
|
||||
// retry for event message sending in PAUSE error handling mode
|
||||
taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
|
||||
--i;
|
||||
continue;
|
||||
} else {
|
||||
// simply ignore the failure in DROP error handling mode
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
continue;
|
||||
startTime = taosGetMonoTimestampMs();
|
||||
for (int32_t i = 0; i < nNotifyAddr; ++i) {
|
||||
if (streamTaskShouldStop(pTask)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
code = sendSingleStreamNotify(pHandle, msg);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tqError("failed to send stream notify handle to %s since %s", url, tstrerror(code));
|
||||
if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
|
||||
// retry for event message sending in PAUSE error handling mode
|
||||
taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
|
||||
--i;
|
||||
} else {
|
||||
// simply ignore the failure in DROP error handling mode
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
const char* url = taosArrayGetP(pTask->notifyInfo.pNotifyAddrUrls, i);
|
||||
code = acquireStreamNotifyHandle(pVnode->pNotifyHandleMap, url, &pHandle);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tqError("failed to get stream notify handle of %s", url);
|
||||
if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
|
||||
// retry for event message sending in PAUSE error handling mode
|
||||
taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
|
||||
--i;
|
||||
continue;
|
||||
} else {
|
||||
// simply ignore the failure in DROP error handling mode
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tqDebug("stream task %s send %d notify events to %s successfully", pTask->notifyInfo.streamName, nNotifyEvents,
|
||||
url);
|
||||
code = sendSingleStreamNotify(pHandle, msg);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tqError("failed to send stream notify handle to %s since %s", url, tstrerror(code));
|
||||
if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
|
||||
// retry for event message sending in PAUSE error handling mode
|
||||
taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
|
||||
--i;
|
||||
} else {
|
||||
// simply ignore the failure in DROP error handling mode
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
}
|
||||
} else {
|
||||
tqDebug("stream task %s send %d notify events to %s successfully", pTask->notifyInfo.streamName, nNotifyEvents,
|
||||
url);
|
||||
}
|
||||
releaseStreamNotifyHandle(&pHandle);
|
||||
}
|
||||
releaseStreamNotifyHandle(&pHandle);
|
||||
|
||||
endTime = taosGetMonoTimestampMs();
|
||||
pTask->notifyEventStat.notifyEventSendTimes++;
|
||||
pTask->notifyEventStat.notifyEventSendElems += nNotifyEvents;
|
||||
pTask->notifyEventStat.notifyEventSendCostSec += (endTime - startTime) / 1000.0;
|
||||
|
||||
taosMemoryFreeClear(msg);
|
||||
}
|
||||
|
||||
_end:
|
||||
|
|
|
@ -87,9 +87,9 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) {
|
|||
return code;
|
||||
}
|
||||
|
||||
code =
|
||||
qSetStreamNotifyInfo(pTask->exec.pExecutor, pTask->notifyInfo.notifyEventTypes,
|
||||
pTask->notifyInfo.pSchemaWrapper, pTask->notifyInfo.stbFullName, IS_NEW_SUBTB_RULE(pTask));
|
||||
code = qSetStreamNotifyInfo(pTask->exec.pExecutor, pTask->notifyInfo.notifyEventTypes,
|
||||
pTask->notifyInfo.pSchemaWrapper, pTask->notifyInfo.stbFullName,
|
||||
IS_NEW_SUBTB_RULE(pTask), &pTask->notifyEventStat);
|
||||
if (code) {
|
||||
tqError("s-task:%s failed to set stream notify info, code:%s", pTask->id.idStr, tstrerror(code));
|
||||
return code;
|
||||
|
|
|
@ -223,6 +223,7 @@ void initStateStoreAPI(SStateStore* pStore) {
|
|||
pStore->updateInfoSerialize = updateInfoSerialize;
|
||||
pStore->updateInfoDeserialize = updateInfoDeserialize;
|
||||
|
||||
pStore->streamStateSessionSeekKeyPrev = streamStateSessionSeekKeyPrev;
|
||||
pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext;
|
||||
pStore->streamStateCountSeekKeyPrev = streamStateCountSeekKeyPrev;
|
||||
pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev;
|
||||
|
|
|
@ -450,16 +450,17 @@ typedef struct STimeWindowAggSupp {
|
|||
} STimeWindowAggSupp;
|
||||
|
||||
typedef struct SStreamNotifyEventSupp {
|
||||
SArray* pWindowEvents; // Array of SStreamNotifyEvent, storing window events and trigger values.
|
||||
SHashObj* pTableNameHashMap; // Hash map from groupid to the dest child table name.
|
||||
SHashObj* pResultHashMap; // Hash map from groupid+skey to the window agg result.
|
||||
SSDataBlock* pEventBlock; // The datablock contains all window events and results.
|
||||
SHashObj* pWindowEventHashMap; // Hash map from gorupid+skey+eventType to the list node of window event.
|
||||
SHashObj* pTableNameHashMap; // Hash map from groupid to the dest child table name.
|
||||
SSDataBlock* pEventBlock; // The datablock contains all window events and results.
|
||||
SArray* pSessionKeys;
|
||||
const char* windowType;
|
||||
} SStreamNotifyEventSupp;
|
||||
|
||||
typedef struct SSteamOpBasicInfo {
|
||||
int32_t primaryPkIndex;
|
||||
bool updateOperatorInfo;
|
||||
SStreamNotifyEventSupp windowEventSup;
|
||||
SStreamNotifyEventSupp notifyEventSup;
|
||||
} SSteamOpBasicInfo;
|
||||
|
||||
typedef struct SStreamFillSupporter {
|
||||
|
@ -1053,7 +1054,7 @@ int32_t saveDeleteRes(SSHashObj* pStDelete, SSessionKey key);
|
|||
void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey);
|
||||
void doBuildDeleteDataBlock(struct SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite);
|
||||
void doBuildSessionResult(struct SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo,
|
||||
SSDataBlock* pBlock);
|
||||
SSDataBlock* pBlock, SArray* pSessionKeys);
|
||||
int32_t getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo);
|
||||
void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin,
|
||||
SResultWindowInfo* pNextWin);
|
||||
|
@ -1090,7 +1091,7 @@ void freeResetOperatorParams(struct SOperatorInfo* pOperator, SOperatorParamT
|
|||
int32_t getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int32_t idx, bool clearParam,
|
||||
SSDataBlock** pResBlock);
|
||||
void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, EStreamType mode, SSessionKey* pDelRange);
|
||||
void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey);
|
||||
void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey);
|
||||
|
||||
int32_t saveDeleteInfo(SArray* pWins, SSessionKey key);
|
||||
void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins);
|
||||
|
|
|
@ -18,9 +18,9 @@
|
|||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "executorInt.h"
|
||||
#endif
|
||||
|
||||
#include "executorInt.h"
|
||||
|
||||
#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str)
|
||||
|
||||
|
@ -59,22 +59,23 @@ typedef struct STaskStopInfo {
|
|||
} STaskStopInfo;
|
||||
|
||||
typedef struct {
|
||||
STqOffsetVal currentOffset; // for tmq
|
||||
SMqBatchMetaRsp btMetaRsp; // for tmq fetching meta
|
||||
int8_t sourceExcluded;
|
||||
int64_t snapshotVer;
|
||||
SSchemaWrapper* schema;
|
||||
char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor
|
||||
int8_t recoverStep;
|
||||
int8_t recoverScanFinished;
|
||||
SQueryTableDataCond tableCond;
|
||||
SVersionRange fillHistoryVer;
|
||||
STimeWindow fillHistoryWindow;
|
||||
SStreamState* pState;
|
||||
int32_t eventTypes; // event types to notify
|
||||
SSchemaWrapper* notifyResultSchema; // agg result to notify
|
||||
char* stbFullName; // used to generate dest child table name
|
||||
bool newSubTableRule; // used to generate dest child table name
|
||||
STqOffsetVal currentOffset; // for tmq
|
||||
SMqBatchMetaRsp btMetaRsp; // for tmq fetching meta
|
||||
int8_t sourceExcluded;
|
||||
int64_t snapshotVer;
|
||||
SSchemaWrapper* schema;
|
||||
char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor
|
||||
int8_t recoverStep;
|
||||
int8_t recoverScanFinished;
|
||||
SQueryTableDataCond tableCond;
|
||||
SVersionRange fillHistoryVer;
|
||||
STimeWindow fillHistoryWindow;
|
||||
SStreamState* pState;
|
||||
int32_t eventTypes; // event types to notify
|
||||
SSchemaWrapper* notifyResultSchema; // agg result to notify
|
||||
char* stbFullName; // used to generate dest child table name
|
||||
bool newSubTableRule; // used to generate dest child table name
|
||||
STaskNotifyEventStat* pNotifyEventStat; // used to store notify event statistics
|
||||
} SStreamTaskInfo;
|
||||
|
||||
struct SExecTaskInfo {
|
||||
|
|
|
@ -30,20 +30,31 @@ extern "C" {
|
|||
#define FILL_POS_MID 2
|
||||
#define FILL_POS_END 3
|
||||
|
||||
#define HAS_NON_ROW_DATA(pRowData) (pRowData->key == INT64_MIN)
|
||||
#define HAS_ROW_DATA(pRowData) (pRowData && pRowData->key != INT64_MIN)
|
||||
#define HAS_NON_ROW_DATA(pRowData) (pRowData->key == INT64_MIN)
|
||||
#define HAS_ROW_DATA(pRowData) (pRowData && pRowData->key != INT64_MIN)
|
||||
|
||||
#define IS_INVALID_WIN_KEY(ts) ((ts) == INT64_MIN)
|
||||
#define IS_VALID_WIN_KEY(ts) ((ts) != INT64_MIN)
|
||||
#define SET_WIN_KEY_INVALID(ts) ((ts) = INT64_MIN)
|
||||
#define IS_INVALID_WIN_KEY(ts) ((ts) == INT64_MIN)
|
||||
#define IS_VALID_WIN_KEY(ts) ((ts) != INT64_MIN)
|
||||
#define SET_WIN_KEY_INVALID(ts) ((ts) = INT64_MIN)
|
||||
|
||||
#define IS_NORMAL_INTERVAL_OP(op) \
|
||||
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || \
|
||||
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
|
||||
|
||||
#define IS_NORMAL_SESSION_OP(op) \
|
||||
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \
|
||||
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
|
||||
|
||||
#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE)
|
||||
|
||||
#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT)
|
||||
|
||||
#define IS_NORMAL_COUNT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT)
|
||||
|
||||
#define IS_CONTINUE_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL)
|
||||
|
||||
#define IS_FILL_CONST_VALUE(type) ((type == TSDB_FILL_NULL || type == TSDB_FILL_NULL_F || type == TSDB_FILL_SET_VALUE || type == TSDB_FILL_SET_VALUE_F))
|
||||
#define IS_FILL_CONST_VALUE(type) \
|
||||
((type == TSDB_FILL_NULL || type == TSDB_FILL_NULL_F || type == TSDB_FILL_SET_VALUE || type == TSDB_FILL_SET_VALUE_F))
|
||||
|
||||
typedef struct SSliceRowData {
|
||||
TSKEY key;
|
||||
|
@ -57,11 +68,13 @@ typedef struct SSlicePoint {
|
|||
SRowBuffPos* pResPos;
|
||||
} SSlicePoint;
|
||||
|
||||
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
|
||||
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);
|
||||
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo);
|
||||
int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo);
|
||||
void destroyStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo);
|
||||
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
|
||||
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);
|
||||
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo);
|
||||
int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo, const struct SOperatorInfo* pOperator);
|
||||
void destroyStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo);
|
||||
int32_t encodeStreamBasicInfo(void** buf, const SSteamOpBasicInfo* pBasicInfo);
|
||||
int32_t decodeStreamBasicInfo(void** buf, SSteamOpBasicInfo* pBasicInfo);
|
||||
|
||||
int64_t getDeleteMarkFromOption(SStreamNodeOption* pOption);
|
||||
void removeDeleteResults(SSHashObj* pUpdatedMap, SArray* pDelWins);
|
||||
|
@ -98,7 +111,7 @@ SResultCellData* getSliceResultCell(SResultCellData* pRowVal, int32_t index, int
|
|||
int32_t getDownstreamRes(struct SOperatorInfo* downstream, SSDataBlock** ppRes, SColumnInfo** ppPkCol);
|
||||
void destroyFlusedppPos(void* ppRes);
|
||||
void doBuildStreamIntervalResult(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
|
||||
SGroupResInfo* pGroupResInfo);
|
||||
SGroupResInfo* pGroupResInfo, SArray* pSessionKeys);
|
||||
void transBlockToSliceResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SSliceRowData* pRowVal,
|
||||
int32_t rowSize, void* pPkData, SColumnInfoData* pPkCol, int32_t* pCellOffsetInfo);
|
||||
int32_t getQualifiedRowNumDesc(SExprSupp* pExprSup, SSDataBlock* pBlock, TSKEY* tsCols, int32_t rowId, bool ignoreNull);
|
||||
|
@ -112,10 +125,25 @@ TSKEY compareTs(void* pKey);
|
|||
|
||||
int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
const SSDataBlock* pInputBlock, const SNodeList* pCondCols, int32_t ri,
|
||||
SStreamNotifyEventSupp* sup);
|
||||
int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SSchemaWrapper* pSchemaWrapper,
|
||||
SStreamNotifyEventSupp* sup);
|
||||
int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup);
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t addStateAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
const SStateKeys* pCurState, const SStateKeys* pAnotherState, bool onlyUpdate,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t addIntervalAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t addSessionAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t addCountAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SArray* pSessionKeys,
|
||||
const SSchemaWrapper* pSchemaWrapper, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t addAggDeleteNotifyEvent(const SSDataBlock* pDeleteBlock, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t removeOutdatedNotifyEvents(STimeWindowAggSupp* pTwSup, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
@ -251,7 +251,7 @@ int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) {
|
|||
}
|
||||
|
||||
int32_t qSetStreamNotifyInfo(qTaskInfo_t tinfo, int32_t eventTypes, const SSchemaWrapper* pSchemaWrapper,
|
||||
const char* stbFullName, bool newSubTableRule) {
|
||||
const char* stbFullName, bool newSubTableRule, STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
SStreamTaskInfo *pStreamInfo = NULL;
|
||||
|
||||
|
@ -267,6 +267,7 @@ int32_t qSetStreamNotifyInfo(qTaskInfo_t tinfo, int32_t eventTypes, const SSchem
|
|||
}
|
||||
pStreamInfo->stbFullName = taosStrdup(stbFullName);
|
||||
pStreamInfo->newSubTableRule = newSubTableRule;
|
||||
pStreamInfo->pNotifyEventStat = pNotifyEventStat;
|
||||
|
||||
_end:
|
||||
return code;
|
||||
|
|
|
@ -25,7 +25,6 @@
|
|||
#include "tlog.h"
|
||||
#include "ttime.h"
|
||||
|
||||
#define IS_NORMAL_COUNT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT)
|
||||
#define STREAM_COUNT_OP_STATE_NAME "StreamCountHistoryState"
|
||||
#define STREAM_COUNT_OP_CHECKPOINT_NAME "StreamCountOperator_Checkpoint"
|
||||
|
||||
|
@ -56,6 +55,8 @@ void destroyStreamCountAggOperatorInfo(void* param) {
|
|||
&pInfo->groupResInfo);
|
||||
pInfo->pOperator = NULL;
|
||||
}
|
||||
|
||||
destroyStreamBasicInfo(&pInfo->basic);
|
||||
destroyStreamAggSupporter(&pInfo->streamAggSup);
|
||||
cleanupExprSupp(&pInfo->scalarSupp);
|
||||
clearGroupResInfo(&pInfo->groupResInfo);
|
||||
|
@ -79,10 +80,9 @@ void destroyStreamCountAggOperatorInfo(void* param) {
|
|||
bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; }
|
||||
|
||||
int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin,
|
||||
SBuffInfo* pBuffInfo) {
|
||||
SBuffInfo* pBuffInfo, int32_t* pWinCode) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
int32_t size = pAggSup->resultRowSize;
|
||||
pCurWin->winInfo.sessionWin.groupId = groupId;
|
||||
pCurWin->winInfo.sessionWin.win.skey = ts;
|
||||
|
@ -90,19 +90,21 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
|
|||
|
||||
if (isSlidingCountWindow(pAggSup)) {
|
||||
if (pBuffInfo->winBuffOp == CREATE_NEW_WINDOW) {
|
||||
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
code =
|
||||
pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
|
||||
pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
winCode = TSDB_CODE_FAILED;
|
||||
*pWinCode = TSDB_CODE_FAILED;
|
||||
} else if (pBuffInfo->winBuffOp == MOVE_NEXT_WINDOW) {
|
||||
QUERY_CHECK_NULL(pBuffInfo->pCur, code, lino, _end, terrno);
|
||||
pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur);
|
||||
winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
if (winCode == TSDB_CODE_FAILED) {
|
||||
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
*pWinCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
if (*pWinCode == TSDB_CODE_FAILED) {
|
||||
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
|
||||
pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos,
|
||||
&size);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
} else {
|
||||
reuseOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->stateStore);
|
||||
|
@ -110,11 +112,12 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
|
|||
} else {
|
||||
pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin,
|
||||
pAggSup->windowCount);
|
||||
winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
if (winCode == TSDB_CODE_FAILED) {
|
||||
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
*pWinCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
if (*pWinCode == TSDB_CODE_FAILED) {
|
||||
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
|
||||
pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos,
|
||||
&size);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
} else {
|
||||
reuseOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->stateStore);
|
||||
|
@ -126,11 +129,11 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
|
|||
} else {
|
||||
code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin,
|
||||
pAggSup->windowCount,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size, &winCode);
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size, pWinCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
if (winCode == TSDB_CODE_SUCCESS) {
|
||||
if (*pWinCode == TSDB_CODE_SUCCESS) {
|
||||
pCurWin->winInfo.isOutput = true;
|
||||
}
|
||||
pCurWin->pWindowCount =
|
||||
|
@ -297,10 +300,18 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
continue;
|
||||
}
|
||||
SCountWindowInfo curWin = {0};
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
buffInfo.rebuildWindow = false;
|
||||
code = setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo);
|
||||
code = setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo, &winCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (winCode != TSDB_CODE_SUCCESS &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
|
||||
code = addCountAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, &pInfo->basic.notifyEventSup,
|
||||
pTaskInfo->streamInfo.pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
if (!inCountSlidingWindow(pAggSup, &curWin.winInfo.sessionWin.win, &pSDataBlock->info)) {
|
||||
buffInfo.winBuffOp = MOVE_NEXT_WINDOW;
|
||||
continue;
|
||||
|
@ -375,23 +386,54 @@ _end:
|
|||
|
||||
static int32_t buildCountResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
SOptrBasicInfo* pBInfo = &pInfo->binfo;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
bool addNotifyEvent = false;
|
||||
addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
|
||||
doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->pDelRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
|
||||
doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes,
|
||||
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
|
||||
if (pBInfo->pRes->info.rows > 0) {
|
||||
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
|
||||
pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pBInfo->pRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
|
||||
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
(*ppRes) = pNotifySup->pEventBlock;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
}
|
||||
(*ppRes) = NULL;
|
||||
return code;
|
||||
}
|
||||
|
@ -423,7 +465,10 @@ int32_t doStreamCountEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
|
|||
// 3.dataVersion
|
||||
tlen += taosEncodeFixedI32(buf, pInfo->dataVersion);
|
||||
|
||||
// 4.checksum
|
||||
// 4.basicInfo
|
||||
tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
|
||||
|
||||
// 5.checksum
|
||||
if (isParent) {
|
||||
if (buf) {
|
||||
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
|
||||
|
@ -441,12 +486,13 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
int32_t lino = 0;
|
||||
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
void* pDataEnd = POINTER_SHIFT(buf, len);
|
||||
if (!pInfo) {
|
||||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
// 4.checksum
|
||||
// 5.checksum
|
||||
if (isParent) {
|
||||
int32_t dataLen = len - sizeof(uint32_t);
|
||||
void* pCksum = POINTER_SHIFT(buf, dataLen);
|
||||
|
@ -454,6 +500,7 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
pDataEnd = pCksum;
|
||||
}
|
||||
|
||||
// 1.streamAggSup.pResultRows
|
||||
|
@ -462,9 +509,10 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
for (int32_t i = 0; i < mapSize; i++) {
|
||||
SSessionKey key = {0};
|
||||
SCountWindowInfo curWin = {0};
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
buf = decodeSSessionKey(buf, &key);
|
||||
SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL};
|
||||
code = setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo);
|
||||
code = setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo, &winCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize);
|
||||
|
@ -479,6 +527,12 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
// 3.dataVersion
|
||||
buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);
|
||||
|
||||
// 4.basicInfo
|
||||
if (buf < pDataEnd) {
|
||||
code = decodeStreamBasicInfo(&buf, &pInfo->basic);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
|
@ -851,7 +905,7 @@ int32_t createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
|
|||
QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);
|
||||
pInfo->binfo.pRes = pResBlock;
|
||||
|
||||
SExprInfo* pExprInfo = NULL;
|
||||
SExprInfo* pExprInfo = NULL;
|
||||
code = createExprInfo(pCountNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
|
@ -925,6 +979,9 @@ int32_t createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
|
|||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
if (downstream) {
|
||||
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
|
||||
&pInfo->twAggSup, &pInfo->basic);
|
||||
|
|
|
@ -30,7 +30,6 @@
|
|||
#include "tlog.h"
|
||||
#include "ttime.h"
|
||||
|
||||
#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT)
|
||||
#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState"
|
||||
#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint"
|
||||
|
||||
|
@ -135,7 +134,8 @@ void reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) {
|
|||
}
|
||||
|
||||
int32_t setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupId, bool* pStart, bool* pEnd,
|
||||
int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey, int32_t* pWinCode) {
|
||||
int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey,
|
||||
int32_t* pWinCode) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
|
@ -179,7 +179,7 @@ int32_t setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t gro
|
|||
SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId};
|
||||
code = pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
(*pWinCode) = TSDB_CODE_FAILED;
|
||||
(*pWinCode) = TSDB_CODE_FAILED;
|
||||
|
||||
setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin);
|
||||
pCurWin->pWinFlag->startFlag = start;
|
||||
|
@ -335,6 +335,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
SColumnInfoData* pColStart = NULL;
|
||||
SColumnInfoData* pColEnd = NULL;
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
|
||||
pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
|
||||
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
|
||||
|
@ -395,17 +397,19 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
&nextWinKey, &winCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN) &&
|
||||
*(bool*)colDataGetNumData(pColStart, i) && winCode != TSDB_CODE_SUCCESS) {
|
||||
if (winCode != TSDB_CODE_SUCCESS &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN) &&
|
||||
*(bool*)colDataGetNumData(pColStart, i)) {
|
||||
code = addEventAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, pSDataBlock,
|
||||
pInfo->pStartCondCols, i, &pInfo->basic.windowEventSup);
|
||||
pInfo->pStartCondCols, i, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);
|
||||
bool rebuild = false;
|
||||
code = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData,
|
||||
rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild, &winRows);
|
||||
code = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData,
|
||||
(bool*)pColEnd->pData, rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild,
|
||||
&winRows);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (rebuild) {
|
||||
|
@ -471,7 +475,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
|
||||
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {
|
||||
code = addEventAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &curWin.winInfo.sessionWin, pSDataBlock,
|
||||
pInfo->pEndCondCols, i + winRows - 1, &pInfo->basic.windowEventSup);
|
||||
pInfo->pEndCondCols, i + winRows - 1, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
}
|
||||
|
@ -513,7 +517,10 @@ int32_t doStreamEventEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
|
|||
// 3.dataVersion
|
||||
tlen += taosEncodeFixedI32(buf, pInfo->dataVersion);
|
||||
|
||||
// 4.checksum
|
||||
// 4.basicInfo
|
||||
tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
|
||||
|
||||
// 5.checksum
|
||||
if (buf) {
|
||||
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
|
||||
tlen += taosEncodeFixedU32(buf, cksum);
|
||||
|
@ -529,13 +536,14 @@ int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
int32_t lino = 0;
|
||||
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
void* pDataEnd = POINTER_SHIFT(buf, len);
|
||||
if (!pInfo) {
|
||||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
|
||||
// 4.checksum
|
||||
// 5.checksum
|
||||
int32_t dataLen = len - sizeof(uint32_t);
|
||||
void* pCksum = POINTER_SHIFT(buf, dataLen);
|
||||
if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {
|
||||
|
@ -543,6 +551,7 @@ int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
pDataEnd = pCksum;
|
||||
|
||||
// 1.streamAggSup.pResultRows
|
||||
int32_t mapSize = 0;
|
||||
|
@ -567,6 +576,12 @@ int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
// 3.dataVersion
|
||||
buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);
|
||||
|
||||
// 4.basicInfo
|
||||
if (buf < pDataEnd) {
|
||||
code = decodeStreamBasicInfo(&buf, &pInfo->basic);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
|
@ -598,33 +613,45 @@ static int32_t buildEventResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
|
|||
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
|
||||
SOptrBasicInfo* pBInfo = &pInfo->binfo;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
bool addNotifyEvent = false;
|
||||
addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
|
||||
doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->pDelRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes);
|
||||
doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes,
|
||||
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
|
||||
if (pBInfo->pRes->info.rows > 0) {
|
||||
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {
|
||||
code = addAggResultNotifyEvent(pBInfo->pRes, pTaskInfo->streamInfo.notifyResultSchema, &pInfo->basic.windowEventSup);
|
||||
if (addNotifyEvent) {
|
||||
code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
|
||||
pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pBInfo->pRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = buildNotifyEventBlock(pTaskInfo, &pInfo->basic.windowEventSup);
|
||||
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (pInfo->basic.windowEventSup.pEventBlock->info.rows > 0) {
|
||||
printDataBlock(pInfo->basic.windowEventSup.pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
(*ppRes) = pInfo->basic.windowEventSup.pEventBlock;
|
||||
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
|
||||
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
(*ppRes) = pNotifySup->pEventBlock;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
_end:
|
||||
(*ppRes) = NULL;
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
@ -800,7 +827,7 @@ void streamEventReleaseState(SOperatorInfo* pOperator) {
|
|||
int32_t resSize = winSize + sizeof(TSKEY);
|
||||
char* pBuff = taosMemoryCalloc(1, resSize);
|
||||
if (!pBuff) {
|
||||
return ;
|
||||
return;
|
||||
}
|
||||
memcpy(pBuff, pInfo->historyWins->pData, winSize);
|
||||
memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));
|
||||
|
@ -954,9 +981,9 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
|
|||
QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);
|
||||
pInfo->binfo.pRes = pResBlock;
|
||||
|
||||
SExprSupp* pExpSup = &pOperator->exprSupp;
|
||||
int32_t numOfCols = 0;
|
||||
SExprInfo* pExprInfo = NULL;
|
||||
SExprSupp* pExpSup = &pOperator->exprSupp;
|
||||
int32_t numOfCols = 0;
|
||||
SExprInfo* pExprInfo = NULL;
|
||||
code = createExprInfo(pEventNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
|
@ -1006,7 +1033,6 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
|
|||
pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);
|
||||
QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno);
|
||||
pInfo->destHasPrimaryKey = pEventNode->window.destHasPrimaryKey;
|
||||
initStreamBasicInfo(&pInfo->basic);
|
||||
|
||||
pInfo->pOperator = pOperator;
|
||||
setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED,
|
||||
|
@ -1026,6 +1052,10 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
|
|||
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAggNext, NULL, destroyStreamEventOperatorInfo,
|
||||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
|
||||
&pInfo->twAggSup, &pInfo->basic);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
|
|
@ -16,53 +16,58 @@
|
|||
#include "streamexecutorInt.h"
|
||||
|
||||
#include "executorInt.h"
|
||||
#include "operator.h"
|
||||
#include "tdatablock.h"
|
||||
|
||||
#define NOTIFY_EVENT_NAME_CACHE_LIMIT_MB 16
|
||||
|
||||
typedef struct SStreamNotifyEvent {
|
||||
uint64_t gid;
|
||||
TSKEY skey;
|
||||
char* content;
|
||||
bool isEnd;
|
||||
int64_t eventType;
|
||||
STimeWindow win;
|
||||
cJSON* pJson;
|
||||
} SStreamNotifyEvent;
|
||||
|
||||
#define NOTIFY_EVENT_KEY_SIZE \
|
||||
((sizeof(((struct SStreamNotifyEvent*)0)->gid) + sizeof(((struct SStreamNotifyEvent*)0)->eventType)) + \
|
||||
sizeof(((struct SStreamNotifyEvent*)0)->win.skey))
|
||||
|
||||
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type) {
|
||||
if (type != STREAM_GET_ALL && type != STREAM_CHECKPOINT) {
|
||||
pBasicInfo->updateOperatorInfo = true;
|
||||
}
|
||||
}
|
||||
|
||||
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo) {
|
||||
return pBasicInfo->updateOperatorInfo;
|
||||
}
|
||||
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo) { return pBasicInfo->updateOperatorInfo; }
|
||||
|
||||
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo) {
|
||||
pBasicInfo->updateOperatorInfo = false;
|
||||
}
|
||||
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo) { pBasicInfo->updateOperatorInfo = false; }
|
||||
|
||||
static void destroyStreamWindowEvent(void* ptr) {
|
||||
SStreamNotifyEvent* pEvent = ptr;
|
||||
if (pEvent == NULL || pEvent->content == NULL) return;
|
||||
cJSON_free(pEvent->content);
|
||||
SStreamNotifyEvent* pEvent = (SStreamNotifyEvent*)ptr;
|
||||
if (pEvent) {
|
||||
if (pEvent->pJson) {
|
||||
cJSON_Delete(pEvent->pJson);
|
||||
}
|
||||
*pEvent = (SStreamNotifyEvent){0};
|
||||
}
|
||||
}
|
||||
|
||||
static void destroyStreamNotifyEventSupp(SStreamNotifyEventSupp* sup) {
|
||||
if (sup == NULL) return;
|
||||
taosArrayDestroyEx(sup->pWindowEvents, destroyStreamWindowEvent);
|
||||
taosHashCleanup(sup->pWindowEventHashMap);
|
||||
taosHashCleanup(sup->pTableNameHashMap);
|
||||
taosHashCleanup(sup->pResultHashMap);
|
||||
blockDataDestroy(sup->pEventBlock);
|
||||
taosArrayDestroy(sup->pSessionKeys);
|
||||
*sup = (SStreamNotifyEventSupp){0};
|
||||
}
|
||||
|
||||
static int32_t initStreamNotifyEventSupp(SStreamNotifyEventSupp *sup) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SSDataBlock* pBlock = NULL;
|
||||
static int32_t initStreamNotifyEventSupp(SStreamNotifyEventSupp* sup, const char* windowType, int32_t resCapacity) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SSDataBlock* pBlock = NULL;
|
||||
SColumnInfoData infoData = {0};
|
||||
|
||||
if (sup == NULL) {
|
||||
if (sup == NULL || sup->pWindowEventHashMap != NULL) {
|
||||
goto _end;
|
||||
}
|
||||
|
||||
|
@ -77,15 +82,18 @@ static int32_t initStreamNotifyEventSupp(SStreamNotifyEventSupp *sup) {
|
|||
code = blockDataAppendColInfo(pBlock, &infoData);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
sup->pWindowEvents = taosArrayInit(0, sizeof(SStreamNotifyEvent));
|
||||
QUERY_CHECK_NULL(sup->pWindowEvents, code, lino, _end, terrno);
|
||||
sup->pWindowEventHashMap = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
|
||||
QUERY_CHECK_NULL(sup->pWindowEventHashMap, code, lino, _end, terrno);
|
||||
taosHashSetFreeFp(sup->pWindowEventHashMap, destroyStreamWindowEvent);
|
||||
sup->pTableNameHashMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
|
||||
QUERY_CHECK_NULL(sup->pTableNameHashMap, code, lino, _end, terrno);
|
||||
sup->pResultHashMap = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
|
||||
QUERY_CHECK_NULL(sup->pResultHashMap, code, lino, _end, terrno);
|
||||
taosHashSetFreeFp(sup->pResultHashMap, destroyStreamWindowEvent);
|
||||
sup->pEventBlock = pBlock;
|
||||
pBlock = NULL;
|
||||
code = blockDataEnsureCapacity(sup->pEventBlock, resCapacity);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
sup->windowType = windowType;
|
||||
sup->pSessionKeys = taosArrayInit(resCapacity, sizeof(SSessionKey));
|
||||
QUERY_CHECK_NULL(sup->pSessionKeys, code, lino, _end, terrno);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
@ -100,17 +108,99 @@ _end:
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo) {
|
||||
int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo, const struct SOperatorInfo* pOperator) {
|
||||
pBasicInfo->primaryPkIndex = -1;
|
||||
pBasicInfo->updateOperatorInfo = false;
|
||||
return initStreamNotifyEventSupp(&pBasicInfo->windowEventSup);
|
||||
const char* windowType = NULL;
|
||||
if (IS_NORMAL_INTERVAL_OP(pOperator)) {
|
||||
windowType = "Time";
|
||||
} else if (IS_NORMAL_SESSION_OP(pOperator)) {
|
||||
windowType = "Session";
|
||||
} else if (IS_NORMAL_STATE_OP(pOperator)) {
|
||||
windowType = "State";
|
||||
} else if (IS_NORMAL_EVENT_OP(pOperator)) {
|
||||
windowType = "Event";
|
||||
} else if (IS_NORMAL_COUNT_OP(pOperator)) {
|
||||
windowType = "Count";
|
||||
} else {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
return initStreamNotifyEventSupp(&pBasicInfo->notifyEventSup, windowType, pOperator->resultInfo.capacity);
|
||||
}
|
||||
|
||||
void destroyStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo) {
|
||||
destroyStreamNotifyEventSupp(&pBasicInfo->windowEventSup);
|
||||
destroyStreamNotifyEventSupp(&pBasicInfo->notifyEventSup);
|
||||
}
|
||||
|
||||
static void streamNotifyGetEventWindowId(const SSessionKey* pSessionKey, char *buf) {
|
||||
static int32_t encodeStreamNotifyEventSupp(void** buf, const SStreamNotifyEventSupp* sup) {
|
||||
int32_t tlen = 0;
|
||||
void* pIter = NULL;
|
||||
char* str = NULL;
|
||||
|
||||
if (sup == NULL) {
|
||||
return tlen;
|
||||
}
|
||||
|
||||
tlen += taosEncodeFixedI32(buf, taosHashGetSize(sup->pWindowEventHashMap));
|
||||
pIter = taosHashIterate(sup->pWindowEventHashMap, NULL);
|
||||
while (pIter) {
|
||||
const SStreamNotifyEvent* pEvent = (const SStreamNotifyEvent*)pIter;
|
||||
str = cJSON_PrintUnformatted(pEvent->pJson);
|
||||
|
||||
tlen += taosEncodeFixedU64(buf, pEvent->gid);
|
||||
tlen += taosEncodeFixedI64(buf, pEvent->eventType);
|
||||
tlen += taosEncodeFixedI64(buf, pEvent->win.skey);
|
||||
tlen += taosEncodeFixedI64(buf, pEvent->win.ekey);
|
||||
tlen += taosEncodeString(buf, str);
|
||||
cJSON_free(str);
|
||||
pIter = taosHashIterate(sup->pWindowEventHashMap, pIter);
|
||||
}
|
||||
return tlen;
|
||||
}
|
||||
|
||||
static int32_t decodeStreamNotifyEventSupp(void** buf, SStreamNotifyEventSupp* sup) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
void* p = *buf;
|
||||
int32_t size = 0;
|
||||
uint64_t len = 0;
|
||||
SStreamNotifyEvent item = {0};
|
||||
|
||||
p = taosDecodeFixedI32(p, &size);
|
||||
for (int32_t i = 0; i < size; i++) {
|
||||
p = taosDecodeFixedU64(p, &item.gid);
|
||||
p = taosDecodeFixedI64(p, &item.eventType);
|
||||
p = taosDecodeFixedI64(p, &item.win.skey);
|
||||
p = taosDecodeFixedI64(p, &item.win.ekey);
|
||||
p = taosDecodeVariantU64(p, &len);
|
||||
item.pJson = cJSON_Parse(p);
|
||||
if (item.pJson == NULL) {
|
||||
qWarn("failed to parse the json content since %s", cJSON_GetErrorPtr());
|
||||
}
|
||||
QUERY_CHECK_NULL(item.pJson, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
|
||||
p = POINTER_SHIFT(p, len);
|
||||
code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
item.pJson = NULL;
|
||||
}
|
||||
*buf = p;
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
destroyStreamWindowEvent(&item);
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t encodeStreamBasicInfo(void** buf, const SSteamOpBasicInfo* pBasicInfo) {
|
||||
return encodeStreamNotifyEventSupp(buf, &pBasicInfo->notifyEventSup);
|
||||
}
|
||||
|
||||
int32_t decodeStreamBasicInfo(void** buf, SSteamOpBasicInfo* pBasicInfo) {
|
||||
return decodeStreamNotifyEventSupp(buf, &pBasicInfo->notifyEventSup);
|
||||
}
|
||||
|
||||
static void streamNotifyGetEventWindowId(const SSessionKey* pSessionKey, char* buf) {
|
||||
uint64_t hash = 0;
|
||||
uint64_t ar[2];
|
||||
|
||||
|
@ -123,60 +213,60 @@ static void streamNotifyGetEventWindowId(const SSessionKey* pSessionKey, char *b
|
|||
#define JSON_CHECK_ADD_ITEM(obj, str, item) \
|
||||
QUERY_CHECK_CONDITION(cJSON_AddItemToObjectCS(obj, str, item), code, lino, _end, TSDB_CODE_OUT_OF_MEMORY)
|
||||
|
||||
static int32_t jsonAddColumnField(const char* colName, const SColumnInfoData* pColData, int32_t ri, cJSON* obj) {
|
||||
static int32_t jsonAddColumnField(const char* colName, int16_t type, bool isNull, const char* pData, cJSON* obj) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
char* temp = NULL;
|
||||
|
||||
QUERY_CHECK_NULL(colName, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_CONDITION(isNull || (pData != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(obj, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
if (colDataIsNull_s(pColData, ri)) {
|
||||
if (isNull) {
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNull());
|
||||
goto _end;
|
||||
}
|
||||
|
||||
switch (pColData->info.type) {
|
||||
switch (type) {
|
||||
case TSDB_DATA_TYPE_BOOL: {
|
||||
bool val = *(bool*)colDataGetNumData(pColData, ri);
|
||||
bool val = *(const bool*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateBool(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_TINYINT: {
|
||||
int8_t val = *(int8_t*)colDataGetNumData(pColData, ri);
|
||||
int8_t val = *(const int8_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_SMALLINT: {
|
||||
int16_t val = *(int16_t*)colDataGetNumData(pColData, ri);
|
||||
int16_t val = *(const int16_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_INT: {
|
||||
int32_t val = *(int32_t*)colDataGetNumData(pColData, ri);
|
||||
int32_t val = *(const int32_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_BIGINT:
|
||||
case TSDB_DATA_TYPE_TIMESTAMP: {
|
||||
int64_t val = *(int64_t*)colDataGetNumData(pColData, ri);
|
||||
int64_t val = *(const int64_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_FLOAT: {
|
||||
float val = *(float*)colDataGetNumData(pColData, ri);
|
||||
float val = *(const float*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_DOUBLE: {
|
||||
double val = *(double*)colDataGetNumData(pColData, ri);
|
||||
double val = *(const double*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
@ -185,8 +275,8 @@ static int32_t jsonAddColumnField(const char* colName, const SColumnInfoData* pC
|
|||
case TSDB_DATA_TYPE_NCHAR: {
|
||||
// cJSON requires null-terminated strings, but this data is not null-terminated,
|
||||
// so we need to manually copy the string and add null termination.
|
||||
const char* src = varDataVal(colDataGetVarData(pColData, ri));
|
||||
int32_t len = varDataLen(colDataGetVarData(pColData, ri));
|
||||
const char* src = varDataVal(pData);
|
||||
int32_t len = varDataLen(pData);
|
||||
temp = cJSON_malloc(len + 1);
|
||||
QUERY_CHECK_NULL(temp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
|
||||
memcpy(temp, src, len);
|
||||
|
@ -202,25 +292,25 @@ static int32_t jsonAddColumnField(const char* colName, const SColumnInfoData* pC
|
|||
}
|
||||
|
||||
case TSDB_DATA_TYPE_UTINYINT: {
|
||||
uint8_t val = *(uint8_t*)colDataGetNumData(pColData, ri);
|
||||
uint8_t val = *(const uint8_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_USMALLINT: {
|
||||
uint16_t val = *(uint16_t*)colDataGetNumData(pColData, ri);
|
||||
uint16_t val = *(const uint16_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_UINT: {
|
||||
uint32_t val = *(uint32_t*)colDataGetNumData(pColData, ri);
|
||||
uint32_t val = *(const uint32_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_UBIGINT: {
|
||||
uint64_t val = *(uint64_t*)colDataGetNumData(pColData, ri);
|
||||
uint64_t val = *(const uint64_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
@ -241,53 +331,91 @@ _end:
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
const SSDataBlock* pInputBlock, const SNodeList* pCondCols, int32_t ri,
|
||||
SStreamNotifyEventSupp* sup) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SNode* node = NULL;
|
||||
cJSON* event = NULL;
|
||||
cJSON* fields = NULL;
|
||||
cJSON* cond = NULL;
|
||||
SStreamNotifyEvent item = {0};
|
||||
char windowId[32];
|
||||
static cJSON* createBasicAggNotifyEvent(const char* windowType, EStreamNotifyEventType eventType,
|
||||
const SSessionKey* pSessionKey) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
const char* eventTypeStr = NULL;
|
||||
cJSON* event = NULL;
|
||||
char windowId[32];
|
||||
|
||||
QUERY_CHECK_NULL(windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pSessionKey, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pInputBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pInputBlock->pDataBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pCondCols, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
qDebug("add stream notify event from event window, type: %s, start: %" PRId64 ", end: %" PRId64,
|
||||
(eventType == SNOTIFY_EVENT_WINDOW_OPEN) ? "WINDOW_OPEN" : "WINDOW_CLOSE", pSessionKey->win.skey,
|
||||
pSessionKey->win.ekey);
|
||||
if (eventType == SNOTIFY_EVENT_WINDOW_OPEN) {
|
||||
eventTypeStr = "WINDOW_OPEN";
|
||||
} else if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
|
||||
eventTypeStr = "WINDOW_CLOSE";
|
||||
} else if (eventType == SNOTIFY_EVENT_WINDOW_INVALIDATION) {
|
||||
eventTypeStr = "WINDOW_INVALIDATION";
|
||||
} else {
|
||||
QUERY_CHECK_CONDITION(false, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
}
|
||||
|
||||
qDebug("add stream notify event from %s Window, type: %s, start: %" PRId64 ", end: %" PRId64, windowType,
|
||||
eventTypeStr, pSessionKey->win.skey, pSessionKey->win.ekey);
|
||||
|
||||
event = cJSON_CreateObject();
|
||||
QUERY_CHECK_NULL(event, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
|
||||
|
||||
// add basic info
|
||||
streamNotifyGetEventWindowId(pSessionKey, windowId);
|
||||
if (eventType == SNOTIFY_EVENT_WINDOW_OPEN) {
|
||||
JSON_CHECK_ADD_ITEM(event, "eventType", cJSON_CreateStringReference("WINDOW_OPEN"));
|
||||
} else if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
|
||||
JSON_CHECK_ADD_ITEM(event, "eventType", cJSON_CreateStringReference("WINDOW_CLOSE"));
|
||||
}
|
||||
JSON_CHECK_ADD_ITEM(event, "eventType", cJSON_CreateStringReference(eventTypeStr));
|
||||
JSON_CHECK_ADD_ITEM(event, "eventTime", cJSON_CreateNumber(taosGetTimestampMs()));
|
||||
JSON_CHECK_ADD_ITEM(event, "windowId", cJSON_CreateStringReference(windowId));
|
||||
JSON_CHECK_ADD_ITEM(event, "windowType", cJSON_CreateStringReference("Event"));
|
||||
JSON_CHECK_ADD_ITEM(event, "windowId", cJSON_CreateString(windowId));
|
||||
JSON_CHECK_ADD_ITEM(event, "windowType", cJSON_CreateStringReference(windowType));
|
||||
JSON_CHECK_ADD_ITEM(event, "windowStart", cJSON_CreateNumber(pSessionKey->win.skey));
|
||||
if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
|
||||
JSON_CHECK_ADD_ITEM(event, "windowEnd", cJSON_CreateNumber(pSessionKey->win.ekey));
|
||||
if (eventType != SNOTIFY_EVENT_WINDOW_OPEN) {
|
||||
if (strcmp(windowType, "Time") == 0) {
|
||||
JSON_CHECK_ADD_ITEM(event, "windowEnd", cJSON_CreateNumber(pSessionKey->win.ekey + 1));
|
||||
} else {
|
||||
JSON_CHECK_ADD_ITEM(event, "windowEnd", cJSON_CreateNumber(pSessionKey->win.ekey));
|
||||
}
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
terrno = code;
|
||||
cJSON_Delete(event);
|
||||
event = NULL;
|
||||
}
|
||||
return event;
|
||||
}
|
||||
|
||||
int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
const SSDataBlock* pInputBlock, const SNodeList* pCondCols, int32_t ri,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
cJSON* event = NULL;
|
||||
cJSON* fields = NULL;
|
||||
cJSON* cond = NULL;
|
||||
const SNode* pNode = NULL;
|
||||
int32_t origSize = 0;
|
||||
int64_t startTime = 0;
|
||||
int64_t endTime = 0;
|
||||
SStreamNotifyEvent item = {0};
|
||||
|
||||
QUERY_CHECK_NULL(pInputBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pInputBlock->pDataBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pCondCols, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup->windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
startTime = taosGetMonoTimestampMs();
|
||||
event = createBasicAggNotifyEvent(sup->windowType, eventType, pSessionKey);
|
||||
QUERY_CHECK_NULL(event, code, lino, _end, terrno);
|
||||
|
||||
// create fields object to store matched column values
|
||||
fields = cJSON_CreateObject();
|
||||
QUERY_CHECK_NULL(fields, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
|
||||
FOREACH(node, pCondCols) {
|
||||
SColumnNode* pColDef = (SColumnNode*)node;
|
||||
SColumnInfoData* pColData = taosArrayGet(pInputBlock->pDataBlock, pColDef->slotId);
|
||||
code = jsonAddColumnField(pColDef->colName, pColData, ri, fields);
|
||||
FOREACH(pNode, pCondCols) {
|
||||
const SColumnNode* pColDef = (const SColumnNode*)pNode;
|
||||
const SColumnInfoData* pColData = taosArrayGet(pInputBlock->pDataBlock, pColDef->slotId);
|
||||
code = jsonAddColumnField(pColDef->colName, pColData->info.type, colDataIsNull_s(pColData, ri),
|
||||
colDataGetData(pColData, ri), fields);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
|
@ -297,16 +425,24 @@ int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionK
|
|||
JSON_CHECK_ADD_ITEM(cond, "conditionIndex", cJSON_CreateNumber(0));
|
||||
JSON_CHECK_ADD_ITEM(cond, "fieldValues", fields);
|
||||
fields = NULL;
|
||||
JSON_CHECK_ADD_ITEM(event, "triggerConditions", cond);
|
||||
JSON_CHECK_ADD_ITEM(event, "triggerCondition", cond);
|
||||
cond = NULL;
|
||||
|
||||
// convert json object to string value
|
||||
item.gid = pSessionKey->groupId;
|
||||
item.skey = pSessionKey->win.skey;
|
||||
item.isEnd = (eventType == SNOTIFY_EVENT_WINDOW_CLOSE);
|
||||
item.content = cJSON_PrintUnformatted(event);
|
||||
QUERY_CHECK_NULL(taosArrayPush(sup->pWindowEvents, &item), code, lino, _end, terrno);
|
||||
item.content = NULL;
|
||||
item.win = pSessionKey->win;
|
||||
item.eventType = eventType;
|
||||
item.pJson = event;
|
||||
event = NULL;
|
||||
origSize = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
item.pJson = NULL;
|
||||
|
||||
endTime = taosGetMonoTimestampMs();
|
||||
pNotifyEventStat->notifyEventAddTimes++;
|
||||
pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
|
||||
pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
|
||||
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
@ -325,50 +461,204 @@ _end:
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SSchemaWrapper* pSchemaWrapper,
|
||||
SStreamNotifyEventSupp* sup) {
|
||||
int32_t addStateAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
const SStateKeys* pCurState, const SStateKeys* pAnotherState, bool onlyUpdate,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SNode * node = NULL;
|
||||
cJSON* event = NULL;
|
||||
cJSON* result = NULL;
|
||||
int32_t origSize = 0;
|
||||
int64_t startTime = 0;
|
||||
int64_t endTime = 0;
|
||||
SStreamNotifyEvent item = {0};
|
||||
|
||||
QUERY_CHECK_NULL(pCurState, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup->windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
item.gid = pSessionKey->groupId;
|
||||
item.win = pSessionKey->win;
|
||||
item.eventType = eventType;
|
||||
// Check if the notify event exists for update
|
||||
if (onlyUpdate && taosHashGet(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE) == NULL) {
|
||||
goto _end;
|
||||
}
|
||||
|
||||
startTime = taosGetMonoTimestampMs();
|
||||
event = createBasicAggNotifyEvent(sup->windowType, eventType, pSessionKey);
|
||||
QUERY_CHECK_NULL(event, code, lino, _end, terrno);
|
||||
|
||||
// add state value
|
||||
if (eventType == SNOTIFY_EVENT_WINDOW_OPEN) {
|
||||
if (pAnotherState) {
|
||||
code = jsonAddColumnField("prevState", pAnotherState->type, pAnotherState->isNull, pAnotherState->pData, event);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
} else {
|
||||
code = jsonAddColumnField("prevState", pCurState->type, true, NULL, event);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
}
|
||||
code = jsonAddColumnField("curState", pCurState->type, pCurState->isNull, pCurState->pData, event);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
|
||||
if (pAnotherState) {
|
||||
code = jsonAddColumnField("nextState", pAnotherState->type, pAnotherState->isNull, pAnotherState->pData, event);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
} else {
|
||||
code = jsonAddColumnField("nextState", pCurState->type, true, NULL, event);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
}
|
||||
|
||||
item.pJson = event;
|
||||
event = NULL;
|
||||
origSize = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
item.pJson = NULL;
|
||||
|
||||
endTime = taosGetMonoTimestampMs();
|
||||
pNotifyEventStat->notifyEventAddTimes++;
|
||||
pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
|
||||
pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
|
||||
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
destroyStreamWindowEvent(&item);
|
||||
if (event != NULL) {
|
||||
cJSON_Delete(event);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t addNormalAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
cJSON* event = NULL;
|
||||
int32_t origSize = 0;
|
||||
int64_t startTime = 0;
|
||||
int64_t endTime = 0;
|
||||
SStreamNotifyEvent item = {0};
|
||||
|
||||
QUERY_CHECK_NULL(pSessionKey, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup->windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
startTime = taosGetMonoTimestampMs();
|
||||
event = createBasicAggNotifyEvent(sup->windowType, eventType, pSessionKey);
|
||||
QUERY_CHECK_NULL(event, code, lino, _end, terrno);
|
||||
|
||||
item.gid = pSessionKey->groupId;
|
||||
item.win = pSessionKey->win;
|
||||
item.eventType = eventType;
|
||||
item.pJson = event;
|
||||
event = NULL;
|
||||
origSize = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
item.pJson = NULL;
|
||||
|
||||
endTime = taosGetMonoTimestampMs();
|
||||
pNotifyEventStat->notifyEventAddTimes++;
|
||||
pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
|
||||
pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
|
||||
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
destroyStreamWindowEvent(&item);
|
||||
if (event != NULL) {
|
||||
cJSON_Delete(event);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t addIntervalAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
|
||||
return addNormalAggNotifyEvent(eventType, pSessionKey, sup, pNotifyEventStat);
|
||||
}
|
||||
|
||||
int32_t addSessionAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
|
||||
return addNormalAggNotifyEvent(eventType, pSessionKey, sup, pNotifyEventStat);
|
||||
}
|
||||
|
||||
int32_t addCountAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
|
||||
return addNormalAggNotifyEvent(eventType, pSessionKey, sup, pNotifyEventStat);
|
||||
}
|
||||
|
||||
int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SArray* pSessionKeys,
|
||||
const SSchemaWrapper* pSchemaWrapper, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
cJSON* result = NULL;
|
||||
int32_t origSize = 0;
|
||||
int64_t startTime = 0;
|
||||
int64_t endTime = 0;
|
||||
SStreamNotifyEvent item = {0};
|
||||
SColumnInfoData* pWstartCol = NULL;
|
||||
|
||||
QUERY_CHECK_NULL(pResultBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pSessionKeys, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pSchemaWrapper, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
qDebug("add %" PRId64 " stream notify results from window agg", pResultBlock->info.rows);
|
||||
startTime = taosGetMonoTimestampMs();
|
||||
origSize = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
|
||||
pWstartCol = taosArrayGet(pResultBlock->pDataBlock, 0);
|
||||
for (int32_t i = 0; i< pResultBlock->info.rows; ++i) {
|
||||
event = cJSON_CreateObject();
|
||||
QUERY_CHECK_NULL(event, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
|
||||
for (int32_t i = 0; i < pResultBlock->info.rows; ++i) {
|
||||
const SSessionKey* pSessionKey = taosArrayGet(pSessionKeys, i);
|
||||
item.gid = pSessionKey->groupId;
|
||||
item.win = pSessionKey->win;
|
||||
item.eventType = SNOTIFY_EVENT_WINDOW_CLOSE;
|
||||
SStreamNotifyEvent* pItem = taosHashGet(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE);
|
||||
if (pItem == NULL) {
|
||||
item.pJson = createBasicAggNotifyEvent(sup->windowType, SNOTIFY_EVENT_WINDOW_CLOSE, pSessionKey);
|
||||
QUERY_CHECK_NULL(item.pJson, code, lino, _end, terrno);
|
||||
if (strcmp(sup->windowType, "Event") == 0) {
|
||||
JSON_CHECK_ADD_ITEM(item.pJson, "triggerCondition", cJSON_CreateNull());
|
||||
} else if (strcmp(sup->windowType, "State") == 0) {
|
||||
JSON_CHECK_ADD_ITEM(item.pJson, "curState", cJSON_CreateNull());
|
||||
JSON_CHECK_ADD_ITEM(item.pJson, "nextState", cJSON_CreateNull());
|
||||
}
|
||||
code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
item.pJson = NULL;
|
||||
pItem = taosHashGet(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE);
|
||||
QUERY_CHECK_NULL(pItem, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
|
||||
}
|
||||
|
||||
// convert the result row into json
|
||||
result = cJSON_CreateObject();
|
||||
QUERY_CHECK_NULL(result, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
|
||||
for (int32_t j = 0; j < pSchemaWrapper->nCols; ++j) {
|
||||
SSchema *pCol = pSchemaWrapper->pSchema + j;
|
||||
SColumnInfoData *pColData = taosArrayGet(pResultBlock->pDataBlock, pCol->colId - 1);
|
||||
code = jsonAddColumnField(pCol->name, pColData, i, result);
|
||||
const SSchema* pCol = pSchemaWrapper->pSchema + j;
|
||||
const SColumnInfoData* pColData = taosArrayGet(pResultBlock->pDataBlock, pCol->colId - 1);
|
||||
code = jsonAddColumnField(pCol->name, pColData->info.type, colDataIsNull_s(pColData, i),
|
||||
colDataGetData(pColData, i), result);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
JSON_CHECK_ADD_ITEM(event, "result", result);
|
||||
JSON_CHECK_ADD_ITEM(pItem->pJson, "result", result);
|
||||
result = NULL;
|
||||
|
||||
item.gid = pResultBlock->info.id.groupId;
|
||||
item.skey = *(uint64_t*)colDataGetNumData(pWstartCol, i);
|
||||
item.content = cJSON_PrintUnformatted(event);
|
||||
code = taosHashPut(sup->pResultHashMap, &item.gid, sizeof(item.gid) + sizeof(item.skey), &item, sizeof(item));
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
item.content = NULL;
|
||||
|
||||
cJSON_Delete(event);
|
||||
event = NULL;
|
||||
}
|
||||
|
||||
endTime = taosGetMonoTimestampMs();
|
||||
pNotifyEventStat->notifyEventAddTimes++;
|
||||
pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
|
||||
pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
|
||||
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
|
@ -377,8 +667,38 @@ _end:
|
|||
if (result != NULL) {
|
||||
cJSON_Delete(result);
|
||||
}
|
||||
if (event != NULL) {
|
||||
cJSON_Delete(event);
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t addAggDeleteNotifyEvent(const SSDataBlock* pDeleteBlock, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SSessionKey sessionKey = {0};
|
||||
SColumnInfoData* pWstartCol = NULL;
|
||||
SColumnInfoData* pWendCol = NULL;
|
||||
SColumnInfoData* pGroupIdCol = NULL;
|
||||
|
||||
QUERY_CHECK_NULL(pDeleteBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
qDebug("add %" PRId64 " stream notify delete events from window agg", pDeleteBlock->info.rows);
|
||||
|
||||
pWstartCol = taosArrayGet(pDeleteBlock->pDataBlock, START_TS_COLUMN_INDEX);
|
||||
pWendCol = taosArrayGet(pDeleteBlock->pDataBlock, END_TS_COLUMN_INDEX);
|
||||
pGroupIdCol = taosArrayGet(pDeleteBlock->pDataBlock, GROUPID_COLUMN_INDEX);
|
||||
for (int32_t i = 0; i < pDeleteBlock->info.rows; ++i) {
|
||||
sessionKey.win.skey = *(int64_t*)colDataGetNumData(pWstartCol, i);
|
||||
sessionKey.win.ekey = *(int64_t*)colDataGetNumData(pWendCol, i);
|
||||
sessionKey.groupId = *(uint64_t*)colDataGetNumData(pGroupIdCol, i);
|
||||
code = addNormalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_INVALIDATION, &sessionKey, sup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
@ -418,97 +738,42 @@ _end:
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t streamNotifyFillTableName(const char* tableName, const SStreamNotifyEvent* pEvent,
|
||||
const SStreamNotifyEvent* pResult, char** pVal) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
static const char* prefix = "{\"tableName\":\"";
|
||||
uint64_t prefixLen = 0;
|
||||
uint64_t nameLen = 0;
|
||||
uint64_t eventLen = 0;
|
||||
uint64_t resultLen = 0;
|
||||
uint64_t valLen = 0;
|
||||
char* val = NULL;
|
||||
char* p = NULL;
|
||||
|
||||
QUERY_CHECK_NULL(tableName, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pEvent, code, lino , _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pVal, code, lino , _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
*pVal = NULL;
|
||||
prefixLen = strlen(prefix);
|
||||
nameLen = strlen(tableName);
|
||||
eventLen = strlen(pEvent->content);
|
||||
|
||||
if (pResult != NULL) {
|
||||
resultLen = strlen(pResult->content);
|
||||
valLen = VARSTR_HEADER_SIZE + prefixLen + nameLen + eventLen + resultLen;
|
||||
} else {
|
||||
valLen = VARSTR_HEADER_SIZE + prefixLen + nameLen + eventLen + 1;
|
||||
}
|
||||
val = taosMemoryMalloc(valLen);
|
||||
QUERY_CHECK_NULL(val, code, lino, _end, terrno);
|
||||
varDataSetLen(val, valLen - VARSTR_HEADER_SIZE);
|
||||
|
||||
p = varDataVal(val);
|
||||
TAOS_STRNCPY(p, prefix, prefixLen);
|
||||
p += prefixLen;
|
||||
TAOS_STRNCPY(p, tableName, nameLen);
|
||||
p += nameLen;
|
||||
*(p++) = '\"';
|
||||
TAOS_STRNCPY(p, pEvent->content, eventLen);
|
||||
*p = ',';
|
||||
|
||||
if (pResult != NULL) {
|
||||
p += eventLen - 1;
|
||||
TAOS_STRNCPY(p, pResult->content, resultLen);
|
||||
*p = ',';
|
||||
}
|
||||
*pVal = val;
|
||||
val = NULL;
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
if (val != NULL) {
|
||||
taosMemoryFreeClear(val);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup) {
|
||||
int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SColumnInfoData* pEventStrCol = NULL;
|
||||
int32_t nWindowEvents = 0;
|
||||
int32_t nWindowResults = 0;
|
||||
char* val = NULL;
|
||||
SColumnInfoData* pEventStrCol = NULL;
|
||||
int64_t startTime = 0;
|
||||
int64_t endTime = 0;
|
||||
void* pIter = NULL;
|
||||
|
||||
if (pTaskInfo == NULL || sup == NULL) {
|
||||
if (pTaskInfo == NULL || sup == NULL || sup->pEventBlock == NULL || pNotifyEventStat == NULL) {
|
||||
goto _end;
|
||||
}
|
||||
|
||||
QUERY_CHECK_NULL(sup->pEventBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
blockDataCleanup(sup->pEventBlock);
|
||||
nWindowEvents = taosArrayGetSize(sup->pWindowEvents);
|
||||
nWindowResults = taosHashGetSize(sup->pResultHashMap);
|
||||
qDebug("start to build stream notify event block, nWindowEvents: %d, nWindowResults: %d", nWindowEvents,
|
||||
nWindowResults);
|
||||
if (nWindowEvents == 0) {
|
||||
goto _end;
|
||||
}
|
||||
|
||||
code = blockDataEnsureCapacity(sup->pEventBlock, nWindowEvents);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
startTime = taosGetMonoTimestampMs();
|
||||
blockDataCleanup(sup->pEventBlock);
|
||||
nWindowEvents = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
qDebug("start to build stream notify event block, nWindowEvents: %d", nWindowEvents);
|
||||
|
||||
pEventStrCol = taosArrayGet(sup->pEventBlock->pDataBlock, NOTIFY_EVENT_STR_COLUMN_INDEX);
|
||||
QUERY_CHECK_NULL(pEventStrCol, code, lino, _end, terrno);
|
||||
|
||||
for (int32_t i = 0; i < nWindowEvents; ++i) {
|
||||
SStreamNotifyEvent* pResult = NULL;
|
||||
SStreamNotifyEvent* pEvent = taosArrayGet(sup->pWindowEvents, i);
|
||||
char* tableName = taosHashGet(sup->pTableNameHashMap, &pEvent->gid, sizeof(pEvent->gid));
|
||||
// Append all events content into data block.
|
||||
pIter = taosHashIterate(sup->pWindowEventHashMap, NULL);
|
||||
while (pIter) {
|
||||
const SStreamNotifyEvent* pEvent = (const SStreamNotifyEvent*)pIter;
|
||||
pIter = taosHashIterate(sup->pWindowEventHashMap, pIter);
|
||||
if (pEvent->eventType == SNOTIFY_EVENT_WINDOW_CLOSE && !cJSON_HasObjectItem(pEvent->pJson, "result")) {
|
||||
// current WINDOW_CLOSE event cannot be pushed yet due to watermark
|
||||
continue;
|
||||
}
|
||||
|
||||
// get name of the dest child table
|
||||
char* tableName = taosHashGet(sup->pTableNameHashMap, &pEvent->gid, sizeof(&pEvent->gid));
|
||||
if (tableName == NULL) {
|
||||
code = streamNotifyGetDestTableName(pTaskInfo, pEvent->gid, &tableName);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
@ -518,32 +783,73 @@ int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEvent
|
|||
tableName = taosHashGet(sup->pTableNameHashMap, &pEvent->gid, sizeof(pEvent->gid));
|
||||
QUERY_CHECK_NULL(tableName, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
|
||||
}
|
||||
if (pEvent->isEnd) {
|
||||
pResult = taosHashGet(sup->pResultHashMap, &pEvent->gid, sizeof(pEvent->gid) + sizeof(pEvent->skey));
|
||||
QUERY_CHECK_NULL(pResult, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
|
||||
}
|
||||
code = streamNotifyFillTableName(tableName, pEvent, pResult, &val);
|
||||
JSON_CHECK_ADD_ITEM(pEvent->pJson, "tableName", cJSON_CreateStringReference(tableName));
|
||||
|
||||
// convert the json object into string and append it into the block
|
||||
char* str = cJSON_PrintUnformatted(pEvent->pJson);
|
||||
QUERY_CHECK_NULL(str, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
|
||||
int32_t len = strlen(str);
|
||||
code = varColSetVarData(pEventStrCol, sup->pEventBlock->info.rows, str, len, false);
|
||||
cJSON_free(str);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
code = colDataSetVal(pEventStrCol, i, val, false);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
taosMemoryFreeClear(val);
|
||||
sup->pEventBlock->info.rows++;
|
||||
code = taosHashRemove(sup->pWindowEventHashMap, pEvent, NOTIFY_EVENT_KEY_SIZE);
|
||||
if (code == TSDB_CODE_NOT_FOUND) {
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
}
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (sup->pEventBlock->info.rows >= sup->pEventBlock->info.capacity) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (taosHashGetMemSize(sup->pTableNameHashMap) >= NOTIFY_EVENT_NAME_CACHE_LIMIT_MB * 1024 * 1024) {
|
||||
taosHashClear(sup->pTableNameHashMap);
|
||||
}
|
||||
|
||||
endTime = taosGetMonoTimestampMs();
|
||||
if (sup->pEventBlock->info.rows > 0) {
|
||||
pNotifyEventStat->notifyEventPushTimes++;
|
||||
pNotifyEventStat->notifyEventPushElems += sup->pEventBlock->info.rows;
|
||||
pNotifyEventStat->notifyEventPushCostSec += (endTime - startTime) / 1000.0;
|
||||
}
|
||||
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
if (val != NULL) {
|
||||
taosMemoryFreeClear(val);
|
||||
}
|
||||
if (sup != NULL) {
|
||||
taosArrayClearEx(sup->pWindowEvents, destroyStreamWindowEvent);
|
||||
taosHashClear(sup->pResultHashMap);
|
||||
if (pIter) {
|
||||
taosHashCancelIterate(sup->pWindowEventHashMap, pIter);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t removeOutdatedNotifyEvents(STimeWindowAggSupp* pTwSup, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
void* pIter = NULL;
|
||||
|
||||
if (pTwSup || sup == NULL || pNotifyEventStat == NULL) {
|
||||
goto _end;
|
||||
}
|
||||
|
||||
pIter = taosHashIterate(sup->pWindowEventHashMap, NULL);
|
||||
while (pIter) {
|
||||
const SStreamNotifyEvent* pEvent = (const SStreamNotifyEvent*)pIter;
|
||||
pIter = taosHashIterate(sup->pWindowEventHashMap, pIter);
|
||||
if (isOverdue(pEvent->win.ekey, pTwSup)) {
|
||||
code = taosHashRemove(sup->pWindowEventHashMap, pEvent, NOTIFY_EVENT_KEY_SIZE);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
}
|
||||
|
||||
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
|
|
@ -87,23 +87,48 @@ static int32_t buildIntervalSliceResult(SOperatorInfo* pOperator, SSDataBlock**
|
|||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
uint16_t opType = pOperator->operatorType;
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
bool addNotifyEvent = false;
|
||||
addNotifyEvent = IS_NORMAL_INTERVAL_OP(pOperator) &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
|
||||
doBuildDeleteResultImpl(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pState, pInfo->pDelWins, &pInfo->delIndex,
|
||||
pInfo->pDelRes);
|
||||
if (pInfo->pDelRes->info.rows != 0) {
|
||||
// process the rest of the data
|
||||
printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->pDelRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
doBuildStreamIntervalResult(pOperator, pInfo->streamAggSup.pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
|
||||
doBuildStreamIntervalResult(pOperator, pInfo->streamAggSup.pState, pInfo->binfo.pRes, &pInfo->groupResInfo,
|
||||
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
|
||||
if (pInfo->binfo.pRes->info.rows != 0) {
|
||||
printDataBlock(pInfo->binfo.pRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggResultNotifyEvent(pInfo->binfo.pRes, pNotifySup->pSessionKeys,
|
||||
pTaskInfo->streamInfo.notifyResultSchema, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->binfo.pRes;
|
||||
goto _end;
|
||||
}
|
||||
|
||||
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
|
||||
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(opType), GET_TASKID(pTaskInfo));
|
||||
(*ppRes) = pNotifySup->pEventBlock;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
|
@ -316,6 +341,14 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc
|
|||
code = setIntervalSliceOutputBuf(&curPoint, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator) &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
|
||||
SSessionKey key = {.win = curWin, .groupId = groupId};
|
||||
code = addIntervalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &key, &pInfo->basic.notifyEventSup,
|
||||
pTaskInfo->streamInfo.pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
resetIntervalSliceFunctionKey(pSup->pCtx, numOfOutput);
|
||||
if (pInfo->hasInterpoFunc && IS_VALID_WIN_KEY(prevPoint.winKey.win.skey) && curPoint.winKey.win.skey != curTs) {
|
||||
doStreamSliceInterpolation(prevPoint.pLastRow, curPoint.winKey.win.skey, curTs, pBlock, startPos, &pOperator->exprSupp, INTERVAL_SLICE_START, pInfo->pOffsetInfo);
|
||||
|
@ -652,8 +685,9 @@ int32_t createStreamIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiN
|
|||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamIntervalSliceReleaseState, streamIntervalSliceReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic);
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
if (downstream) {
|
||||
code = initIntervalSliceDownStream(downstream, &pInfo->streamAggSup, pPhyNode->type, pInfo->primaryTsIndex,
|
||||
&pInfo->twAggSup, &pInfo->basic, &pInfo->interval, pInfo->hasInterpoFunc);
|
||||
|
|
|
@ -150,7 +150,6 @@ void destroyStreamTimeSliceOperatorInfo(void* param) {
|
|||
&pInfo->groupResInfo);
|
||||
pInfo->pOperator = NULL;
|
||||
}
|
||||
destroyStreamBasicInfo(&pInfo->basic);
|
||||
colDataDestroy(&pInfo->twAggSup.timeWindowData);
|
||||
destroyStreamAggSupporter(&pInfo->streamAggSup);
|
||||
resetPrevAndNextWindow(pInfo->pFillSup);
|
||||
|
@ -2202,7 +2201,7 @@ int32_t createStreamTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
|
|||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamTimeSliceReleaseState, streamTimeSliceReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic);
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
if (downstream) {
|
||||
code = initTimeSliceDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
|
||||
|
|
|
@ -32,11 +32,6 @@
|
|||
#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL)
|
||||
|
||||
#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
|
||||
#define IS_NORMAL_SESSION_OP(op) \
|
||||
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \
|
||||
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
|
||||
|
||||
#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE)
|
||||
|
||||
#define DEAULT_DELETE_MARK INT64_MAX
|
||||
#define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState"
|
||||
|
@ -480,6 +475,8 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) {
|
|||
false);
|
||||
pInfo->pOperator = NULL;
|
||||
}
|
||||
|
||||
destroyStreamBasicInfo(&pInfo->basic);
|
||||
cleanupAggSup(&pInfo->aggSup);
|
||||
clearGroupResInfo(&pInfo->groupResInfo);
|
||||
taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);
|
||||
|
@ -917,7 +914,7 @@ int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SSta
|
|||
}
|
||||
|
||||
void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,
|
||||
SGroupResInfo* pGroupResInfo) {
|
||||
SGroupResInfo* pGroupResInfo, SArray* pSessionKeys) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
@ -991,6 +988,14 @@ void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBl
|
|||
}
|
||||
}
|
||||
|
||||
if (pSessionKeys) {
|
||||
SSessionKey key = {.groupId = groupId, .win = pRow->win};
|
||||
for (int32_t j = 0; j < pRow->numOfRows; ++j) {
|
||||
const void* px = taosArrayPush(pSessionKeys, &key);
|
||||
QUERY_CHECK_NULL(px, code, lino, _end, terrno);
|
||||
}
|
||||
}
|
||||
|
||||
pBlock->info.rows += pRow->numOfRows;
|
||||
}
|
||||
|
||||
|
@ -1005,19 +1010,20 @@ _end:
|
|||
}
|
||||
|
||||
void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
|
||||
SGroupResInfo* pGroupResInfo) {
|
||||
SGroupResInfo* pGroupResInfo, SArray* pSessionKeys) {
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
// set output datablock version
|
||||
pBlock->info.version = pTaskInfo->version;
|
||||
|
||||
blockDataCleanup(pBlock);
|
||||
taosArrayClear(pSessionKeys);
|
||||
if (!hasRemainResults(pGroupResInfo)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// clear the existed group id
|
||||
pBlock->info.id.groupId = 0;
|
||||
buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo);
|
||||
buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);
|
||||
}
|
||||
|
||||
static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
|
||||
|
@ -1150,6 +1156,14 @@ static int32_t doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS
|
|||
pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->stateStore, &winCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator) &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
|
||||
SSessionKey key = {.win = nextWin, .groupId = groupId};
|
||||
code = addIntervalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &key, &pInfo->basic.notifyEventSup,
|
||||
pTaskInfo->streamInfo.pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
pResult = (SResultRow*)pResPos->pRowBuff;
|
||||
|
||||
if (IS_FINAL_INTERVAL_OP(pOperator)) {
|
||||
|
@ -1371,7 +1385,10 @@ int32_t doStreamIntervalEncodeOpState(void** buf, int32_t len, SOperatorInfo* pO
|
|||
// 5.dataVersion
|
||||
tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);
|
||||
|
||||
// 6.checksum
|
||||
// 6.basicInfo
|
||||
tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
|
||||
|
||||
// 7.checksum
|
||||
if (buf) {
|
||||
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
|
||||
tlen += taosEncodeFixedU32(buf, cksum);
|
||||
|
@ -1387,18 +1404,20 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
int32_t lino = 0;
|
||||
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
void* pDataEnd = POINTER_SHIFT(buf, len);
|
||||
if (!pInfo) {
|
||||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
// 6.checksum
|
||||
// 7.checksum
|
||||
int32_t dataLen = len - sizeof(uint32_t);
|
||||
void* pCksum = POINTER_SHIFT(buf, dataLen);
|
||||
if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {
|
||||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
pDataEnd = pCksum;
|
||||
|
||||
// 1.pResultRowHashTable
|
||||
int32_t mapSize = 0;
|
||||
|
@ -1454,6 +1473,12 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
// 5.dataVersion
|
||||
buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);
|
||||
|
||||
// 6.basicInfo
|
||||
if (buf < pDataEnd) {
|
||||
code = decodeStreamBasicInfo(&buf, &pInfo->basic);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
|
@ -1503,8 +1528,12 @@ _end:
|
|||
static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
|
||||
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
uint16_t opType = pOperator->operatorType;
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
bool addNotifyEvent = false;
|
||||
|
||||
// check if query task is closed or not
|
||||
if (isTaskKilled(pTaskInfo)) {
|
||||
|
@ -1512,6 +1541,8 @@ static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes
|
|||
return code;
|
||||
}
|
||||
|
||||
addNotifyEvent = IS_NORMAL_INTERVAL_OP(pOperator) &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
|
||||
if (IS_FINAL_INTERVAL_OP(pOperator)) {
|
||||
doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
|
||||
if (pInfo->pPullDataRes->info.rows != 0) {
|
||||
|
@ -1526,17 +1557,42 @@ static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes
|
|||
if (pInfo->pDelRes->info.rows != 0) {
|
||||
// process the rest of the data
|
||||
printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->pDelRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
|
||||
doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo,
|
||||
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
|
||||
if (pInfo->binfo.pRes->info.rows != 0) {
|
||||
printDataBlock(pInfo->binfo.pRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggResultNotifyEvent(pInfo->binfo.pRes, pNotifySup->pSessionKeys,
|
||||
pTaskInfo->streamInfo.notifyResultSchema, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->binfo.pRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
|
||||
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
(*ppRes) = pNotifySup->pEventBlock;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
}
|
||||
(*ppRes) = NULL;
|
||||
return code;
|
||||
}
|
||||
|
@ -2023,7 +2079,7 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN
|
|||
pInfo->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;
|
||||
pInfo->pState->pResultRowStore.resultRowPut = putResultRowToBuf;
|
||||
pInfo->pState->pExprSupp = &pOperator->exprSupp;
|
||||
|
||||
|
||||
code =
|
||||
pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,
|
||||
compareTs, pInfo->pState, pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo),
|
||||
|
@ -2069,6 +2125,10 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN
|
|||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
}
|
||||
setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
|
||||
pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
|
||||
pInfo->basic.primaryPkIndex = -1;
|
||||
|
@ -2120,6 +2180,8 @@ void destroyStreamSessionAggOperatorInfo(void* param) {
|
|||
&pInfo->groupResInfo);
|
||||
pInfo->pOperator = NULL;
|
||||
}
|
||||
|
||||
destroyStreamBasicInfo(&pInfo->basic);
|
||||
destroyStreamAggSupporter(&pInfo->streamAggSup);
|
||||
cleanupExprSupp(&pInfo->scalarSupp);
|
||||
clearGroupResInfo(&pInfo->groupResInfo);
|
||||
|
@ -2248,8 +2310,8 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, in
|
|||
}
|
||||
|
||||
if (stateType == STREAM_STATE_BUFF_SORT) {
|
||||
pSup->pState->pFileState = NULL;
|
||||
code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize,
|
||||
pSup->pState->pFileState = NULL;
|
||||
code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize,
|
||||
funResSize, sesionTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr,
|
||||
pHandle->checkpointId, stateType, &pSup->pState->pFileState);
|
||||
} else if (stateType == STREAM_STATE_BUFF_HASH_SORT || stateType == STREAM_STATE_BUFF_HASH_SEARCH) {
|
||||
|
@ -2309,24 +2371,23 @@ bool inWinRange(STimeWindow* range, STimeWindow* cur) {
|
|||
void clearOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { pAPI->streamStateClearBuff(pState, pPos); }
|
||||
|
||||
int32_t setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
|
||||
SResultWindowInfo* pCurWin) {
|
||||
SResultWindowInfo* pCurWin, int32_t* pWinCode) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
pCurWin->sessionWin.groupId = groupId;
|
||||
pCurWin->sessionWin.win.skey = startTs;
|
||||
pCurWin->sessionWin.win.ekey = endTs;
|
||||
int32_t size = pAggSup->resultRowSize;
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap,
|
||||
(void**)&pCurWin->pStatePos, &size, &winCode);
|
||||
(void**)&pCurWin->pStatePos, &size, pWinCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) {
|
||||
winCode = TSDB_CODE_FAILED;
|
||||
if (*pWinCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) {
|
||||
*pWinCode = TSDB_CODE_FAILED;
|
||||
clearOutputBuf(pAggSup->pState, pCurWin->pStatePos, &pAggSup->pSessionAPI->stateStore);
|
||||
}
|
||||
|
||||
if (winCode == TSDB_CODE_SUCCESS) {
|
||||
if (*pWinCode == TSDB_CODE_SUCCESS) {
|
||||
pCurWin->isOutput = true;
|
||||
if (pCurWin->pStatePos->needFree) {
|
||||
pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->sessionWin);
|
||||
|
@ -2692,9 +2753,17 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
|
|||
continue;
|
||||
}
|
||||
SResultWindowInfo winInfo = {0};
|
||||
code = setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo);
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
code = setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo, &winCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_SESSION_OP(pOperator) &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
|
||||
code = addSessionAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &winInfo.sessionWin, &pInfo->basic.notifyEventSup,
|
||||
pTaskInfo->streamInfo.pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
// coverity scan error
|
||||
if (!winInfo.pStatePos) {
|
||||
continue;
|
||||
|
@ -2908,7 +2977,9 @@ static int32_t rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray,
|
|||
|
||||
if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pWinKey->win, &childWin.sessionWin.win)) {
|
||||
if (num == 0) {
|
||||
code = setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin);
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
code = setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin,
|
||||
&winCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
parentWin.sessionWin = childWin.sessionWin;
|
||||
|
@ -3051,7 +3122,7 @@ void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayL
|
|||
}
|
||||
|
||||
int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,
|
||||
SGroupResInfo* pGroupResInfo) {
|
||||
SGroupResInfo* pGroupResInfo, SArray* pSessionKeys) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
@ -3131,6 +3202,13 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa
|
|||
}
|
||||
}
|
||||
|
||||
if (pSessionKeys) {
|
||||
for (int32_t j = 0; j < pRow->numOfRows; ++j) {
|
||||
const void* px = taosArrayPush(pSessionKeys, pKey);
|
||||
QUERY_CHECK_NULL(px, code, lino, _end, terrno);
|
||||
}
|
||||
}
|
||||
|
||||
pBlock->info.dataLoad = 1;
|
||||
pBlock->info.rows += pRow->numOfRows;
|
||||
}
|
||||
|
@ -3144,7 +3222,8 @@ _end:
|
|||
return code;
|
||||
}
|
||||
|
||||
void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock) {
|
||||
void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock,
|
||||
SArray* pSessionKeys) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
@ -3152,6 +3231,7 @@ void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo*
|
|||
pBlock->info.version = pTaskInfo->version;
|
||||
|
||||
blockDataCleanup(pBlock);
|
||||
taosArrayClear(pSessionKeys);
|
||||
if (!hasRemainResults(pGroupResInfo)) {
|
||||
cleanupGroupResInfo(pGroupResInfo);
|
||||
goto _end;
|
||||
|
@ -3159,7 +3239,7 @@ void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo*
|
|||
|
||||
// clear the existed group id
|
||||
pBlock->info.id.groupId = 0;
|
||||
code = buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo);
|
||||
code = buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (pBlock->info.rows == 0) {
|
||||
|
@ -3174,23 +3254,60 @@ _end:
|
|||
|
||||
static int32_t buildSessionResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
SOptrBasicInfo* pBInfo = &pInfo->binfo;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
bool addNotifyEvent = false;
|
||||
addNotifyEvent = IS_NORMAL_SESSION_OP(pOperator) &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
|
||||
doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->pDelRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
|
||||
doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes,
|
||||
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
|
||||
if (pBInfo->pRes->info.rows > 0) {
|
||||
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
// Adjust the window end time based on the Session Window gap
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pNotifySup->pSessionKeys); ++i) {
|
||||
SSessionKey* pKey = taosArrayGet(pNotifySup->pSessionKeys, i);
|
||||
pKey->win.ekey += pAggSup->gap;
|
||||
}
|
||||
code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
|
||||
pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pBInfo->pRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
|
||||
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
(*ppRes) = pNotifySup->pEventBlock;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
}
|
||||
(*ppRes) = NULL;
|
||||
return code;
|
||||
}
|
||||
|
@ -3295,7 +3412,10 @@ int32_t doStreamSessionEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOp
|
|||
// 4.dataVersion
|
||||
tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);
|
||||
|
||||
// 5.checksum
|
||||
// 5.basicInfo
|
||||
tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
|
||||
|
||||
// 6.checksum
|
||||
if (isParent) {
|
||||
if (buf) {
|
||||
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
|
||||
|
@ -3313,13 +3433,14 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe
|
|||
int32_t lino = 0;
|
||||
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
void* pDataEnd = POINTER_SHIFT(buf, len);
|
||||
if (!pInfo) {
|
||||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
|
||||
// 5.checksum
|
||||
// 6.checksum
|
||||
if (isParent) {
|
||||
int32_t dataLen = len - sizeof(uint32_t);
|
||||
void* pCksum = POINTER_SHIFT(buf, dataLen);
|
||||
|
@ -3328,6 +3449,7 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe
|
|||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
pDataEnd = pCksum;
|
||||
}
|
||||
|
||||
// 1.streamAggSup.pResultRows
|
||||
|
@ -3366,6 +3488,12 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe
|
|||
(*ppBuf) = buf;
|
||||
}
|
||||
|
||||
// 5.basicInfo
|
||||
if (buf < pDataEnd) {
|
||||
code = decodeStreamBasicInfo(&buf, &pInfo->basic);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
|
@ -3926,6 +4054,9 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode
|
|||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
if (downstream) {
|
||||
pInfo->basic.primaryPkIndex = -1;
|
||||
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
|
||||
|
@ -4162,6 +4293,10 @@ int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhys
|
|||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionSemiReloadState);
|
||||
}
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), pPhyNode->type, false, OP_NOT_OPENED, pInfo,
|
||||
pTaskInfo);
|
||||
|
||||
|
@ -4230,6 +4365,8 @@ void destroyStreamStateOperatorInfo(void* param) {
|
|||
&pInfo->groupResInfo);
|
||||
pInfo->pOperator = NULL;
|
||||
}
|
||||
|
||||
destroyStreamBasicInfo(&pInfo->basic);
|
||||
destroyStreamAggSupporter(&pInfo->streamAggSup);
|
||||
clearGroupResInfo(&pInfo->groupResInfo);
|
||||
taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);
|
||||
|
@ -4284,12 +4421,39 @@ bool compareWinStateKey(SStateKeys* left, SStateKeys* right) {
|
|||
return compareVal(left->pData, right);
|
||||
}
|
||||
|
||||
static void getNextStateWin(const SStreamAggSupporter* pAggSup, SStateWindowInfo* pNextWin, bool asc) {
|
||||
SStreamStateCur* pCur = NULL;
|
||||
|
||||
if (pAggSup == NULL || pNextWin == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (asc)
|
||||
pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);
|
||||
else
|
||||
pCur = pAggSup->stateStore.streamStateSessionSeekKeyPrev(pAggSup->pState, &pNextWin->winInfo.sessionWin);
|
||||
int32_t nextSize = pAggSup->resultRowSize;
|
||||
int32_t winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,
|
||||
(void**)&pNextWin->winInfo.pStatePos, &nextSize);
|
||||
if (winCode != TSDB_CODE_SUCCESS) {
|
||||
SET_SESSION_WIN_INVALID(pNextWin->winInfo);
|
||||
} else {
|
||||
pNextWin->pStateKey =
|
||||
(SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));
|
||||
pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);
|
||||
pNextWin->pStateKey->type = pAggSup->stateKeyType;
|
||||
pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);
|
||||
pNextWin->pStateKey->isNull = false;
|
||||
pNextWin->winInfo.isOutput = true;
|
||||
}
|
||||
pAggSup->stateStore.streamStateFreeCur(pCur);
|
||||
}
|
||||
|
||||
int32_t getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SStateWindowInfo* pCurWin,
|
||||
SStateWindowInfo* pNextWin) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SStreamStateCur* pCur = NULL;
|
||||
int32_t size = pAggSup->resultRowSize;
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
int32_t size = pAggSup->resultRowSize;
|
||||
pCurWin->winInfo.sessionWin.groupId = pKey->groupId;
|
||||
pCurWin->winInfo.sessionWin.win.skey = pKey->win.skey;
|
||||
pCurWin->winInfo.sessionWin.win.ekey = pKey->win.ekey;
|
||||
|
@ -4313,24 +4477,9 @@ int32_t getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey,
|
|||
pCurWin->winInfo.sessionWin.win.ekey);
|
||||
|
||||
pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;
|
||||
pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);
|
||||
int32_t nextSize = pAggSup->resultRowSize;
|
||||
int32_t winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,
|
||||
(void**)&pNextWin->winInfo.pStatePos, &nextSize);
|
||||
if (winCode != TSDB_CODE_SUCCESS) {
|
||||
SET_SESSION_WIN_INVALID(pNextWin->winInfo);
|
||||
} else {
|
||||
pNextWin->pStateKey =
|
||||
(SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));
|
||||
pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);
|
||||
pNextWin->pStateKey->type = pAggSup->stateKeyType;
|
||||
pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);
|
||||
pNextWin->pStateKey->isNull = false;
|
||||
pNextWin->winInfo.isOutput = true;
|
||||
}
|
||||
getNextStateWin(pAggSup, pNextWin, true);
|
||||
|
||||
_end:
|
||||
pAggSup->stateStore.streamStateFreeCur(pCur);
|
||||
qDebug("===stream===get state next win buff. skey:%" PRId64 ", endkey:%" PRId64,
|
||||
pNextWin->winInfo.sessionWin.win.skey, pNextWin->winInfo.sessionWin.win.ekey);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
@ -4340,9 +4489,8 @@ _end:
|
|||
}
|
||||
|
||||
int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,
|
||||
SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin) {
|
||||
int32_t size = pAggSup->resultRowSize;
|
||||
SStreamStateCur* pCur = NULL;
|
||||
SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin, int32_t* pWinCode) {
|
||||
int32_t size = pAggSup->resultRowSize;
|
||||
pCurWin->winInfo.sessionWin.groupId = groupId;
|
||||
pCurWin->winInfo.sessionWin.win.skey = ts;
|
||||
pCurWin->winInfo.sessionWin.win.ekey = ts;
|
||||
|
@ -4390,29 +4538,16 @@ int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
|
|||
}
|
||||
}
|
||||
|
||||
*pWinCode = winCode;
|
||||
|
||||
qDebug("===stream===set state cur win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,
|
||||
pCurWin->winInfo.sessionWin.win.ekey);
|
||||
|
||||
pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;
|
||||
pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);
|
||||
int32_t nextSize = pAggSup->resultRowSize;
|
||||
winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,
|
||||
(void**)&pNextWin->winInfo.pStatePos, &nextSize);
|
||||
if (winCode != TSDB_CODE_SUCCESS) {
|
||||
SET_SESSION_WIN_INVALID(pNextWin->winInfo);
|
||||
} else {
|
||||
pNextWin->pStateKey =
|
||||
(SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));
|
||||
pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);
|
||||
pNextWin->pStateKey->type = pAggSup->stateKeyType;
|
||||
pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);
|
||||
pNextWin->pStateKey->isNull = false;
|
||||
pNextWin->winInfo.isOutput = true;
|
||||
}
|
||||
getNextStateWin(pAggSup, pNextWin, true);
|
||||
qDebug("===stream===set state next win buff. skey:%" PRId64 ", endkey:%" PRId64,
|
||||
pNextWin->winInfo.sessionWin.win.skey, pNextWin->winInfo.sessionWin.win.ekey);
|
||||
_end:
|
||||
pAggSup->stateStore.streamStateFreeCur(pCur);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
|
@ -4472,7 +4607,7 @@ static bool isWinResult(SSessionKey* pKey, SSHashObj* pSeUpdate, SSHashObj* pRes
|
|||
if (tSimpleHashGet(pSeUpdate, &checkKey, sizeof(SSessionKey)) != NULL) {
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
if (tSimpleHashGet(pResults, &checkKey, sizeof(SSessionKey)) != NULL) {
|
||||
return true;
|
||||
}
|
||||
|
@ -4493,6 +4628,8 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
SResultRow* pResult = NULL;
|
||||
int32_t winRows = 0;
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
|
||||
pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
|
||||
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
|
||||
|
@ -4528,9 +4665,31 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
bool allEqual = true;
|
||||
SStateWindowInfo curWin = {0};
|
||||
SStateWindowInfo nextWin = {0};
|
||||
code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin);
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin, &winCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (winCode != TSDB_CODE_SUCCESS && pTaskInfo->streamInfo.eventTypes) {
|
||||
SStateWindowInfo prevWin = {.winInfo.sessionWin = curWin.winInfo.sessionWin};
|
||||
getNextStateWin(pAggSup, &prevWin, false);
|
||||
qDebug("===stream===get state prev win buff. skey:%" PRId64 ", endkey:%" PRId64,
|
||||
prevWin.winInfo.sessionWin.win.skey, prevWin.winInfo.sessionWin.win.ekey);
|
||||
releaseOutputBuf(pAggSup->pState, prevWin.winInfo.pStatePos, &pAPI->stateStore);
|
||||
// For ordered data, the previous window's closure did not record the corresponding state values, so they need to
|
||||
// be added here.
|
||||
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE) &&
|
||||
IS_VALID_SESSION_WIN(prevWin.winInfo)) {
|
||||
code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &prevWin.winInfo.sessionWin, prevWin.pStateKey,
|
||||
curWin.pStateKey, true, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
|
||||
code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, curWin.pStateKey,
|
||||
prevWin.pStateKey, false, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
}
|
||||
|
||||
if (isWinResult(&nextWin.winInfo.sessionWin, pSeUpdated, pAggSup->pResultRows) == false) {
|
||||
releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore);
|
||||
}
|
||||
|
@ -4578,6 +4737,14 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
// If this is a windown recalculation, add the corresponding state values here since the next window may not require
|
||||
// recalculation.
|
||||
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {
|
||||
code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &curWin.winInfo.sessionWin, curWin.pStateKey,
|
||||
nextWin.pStateKey, false, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
}
|
||||
|
||||
_end:
|
||||
|
@ -4621,7 +4788,10 @@ int32_t doStreamStateEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
|
|||
// 4.dataVersion
|
||||
tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);
|
||||
|
||||
// 5.checksum
|
||||
// 5.basicInfo
|
||||
tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
|
||||
|
||||
// 6.checksum
|
||||
if (isParent) {
|
||||
if (buf) {
|
||||
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
|
||||
|
@ -4640,12 +4810,13 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
int32_t lino = 0;
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
void* pDataEnd = POINTER_SHIFT(buf, len);
|
||||
if (!pInfo) {
|
||||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
// 5.checksum
|
||||
// 6.checksum
|
||||
if (isParent) {
|
||||
int32_t dataLen = len - sizeof(uint32_t);
|
||||
void* pCksum = POINTER_SHIFT(buf, dataLen);
|
||||
|
@ -4654,6 +4825,7 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
pDataEnd = pCksum;
|
||||
}
|
||||
|
||||
// 1.streamAggSup.pResultRows
|
||||
|
@ -4693,6 +4865,12 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
(*ppBuf) = buf;
|
||||
}
|
||||
|
||||
// 5.basicInfo
|
||||
if (buf < pDataEnd) {
|
||||
code = decodeStreamBasicInfo(&buf, &pInfo->basic);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
|
@ -4720,23 +4898,53 @@ void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) {
|
|||
|
||||
static int32_t buildStateResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
|
||||
SOptrBasicInfo* pBInfo = &pInfo->binfo;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
bool addNotifyEvent = false;
|
||||
addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
|
||||
doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->pDelRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes);
|
||||
doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes,
|
||||
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
|
||||
if (pBInfo->pRes->info.rows > 0) {
|
||||
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
|
||||
pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pBInfo->pRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
|
||||
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
(*ppRes) = pNotifySup->pEventBlock;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
}
|
||||
(*ppRes) = NULL;
|
||||
return code;
|
||||
}
|
||||
|
@ -5122,6 +5330,10 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
|
|||
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAggNext, NULL, destroyStreamStateOperatorInfo,
|
||||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
|
||||
&pInfo->twAggSup, &pInfo->basic);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
@ -5331,8 +5543,9 @@ _end:
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,
|
||||
SReadHandle* pHandle, SOperatorInfo** pOptrInfo) {
|
||||
static int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
|
||||
SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
|
||||
SOperatorInfo** pOptrInfo) {
|
||||
QRY_PARAM_CHECK(pOptrInfo);
|
||||
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
@ -5455,6 +5668,9 @@ static int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream,
|
|||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
pInfo->recvGetAll = false;
|
||||
|
||||
code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);
|
||||
|
|
|
@ -11420,7 +11420,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
|
|||
!hasTbnameFunction(pSelect->pPartitionByList) && pSelect->pWindow != NULL &&
|
||||
pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW) {
|
||||
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
|
||||
"Event window for stream on super table must patitioned by table name");
|
||||
"Event window for stream on super table must partitioned by table name");
|
||||
}
|
||||
|
||||
if (pSelect->pWindow != NULL && pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW &&
|
||||
|
|
|
@ -562,6 +562,33 @@ SStreamStateCur* sessionWinStateSeekKeyCurrentPrev(SStreamFileState* pFileState,
|
|||
pCur->pStreamFileState = pFileState;
|
||||
return pCur;
|
||||
}
|
||||
|
||||
SStreamStateCur* sessionWinStateSeekKeyPrev(SStreamFileState *pFileState, const SSessionKey *pWinKey) {
|
||||
SArray* pWinStates = NULL;
|
||||
int32_t index = -1;
|
||||
SStreamStateCur *pCur = seekKeyCurrentPrev_buff(pFileState, pWinKey, &pWinStates, &index);
|
||||
if (pCur) {
|
||||
int32_t cmpRes= sessionStateRangeKeyCompare(pWinKey, pWinStates, index);
|
||||
if (cmpRes > 0) {
|
||||
return pCur;
|
||||
} else if (cmpRes == 0 && index > 0) {
|
||||
sessionWinStateMoveToPrev(pCur);
|
||||
return pCur;
|
||||
}
|
||||
streamStateFreeCur(pCur);
|
||||
pCur = NULL;
|
||||
}
|
||||
|
||||
void* pFileStore = getStateFileStore(pFileState);
|
||||
pCur = streamStateSessionSeekKeyPrev_rocksdb(pFileStore, pWinKey);
|
||||
if (!pCur) {
|
||||
return NULL;
|
||||
}
|
||||
pCur->buffIndex = -1;
|
||||
pCur->pStreamFileState = pFileState;
|
||||
return pCur;
|
||||
}
|
||||
|
||||
static void transformCursor(SStreamFileState* pFileState, SStreamStateCur* pCur) {
|
||||
if (!pCur) {
|
||||
return;
|
||||
|
@ -747,6 +774,15 @@ void sessionWinStateMoveToNext(SStreamStateCur* pCur) {
|
|||
}
|
||||
}
|
||||
|
||||
void sessionWinStateMoveToPrev(SStreamStateCur* pCur) {
|
||||
qTrace("move cursor to prev");
|
||||
if (pCur && pCur->buffIndex >= 1) {
|
||||
pCur->buffIndex--;
|
||||
} else {
|
||||
streamStateCurPrev_rocksdb(pCur);
|
||||
}
|
||||
}
|
||||
|
||||
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey,
|
||||
range_cmpr_fn cmpFn) {
|
||||
SStreamStateCur* pCur = sessionWinStateSeekKeyCurrentPrev(pFileState, key);
|
||||
|
|
|
@ -440,6 +440,10 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, cons
|
|||
return sessionWinStateSeekKeyCurrentNext(pState->pFileState, key);
|
||||
}
|
||||
|
||||
SStreamStateCur *streamStateSessionSeekKeyPrev(SStreamState *pState, const SSessionKey *key) {
|
||||
return sessionWinStateSeekKeyPrev(pState->pFileState, key);
|
||||
}
|
||||
|
||||
SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key) {
|
||||
return sessionWinStateSeekKeyNext(pState->pFileState, key);
|
||||
}
|
||||
|
|
|
@ -331,6 +331,8 @@ void tFreeStreamTask(void* pParam) {
|
|||
taosMemoryFreeClear(pTask->notifyInfo.stbFullName);
|
||||
tDeleteSchemaWrapper(pTask->notifyInfo.pSchemaWrapper);
|
||||
|
||||
pTask->notifyEventStat = (STaskNotifyEventStat){0};
|
||||
|
||||
taosMemoryFree(pTask);
|
||||
stDebug("s-task:0x%x free task completed", taskId);
|
||||
}
|
||||
|
@ -988,6 +990,7 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc)
|
|||
|
||||
pDst->startTime = pSrc->startTime;
|
||||
pDst->hTaskId = pSrc->hTaskId;
|
||||
pDst->notifyEventStat = pSrc->notifyEventStat;
|
||||
}
|
||||
|
||||
STaskStatusEntry streamTaskGetStatusEntry(SStreamTask* pTask) {
|
||||
|
@ -1016,6 +1019,7 @@ STaskStatusEntry streamTaskGetStatusEntry(SStreamTask* pTask) {
|
|||
.outputThroughput = SIZE_IN_KiB(pExecInfo->outputThroughput),
|
||||
.startCheckpointId = pExecInfo->startCheckpointId,
|
||||
.startCheckpointVer = pExecInfo->startCheckpointVer,
|
||||
.notifyEventStat = pTask->notifyEventStat,
|
||||
};
|
||||
return entry;
|
||||
}
|
||||
|
|
|
@ -0,0 +1,71 @@
|
|||
{
|
||||
"filetype": "insert",
|
||||
"cfgdir": "/etc/taos",
|
||||
"host": "127.0.0.1",
|
||||
"port": 6030,
|
||||
"user": "root",
|
||||
"password": "taosdata",
|
||||
"thread_count": 4,
|
||||
"create_table_thread_count": 4,
|
||||
"result_file": "./insert_res.txt",
|
||||
"confirm_parameter_prompt": "no",
|
||||
"num_of_records_per_req": 10000,
|
||||
"prepared_rand": 10000,
|
||||
"chinese": "no",
|
||||
"escape_character": "yes",
|
||||
"continue_if_fail": "no",
|
||||
"databases": [
|
||||
{
|
||||
"dbinfo": {
|
||||
"name": "test",
|
||||
"drop": "no",
|
||||
"vgroups": 4,
|
||||
"precision": "ms"
|
||||
},
|
||||
"super_tables": [
|
||||
{
|
||||
"name": "st",
|
||||
"child_table_exists": "no",
|
||||
"childtable_count": 5,
|
||||
"childtable_prefix": "ct",
|
||||
"auto_create_table": "yes",
|
||||
"batch_create_tbl_num": 5,
|
||||
"data_source": "rand",
|
||||
"insert_mode": "taosc",
|
||||
"non_stop_mode": "no",
|
||||
"line_protocol": "line",
|
||||
"insert_rows": 10000,
|
||||
"childtable_limit": 0,
|
||||
"childtable_offset": 0,
|
||||
"interlace_rows": 50,
|
||||
"insert_interval": 10,
|
||||
"partial_col_num": 0,
|
||||
"timestamp_step": 500,
|
||||
"start_timestamp": "2025-01-13 17:30:00.000",
|
||||
"sample_format": "csv",
|
||||
"sample_file": "./sample.csv",
|
||||
"use_sample_ts": "no",
|
||||
"tags_file": "",
|
||||
"columns": [
|
||||
{"type": "TINYINT", "name": "c0"},
|
||||
{"type": "SMALLINT", "name": "c1"},
|
||||
{"type": "INT", "name": "c2"},
|
||||
{"type": "BIGINT", "name": "c3"},
|
||||
{"type": "DOUBLE", "name": "c4"},
|
||||
{"type": "FLOAT", "name": "c5"},
|
||||
{"type": "BOOL", "name": "c6"},
|
||||
{"type": "VARCHAR", "name": "c7", "len": 10},
|
||||
{"type": "NCHAR", "name": "c8", "len": 10},
|
||||
{"type": "UTINYINT", "name": "c9"},
|
||||
{"type": "USMALLINT", "name": "c10"},
|
||||
{"type": "UINT", "name": "c11"},
|
||||
{"type": "UBIGINT", "name": "c12"}
|
||||
],
|
||||
"tags": [
|
||||
{"type": "INT", "name": "groupid", "max": 100, "min": 1}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,72 @@
|
|||
{
|
||||
"filetype": "insert",
|
||||
"cfgdir": "/etc/taos",
|
||||
"host": "127.0.0.1",
|
||||
"port": 6030,
|
||||
"user": "root",
|
||||
"password": "taosdata",
|
||||
"thread_count": 4,
|
||||
"create_table_thread_count": 4,
|
||||
"result_file": "./insert_res.txt",
|
||||
"confirm_parameter_prompt": "no",
|
||||
"num_of_records_per_req": 10000,
|
||||
"prepared_rand": 10000,
|
||||
"chinese": "no",
|
||||
"escape_character": "yes",
|
||||
"continue_if_fail": "no",
|
||||
"databases": [
|
||||
{
|
||||
"dbinfo": {
|
||||
"name": "test",
|
||||
"drop": "no",
|
||||
"vgroups": 4,
|
||||
"precision": "ms"
|
||||
},
|
||||
"super_tables": [
|
||||
{
|
||||
"name": "st",
|
||||
"child_table_exists": "no",
|
||||
"childtable_count": 5,
|
||||
"childtable_prefix": "ct",
|
||||
"auto_create_table": "yes",
|
||||
"batch_create_tbl_num": 5,
|
||||
"data_source": "rand",
|
||||
"insert_mode": "taosc",
|
||||
"non_stop_mode": "no",
|
||||
"line_protocol": "line",
|
||||
"insert_rows": 10000,
|
||||
"disorder_ratio": 10,
|
||||
"childtable_limit": 0,
|
||||
"childtable_offset": 0,
|
||||
"interlace_rows": 50,
|
||||
"insert_interval": 10,
|
||||
"partial_col_num": 0,
|
||||
"timestamp_step": 500,
|
||||
"start_timestamp": "2025-01-13 17:30:00.000",
|
||||
"sample_format": "csv",
|
||||
"sample_file": "./sample.csv",
|
||||
"use_sample_ts": "no",
|
||||
"tags_file": "",
|
||||
"columns": [
|
||||
{"type": "TINYINT", "name": "c0"},
|
||||
{"type": "SMALLINT", "name": "c1"},
|
||||
{"type": "INT", "name": "c2"},
|
||||
{"type": "BIGINT", "name": "c3"},
|
||||
{"type": "DOUBLE", "name": "c4"},
|
||||
{"type": "FLOAT", "name": "c5"},
|
||||
{"type": "BOOL", "name": "c6"},
|
||||
{"type": "VARCHAR", "name": "c7", "len": 10},
|
||||
{"type": "NCHAR", "name": "c8", "len": 10},
|
||||
{"type": "UTINYINT", "name": "c9"},
|
||||
{"type": "USMALLINT", "name": "c10"},
|
||||
{"type": "UINT", "name": "c11"},
|
||||
{"type": "UBIGINT", "name": "c12"}
|
||||
],
|
||||
"tags": [
|
||||
{"type": "INT", "name": "groupid", "max": 100, "min": 1}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,55 @@
|
|||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import asyncio
|
||||
import signal
|
||||
import websockets
|
||||
import argparse
|
||||
|
||||
stop_event = asyncio.Event()
|
||||
|
||||
async def handle_websocket(websocket, log_file):
|
||||
try:
|
||||
# Write the message to the specified log file
|
||||
if log_file != "":
|
||||
with open(log_file, "a", encoding="utf-8") as f:
|
||||
async for message in websocket:
|
||||
f.write(message + "\n")
|
||||
if stop_event.is_set():
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"Connection closed with error: {e}")
|
||||
|
||||
async def listen(port, log_file):
|
||||
async with websockets.serve(
|
||||
lambda ws: handle_websocket(ws, log_file),
|
||||
"0.0.0.0",
|
||||
port,
|
||||
ping_timeout = None,
|
||||
max_size= 10 * 1024 * 1024 # 10MB,
|
||||
):
|
||||
print(f"WebSocket server listening on port {port}...")
|
||||
await stop_event.wait() # Run forever (until canceled)
|
||||
|
||||
def signal_handler(sig, frame):
|
||||
stop_event.set()
|
||||
|
||||
if __name__ == '__main__':
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-d', '--log_file', type=str, default='stream_notify_server.log', help='log file')
|
||||
parser.add_argument('-p', '--port', type=int, default=12345, help='port number')
|
||||
args = parser.parse_args()
|
||||
|
||||
asyncio.run(listen(args.port, args.log_file))
|
|
@ -0,0 +1,473 @@
|
|||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from frame import etool
|
||||
from frame.etool import *
|
||||
from frame.log import *
|
||||
from frame.cases import *
|
||||
from frame.sql import *
|
||||
from frame.caseBase import *
|
||||
from frame.common import *
|
||||
import signal
|
||||
import subprocess
|
||||
|
||||
class StreamNotifyServer:
|
||||
def __init__(self):
|
||||
self.log_file = ""
|
||||
self.sub_process = None
|
||||
|
||||
def __del__(self):
|
||||
self.stop()
|
||||
|
||||
def run(self, port, log_file):
|
||||
tdLog.info(f"Start notify server: python3 {etool.curFile(__file__, 'stream_notify_server.py')} -p {port} -d {log_file}")
|
||||
self.sub_process = subprocess.Popen(['python3', etool.curFile(__file__, 'stream_notify_server.py'), '-p', str(port), '-d', log_file])
|
||||
self.log_file = log_file
|
||||
|
||||
def stop(self):
|
||||
if self.sub_process is not None:
|
||||
self.sub_process.send_signal(signal.SIGINT)
|
||||
try:
|
||||
self.sub_process.wait(60)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.sub_process.kill()
|
||||
|
||||
class TestStreamNotifySinglePass():
|
||||
def __init__(self, num_addr_per_stream, trigger_mode, notify_event, disorder):
|
||||
self.current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
self.num_addr_per_stream = num_addr_per_stream
|
||||
self.trigger_mode = trigger_mode
|
||||
self.notify_event = notify_event
|
||||
self.disorder = disorder
|
||||
self.streams = []
|
||||
self.id = ''.join(random.choices(string.ascii_letters + string.digits, k=8))
|
||||
|
||||
def is_port_in_use(self, port):
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
return s.connect_ex(("127.0.0.1", port)) == 0
|
||||
|
||||
def gen_streams(self):
|
||||
self.streams = [
|
||||
{
|
||||
"stream_name": "s_time_par",
|
||||
"dest_table": "dst_time_par",
|
||||
"window_clause": "interval(5s)",
|
||||
"partitioned": True,
|
||||
},
|
||||
{
|
||||
"stream_name": "s_time",
|
||||
"dest_table": "dst_time",
|
||||
"window_clause": "interval(5s)",
|
||||
"partitioned": False,
|
||||
},
|
||||
{
|
||||
"stream_name": "s_state_par",
|
||||
"dest_table": "dst_state_par",
|
||||
"window_clause": "state_window(c6)",
|
||||
"partitioned": True,
|
||||
},
|
||||
{
|
||||
"stream_name": "s_session_par",
|
||||
"dest_table": "dst_session_par",
|
||||
"window_clause": "session(ts, 50a)",
|
||||
"partitioned": True,
|
||||
},
|
||||
{
|
||||
"stream_name": "s_session",
|
||||
"dest_table": "dst_session",
|
||||
"window_clause": "session(ts, 50a)",
|
||||
"partitioned": False,
|
||||
},
|
||||
{
|
||||
"stream_name": "s_event_par",
|
||||
"dest_table": "dst_event_par",
|
||||
"window_clause": "event_window start with c6 = true end with c6 = false",
|
||||
"partitioned": True,
|
||||
},
|
||||
{
|
||||
"stream_name": "s_count_par",
|
||||
"dest_table": "dst_count_par",
|
||||
"window_clause": "count_window(10)",
|
||||
"partitioned": True,
|
||||
},
|
||||
]
|
||||
# set port to random number between 10000 and 20000
|
||||
port = random.randint(10000, 20000)
|
||||
for stream in self.streams:
|
||||
stream["notify_address"] = ""
|
||||
stream["notify_server"] = []
|
||||
if self.trigger_mode == "FORCE_WINDOW_CLOSE":
|
||||
if stream["stream_name"] == "s_time" or stream["stream_name"] == "s_session":
|
||||
continue
|
||||
elif "MAX_DELAY" in self.trigger_mode or "AT_ONCE" in self.trigger_mode or self.disorder:
|
||||
if stream["stream_name"] == "s_session_par" or stream["stream_name"] == "s_state_par":
|
||||
continue
|
||||
for i in range(self.num_addr_per_stream):
|
||||
# Find an available port
|
||||
while self.is_port_in_use(port):
|
||||
port += 1
|
||||
# Start the stream notify server and add the address to the stream
|
||||
log_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}_{i}.log"
|
||||
if os.path.exists(log_file):
|
||||
os.remove(log_file)
|
||||
server = StreamNotifyServer()
|
||||
server.run(port, log_file)
|
||||
stream["notify_address"] += f"'ws://127.0.0.1:{port}',"
|
||||
stream["notify_server"].append(server)
|
||||
port += 1
|
||||
stream["notify_address"] = stream["notify_address"][:-1]
|
||||
|
||||
def create_streams(self):
|
||||
tdLog.info("==========step1:create table")
|
||||
tdSql.execute("drop database if exists test;")
|
||||
tdSql.execute("create database test keep 3650;")
|
||||
tdSql.execute("use test;")
|
||||
tdSql.execute(
|
||||
f"""create stable if not exists test.st
|
||||
(ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10), c9 tinyint unsigned, c10 smallint unsigned, c11 int unsigned, c12 bigint unsigned)
|
||||
tags(groupid int);
|
||||
"""
|
||||
)
|
||||
for stream in self.streams:
|
||||
if len(stream["notify_server"]) == 0:
|
||||
continue
|
||||
stream_option = f"TRIGGER {self.trigger_mode}"
|
||||
if self.trigger_mode != "FORCE_WINDOW_CLOSE":
|
||||
stream_option += " IGNORE UPDATE 0"
|
||||
if not stream["stream_name"].startswith("s_count"):
|
||||
stream_option += " IGNORE EXPIRED 0"
|
||||
if stream["stream_name"].startswith("s_count"):
|
||||
stream_option += " WATERMARK 1a"
|
||||
stream_sql = f"""create stream {stream["stream_name"]} {stream_option} into {stream["dest_table"]} as
|
||||
select _wstart, _wend, min(c0), max(c1), min(c2), max(c3), min(c4), max(c5), first(c6), first(c7), last(c8), min(c9), max(c10), min(c11), max(c12)
|
||||
from test.st {stream["partitioned"] and "partition by tbname" or ""}
|
||||
{stream["window_clause"]}
|
||||
notify ({stream["notify_address"]}) on ({self.notify_event});
|
||||
"""
|
||||
tdSql.execute(stream_sql, show=True)
|
||||
# Wait for the stream tasks to be ready
|
||||
for i in range(50):
|
||||
tdLog.info(f"i={i} wait for stream tasks ready ...")
|
||||
time.sleep(1)
|
||||
rows = tdSql.query("select * from information_schema.ins_stream_tasks where status <> 'ready';")
|
||||
if rows == 0:
|
||||
break
|
||||
|
||||
def insert_data(self):
|
||||
tdLog.info("insert stream notify test data.")
|
||||
# taosBenchmark run
|
||||
json_file = self.disorder and "stream_notify_disorder.json" or "stream_notify.json"
|
||||
json = etool.curFile(__file__, json_file)
|
||||
etool.benchMark(json=json)
|
||||
|
||||
def wait_all_streams_done(self):
|
||||
while True:
|
||||
tdLog.info("wait for all streams done ...")
|
||||
time.sleep(10)
|
||||
rows = tdSql.query("select stream_name, level, notify_event_stat from information_schema.ins_stream_tasks where notify_event_stat is not null;")
|
||||
num_pushed = 0
|
||||
num_sent = 0
|
||||
for i in range(rows):
|
||||
tdLog.printNoPrefix(f"{tdSql.getData(i, 0)}, {tdSql.getData(i, 1)}, {tdSql.getData(i, 2)}")
|
||||
notify_event_stat = tdSql.getData(i, 2)
|
||||
match = re.search(r"Push (\d+)x, (\d+) elems", notify_event_stat)
|
||||
if match:
|
||||
num_pushed += int(match.group(2))
|
||||
match = re.search(r"Send (\d+)x, (\d+) elems", notify_event_stat)
|
||||
if match:
|
||||
num_sent += int(match.group(2))
|
||||
if num_pushed == num_sent:
|
||||
break
|
||||
tdLog.info("wait for all notify servers stop ...")
|
||||
for stream in self.streams:
|
||||
for server in stream["notify_server"]:
|
||||
server.stop()
|
||||
|
||||
def parse(self, log_file, out_file, stream_name):
|
||||
message_ids = set()
|
||||
events_map = {}
|
||||
has_open = "window_open" in self.notify_event
|
||||
has_close = "window_close" in self.notify_event
|
||||
with open(log_file, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
data = json.loads(line)
|
||||
|
||||
# Check if the data has the required fields: messageId, timestamp, streams
|
||||
if "messageId" not in data:
|
||||
print(f"Error: Missing 'messageId' in data {data}")
|
||||
return False
|
||||
if "timestamp" not in data:
|
||||
print(f"Error: Missing 'timestamp' in data {data}")
|
||||
return False
|
||||
if "streams" not in data:
|
||||
print(f"Error: Missing 'streams' in data {data}")
|
||||
return False
|
||||
|
||||
# Check if the message id is duplicated
|
||||
if message_ids.__contains__(data["messageId"]):
|
||||
print(f"Error: Duplicate message id {data['messageId']}")
|
||||
return False
|
||||
message_ids.add(data["messageId"])
|
||||
|
||||
# Check if the streams is correct
|
||||
for stream in data["streams"]:
|
||||
# Check if the stream has the required fields: streamName, events
|
||||
if "streamName" not in stream:
|
||||
print(f"Error: Missing 'streamName' in stream {stream}")
|
||||
return False
|
||||
if "events" not in stream:
|
||||
print(f"Error: Missing 'events' in stream {stream}")
|
||||
return False
|
||||
|
||||
# Check if the stream name is correct
|
||||
if stream["streamName"] != stream_name:
|
||||
print(f"Error: Incorrect stream name {stream['streamName']}")
|
||||
return False
|
||||
|
||||
# Check if the events are correct
|
||||
for event in stream["events"]:
|
||||
# Check if the event has the required fields: tableName, eventType, eventTime, windowId, windowType
|
||||
if "tableName" not in event:
|
||||
print(f"Error: Missing 'tableName' in event {event}")
|
||||
return False
|
||||
if "eventType" not in event:
|
||||
print(f"Error: Missing 'eventType' in event {event}")
|
||||
return False
|
||||
if "eventTime" not in event:
|
||||
print(f"Error: Missing 'eventTime' in event {event}")
|
||||
return False
|
||||
if "windowId" not in event:
|
||||
print(f"Error: Missing 'windowId' in event {event}")
|
||||
return False
|
||||
if "windowType" not in event:
|
||||
print(f"Error: Missing 'windowType' in event {event}")
|
||||
return False
|
||||
if event["eventType"] not in [
|
||||
"WINDOW_OPEN",
|
||||
"WINDOW_CLOSE",
|
||||
"WINDOW_INVALIDATION",
|
||||
]:
|
||||
print(f"Error: Invalid event type {event['eventType']}")
|
||||
return False
|
||||
if event["windowType"] not in [
|
||||
"Time",
|
||||
"State",
|
||||
"Session",
|
||||
"Event",
|
||||
"Count",
|
||||
]:
|
||||
print(f"Error: Invalid window type {event['windowType']}")
|
||||
return False
|
||||
|
||||
if event["eventType"] == "WINDOW_INVALIDATION":
|
||||
if not has_close:
|
||||
print(f"Error: WINDOW_INVALIDATION event is not allowed")
|
||||
return False
|
||||
# WINDOW_INVALIDATION must have fields: windowStart, windowEnd
|
||||
if "windowStart" not in event:
|
||||
print(f"Error: Missing 'windowStart' in event {event}")
|
||||
return False
|
||||
if "windowEnd" not in event:
|
||||
print(f"Error: Missing 'windowEnd' in event {event}")
|
||||
return False
|
||||
events_map.pop(
|
||||
(event["tableName"], event["windowId"]), None
|
||||
)
|
||||
continue
|
||||
|
||||
# Get the event from the event map; if it doesn't exist, create a new one
|
||||
e = events_map.get((event["tableName"], event["windowId"]))
|
||||
if e is None:
|
||||
events_map[(event["tableName"], event["windowId"])] = {
|
||||
"opened": False,
|
||||
"closed": False,
|
||||
"wstart": 0,
|
||||
"wend": 0,
|
||||
}
|
||||
e = events_map.get((event["tableName"], event["windowId"]))
|
||||
|
||||
if event["eventType"] == "WINDOW_OPEN":
|
||||
if not has_open:
|
||||
print(f"Error: WINDOW_OPEN event is not allowed")
|
||||
return False
|
||||
# WINDOW_OPEN for all windows must have field: windowStart
|
||||
if "windowStart" not in event:
|
||||
print(f"Error: Missing 'windowStart' in event {event}")
|
||||
return False
|
||||
if event["windowType"] == "State":
|
||||
# WINDOW_OPEN for State window must also have fields: prevState, curState
|
||||
if "prevState" not in event:
|
||||
print(
|
||||
f"Error: Missing 'prevState' in event {event}"
|
||||
)
|
||||
return False
|
||||
if "curState" not in event:
|
||||
print(f"Error: Missing 'curState' in event {event}")
|
||||
return False
|
||||
elif event["windowType"] == "Event":
|
||||
# WINDOW_OPEN for Event window must also have fields: triggerCondition
|
||||
if "triggerCondition" not in event:
|
||||
print(
|
||||
f"Error: Missing 'triggerCondition' in event {event}"
|
||||
)
|
||||
return False
|
||||
e["opened"] = True
|
||||
e["wstart"] = event["windowStart"]
|
||||
elif event["eventType"] == "WINDOW_CLOSE":
|
||||
if not has_close:
|
||||
print(f"Error: WINDOW_CLOSE event is not allowed")
|
||||
return False
|
||||
# WINDOW_CLOSE for all windows must have fields: windowStart, windowEnd, result
|
||||
if "windowStart" not in event:
|
||||
print(f"Error: Missing 'windowStart' in event {event}")
|
||||
return False
|
||||
if "windowEnd" not in event:
|
||||
print(f"Error: Missing 'windowEnd' in event {event}")
|
||||
return False
|
||||
if "result" not in event:
|
||||
print(f"Error: Missing 'result' in event {event}")
|
||||
return False
|
||||
if event["windowType"] == "State":
|
||||
# WINDOW_CLOSE for State window must also have fields: curState, nextState
|
||||
if "curState" not in event:
|
||||
print(f"Error: Missing 'curState' in event {event}")
|
||||
return False
|
||||
if "nextState" not in event:
|
||||
print(
|
||||
f"Error: Missing 'nextState' in event {event}"
|
||||
)
|
||||
return False
|
||||
elif event["windowType"] == "Event":
|
||||
# WINDOW_CLOSE for Event window must also have fields: triggerCondition
|
||||
if "triggerCondition" not in event:
|
||||
print(
|
||||
f"Error: Missing 'triggerCondition' in event {event}"
|
||||
)
|
||||
return False
|
||||
e["closed"] = True
|
||||
e["wstart"] = event["windowStart"]
|
||||
e["wend"] = event["windowEnd"]
|
||||
|
||||
# Collect all the windows that closed
|
||||
windows_map = {}
|
||||
for k, v in events_map.items():
|
||||
if not v["closed"]:
|
||||
continue
|
||||
e = windows_map.get(k[0])
|
||||
if e is None:
|
||||
windows_map[k[0]] = []
|
||||
e = windows_map.get(k[0])
|
||||
e.append((v["wstart"], v["wend"]))
|
||||
|
||||
# Sort the windows by start time
|
||||
for k, v in windows_map.items():
|
||||
v.sort(key=lambda x: x[0])
|
||||
|
||||
# Write all collected window info to the specified output file in sorted order as csv format
|
||||
with open(out_file, "w", encoding="utf-8") as f:
|
||||
f.write("wstart,wend,tbname\n")
|
||||
for k, v in sorted(windows_map.items()):
|
||||
for w in v:
|
||||
f.write(f"{w[0]},{w[1]},\"{k}\"\n")
|
||||
return True
|
||||
|
||||
def check_notify_result(self):
|
||||
all_right = True
|
||||
for stream in self.streams:
|
||||
if len(stream["notify_server"]) == 0 or not "window_close" in self.notify_event:
|
||||
continue
|
||||
query_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}.csv"
|
||||
query_sql = f"select cast(_wstart as bigint) as wstart, cast(_wend as bigint) as wend, tbname from test.{stream['dest_table']} order by tbname, wstart >> {query_file};"
|
||||
tdLog.info("query_sql: " + query_sql)
|
||||
os.system(f"taos -c {tdCom.getClientCfgPath()} -s '{query_sql}'")
|
||||
for i in range(self.num_addr_per_stream):
|
||||
server = stream["notify_server"][i]
|
||||
parse_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}_{i}.csv"
|
||||
if os.path.exists(parse_file):
|
||||
os.remove(parse_file)
|
||||
if not self.parse(f"{self.current_dir}/{self.id}_{stream['stream_name']}_{i}.log", parse_file, stream["stream_name"]):
|
||||
tdLog.exit(f"Error: {stream['stream_name']}_{i} parse notify result failed")
|
||||
# Compare the result using diff command
|
||||
diff_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}_{i}.diff"
|
||||
if os.path.exists(diff_file):
|
||||
os.remove(diff_file)
|
||||
os.system(f"diff --strip-trailing-cr {query_file} {parse_file} > {diff_file}")
|
||||
if os.path.getsize(diff_file) != 0:
|
||||
tdLog.info(f"Error: {stream['stream_name']}_{i} notify result is not correct")
|
||||
all_right = False
|
||||
if not all_right:
|
||||
raise Exception("Error: notify result is not correct")
|
||||
|
||||
def drop_all_streams(self):
|
||||
for stream in self.streams:
|
||||
tdSql.execute(f"drop stream if exists {stream['stream_name']};")
|
||||
# Also remove all generaetd files
|
||||
query_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}.csv"
|
||||
if os.path.exists(query_file):
|
||||
os.remove(query_file)
|
||||
for i in range(self.num_addr_per_stream):
|
||||
log_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}_{i}.log"
|
||||
parse_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}_{i}.csv"
|
||||
diff_file = f"{self.current_dir}/{self.id}_{stream['stream_name']}_{i}.diff"
|
||||
if os.path.exists(log_file):
|
||||
os.remove(log_file)
|
||||
if os.path.exists(parse_file):
|
||||
os.remove(parse_file)
|
||||
if os.path.exists(diff_file):
|
||||
os.remove(diff_file)
|
||||
|
||||
def run(self):
|
||||
tdLog.info(f"Start to execute TestStreamNotifySinglePass({self.num_addr_per_stream}, {self.trigger_mode}, {self.notify_event}, {self.disorder})")
|
||||
self.gen_streams()
|
||||
self.create_streams()
|
||||
self.insert_data()
|
||||
self.wait_all_streams_done()
|
||||
self.check_notify_result()
|
||||
self.drop_all_streams()
|
||||
tdLog.info(f"TestStreamNotifySinglePass({self.num_addr_per_stream}, {self.trigger_mode}, {self.notify_event}, {self.disorder}) successfully executed")
|
||||
|
||||
class TDTestCase(TBase):
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor(), True)
|
||||
|
||||
def run(self):
|
||||
# Disable many tests due to long execution time
|
||||
|
||||
# TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="AT_ONCE", notify_event="'window_open', 'window_close'", disorder=False).run()
|
||||
# TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="MAX_DELAY 10s", notify_event="'window_open', 'window_close'", disorder=False).run()
|
||||
TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="WINDOW_CLOSE", notify_event="'window_open', 'window_close'", disorder=False).run()
|
||||
# TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="FORCE_WINDOW_CLOSE", notify_event="'window_open', 'window_close'", disorder=False).run()
|
||||
|
||||
# TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="AT_ONCE", notify_event="'window_open', 'window_close'", disorder=True).run()
|
||||
# TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="MAX_DELAY 10s", notify_event="'window_open', 'window_close'", disorder=True).run()
|
||||
TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="WINDOW_CLOSE", notify_event="'window_open', 'window_close'", disorder=True).run()
|
||||
# TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="FORCE_WINDOW_CLOSE", notify_event="'window_open', 'window_close'", disorder=True).run()
|
||||
|
||||
# TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="AT_ONCE", notify_event="'window_close'", disorder=False).run()
|
||||
TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="MAX_DELAY 10s", notify_event="'window_close'", disorder=False).run()
|
||||
# TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="WINDOW_CLOSE", notify_event="'window_close'", disorder=False).run()
|
||||
# TestStreamNotifySinglePass(num_addr_per_stream=1, trigger_mode="FORCE_WINDOW_CLOSE", notify_event="'window_close'", disorder=False).run()
|
||||
|
||||
TestStreamNotifySinglePass(num_addr_per_stream=3, trigger_mode="AT_ONCE", notify_event="'window_open'", disorder=False).run()
|
||||
# TestStreamNotifySinglePass(num_addr_per_stream=3, trigger_mode="MAX_DELAY 10s", notify_event="'window_open'", disorder=False).run()
|
||||
# TestStreamNotifySinglePass(num_addr_per_stream=3, trigger_mode="WINDOW_CLOSE", notify_event="'window_open'", disorder=False).run()
|
||||
# TestStreamNotifySinglePass(num_addr_per_stream=3, trigger_mode="FORCE_WINDOW_CLOSE", notify_event="'window_open'", disorder=False).run()
|
||||
|
||||
def stop(self):
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
|
@ -10,6 +10,7 @@
|
|||
|
||||
# army-test
|
||||
#,,y,army,./pytest.sh python3 ./test.py -f multi-level/mlevel_basic.py -N 3 -L 3 -D 2
|
||||
,,y,army,./pytest.sh python3 ./test.py -f stream/test_stream_notify.py
|
||||
|
||||
#tsim test
|
||||
#,,y,script,./test.sh -f tsim/query/timeline.sim
|
||||
|
|
|
@ -222,7 +222,7 @@ class TDTestCase:
|
|||
|
||||
tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'")
|
||||
tdLog.info(len(tdSql.queryResult))
|
||||
tdSql.checkEqual(True, len(tdSql.queryResult) in range(313, 314))
|
||||
tdSql.checkEqual(True, len(tdSql.queryResult) in range(314, 315))
|
||||
|
||||
tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'")
|
||||
tdSql.checkEqual(61, len(tdSql.queryResult))
|
||||
|
|
|
@ -1033,18 +1033,26 @@ int convertStringToDatatype(char *type, int length) {
|
|||
return TSDB_DATA_TYPE_BOOL;
|
||||
} else if (0 == strCompareN(type, "tinyint", length)) {
|
||||
return TSDB_DATA_TYPE_TINYINT;
|
||||
} else if (0 == strCompareN(type, "tinyint unsigned", length)) {
|
||||
return TSDB_DATA_TYPE_UTINYINT;
|
||||
} else if (0 == strCompareN(type, "utinyint", length)) {
|
||||
return TSDB_DATA_TYPE_UTINYINT;
|
||||
} else if (0 == strCompareN(type, "smallint", length)) {
|
||||
return TSDB_DATA_TYPE_SMALLINT;
|
||||
} else if (0 == strCompareN(type, "smallint unsigned", length)) {
|
||||
return TSDB_DATA_TYPE_USMALLINT;
|
||||
} else if (0 == strCompareN(type, "usmallint", length)) {
|
||||
return TSDB_DATA_TYPE_USMALLINT;
|
||||
} else if (0 == strCompareN(type, "int", length)) {
|
||||
return TSDB_DATA_TYPE_INT;
|
||||
} else if (0 == strCompareN(type, "int unsigned", length)) {
|
||||
return TSDB_DATA_TYPE_UINT;
|
||||
} else if (0 == strCompareN(type, "uint", length)) {
|
||||
return TSDB_DATA_TYPE_UINT;
|
||||
} else if (0 == strCompareN(type, "bigint", length)) {
|
||||
return TSDB_DATA_TYPE_BIGINT;
|
||||
} else if (0 == strCompareN(type, "bigint unsigned", length)) {
|
||||
return TSDB_DATA_TYPE_UBIGINT;
|
||||
} else if (0 == strCompareN(type, "ubigint", length)) {
|
||||
return TSDB_DATA_TYPE_UBIGINT;
|
||||
} else if (0 == strCompareN(type, "float", length)) {
|
||||
|
|
Loading…
Reference in New Issue