Merge branch '3.0' of https://github.com/taosdata/TDengine into enh/tsdb_optimize
This commit is contained in:
Hongze Cheng
commit
e731a08f2d
|
|
@ -18,7 +18,7 @@
|
|||
注意：修改文档的分支要以`docs/`为开头，以免进行不必要的测试。
|
||||
4. 创建pull request，将自己的分支合并到开发分支`3.0`，我们开发团队将尽快审核。
|
||||
|
||||
如遇任何问题，请添加官方微信TDengineECO。我们的团队会帮忙解决。
|
||||
如遇任何问题，请添加官方微信 tdengine1。我们的团队会帮忙解决。
|
||||
|
||||
## 给贡献者的礼品
|
||||
|
||||
|
|
@ -48,4 +48,4 @@ TDengine 社区致力于让更多的开发者理解和使用它。
|
|||
|
||||
## 联系我们
|
||||
|
||||
如果您有什么问题需要解决，或者有什么问题需要解答，可以添加微信：TDengineECO
|
||||
如果您有什么问题需要解决，或者有什么问题需要解答，可以添加微信：tdengine1。
|
||||
|
|
|
|||
|
|
@ -285,10 +285,10 @@ You configure the following parameters when creating a consumer:
|
|||
|
||||
| Parameter | Type | Description | Remarks |
|
||||
| :----------------------------: | :-----: | -------------------------------------------------------- | ------------------------------------------- |
|
||||
| `td.connect.ip` | string | Used in establishing a connection; same as `taos_connect` | Only valid for establishing native connection |
|
||||
| `td.connect.user` | string | Used in establishing a connection; same as `taos_connect` | Only valid for establishing native connection |
|
||||
| `td.connect.pass` | string | Used in establishing a connection; same as `taos_connect` | Only valid for establishing native connection |
|
||||
| `td.connect.port` | string | Used in establishing a connection; same as `taos_connect` | Only valid for establishing native connection |
|
||||
| `td.connect.ip` | string | IP address of the server side | |
|
||||
| `td.connect.user` | string | User Name | |
|
||||
| `td.connect.pass` | string | Password | |
|
||||
| `td.connect.port` | string | Port of the server side | |
|
||||
| `group.id` | string | Consumer group ID; consumers with the same ID are in the same group | **Required**. Maximum length: 192. |
|
||||
| `client.id` | string | Client ID | Maximum length: 192. |
|
||||
| `auto.offset.reset` | enum | Initial offset for the consumer group | Specify `earliest`, `latest`, or `none`(default) |
|
||||
|
|
@ -325,6 +325,7 @@ Java programs use the following parameters:
|
|||
|
||||
| Parameter | Type | Description | Remarks |
|
||||
| ----------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `td.connect.type` | string | connection type: "jni" means native connection, "ws" means websocket connection, the default is "jni" |
|
||||
| `bootstrap.servers` | string |Connection address, such as `localhost:6030` |
|
||||
| `value.deserializer` | string | Value deserializer; to use this method, implement the `com.taosdata.jdbc.tmq.Deserializer` interface or inherit the `com.taosdata.jdbc.tmq.ReferenceDeserializer` type |
|
||||
| `value.deserializer.encoding` | string | Specify the encoding for string deserialization | |
|
||||
|
|
@ -399,22 +400,6 @@ from taos.tmq import Consumer
|
|||
consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"})
|
||||
```
|
||||
|
||||
Python programs use the following parameters:
|
||||
|
||||
| Parameter | Type | Description | Remarks |
|
||||
|:---------:|:----:|:-----------:|:-------:|
|
||||
| `td.connect.ip` | string | Used in establishing a connection||
|
||||
| `td.connect.user` | string | Used in establishing a connection||
|
||||
| `td.connect.pass` | string | Used in establishing a connection||
|
||||
| `td.connect.port` | string | Used in establishing a connection||
|
||||
| `group.id` | string | Consumer group ID; consumers with the same ID are in the same group | **Required**. Maximum length: 192 |
|
||||
| `client.id` | string | Client ID | Maximum length: 192 |
|
||||
| `msg.with.table.name` | string | Specify whether to deserialize table names from messages | pecify `true` or `false` |
|
||||
| `enable.auto.commit` | string | Commit automatically | pecify `true` or `false` |
|
||||
| `auto.commit.interval.ms` | string | Interval for automatic commits, in milliseconds | |
|
||||
| `auto.offset.reset` | string | Initial offset for the consumer group | Specify `earliest`, `latest`, or `none`(default) |
|
||||
| `enable.heartbeat.background` | string | Backend heartbeat; if enabled, the consumer does not go offline even if it has not polled for a long time | Specify `true` or `false` |
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem label="Node.JS" value="Node.JS">
|
||||
|
|
|
|||
|
|
@ -867,10 +867,16 @@ FIRST(expr)
|
|||
### INTERP
|
||||
|
||||
```sql
|
||||
INTERP(expr)
|
||||
INTERP(expr [, ignore_null_values])
|
||||
|
||||
ignore_null_values: {
|
||||
0
|
||||
| 1
|
||||
}
|
||||
```
|
||||
|
||||
**Description**: The value that matches the specified timestamp range is returned, if existing; or an interpolation value is returned.
|
||||
**Description**: The value that matches the specified timestamp range is returned, if existing; or an interpolation value is returned. The value of `ignore_null_values` can be 0 or 1, 1 means null values are ignored. The default value of this parameter is 0.
|
||||
|
||||
|
||||
**Return value type**: Same as the column being operated upon
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ CREATE [OR REPLACE] FUNCTION function_name AS library_path OUTPUTTYPE output_typ
|
|||
```
|
||||
- OR REPLACE: if the UDF exists, the UDF properties are modified
|
||||
- function_name: The scalar function name to be used in the SQL statement
|
||||
- LANGUAGE 'C|Python': the programming language of UDF. Now C or Python is supported. If this clause is omitted, C is assumed as the programming language.
|
||||
- LANGUAGE 'C|Python': the programming language of UDF. Now C or Python (v3.7+) is supported. If this clause is omitted, C is assumed as the programming language.
|
||||
- library_path: For C programming language, The absolute path of the DLL file including the name of the shared object file (.so). For Python programming language, the absolute path of the Python UDF script. The path must be quoted with single or double quotes.
|
||||
- output_type: The data type of the results of the UDF.
|
||||
|
||||
|
|
|
|||
|
|
@ -285,10 +285,10 @@ CREATE TOPIC topic_name AS DATABASE db_name;
|
|||
|
||||
| 参数名称 | 类型 | 参数说明 | 备注 |
|
||||
| :----------------------------: | :-----: | -------------------------------------------------------- | ------------------------------------------- |
|
||||
| `td.connect.ip` | string | 用于创建连接，同 `taos_connect` | 仅用于建立原生连接 |
|
||||
| `td.connect.user` | string | 用于创建连接，同 `taos_connect` | 仅用于建立原生连接 |
|
||||
| `td.connect.pass` | string | 用于创建连接，同 `taos_connect` | 仅用于建立原生连接 |
|
||||
| `td.connect.port` | integer | 用于创建连接，同 `taos_connect` | 仅用于建立原生连接 |
|
||||
| `td.connect.ip` | string | 服务端的 IP 地址 | |
|
||||
| `td.connect.user` | string | 用户名 | |
|
||||
| `td.connect.pass` | string | 密码 | |
|
||||
| `td.connect.port` | integer | 服务端的端口号 | |
|
||||
| `group.id` | string | 消费组 ID，同一消费组共享消费进度 | **必填项**。最大长度：192。 |
|
||||
| `client.id` | string | 客户端 ID | 最大长度：192。 |
|
||||
| `auto.offset.reset` | enum | 消费组订阅的初始位置 | <br />`earliest`: default;从头开始订阅; <br/>`latest`: 仅从最新数据开始订阅; <br/>`none`: 没有提交的 offset 无法订阅 |
|
||||
|
|
@ -321,10 +321,11 @@ tmq_conf_destroy(conf);
|
|||
</TabItem>
|
||||
<TabItem value="java" label="Java">
|
||||
|
||||
对于 Java 程序，使用如下配置项：
|
||||
对于 Java 程序，还可以使用如下配置项：
|
||||
|
||||
| 参数名称 | 类型 | 参数说明 |
|
||||
| ----------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `td.connect.type` | string | 连接类型，"jni" 指原生连接，"ws" 指 websocket 连接，默认值为 "jni" |
|
||||
| `bootstrap.servers` | string | 连接地址，如 `localhost:6030` |
|
||||
| `value.deserializer` | string | 值解析方法，使用此方法应实现 `com.taosdata.jdbc.tmq.Deserializer` 接口或继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer` 类 |
|
||||
| `value.deserializer.encoding` | string | 指定字符串解析的字符集 | |
|
||||
|
|
@ -401,21 +402,6 @@ from taos.tmq import Consumer
|
|||
consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"})
|
||||
```
|
||||
|
||||
其中，`configs` 为 dict 类型，传递创建 Consumer 的参数。可以配置的参数有：
|
||||
|
||||
| 参数名称 | 类型 | 参数说明 | 备注 |
|
||||
|:------:|:----:|:-------:|:---:|
|
||||
| `td.connect.ip` | string | 用于创建连接||
|
||||
| `td.connect.user` | string | 用于创建连接||
|
||||
| `td.connect.pass` | string | 用于创建连接||
|
||||
| `td.connect.port` | string | 用于创建连接||
|
||||
| `group.id` | string | 消费组 ID，同一消费组共享消费进度 | **必填项**。最大长度：192 |
|
||||
| `client.id` | string | 客户端 ID | 最大长度：192 |
|
||||
| `msg.with.table.name` | string | 是否允许从消息中解析表名,不适用于列订阅 | 合法值：`true`, `false` |
|
||||
| `enable.auto.commit` | string | 启用自动提交 | 合法值：`true`, `false` |
|
||||
| `auto.commit.interval.ms` | string | 以毫秒为单位的自动提交时间间隔 | 默认值：5000 ms |
|
||||
| `auto.offset.reset` | string | 消费组订阅的初始位置 | 可选：`earliest`(default), `latest`, `none` |
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem label="Node.JS" value="Node.JS">
|
||||
|
|
|
|||
|
|
@ -869,10 +869,15 @@ FIRST(expr)
|
|||
### INTERP
|
||||
|
||||
```sql
|
||||
INTERP(expr)
|
||||
INTERP(expr [, ignore_null_values])
|
||||
|
||||
ignore_null_values: {
|
||||
0
|
||||
| 1
|
||||
}
|
||||
```
|
||||
|
||||
**功能说明**：返回指定时间截面指定列的记录值或插值。
|
||||
**功能说明**：返回指定时间截面指定列的记录值或插值。ignore_null_values 参数的值可以是 0 或 1，为 1 时表示忽略 NULL 值, 缺省值为0。
|
||||
|
||||
**返回数据类型**：同字段类型。
|
||||
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ CREATE [OR REPLACE] AGGREGATE FUNCTION function_name AS library_path OUTPUTTYPE
|
|||
```
|
||||
- OR REPLACE: 如果函数已经存在，会修改已有的函数属性。
|
||||
- function_name：聚合函数未来在 SQL 中被调用时的函数名，必须与函数实现中 udfNormalFunc 的实际名称一致；
|
||||
- LANGUAGE 'C|Python'：函数编程语言，目前支持C语言和Python语言。
|
||||
- LANGUAGE 'C|Python'：函数编程语言，目前支持C语言和Python语言（v3.7+）。
|
||||
- library_path：如果编程语言是C，路径是包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件）。如果编程语言是Python，路径是包含 UDF 函数实现的Python文件路径。这个路径需要用英文单引号或英文双引号括起来；；
|
||||
- output_type：此函数计算结果的数据类型名称；
|
||||
- buffer_size：中间计算结果的缓冲区大小，单位是字节。如果不使用可以不设置。
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ static int32_t init_env() {
|
|||
taos_free_result(pRes);
|
||||
|
||||
// create database
|
||||
pRes = taos_query(pConn, "create database tmqdb precision 'ns'");
|
||||
pRes = taos_query(pConn, "create database tmqdb precision 'ns' WAL_RETENTION_PERIOD 3600");
|
||||
if (taos_errno(pRes) != 0) {
|
||||
printf("error in create tmqdb, reason:%s\n", taos_errstr(pRes));
|
||||
goto END;
|
||||
|
|
@ -289,7 +289,7 @@ void consume_repeatly(tmq_t* tmq) {
|
|||
}
|
||||
}
|
||||
|
||||
free(pAssign);
|
||||
tmq_free_assignment(pAssign);
|
||||
|
||||
// let's do it again
|
||||
basic_consume_loop(tmq);
|
||||
|
|
|
|||
|
|
@ -288,6 +288,7 @@ DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg);
|
|||
DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param);
|
||||
DLL_EXPORT int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment,
|
||||
int32_t *numOfAssignment);
|
||||
DLL_EXPORT void tmq_free_assignment(tmq_topic_assignment* pAssignment);
|
||||
DLL_EXPORT int32_t tmq_offset_seek(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset);
|
||||
|
||||
/* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */
|
||||
|
|
|
|||
|
|
@ -215,7 +215,7 @@ void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);
|
|||
void blockDataCleanup(SSDataBlock* pDataBlock);
|
||||
void blockDataEmpty(SSDataBlock* pDataBlock);
|
||||
|
||||
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
|
||||
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize, int32_t extraSize);
|
||||
|
||||
int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n);
|
||||
int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n);
|
||||
|
|
|
|||
|
|
@ -372,6 +372,7 @@ typedef struct {
|
|||
int32_t upstreamChildId;
|
||||
int32_t upstreamNodeId;
|
||||
int32_t blockNum;
|
||||
int64_t totalLen;
|
||||
SArray* dataLen; // SArray<int32_t>
|
||||
SArray* data; // SArray<SRetrieveTableRsp*>
|
||||
} SStreamDispatchReq;
|
||||
|
|
@ -527,7 +528,7 @@ void tDeleteStreamDispatchReq(SStreamDispatchReq* pReq);
|
|||
int32_t streamSetupTrigger(SStreamTask* pTask);
|
||||
|
||||
int32_t streamProcessRunReq(SStreamTask* pTask);
|
||||
int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg, bool exec);
|
||||
int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg, bool exec);
|
||||
int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code);
|
||||
|
||||
int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg);
|
||||
|
|
@ -536,7 +537,7 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, S
|
|||
void streamTaskInputFail(SStreamTask* pTask);
|
||||
int32_t streamTryExec(SStreamTask* pTask);
|
||||
int32_t streamSchedExec(SStreamTask* pTask);
|
||||
int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock);
|
||||
int32_t streamTaskOutputResultBlock(SStreamTask* pTask, SStreamDataBlock* pBlock);
|
||||
bool streamTaskShouldStop(const SStreamStatus* pStatus);
|
||||
bool streamTaskShouldPause(const SStreamStatus* pStatus);
|
||||
|
||||
|
|
|
|||
|
|
@ -267,8 +267,8 @@ typedef enum ELogicConditionType {
|
|||
#define TSDB_DNODE_CONFIG_LEN 128
|
||||
#define TSDB_DNODE_VALUE_LEN 256
|
||||
|
||||
#define TSDB_ACTIVE_KEY_LEN 109 // history 109:?
|
||||
#define TSDB_CONN_ACTIVE_KEY_LEN 257 // history 257:?
|
||||
#define TSDB_ACTIVE_KEY_LEN 109
|
||||
#define TSDB_CONN_ACTIVE_KEY_LEN 255
|
||||
|
||||
#define TSDB_DEFAULT_PKT_SIZE 65480 // same as RPC_MAX_UDP_SIZE
|
||||
|
||||
|
|
|
|||
|
|
@ -2584,6 +2584,14 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
|
|||
}
|
||||
}
|
||||
|
||||
void tmq_free_assignment(tmq_topic_assignment* pAssignment) {
|
||||
if (pAssignment == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
taosMemoryFree(pAssignment);
|
||||
}
|
||||
|
||||
int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_t offset) {
|
||||
if (tmq == NULL) {
|
||||
tscError("invalid tmq handle, null");
|
||||
|
|
|
|||
|
|
@ -445,12 +445,11 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd
|
|||
|
||||
size_t headerSize = sizeof(int32_t);
|
||||
size_t colHeaderSize = sizeof(int32_t) * numOfCols;
|
||||
size_t payloadSize = pageSize - (headerSize + colHeaderSize);
|
||||
|
||||
// TODO speedup by checking if the whole page can fit in firstly.
|
||||
if (!hasVarCol) {
|
||||
size_t rowSize = blockDataGetRowSize(pBlock);
|
||||
int32_t capacity = payloadSize / (rowSize + numOfCols * bitmapChar / 8.0);
|
||||
int32_t capacity = blockDataGetCapacityInRow(pBlock, pageSize, headerSize + colHeaderSize);
|
||||
if (capacity <= 0) {
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
|
@ -1532,10 +1531,10 @@ SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index) {
|
|||
return taosArrayGet(pBlock->pDataBlock, index);
|
||||
}
|
||||
|
||||
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
|
||||
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize, int32_t extraSize) {
|
||||
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
|
||||
|
||||
int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(numOfCols);
|
||||
int32_t payloadSize = pageSize - extraSize;
|
||||
int32_t rowSize = pBlock->info.rowSize;
|
||||
int32_t nRows = payloadSize / rowSize;
|
||||
ASSERT(nRows >= 1);
|
||||
|
|
@ -2514,9 +2513,6 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) {
|
|||
*actualLen = dataLen;
|
||||
*groupId = pBlock->info.id.groupId;
|
||||
ASSERT(dataLen > 0);
|
||||
|
||||
uDebug("build data block, actualLen:%d, rows:%d, cols:%d", dataLen, *rows, *cols);
|
||||
|
||||
return dataLen;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -44,6 +44,11 @@ static const char *offlineReason[] = {
|
|||
"unknown",
|
||||
};
|
||||
|
||||
enum {
|
||||
DND_ACTIVE_CODE,
|
||||
DND_CONN_ACTIVE_CODE,
|
||||
};
|
||||
|
||||
static int32_t mndCreateDefaultDnode(SMnode *pMnode);
|
||||
static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode);
|
||||
static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw);
|
||||
|
|
@ -227,6 +232,14 @@ static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode) {
|
|||
static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOld, SDnodeObj *pNew) {
|
||||
mTrace("dnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew);
|
||||
pOld->updateTime = pNew->updateTime;
|
||||
#ifdef TD_ENTERPRISE
|
||||
if (strncmp(pOld->active, pNew->active, TSDB_ACTIVE_KEY_LEN) != 0) {
|
||||
strncpy(pOld->active, pNew->active, TSDB_ACTIVE_KEY_LEN);
|
||||
}
|
||||
if (strncmp(pOld->connActive, pNew->connActive, TSDB_CONN_ACTIVE_KEY_LEN) != 0) {
|
||||
strncpy(pOld->connActive, pNew->connActive, TSDB_CONN_ACTIVE_KEY_LEN);
|
||||
}
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -635,6 +648,69 @@ _OVER:
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfgReq, int8_t action) {
|
||||
SSdbRaw *pRaw = NULL;
|
||||
STrans *pTrans = NULL;
|
||||
SDnodeObj *pDnode = NULL;
|
||||
bool cfgAll = pCfgReq->dnodeId == -1;
|
||||
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
void *pIter = NULL;
|
||||
while (1) {
|
||||
if (cfgAll) {
|
||||
pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode);
|
||||
if (pIter == NULL) break;
|
||||
} else if(!(pDnode = mndAcquireDnode(pMnode, pCfgReq->dnodeId))) {
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
if (!pTrans) {
|
||||
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "config-dnode");
|
||||
if (!pTrans) goto _OVER;
|
||||
if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER;
|
||||
}
|
||||
|
||||
SDnodeObj tmpDnode = *pDnode;
|
||||
if (action == DND_ACTIVE_CODE) {
|
||||
strncpy(tmpDnode.active, pCfgReq->value, TSDB_ACTIVE_KEY_LEN);
|
||||
} else if (action == DND_CONN_ACTIVE_CODE) {
|
||||
strncpy(tmpDnode.connActive, pCfgReq->value, TSDB_CONN_ACTIVE_KEY_LEN);
|
||||
} else {
|
||||
terrno = TSDB_CODE_INVALID_CFG;
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
pRaw = mndDnodeActionEncode(&tmpDnode);
|
||||
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
|
||||
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
|
||||
pRaw = NULL;
|
||||
|
||||
mInfo("dnode:%d, config dnode, cfg:%d, app:%p config:%s value:%s", pDnode->id, pCfgReq->dnodeId, pReq->info.ahandle,
|
||||
pCfgReq->config, pCfgReq->value);
|
||||
|
||||
if (cfgAll) {
|
||||
sdbRelease(pSdb, pDnode);
|
||||
pDnode = NULL;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (pTrans && mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
|
||||
|
||||
terrno = 0;
|
||||
|
||||
_OVER:
|
||||
if (cfgAll) {
|
||||
sdbRelease(pSdb, pDnode);
|
||||
} else {
|
||||
mndReleaseDnode(pMnode, pDnode);
|
||||
}
|
||||
mndTransDrop(pTrans);
|
||||
sdbFreeRaw(pRaw);
|
||||
return terrno;
|
||||
}
|
||||
|
||||
static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) {
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
|
|
@ -977,6 +1053,34 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
|
|||
|
||||
strcpy(dcfgReq.config, "monitor");
|
||||
snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
|
||||
#ifdef TD_ENTERPRISE
|
||||
} else if (strncasecmp(cfgReq.config, "activeCode", 10) == 0 || strncasecmp(cfgReq.config, "cActiveCode", 11) == 0) {
|
||||
int8_t opt = strncasecmp(cfgReq.config, "a", 1) == 0 ? DND_ACTIVE_CODE : DND_CONN_ACTIVE_CODE;
|
||||
int8_t index = opt == DND_ACTIVE_CODE ? 10 : 11;
|
||||
if (' ' != cfgReq.config[index] && 0 != cfgReq.config[index]) {
|
||||
mError("dnode:%d, failed to config activeCode since invalid conf:%s", cfgReq.dnodeId, cfgReq.config);
|
||||
terrno = TSDB_CODE_INVALID_CFG;
|
||||
return -1;
|
||||
}
|
||||
int32_t vlen = strlen(cfgReq.value);
|
||||
if (vlen > 0 && ((opt == DND_ACTIVE_CODE && vlen != (TSDB_ACTIVE_KEY_LEN - 1)) ||
|
||||
(opt == DND_CONN_ACTIVE_CODE &&
|
||||
(vlen > (TSDB_CONN_ACTIVE_KEY_LEN - 1) || vlen < (TSDB_ACTIVE_KEY_LEN - 1))))) {
|
||||
mError("dnode:%d, failed to config activeCode since invalid vlen:%d. conf:%s, val:%s", cfgReq.dnodeId, vlen,
|
||||
cfgReq.config, cfgReq.value);
|
||||
terrno = TSDB_CODE_INVALID_OPTION;
|
||||
return -1;
|
||||
}
|
||||
|
||||
strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? "activeCode" : "cActiveCode");
|
||||
snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%s", cfgReq.value);
|
||||
|
||||
if (mndConfigDnode(pMnode, pReq, &cfgReq, opt) != 0) {
|
||||
mError("dnode:%d, failed to config activeCode since %s", cfgReq.dnodeId, terrstr());
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
#endif
|
||||
} else {
|
||||
bool findOpt = false;
|
||||
for (int32_t d = 0; d < optionSize; ++d) {
|
||||
|
|
@ -1030,7 +1134,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
|
|||
tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq);
|
||||
mInfo("dnode:%d, send config req to dnode, app:%p config:%s value:%s", cfgReq.dnodeId, pReq->info.ahandle,
|
||||
dcfgReq.config, dcfgReq.value);
|
||||
SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen, .info = pReq->info};
|
||||
SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen};
|
||||
tmsgSendReq(&epSet, &rpcMsg);
|
||||
code = 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) {
|
|||
.info = pMsg->info,
|
||||
.code = 0,
|
||||
};
|
||||
streamProcessDispatchReq(pTask, &req, &rsp, false);
|
||||
streamProcessDispatchMsg(pTask, &req, &rsp, false);
|
||||
streamMetaReleaseTask(pSnode->pMeta, pTask);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
|
|
@ -203,17 +203,13 @@ int32_t sndProcessTaskDispatchReq(SSnode *pSnode, SRpcMsg *pMsg, bool exec) {
|
|||
|
||||
SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, taskId);
|
||||
if (pTask) {
|
||||
SRpcMsg rsp = {
|
||||
.info = pMsg->info,
|
||||
.code = 0,
|
||||
};
|
||||
streamProcessDispatchReq(pTask, &req, &rsp, exec);
|
||||
SRpcMsg rsp = { .info = pMsg->info, .code = 0 };
|
||||
streamProcessDispatchMsg(pTask, &req, &rsp, exec);
|
||||
streamMetaReleaseTask(pSnode->pMeta, pTask);
|
||||
return 0;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t sndProcessTaskRetrieveReq(SSnode *pSnode, SRpcMsg *pMsg) {
|
||||
|
|
@ -227,11 +223,9 @@ int32_t sndProcessTaskRetrieveReq(SSnode *pSnode, SRpcMsg *pMsg) {
|
|||
tDecoderClear(&decoder);
|
||||
int32_t taskId = req.dstTaskId;
|
||||
SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, taskId);
|
||||
|
||||
if (pTask) {
|
||||
SRpcMsg rsp = {
|
||||
.info = pMsg->info,
|
||||
.code = 0,
|
||||
};
|
||||
SRpcMsg rsp = { .info = pMsg->info, .code = 0};
|
||||
streamProcessRetrieveReq(pTask, &req, &rsp);
|
||||
streamMetaReleaseTask(pSnode->pMeta, pTask);
|
||||
tDeleteStreamRetrieveReq(&req);
|
||||
|
|
|
|||
|
|
@ -198,6 +198,7 @@ void *tsdbGetIdx(SMeta *pMeta);
|
|||
void *tsdbGetIvtIdx(SMeta *pMeta);
|
||||
uint64_t tsdbGetReaderMaxVersion(STsdbReader *pReader);
|
||||
void tsdbReaderSetCloseFlag(STsdbReader *pReader);
|
||||
int64_t tsdbGetLastTimestamp(SVnode* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr);
|
||||
|
||||
int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables);
|
||||
int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
|
||||
|
|
|
|||
|
|
@ -760,6 +760,7 @@ void freePtr(void *ptr) {
|
|||
|
||||
int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
|
||||
int32_t vgId = TD_VID(pTq->pVnode);
|
||||
|
||||
pTask->id.idStr = createStreamTaskIdStr(pTask->id.streamId, pTask->id.taskId);
|
||||
pTask->refCnt = 1;
|
||||
pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
|
||||
|
|
@ -837,7 +838,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
|
|||
}
|
||||
|
||||
if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
|
||||
SWalFilterCond cond = {.deleteMsg = 1};
|
||||
SWalFilterCond cond = {.deleteMsg = 1}; // delete msg also extract from wal files
|
||||
pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond);
|
||||
}
|
||||
|
||||
|
|
@ -855,11 +856,14 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
|
|||
char* msgStr = pMsg->pCont;
|
||||
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
|
||||
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
|
||||
|
||||
SStreamTaskCheckReq req;
|
||||
SDecoder decoder;
|
||||
|
||||
tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen);
|
||||
tDecodeSStreamTaskCheckReq(&decoder, &req);
|
||||
tDecoderClear(&decoder);
|
||||
|
||||
int32_t taskId = req.downstreamTaskId;
|
||||
SStreamTaskCheckRsp rsp = {
|
||||
.reqId = req.reqId,
|
||||
|
|
@ -873,18 +877,18 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
|
|||
|
||||
SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
|
||||
|
||||
if (pTask) {
|
||||
if (pTask != NULL) {
|
||||
rsp.status = streamTaskCheckStatus(pTask);
|
||||
streamMetaReleaseTask(pTq->pStreamMeta, pTask);
|
||||
|
||||
tqDebug("tq recv task check req(reqId:0x%" PRIx64
|
||||
tqDebug("s-task:%s recv task check req(reqId:0x%" PRIx64
|
||||
") %d at node %d task status:%d, check req from task %d at node %d, rsp status %d",
|
||||
rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, pTask->status.taskStatus, rsp.upstreamTaskId,
|
||||
rsp.upstreamNodeId, rsp.status);
|
||||
pTask->id.idStr, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, pTask->status.taskStatus,
|
||||
rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
|
||||
} else {
|
||||
rsp.status = 0;
|
||||
tqDebug("tq recv task check(taskId:%d not built yet) req(reqId:0x%" PRIx64
|
||||
") %d at node %d, check req from task %d at node %d, rsp status %d",
|
||||
tqDebug("tq recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64
|
||||
") %d at node %d, check req from task:0x%x at node %d, rsp status %d",
|
||||
taskId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId,
|
||||
rsp.status);
|
||||
}
|
||||
|
|
@ -892,9 +896,10 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
|
|||
SEncoder encoder;
|
||||
int32_t code;
|
||||
int32_t len;
|
||||
|
||||
tEncodeSize(tEncodeSStreamTaskCheckRsp, &rsp, len, code);
|
||||
if (code < 0) {
|
||||
tqError("unable to encode rsp %d", __LINE__);
|
||||
tqError("vgId:%d failed to encode task check rsp, task:0x%x", pTq->pStreamMeta->vgId, taskId);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -907,6 +912,7 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
|
|||
tEncoderClear(&encoder);
|
||||
|
||||
SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = pMsg->info};
|
||||
|
||||
tmsgSendRsp(&rspMsg);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -918,17 +924,20 @@ int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t sversion, char* msg, int32
|
|||
SDecoder decoder;
|
||||
tDecoderInit(&decoder, (uint8_t*)msg, msgLen);
|
||||
code = tDecodeSStreamTaskCheckRsp(&decoder, &rsp);
|
||||
|
||||
if (code < 0) {
|
||||
tDecoderClear(&decoder);
|
||||
return -1;
|
||||
}
|
||||
|
||||
tDecoderClear(&decoder);
|
||||
tqDebug("tq recv task check rsp(reqId:0x%" PRIx64 ") %d at node %d check req from task %d at node %d, status %d",
|
||||
tqDebug("tq recv task check rsp(reqId:0x%" PRIx64 ") %d at node %d check req from task:0x%x at node %d, status %d",
|
||||
rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
|
||||
|
||||
SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, rsp.upstreamTaskId);
|
||||
if (pTask == NULL) {
|
||||
tqError("tq failed to locate the stream task:0x%x vgId:%d, it may have been destroyed", rsp.upstreamTaskId,
|
||||
pTq->pStreamMeta->vgId);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -948,6 +957,8 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
|
|||
// 1.deserialize msg and build task
|
||||
SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
|
||||
if (pTask == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
tqError("vgId:%d failed to create stream task due to out of memory, alloc size:%d", vgId, (int32_t) sizeof(SStreamTask));
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -965,9 +976,9 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
|
|||
// 2.save task, use the newest commit version as the initial start version of stream task.
|
||||
taosWLockLatch(&pTq->pStreamMeta->lock);
|
||||
code = streamMetaAddDeployedTask(pTq->pStreamMeta, sversion, pTask);
|
||||
int32_t numOfTasks = streamMetaGetNumOfTasks(pTq->pStreamMeta);
|
||||
if (code < 0) {
|
||||
tqError("vgId:%d failed to add s-task:%s, total:%d", vgId, pTask->id.idStr,
|
||||
streamMetaGetNumOfTasks(pTq->pStreamMeta));
|
||||
tqError("vgId:%d failed to add s-task:%s, total:%d", vgId, pTask->id.idStr, numOfTasks);
|
||||
taosWUnLockLatch(&pTq->pStreamMeta->lock);
|
||||
return -1;
|
||||
}
|
||||
|
|
@ -980,7 +991,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
|
|||
}
|
||||
|
||||
tqDebug("vgId:%d s-task:%s is deployed and add meta from mnd, status:%d, total:%d", vgId, pTask->id.idStr,
|
||||
pTask->status.taskStatus, streamMetaGetNumOfTasks(pTq->pStreamMeta));
|
||||
pTask->status.taskStatus, numOfTasks);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -1229,7 +1240,9 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) {
|
|||
char* msgStr = pMsg->pCont;
|
||||
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
|
||||
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
|
||||
SStreamDispatchReq req;
|
||||
|
||||
SStreamDispatchReq req = {0};
|
||||
|
||||
SDecoder decoder;
|
||||
tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen);
|
||||
tDecodeStreamDispatchReq(&decoder, &req);
|
||||
|
|
@ -1237,7 +1250,7 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) {
|
|||
SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.taskId);
|
||||
if (pTask) {
|
||||
SRpcMsg rsp = {.info = pMsg->info, .code = 0};
|
||||
streamProcessDispatchReq(pTask, &req, &rsp, exec);
|
||||
streamProcessDispatchMsg(pTask, &req, &rsp, exec);
|
||||
streamMetaReleaseTask(pTq->pStreamMeta, pTask);
|
||||
return 0;
|
||||
} else {
|
||||
|
|
@ -1356,7 +1369,7 @@ int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) {
|
|||
SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
|
||||
if (pTask) {
|
||||
SRpcMsg rsp = {.info = pMsg->info, .code = 0};
|
||||
streamProcessDispatchReq(pTask, &req, &rsp, false);
|
||||
streamProcessDispatchMsg(pTask, &req, &rsp, false);
|
||||
streamMetaReleaseTask(pTq->pStreamMeta, pTask);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ int32_t createStreamTaskRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) {
|
|||
|
||||
int32_t status = pTask->status.taskStatus;
|
||||
if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
|
||||
tqDebug("s-task:%s level:%d not source task, no need to start", pTask->id.idStr, pTask->taskLevel);
|
||||
// tqTrace("s-task:%s level:%d not source task, no need to start", pTask->id.idStr, pTask->taskLevel);
|
||||
streamMetaReleaseTask(pStreamMeta, pTask);
|
||||
continue;
|
||||
}
|
||||
|
|
@ -132,7 +132,7 @@ int32_t createStreamTaskRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) {
|
|||
}
|
||||
|
||||
if (tInputQueueIsFull(pTask)) {
|
||||
tqDebug("s-task:%s input queue is full, do nothing", pTask->id.idStr);
|
||||
tqTrace("s-task:%s input queue is full, do nothing", pTask->id.idStr);
|
||||
streamMetaReleaseTask(pStreamMeta, pTask);
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5506,3 +5506,58 @@ void tsdbReaderSetId(STsdbReader* pReader, const char* idstr) {
|
|||
}
|
||||
|
||||
void tsdbReaderSetCloseFlag(STsdbReader* pReader) { pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED; }
|
||||
|
||||
/*-------------todo:refactor the implementation of those APIs in this file to seperate the API into two files------*/
|
||||
// opt perf, do NOT create so many readers
|
||||
int64_t tsdbGetLastTimestamp(SVnode* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr) {
|
||||
SQueryTableDataCond cond = {.type = TIMEWINDOW_RANGE_CONTAINED, .numOfCols = 1, .order = TSDB_ORDER_DESC,
|
||||
.startVersion = -1, .endVersion = -1};
|
||||
cond.twindows.skey = INT64_MIN;
|
||||
cond.twindows.ekey = INT64_MAX;
|
||||
|
||||
cond.colList = taosMemoryCalloc(1, sizeof(SColumnInfo));
|
||||
cond.pSlotList = taosMemoryMalloc(sizeof(int32_t) * cond.numOfCols);
|
||||
if (cond.colList == NULL || cond.pSlotList == NULL) {
|
||||
// todo
|
||||
}
|
||||
|
||||
cond.colList[0].colId = 1;
|
||||
cond.colList[0].slotId = 0;
|
||||
cond.colList[0].type = TSDB_DATA_TYPE_TIMESTAMP;
|
||||
|
||||
cond.pSlotList[0] = 0;
|
||||
|
||||
STableKeyInfo* pTableKeyInfo = pTableList;
|
||||
STsdbReader* pReader = NULL;
|
||||
SSDataBlock* pBlock = createDataBlock();
|
||||
|
||||
SColumnInfoData data = {0};
|
||||
data.info = (SColumnInfo) {.type = TSDB_DATA_TYPE_TIMESTAMP, .colId = 1, .bytes = TSDB_KEYSIZE};
|
||||
blockDataAppendColInfo(pBlock, &data);
|
||||
|
||||
int64_t key = INT64_MIN;
|
||||
|
||||
for(int32_t i = 0; i < numOfTables; ++i) {
|
||||
int32_t code = tsdbReaderOpen(pVnode, &cond, &pTableKeyInfo[i], 1, pBlock, &pReader, pIdStr, false, NULL);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
}
|
||||
|
||||
bool hasData = false;
|
||||
code = tsdbNextDataBlock(pReader, &hasData);
|
||||
if (!hasData || code != TSDB_CODE_SUCCESS) {
|
||||
continue;
|
||||
}
|
||||
|
||||
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, 0);
|
||||
int64_t k = *(int64_t*)pCol->pData;
|
||||
|
||||
if (key < k) {
|
||||
key = k;
|
||||
}
|
||||
|
||||
tsdbReaderClose(pReader);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -551,9 +551,9 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
|
|||
|
||||
// start to restore all stream tasks
|
||||
if (tsDisableStream) {
|
||||
vInfo("vgId:%d, not restore stream tasks, since disabled", pVnode->config.vgId);
|
||||
vInfo("vgId:%d, not launch stream tasks, since stream tasks are disabled", pVnode->config.vgId);
|
||||
} else {
|
||||
vInfo("vgId:%d start to restore stream tasks", pVnode->config.vgId);
|
||||
vInfo("vgId:%d start to launch stream tasks", pVnode->config.vgId);
|
||||
tqStartStreamTasks(pVnode->pTq);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -885,7 +885,8 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
|
|||
goto _error;
|
||||
}
|
||||
|
||||
pInfo->rowCapacity = blockDataGetCapacityInRow(pInfo->binfo.pRes, getBufPageSize(pInfo->pBuf));
|
||||
pInfo->rowCapacity = blockDataGetCapacityInRow(pInfo->binfo.pRes, getBufPageSize(pInfo->pBuf),
|
||||
blockDataGetSerialMetaSize(taosArrayGetSize(pInfo->binfo.pRes->pDataBlock)));
|
||||
pInfo->columnOffset = setupColumnOffset(pInfo->binfo.pRes, pInfo->rowCapacity);
|
||||
code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pInfo->pGroupCols);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
|
|
|||
|
|
@ -209,6 +209,45 @@ static bool isGroupKeyFunc(SExprInfo* pExprInfo) {
|
|||
return (functionType == FUNCTION_TYPE_GROUP_KEY);
|
||||
}
|
||||
|
||||
static bool getIgoreNullRes(SExprSupp* pExprSup) {
|
||||
for (int32_t i = 0; i < pExprSup->numOfExprs; ++i) {
|
||||
SExprInfo* pExprInfo = &pExprSup->pExprInfo[i];
|
||||
|
||||
if (isInterpFunc(pExprInfo)) {
|
||||
for (int32_t j = 0; j < pExprInfo->base.numOfParams; ++j) {
|
||||
SFunctParam *pFuncParam = &pExprInfo->base.pParam[j];
|
||||
if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) {
|
||||
return pFuncParam->param.i ? true : false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool checkNullRow(SExprSupp* pExprSup, SSDataBlock* pSrcBlock, int32_t index, bool ignoreNull) {
|
||||
if (!ignoreNull) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
|
||||
SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
|
||||
|
||||
if (isInterpFunc(pExprInfo)) {
|
||||
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
|
||||
SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot);
|
||||
|
||||
if (colDataIsNull_s(pSrc, index)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock,
|
||||
SSDataBlock* pSrcBlock, int32_t index, bool beforeTs) {
|
||||
int32_t rows = pResBlock->info.rows;
|
||||
|
|
@ -271,15 +310,27 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
|
|||
|
||||
if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) {
|
||||
float v = 0;
|
||||
if (!IS_VAR_DATA_TYPE(pVar->nType)) {
|
||||
GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
|
||||
} else {
|
||||
v = taosStr2Float(varDataVal(pVar->pz), NULL);
|
||||
}
|
||||
colDataSetVal(pDst, rows, (char*)&v, false);
|
||||
} else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) {
|
||||
double v = 0;
|
||||
if (!IS_VAR_DATA_TYPE(pVar->nType)) {
|
||||
GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
|
||||
} else {
|
||||
v = taosStr2Double(varDataVal(pVar->pz), NULL);
|
||||
}
|
||||
colDataSetVal(pDst, rows, (char*)&v, false);
|
||||
} else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) {
|
||||
int64_t v = 0;
|
||||
if (!IS_VAR_DATA_TYPE(pVar->nType)) {
|
||||
GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
|
||||
} else {
|
||||
v = taosStr2int64(varDataVal(pVar->pz));
|
||||
}
|
||||
colDataSetVal(pDst, rows, (char*)&v, false);
|
||||
} else if (IS_BOOLEAN_TYPE(pDst->info.type)) {
|
||||
bool v = false;
|
||||
|
|
@ -590,7 +641,7 @@ static int32_t resetKeeperInfo(STimeSliceOperatorInfo* pInfo) {
|
|||
}
|
||||
|
||||
static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pSliceInfo, SSDataBlock* pBlock,
|
||||
SExecTaskInfo* pTaskInfo) {
|
||||
SExecTaskInfo* pTaskInfo, bool ignoreNull) {
|
||||
SSDataBlock* pResBlock = pSliceInfo->pRes;
|
||||
SInterval* pInterval = &pSliceInfo->interval;
|
||||
|
||||
|
|
@ -603,6 +654,10 @@ static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pS
|
|||
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_FUNC_DUP_TIMESTAMP);
|
||||
}
|
||||
|
||||
if (checkNullRow(&pOperator->exprSupp, pBlock, i, ignoreNull)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (pSliceInfo->current > pSliceInfo->win.ekey) {
|
||||
break;
|
||||
}
|
||||
|
|
@ -732,6 +787,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
|
|||
STimeSliceOperatorInfo* pSliceInfo = pOperator->info;
|
||||
SSDataBlock* pResBlock = pSliceInfo->pRes;
|
||||
SExprSupp* pSup = &pOperator->exprSupp;
|
||||
bool ignoreNull = getIgoreNullRes(pSup);
|
||||
|
||||
int32_t order = TSDB_ORDER_ASC;
|
||||
SInterval* pInterval = &pSliceInfo->interval;
|
||||
|
|
@ -742,7 +798,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
|
|||
while (1) {
|
||||
if (pSliceInfo->pNextGroupRes != NULL) {
|
||||
setInputDataBlock(pSup, pSliceInfo->pNextGroupRes, order, MAIN_SCAN, true);
|
||||
doTimesliceImpl(pOperator, pSliceInfo, pSliceInfo->pNextGroupRes, pTaskInfo);
|
||||
doTimesliceImpl(pOperator, pSliceInfo, pSliceInfo->pNextGroupRes, pTaskInfo, ignoreNull);
|
||||
copyPrevGroupKey(&pOperator->exprSupp, pSliceInfo->pPrevGroupKey, pSliceInfo->pNextGroupRes);
|
||||
pSliceInfo->pNextGroupRes = NULL;
|
||||
}
|
||||
|
|
@ -776,7 +832,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
|
|||
|
||||
// the pDataBlock are always the same one, no need to call this again
|
||||
setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true);
|
||||
doTimesliceImpl(pOperator, pSliceInfo, pBlock, pTaskInfo);
|
||||
doTimesliceImpl(pOperator, pSliceInfo, pBlock, pTaskInfo, ignoreNull);
|
||||
copyPrevGroupKey(&pOperator->exprSupp, pSliceInfo->pPrevGroupKey, pBlock);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -533,7 +533,8 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
|
|||
pHandle->numOfPages);
|
||||
}
|
||||
|
||||
int32_t numOfRows = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize);
|
||||
int32_t numOfRows = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize,
|
||||
blockDataGetSerialMetaSize(taosArrayGetSize(pHandle->pDataBlock->pDataBlock)));
|
||||
blockDataEnsureCapacity(pHandle->pDataBlock, numOfRows);
|
||||
|
||||
// the initial pass + sortPass + final mergePass
|
||||
|
|
|
|||
|
|
@ -1568,8 +1568,7 @@ static int32_t translateInterp(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
|
|||
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
|
||||
// if (1 != numOfParams && 3 != numOfParams && 4 != numOfParams) {
|
||||
if (1 != numOfParams) {
|
||||
if (2 < numOfParams) {
|
||||
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
|
|
@ -1579,6 +1578,22 @@ static int32_t translateInterp(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
|
|||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
if (2 == numOfParams) {
|
||||
nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, 1));
|
||||
paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type;
|
||||
if (!IS_INTEGER_TYPE(paraType) || QUERY_NODE_VALUE != nodeType) {
|
||||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1);
|
||||
if (pValue->datum.i != 0 && pValue->datum.i != 1) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"INTERP function second parameter should be 0/1");
|
||||
}
|
||||
|
||||
pValue->notReserved = true;
|
||||
}
|
||||
|
||||
#if 0
|
||||
if (3 <= numOfParams) {
|
||||
int64_t timeVal[2] = {0};
|
||||
|
|
@ -1624,6 +1639,15 @@ static int32_t translateInterp(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static EFuncReturnRows interpEstReturnRows(SFunctionNode* pFunc) {
|
||||
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
|
||||
if (1 < numOfParams && 1 == ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 1))->datum.i) {
|
||||
return FUNC_RETURN_ROWS_INDEFINITE;
|
||||
} else {
|
||||
return FUNC_RETURN_ROWS_N;
|
||||
}
|
||||
}
|
||||
|
||||
static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
|
||||
// forbid null as first/last input, since first(c0, null, 1) may have different number of input
|
||||
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
|
||||
|
|
@ -2484,7 +2508,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
.getEnvFunc = getSelectivityFuncEnv,
|
||||
.initFunc = functionSetup,
|
||||
.processFunc = NULL,
|
||||
.finalizeFunc = NULL
|
||||
.finalizeFunc = NULL,
|
||||
.estimateReturnRowsFunc = interpEstReturnRows
|
||||
},
|
||||
{
|
||||
.name = "derivative",
|
||||
|
|
|
|||
|
|
@ -1523,14 +1523,15 @@ static int32_t translateInterpFunc(STranslateContext* pCxt, SFunctionNode* pFunc
|
|||
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
|
||||
SNode* pTable = pSelect->pFromTable;
|
||||
|
||||
if ((NULL != pTable && QUERY_NODE_REAL_TABLE != nodeType(pTable))) {
|
||||
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE,
|
||||
"%s is only supported in single table query", pFunc->functionName);
|
||||
}
|
||||
|
||||
if (pSelect->hasAggFuncs || pSelect->hasMultiRowsFunc || pSelect->hasIndefiniteRowsFunc) {
|
||||
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
|
||||
}
|
||||
|
||||
if (pSelect->hasInterpFunc && (FUNC_RETURN_ROWS_INDEFINITE == pSelect->returnRows || pSelect->returnRows != fmGetFuncReturnRows(pFunc))) {
|
||||
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
|
||||
"%s ignoring null value options cannot be used when applying to multiple columns", pFunc->functionName);
|
||||
}
|
||||
|
||||
if (NULL != pSelect->pWindow || NULL != pSelect->pGroupByList) {
|
||||
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
|
||||
"%s function is not supported in window query or group query", pFunc->functionName);
|
||||
|
|
@ -1731,7 +1732,10 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) {
|
|||
if (fmIsIndefiniteRowsFunc(pFunc->funcId)) {
|
||||
pSelect->hasIndefiniteRowsFunc = true;
|
||||
pSelect->returnRows = fmGetFuncReturnRows(pFunc);
|
||||
} else if (fmIsInterpFunc(pFunc->funcId)) {
|
||||
pSelect->returnRows = fmGetFuncReturnRows(pFunc);
|
||||
}
|
||||
|
||||
pSelect->hasMultiRowsFunc = pSelect->hasMultiRowsFunc ? true : fmIsMultiRowsFunc(pFunc->funcId);
|
||||
if (fmIsSelectFunc(pFunc->funcId)) {
|
||||
pSelect->hasSelectFunc = true;
|
||||
|
|
|
|||
|
|
@ -305,6 +305,7 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) {
|
|||
dst[j] = src[k];
|
||||
j++;
|
||||
}
|
||||
if (j >= dlen) j = dlen - 1;
|
||||
dst[j] = '\0';
|
||||
return j;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,8 +33,12 @@ typedef struct {
|
|||
|
||||
static SStreamGlobalEnv streamEnv;
|
||||
|
||||
int32_t streamDispatch(SStreamTask* pTask);
|
||||
int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData);
|
||||
int32_t streamDispatchStreamBlock(SStreamTask* pTask);
|
||||
|
||||
SStreamDataBlock* createStreamDataFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg);
|
||||
SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize, SArray* pRes);
|
||||
void destroyStreamDataBlock(SStreamDataBlock* pBlock);
|
||||
|
||||
int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData);
|
||||
int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* data);
|
||||
|
||||
|
|
|
|||
|
|
@ -120,39 +120,35 @@ int32_t streamSchedExec(SStreamTask* pTask) {
|
|||
}
|
||||
|
||||
int32_t streamTaskEnqueueBlocks(SStreamTask* pTask, const SStreamDispatchReq* pReq, SRpcMsg* pRsp) {
|
||||
SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0);
|
||||
int8_t status;
|
||||
int8_t status = 0;
|
||||
|
||||
// enqueue data block
|
||||
if (pData != NULL) {
|
||||
pData->type = STREAM_INPUT__DATA_BLOCK;
|
||||
pData->srcVgId = pReq->dataSrcVgId;
|
||||
// decode
|
||||
/*pData->blocks = pReq->data;*/
|
||||
/*pBlock->sourceVer = pReq->sourceVer;*/
|
||||
streamDispatchReqToData(pReq, pData);
|
||||
if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pData) == 0) {
|
||||
SStreamDataBlock* pBlock = createStreamDataFromDispatchMsg(pReq, STREAM_INPUT__DATA_BLOCK, pReq->dataSrcVgId);
|
||||
if (pBlock == NULL) {
|
||||
streamTaskInputFail(pTask);
|
||||
status = TASK_INPUT_STATUS__FAILED;
|
||||
qDebug("vgId:%d, s-task:%s failed to receive dispatch msg, reason: out of memory", pTask->pMeta->vgId,
|
||||
pTask->id.idStr);
|
||||
} else {
|
||||
if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pBlock) == 0) {
|
||||
status = TASK_INPUT_STATUS__NORMAL;
|
||||
} else { // input queue is full, upstream is blocked now
|
||||
status = TASK_INPUT_STATUS__BLOCKED;
|
||||
}
|
||||
} else {
|
||||
streamTaskInputFail(pTask);
|
||||
status = TASK_INPUT_STATUS__FAILED;
|
||||
}
|
||||
|
||||
// rsp by input status
|
||||
void* buf = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp));
|
||||
((SMsgHead*)buf)->vgId = htonl(pReq->upstreamNodeId);
|
||||
SStreamDispatchRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead));
|
||||
pCont->inputStatus = status;
|
||||
pCont->streamId = htobe64(pReq->streamId);
|
||||
pCont->upstreamNodeId = htonl(pReq->upstreamNodeId);
|
||||
pCont->upstreamTaskId = htonl(pReq->upstreamTaskId);
|
||||
pCont->downstreamNodeId = htonl(pTask->nodeId);
|
||||
pCont->downstreamTaskId = htonl(pTask->id.taskId);
|
||||
pRsp->pCont = buf;
|
||||
SStreamDispatchRsp* pDispatchRsp = POINTER_SHIFT(buf, sizeof(SMsgHead));
|
||||
|
||||
pDispatchRsp->inputStatus = status;
|
||||
pDispatchRsp->streamId = htobe64(pReq->streamId);
|
||||
pDispatchRsp->upstreamNodeId = htonl(pReq->upstreamNodeId);
|
||||
pDispatchRsp->upstreamTaskId = htonl(pReq->upstreamTaskId);
|
||||
pDispatchRsp->downstreamNodeId = htonl(pTask->nodeId);
|
||||
pDispatchRsp->downstreamTaskId = htonl(pTask->id.taskId);
|
||||
|
||||
pRsp->pCont = buf;
|
||||
pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp);
|
||||
tmsgSendRsp(pRsp);
|
||||
|
||||
|
|
@ -165,7 +161,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq,
|
|||
|
||||
// enqueue
|
||||
if (pData != NULL) {
|
||||
qDebug("task %d(child %d) recv retrieve req from task %d, reqId %" PRId64, pTask->id.taskId, pTask->selfChildId,
|
||||
qDebug("s-task:%s (child %d) recv retrieve req from task:0x%x, reqId %" PRId64, pTask->id.idStr, pTask->selfChildId,
|
||||
pReq->srcTaskId, pReq->reqId);
|
||||
|
||||
pData->type = STREAM_INPUT__DATA_RETRIEVE;
|
||||
|
|
@ -197,30 +193,31 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq,
|
|||
return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1;
|
||||
}
|
||||
|
||||
int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) {
|
||||
// todo add log
|
||||
int32_t streamTaskOutputResultBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) {
|
||||
int32_t code = 0;
|
||||
if (pTask->outputType == TASK_OUTPUT__TABLE) {
|
||||
pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks);
|
||||
taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes);
|
||||
taosFreeQitem(pBlock);
|
||||
destroyStreamDataBlock(pBlock);
|
||||
} else if (pTask->outputType == TASK_OUTPUT__SMA) {
|
||||
pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks);
|
||||
taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes);
|
||||
taosFreeQitem(pBlock);
|
||||
destroyStreamDataBlock(pBlock);
|
||||
} else {
|
||||
ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH);
|
||||
code = taosWriteQitem(pTask->outputQueue->queue, pBlock);
|
||||
if (code != 0) {
|
||||
if (code != 0) { // todo failed to add it into the output queue, free it.
|
||||
return code;
|
||||
}
|
||||
streamDispatch(pTask);
|
||||
|
||||
streamDispatchStreamBlock(pTask);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp, bool exec) {
|
||||
qDebug("s-task:%s receive dispatch msg from taskId:%d(vgId:%d)", pTask->id.idStr, pReq->upstreamTaskId,
|
||||
pReq->upstreamNodeId);
|
||||
int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp, bool exec) {
|
||||
qDebug("s-task:%s receive dispatch msg from taskId:0x%x(vgId:%d), msgLen:%" PRId64, pTask->id.idStr,
|
||||
pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->totalLen);
|
||||
|
||||
// todo add the input queue buffer limitation
|
||||
streamTaskEnqueueBlocks(pTask, pReq, pRsp);
|
||||
|
|
@ -257,8 +254,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
|
|||
return 0;
|
||||
}
|
||||
|
||||
// continue dispatch
|
||||
streamDispatch(pTask);
|
||||
// continue dispatch one block to down stream in pipeline
|
||||
streamDispatchStreamBlock(pTask);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -268,13 +265,13 @@ int32_t streamProcessRunReq(SStreamTask* pTask) {
|
|||
}
|
||||
|
||||
/*if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {*/
|
||||
/*streamDispatch(pTask);*/
|
||||
/*streamDispatchStreamBlock(pTask);*/
|
||||
/*}*/
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) {
|
||||
qDebug("s-task:%s receive retrieve req from node %d taskId:%d", pTask->id.idStr, pReq->srcNodeId, pReq->srcTaskId);
|
||||
qDebug("s-task:%s receive retrieve req from node %d taskId:0x%x", pTask->id.idStr, pReq->srcNodeId, pReq->srcTaskId);
|
||||
streamTaskEnqueueRetrieve(pTask, pReq, pRsp);
|
||||
|
||||
ASSERT(pTask->taskLevel != TASK_LEVEL__SINK);
|
||||
|
|
@ -294,16 +291,13 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) {
|
|||
double size = QUEUE_MEM_SIZE_IN_MB(pTask->inputQueue->queue);
|
||||
|
||||
if (type == STREAM_INPUT__DATA_SUBMIT) {
|
||||
int32_t numOfBlocks = taosQueueItemSize(pTask->inputQueue->queue) + 1;
|
||||
double size = QUEUE_MEM_SIZE_IN_MB(pTask->inputQueue->queue);
|
||||
|
||||
SStreamDataSubmit* px = (SStreamDataSubmit*)pItem;
|
||||
qDebug("s-task:%s submit enqueue msgLen:%d ver:%" PRId64 ", total in queue:%d, size:%.2fMiB", pTask->id.idStr,
|
||||
px->submit.msgLen, px->submit.ver, numOfBlocks, size);
|
||||
px->submit.msgLen, px->submit.ver, total, size);
|
||||
|
||||
if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && tInputQueueIsFull(pTask)) {
|
||||
qError("s-task:%s input queue is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) abort",
|
||||
pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, numOfBlocks,
|
||||
pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, total,
|
||||
size);
|
||||
streamDataSubmitDestroy(px);
|
||||
taosFreeQitem(pItem);
|
||||
|
|
@ -312,22 +306,20 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) {
|
|||
taosWriteQitem(pTask->inputQueue->queue, pItem);
|
||||
} else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE ||
|
||||
type == STREAM_INPUT__REF_DATA_BLOCK) {
|
||||
int32_t numOfBlocks = taosQueueItemSize(pTask->inputQueue->queue) + 1;
|
||||
double size = QUEUE_MEM_SIZE_IN_MB(pTask->inputQueue->queue);
|
||||
|
||||
if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && (tInputQueueIsFull(pTask))) {
|
||||
qError("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort",
|
||||
pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, numOfBlocks,
|
||||
pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, total,
|
||||
size);
|
||||
return -1;
|
||||
}
|
||||
|
||||
qDebug("s-task:%s data block enqueue, total in queue:%d", pTask->id.idStr, numOfBlocks);
|
||||
qDebug("s-task:%s data block enqueue, current(blocks:%d, size:%.2fMiB)", pTask->id.idStr, total, size);
|
||||
taosWriteQitem(pTask->inputQueue->queue, pItem);
|
||||
} else if (type == STREAM_INPUT__CHECKPOINT) {
|
||||
taosWriteQitem(pTask->inputQueue->queue, pItem);
|
||||
} else if (type == STREAM_INPUT__GET_RES) {
|
||||
taosWriteQitem(pTask->inputQueue->queue, pItem);
|
||||
qDebug("s-task:%s data res enqueue, current(blocks:%d, size:%.2fMiB)", pTask->id.idStr, total, size);
|
||||
}
|
||||
|
||||
if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) {
|
||||
|
|
|
|||
|
|
@ -15,20 +15,28 @@
|
|||
|
||||
#include "streamInc.h"
|
||||
|
||||
int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData) {
|
||||
SStreamDataBlock* createStreamDataFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg) {
|
||||
SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, pReq->totalLen);
|
||||
if (pData == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
pData->type = blockType;
|
||||
pData->srcVgId = srcVg;
|
||||
|
||||
int32_t blockNum = pReq->blockNum;
|
||||
SArray* pArray = taosArrayInit_s(sizeof(SSDataBlock), blockNum);
|
||||
if (pArray == NULL) {
|
||||
return -1;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ASSERT(pReq->blockNum == taosArrayGetSize(pReq->data));
|
||||
ASSERT(pReq->blockNum == taosArrayGetSize(pReq->dataLen));
|
||||
ASSERT((pReq->blockNum == taosArrayGetSize(pReq->data)) && (pReq->blockNum == taosArrayGetSize(pReq->dataLen)));
|
||||
|
||||
for (int32_t i = 0; i < blockNum; i++) {
|
||||
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*) taosArrayGetP(pReq->data, i);
|
||||
SSDataBlock* pDataBlock = taosArrayGet(pArray, i);
|
||||
blockDecode(pDataBlock, pRetrieve->data);
|
||||
|
||||
// TODO: refactor
|
||||
pDataBlock->info.window.skey = be64toh(pRetrieve->skey);
|
||||
pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey);
|
||||
|
|
@ -39,8 +47,41 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock
|
|||
pDataBlock->info.type = pRetrieve->streamBlockType;
|
||||
pDataBlock->info.childId = pReq->upstreamChildId;
|
||||
}
|
||||
|
||||
pData->blocks = pArray;
|
||||
return 0;
|
||||
return pData;
|
||||
}
|
||||
|
||||
SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize, SArray* pRes) {
|
||||
SStreamDataBlock* pStreamBlocks = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, resultSize);
|
||||
if (pStreamBlocks == NULL) {
|
||||
taosArrayClearEx(pRes, (FDelete)blockDataFreeRes);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
pStreamBlocks->type = STREAM_INPUT__DATA_BLOCK;
|
||||
pStreamBlocks->blocks = pRes;
|
||||
|
||||
if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
|
||||
SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)pItem;
|
||||
pStreamBlocks->childId = pTask->selfChildId;
|
||||
pStreamBlocks->sourceVer = pSubmit->ver;
|
||||
} else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) {
|
||||
SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)pItem;
|
||||
pStreamBlocks->childId = pTask->selfChildId;
|
||||
pStreamBlocks->sourceVer = pMerged->ver;
|
||||
}
|
||||
|
||||
return pStreamBlocks;
|
||||
}
|
||||
|
||||
void destroyStreamDataBlock(SStreamDataBlock* pBlock) {
|
||||
if (pBlock == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes);
|
||||
taosFreeQitem(pBlock);
|
||||
}
|
||||
|
||||
int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData) {
|
||||
|
|
@ -184,11 +225,13 @@ void streamFreeQitem(SStreamQueueItem* data) {
|
|||
taosFreeQitem(data);
|
||||
} else if (type == STREAM_INPUT__MERGED_SUBMIT) {
|
||||
SStreamMergedSubmit* pMerge = (SStreamMergedSubmit*)data;
|
||||
|
||||
int32_t sz = taosArrayGetSize(pMerge->submits);
|
||||
for (int32_t i = 0; i < sz; i++) {
|
||||
int32_t* pRef = taosArrayGetP(pMerge->dataRefs, i);
|
||||
int32_t ref = atomic_sub_fetch_32(pRef, 1);
|
||||
ASSERT(ref >= 0);
|
||||
|
||||
if (ref == 0) {
|
||||
SPackedData* pSubmit = (SPackedData*)taosArrayGet(pMerge->submits, i);
|
||||
taosMemoryFree(pSubmit->msgStr);
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* p
|
|||
if (tEncodeI32(pEncoder, pReq->upstreamChildId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1;
|
||||
if (tEncodeI32(pEncoder, pReq->blockNum) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pReq->totalLen) < 0) return -1;
|
||||
ASSERT(taosArrayGetSize(pReq->data) == pReq->blockNum);
|
||||
ASSERT(taosArrayGetSize(pReq->dataLen) == pReq->blockNum);
|
||||
for (int32_t i = 0; i < pReq->blockNum; i++) {
|
||||
|
|
@ -45,6 +46,8 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) {
|
|||
if (tDecodeI32(pDecoder, &pReq->upstreamChildId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1;
|
||||
if (tDecodeI32(pDecoder, &pReq->blockNum) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pReq->totalLen) < 0) return -1;
|
||||
|
||||
ASSERT(pReq->blockNum > 0);
|
||||
pReq->data = taosArrayInit(pReq->blockNum, sizeof(void*));
|
||||
pReq->dataLen = taosArrayInit(pReq->blockNum, sizeof(int32_t));
|
||||
|
|
@ -135,7 +138,6 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
|
|||
SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->childEpInfo, i);
|
||||
req.dstNodeId = pEpInfo->nodeId;
|
||||
req.dstTaskId = pEpInfo->taskId;
|
||||
int32_t code;
|
||||
int32_t len;
|
||||
tEncodeSize(tEncodeStreamRetrieveReq, &req, len, code);
|
||||
if (code < 0) {
|
||||
|
|
@ -155,30 +157,25 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
|
|||
tEncodeStreamRetrieveReq(&encoder, &req);
|
||||
tEncoderClear(&encoder);
|
||||
|
||||
SRpcMsg rpcMsg = {
|
||||
.code = 0,
|
||||
.msgType = TDMT_STREAM_RETRIEVE,
|
||||
.pCont = buf,
|
||||
.contLen = sizeof(SMsgHead) + len,
|
||||
};
|
||||
|
||||
SRpcMsg rpcMsg = { .code = 0, .msgType = TDMT_STREAM_RETRIEVE, .pCont = buf, .contLen = sizeof(SMsgHead) + len };
|
||||
if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) {
|
||||
ASSERT(0);
|
||||
goto CLEAR;
|
||||
}
|
||||
buf = NULL;
|
||||
|
||||
qDebug("s-task:%s (child %d) send retrieve req to task %d at node %d, reqId %" PRId64, pTask->id.idStr,
|
||||
buf = NULL;
|
||||
qDebug("s-task:%s (child %d) send retrieve req to task %d at node %d, reqId:0x%" PRIx64, pTask->id.idStr,
|
||||
pTask->selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req.reqId);
|
||||
}
|
||||
code = 0;
|
||||
|
||||
CLEAR:
|
||||
taosMemoryFree(pRetrieve);
|
||||
rpcFreeCont(buf);
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) {
|
||||
static int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) {
|
||||
int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
|
||||
void* buf = taosMemoryCalloc(1, dataStrLen);
|
||||
if (buf == NULL) return -1;
|
||||
|
|
@ -205,6 +202,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis
|
|||
taosArrayPush(pReq->dataLen, &actualLen);
|
||||
taosArrayPush(pReq->data, &buf);
|
||||
|
||||
pReq->totalLen += dataStrLen;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -291,7 +289,7 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t streamDispatchOneDataReq(SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t vgId, SEpSet* pEpSet) {
|
||||
int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t vgId, SEpSet* pEpSet) {
|
||||
void* buf = NULL;
|
||||
int32_t code = -1;
|
||||
SRpcMsg msg = {0};
|
||||
|
|
@ -320,11 +318,12 @@ int32_t streamDispatchOneDataReq(SStreamTask* pTask, const SStreamDispatchReq* p
|
|||
msg.pCont = buf;
|
||||
msg.msgType = pTask->dispatchMsgType;
|
||||
|
||||
qDebug("dispatch from s-task:%s to taskId:%d vgId:%d data msg", pTask->id.idStr, pReq->taskId, vgId);
|
||||
qDebug("dispatch from s-task:%s to taskId:0x%x vgId:%d data msg", pTask->id.idStr, pReq->taskId, vgId);
|
||||
tmsgSendReq(pEpSet, &msg);
|
||||
|
||||
code = 0;
|
||||
return 0;
|
||||
|
||||
FAIL:
|
||||
if (buf) rpcFreeCont(buf);
|
||||
return code;
|
||||
|
|
@ -360,7 +359,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
|
|||
SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j);
|
||||
ASSERT(pVgInfo->vgId > 0);
|
||||
if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) {
|
||||
if (streamAddBlockToDispatchMsg(pDataBlock, &pReqs[j]) < 0) {
|
||||
if (streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]) < 0) {
|
||||
return -1;
|
||||
}
|
||||
if (pReqs[j].blockNum == 0) {
|
||||
|
|
@ -376,9 +375,9 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
|
|||
}
|
||||
|
||||
int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pData) {
|
||||
int32_t code = -1;
|
||||
int32_t blockNum = taosArrayGetSize(pData->blocks);
|
||||
ASSERT(blockNum != 0);
|
||||
int32_t code = 0;
|
||||
int32_t numOfBlocks = taosArrayGetSize(pData->blocks);
|
||||
ASSERT(numOfBlocks != 0);
|
||||
|
||||
if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
|
||||
SStreamDispatchReq req = {
|
||||
|
|
@ -387,19 +386,25 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat
|
|||
.upstreamTaskId = pTask->id.taskId,
|
||||
.upstreamChildId = pTask->selfChildId,
|
||||
.upstreamNodeId = pTask->nodeId,
|
||||
.blockNum = blockNum,
|
||||
.blockNum = numOfBlocks,
|
||||
};
|
||||
|
||||
req.data = taosArrayInit(blockNum, sizeof(void*));
|
||||
req.dataLen = taosArrayInit(blockNum, sizeof(int32_t));
|
||||
req.data = taosArrayInit(numOfBlocks, sizeof(void*));
|
||||
req.dataLen = taosArrayInit(numOfBlocks, sizeof(int32_t));
|
||||
if (req.data == NULL || req.dataLen == NULL) {
|
||||
goto FAIL_FIXED_DISPATCH;
|
||||
taosArrayDestroyP(req.data, taosMemoryFree);
|
||||
taosArrayDestroy(req.dataLen);
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < blockNum; i++) {
|
||||
for (int32_t i = 0; i < numOfBlocks; i++) {
|
||||
SSDataBlock* pDataBlock = taosArrayGet(pData->blocks, i);
|
||||
if (streamAddBlockToDispatchMsg(pDataBlock, &req) < 0) {
|
||||
goto FAIL_FIXED_DISPATCH;
|
||||
code = streamAddBlockIntoDispatchMsg(pDataBlock, &req);
|
||||
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
taosArrayDestroyP(req.data, taosMemoryFree);
|
||||
taosArrayDestroy(req.dataLen);
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -410,19 +415,12 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat
|
|||
req.taskId = downstreamTaskId;
|
||||
|
||||
qDebug("s-task:%s (child taskId:%d) fix-dispatch blocks:%d to down stream s-task:%d in vgId:%d", pTask->id.idStr,
|
||||
pTask->selfChildId, blockNum, downstreamTaskId, vgId);
|
||||
pTask->selfChildId, numOfBlocks, downstreamTaskId, vgId);
|
||||
|
||||
if (streamDispatchOneDataReq(pTask, &req, vgId, pEpSet) < 0) {
|
||||
goto FAIL_FIXED_DISPATCH;
|
||||
}
|
||||
|
||||
code = 0;
|
||||
|
||||
FAIL_FIXED_DISPATCH:
|
||||
code = doSendDispatchMsg(pTask, &req, vgId, pEpSet);
|
||||
taosArrayDestroyP(req.data, taosMemoryFree);
|
||||
taosArrayDestroy(req.dataLen);
|
||||
return code;
|
||||
|
||||
} else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
int32_t rspCnt = atomic_load_32(&pTask->shuffleDispatcher.waitingRspCnt);
|
||||
ASSERT(rspCnt == 0);
|
||||
|
|
@ -452,13 +450,13 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat
|
|||
pReqs[i].taskId = pVgInfo->taskId;
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < blockNum; i++) {
|
||||
for (int32_t i = 0; i < numOfBlocks; i++) {
|
||||
SSDataBlock* pDataBlock = taosArrayGet(pData->blocks, i);
|
||||
|
||||
// TODO: do not use broadcast
|
||||
if (pDataBlock->info.type == STREAM_DELETE_RESULT) {
|
||||
for (int32_t j = 0; j < vgSz; j++) {
|
||||
if (streamAddBlockToDispatchMsg(pDataBlock, &pReqs[j]) < 0) {
|
||||
if (streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]) < 0) {
|
||||
goto FAIL_SHUFFLE_DISPATCH;
|
||||
}
|
||||
if (pReqs[j].blockNum == 0) {
|
||||
|
|
@ -475,7 +473,7 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat
|
|||
}
|
||||
|
||||
qDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to %d vgroups", pTask->id.idStr, pTask->selfChildId,
|
||||
blockNum, vgSz);
|
||||
numOfBlocks, vgSz);
|
||||
|
||||
for (int32_t i = 0; i < vgSz; i++) {
|
||||
if (pReqs[i].blockNum > 0) {
|
||||
|
|
@ -483,7 +481,7 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat
|
|||
qDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to vgId:%d", pTask->id.idStr, pTask->selfChildId,
|
||||
pReqs[i].blockNum, pVgInfo->vgId);
|
||||
|
||||
if (streamDispatchOneDataReq(pTask, &pReqs[i], pVgInfo->vgId, &pVgInfo->epSet) < 0) {
|
||||
if (doSendDispatchMsg(pTask, &pReqs[i], pVgInfo->vgId, &pVgInfo->epSet) < 0) {
|
||||
goto FAIL_SHUFFLE_DISPATCH;
|
||||
}
|
||||
}
|
||||
|
|
@ -501,9 +499,8 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t streamDispatch(SStreamTask* pTask) {
|
||||
int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
|
||||
ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH);
|
||||
|
||||
int32_t numOfElems = taosQueueItemSize(pTask->outputQueue->queue);
|
||||
if (numOfElems > 0) {
|
||||
qDebug("s-task:%s try to dispatch intermediate result block to downstream, elem in outputQ:%d", pTask->id.idStr,
|
||||
|
|
@ -517,23 +514,22 @@ int32_t streamDispatch(SStreamTask* pTask) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
SStreamDataBlock* pBlock = streamQueueNextItem(pTask->outputQueue);
|
||||
if (pBlock == NULL) {
|
||||
SStreamDataBlock* pDispatchedBlock = streamQueueNextItem(pTask->outputQueue);
|
||||
if (pDispatchedBlock == NULL) {
|
||||
qDebug("s-task:%s stop dispatching since no output in output queue", pTask->id.idStr);
|
||||
atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL);
|
||||
return 0;
|
||||
}
|
||||
|
||||
ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK);
|
||||
ASSERT(pDispatchedBlock->type == STREAM_INPUT__DATA_BLOCK);
|
||||
|
||||
int32_t code = 0;
|
||||
if (streamDispatchAllBlocks(pTask, pBlock) < 0) {
|
||||
code = -1;
|
||||
int32_t code = streamDispatchAllBlocks(pTask, pDispatchedBlock);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
streamQueueProcessFail(pTask->outputQueue);
|
||||
atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL);
|
||||
}
|
||||
|
||||
taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes);
|
||||
taosFreeQitem(pBlock);
|
||||
// this block can be freed only when it has been pushed to down stream.
|
||||
destroyStreamDataBlock(pDispatchedBlock);
|
||||
return code;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,9 @@
|
|||
// maximum allowed processed block batches. One block may include several submit blocks
|
||||
#define MAX_STREAM_EXEC_BATCH_NUM 128
|
||||
#define MIN_STREAM_EXEC_BATCH_NUM 16
|
||||
#define MAX_STREAM_RESULT_DUMP_THRESHOLD 1000
|
||||
|
||||
static int32_t updateCheckPointInfo (SStreamTask* pTask);
|
||||
|
||||
bool streamTaskShouldStop(const SStreamStatus* pStatus) {
|
||||
int32_t status = atomic_load_8((int8_t*)&pStatus->taskStatus);
|
||||
|
|
@ -29,56 +32,57 @@ bool streamTaskShouldPause(const SStreamStatus* pStatus) {
|
|||
return (status == TASK_STATUS__PAUSE);
|
||||
}
|
||||
|
||||
static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) {
|
||||
static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* pRes, int32_t size, int64_t* totalSize,
|
||||
int32_t* totalBlocks) {
|
||||
int32_t code = updateCheckPointInfo(pTask);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t numOfBlocks = taosArrayGetSize(pRes);
|
||||
if (numOfBlocks > 0) {
|
||||
SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(pItem, pTask, size, pRes);
|
||||
if (pStreamBlocks == NULL) {
|
||||
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
|
||||
return -1;
|
||||
}
|
||||
|
||||
qDebug("s-task:%s dump stream result data blocks, num:%d, size:%.2fMiB", pTask->id.idStr, numOfBlocks, size/1048576.0);
|
||||
|
||||
code = streamTaskOutputResultBlock(pTask, pStreamBlocks);
|
||||
if (code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY) { // back pressure and record position
|
||||
destroyStreamDataBlock(pStreamBlocks);
|
||||
return -1;
|
||||
}
|
||||
|
||||
*totalSize += size;
|
||||
*totalBlocks += numOfBlocks;
|
||||
} else {
|
||||
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
void* pExecutor = pTask->exec.pExecutor;
|
||||
|
||||
while (pTask->taskLevel == TASK_LEVEL__SOURCE) {
|
||||
int8_t status = atomic_load_8(&pTask->status.taskStatus);
|
||||
if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__PAUSE) {
|
||||
qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr,
|
||||
atomic_load_8(&pTask->status.taskStatus));
|
||||
taosMsleep(2);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
*totalBlocks = 0;
|
||||
*totalSize = 0;
|
||||
|
||||
// set input
|
||||
const SStreamQueueItem* pItem = (const SStreamQueueItem*)data;
|
||||
if (pItem->type == STREAM_INPUT__GET_RES) {
|
||||
const SStreamTrigger* pTrigger = (const SStreamTrigger*)data;
|
||||
qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
|
||||
} else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
|
||||
ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE);
|
||||
const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)data;
|
||||
qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT);
|
||||
qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, pTask->id.idStr, pSubmit,
|
||||
pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver);
|
||||
} else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
|
||||
const SStreamDataBlock* pBlock = (const SStreamDataBlock*)data;
|
||||
int32_t size = 0;
|
||||
int32_t numOfBlocks = 0;
|
||||
SArray* pRes = NULL;
|
||||
|
||||
SArray* pBlockList = pBlock->blocks;
|
||||
int32_t numOfBlocks = taosArrayGetSize(pBlockList);
|
||||
qDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, pTask->id.idStr, numOfBlocks, pBlock->sourceVer);
|
||||
qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK);
|
||||
} else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) {
|
||||
const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)data;
|
||||
|
||||
SArray* pBlockList = pMerged->submits;
|
||||
int32_t numOfBlocks = taosArrayGetSize(pBlockList);
|
||||
qDebug("s-task:%s %p set submit input (merged), batch num:%d", pTask->id.idStr, pTask, numOfBlocks);
|
||||
qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT);
|
||||
} else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) {
|
||||
const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)data;
|
||||
qSetMultiStreamInput(pExecutor, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
|
||||
// pExecutor
|
||||
while (1) {
|
||||
if (pRes == NULL) {
|
||||
pRes = taosArrayInit(4, sizeof(SSDataBlock));
|
||||
}
|
||||
|
||||
if (streamTaskShouldStop(&pTask->status)) {
|
||||
taosArrayDestroy(pRes); // memory leak
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -97,17 +101,18 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
|
|||
if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
|
||||
SSDataBlock block = {0};
|
||||
|
||||
const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)data;
|
||||
const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*) pItem;
|
||||
ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1);
|
||||
|
||||
assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0));
|
||||
block.info.type = STREAM_PULL_OVER;
|
||||
block.info.childId = pTask->selfChildId;
|
||||
taosArrayPush(pRes, &block);
|
||||
|
||||
qDebug("task %d(child %d) processed retrieve, reqId %" PRId64, pTask->id.taskId, pTask->selfChildId,
|
||||
numOfBlocks += 1;
|
||||
qDebug("s-task:%s(child %d) processed retrieve, reqId:0x%" PRIx64, pTask->id.idStr, pTask->selfChildId,
|
||||
pRetrieveBlock->reqId);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -118,15 +123,40 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
|
|||
continue;
|
||||
}
|
||||
|
||||
qDebug("s-task:%s (child %d) executed and get block", pTask->id.idStr, pTask->selfChildId);
|
||||
|
||||
SSDataBlock block = {0};
|
||||
assignOneDataBlock(&block, output);
|
||||
block.info.childId = pTask->selfChildId;
|
||||
|
||||
size += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block);
|
||||
numOfBlocks += 1;
|
||||
|
||||
taosArrayPush(pRes, &block);
|
||||
|
||||
qDebug("s-task:%s (child %d) executed and get block, total blocks:%d, size:%.2fMiB", pTask->id.idStr,
|
||||
pTask->selfChildId, numOfBlocks, size / 1048576.0);
|
||||
|
||||
// current output should be dispatched to down stream nodes
|
||||
if (numOfBlocks >= MAX_STREAM_RESULT_DUMP_THRESHOLD) {
|
||||
ASSERT(numOfBlocks == taosArrayGetSize(pRes));
|
||||
code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
}
|
||||
|
||||
return 0;
|
||||
pRes = NULL;
|
||||
size = 0;
|
||||
numOfBlocks = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (numOfBlocks > 0) {
|
||||
ASSERT(numOfBlocks == taosArrayGetSize(pRes));
|
||||
code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks);
|
||||
} else {
|
||||
taosArrayDestroy(pRes);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
|
||||
|
|
@ -200,7 +230,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
|
|||
|
||||
qRes->type = STREAM_INPUT__DATA_BLOCK;
|
||||
qRes->blocks = pRes;
|
||||
code = streamTaskOutput(pTask, qRes);
|
||||
code = streamTaskOutputResultBlock(pTask, qRes);
|
||||
if (code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY) {
|
||||
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
|
||||
taosFreeQitem(qRes);
|
||||
|
|
@ -209,7 +239,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
|
|||
|
||||
if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
qDebug("s-task:%s scan exec dispatch blocks:%d", pTask->id.idStr, batchCnt);
|
||||
streamDispatch(pTask);
|
||||
streamDispatchStreamBlock(pTask);
|
||||
}
|
||||
|
||||
if (finished) {
|
||||
|
|
@ -246,7 +276,7 @@ int32_t streamBatchExec(SStreamTask* pTask, int32_t batchLimit) {
|
|||
|
||||
if (pTask->taskLevel == TASK_LEVEL__SINK) {
|
||||
ASSERT(((SStreamQueueItem*)pItem)->type == STREAM_INPUT__DATA_BLOCK);
|
||||
streamTaskOutput(pTask, (SStreamDataBlock*)pItem);
|
||||
streamTaskOutputResultBlock(pTask, (SStreamDataBlock*)pItem);
|
||||
}
|
||||
|
||||
// exec impl
|
||||
|
|
@ -257,6 +287,34 @@ int32_t streamBatchExec(SStreamTask* pTask, int32_t batchLimit) {
|
|||
}
|
||||
#endif
|
||||
|
||||
int32_t updateCheckPointInfo (SStreamTask* pTask) {
|
||||
int64_t ckId = 0;
|
||||
int64_t dataVer = 0;
|
||||
qGetCheckpointVersion(pTask->exec.pExecutor, &dataVer, &ckId);
|
||||
|
||||
SCheckpointInfo* pCkInfo = &pTask->chkInfo;
|
||||
if (ckId > pCkInfo->id) { // save it since the checkpoint is updated
|
||||
qDebug("s-task:%s exec end, start to update check point, ver from %" PRId64 " to %" PRId64
|
||||
", checkPoint id:%" PRId64 " -> %" PRId64, pTask->id.idStr, pCkInfo->version, dataVer, pCkInfo->id, ckId);
|
||||
|
||||
pTask->chkInfo = (SCheckpointInfo){.version = dataVer, .id = ckId, .currentVer = pCkInfo->currentVer};
|
||||
|
||||
taosWLockLatch(&pTask->pMeta->lock);
|
||||
|
||||
streamMetaSaveTask(pTask->pMeta, pTask);
|
||||
if (streamMetaCommit(pTask->pMeta) < 0) {
|
||||
taosWUnLockLatch(&pTask->pMeta->lock);
|
||||
qError("s-task:%s failed to commit stream meta, since %s", pTask->id.idStr, terrstr());
|
||||
return -1;
|
||||
} else {
|
||||
taosWUnLockLatch(&pTask->pMeta->lock);
|
||||
qDebug("s-task:%s update checkpoint ver succeed", pTask->id.idStr);
|
||||
}
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t streamExecForAll(SStreamTask* pTask) {
|
||||
int32_t code = 0;
|
||||
while (1) {
|
||||
|
|
@ -272,6 +330,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
|
|||
if (streamTaskShouldPause(&pTask->status)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue);
|
||||
if (qItem == NULL) {
|
||||
if (pTask->taskLevel == TASK_LEVEL__SOURCE && batchSize < MIN_STREAM_EXEC_BATCH_NUM && times < 5) {
|
||||
|
|
@ -324,74 +383,70 @@ int32_t streamExecForAll(SStreamTask* pTask) {
|
|||
if (pTask->taskLevel == TASK_LEVEL__SINK) {
|
||||
ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK);
|
||||
qDebug("s-task:%s sink node start to sink result. numOfBlocks:%d", pTask->id.idStr, batchSize);
|
||||
streamTaskOutput(pTask, (SStreamDataBlock*)pInput);
|
||||
streamTaskOutputResultBlock(pTask, (SStreamDataBlock*)pInput);
|
||||
continue;
|
||||
}
|
||||
|
||||
SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
|
||||
// wait for the task to be ready to go
|
||||
while (pTask->taskLevel == TASK_LEVEL__SOURCE) {
|
||||
int8_t status = atomic_load_8(&pTask->status.taskStatus);
|
||||
if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__PAUSE) {
|
||||
qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr,
|
||||
atomic_load_8(&pTask->status.taskStatus));
|
||||
taosMsleep(2);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
int64_t st = taosGetTimestampMs();
|
||||
qDebug("s-task:%s start to execute, block batches:%d", pTask->id.idStr, batchSize);
|
||||
|
||||
streamTaskExecImpl(pTask, pInput, pRes);
|
||||
{
|
||||
// set input
|
||||
void* pExecutor = pTask->exec.pExecutor;
|
||||
|
||||
int64_t ckId = 0;
|
||||
int64_t dataVer = 0;
|
||||
qGetCheckpointVersion(pTask->exec.pExecutor, &dataVer, &ckId);
|
||||
if (ckId > pTask->chkInfo.id) { // save it since the checkpoint is updated
|
||||
qDebug("s-task:%s exec end, start to update check point, ver from %" PRId64 " to %" PRId64
|
||||
", checkPoint id:%" PRId64 " -> %" PRId64,
|
||||
pTask->id.idStr, pTask->chkInfo.version, dataVer, pTask->chkInfo.id, ckId);
|
||||
const SStreamQueueItem* pItem = pInput;
|
||||
if (pItem->type == STREAM_INPUT__GET_RES) {
|
||||
const SStreamTrigger* pTrigger = (const SStreamTrigger*)pInput;
|
||||
qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
|
||||
} else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
|
||||
ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE);
|
||||
const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput;
|
||||
qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT);
|
||||
qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, pTask->id.idStr, pSubmit,
|
||||
pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver);
|
||||
} else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
|
||||
const SStreamDataBlock* pBlock = (const SStreamDataBlock*)pInput;
|
||||
|
||||
pTask->chkInfo = (SCheckpointInfo){.version = dataVer, .id = ckId, .currentVer = pTask->chkInfo.currentVer};
|
||||
SArray* pBlockList = pBlock->blocks;
|
||||
int32_t numOfBlocks = taosArrayGetSize(pBlockList);
|
||||
qDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, pTask->id.idStr, numOfBlocks, pBlock->sourceVer);
|
||||
qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK);
|
||||
} else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) {
|
||||
const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)pInput;
|
||||
|
||||
taosWLockLatch(&pTask->pMeta->lock);
|
||||
|
||||
streamMetaSaveTask(pTask->pMeta, pTask);
|
||||
if (streamMetaCommit(pTask->pMeta) < 0) {
|
||||
taosWUnLockLatch(&pTask->pMeta->lock);
|
||||
qError("s-task:%s failed to commit stream meta, since %s", pTask->id.idStr, terrstr());
|
||||
return -1;
|
||||
SArray* pBlockList = pMerged->submits;
|
||||
int32_t numOfBlocks = taosArrayGetSize(pBlockList);
|
||||
qDebug("s-task:%s %p set submit input (merged), batch num:%d", pTask->id.idStr, pTask, numOfBlocks);
|
||||
qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT);
|
||||
} else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) {
|
||||
const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)pInput;
|
||||
qSetMultiStreamInput(pExecutor, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
|
||||
} else {
|
||||
taosWUnLockLatch(&pTask->pMeta->lock);
|
||||
qDebug("s-task:%s update checkpoint ver succeed", pTask->id.idStr);
|
||||
ASSERT(0);
|
||||
}
|
||||
} else {
|
||||
qDebug("s-task:%s exec end", pTask->id.idStr);
|
||||
}
|
||||
|
||||
if (taosArrayGetSize(pRes) != 0) {
|
||||
SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0);
|
||||
if (qRes == NULL) {
|
||||
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
|
||||
streamFreeQitem(pInput);
|
||||
return -1;
|
||||
}
|
||||
int64_t resSize = 0;
|
||||
int32_t totalBlocks = 0;
|
||||
streamTaskExecImpl(pTask, pInput, &resSize, &totalBlocks);
|
||||
|
||||
qRes->type = STREAM_INPUT__DATA_BLOCK;
|
||||
qRes->blocks = pRes;
|
||||
|
||||
if (((SStreamQueueItem*)pInput)->type == STREAM_INPUT__DATA_SUBMIT) {
|
||||
SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)pInput;
|
||||
qRes->childId = pTask->selfChildId;
|
||||
qRes->sourceVer = pSubmit->ver;
|
||||
} else if (((SStreamQueueItem*)pInput)->type == STREAM_INPUT__MERGED_SUBMIT) {
|
||||
SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)pInput;
|
||||
qRes->childId = pTask->selfChildId;
|
||||
qRes->sourceVer = pMerged->ver;
|
||||
}
|
||||
|
||||
code = streamTaskOutput(pTask, qRes);
|
||||
if (code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY) {
|
||||
// backpressure and record position
|
||||
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
|
||||
streamFreeQitem(pInput);
|
||||
taosFreeQitem(qRes);
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
taosArrayDestroy(pRes);
|
||||
}
|
||||
double el = (taosGetTimestampMs() - st) / 1000.0;
|
||||
qDebug("s-task:%s exec end, elapsed time:%.2fs, result size:%.2fMiB, numOfBlocks:%d", pTask->id.idStr, el, resSize / 1048576.0, totalBlocks);
|
||||
streamFreeQitem(pInput);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -296,6 +296,7 @@ int32_t streamMetaBegin(SStreamMeta* pMeta) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
// todo add error log
|
||||
int32_t streamMetaCommit(SStreamMeta* pMeta) {
|
||||
if (tdbCommit(pMeta->db, pMeta->txn) < 0) {
|
||||
qError("failed to commit stream meta");
|
||||
|
|
@ -311,6 +312,7 @@ int32_t streamMetaCommit(SStreamMeta* pMeta) {
|
|||
TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -373,7 +375,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) {
|
|||
}
|
||||
|
||||
if (pTask->fillHistory) {
|
||||
pTask->status.taskStatus = TASK_STATUS__WAIT_DOWNSTREAM;
|
||||
ASSERT(pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM);
|
||||
streamTaskCheckDownstream(pTask, ver);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) {
|
|||
|
||||
if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
|
||||
atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__RECOVER_PREPARE);
|
||||
qDebug("s-task:%s set task status:%d and start recover", pTask->id.idStr, pTask->status.taskStatus);
|
||||
qDebug("s-task:%s set task status:%d and start to recover", pTask->id.idStr, pTask->status.taskStatus);
|
||||
|
||||
streamSetParamForRecover(pTask);
|
||||
streamSourceRecoverPrepareStep1(pTask, version);
|
||||
|
|
@ -46,6 +46,7 @@ int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) {
|
|||
streamSetParamForRecover(pTask);
|
||||
streamAggRecoverPrepare(pTask);
|
||||
} else if (pTask->taskLevel == TASK_LEVEL__SINK) {
|
||||
// sink nodes has no specified operation for fill history
|
||||
atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
|
||||
}
|
||||
|
||||
|
|
@ -54,7 +55,7 @@ int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) {
|
|||
|
||||
// checkstatus
|
||||
int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) {
|
||||
qDebug("s-taks:%s in fill history stage, ver:%"PRId64, pTask->id.idStr, version);
|
||||
qDebug("s-task:%s in fill history stage, ver:%"PRId64, pTask->id.idStr, version);
|
||||
|
||||
SStreamTaskCheckReq req = {
|
||||
.streamId = pTask->id.streamId,
|
||||
|
|
@ -71,23 +72,23 @@ int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) {
|
|||
req.downstreamTaskId = pTask->fixedEpDispatcher.taskId;
|
||||
pTask->checkReqId = req.reqId;
|
||||
|
||||
qDebug("s-task:%s at node %d check downstream task %d at node %d", pTask->id.idStr, pTask->nodeId, req.downstreamTaskId,
|
||||
qDebug("s-task:%s at node %d check downstream task:0x%x at node %d", pTask->id.idStr, pTask->nodeId, req.downstreamTaskId,
|
||||
req.downstreamNodeId);
|
||||
streamDispatchCheckMsg(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet);
|
||||
} else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
|
||||
|
||||
int32_t vgSz = taosArrayGetSize(vgInfo);
|
||||
pTask->recoverTryingDownstream = vgSz;
|
||||
pTask->checkReqIds = taosArrayInit(vgSz, sizeof(int64_t));
|
||||
int32_t numOfVgs = taosArrayGetSize(vgInfo);
|
||||
pTask->recoverTryingDownstream = numOfVgs;
|
||||
pTask->checkReqIds = taosArrayInit(numOfVgs, sizeof(int64_t));
|
||||
|
||||
for (int32_t i = 0; i < vgSz; i++) {
|
||||
for (int32_t i = 0; i < numOfVgs; i++) {
|
||||
SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
|
||||
req.reqId = tGenIdPI64();
|
||||
taosArrayPush(pTask->checkReqIds, &req.reqId);
|
||||
req.downstreamNodeId = pVgInfo->vgId;
|
||||
req.downstreamTaskId = pVgInfo->taskId;
|
||||
qDebug("s-task:%s at node %d check downstream task %d at node %d (shuffle)", pTask->id.idStr, pTask->nodeId,
|
||||
qDebug("s-task:%s at node %d check downstream task:0x%x at node %d (shuffle)", pTask->id.idStr, pTask->nodeId,
|
||||
req.downstreamTaskId, req.downstreamNodeId);
|
||||
streamDispatchCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
|
||||
}
|
||||
|
|
@ -110,15 +111,16 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp
|
|||
.childId = pRsp->childId,
|
||||
};
|
||||
|
||||
qDebug("s-task:%s at node %d check downstream task %d at node %d (recheck)", pTask->id.idStr, pTask->nodeId,
|
||||
qDebug("s-task:%s at node %d check downstream task:0x%x at node %d (recheck)", pTask->id.idStr, pTask->nodeId,
|
||||
req.downstreamTaskId, req.downstreamNodeId);
|
||||
|
||||
if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
|
||||
streamDispatchCheckMsg(pTask, &req, pRsp->downstreamNodeId, &pTask->fixedEpDispatcher.epSet);
|
||||
} else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
|
||||
SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
|
||||
int32_t vgSz = taosArrayGetSize(vgInfo);
|
||||
for (int32_t i = 0; i < vgSz; i++) {
|
||||
|
||||
int32_t numOfVgs = taosArrayGetSize(vgInfo);
|
||||
for (int32_t i = 0; i < numOfVgs; i++) {
|
||||
SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
|
||||
if (pVgInfo->taskId == req.downstreamTaskId) {
|
||||
streamDispatchCheckMsg(pTask, &req, pRsp->downstreamNodeId, &pVgInfo->epSet);
|
||||
|
|
@ -134,7 +136,9 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask) {
|
|||
}
|
||||
|
||||
int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version) {
|
||||
qDebug("task %d at node %d recv check rsp from task %d at node %d: status %d", pRsp->upstreamTaskId,
|
||||
ASSERT(pTask->id.taskId == pRsp->upstreamTaskId);
|
||||
|
||||
qDebug("s-task:%s at node %d recv check rsp from task:0x%x at node %d: status %d", pTask->id.idStr,
|
||||
pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status);
|
||||
|
||||
if (pRsp->status == 1) {
|
||||
|
|
@ -161,7 +165,7 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp*
|
|||
taosArrayDestroy(pTask->checkReqIds);
|
||||
pTask->checkReqIds = NULL;
|
||||
|
||||
qDebug("s-task:%s all downstream tasks:%d are ready, now enter into recover stage", pTask->id.idStr, numOfReqs);
|
||||
qDebug("s-task:%s all %d downstream tasks are ready, now enter into recover stage", pTask->id.idStr, numOfReqs);
|
||||
streamTaskLaunchRecover(pTask, version);
|
||||
}
|
||||
} else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
|
||||
|
|
@ -174,9 +178,10 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp*
|
|||
ASSERT(0);
|
||||
}
|
||||
} else { // not ready, wait for 100ms and retry
|
||||
qDebug("s-task:%s downstream taskId:%d (vgId:%d) not ready, wait for 100ms and retry", pTask->id.idStr,
|
||||
qDebug("s-task:%s downstream taskId:0x%x (vgId:%d) not ready, wait for 100ms and retry", pTask->id.idStr,
|
||||
pRsp->downstreamTaskId, pRsp->downstreamNodeId);
|
||||
taosMsleep(100);
|
||||
|
||||
streamRecheckOneDownstream(pTask, pRsp);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -163,6 +163,52 @@ class TDTestCase:
|
|||
tdSql.query(f'select * from information_schema.ins_columns where db_name="db2" and table_type=="NORMAL_TABLE"')
|
||||
tdSql.checkEqual(20470,len(tdSql.queryResult))
|
||||
|
||||
def ins_dnodes_check(self):
|
||||
tdSql.execute('drop database if exists db2')
|
||||
tdSql.execute('create database if not exists db2 vgroups 1 replica 1')
|
||||
tdSql.query(f'select * from information_schema.ins_dnodes')
|
||||
result = tdSql.queryResult
|
||||
tdSql.checkEqual(result[0][0],1)
|
||||
tdSql.checkEqual(result[0][8],"")
|
||||
tdSql.checkEqual(result[0][9],"")
|
||||
self.str107 = 'Hc7VCc+'
|
||||
for t in range (10):
|
||||
self.str107 += 'tP+2soIXpP'
|
||||
self.str108 = self.str107 + '='
|
||||
self.str109 = self.str108 + '+'
|
||||
self.str254 = self.str108 + self.str108 + '01234567890123456789012345678901234567'
|
||||
self.str255 = self.str254 + '='
|
||||
self.str256 = self.str254 + '=('
|
||||
self.str257 = self.str254 + '=()'
|
||||
self.str510 = self.str255 + self.str255
|
||||
tdSql.error('alter dnode 1 "activeCode" "a"')
|
||||
tdSql.error('alter dnode 1 "activeCode" "' + self.str107 + '"')
|
||||
tdSql.execute('alter all dnodes "activeCode" "' + self.str108 + '"')
|
||||
tdSql.error('alter dnode 1 "activeCode" "' + self.str109 + '"')
|
||||
tdSql.error('alter all dnodes "activeCode" "' + self.str510 + '"')
|
||||
tdSql.query(f'select * from information_schema.ins_dnodes')
|
||||
tdSql.checkEqual(tdSql.queryResult[0][8],self.str108)
|
||||
tdSql.execute('alter dnode 1 "activeCode" ""')
|
||||
tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes')
|
||||
tdSql.checkEqual(tdSql.queryResult[0][0],"")
|
||||
tdSql.checkEqual(tdSql.queryResult[0][1],'')
|
||||
tdSql.error('alter dnode 1 "cActiveCode" "a"')
|
||||
tdSql.error('alter dnode 1 "cActiveCode" "' + self.str107 + '"')
|
||||
tdSql.error('alter dnode 1 "cActiveCode" "' + self.str256 + '"')
|
||||
tdSql.error('alter all dnodes "cActiveCode" "' + self.str255 + '"')
|
||||
tdSql.error('alter all dnodes "cActiveCode" "' + self.str256 + '"')
|
||||
tdSql.error('alter all dnodes "cActiveCode" "' + self.str257 + '"')
|
||||
tdSql.execute('alter all dnodes "cActiveCode" "' + self.str254 + '"')
|
||||
tdSql.error('alter dnode 1 "cActiveCode" "' + self.str510 + '"')
|
||||
tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes')
|
||||
tdSql.checkEqual(tdSql.queryResult[0][0],"")
|
||||
tdSql.checkEqual(tdSql.queryResult[0][1],self.str254)
|
||||
tdSql.execute('alter dnode 1 "cActiveCode" "' + self.str109 + '"')
|
||||
tdSql.query(f'show dnodes')
|
||||
tdSql.checkEqual(tdSql.queryResult[0][9],self.str109)
|
||||
tdSql.execute('alter all dnodes "cActiveCode" ""')
|
||||
tdSql.query(f'select c_active_code from information_schema.ins_dnodes')
|
||||
tdSql.checkEqual(tdSql.queryResult[0][0],'')
|
||||
|
||||
def run(self):
|
||||
self.prepare_data()
|
||||
|
|
@ -170,6 +216,7 @@ class TDTestCase:
|
|||
self.ins_columns_check()
|
||||
# self.ins_col_check_4096()
|
||||
self.ins_stable_check()
|
||||
self.ins_dnodes_check()
|
||||
|
||||
|
||||
def stop(self):
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue