diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index a2e6bca46c..074fbfbc8d 100755 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] [TAGS] select_list hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */ hint: - BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT + BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT | PARTITION_FIRST | SMALLDATA_TS_SORT select_list: select_expr [, select_expr] ... @@ -94,6 +94,7 @@ The list of currently supported Hints is as follows: | SORT_FOR_GROUP| None | Use sort for partition, conflict with PARTITION_FIRST | With normal column in partition by list | | PARTITION_FIRST| None | Use Partition before aggregate, conflict with SORT_FOR_GROUP | With normal column in partition by list | | PARA_TABLES_SORT| None | When sorting the supertable rows by timestamp, No temporary disk space is used. When there are numerous tables, each with long rows, the corresponding algorithm associated with this prompt may consume a substantial amount of memory, potentially leading to an Out Of Memory (OOM) situation. | Sorting the supertable rows by timestamp | +| SMALLDATA_TS_SORT| None | When sorting the supertable rows by timestamp, if the length of query columns >= 256, and there are relatively few rows, this hint can improve performance. | Sorting the supertable rows by timestamp | For example: @@ -102,6 +103,7 @@ SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0 SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1; SELECT /*+ PARTITION_FIRST() */ count(*), c1 FROM stable1 PARTITION BY c1; SELECT /*+ PARA_TABLES_SORT() */ * from stable1 order by ts; +SELECT /*+ SMALLDATA_TS_SORT() */ * from stable1 order by ts; ``` ## Lists diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index eec947ea23..573e854864 100755 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] [TAGS] select_list hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */ hint: - BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT + BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARTITION_FIRST | PARA_TABLES_SORT | SMALLDATA_TS_SORT select_list: select_expr [, select_expr] ... @@ -94,6 +94,8 @@ Hints 是用户控制单个语句查询优化的一种手段，当 Hint 不适 | SORT_FOR_GROUP| 无 | 采用sort方式进行分组, 与PARTITION_FIRST冲突 | partition by 列表有普通列时 | | PARTITION_FIRST| 无 | 在聚合之前使用PARTITION计算分组, 与SORT_FOR_GROUP冲突 | partition by 列表有普通列时 | | PARA_TABLES_SORT| 无 | 超级表的数据按时间戳排序时, 不使用临时磁盘空间, 只使用内存。当子表数量多, 行长比较大时候, 会使用大量内存, 可能发生OOM | 超级表的数据按时间戳排序时 | +| SMALLDATA_TS_SORT| 无 | 超级表的数据按时间戳排序时, 查询列长度大于等于256, 但是行数不多, 使用这个提示, 可以提高性能 | 超级表的数据按时间戳排序时 | + 举例： ```sql @@ -101,6 +103,7 @@ SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0 SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1; SELECT /*+ PARTITION_FIRST() */ count(*), c1 FROM stable1 PARTITION BY c1; SELECT /*+ PARA_TABLES_SORT() */ * from stable1 order by ts; +SELECT /*+ SMALLDATA_TS_SORT() */ * from stable1 order by ts; ``` ## 列表 diff --git a/docs/zh/12-taos-sql/12-distinguished.md b/docs/zh/12-taos-sql/12-distinguished.md index 8ae3f900f4..f73919a970 100755 --- a/docs/zh/12-taos-sql/12-distinguished.md +++ b/docs/zh/12-taos-sql/12-distinguished.md @@ -49,6 +49,7 @@ window_clause: { | STATE_WINDOW(col) | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [FILL(fill_mod_and_val)] | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition + | COUNT_WINDOW(count_val[, sliding_val]) } ``` @@ -180,6 +181,19 @@ select _wstart, _wend, count(*) from t event_window start with c1 > 0 end with c ![TDengine Database 事件窗口示意图](./event_window.webp) +### 计数窗口 + +计数窗口按固定的数据行数来划分窗口。默认将数据按时间戳排序，再按照count_val的值，将数据划分为多个窗口，然后做聚合计算。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。sliding_val是常量，表示窗口滑动的数量，类似于 interval的SLIDING。 + +以下面的 SQL 语句为例，计数窗口切分如图所示： +```sql +select _wstart, _wend, count(*) from t count_window(4); +``` + +![count_window](https://github.com/taosdata/TDengine/assets/38781207/ff86278e-cbff-477f-a866-6e5cc150b712) + + + ### 时间戳伪列 窗口聚合查询结果中，如果 SQL 语句中没有指定输出查询结果中的时间戳列，那么最终结果中不会自动包含窗口的时间列信息。如果需要在结果中输出聚合结果所对应的时间窗口信息，需要在 SELECT 子句中使用时间戳相关的伪列: 时间窗口起始时间 (\_WSTART), 时间窗口结束时间 (\_WEND), 时间窗口持续时间 (\_WDURATION), 以及查询整体窗口相关的伪列: 查询窗口起始时间(\_QSTART) 和查询窗口结束时间(\_QEND)。需要注意的是时间窗口起始时间和结束时间均是闭区间，时间窗口持续时间是数据当前时间分辨率下的数值。例如，如果当前数据库的时间分辨率是毫秒，那么结果中 500 就表示当前时间窗口的持续时间是 500毫秒 (500 ms)。 diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 958789178a..db2c51fb01 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3820,6 +3820,7 @@ typedef struct { uint32_t phyLen; char* sql; char* msg; + int8_t source; } SVDeleteReq; int32_t tSerializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq); @@ -3841,6 +3842,7 @@ typedef struct SDeleteRes { char tableFName[TSDB_TABLE_NAME_LEN]; char tsColName[TSDB_COL_NAME_LEN]; int64_t ctimeMs; // fill by vnode + int8_t source; } SDeleteRes; int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes); diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 8f89857d33..020be9d447 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -380,7 +380,7 @@ #define TK_SORT_FOR_GROUP 608 #define TK_PARTITION_FIRST 609 #define TK_PARA_TABLES_SORT 610 - +#define TK_SMALLDATA_TS_SORT 611 #define TK_NK_NIL 65535 diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index e66ffe5521..3433f98d38 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -223,10 +223,10 @@ typedef struct SStoreTqReader { bool (*tqReaderCurrentBlockConsumed)(); struct SWalReader* (*tqReaderGetWalReader)(); // todo remove it - int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it +// int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it int32_t (*tqReaderSetSubmitMsg)(); // todo remove it - bool (*tqReaderNextBlockFilterOut)(); +// bool (*tqReaderNextBlockFilterOut)(); } SStoreTqReader; typedef struct SStoreSnapshotFn { diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 0bc3ce04ef..cbf38102de 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -122,6 +122,7 @@ typedef struct SScanLogicNode { bool isCountByTag; // true if selectstmt hasCountFunc & part by tag/tbname SArray* pFuncTypes; // for last, last_row bool paraTablesSort; // for table merge scan + bool smallDataTsSort; // disable row id sort for table merge scan } SScanLogicNode; typedef struct SJoinLogicNode { @@ -445,6 +446,7 @@ typedef struct STableScanPhysiNode { bool filesetDelimited; bool needCountEmptyTable; bool paraTablesSort; + bool smallDataTsSort; } STableScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 7ceb7e0278..97ac4ff3b9 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -128,7 +128,8 @@ typedef enum EHintOption { HINT_BATCH_SCAN, HINT_SORT_FOR_GROUP, HINT_PARTITION_FIRST, - HINT_PARA_TABLES_SORT + HINT_PARA_TABLES_SORT, + HINT_SMALLDATA_TS_SORT, } EHintOption; typedef struct SHintNode { diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 958d63349d..952af3c443 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -78,6 +78,7 @@ typedef struct SSchedulerReq { void* chkKillParam; SExecResult* pExecRes; void** pFetchRes; + int8_t source; } SSchedulerReq; int32_t schedulerInit(void); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 64ce735843..5f3761d7b7 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -56,7 +56,6 @@ extern "C" { #define STREAM_EXEC_T_RESTART_ALL_TASKS (-4) #define STREAM_EXEC_T_STOP_ALL_TASKS (-5) #define STREAM_EXEC_T_RESUME_TASK (-6) -#define STREAM_EXEC_T_UPDATE_TASK_EPSET (-7) typedef struct SStreamTask SStreamTask; typedef struct SStreamQueue SStreamQueue; @@ -783,11 +782,14 @@ bool streamTaskIsAllUpstreamClosed(SStreamTask* pTask); bool streamTaskSetSchedStatusWait(SStreamTask* pTask); int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask); int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask); -int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt, bool metaLock); +int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt); int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event); -int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event); -void streamTaskRestoreStatus(SStreamTask* pTask); + +typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param); +int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param); +int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param); +int32_t streamTaskRestoreStatus(SStreamTask* pTask); int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp, SRpcHandleInfo* pRpcInfo, int32_t taskId); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index e7b0b25653..460b8962ea 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -62,6 +62,7 @@ typedef struct SRpcHandleInfo { SRpcConnInfo conn; int8_t forbiddenIp; + int8_t notFreeAhandle; } SRpcHandleInfo; diff --git a/include/os/osFile.h b/include/os/osFile.h index eb0862a719..9c9027e931 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -119,6 +119,13 @@ int32_t taosSetFileHandlesLimit(); int32_t taosLinkFile(char *src, char *dst); +FILE* taosOpenCFile(const char* filename, const char* mode); +int taosSeekCFile(FILE* file, int64_t offset, int whence); +size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream ); +size_t taosWriteToCFile(const void* ptr, size_t size, size_t nitems, FILE* stream); +int taosCloseCFile(FILE *); +int taosSetAutoDelFile(char* path); + bool lastErrorIsFileNotExist(); #ifdef __cplusplus diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 989c6614a6..6c3603b4e0 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -284,6 +284,7 @@ typedef struct SRequestObj { void* pWrapper; SMetaData parseMeta; char* effectiveUser; + int8_t source; } SRequestObj; typedef struct SSyncQueryParam { @@ -297,8 +298,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); void doSetOneRowPtr(SReqResultInfo* pResultInfo); void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); -int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, - bool freeAfterUse); +int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4); int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows, bool convertUcs4); void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); @@ -306,10 +306,10 @@ void doFreeReqResultInfo(SReqResultInfo* pResInfo); int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq); void syncCatalogFn(SMetaData* pResult, void* param, int32_t code); -TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly); +TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t source); TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly, int64_t reqid); -void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly); +void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source); void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int64_t reqid); void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param); @@ -354,6 +354,7 @@ SRequestObj* acquireRequest(int64_t rid); int32_t releaseRequest(int64_t rid); int32_t removeRequest(int64_t rid); void doDestroyRequest(void* p); +int64_t removeFromMostPrevReq(SRequestObj* pRequest); char* getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 1df50a51da..6c20813118 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -385,6 +385,33 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); } +/// return the most previous req ref id +int64_t removeFromMostPrevReq(SRequestObj* pRequest) { + int64_t mostPrevReqRefId = pRequest->self; + SRequestObj* pTmp = pRequest; + while (pTmp->relation.prevRefId) { + pTmp = acquireRequest(pTmp->relation.prevRefId); + if (pTmp) { + mostPrevReqRefId = pTmp->self; + releaseRequest(mostPrevReqRefId); + } else { + break; + } + } + removeRequest(mostPrevReqRefId); + return mostPrevReqRefId; +} + +void destroyNextReq(int64_t nextRefId) { + if (nextRefId) { + SRequestObj* pObj = acquireRequest(nextRefId); + if (pObj) { + releaseRequest(nextRefId); + releaseRequest(nextRefId); + } + } +} + void destroySubRequests(SRequestObj *pRequest) { int32_t reqIdx = -1; SRequestObj *pReqList[16] = {NULL}; @@ -435,7 +462,7 @@ void doDestroyRequest(void *p) { uint64_t reqId = pRequest->requestId; tscTrace("begin to destroy request %" PRIx64 " p:%p", reqId, pRequest); - destroySubRequests(pRequest); + int64_t nextReqRefId = pRequest->relation.nextRefId; taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self)); @@ -471,6 +498,7 @@ void doDestroyRequest(void *p) { taosMemoryFreeClear(pRequest->sqlstr); taosMemoryFree(pRequest); tscTrace("end to destroy request %" PRIx64 " p:%p", reqId, pRequest); + destroyNextReq(nextReqRefId); } void destroyRequest(SRequestObj *pRequest) { @@ -479,7 +507,7 @@ void destroyRequest(SRequestObj *pRequest) { } taos_stop_query(pRequest); - removeRequest(pRequest->self); + removeFromMostPrevReq(pRequest); } void taosStopQueryImpl(SRequestObj *pRequest) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index de4f9f16e4..e87e676f08 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -302,7 +302,7 @@ int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { int8_t biMode = atomic_load_8(&pRequest->pTscObj->biMode); int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp, biMode); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { - code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, true); + code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false); } return code; @@ -341,7 +341,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp, atomic_load_8(&pRequest->pTscObj->biMode)); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { - code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, true); + code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false); } SReqResultInfo* pResultInfo = &pRequest->body.resInfo; @@ -743,6 +743,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList .chkKillFp = chkRequestKilled, .chkKillParam = (void*)pRequest->self, .pExecRes = &res, + .source = pRequest->source, }; int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob); @@ -1212,6 +1213,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat .chkKillFp = chkRequestKilled, .chkKillParam = (void*)pRequest->self, .pExecRes = NULL, + .source = pRequest->source, }; code = schedulerExecJob(&req, &pRequest->body.queryJob); taosArrayDestroy(pNodeList); @@ -1719,7 +1721,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) } pRequest->code = - setQueryResultFromRsp(&pRequest->body.resInfo, (const SRetrieveTableRsp*)pResInfo->pData, convertUcs4, true); + setQueryResultFromRsp(&pRequest->body.resInfo, (const SRetrieveTableRsp*)pResInfo->pData, convertUcs4); if (pRequest->code != TSDB_CODE_SUCCESS) { pResultInfo->numOfRows = 0; return NULL; @@ -2178,15 +2180,13 @@ void resetConnectDB(STscObj* pTscObj) { taosThreadMutexUnlock(&pTscObj->mutex); } -int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, - bool freeAfterUse) { +int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4) { if (pResultInfo == NULL || pRsp == NULL) { tscError("setQueryResultFromRsp paras is null"); return TSDB_CODE_TSC_INTERNAL_ERROR; } - if (freeAfterUse) taosMemoryFreeClear(pResultInfo->pRspMsg); - + taosMemoryFreeClear(pResultInfo->pRspMsg); pResultInfo->pRspMsg = (const char*)pRsp; pResultInfo->pData = (void*)pRsp->data; pResultInfo->numOfRows = htobe64(pRsp->numOfRows); @@ -2475,7 +2475,7 @@ void syncQueryFn(void* param, void* res, int32_t code) { tsem_post(&pParam->sem); } -void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) { +void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source) { if (sql == NULL || NULL == fp) { terrno = TSDB_CODE_INVALID_PARA; if (fp) { @@ -2501,6 +2501,7 @@ void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, return; } + pRequest->source = source; pRequest->body.queryFp = fp; doAsyncQuery(pRequest, false); } @@ -2535,7 +2536,7 @@ void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_ doAsyncQuery(pRequest, false); } -TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { +TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t source) { if (NULL == taos) { terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; @@ -2550,7 +2551,7 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { } tsem_init(¶m->sem, 0, 0); - taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly); + taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, source); tsem_wait(¶m->sem); SRequestObj* pRequest = NULL; @@ -2615,7 +2616,7 @@ static void fetchCallback(void* pResult, void* param, int32_t code) { } pRequest->code = - setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4, true); + setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4); if (pRequest->code != TSDB_CODE_SUCCESS) { pResultInfo->numOfRows = 0; pRequest->code = code; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 275ca0d2aa..a49d2091ac 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -350,7 +350,6 @@ void taos_free_result(TAOS_RES *res) { taosArrayDestroy(pRsp->rsp.createTableLen); taosArrayDestroyP(pRsp->rsp.createTableReq, taosMemoryFree); - pRsp->resInfo.pRspMsg = NULL; doFreeReqResultInfo(&pRsp->resInfo); taosMemoryFree(pRsp); } else if (TD_RES_TMQ(res)) { @@ -359,7 +358,6 @@ void taos_free_result(TAOS_RES *res) { taosArrayDestroy(pRsp->rsp.blockDataLen); taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree); taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSchemaWrapper); - pRsp->resInfo.pRspMsg = NULL; doFreeReqResultInfo(&pRsp->resInfo); taosMemoryFree(pRsp); } else if (TD_RES_TMQ_META(res)) { @@ -402,7 +400,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { return pResInfo->userFields; } -TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false); } +TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false, TD_REQ_FROM_APP); } TAOS_RES *taos_query_with_reqid(TAOS *taos, const char *sql, int64_t reqid) { return taosQueryImplWithReqid(taos, sql, false, reqid); } @@ -828,7 +826,7 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { } int taos_validate_sql(TAOS *taos, const char *sql) { - TAOS_RES *pObj = taosQueryImpl(taos, sql, true); + TAOS_RES *pObj = taosQueryImpl(taos, sql, true, TD_REQ_FROM_APP); int code = taos_errno(pObj); @@ -1126,7 +1124,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { int64_t connId = *(int64_t *)taos; tscDebug("taos_query_a start with sql:%s", sql); - taosAsyncQueryImpl(connId, sql, fp, param, false); + taosAsyncQueryImpl(connId, sql, fp, param, false, TD_REQ_FROM_APP); tscDebug("taos_query_a end with sql:%s", sql); } @@ -1254,54 +1252,34 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { } void restartAsyncQuery(SRequestObj *pRequest, int32_t code) { - int32_t reqIdx = 0; - SRequestObj *pReqList[16] = {NULL}; - SRequestObj *pUserReq = NULL; - pReqList[0] = pRequest; - uint64_t tmpRefId = 0; - SRequestObj *pTmp = pRequest; - while (pTmp->relation.prevRefId) { - tmpRefId = pTmp->relation.prevRefId; - pTmp = acquireRequest(tmpRefId); - if (pTmp) { - pReqList[++reqIdx] = pTmp; - releaseRequest(tmpRefId); - } else { - tscError("prev req ref 0x%" PRIx64 " is not there", tmpRefId); + tscInfo("restart request: %s p: %p", pRequest->sqlstr, pRequest); + SRequestObj* pUserReq = pRequest; + acquireRequest(pRequest->self); + while (pUserReq) { + if (pUserReq->self == pUserReq->relation.userRefId || pUserReq->relation.userRefId == 0) { break; - } - } - - tmpRefId = pRequest->relation.nextRefId; - while (tmpRefId) { - pTmp = acquireRequest(tmpRefId); - if (pTmp) { - tmpRefId = pTmp->relation.nextRefId; - removeRequest(pTmp->self); - releaseRequest(pTmp->self); } else { - tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId); - break; + int64_t nextRefId = pUserReq->relation.nextRefId; + releaseRequest(pUserReq->self); + if (nextRefId) { + pUserReq = acquireRequest(nextRefId); + } } } - - for (int32_t i = reqIdx; i >= 0; i--) { - destroyCtxInRequest(pReqList[i]); - if (pReqList[i]->relation.userRefId == pReqList[i]->self || 0 == pReqList[i]->relation.userRefId) { - pUserReq = pReqList[i]; - } else { - removeRequest(pReqList[i]->self); - } - } - + bool hasSubRequest = pUserReq != pRequest || pRequest->relation.prevRefId != 0; if (pUserReq) { + destroyCtxInRequest(pUserReq); pUserReq->prevCode = code; memset(&pUserReq->relation, 0, sizeof(pUserReq->relation)); } else { - tscError("user req is missing"); + tscError("User req is missing"); + removeFromMostPrevReq(pRequest); return; } - + if (hasSubRequest) + removeFromMostPrevReq(pRequest); + else + releaseRequest(pUserReq->self); doAsyncQuery(pUserReq, true); } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 324b99022b..360a346fb7 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -538,7 +538,7 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) { code = buildShowVariablesRsp(rsp.variables, &pRes); } if (TSDB_CODE_SUCCESS == code) { - code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true); + code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false); } if (code != 0) { @@ -651,7 +651,7 @@ int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) { code = buildRetriveTableRspForCompactDb(&rsp, &pRes); } if (TSDB_CODE_SUCCESS == code) { - code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true); + code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false); } if (code != 0) { diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index f143624bab..adc8c361cd 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1256,7 +1256,7 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName, req.tsColName, req.skey, req.tsColName, req.ekey); - TAOS_RES* res = taos_query(taos, sql); + TAOS_RES* res = taosQueryImpl(taos, sql, false, TD_REQ_FROM_TAOX); SRequestObj* pRequest = (SRequestObj*)res; code = pRequest->code; if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_PAR_GET_META_ERROR) { diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index a893b27896..fb1882e472 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1010,19 +1010,8 @@ int32_t tmq_unsubscribe(tmq_t* tmq) { } taosSsleep(2); // sleep 2s for hb to send offset and rows to server - int32_t rsp; - int32_t retryCnt = 0; tmq_list_t* lst = tmq_list_new(); - while (1) { - rsp = tmq_subscribe(tmq, lst); - if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) { - break; - } else { - retryCnt++; - taosMsleep(500); - } - } - + int32_t rsp = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); return rsp; } @@ -1272,10 +1261,9 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } int32_t retryCnt = 0; - while (syncAskEp(tmq) != 0) { - if (retryCnt++ > MAX_RETRY_COUNT) { + while ((code = syncAskEp(tmq)) != 0) { + if (retryCnt++ > MAX_RETRY_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes", tmq->consumerId); - code = TSDB_CODE_MND_CONSUMER_NOT_READY; goto FAIL; } @@ -2148,26 +2136,19 @@ int32_t tmq_consumer_close(tmq_t* tmq) { if (tmq->status == TMQ_CONSUMER_STATUS__READY) { // if auto commit is set, commit before close consumer. Otherwise, do nothing. if (tmq->autoCommit) { - int32_t rsp = tmq_commit_sync(tmq, NULL); - if (rsp != 0) { - return rsp; + int32_t code = tmq_commit_sync(tmq, NULL); + if (code != 0) { + return code; } } taosSsleep(2); // sleep 2s for hb to send offset and rows to server - int32_t retryCnt = 0; tmq_list_t* lst = tmq_list_new(); - while (1) { - int32_t rsp = tmq_subscribe(tmq, lst); - if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) { - break; - } else { - retryCnt++; - taosMsleep(500); - } - } - + int32_t code = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); + if (code != 0) { + return code; + } } else { tscInfo("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId); } @@ -2647,13 +2628,9 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { SRetrieveTableRspForTmq* pRetrieveTmq = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter); if (pRspObj->rsp.withSchema) { + doFreeReqResultInfo(&pRspObj->resInfo); SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter); setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols); - taosMemoryFreeClear(pRspObj->resInfo.row); - taosMemoryFreeClear(pRspObj->resInfo.pCol); - taosMemoryFreeClear(pRspObj->resInfo.length); - taosMemoryFreeClear(pRspObj->resInfo.convertBuf); - taosMemoryFreeClear(pRspObj->resInfo.convertJson); } pRspObj->resInfo.pData = (void*)pRetrieveTmq->data; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 271fa20c54..ad65ee9b98 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -58,7 +58,7 @@ int32_t tsNumOfMnodeQueryThreads = 4; int32_t tsNumOfMnodeFetchThreads = 1; int32_t tsNumOfMnodeReadThreads = 1; int32_t tsNumOfVnodeQueryThreads = 4; -float tsRatioOfVnodeStreamThreads = 1.5F; +float tsRatioOfVnodeStreamThreads = 0.5F; int32_t tsNumOfVnodeFetchThreads = 4; int32_t tsNumOfVnodeRsmaThreads = 2; int32_t tsNumOfQnodeQueryThreads = 4; @@ -502,7 +502,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1; @@ -538,8 +538,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { return -1; if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; return 0; } @@ -586,7 +586,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfSupportVnodes = tsNumOfCores * 2; tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); - if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) @@ -600,18 +600,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { return -1; if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - tsNumOfRpcThreads = tsNumOfCores / 2; - tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); - if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; - - tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 10000); - if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) - return -1; - - tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 1000000); - if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsNumOfRpcSessions, 20, 1000000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) - return -1; - tsNumOfCommitThreads = tsNumOfCores / 2; tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4); if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) @@ -691,9 +679,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "grantMode", tsMndGrantMode, 0, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) - return -1; if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; @@ -707,7 +692,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; @@ -814,8 +798,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { return -1; if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; - // GRANT_CFG_ADD; return 0; } @@ -1353,6 +1335,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile if (taosAddClientLogCfg(tsCfg) != 0) return -1; if (taosAddServerLogCfg(tsCfg) != 0) return -1; } + taosAddSystemCfg(tsCfg); if (taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) { @@ -1379,7 +1362,9 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile if (taosSetTfsCfg(tsCfg) != 0) return -1; if (taosSetS3Cfg(tsCfg) != 0) return -1; } + taosSetSystemCfg(tsCfg); + if (taosSetFileHandlesLimit() != 0) return -1; taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32); @@ -1626,6 +1611,10 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) { tsLogSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024); uInfo("%s set to %" PRId64, name, tsLogSpace.reserved); matched = true; + } else if (strcasecmp("monitor", name) == 0) { + tsEnableMonitor = pItem->bval; + uInfo("%s set to %d", name, tsEnableMonitor); + matched = true; } break; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index f8df2edd61..9138d7c983 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -7192,6 +7192,7 @@ int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { if (tEncodeU32(&encoder, pReq->sqlLen) < 0) return -1; if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (tEncodeBinary(&encoder, pReq->msg, pReq->phyLen) < 0) return -1; + if (tEncodeI8(&encoder, pReq->source) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -7228,6 +7229,9 @@ int32_t tDeserializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->msg, &msgLen) < 0) return -1; pReq->phyLen = msgLen; + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->source) < 0) return -1; + } tEndDecode(&decoder); tDecoderClear(&decoder); @@ -8427,6 +8431,7 @@ int32_t tEncodeDeleteRes(SEncoder *pCoder, const SDeleteRes *pRes) { if (tEncodeCStr(pCoder, pRes->tableFName) < 0) return -1; if (tEncodeCStr(pCoder, pRes->tsColName) < 0) return -1; if (tEncodeI64(pCoder, pRes->ctimeMs) < 0) return -1; + if (tEncodeI8(pCoder, pRes->source) < 0) return -1; return 0; } @@ -8451,6 +8456,9 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) { if (!tDecodeIsEnd(pCoder)) { if (tDecodeI64(pCoder, &pRes->ctimeMs) < 0) return -1; } + if (!tDecodeIsEnd(pCoder)) { + if (tDecodeI8(pCoder, &pRes->source) < 0) return -1; + } return 0; } diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 57fd187da3..5307ff4b05 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -111,7 +111,7 @@ STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status); SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo); -int32_t mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans); +int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans); SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId); int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 79a5f5fd83..befb6d3521 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -610,7 +610,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { for(int32_t j = 0; j < tagSize; j++){ SJson* item = tjsonGetArrayItem(arrayTag, j); - *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); + *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); tjsonGetStringValue(item, "name", *(labels + j)); *(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN); @@ -626,7 +626,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { for(int32_t j = 0; j < metricLen; j++){ SJson *item = tjsonGetArrayItem(metrics, j); - char name[MONITOR_METRIC_NAME_LEN] = {0}; + char name[MONITOR_METRIC_NAME_LEN] = {0}; tjsonGetStringValue(item, "name", name); double value = 0; @@ -636,7 +636,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { tjsonGetDoubleValue(item, "type", &type); int32_t metricNameLen = strlen(name) + strlen(tableName) + 2; - char* metricName = taosMemoryMalloc(metricNameLen); + char* metricName = taosMemoryMalloc(metricNameLen); memset(metricName, 0, metricNameLen); sprintf(metricName, "%s:%s", tableName, name); @@ -669,7 +669,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { else{ mTrace("get metric from registry:%p", metric); } - + if(type == 0){ taos_counter_add(metric, value, (const char**)sample_labels); } @@ -689,7 +689,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { taosMemoryFreeClear(labels); } } - + } code = 0; @@ -1409,24 +1409,6 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { if (strcasecmp(cfgReq.config, "resetlog") == 0) { strcpy(dcfgReq.config, "resetlog"); #ifdef TD_ENTERPRISE - } else if (strncasecmp(cfgReq.config, "supportvnodes", 13) == 0) { - int32_t optLen = strlen("supportvnodes"); - int32_t flag = -1; - int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); - if (code < 0) return code; - - if (flag < 0 || flag > 4096) { - mError("dnode:%d, failed to config supportVnodes since value:%d. Valid range: [0, 4096]", cfgReq.dnodeId, flag); - terrno = TSDB_CODE_OUT_OF_RANGE; - goto _err_out; - } - if (flag == 0) { - flag = tsNumOfCores * 2; - } - flag = TMAX(flag, 2); - - strcpy(dcfgReq.config, "supportvnodes"); - snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) { int32_t optLen = strlen("s3blocksize"); int32_t flag = -1; diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index 622ed0080f..f2e2c556cf 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -597,6 +597,7 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb taosRUnLockLatch(&pOld->lock); pNew->pTags = NULL; + pNew->pColumns = NULL; pNew->updateTime = taosGetTimestampMs(); pNew->lock = 0; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 3ef2f64df7..50e3dd0d03 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -721,6 +721,8 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } + // add into buffer firstly + // to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already. taosThreadMutexLock(&execInfo.lock); mDebug("stream stream:%s tasks register into node list", createReq.name); saveStreamTasksInfo(&streamObj, &execInfo); @@ -1811,7 +1813,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid, pStream->name, pTrans->id); - int32_t code = mndStreamSetUpdateEpsetAction(pStream, pChangeInfo, pTrans); + int32_t code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans); // todo: not continue, drop all and retry again if (code != TSDB_CODE_SUCCESS) { diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 1ae85a2cc6..a124b4052c 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -115,6 +115,7 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) { char buf[256] = {0}; EPSET_TO_STR(&entry.epset, buf); + mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf); taosArrayPush(pVgroupListSnapshot, &entry); sdbRelease(pSdb, pVgroup); @@ -300,7 +301,10 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa return code; } - mDebug("pause node:%d, epset:%d", pTask->info.nodeId, epset.numOfEps); + char buf[256] = {0}; + EPSET_TO_STR(&epset, buf); + mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf); + code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0); if (code != 0) { taosMemoryFree(pReq); @@ -462,14 +466,22 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha return TSDB_CODE_SUCCESS; } -static int32_t doSetUpdateTaskAction(STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) { +static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) { void *pBuf = NULL; int32_t len = 0; streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList); doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id); - int32_t code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet, 0); + SEpSet epset = {0}; + bool hasEpset = false; + int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); + if (code != TSDB_CODE_SUCCESS || !hasEpset) { + terrno = code; + return code; + } + + code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &epset, TSDB_CODE_VND_INVALID_VGROUP_ID); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(pBuf); } @@ -478,14 +490,14 @@ static int32_t doSetUpdateTaskAction(STrans *pTrans, SStreamTask *pTask, SVgroup } // build trans to update the epset -int32_t mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) { +int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) { mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid); taosWLockLatch(&pStream->lock); SStreamTaskIter *pIter = createStreamTaskIter(pStream); while (streamTaskIterNextTask(pIter)) { SStreamTask *pTask = streamTaskIterGetCurrent(pIter); - int32_t code = doSetUpdateTaskAction(pTrans, pTask, pInfo); + int32_t code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo); if (code != TSDB_CODE_SUCCESS) { destroyStreamTaskIter(pIter); taosWUnLockLatch(&pStream->lock); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 04d74112d4..f7db0d7b15 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -15,11 +15,11 @@ #define _DEFAULT_SOURCE #include "mndTrans.h" -#include "mndSubscribe.h" #include "mndDb.h" #include "mndPrivilege.h" #include "mndShow.h" #include "mndStb.h" +#include "mndSubscribe.h" #include "mndSync.h" #include "mndUser.h" @@ -801,16 +801,17 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { if (pNew->conflict == TRN_CONFLICT_TOPIC) { if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; if (pTrans->conflict == TRN_CONFLICT_TOPIC || pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) { - if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 ) conflict = true; + if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true; } } if (pNew->conflict == TRN_CONFLICT_TOPIC_INSIDE) { if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; - if (pTrans->conflict == TRN_CONFLICT_TOPIC ) { - if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 ) conflict = true; + if (pTrans->conflict == TRN_CONFLICT_TOPIC) { + if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true; } if (pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) { - if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 && strcasecmp(pNew->stbname, pTrans->stbname) == 0) conflict = true; + if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 && strcasecmp(pNew->stbname, pTrans->stbname) == 0) + conflict = true; } } @@ -847,7 +848,7 @@ int32_t mndTransCheckConflict(SMnode *pMnode, STrans *pTrans) { } int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { - if(pTrans == NULL) return -1; + if (pTrans == NULL) return -1; if (mndTransCheckConflict(pMnode, pTrans) != 0) { return -1; @@ -1142,6 +1143,7 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio return -1; } rpcMsg.info.traceId.rootId = pTrans->mTraceId; + rpcMsg.info.notFreeAhandle = 1; memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); @@ -1156,7 +1158,7 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio int32_t code = tmsgSendReq(&pAction->epSet, &rpcMsg); if (code == 0) { pAction->msgSent = 1; - //pAction->msgReceived = 0; + // pAction->msgReceived = 0; pAction->errCode = TSDB_CODE_ACTION_IN_PROGRESS; mInfo("trans:%d, %s:%d is sent, %s", pTrans->id, mndTransStr(pAction->stage), pAction->id, detail); @@ -1253,16 +1255,16 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA for (int32_t action = 0; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pArray, action); - mDebug("trans:%d, %s:%d Sent:%d, Received:%d, errCode:0x%x, acceptableCode:0x%x, retryCode:0x%x", - pTrans->id, mndTransStr(pAction->stage), pAction->id, pAction->msgSent, pAction->msgReceived, - pAction->errCode, pAction->acceptableCode, pAction->retryCode); + mDebug("trans:%d, %s:%d Sent:%d, Received:%d, errCode:0x%x, acceptableCode:0x%x, retryCode:0x%x", pTrans->id, + mndTransStr(pAction->stage), pAction->id, pAction->msgSent, pAction->msgReceived, pAction->errCode, + pAction->acceptableCode, pAction->retryCode); if (pAction->msgSent) { if (pAction->msgReceived) { if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) { mndTransResetAction(pMnode, pTrans, pAction); mInfo("trans:%d, %s:%d reset", pTrans->id, mndTransStr(pAction->stage), pAction->id); } - } + } } } return TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index e610161544..9fdb4993bd 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -106,7 +106,7 @@ typedef struct SQueryNode SQueryNode; #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" -#define VNODE_METRIC_SQL_COUNT "taos_sql_req:count" +#define VNODE_METRIC_SQL_COUNT "taosd_sql_req:count" #define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type" #define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id" diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 011e62cb89..47900d540c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -917,6 +917,22 @@ static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask } } +int32_t handleStep2Async(SStreamTask* pStreamTask, void* param) { + STQ* pTq = param; + + SStreamMeta* pMeta = pStreamTask->pMeta; + STaskId hId = pStreamTask->hTaskInfo.id; + SStreamTask* pTask = streamMetaAcquireTask(pStreamTask->pMeta, hId.streamId, hId.taskId); + if (pTask == NULL) { + // todo handle error + } + + doStartFillhistoryStep2(pTask, pStreamTask, pTq); + + streamMetaReleaseTask(pMeta, pTask); + return 0; +} + // this function should be executed by only one thread, so we set an sentinel to protect this function int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { SStreamScanHistoryReq* pReq = (SStreamScanHistoryReq*)pMsg->pCont; @@ -1007,37 +1023,27 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { // the following procedure should be executed, no matter status is stop/pause or not tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, pTask->execInfo.step1El); - if (pTask->info.fillHistory) { - SStreamTask* pStreamTask = NULL; + ASSERT(pTask->info.fillHistory == 1); - // 1. get the related stream task - pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); - if (pStreamTask == NULL) { - tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s", - pTask->streamTaskId.taskId, pTask->id.idStr); + // 1. get the related stream task + SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); + if (pStreamTask == NULL) { + tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s", + pTask->streamTaskId.taskId, pTask->id.idStr); tqDebug("s-task:%s fill-history task set status to be dropping and drop it", id); streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0); - atomic_store_32(&pTask->status.inScanHistorySentinel, 0); - streamMetaReleaseTask(pMeta, pTask); - return -1; - } - - ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE); - - code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); - if (code == TSDB_CODE_SUCCESS) { - doStartFillhistoryStep2(pTask, pStreamTask, pTq); - } else { - tqError("s-task:%s failed to halt s-task:%s, not launch step2", id, pStreamTask->id.idStr); - } - - streamMetaReleaseTask(pMeta, pStreamTask); - } else { - ASSERT(0); + atomic_store_32(&pTask->status.inScanHistorySentinel, 0); + streamMetaReleaseTask(pMeta, pTask); + return -1; } + ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE); + code = streamTaskHandleEventAsync(pStreamTask->status.pSM, TASK_EVENT_HALT, handleStep2Async, pTq); + + streamMetaReleaseTask(pMeta, pStreamTask); + atomic_store_32(&pTask->status.inScanHistorySentinel, 0); streamMetaReleaseTask(pMeta, pTask); return code; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 7a737fbb5d..110cac152d 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -368,24 +368,11 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con } } -// todo ignore the error in wal? bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { SWalReader* pWalReader = pReader->pWalReader; - SSDataBlock* pDataBlock = NULL; uint64_t st = taosGetTimestampMs(); while (1) { - // try next message in wal file - if (walNextValidMsg(pWalReader) < 0) { - return false; - } - - void* pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); - int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); - int64_t ver = pWalReader->pHead->head.version; - - tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver); - pReader->nextBlk = 0; int32_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < numOfBlocks) { tqTrace("tq reader next data block %d/%d, len:%d %" PRId64, pReader->nextBlk, numOfBlocks, pReader->msg.msgLen, @@ -400,33 +387,32 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { tqTrace("tq reader return submit block, uid:%" PRId64, pSubmitTbData->uid); SSDataBlock* pRes = NULL; int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); - if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) { - if (pDataBlock == NULL) { - pDataBlock = createOneDataBlock(pRes, true); - } else { - blockDataMerge(pDataBlock, pRes); - } + if (code == TSDB_CODE_SUCCESS) { + return true; } } else { pReader->nextBlk += 1; tqTrace("tq reader discard submit block, uid:%" PRId64 ", continue", pSubmitTbData->uid); } } + tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); pReader->msg.msgStr = NULL; - if (pDataBlock != NULL) { - blockDataCleanup(pReader->pResBlock); - copyDataBlock(pReader->pResBlock, pDataBlock); - blockDataDestroy(pDataBlock); - return true; - } else { - qTrace("stream scan return empty, all %d submit blocks consumed, %s", numOfBlocks, id); - } - if (taosGetTimestampMs() - st > 1000) { return false; } + + // try next message in wal file + if (walNextValidMsg(pWalReader) < 0) { + return false; + } + + void* pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); + int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); + int64_t ver = pWalReader->pHead->head.version; + tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver); + pReader->nextBlk = 0; } } diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 280c110711..73508202d9 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -28,8 +28,8 @@ static int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDurat // extract data blocks(submit/delete) from WAL, and add them into the input queue for all the sources tasks. int32_t tqScanWal(STQ* pTq) { - int32_t vgId = TD_VID(pTq->pVnode); SStreamMeta* pMeta = pTq->pStreamMeta; + int32_t vgId = pMeta->vgId; int64_t st = taosGetTimestampMs(); tqDebug("vgId:%d continue to check if data in wal are available, scanCounter:%d", vgId, pMeta->scanInfo.scanCounter); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index dad1211326..6029575e2c 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -263,8 +263,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } else if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB) { PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq) } else if (pHead->msgType == TDMT_VND_DELETE) { - fetchVer++; - continue; + PROCESS_EXCLUDED_MSG(SDeleteRes, tDecodeDeleteRes) } } diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index a2d45062b9..1b67dce9b0 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -142,8 +142,10 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); if (ppHTask == NULL || *ppHTask == NULL) { - tqError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already", - vgId, req.taskId); + tqError( + "vgId:%d failed to acquire fill-history task:0x%x when handling update, may have been dropped already, rel " + "stream task:0x%x", + vgId, (uint32_t)pTask->hTaskInfo.id.taskId, req.taskId); CLEAR_RELATED_FILLHISTORY_TASK(pTask); } else { tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr); @@ -612,23 +614,35 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) { SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; + int32_t vgId = pMeta->vgId; + STaskId hTaskId = {0}; - int32_t vgId = pMeta->vgId; tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); - if (pTask != NULL) { - // drop the related fill-history task firstly + streamMetaWLock(pMeta); + + STaskId id = {.streamId = pReq->streamId, .taskId = pReq->taskId}; + SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + if ((ppTask != NULL) && ((*ppTask) != NULL)) { + streamMetaAcquireOneTask(*ppTask); + SStreamTask* pTask = *ppTask; + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - STaskId* pHTaskId = &pTask->hTaskInfo.id; - streamMetaUnregisterTask(pMeta, pHTaskId->streamId, pHTaskId->taskId); - tqDebug("s-task:0x%x vgId:%d drop fill-history task:0x%x firstly", pReq->taskId, vgId, - (int32_t)pHTaskId->taskId); + hTaskId.streamId = pTask->hTaskInfo.id.streamId; + hTaskId.taskId = pTask->hTaskInfo.id.taskId; } + + streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt); streamMetaReleaseTask(pMeta, pTask); } - streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt, true); + streamMetaWUnLock(pMeta); + + // drop the related fill-history task firstly + if (hTaskId.taskId != 0 && hTaskId.streamId != 0) { + streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId); + tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId); + } // drop the stream task now streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId); @@ -865,7 +879,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg){ pHistoryTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId); if (pHistoryTask == NULL) { tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64 - ", it may have been dropped already", + ", it may have been dropped already", pMeta->vgId, pTask->hTaskInfo.id.taskId); streamMetaReleaseTask(pMeta, pTask); @@ -915,6 +929,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t } if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) { + pTask->hTaskInfo.operatorOpen = false; streamStartScanHistoryAsync(pTask, igUntreated); } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) { tqScanWalAsync((STQ*)handle, false); diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index aff7e4642b..16abe69def 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -128,12 +128,12 @@ void initTqAPI(SStoreTqReader* pTq) { pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed; pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it - pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it +// pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it pTq->tqGetResultBlock = tqGetResultBlock; - pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; +// pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; pTq->tqGetResultBlockTime = tqGetResultBlockTime; pTq->tqGetStreamExecProgress = tqGetStreamExecInfo; diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 43ca8a3c38..5263546765 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -352,6 +352,8 @@ typedef struct STableMergeScanInfo { SSDataBlock* nextDurationBlocks[2]; bool rtnNextDurationBlocks; int32_t nextDurationBlocksIdx; + + bool bSortRowId; STmsSubTablesMergeInfo* pSubTablesMergeInfo; } STableMergeScanInfo; diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index 436d1cefb8..ca799673ea 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -194,6 +194,9 @@ void tsortSetClosed(SSortHandle* pHandle); void tsortSetSingleTableMerge(SSortHandle* pHandle); void tsortSetAbortCheckFn(SSortHandle* pHandle, bool (*checkFn)(void* param), void* param); +int32_t tsortSetSortByRowId(SSortHandle* pHandle, int32_t extRowsSize); + +void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHandle* pTupleHandle); /** * @brief comp the tuple with keyBuf, if not equal, new keys will be built in keyBuf, newLen will be stored in keyLen * @param [in] pSortCols cols to comp and build diff --git a/source/libs/executor/src/dynqueryctrloperator.c b/source/libs/executor/src/dynqueryctrloperator.c index 9e7f1144f8..e7c9c17a0b 100755 --- a/source/libs/executor/src/dynqueryctrloperator.c +++ b/source/libs/executor/src/dynqueryctrloperator.c @@ -456,7 +456,8 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS code = pInfo->stbJoin.basic.srcScan[1] ? buildSingleTableScanOperatorParam(&pSrcParam1, 1, rightVg, rightUid) : buildExchangeOperatorParam(&pSrcParam1, 1, rightVg, rightUid); } } - + + bool initParam = pSrcParam0 ? true : false; if (TSDB_CODE_SUCCESS == code) { code = buildGroupCacheOperatorParam(&pGcParam0, 0, *leftVg, *leftUid, pPost->leftNeedCache, pSrcParam0); pSrcParam0 = NULL; @@ -466,7 +467,7 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS pSrcParam1 = NULL; } if (TSDB_CODE_SUCCESS == code) { - code = buildMergeJoinOperatorParam(ppParam, pSrcParam0 ? true : false, pGcParam0, pGcParam1); + code = buildMergeJoinOperatorParam(ppParam, initParam, pGcParam0, pGcParam1); } if (TSDB_CODE_SUCCESS != code) { if (pSrcParam0) { diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index ecc2526dd9..e1aa75d828 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -32,12 +32,6 @@ #include "operator.h" #include "querytask.h" - -#define FILL_POS_INVALID 0 -#define FILL_POS_START 1 -#define FILL_POS_MID 2 -#define FILL_POS_END 3 - typedef struct STimeRange { TSKEY skey; TSKEY ekey; @@ -474,1139 +468,3 @@ _error: taosMemoryFreeClear(pOperator); return NULL; } - -TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) { - STimeWindow win = {.skey = ts, .ekey = ts}; - getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); - return win.skey; -} - -TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { - STimeWindow win = {.skey = ts, .ekey = ts}; - getNextTimeWindow(pInterval, &win, TSDB_ORDER_DESC); - return win.skey; -} - -void setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) { - colDataSetVal(pCol, rowId, pCell->pData, pCell->isNull); -} - -SResultCellData* getResultCell(SResultRowData* pRaw, int32_t index) { - if (!pRaw || !pRaw->pRowVal) { - return NULL; - } - char* pData = (char*)pRaw->pRowVal; - SResultCellData* pCell = pRaw->pRowVal; - for (int32_t i = 0; i < index; i++) { - pData += (pCell->bytes + sizeof(SResultCellData)); - pCell = (SResultCellData*)pData; - } - return pCell; -} - -void* destroyFillColumnInfo(SFillColInfo* pFillCol, int32_t start, int32_t end) { - for (int32_t i = start; i < end; i++) { - destroyExprInfo(pFillCol[i].pExpr, 1); - taosVariantDestroy(&pFillCol[i].fillVal); - } - taosMemoryFreeClear(pFillCol[start].pExpr); - taosMemoryFree(pFillCol); - return NULL; -} - -void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { - pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); - tSimpleHashCleanup(pFillSup->pResMap); - pFillSup->pResMap = NULL; - cleanupExprSupp(&pFillSup->notFillExprSup); - if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) { - taosMemoryFree(pFillSup->cur.pRowVal); - } - taosMemoryFree(pFillSup->prev.pRowVal); - taosMemoryFree(pFillSup->next.pRowVal); - taosMemoryFree(pFillSup->nextNext.pRowVal); - - taosMemoryFree(pFillSup); - return NULL; -} - -void destroySPoint(void* ptr) { - SPoint* point = (SPoint*) ptr; - taosMemoryFreeClear(point->val); -} - -void* destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) { - taosArrayDestroyEx(pFillLinear->pEndPoints, destroySPoint); - taosArrayDestroyEx(pFillLinear->pNextEndPoints, destroySPoint); - taosMemoryFree(pFillLinear); - return NULL; -} -void* destroyStreamFillInfo(SStreamFillInfo* pFillInfo) { - if (pFillInfo->type == TSDB_FILL_SET_VALUE || pFillInfo->type == TSDB_FILL_SET_VALUE_F || - pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) { - taosMemoryFreeClear(pFillInfo->pResRow->pRowVal); - taosMemoryFreeClear(pFillInfo->pResRow); - } - pFillInfo->pLinearInfo = destroyStreamFillLinearInfo(pFillInfo->pLinearInfo); - taosArrayDestroy(pFillInfo->delRanges); - taosMemoryFree(pFillInfo); - return NULL; -} - -static void destroyStreamFillOperatorInfo(void* param) { - SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param; - pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo); - pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup); - pInfo->pRes = blockDataDestroy(pInfo->pRes); - pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); - pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); - pInfo->matchInfo.pList = taosArrayDestroy(pInfo->matchInfo.pList); - taosMemoryFree(pInfo); -} - -static void resetFillWindow(SResultRowData* pRowData) { - pRowData->key = INT64_MIN; - taosMemoryFreeClear(pRowData->pRowVal); -} - -void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStorageAPI* pAPI) { - if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) { - resetFillWindow(&pFillSup->cur); - } else { - pFillSup->cur.key = INT64_MIN; - pFillSup->cur.pRowVal = NULL; - } - resetFillWindow(&pFillSup->prev); - resetFillWindow(&pFillSup->next); - resetFillWindow(&pFillSup->nextNext); -} - -void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - - void* pState = pOperator->pTaskInfo->streamInfo.pState; - resetPrevAndNextWindow(pFillSup, pState, pAPI); - - SWinKey key = {.ts = ts, .groupId = groupId}; - int32_t curVLen = 0; - - int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); - ASSERT(code == TSDB_CODE_SUCCESS); - pFillSup->cur.key = key.ts; -} - -void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - void* pState = pOperator->pTaskInfo->streamInfo.pState; - resetPrevAndNextWindow(pFillSup, pState, pAPI); - - SWinKey key = {.ts = ts, .groupId = groupId}; - void* curVal = NULL; - int32_t curVLen = 0; - int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); - ASSERT(code == TSDB_CODE_SUCCESS); - pFillSup->cur.key = key.ts; - pFillSup->cur.pRowVal = curVal; - - SStreamStateCur* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key); - SWinKey preKey = {.ts = INT64_MIN, .groupId = groupId}; - void* preVal = NULL; - int32_t preVLen = 0; - code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); - - if (code == TSDB_CODE_SUCCESS) { - pFillSup->prev.key = preKey.ts; - pFillSup->prev.pRowVal = preVal; - - code = pAPI->stateStore.streamStateCurNext(pState, pCur); - ASSERT(code == TSDB_CODE_SUCCESS); - - code = pAPI->stateStore.streamStateCurNext(pState, pCur); - if (code != TSDB_CODE_SUCCESS) { - pAPI->stateStore.streamStateFreeCur(pCur); - pCur = NULL; - } - } else { - pAPI->stateStore.streamStateFreeCur(pCur); - pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key); - } - - SWinKey nextKey = {.ts = INT64_MIN, .groupId = groupId}; - void* nextVal = NULL; - int32_t nextVLen = 0; - code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); - if (code == TSDB_CODE_SUCCESS) { - pFillSup->next.key = nextKey.ts; - pFillSup->next.pRowVal = nextVal; - if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { - code = pAPI->stateStore.streamStateCurNext(pState, pCur); - if (code == TSDB_CODE_SUCCESS) { - SWinKey nextNextKey = {.groupId = groupId}; - void* nextNextVal = NULL; - int32_t nextNextVLen = 0; - code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); - if (code == TSDB_CODE_SUCCESS) { - pFillSup->nextNext.key = nextNextKey.ts; - pFillSup->nextNext.pRowVal = nextNextVal; - } - } - } - } - pAPI->stateStore.streamStateFreeCur(pCur); -} - -static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; } -static bool hasNextWindow(SStreamFillSupporter* pFillSup) { return pFillSup->next.key != INT64_MIN; } -static bool hasNextNextWindow(SStreamFillSupporter* pFillSup) { - return pFillSup->nextNext.key != INT64_MIN; - return false; -} - -static void transBlockToResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SResultRowData* pRowVal) { - int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); - SResultCellData* pCell = getResultCell(pRowVal, i); - if (!colDataIsNull_s(pColData, rowId)) { - pCell->isNull = false; - pCell->type = pColData->info.type; - pCell->bytes = pColData->info.bytes; - char* val = colDataGetData(pColData, rowId); - if (IS_VAR_DATA_TYPE(pCell->type)) { - memcpy(pCell->pData, val, varDataTLen(val)); - } else { - memcpy(pCell->pData, val, pCell->bytes); - } - } else { - pCell->isNull = true; - } - } - pRowVal->key = ts; -} - -static void calcDeltaData(SSDataBlock* pBlock, int32_t rowId, SResultRowData* pRowVal, SArray* pDelta, - SFillColInfo* pFillCol, int32_t numOfCol, int32_t winCount, int32_t order) { - for (int32_t i = 0; i < numOfCol; i++) { - if (!pFillCol[i].notFillCol) { - int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); - SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - char* var = colDataGetData(pCol, rowId); - double start = 0; - GET_TYPED_DATA(start, double, pCol->info.type, var); - SResultCellData* pCell = getResultCell(pRowVal, slotId); - double end = 0; - GET_TYPED_DATA(end, double, pCell->type, pCell->pData); - double delta = 0; - if (order == TSDB_ORDER_ASC) { - delta = (end - start) / winCount; - } else { - delta = (start - end) / winCount; - } - taosArraySet(pDelta, slotId, &delta); - } - } -} - -static void calcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, - int32_t numOfCol) { - for (int32_t i = 0; i < numOfCol; i++) { - if (!pFillCol[i].notFillCol) { - int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); - SResultCellData* pECell = getResultCell(pEndRow, slotId); - SPoint* pPoint = taosArrayGet(pEndPoins, slotId); - pPoint->key = pEndRow->key; - memcpy(pPoint->val, pECell->pData, pECell->bytes); - } - } -} - -static void setFillInfoStart(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { - ts = taosTimeAdd(ts, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); - pFillInfo->start = ts; -} - -static void setFillInfoEnd(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { - ts = taosTimeAdd(ts, pInterval->sliding * -1, pInterval->slidingUnit, pInterval->precision); - pFillInfo->end = ts; -} - -static void setFillKeyInfo(TSKEY start, TSKEY end, SInterval* pInterval, SStreamFillInfo* pFillInfo) { - setFillInfoStart(start, pInterval, pFillInfo); - pFillInfo->current = pFillInfo->start; - setFillInfoEnd(end, pInterval, pFillInfo); -} - -void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) { - if (!hasPrevWindow(pFillSup) || !hasNextWindow(pFillSup)) { - pFillInfo->needFill = false; - return; - } - - TSKEY realStart = taosTimeAdd(pFillSup->prev.key, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); - - pFillInfo->needFill = true; - pFillInfo->start = realStart; - pFillInfo->current = pFillInfo->start; - pFillInfo->end = end; - pFillInfo->pos = FILL_POS_INVALID; - switch (pFillInfo->type) { - case TSDB_FILL_NULL: - case TSDB_FILL_NULL_F: - case TSDB_FILL_SET_VALUE: - case TSDB_FILL_SET_VALUE_F: - break; - case TSDB_FILL_PREV: - pFillInfo->pResRow = &pFillSup->prev; - break; - case TSDB_FILL_NEXT: - pFillInfo->pResRow = &pFillSup->next; - break; - case TSDB_FILL_LINEAR: { - setFillKeyInfo(pFillSup->prev.key, pFillSup->next.key, &pFillSup->interval, pFillInfo); - pFillInfo->pLinearInfo->hasNext = false; - pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, - pFillSup->numOfAllCols); - pFillInfo->pResRow = &pFillSup->prev; - pFillInfo->pLinearInfo->winIndex = 0; - } break; - default: - ASSERT(0); - break; - } -} - -void copyNotFillExpData(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) { - for (int32_t i = pFillSup->numOfFillCols; i < pFillSup->numOfAllCols; ++i) { - SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; - int32_t slotId = GET_DEST_SLOT_ID(pFillCol); - SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId); - SResultCellData* pCurCell = getResultCell(&pFillSup->cur, slotId); - pCell->isNull = pCurCell->isNull; - if (!pCurCell->isNull) { - memcpy(pCell->pData, pCurCell->pData, pCell->bytes); - } - } -} - -void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillSupporter* pFillSup, - SStreamFillInfo* pFillInfo) { - pFillInfo->preRowKey = pFillSup->cur.key; - if (!hasPrevWindow(pFillSup) && !hasNextWindow(pFillSup)) { - pFillInfo->needFill = false; - pFillInfo->pos = FILL_POS_START; - return; - } - TSKEY prevWKey = INT64_MIN; - TSKEY nextWKey = INT64_MIN; - if (hasPrevWindow(pFillSup)) { - prevWKey = pFillSup->prev.key; - } - if (hasNextWindow(pFillSup)) { - nextWKey = pFillSup->next.key; - } - - pFillInfo->needFill = true; - pFillInfo->pos = FILL_POS_INVALID; - switch (pFillInfo->type) { - case TSDB_FILL_NULL: - case TSDB_FILL_NULL_F: - case TSDB_FILL_SET_VALUE: - case TSDB_FILL_SET_VALUE_F: { - if (pFillSup->prev.key == pFillInfo->preRowKey) { - resetFillWindow(&pFillSup->prev); - } - if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) { - if (pFillSup->next.key == pFillInfo->nextRowKey) { - pFillInfo->preRowKey = INT64_MIN; - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_END; - } else { - pFillInfo->needFill = false; - pFillInfo->pos = FILL_POS_START; - } - } else if (hasPrevWindow(pFillSup)) { - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_END; - } else { - setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_START; - } - copyNotFillExpData(pFillSup, pFillInfo); - } break; - case TSDB_FILL_PREV: { - if (hasNextWindow(pFillSup) && ((pFillSup->next.key != pFillInfo->nextRowKey) || - (pFillSup->next.key == pFillInfo->nextRowKey && hasNextNextWindow(pFillSup)) || - (pFillSup->next.key == pFillInfo->nextRowKey && !hasPrevWindow(pFillSup)))) { - setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_START; - resetFillWindow(&pFillSup->prev); - pFillSup->prev.key = pFillSup->cur.key; - pFillSup->prev.pRowVal = pFillSup->cur.pRowVal; - } else if (hasPrevWindow(pFillSup)) { - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_END; - pFillInfo->preRowKey = INT64_MIN; - } - pFillInfo->pResRow = &pFillSup->prev; - } break; - case TSDB_FILL_NEXT: { - if (hasPrevWindow(pFillSup)) { - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_END; - resetFillWindow(&pFillSup->next); - pFillSup->next.key = pFillSup->cur.key; - pFillSup->next.pRowVal = pFillSup->cur.pRowVal; - pFillInfo->preRowKey = INT64_MIN; - } else { - ASSERT(hasNextWindow(pFillSup)); - setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_START; - } - pFillInfo->pResRow = &pFillSup->next; - } break; - case TSDB_FILL_LINEAR: { - pFillInfo->pLinearInfo->winIndex = 0; - if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) { - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_MID; - pFillInfo->pLinearInfo->nextEnd = nextWKey; - calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, - pFillSup->numOfAllCols); - pFillInfo->pResRow = &pFillSup->prev; - - calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pNextEndPoints, pFillSup->pAllColInfo, - pFillSup->numOfAllCols); - pFillInfo->pLinearInfo->hasNext = true; - } else if (hasPrevWindow(pFillSup)) { - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_END; - pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, - pFillSup->numOfAllCols); - pFillInfo->pResRow = &pFillSup->prev; - pFillInfo->pLinearInfo->hasNext = false; - } else { - ASSERT(hasNextWindow(pFillSup)); - setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_START; - pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, - pFillSup->numOfAllCols); - pFillInfo->pResRow = &pFillSup->cur; - pFillInfo->pLinearInfo->hasNext = false; - } - } break; - default: - ASSERT(0); - break; - } - ASSERT(pFillInfo->pos != FILL_POS_INVALID); -} - -static bool checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t groupId) { - SWinKey key = {.groupId = groupId, .ts = ts}; - if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) { - return false; - } - tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); - return true; -} - -static bool buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFillSup, TSKEY ts, SSDataBlock* pBlock) { - if (pBlock->info.rows >= pBlock->info.capacity) { - return false; - } - uint64_t groupId = pBlock->info.id.groupId; - if (pFillSup->hasDelete && !checkResult(pFillSup, ts, groupId)) { - return true; - } - for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { - SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; - int32_t slotId = GET_DEST_SLOT_ID(pFillCol); - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId); - SFillInfo tmpInfo = { - .currentKey = ts, - .order = TSDB_ORDER_ASC, - .interval = pFillSup->interval, - }; - bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pColData, pBlock->info.rows); - if (!filled) { - SResultCellData* pCell = getResultCell(pResRow, slotId); - setRowCell(pColData, pBlock->info.rows, pCell); - } - } - pBlock->info.rows++; - return true; -} - -static bool hasRemainCalc(SStreamFillInfo* pFillInfo) { - if (pFillInfo->current != INT64_MIN && pFillInfo->current <= pFillInfo->end) { - return true; - } - return false; -} - -static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { - while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { - STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; - if (inWinRange(&pFillSup->winRange, &st)) { - buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); - } - pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); - } -} - -static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { - while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { - uint64_t groupId = pBlock->info.id.groupId; - SWinKey key = {.groupId = groupId, .ts = pFillInfo->current}; - STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; - if ( ( pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId) ) || !inWinRange(&pFillSup->winRange, &st) ) { - pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); - pFillInfo->pLinearInfo->winIndex++; - continue; - } - pFillInfo->pLinearInfo->winIndex++; - for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { - SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; - SFillInfo tmp = { - .currentKey = pFillInfo->current, - .order = TSDB_ORDER_ASC, - .interval = pFillSup->interval, - }; - - int32_t slotId = GET_DEST_SLOT_ID(pFillCol); - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId); - int16_t type = pColData->info.type; - SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId); - int32_t index = pBlock->info.rows; - if (pFillCol->notFillCol) { - bool filled = fillIfWindowPseudoColumn(&tmp, pFillCol, pColData, index); - if (!filled) { - setRowCell(pColData, index, pCell); - } - } else { - if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) { - colDataSetNULL(pColData, index); - continue; - } - SPoint* pEnd = taosArrayGet(pFillInfo->pLinearInfo->pEndPoints, slotId); - double vCell = 0; - SPoint start = {0}; - start.key = pFillInfo->pResRow->key; - start.val = pCell->pData; - - SPoint cur = {0}; - cur.key = pFillInfo->current; - cur.val = taosMemoryCalloc(1, pCell->bytes); - taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type); - colDataSetVal(pColData, index, (const char*)cur.val, false); - destroySPoint(&cur); - } - } - pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); - pBlock->info.rows++; - } -} - -static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - - SWinKey key = {.groupId = groupId, .ts = pRow->key}; - int32_t code = pAPI->stateStore.streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len); - qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 " code:%d", key.ts, key.groupId, code); - ASSERT(code == TSDB_CODE_SUCCESS); -} - -static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { - if (pFillInfo->needFill == false) { - buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); - return; - } - - if (pFillInfo->pos == FILL_POS_START) { - if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { - pFillInfo->pos = FILL_POS_INVALID; - } - } - if (pFillInfo->type != TSDB_FILL_LINEAR) { - doStreamFillNormal(pFillSup, pFillInfo, pRes); - } else { - doStreamFillLinear(pFillSup, pFillInfo, pRes); - - if (pFillInfo->pos == FILL_POS_MID) { - if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { - pFillInfo->pos = FILL_POS_INVALID; - } - } - - if (pFillInfo->current > pFillInfo->end && pFillInfo->pLinearInfo->hasNext) { - pFillInfo->pLinearInfo->hasNext = false; - pFillInfo->pLinearInfo->winIndex = 0; - taosArraySwap(pFillInfo->pLinearInfo->pEndPoints, pFillInfo->pLinearInfo->pNextEndPoints); - pFillInfo->pResRow = &pFillSup->cur; - setFillKeyInfo(pFillSup->cur.key, pFillInfo->pLinearInfo->nextEnd, &pFillSup->interval, pFillInfo); - doStreamFillLinear(pFillSup, pFillInfo, pRes); - } - } - if (pFillInfo->pos == FILL_POS_END) { - if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { - pFillInfo->pos = FILL_POS_INVALID; - } - } -} - -void keepBlockRowInDiscBuf(SOperatorInfo* pOperator, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, TSKEY* tsCol, - int32_t rowId, uint64_t groupId, int32_t rowSize) { - TSKEY ts = tsCol[rowId]; - pFillInfo->nextRowKey = ts; - SResultRowData tmpNextRow = {.key = ts}; - tmpNextRow.pRowVal = taosMemoryCalloc(1, rowSize); - transBlockToResultRow(pBlock, rowId, ts, &tmpNextRow); - keepResultInDiscBuf(pOperator, groupId, &tmpNextRow, rowSize); - taosMemoryFreeClear(tmpNextRow.pRowVal); -} - -static void doFillResults(SOperatorInfo* pOperator, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, - SSDataBlock* pBlock, TSKEY* tsCol, int32_t rowId, SSDataBlock* pRes) { - uint64_t groupId = pBlock->info.id.groupId; - getWindowFromDiscBuf(pOperator, tsCol[rowId], groupId, pFillSup); - if (pFillSup->prev.key == pFillInfo->preRowKey) { - resetFillWindow(&pFillSup->prev); - } - setFillValueInfo(pBlock, tsCol[rowId], rowId, pFillSup, pFillInfo); - doStreamFillRange(pFillInfo, pFillSup, pRes); -} - -static void doStreamFillImpl(SOperatorInfo* pOperator) { - SStreamFillOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStreamFillSupporter* pFillSup = pInfo->pFillSup; - SStreamFillInfo* pFillInfo = pInfo->pFillInfo; - SSDataBlock* pBlock = pInfo->pSrcBlock; - uint64_t groupId = pBlock->info.id.groupId; - SSDataBlock* pRes = pInfo->pRes; - SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol); - TSKEY* tsCol = (TSKEY*)pTsCol->pData; - pRes->info.id.groupId = groupId; - pInfo->srcRowIndex++; - - if (pInfo->srcRowIndex == 0) { - keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); - pInfo->srcRowIndex++; - } - - while (pInfo->srcRowIndex < pBlock->info.rows) { - TSKEY ts = tsCol[pInfo->srcRowIndex]; - keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); - doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); - if (pInfo->pRes->info.rows == pInfo->pRes->info.capacity) { - blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); - return; - } - pInfo->srcRowIndex++; - } - doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); - blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); - blockDataCleanup(pInfo->pSrcBlock); -} - -static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { - SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; - void* pState = pOp->pTaskInfo->streamInfo.pState; - - SSDataBlock* pBlock = delRes; - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); - SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); - SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - SColumnInfoData* pTbNameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - colDataSetVal(pStartCol, pBlock->info.rows, (const char*)&start, false); - colDataSetVal(pEndCol, pBlock->info.rows, (const char*)&end, false); - colDataSetNULL(pUidCol, pBlock->info.rows); - colDataSetVal(pGroupCol, pBlock->info.rows, (const char*)&groupId, false); - colDataSetNULL(pCalStartCol, pBlock->info.rows); - colDataSetNULL(pCalEndCol, pBlock->info.rows); - - SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - - void* tbname = NULL; - pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname); - if (tbname == NULL) { - colDataSetNULL(pTableCol, pBlock->info.rows); - } else { - char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; - STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); - colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); - pAPI->stateStore.streamStateFreeVal(tbname); - } - - pBlock->info.rows++; -} - -static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId, - SSDataBlock* delRes) { - SStreamFillOperatorInfo* pInfo = pOperator->info; - SStreamFillSupporter* pFillSup = pInfo->pFillSup; - if (hasPrevWindow(pFillSup)) { - TSKEY start = getNextWindowTs(pFillSup->prev.key, &pFillSup->interval); - buildDeleteRange(pOperator, start, endTs, groupId, delRes); - } else if (hasNextWindow(pFillSup)) { - TSKEY end = getPrevWindowTs(pFillSup->next.key, &pFillSup->interval); - buildDeleteRange(pOperator, startTs, end, groupId, delRes); - } else { - buildDeleteRange(pOperator, startTs, endTs, groupId, delRes); - } -} - -static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - SStreamFillOperatorInfo* pInfo = pOperator->info; - getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup); - setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo); - SWinKey key = {.ts = startTs, .groupId = groupId}; - if (!pInfo->pFillInfo->needFill) { - pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); - buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes); - } else { - STimeRange tw = { - .skey = startTs, - .ekey = endTs, - .groupId = groupId, - }; - taosArrayPush(pInfo->pFillInfo->delRanges, &tw); - while (key.ts <= endTs) { - key.ts = taosTimeAdd(key.ts, pInfo->pFillSup->interval.sliding, pInfo->pFillSup->interval.slidingUnit, - pInfo->pFillSup->interval.precision); - tSimpleHashPut(pInfo->pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); - } - } -} - -static void doDeleteFillFinalize(SOperatorInfo* pOperator) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - - SStreamFillOperatorInfo* pInfo = pOperator->info; - SStreamFillInfo* pFillInfo = pInfo->pFillInfo; - int32_t size = taosArrayGetSize(pFillInfo->delRanges); - tSimpleHashClear(pInfo->pFillSup->pResMap); - for (; pFillInfo->delIndex < size; pFillInfo->delIndex++) { - STimeRange* range = taosArrayGet(pFillInfo->delRanges, pFillInfo->delIndex); - if (pInfo->pRes->info.id.groupId != 0 && pInfo->pRes->info.id.groupId != range->groupId) { - return; - } - getWindowFromDiscBuf(pOperator, range->skey, range->groupId, pInfo->pFillSup); - setDeleteFillValueInfo(range->skey, range->ekey, pInfo->pFillSup, pInfo->pFillInfo); - if (pInfo->pFillInfo->needFill) { - doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes); - pInfo->pRes->info.id.groupId = range->groupId; - } - SWinKey key = {.ts = range->skey, .groupId = range->groupId}; - pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); - } -} - -static void doDeleteFillResult(SOperatorInfo* pOperator) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - - SStreamFillOperatorInfo* pInfo = pOperator->info; - SStreamFillInfo* pFillInfo = pInfo->pFillInfo; - SSDataBlock* pBlock = pInfo->pSrcDelBlock; - - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - TSKEY* tsStarts = (TSKEY*)pStartCol->pData; - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - uint64_t* groupIds = (uint64_t*)pGroupCol->pData; - while (pInfo->srcDelRowIndex < pBlock->info.rows) { - TSKEY ts = tsStarts[pInfo->srcDelRowIndex]; - TSKEY endTs = ts; - uint64_t groupId = groupIds[pInfo->srcDelRowIndex]; - SWinKey key = {.ts = ts, .groupId = groupId}; - SStreamStateCur* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); - - if (!pCur) { - pInfo->srcDelRowIndex++; - continue; - } - - SWinKey nextKey = {.groupId = groupId, .ts = ts}; - while (pInfo->srcDelRowIndex < pBlock->info.rows) { - TSKEY delTs = tsStarts[pInfo->srcDelRowIndex]; - uint64_t delGroupId = groupIds[pInfo->srcDelRowIndex]; - int32_t code = TSDB_CODE_SUCCESS; - if (groupId != delGroupId) { - break; - } - if (delTs > nextKey.ts) { - break; - } - - SWinKey delKey = {.groupId = delGroupId, .ts = delTs}; - if (delTs == nextKey.ts) { - code = pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); - if (code == TSDB_CODE_SUCCESS) { - code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, NULL, NULL); - } - // ts will be deleted later - if (delTs != ts) { - pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); - pAPI->stateStore.streamStateFreeCur(pCur); - pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); - } - endTs = TMAX(delTs, nextKey.ts - 1); - if (code != TSDB_CODE_SUCCESS) { - break; - } - } - pInfo->srcDelRowIndex++; - } - - pAPI->stateStore.streamStateFreeCur(pCur); - doDeleteFillResultImpl(pOperator, ts, endTs, groupId); - } - - pFillInfo->current = pFillInfo->end + 1; -} - -static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) { - tSimpleHashClear(pInfo->pFillSup->pResMap); - pInfo->pFillSup->hasDelete = false; - taosArrayClear(pInfo->pFillInfo->delRanges); - pInfo->pFillInfo->delIndex = 0; -} - -static void doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock, SSDataBlock* pDstBlock) { - SStreamFillOperatorInfo* pInfo = pOperator->info; - SExprSupp* pSup = &pOperator->exprSupp; - - blockDataCleanup(pDstBlock); - blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows); - setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); - projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); - - pDstBlock->info.rows = 0; - pSup = &pInfo->pFillSup->notFillExprSup; - setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); - projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); - pDstBlock->info.id.groupId = pSrcBlock->info.id.groupId; - - blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol); -} - -static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { - SStreamFillOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - blockDataCleanup(pInfo->pRes); - if (hasRemainCalc(pInfo->pFillInfo) || - (pInfo->pFillInfo->pos != FILL_POS_INVALID && pInfo->pFillInfo->needFill == true)) { - doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes); - if (pInfo->pRes->info.rows > 0) { - printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); - return pInfo->pRes; - } - } - if (pOperator->status == OP_RES_TO_RETURN) { - doDeleteFillFinalize(pOperator); - if (pInfo->pRes->info.rows > 0) { - printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); - return pInfo->pRes; - } - setOperatorCompleted(pOperator); - resetStreamFillInfo(pInfo); - return NULL; - } - - SSDataBlock* fillResult = NULL; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - while (1) { - if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows || pInfo->pSrcBlock->info.rows == 0) { - // If there are delete datablocks, we receive them first. - SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0); - if (pBlock == NULL) { - pOperator->status = OP_RES_TO_RETURN; - pInfo->pFillInfo->preRowKey = INT64_MIN; - if (pInfo->pRes->info.rows > 0) { - printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); - return pInfo->pRes; - } - break; - } - printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo)); - - if (pInfo->pFillInfo->curGroupId != pBlock->info.id.groupId) { - pInfo->pFillInfo->curGroupId = pBlock->info.id.groupId; - pInfo->pFillInfo->preRowKey = INT64_MIN; - } - - pInfo->pFillSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; - if (pInfo->pFillSup->winRange.ekey <= 0) { - pInfo->pFillSup->winRange.ekey = INT64_MAX; - } - - switch (pBlock->info.type) { - case STREAM_RETRIEVE: - return pBlock; - case STREAM_DELETE_RESULT: { - pInfo->pSrcDelBlock = pBlock; - pInfo->srcDelRowIndex = 0; - blockDataCleanup(pInfo->pDelRes); - pInfo->pFillSup->hasDelete = true; - doDeleteFillResult(pOperator); - if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); - return pInfo->pDelRes; - } - continue; - } break; - case STREAM_NORMAL: - case STREAM_INVALID: - case STREAM_PULL_DATA: { - doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock); - memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); - pInfo->srcRowIndex = -1; - } break; - case STREAM_CHECKPOINT: - case STREAM_CREATE_CHILD_TABLE: { - return pBlock; - } break; - default: - ASSERTS(false, "invalid SSDataBlock type"); - } - } - - doStreamFillImpl(pOperator); - doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo); - memcpy(pInfo->pRes->info.parTbName, pInfo->pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN); - pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; - if (pInfo->pRes->info.rows > 0) { - break; - } - } - if (pOperator->status == OP_RES_TO_RETURN) { - doDeleteFillFinalize(pOperator); - } - - if (pInfo->pRes->info.rows == 0) { - setOperatorCompleted(pOperator); - resetStreamFillInfo(pInfo); - return NULL; - } - - pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; - printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); - return pInfo->pRes; -} - -static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { - pFillSup->rowSize = sizeof(SResultCellData) * pFillSup->numOfAllCols; - for (int i = 0; i < pFillSup->numOfAllCols; i++) { - SFillColInfo* pCol = &pFillSup->pAllColInfo[i]; - SResSchema* pSchema = &pCol->pExpr->base.resSchema; - pFillSup->rowSize += pSchema->bytes; - } - pFillSup->next.key = INT64_MIN; - pFillSup->nextNext.key = INT64_MIN; - pFillSup->prev.key = INT64_MIN; - pFillSup->cur.key = INT64_MIN; - pFillSup->next.pRowVal = NULL; - pFillSup->nextNext.pRowVal = NULL; - pFillSup->prev.pRowVal = NULL; - pFillSup->cur.pRowVal = NULL; - - return TSDB_CODE_SUCCESS; -} - -static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, - SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) { - SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); - if (!pFillSup) { - return NULL; - } - pFillSup->numOfFillCols = numOfFillCols; - int32_t numOfNotFillCols = 0; - SExprInfo* noFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); - pFillSup->pAllColInfo = createFillColInfo(pFillExprInfo, pFillSup->numOfFillCols, noFillExprInfo, numOfNotFillCols, - (const SNodeListNode*)(pPhyFillNode->pValues)); - pFillSup->type = convertFillType(pPhyFillNode->mode); - pFillSup->numOfAllCols = pFillSup->numOfFillCols + numOfNotFillCols; - pFillSup->interval = *pInterval; - pFillSup->pAPI = pAPI; - - int32_t code = initResultBuf(pFillSup); - if (code != TSDB_CODE_SUCCESS) { - destroyStreamFillSupporter(pFillSup); - return NULL; - } - - SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); - code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore); - if (code != TSDB_CODE_SUCCESS) { - destroyStreamFillSupporter(pFillSup); - return NULL; - } - - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pFillSup->pResMap = tSimpleHashInit(16, hashFn); - pFillSup->hasDelete = false; - return pFillSup; -} - -SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { - SStreamFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SStreamFillInfo)); - pFillInfo->start = INT64_MIN; - pFillInfo->current = INT64_MIN; - pFillInfo->end = INT64_MIN; - pFillInfo->preRowKey = INT64_MIN; - pFillInfo->needFill = false; - pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo)); - pFillInfo->pLinearInfo->hasNext = false; - pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - pFillInfo->pLinearInfo->pEndPoints = NULL; - pFillInfo->pLinearInfo->pNextEndPoints = NULL; - if (pFillSup->type == TSDB_FILL_LINEAR) { - pFillInfo->pLinearInfo->pEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); - pFillInfo->pLinearInfo->pNextEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); - for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) { - SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); - SPoint value = {0}; - value.val = taosMemoryCalloc(1, pColData->info.bytes); - taosArrayPush(pFillInfo->pLinearInfo->pEndPoints, &value); - - value.val = taosMemoryCalloc(1, pColData->info.bytes); - taosArrayPush(pFillInfo->pLinearInfo->pNextEndPoints, &value); - } - } - pFillInfo->pLinearInfo->winIndex = 0; - - pFillInfo->pResRow = NULL; - if (pFillSup->type == TSDB_FILL_SET_VALUE || pFillSup->type == TSDB_FILL_SET_VALUE_F || - pFillSup->type == TSDB_FILL_NULL || pFillSup->type == TSDB_FILL_NULL_F) { - pFillInfo->pResRow = taosMemoryCalloc(1, sizeof(SResultRowData)); - pFillInfo->pResRow->key = INT64_MIN; - pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize); - for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { - SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); - SResultCellData* pCell = getResultCell(pFillInfo->pResRow, i); - pCell->bytes = pColData->info.bytes; - pCell->type = pColData->info.type; - } - } - - pFillInfo->type = pFillSup->type; - pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeRange)); - pFillInfo->delIndex = 0; - pFillInfo->curGroupId = 0; - return pFillInfo; -} - -SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, - SExecTaskInfo* pTaskInfo) { - SStreamFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFillOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval; - int32_t numOfFillCols = 0; - SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); - pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI); - if (!pInfo->pFillSup) { - goto _error; - } - - initResultSizeInfo(&pOperator->resultInfo, 4096); - pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); - pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); - blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity); - - pInfo->pFillInfo = initStreamFillInfo(pInfo->pFillSup, pInfo->pRes); - if (!pInfo->pFillInfo) { - goto _error; - } - - if (pInfo->pFillInfo->type == TSDB_FILL_SET_VALUE || pInfo->pFillInfo->type == TSDB_FILL_SET_VALUE_F) { - for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) { - SFillColInfo* pFillCol = pInfo->pFillSup->pAllColInfo + i; - int32_t slotId = GET_DEST_SLOT_ID(pFillCol); - SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId); - SVariant* pVar = &(pFillCol->fillVal); - if (pCell->type == TSDB_DATA_TYPE_FLOAT) { - float v = 0; - GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); - SET_TYPED_DATA(pCell->pData, pCell->type, v); - } else if (IS_FLOAT_TYPE(pCell->type)) { - double v = 0; - GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); - SET_TYPED_DATA(pCell->pData, pCell->type, v); - } else if (IS_INTEGER_TYPE(pCell->type)) { - int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); - SET_TYPED_DATA(pCell->pData, pCell->type, v); - } else { - pCell->isNull = true; - } - } - } else if (pInfo->pFillInfo->type == TSDB_FILL_NULL || pInfo->pFillInfo->type == TSDB_FILL_NULL_F) { - for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) { - SFillColInfo* pFillCol = pInfo->pFillSup->pAllColInfo + i; - int32_t slotId = GET_DEST_SLOT_ID(pFillCol); - SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId); - pCell->isNull = true; - } - } - - pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); - blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity); - - pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId; - pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId; - - int32_t numOfOutputCols = 0; - int32_t code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, - &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pInfo->srcRowIndex = -1; - setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, - pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); - setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); - - code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - return pOperator; - -_error: - destroyStreamFillOperatorInfo(pInfo); - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c index 093b6ab11e..c1a51898bc 100755 --- a/source/libs/executor/src/mergeoperator.c +++ b/source/libs/executor/src/mergeoperator.c @@ -269,6 +269,7 @@ SSDataBlock* doNonSortMerge(SOperatorInfo* pOperator) { SMultiwayMergeOperatorInfo* pInfo = pOperator->info; SNonSortMergeInfo* pNonSortMerge = &pInfo->nsortMergeInfo; SSDataBlock* pBlock = NULL; + SSDataBlock* pRes = pInfo->binfo.pRes; qDebug("start to merge no sorted rows, %s", GET_TASKID(pTaskInfo)); @@ -278,13 +279,18 @@ SSDataBlock* doNonSortMerge(SOperatorInfo* pOperator) { if (NULL == pBlock) { TSWAP(pNonSortMerge->pSourceStatus[pNonSortMerge->sourceWorkIdx], pNonSortMerge->pSourceStatus[idx]); pNonSortMerge->sourceWorkIdx++; - idx = NON_SORT_NEXT_SRC(pNonSortMerge, idx); + idx = NON_SORT_NEXT_SRC(pNonSortMerge, pNonSortMerge->lastSourceIdx); continue; } break; } - return pBlock; + if (!pBlock) { + return NULL; + } + copyDataBlock(pRes, pBlock); + + return pRes; } void destroyNonSortMergeOperatorInfo(void* param) { @@ -491,6 +497,9 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size } case MERGE_TYPE_NON_SORT: { SNonSortMergeInfo* pNonSortMerge = &pInfo->nsortMergeInfo; + pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode); + initResultSizeInfo(&pOperator->resultInfo, 1024); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); break; } case MERGE_TYPE_COLUMNS: { diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 691210e5ba..c6ebb04446 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -28,6 +28,7 @@ typedef struct SProjectOperatorInfo { bool mergeDataBlocks; SSDataBlock* pFinalRes; bool inputIgnoreGroup; + bool outputIgnoreGroup; } SProjectOperatorInfo; typedef struct SIndefOperatorInfo { @@ -111,6 +112,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys pInfo->binfo.inputTsOrder = pProjPhyNode->node.inputTsOrder; pInfo->binfo.outputTsOrder = pProjPhyNode->node.outputTsOrder; pInfo->inputIgnoreGroup = pProjPhyNode->inputIgnoreGroup; + pInfo->outputIgnoreGroup = pProjPhyNode->ignoreGroupId; if (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM || pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { pInfo->mergeDataBlocks = false; @@ -276,6 +278,10 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } + if (pProjectInfo->outputIgnoreGroup) { + pRes->info.id.groupId = 0; + } + return (pRes->info.rows > 0) ? pRes : NULL; } @@ -385,6 +391,10 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { printDataBlock(p, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); } + if (pProjectInfo->outputIgnoreGroup) { + p->info.id.groupId = 0; + } + return (p->info.rows > 0) ? p : NULL; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 9024f7a341..51edfcb42c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2269,6 +2269,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__SCAN1) { if (isTaskKilled(pTaskInfo)) { + qInfo("===stream===stream scan is killed. task id:%s, code %s", id, tstrerror(pTaskInfo->code)); return NULL; } @@ -3870,6 +3871,7 @@ static void doGetBlockForTableMergeScan(SOperatorInfo* pOperator, bool* pFinishe uint32_t status = 0; code = loadDataBlock(pOperator, &pInfo->base, pBlock, &status); + if (code != TSDB_CODE_SUCCESS) { qInfo("table merge scan load datablock code %d, %s", code, GET_TASKID(pTaskInfo)); T_LONG_JMP(pTaskInfo->env, code); @@ -3956,7 +3958,9 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { pBlock->info.id.groupId = tableListGetTableGroupId(pInfo->base.pTableListInfo, pBlock->info.id.uid); pOperator->resultInfo.totalRows += pBlock->info.rows; + pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; + return pBlock; } @@ -4008,9 +4012,16 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) { pInfo->sortBufSize = 2048 * pInfo->bufPageSize; int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage, - pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0); + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage, + pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0); + if (pInfo->bSortRowId && numOfTable != 1) { + int32_t memSize = 512 * 1024 * 1024; + code = tsortSetSortByRowId(pInfo->pSortHandle, memSize); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } tsortSetMergeLimit(pInfo->pSortHandle, pInfo->mergeLimit); tsortSetMergeLimitReachedFp(pInfo->pSortHandle, tableMergeScanDoSkipTable, pInfo); tsortSetAbortCheckFn(pInfo->pSortHandle, isTaskKilled, pOperator->pTaskInfo); @@ -4047,6 +4058,7 @@ void stopDurationForGroupTableMergeScan(SOperatorInfo* pOperator) { tsortDestroySortHandle(pInfo->pSortHandle); pInfo->pSortHandle = NULL; + } int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { @@ -4131,8 +4143,7 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* if (pTupleHandle == NULL) { break; } - - appendOneRowToDataBlock(pResBlock, pTupleHandle); + tsortAppendTupleToBlock(pInfo->pSortHandle, pResBlock, pTupleHandle); if (pResBlock->info.rows >= capacity) { break; } @@ -4199,7 +4210,10 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { } else { if (pInfo->bNewFilesetEvent) { stopDurationForGroupTableMergeScan(pOperator); - startDurationForGroupTableMergeScan(pOperator); + code = startDurationForGroupTableMergeScan(pOperator); + if (code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, terrno); + } } else { // Data of this group are all dumped, let's try the next group stopGroupTableMergeScan(pOperator); @@ -4330,10 +4344,15 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->mergeLimit = pInfo->limitInfo.limit.limit + pInfo->limitInfo.limit.offset; pInfo->mSkipTables = NULL; } - + initResultSizeInfo(&pOperator->resultInfo, 1024); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); + if (!hasLimit && blockDataGetRowSize(pInfo->pResBlock) >= 256 && !pTableScanNode->smallDataTsSort) { + pInfo->bSortRowId = true; + } else { + pInfo->bSortRowId = false; + } pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order); pInfo->pReaderBlock = createOneDataBlock(pInfo->pResBlock, false); @@ -4342,6 +4361,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN int32_t rowSize = pInfo->pResBlock->info.rowSize; uint32_t nCols = taosArrayGetSize(pInfo->pResBlock->pDataBlock); + pInfo->bufPageSize = getProperSortPageSize(rowSize, nCols); //start one reader variable diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index f9c7b51316..720734431f 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -390,10 +390,12 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato buf = taosDecodeFixedI32(buf, &mapSize); for (int32_t i = 0; i < mapSize; i++) { SSessionKey key = {0}; - SResultWindowInfo winfo = {0}; + SCountWindowInfo curWin = {0}; buf = decodeSSessionKey(buf, &key); - buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL}; + setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo); + buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize); + tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); } // 2.twAggSup @@ -694,6 +696,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->recvGetAll = false; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT; + setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, true, + OP_NOT_OPENED, pInfo, pTaskInfo); // for stream void* buff = NULL; int32_t len = 0; @@ -704,8 +708,6 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys doStreamCountDecodeOpState(buff, len, pOperator, true); taosMemoryFree(buff); } - setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, true, - OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamCountAgg, NULL, destroyStreamCountAggOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState); diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 4a2fe2416f..11d8d1487a 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -406,6 +406,7 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato if (!pInfo) { return buf; } + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; // 4.checksum int32_t dataLen = len - sizeof(uint32_t); @@ -423,6 +424,8 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); + pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &winfo.sessionWin, pAggSup->gap, + (void**)&winfo.pStatePos, &pAggSup->resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); } @@ -735,6 +738,8 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->reCkBlock = false; pInfo->recvGetAll = false; + setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED, + pInfo, pTaskInfo); // for stream void* buff = NULL; int32_t len = 0; @@ -746,8 +751,6 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys taosMemoryFree(buff); } - setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED, - pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState); diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c new file mode 100644 index 0000000000..b54802e2c6 --- /dev/null +++ b/source/libs/executor/src/streamfilloperator.c @@ -0,0 +1,1181 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "filter.h" +#include "os.h" +#include "query.h" +#include "taosdef.h" +#include "tmsg.h" +#include "ttypes.h" + +#include "executorInt.h" +#include "tcommon.h" +#include "thash.h" +#include "ttime.h" + +#include "function.h" +#include "querynodes.h" +#include "tdatablock.h" +#include "tfill.h" +#include "operator.h" +#include "querytask.h" + + +#define FILL_POS_INVALID 0 +#define FILL_POS_START 1 +#define FILL_POS_MID 2 +#define FILL_POS_END 3 + +typedef struct STimeRange { + TSKEY skey; + TSKEY ekey; + uint64_t groupId; +} STimeRange; + +TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) { + STimeWindow win = {.skey = ts, .ekey = ts}; + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); + return win.skey; +} + +TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { + STimeWindow win = {.skey = ts, .ekey = ts}; + getNextTimeWindow(pInterval, &win, TSDB_ORDER_DESC); + return win.skey; +} + +void setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) { + colDataSetVal(pCol, rowId, pCell->pData, pCell->isNull); +} + +SResultCellData* getResultCell(SResultRowData* pRaw, int32_t index) { + if (!pRaw || !pRaw->pRowVal) { + return NULL; + } + char* pData = (char*)pRaw->pRowVal; + SResultCellData* pCell = pRaw->pRowVal; + for (int32_t i = 0; i < index; i++) { + pData += (pCell->bytes + sizeof(SResultCellData)); + pCell = (SResultCellData*)pData; + } + return pCell; +} + +void* destroyFillColumnInfo(SFillColInfo* pFillCol, int32_t start, int32_t end) { + for (int32_t i = start; i < end; i++) { + destroyExprInfo(pFillCol[i].pExpr, 1); + taosVariantDestroy(&pFillCol[i].fillVal); + } + taosMemoryFreeClear(pFillCol[start].pExpr); + taosMemoryFree(pFillCol); + return NULL; +} + +void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { + pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); + tSimpleHashCleanup(pFillSup->pResMap); + pFillSup->pResMap = NULL; + cleanupExprSupp(&pFillSup->notFillExprSup); + if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) { + taosMemoryFree(pFillSup->cur.pRowVal); + } + taosMemoryFree(pFillSup->prev.pRowVal); + taosMemoryFree(pFillSup->next.pRowVal); + taosMemoryFree(pFillSup->nextNext.pRowVal); + + taosMemoryFree(pFillSup); + return NULL; +} + +void destroySPoint(void* ptr) { + SPoint* point = (SPoint*) ptr; + taosMemoryFreeClear(point->val); +} + +void* destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) { + taosArrayDestroyEx(pFillLinear->pEndPoints, destroySPoint); + taosArrayDestroyEx(pFillLinear->pNextEndPoints, destroySPoint); + taosMemoryFree(pFillLinear); + return NULL; +} +void* destroyStreamFillInfo(SStreamFillInfo* pFillInfo) { + if (pFillInfo->type == TSDB_FILL_SET_VALUE || pFillInfo->type == TSDB_FILL_SET_VALUE_F || + pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) { + taosMemoryFreeClear(pFillInfo->pResRow->pRowVal); + taosMemoryFreeClear(pFillInfo->pResRow); + } + pFillInfo->pLinearInfo = destroyStreamFillLinearInfo(pFillInfo->pLinearInfo); + taosArrayDestroy(pFillInfo->delRanges); + taosMemoryFree(pFillInfo); + return NULL; +} + +static void destroyStreamFillOperatorInfo(void* param) { + SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param; + pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo); + pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup); + pInfo->pRes = blockDataDestroy(pInfo->pRes); + pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + pInfo->matchInfo.pList = taosArrayDestroy(pInfo->matchInfo.pList); + taosMemoryFree(pInfo); +} + +static void resetFillWindow(SResultRowData* pRowData) { + pRowData->key = INT64_MIN; + taosMemoryFreeClear(pRowData->pRowVal); +} + +void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStorageAPI* pAPI) { + if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) { + resetFillWindow(&pFillSup->cur); + } else { + pFillSup->cur.key = INT64_MIN; + pFillSup->cur.pRowVal = NULL; + } + resetFillWindow(&pFillSup->prev); + resetFillWindow(&pFillSup->next); + resetFillWindow(&pFillSup->nextNext); +} + +void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + + void* pState = pOperator->pTaskInfo->streamInfo.pState; + resetPrevAndNextWindow(pFillSup, pState, pAPI); + + SWinKey key = {.ts = ts, .groupId = groupId}; + int32_t curVLen = 0; + + int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); + ASSERT(code == TSDB_CODE_SUCCESS); + pFillSup->cur.key = key.ts; +} + +void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + void* pState = pOperator->pTaskInfo->streamInfo.pState; + resetPrevAndNextWindow(pFillSup, pState, pAPI); + + SWinKey key = {.ts = ts, .groupId = groupId}; + void* curVal = NULL; + int32_t curVLen = 0; + int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); + ASSERT(code == TSDB_CODE_SUCCESS); + pFillSup->cur.key = key.ts; + pFillSup->cur.pRowVal = curVal; + + SStreamStateCur* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key); + SWinKey preKey = {.ts = INT64_MIN, .groupId = groupId}; + void* preVal = NULL; + int32_t preVLen = 0; + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); + + if (code == TSDB_CODE_SUCCESS) { + pFillSup->prev.key = preKey.ts; + pFillSup->prev.pRowVal = preVal; + + code = pAPI->stateStore.streamStateCurNext(pState, pCur); + ASSERT(code == TSDB_CODE_SUCCESS); + + code = pAPI->stateStore.streamStateCurNext(pState, pCur); + if (code != TSDB_CODE_SUCCESS) { + pAPI->stateStore.streamStateFreeCur(pCur); + pCur = NULL; + } + } else { + pAPI->stateStore.streamStateFreeCur(pCur); + pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key); + } + + SWinKey nextKey = {.ts = INT64_MIN, .groupId = groupId}; + void* nextVal = NULL; + int32_t nextVLen = 0; + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); + if (code == TSDB_CODE_SUCCESS) { + pFillSup->next.key = nextKey.ts; + pFillSup->next.pRowVal = nextVal; + if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { + code = pAPI->stateStore.streamStateCurNext(pState, pCur); + if (code == TSDB_CODE_SUCCESS) { + SWinKey nextNextKey = {.groupId = groupId}; + void* nextNextVal = NULL; + int32_t nextNextVLen = 0; + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); + if (code == TSDB_CODE_SUCCESS) { + pFillSup->nextNext.key = nextNextKey.ts; + pFillSup->nextNext.pRowVal = nextNextVal; + } + } + } + } + pAPI->stateStore.streamStateFreeCur(pCur); +} + +static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; } +static bool hasNextWindow(SStreamFillSupporter* pFillSup) { return pFillSup->next.key != INT64_MIN; } +static bool hasNextNextWindow(SStreamFillSupporter* pFillSup) { + return pFillSup->nextNext.key != INT64_MIN; + return false; +} + +static void transBlockToResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SResultRowData* pRowVal) { + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); + SResultCellData* pCell = getResultCell(pRowVal, i); + if (!colDataIsNull_s(pColData, rowId)) { + pCell->isNull = false; + pCell->type = pColData->info.type; + pCell->bytes = pColData->info.bytes; + char* val = colDataGetData(pColData, rowId); + if (IS_VAR_DATA_TYPE(pCell->type)) { + memcpy(pCell->pData, val, varDataTLen(val)); + } else { + memcpy(pCell->pData, val, pCell->bytes); + } + } else { + pCell->isNull = true; + } + } + pRowVal->key = ts; +} + +static void calcDeltaData(SSDataBlock* pBlock, int32_t rowId, SResultRowData* pRowVal, SArray* pDelta, + SFillColInfo* pFillCol, int32_t numOfCol, int32_t winCount, int32_t order) { + for (int32_t i = 0; i < numOfCol; i++) { + if (!pFillCol[i].notFillCol) { + int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + char* var = colDataGetData(pCol, rowId); + double start = 0; + GET_TYPED_DATA(start, double, pCol->info.type, var); + SResultCellData* pCell = getResultCell(pRowVal, slotId); + double end = 0; + GET_TYPED_DATA(end, double, pCell->type, pCell->pData); + double delta = 0; + if (order == TSDB_ORDER_ASC) { + delta = (end - start) / winCount; + } else { + delta = (start - end) / winCount; + } + taosArraySet(pDelta, slotId, &delta); + } + } +} + +static void calcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, + int32_t numOfCol) { + for (int32_t i = 0; i < numOfCol; i++) { + if (!pFillCol[i].notFillCol) { + int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); + SResultCellData* pECell = getResultCell(pEndRow, slotId); + SPoint* pPoint = taosArrayGet(pEndPoins, slotId); + pPoint->key = pEndRow->key; + memcpy(pPoint->val, pECell->pData, pECell->bytes); + } + } +} + +static void setFillInfoStart(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { + ts = taosTimeAdd(ts, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + pFillInfo->start = ts; +} + +static void setFillInfoEnd(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { + ts = taosTimeAdd(ts, pInterval->sliding * -1, pInterval->slidingUnit, pInterval->precision); + pFillInfo->end = ts; +} + +static void setFillKeyInfo(TSKEY start, TSKEY end, SInterval* pInterval, SStreamFillInfo* pFillInfo) { + setFillInfoStart(start, pInterval, pFillInfo); + pFillInfo->current = pFillInfo->start; + setFillInfoEnd(end, pInterval, pFillInfo); +} + +void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) { + if (!hasPrevWindow(pFillSup) || !hasNextWindow(pFillSup)) { + pFillInfo->needFill = false; + return; + } + + TSKEY realStart = taosTimeAdd(pFillSup->prev.key, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + + pFillInfo->needFill = true; + pFillInfo->start = realStart; + pFillInfo->current = pFillInfo->start; + pFillInfo->end = end; + pFillInfo->pos = FILL_POS_INVALID; + switch (pFillInfo->type) { + case TSDB_FILL_NULL: + case TSDB_FILL_NULL_F: + case TSDB_FILL_SET_VALUE: + case TSDB_FILL_SET_VALUE_F: + break; + case TSDB_FILL_PREV: + pFillInfo->pResRow = &pFillSup->prev; + break; + case TSDB_FILL_NEXT: + pFillInfo->pResRow = &pFillSup->next; + break; + case TSDB_FILL_LINEAR: { + setFillKeyInfo(pFillSup->prev.key, pFillSup->next.key, &pFillSup->interval, pFillInfo); + pFillInfo->pLinearInfo->hasNext = false; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, + pFillSup->numOfAllCols); + pFillInfo->pResRow = &pFillSup->prev; + pFillInfo->pLinearInfo->winIndex = 0; + } break; + default: + ASSERT(0); + break; + } +} + +void copyNotFillExpData(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) { + for (int32_t i = pFillSup->numOfFillCols; i < pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId); + SResultCellData* pCurCell = getResultCell(&pFillSup->cur, slotId); + pCell->isNull = pCurCell->isNull; + if (!pCurCell->isNull) { + memcpy(pCell->pData, pCurCell->pData, pCell->bytes); + } + } +} + +void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillSupporter* pFillSup, + SStreamFillInfo* pFillInfo) { + pFillInfo->preRowKey = pFillSup->cur.key; + if (!hasPrevWindow(pFillSup) && !hasNextWindow(pFillSup)) { + pFillInfo->needFill = false; + pFillInfo->pos = FILL_POS_START; + return; + } + TSKEY prevWKey = INT64_MIN; + TSKEY nextWKey = INT64_MIN; + if (hasPrevWindow(pFillSup)) { + prevWKey = pFillSup->prev.key; + } + if (hasNextWindow(pFillSup)) { + nextWKey = pFillSup->next.key; + } + + pFillInfo->needFill = true; + pFillInfo->pos = FILL_POS_INVALID; + switch (pFillInfo->type) { + case TSDB_FILL_NULL: + case TSDB_FILL_NULL_F: + case TSDB_FILL_SET_VALUE: + case TSDB_FILL_SET_VALUE_F: { + if (pFillSup->prev.key == pFillInfo->preRowKey) { + resetFillWindow(&pFillSup->prev); + } + if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) { + if (pFillSup->next.key == pFillInfo->nextRowKey) { + pFillInfo->preRowKey = INT64_MIN; + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + } else { + pFillInfo->needFill = false; + pFillInfo->pos = FILL_POS_START; + } + } else if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + } else { + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + } + copyNotFillExpData(pFillSup, pFillInfo); + } break; + case TSDB_FILL_PREV: { + if (hasNextWindow(pFillSup) && ((pFillSup->next.key != pFillInfo->nextRowKey) || + (pFillSup->next.key == pFillInfo->nextRowKey && hasNextNextWindow(pFillSup)) || + (pFillSup->next.key == pFillInfo->nextRowKey && !hasPrevWindow(pFillSup)))) { + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + resetFillWindow(&pFillSup->prev); + pFillSup->prev.key = pFillSup->cur.key; + pFillSup->prev.pRowVal = pFillSup->cur.pRowVal; + } else if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + pFillInfo->preRowKey = INT64_MIN; + } + pFillInfo->pResRow = &pFillSup->prev; + } break; + case TSDB_FILL_NEXT: { + if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + resetFillWindow(&pFillSup->next); + pFillSup->next.key = pFillSup->cur.key; + pFillSup->next.pRowVal = pFillSup->cur.pRowVal; + pFillInfo->preRowKey = INT64_MIN; + } else { + ASSERT(hasNextWindow(pFillSup)); + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + } + pFillInfo->pResRow = &pFillSup->next; + } break; + case TSDB_FILL_LINEAR: { + pFillInfo->pLinearInfo->winIndex = 0; + if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_MID; + pFillInfo->pLinearInfo->nextEnd = nextWKey; + calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, + pFillSup->numOfAllCols); + pFillInfo->pResRow = &pFillSup->prev; + + calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pNextEndPoints, pFillSup->pAllColInfo, + pFillSup->numOfAllCols); + pFillInfo->pLinearInfo->hasNext = true; + } else if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, + pFillSup->numOfAllCols); + pFillInfo->pResRow = &pFillSup->prev; + pFillInfo->pLinearInfo->hasNext = false; + } else { + ASSERT(hasNextWindow(pFillSup)); + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, + pFillSup->numOfAllCols); + pFillInfo->pResRow = &pFillSup->cur; + pFillInfo->pLinearInfo->hasNext = false; + } + } break; + default: + ASSERT(0); + break; + } + ASSERT(pFillInfo->pos != FILL_POS_INVALID); +} + +static bool checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t groupId) { + SWinKey key = {.groupId = groupId, .ts = ts}; + if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) { + return false; + } + tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); + return true; +} + +static bool buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFillSup, TSKEY ts, SSDataBlock* pBlock) { + if (pBlock->info.rows >= pBlock->info.capacity) { + return false; + } + uint64_t groupId = pBlock->info.id.groupId; + if (pFillSup->hasDelete && !checkResult(pFillSup, ts, groupId)) { + return true; + } + for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId); + SFillInfo tmpInfo = { + .currentKey = ts, + .order = TSDB_ORDER_ASC, + .interval = pFillSup->interval, + }; + bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pColData, pBlock->info.rows); + if (!filled) { + SResultCellData* pCell = getResultCell(pResRow, slotId); + setRowCell(pColData, pBlock->info.rows, pCell); + } + } + pBlock->info.rows++; + return true; +} + +static bool hasRemainCalc(SStreamFillInfo* pFillInfo) { + if (pFillInfo->current != INT64_MIN && pFillInfo->current <= pFillInfo->end) { + return true; + } + return false; +} + +static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { + while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { + STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; + if (inWinRange(&pFillSup->winRange, &st)) { + buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); + } + pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + } +} + +static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { + while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { + uint64_t groupId = pBlock->info.id.groupId; + SWinKey key = {.groupId = groupId, .ts = pFillInfo->current}; + STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; + if ( ( pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId) ) || !inWinRange(&pFillSup->winRange, &st) ) { + pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + pFillInfo->pLinearInfo->winIndex++; + continue; + } + pFillInfo->pLinearInfo->winIndex++; + for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; + SFillInfo tmp = { + .currentKey = pFillInfo->current, + .order = TSDB_ORDER_ASC, + .interval = pFillSup->interval, + }; + + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId); + int16_t type = pColData->info.type; + SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId); + int32_t index = pBlock->info.rows; + if (pFillCol->notFillCol) { + bool filled = fillIfWindowPseudoColumn(&tmp, pFillCol, pColData, index); + if (!filled) { + setRowCell(pColData, index, pCell); + } + } else { + if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) { + colDataSetNULL(pColData, index); + continue; + } + SPoint* pEnd = taosArrayGet(pFillInfo->pLinearInfo->pEndPoints, slotId); + double vCell = 0; + SPoint start = {0}; + start.key = pFillInfo->pResRow->key; + start.val = pCell->pData; + + SPoint cur = {0}; + cur.key = pFillInfo->current; + cur.val = taosMemoryCalloc(1, pCell->bytes); + taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type); + colDataSetVal(pColData, index, (const char*)cur.val, false); + destroySPoint(&cur); + } + } + pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + pBlock->info.rows++; + } +} + +static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + + SWinKey key = {.groupId = groupId, .ts = pRow->key}; + int32_t code = pAPI->stateStore.streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len); + qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 " code:%d", key.ts, key.groupId, code); + ASSERT(code == TSDB_CODE_SUCCESS); +} + +static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { + if (pFillInfo->needFill == false) { + buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); + return; + } + + if (pFillInfo->pos == FILL_POS_START) { + if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { + pFillInfo->pos = FILL_POS_INVALID; + } + } + if (pFillInfo->type != TSDB_FILL_LINEAR) { + doStreamFillNormal(pFillSup, pFillInfo, pRes); + } else { + doStreamFillLinear(pFillSup, pFillInfo, pRes); + + if (pFillInfo->pos == FILL_POS_MID) { + if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { + pFillInfo->pos = FILL_POS_INVALID; + } + } + + if (pFillInfo->current > pFillInfo->end && pFillInfo->pLinearInfo->hasNext) { + pFillInfo->pLinearInfo->hasNext = false; + pFillInfo->pLinearInfo->winIndex = 0; + taosArraySwap(pFillInfo->pLinearInfo->pEndPoints, pFillInfo->pLinearInfo->pNextEndPoints); + pFillInfo->pResRow = &pFillSup->cur; + setFillKeyInfo(pFillSup->cur.key, pFillInfo->pLinearInfo->nextEnd, &pFillSup->interval, pFillInfo); + doStreamFillLinear(pFillSup, pFillInfo, pRes); + } + } + if (pFillInfo->pos == FILL_POS_END) { + if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { + pFillInfo->pos = FILL_POS_INVALID; + } + } +} + +void keepBlockRowInDiscBuf(SOperatorInfo* pOperator, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, TSKEY* tsCol, + int32_t rowId, uint64_t groupId, int32_t rowSize) { + TSKEY ts = tsCol[rowId]; + pFillInfo->nextRowKey = ts; + SResultRowData tmpNextRow = {.key = ts}; + tmpNextRow.pRowVal = taosMemoryCalloc(1, rowSize); + transBlockToResultRow(pBlock, rowId, ts, &tmpNextRow); + keepResultInDiscBuf(pOperator, groupId, &tmpNextRow, rowSize); + taosMemoryFreeClear(tmpNextRow.pRowVal); +} + +static void doFillResults(SOperatorInfo* pOperator, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, + SSDataBlock* pBlock, TSKEY* tsCol, int32_t rowId, SSDataBlock* pRes) { + uint64_t groupId = pBlock->info.id.groupId; + getWindowFromDiscBuf(pOperator, tsCol[rowId], groupId, pFillSup); + if (pFillSup->prev.key == pFillInfo->preRowKey) { + resetFillWindow(&pFillSup->prev); + } + setFillValueInfo(pBlock, tsCol[rowId], rowId, pFillSup, pFillInfo); + doStreamFillRange(pFillInfo, pFillSup, pRes); +} + +static void doStreamFillImpl(SOperatorInfo* pOperator) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStreamFillSupporter* pFillSup = pInfo->pFillSup; + SStreamFillInfo* pFillInfo = pInfo->pFillInfo; + SSDataBlock* pBlock = pInfo->pSrcBlock; + uint64_t groupId = pBlock->info.id.groupId; + SSDataBlock* pRes = pInfo->pRes; + SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol); + TSKEY* tsCol = (TSKEY*)pTsCol->pData; + pRes->info.id.groupId = groupId; + pInfo->srcRowIndex++; + + if (pInfo->srcRowIndex == 0) { + keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); + pInfo->srcRowIndex++; + } + + while (pInfo->srcRowIndex < pBlock->info.rows) { + TSKEY ts = tsCol[pInfo->srcRowIndex]; + keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); + doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); + if (pInfo->pRes->info.rows == pInfo->pRes->info.capacity) { + blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + return; + } + pInfo->srcRowIndex++; + } + doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); + blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + blockDataCleanup(pInfo->pSrcBlock); +} + +static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { + SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; + void* pState = pOp->pTaskInfo->streamInfo.pState; + + SSDataBlock* pBlock = delRes; + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + SColumnInfoData* pTbNameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); + colDataSetVal(pStartCol, pBlock->info.rows, (const char*)&start, false); + colDataSetVal(pEndCol, pBlock->info.rows, (const char*)&end, false); + colDataSetNULL(pUidCol, pBlock->info.rows); + colDataSetVal(pGroupCol, pBlock->info.rows, (const char*)&groupId, false); + colDataSetNULL(pCalStartCol, pBlock->info.rows); + colDataSetNULL(pCalEndCol, pBlock->info.rows); + + SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); + + void* tbname = NULL; + pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname); + if (tbname == NULL) { + colDataSetNULL(pTableCol, pBlock->info.rows); + } else { + char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; + STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); + colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + pAPI->stateStore.streamStateFreeVal(tbname); + } + + pBlock->info.rows++; +} + +static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId, + SSDataBlock* delRes) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SStreamFillSupporter* pFillSup = pInfo->pFillSup; + if (hasPrevWindow(pFillSup)) { + TSKEY start = getNextWindowTs(pFillSup->prev.key, &pFillSup->interval); + buildDeleteRange(pOperator, start, endTs, groupId, delRes); + } else if (hasNextWindow(pFillSup)) { + TSKEY end = getPrevWindowTs(pFillSup->next.key, &pFillSup->interval); + buildDeleteRange(pOperator, startTs, end, groupId, delRes); + } else { + buildDeleteRange(pOperator, startTs, endTs, groupId, delRes); + } +} + +static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamFillOperatorInfo* pInfo = pOperator->info; + getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup); + setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo); + SWinKey key = {.ts = startTs, .groupId = groupId}; + if (!pInfo->pFillInfo->needFill) { + pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); + buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes); + } else { + STimeRange tw = { + .skey = startTs, + .ekey = endTs, + .groupId = groupId, + }; + taosArrayPush(pInfo->pFillInfo->delRanges, &tw); + while (key.ts <= endTs) { + key.ts = taosTimeAdd(key.ts, pInfo->pFillSup->interval.sliding, pInfo->pFillSup->interval.slidingUnit, + pInfo->pFillSup->interval.precision); + tSimpleHashPut(pInfo->pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); + } + } +} + +static void doDeleteFillFinalize(SOperatorInfo* pOperator) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + + SStreamFillOperatorInfo* pInfo = pOperator->info; + SStreamFillInfo* pFillInfo = pInfo->pFillInfo; + int32_t size = taosArrayGetSize(pFillInfo->delRanges); + tSimpleHashClear(pInfo->pFillSup->pResMap); + for (; pFillInfo->delIndex < size; pFillInfo->delIndex++) { + STimeRange* range = taosArrayGet(pFillInfo->delRanges, pFillInfo->delIndex); + if (pInfo->pRes->info.id.groupId != 0 && pInfo->pRes->info.id.groupId != range->groupId) { + return; + } + getWindowFromDiscBuf(pOperator, range->skey, range->groupId, pInfo->pFillSup); + setDeleteFillValueInfo(range->skey, range->ekey, pInfo->pFillSup, pInfo->pFillInfo); + if (pInfo->pFillInfo->needFill) { + doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes); + pInfo->pRes->info.id.groupId = range->groupId; + } + SWinKey key = {.ts = range->skey, .groupId = range->groupId}; + pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); + } +} + +static void doDeleteFillResult(SOperatorInfo* pOperator) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + + SStreamFillOperatorInfo* pInfo = pOperator->info; + SStreamFillInfo* pFillInfo = pInfo->pFillInfo; + SSDataBlock* pBlock = pInfo->pSrcDelBlock; + + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + TSKEY* tsStarts = (TSKEY*)pStartCol->pData; + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + uint64_t* groupIds = (uint64_t*)pGroupCol->pData; + while (pInfo->srcDelRowIndex < pBlock->info.rows) { + TSKEY ts = tsStarts[pInfo->srcDelRowIndex]; + TSKEY endTs = ts; + uint64_t groupId = groupIds[pInfo->srcDelRowIndex]; + SWinKey key = {.ts = ts, .groupId = groupId}; + SStreamStateCur* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); + + if (!pCur) { + pInfo->srcDelRowIndex++; + continue; + } + + SWinKey nextKey = {.groupId = groupId, .ts = ts}; + while (pInfo->srcDelRowIndex < pBlock->info.rows) { + TSKEY delTs = tsStarts[pInfo->srcDelRowIndex]; + uint64_t delGroupId = groupIds[pInfo->srcDelRowIndex]; + int32_t code = TSDB_CODE_SUCCESS; + if (groupId != delGroupId) { + break; + } + if (delTs > nextKey.ts) { + break; + } + + SWinKey delKey = {.groupId = delGroupId, .ts = delTs}; + if (delTs == nextKey.ts) { + code = pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); + if (code == TSDB_CODE_SUCCESS) { + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, NULL, NULL); + } + // ts will be deleted later + if (delTs != ts) { + pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); + pAPI->stateStore.streamStateFreeCur(pCur); + pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); + } + endTs = TMAX(delTs, nextKey.ts - 1); + if (code != TSDB_CODE_SUCCESS) { + break; + } + } + pInfo->srcDelRowIndex++; + } + + pAPI->stateStore.streamStateFreeCur(pCur); + doDeleteFillResultImpl(pOperator, ts, endTs, groupId); + } + + pFillInfo->current = pFillInfo->end + 1; +} + +static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) { + tSimpleHashClear(pInfo->pFillSup->pResMap); + pInfo->pFillSup->hasDelete = false; + taosArrayClear(pInfo->pFillInfo->delRanges); + pInfo->pFillInfo->delIndex = 0; +} + +static void doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock, SSDataBlock* pDstBlock) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SExprSupp* pSup = &pOperator->exprSupp; + + blockDataCleanup(pDstBlock); + blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows); + setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); + projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + + pDstBlock->info.rows = 0; + pSup = &pInfo->pFillSup->notFillExprSup; + setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); + projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + pDstBlock->info.id.groupId = pSrcBlock->info.id.groupId; + + blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol); +} + +static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + blockDataCleanup(pInfo->pRes); + if (hasRemainCalc(pInfo->pFillInfo) || + (pInfo->pFillInfo->pos != FILL_POS_INVALID && pInfo->pFillInfo->needFill == true)) { + doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes); + if (pInfo->pRes->info.rows > 0) { + printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pInfo->pRes; + } + } + if (pOperator->status == OP_RES_TO_RETURN) { + doDeleteFillFinalize(pOperator); + if (pInfo->pRes->info.rows > 0) { + printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pInfo->pRes; + } + setOperatorCompleted(pOperator); + resetStreamFillInfo(pInfo); + return NULL; + } + + SSDataBlock* fillResult = NULL; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + while (1) { + if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows || pInfo->pSrcBlock->info.rows == 0) { + // If there are delete datablocks, we receive them first. + SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0); + if (pBlock == NULL) { + pOperator->status = OP_RES_TO_RETURN; + pInfo->pFillInfo->preRowKey = INT64_MIN; + if (pInfo->pRes->info.rows > 0) { + printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pInfo->pRes; + } + break; + } + printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo)); + + if (pInfo->pFillInfo->curGroupId != pBlock->info.id.groupId) { + pInfo->pFillInfo->curGroupId = pBlock->info.id.groupId; + pInfo->pFillInfo->preRowKey = INT64_MIN; + } + + pInfo->pFillSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; + if (pInfo->pFillSup->winRange.ekey <= 0) { + pInfo->pFillSup->winRange.ekey = INT64_MAX; + } + + switch (pBlock->info.type) { + case STREAM_RETRIEVE: + return pBlock; + case STREAM_DELETE_RESULT: { + pInfo->pSrcDelBlock = pBlock; + pInfo->srcDelRowIndex = 0; + blockDataCleanup(pInfo->pDelRes); + pInfo->pFillSup->hasDelete = true; + doDeleteFillResult(pOperator); + if (pInfo->pDelRes->info.rows > 0) { + printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pInfo->pDelRes; + } + continue; + } break; + case STREAM_NORMAL: + case STREAM_INVALID: + case STREAM_PULL_DATA: { + doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock); + memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); + pInfo->srcRowIndex = -1; + } break; + case STREAM_CHECKPOINT: + case STREAM_CREATE_CHILD_TABLE: { + return pBlock; + } break; + default: + ASSERTS(false, "invalid SSDataBlock type"); + } + } + + doStreamFillImpl(pOperator); + doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo); + memcpy(pInfo->pRes->info.parTbName, pInfo->pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN); + pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; + if (pInfo->pRes->info.rows > 0) { + break; + } + } + if (pOperator->status == OP_RES_TO_RETURN) { + doDeleteFillFinalize(pOperator); + } + + if (pInfo->pRes->info.rows == 0) { + setOperatorCompleted(pOperator); + resetStreamFillInfo(pInfo); + return NULL; + } + + pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; + printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pInfo->pRes; +} + +static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { + pFillSup->rowSize = sizeof(SResultCellData) * pFillSup->numOfAllCols; + for (int i = 0; i < pFillSup->numOfAllCols; i++) { + SFillColInfo* pCol = &pFillSup->pAllColInfo[i]; + SResSchema* pSchema = &pCol->pExpr->base.resSchema; + pFillSup->rowSize += pSchema->bytes; + } + pFillSup->next.key = INT64_MIN; + pFillSup->nextNext.key = INT64_MIN; + pFillSup->prev.key = INT64_MIN; + pFillSup->cur.key = INT64_MIN; + pFillSup->next.pRowVal = NULL; + pFillSup->nextNext.pRowVal = NULL; + pFillSup->prev.pRowVal = NULL; + pFillSup->cur.pRowVal = NULL; + + return TSDB_CODE_SUCCESS; +} + +static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, + SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) { + SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); + if (!pFillSup) { + return NULL; + } + pFillSup->numOfFillCols = numOfFillCols; + int32_t numOfNotFillCols = 0; + SExprInfo* noFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); + pFillSup->pAllColInfo = createFillColInfo(pFillExprInfo, pFillSup->numOfFillCols, noFillExprInfo, numOfNotFillCols, + (const SNodeListNode*)(pPhyFillNode->pValues)); + pFillSup->type = convertFillType(pPhyFillNode->mode); + pFillSup->numOfAllCols = pFillSup->numOfFillCols + numOfNotFillCols; + pFillSup->interval = *pInterval; + pFillSup->pAPI = pAPI; + + int32_t code = initResultBuf(pFillSup); + if (code != TSDB_CODE_SUCCESS) { + destroyStreamFillSupporter(pFillSup); + return NULL; + } + + SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); + code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore); + if (code != TSDB_CODE_SUCCESS) { + destroyStreamFillSupporter(pFillSup); + return NULL; + } + + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pFillSup->pResMap = tSimpleHashInit(16, hashFn); + pFillSup->hasDelete = false; + return pFillSup; +} + +SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { + SStreamFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SStreamFillInfo)); + pFillInfo->start = INT64_MIN; + pFillInfo->current = INT64_MIN; + pFillInfo->end = INT64_MIN; + pFillInfo->preRowKey = INT64_MIN; + pFillInfo->needFill = false; + pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo)); + pFillInfo->pLinearInfo->hasNext = false; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + pFillInfo->pLinearInfo->pEndPoints = NULL; + pFillInfo->pLinearInfo->pNextEndPoints = NULL; + if (pFillSup->type == TSDB_FILL_LINEAR) { + pFillInfo->pLinearInfo->pEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); + pFillInfo->pLinearInfo->pNextEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); + for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) { + SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); + SPoint value = {0}; + value.val = taosMemoryCalloc(1, pColData->info.bytes); + taosArrayPush(pFillInfo->pLinearInfo->pEndPoints, &value); + + value.val = taosMemoryCalloc(1, pColData->info.bytes); + taosArrayPush(pFillInfo->pLinearInfo->pNextEndPoints, &value); + } + } + pFillInfo->pLinearInfo->winIndex = 0; + + pFillInfo->pResRow = NULL; + if (pFillSup->type == TSDB_FILL_SET_VALUE || pFillSup->type == TSDB_FILL_SET_VALUE_F || + pFillSup->type == TSDB_FILL_NULL || pFillSup->type == TSDB_FILL_NULL_F) { + pFillInfo->pResRow = taosMemoryCalloc(1, sizeof(SResultRowData)); + pFillInfo->pResRow->key = INT64_MIN; + pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize); + for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { + SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); + SResultCellData* pCell = getResultCell(pFillInfo->pResRow, i); + pCell->bytes = pColData->info.bytes; + pCell->type = pColData->info.type; + } + } + + pFillInfo->type = pFillSup->type; + pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeRange)); + pFillInfo->delIndex = 0; + pFillInfo->curGroupId = 0; + return pFillInfo; +} + +SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, + SExecTaskInfo* pTaskInfo) { + SStreamFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFillOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval; + int32_t numOfFillCols = 0; + SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); + pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI); + if (!pInfo->pFillSup) { + goto _error; + } + + initResultSizeInfo(&pOperator->resultInfo, 4096); + pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); + pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity); + + pInfo->pFillInfo = initStreamFillInfo(pInfo->pFillSup, pInfo->pRes); + if (!pInfo->pFillInfo) { + goto _error; + } + + if (pInfo->pFillInfo->type == TSDB_FILL_SET_VALUE || pInfo->pFillInfo->type == TSDB_FILL_SET_VALUE_F) { + for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pInfo->pFillSup->pAllColInfo + i; + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId); + SVariant* pVar = &(pFillCol->fillVal); + if (pCell->type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); + SET_TYPED_DATA(pCell->pData, pCell->type, v); + } else if (IS_FLOAT_TYPE(pCell->type)) { + double v = 0; + GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); + SET_TYPED_DATA(pCell->pData, pCell->type, v); + } else if (IS_INTEGER_TYPE(pCell->type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); + SET_TYPED_DATA(pCell->pData, pCell->type, v); + } else { + pCell->isNull = true; + } + } + } else if (pInfo->pFillInfo->type == TSDB_FILL_NULL || pInfo->pFillInfo->type == TSDB_FILL_NULL_F) { + for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pInfo->pFillSup->pAllColInfo + i; + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId); + pCell->isNull = true; + } + } + + pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); + blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity); + + pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId; + pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId; + + int32_t numOfOutputCols = 0; + int32_t code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, + &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->srcRowIndex = -1; + setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, + pTaskInfo); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + return pOperator; + +_error: + destroyStreamFillOperatorInfo(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index bac1f9de05..6f920d84a8 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -1277,16 +1277,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { while (1) { if (isTaskKilled(pTaskInfo)) { - if (pInfo->pUpdated != NULL) { - pInfo->pUpdated = taosArrayDestroy(pInfo->pUpdated); - } - - if (pInfo->pUpdatedMap != NULL) { - tSimpleHashCleanup(pInfo->pUpdatedMap); - pInfo->pUpdatedMap = NULL; - } - qInfo("%s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); - T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); + qInfo("===stream=== %s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); + return NULL; } SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); @@ -2539,7 +2531,6 @@ int32_t encodeSResultWindowInfo(void** buf, SResultWindowInfo* key, int32_t outL void* decodeSResultWindowInfo(void* buf, SResultWindowInfo* key, int32_t outLen) { buf = taosDecodeFixedBool(buf, &key->isOutput); - key->pStatePos->pRowBuff = NULL; buf = decodeSSessionKey(buf, &key->sessionWin); return buf; } @@ -2597,6 +2588,7 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera if (!pInfo) { return buf; } + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; // 5.checksum if (isParent) { @@ -2615,6 +2607,8 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); + pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &winfo.sessionWin, pAggSup->gap, + (void**)&winfo.pStatePos, &pAggSup->resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); } @@ -2998,6 +2992,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->recvGetAll = false; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; + setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, + OP_NOT_OPENED, pInfo, pTaskInfo); // for stream void* buff = NULL; int32_t len = 0; @@ -3008,8 +3004,6 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh doStreamSessionDecodeOpState(buff, len, pOperator, true); taosMemoryFree(buff); } - setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, - OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); @@ -3544,6 +3538,7 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato if (!pInfo) { return buf; } + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; // 5.checksum if (isParent) { @@ -3562,6 +3557,9 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); + pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, + pAggSup->stateKeySize, compareStateKey, + (void**)&winfo.pStatePos, &pAggSup->resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); } @@ -3879,6 +3877,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT); pInfo->recvGetAll = false; + setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, + pInfo, pTaskInfo); // for stream void* buff = NULL; int32_t len = 0; @@ -3890,8 +3890,6 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys taosMemoryFree(buff); } - setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, - pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState); @@ -4350,16 +4348,8 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { while (1) { if (isTaskKilled(pTaskInfo)) { - if (pInfo->pUpdated != NULL) { - pInfo->pUpdated = taosArrayDestroy(pInfo->pUpdated); - } - - if (pInfo->pUpdatedMap != NULL) { - tSimpleHashCleanup(pInfo->pUpdatedMap); - pInfo->pUpdatedMap = NULL; - } - - T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); + qInfo("===stream=== %s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); + return NULL; } SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 9ff903cdb9..82881dcd14 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -32,6 +32,32 @@ struct STupleHandle { int32_t rowIndex; }; +typedef struct SSortMemFileRegion { + int64_t fileOffset; + int32_t regionSize; + + int32_t bufRegOffset; + int32_t bufLen; + char* buf; +} SSortMemFileRegion; + +typedef struct SSortMemFile { + char* writeBuf; + int32_t writeBufSize; + int64_t writeFileOffset; + + int32_t currRegionId; + int32_t currRegionOffset; + bool bRegionDirty; + + SArray* aFileRegions; + int32_t cacheSize; + int32_t blockSize; + + FILE* pTdFile; + char memFilePath[PATH_MAX]; +} SSortMemFile; + struct SSortHandle { int32_t type; int32_t pageSize; @@ -76,10 +102,21 @@ struct SSortHandle { bool (*abortCheckFn)(void* param); void* abortCheckParam; + bool bSortByRowId; + SSortMemFile* pExtRowsMemFile; + int32_t extRowBytes; + int32_t extRowsPageSize; + int32_t extRowsMemSize; + int32_t srcTsSlotId; + SBlockOrderInfo extRowsOrderInfo; + void (*mergeLimitReachedFn)(uint64_t tableUid, void* param); void* mergeLimitReachedParam; }; +static int32_t destroySortMemFile(SSortHandle* pHandle); +static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, int32_t tupleOffset, int32_t rowLen, + char** ppRow, bool* pFreeRow); void tsortSetSingleTableMerge(SSortHandle* pHandle) { pHandle->singleTableMerge = true; } @@ -189,6 +226,7 @@ void destroyTuple(void* t) { } } + /** * * @param type @@ -202,7 +240,7 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page pSortHandle->type = type; pSortHandle->pageSize = pageSize; pSortHandle->numOfPages = numOfPages; - pSortHandle->pSortInfo = pSortInfo; + pSortHandle->pSortInfo = taosArrayDup(pSortInfo, NULL); pSortHandle->loops = 0; pSortHandle->pqMaxTupleLength = pqMaxTupleLength; @@ -305,6 +343,10 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) { qDebug("all source fetch time: %" PRId64 "us num:%" PRId64 " %s", fetchUs, fetchNum, pSortHandle->idStr); taosArrayDestroy(pSortHandle->pOrderedSource); + if (pSortHandle->pExtRowsMemFile != NULL) { + destroySortMemFile(pSortHandle); + } + taosArrayDestroy(pSortHandle->pSortInfo); taosMemoryFreeClear(pSortHandle); } @@ -851,6 +893,389 @@ static int32_t createPageBuf(SSortHandle* pHandle) { return 0; } +void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHandle* pTupleHandle) { + if (pHandle->bSortByRowId) { + int32_t regionId = *(int32_t*)tsortGetValue(pTupleHandle, 1); + int32_t offset = *(int32_t*)tsortGetValue(pTupleHandle, 2); + int32_t length = *(int32_t*)tsortGetValue(pTupleHandle, 3); + + char* buf = NULL; + bool bFreeRow = false; + getRowBufFromExtMemFile(pHandle, regionId, offset, length, &buf, &bFreeRow); + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + char* isNull = (char*)buf; + char* pStart = (char*)buf + sizeof(int8_t) * numOfCols; + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + + if (!isNull[i]) { + colDataSetVal(pColInfo, pBlock->info.rows, pStart, false); + if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) { + int32_t dataLen = getJsonValueLen(pStart); + pStart += dataLen; + } else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { + pStart += varDataTLen(pStart); + } else { + int32_t bytes = pColInfo->info.bytes; + pStart += bytes; + } + } else { + colDataSetNULL(pColInfo, pBlock->info.rows); + } + } + if (bFreeRow) { + taosMemoryFree(buf); + } + if (*(int32_t*)pStart != pStart - buf) { + qError("table merge scan row buf deserialization. length error %d != %d ", *(int32_t*)pStart, + (int32_t)(pStart - buf)); + }; + + pBlock->info.dataLoad = 1; + pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag; + pBlock->info.rows += 1; + + } else { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + bool isNull = tsortIsNullVal(pTupleHandle, i); + if (isNull) { + colDataSetNULL(pColInfo, pBlock->info.rows); + } else { + char* pData = tsortGetValue(pTupleHandle, i); + if (pData != NULL) { + colDataSetVal(pColInfo, pBlock->info.rows, pData, false); + } + } + } + + pBlock->info.dataLoad = 1; + pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag; + pBlock->info.rows += 1; + } +} + +static int32_t blockRowToBuf(SSDataBlock* pBlock, int32_t rowIdx, char* buf) { + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + + char* isNull = (char*)buf; + char* pStart = (char*)buf + sizeof(int8_t) * numOfCols; + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); + if (colDataIsNull_s(pCol, rowIdx)) { + isNull[i] = 1; + continue; + } + + isNull[i] = 0; + char* pData = colDataGetData(pCol, rowIdx); + if (pCol->info.type == TSDB_DATA_TYPE_JSON) { + if (pCol->pData) { + int32_t dataLen = getJsonValueLen(pData); + memcpy(pStart, pData, dataLen); + pStart += dataLen; + } else { + // the column that is pre-allocated has no data and has offset + *pStart = 0; + pStart += 1; + } + } else if (IS_VAR_DATA_TYPE(pCol->info.type)) { + if (pCol->pData) { + varDataCopy(pStart, pData); + pStart += varDataTLen(pData); + } else { + // the column that is pre-allocated has no data and has offset + *(VarDataLenT*)(pStart) = 0; + pStart += VARSTR_HEADER_SIZE; + } + } else { + int32_t bytes = pCol->info.bytes; + memcpy(pStart, pData, bytes); + pStart += bytes; + } + } + *(int32_t*)pStart = (char*)pStart - (char*)buf; + pStart += sizeof(int32_t); + return (int32_t)(pStart - (char*)buf); +} + +static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, int32_t tupleOffset, int32_t rowLen, + char** ppRow, bool* pFreeRow) { + SSortMemFile* pMemFile = pHandle->pExtRowsMemFile; + SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, regionId); + if (pRegion->buf == NULL) { + pRegion->bufRegOffset = 0; + pRegion->buf = taosMemoryMalloc(pMemFile->blockSize); + if (pRegion->buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + taosSeekCFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET); + int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize); + int ret = taosReadFromCFile(pRegion->buf, readBytes, 1, pMemFile->pTdFile); + if (ret != 1) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } + pRegion->bufLen = readBytes; + } + ASSERT(pRegion->bufRegOffset <= tupleOffset); + if (pRegion->bufRegOffset + pRegion->bufLen >= tupleOffset + rowLen) { + *pFreeRow = false; + *ppRow = pRegion->buf + tupleOffset - pRegion->bufRegOffset; + } else { + *ppRow = taosMemoryMalloc(rowLen); + if (*ppRow == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t szThisBlock = pRegion->bufLen - (tupleOffset - pRegion->bufRegOffset); + memcpy(*ppRow, pRegion->buf + tupleOffset - pRegion->bufRegOffset, szThisBlock); + taosSeekCFile(pMemFile->pTdFile, pRegion->fileOffset + pRegion->bufRegOffset + pRegion->bufLen, SEEK_SET); + int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize - (pRegion->bufRegOffset + pRegion->bufLen)); + int ret = taosReadFromCFile(pRegion->buf, readBytes, 1, pMemFile->pTdFile); + if (ret != 1) { + taosMemoryFreeClear(*ppRow); + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } + memcpy(*ppRow + szThisBlock, pRegion->buf, rowLen - szThisBlock); + *pFreeRow = true; + pRegion->bufRegOffset += pRegion->bufLen; + pRegion->bufLen = readBytes; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t createSortMemFile(SSortHandle* pHandle) { + if (pHandle->pExtRowsMemFile != NULL) { + return TSDB_CODE_SUCCESS; + } + int32_t code = TSDB_CODE_SUCCESS; + SSortMemFile* pMemFile = taosMemoryCalloc(1, sizeof(SSortMemFile)); + if (pMemFile == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (code == TSDB_CODE_SUCCESS) { + taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath); + pMemFile->pTdFile = taosOpenCFile(pMemFile->memFilePath, "w+"); + if (pMemFile->pTdFile == NULL) { + code = terrno = TAOS_SYSTEM_ERROR(errno); + } + } + if (code == TSDB_CODE_SUCCESS) { + taosSetAutoDelFile(pMemFile->memFilePath); + + pMemFile->currRegionId = -1; + pMemFile->currRegionOffset = -1; + + pMemFile->writeBufSize = 4 * 1024 * 1024; + pMemFile->writeFileOffset = -1; + pMemFile->bRegionDirty = false; + + pMemFile->writeBuf = taosMemoryMalloc(pMemFile->writeBufSize); + if (pMemFile->writeBuf == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (code == TSDB_CODE_SUCCESS) { + pMemFile->cacheSize = pHandle->extRowsMemSize; + pMemFile->aFileRegions = taosArrayInit(64, sizeof(SSortMemFileRegion)); + if (pMemFile->aFileRegions == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (code == TSDB_CODE_SUCCESS) { + pHandle->pExtRowsMemFile = pMemFile; + } else { + if (pMemFile) { + if (pMemFile->aFileRegions) taosMemoryFreeClear(pMemFile->aFileRegions); + if (pMemFile->writeBuf) taosMemoryFreeClear(pMemFile->writeBuf); + if (pMemFile->pTdFile) { + taosCloseCFile(pMemFile->pTdFile); + pMemFile->pTdFile = NULL; + } + taosMemoryFreeClear(pMemFile); + } + } + return code; +} + +static int32_t destroySortMemFile(SSortHandle* pHandle) { + if (pHandle->pExtRowsMemFile == NULL) return TSDB_CODE_SUCCESS; + + SSortMemFile* pMemFile = pHandle->pExtRowsMemFile; + for (int32_t i = 0; i < taosArrayGetSize(pMemFile->aFileRegions); ++i) { + SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, i); + taosMemoryFree(pRegion->buf); + } + taosArrayDestroy(pMemFile->aFileRegions); + pMemFile->aFileRegions = NULL; + + taosMemoryFree(pMemFile->writeBuf); + pMemFile->writeBuf = NULL; + + taosCloseCFile(pMemFile->pTdFile); + pMemFile->pTdFile = NULL; + taosRemoveFile(pMemFile->memFilePath); + taosMemoryFree(pMemFile); + pHandle->pExtRowsMemFile = NULL; + return TSDB_CODE_SUCCESS; +} + +static int32_t tsortOpenRegion(SSortHandle* pHandle) { + SSortMemFile* pMemFile = pHandle->pExtRowsMemFile; + if (pMemFile->currRegionId == -1) { + SSortMemFileRegion region = {0}; + region.fileOffset = 0; + region.bufRegOffset = 0; + taosArrayPush(pMemFile->aFileRegions, ®ion); + pMemFile->currRegionId = 0; + pMemFile->currRegionOffset = 0; + pMemFile->writeFileOffset = 0; + } else { + SSortMemFileRegion regionNew = {0}; + SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, pMemFile->currRegionId); + regionNew.fileOffset = pRegion->fileOffset + pRegion->regionSize; + regionNew.bufRegOffset = 0; + taosArrayPush(pMemFile->aFileRegions, ®ionNew); + ++pMemFile->currRegionId; + pMemFile->currRegionOffset = 0; + pMemFile->writeFileOffset = regionNew.fileOffset; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t tsortCloseRegion(SSortHandle* pHandle) { + SSortMemFile* pMemFile = pHandle->pExtRowsMemFile; + SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, pMemFile->currRegionId); + pRegion->regionSize = pMemFile->currRegionOffset; + int32_t writeBytes = pRegion->regionSize - (pMemFile->writeFileOffset - pRegion->fileOffset); + if (writeBytes > 0) { + int ret = fwrite(pMemFile->writeBuf, writeBytes, 1, pMemFile->pTdFile); + if (ret != 1) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } + pMemFile->bRegionDirty = false; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t tsortFinalizeRegions(SSortHandle* pHandle) { + SSortMemFile* pMemFile = pHandle->pExtRowsMemFile; + size_t numRegions = taosArrayGetSize(pMemFile->aFileRegions); + ASSERT(numRegions == (pMemFile->currRegionId + 1)); + if (numRegions == 0) return TSDB_CODE_SUCCESS; + int32_t blockReadBytes = (pMemFile->cacheSize / numRegions + 4095) & ~4095; + pMemFile->blockSize = blockReadBytes; + + for (int32_t i = 0; i < numRegions; ++i) { + SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, i); + pRegion->bufRegOffset = 0; + } + taosMemoryFree(pMemFile->writeBuf); + pMemFile->writeBuf = NULL; + return TSDB_CODE_SUCCESS; +} + +static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* pBlock, int32_t rowIdx, int32_t* pRegionId, int32_t* pOffset, int32_t* pLength) { + SSortMemFile* pMemFile = pHandle->pExtRowsMemFile; + SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, pMemFile->currRegionId); + { + if (pMemFile->currRegionOffset + pHandle->extRowBytes >= pMemFile->writeBufSize) { + int32_t writeBytes = pMemFile->currRegionOffset - (pMemFile->writeFileOffset - pRegion->fileOffset); + int ret = fwrite(pMemFile->writeBuf, writeBytes, 1, pMemFile->pTdFile); + if (ret != 1) { + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } + pMemFile->writeFileOffset = pRegion->fileOffset + pMemFile->currRegionOffset; + } + } + *pRegionId = pMemFile->currRegionId; + *pOffset = pMemFile->currRegionOffset; + int32_t writeBufOffset = pMemFile->currRegionOffset - (pMemFile->writeFileOffset - pRegion->fileOffset); + int32_t blockLen = blockRowToBuf(pBlock, rowIdx, pMemFile->writeBuf + writeBufOffset); + *pLength = blockLen; + pMemFile->currRegionOffset += blockLen; + pMemFile->bRegionDirty = true; + return TSDB_CODE_SUCCESS; +} + +static void appendToRowIndexDataBlock(SSortHandle* pHandle, SSDataBlock* pSource, int32_t* rowIndex) { + int32_t pageId = -1; + int32_t offset = -1; + int32_t length = -1; + saveBlockRowToExtRowsMemFile(pHandle, pSource, *rowIndex, &pageId, &offset, &length); + + SSDataBlock* pBlock = pHandle->pDataBlock; + SColumnInfoData* pSrcTsCol = taosArrayGet(pSource->pDataBlock, pHandle->extRowsOrderInfo.slotId); + SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, 0); + char* pData = colDataGetData(pSrcTsCol, *rowIndex); + colDataSetVal(pTsCol, pBlock->info.rows, pData, false); + + SColumnInfoData* pRegionIdCol = taosArrayGet(pBlock->pDataBlock, 1); + colDataSetInt32(pRegionIdCol, pBlock->info.rows, &pageId); + + SColumnInfoData* pOffsetCol = taosArrayGet(pBlock->pDataBlock, 2); + colDataSetInt32(pOffsetCol, pBlock->info.rows, &offset); + + SColumnInfoData* pLengthCol = taosArrayGet(pBlock->pDataBlock, 3); + colDataSetInt32(pLengthCol, pBlock->info.rows, &length); + + pBlock->info.rows += 1; + *rowIndex += 1; +} + +static void initRowIdSort(SSortHandle* pHandle) { + + SSDataBlock* pSortInput = createDataBlock(); + SColumnInfoData tsCol = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, 8, 1); + blockDataAppendColInfo(pSortInput, &tsCol); + SColumnInfoData regionIdCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2); + blockDataAppendColInfo(pSortInput, ®ionIdCol); + SColumnInfoData offsetCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3); + blockDataAppendColInfo(pSortInput, &offsetCol); + SColumnInfoData lengthCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 4); + blockDataAppendColInfo(pSortInput, &lengthCol); + + blockDataDestroy(pHandle->pDataBlock); + pHandle->pDataBlock = pSortInput; + + int32_t rowSize = blockDataGetRowSize(pHandle->pDataBlock); + size_t nCols = taosArrayGetSize(pHandle->pDataBlock->pDataBlock); + pHandle->pageSize = 256 * 1024; // 256k + pHandle->numOfPages = 256; + + SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0); + SBlockOrderInfo bi = {0}; + bi.order = pOrder->order; + bi.slotId = 0; + bi.nullFirst = NULL_ORDER_FIRST; + + SArray* aOrder = taosArrayInit(1, sizeof(SBlockOrderInfo)); + taosArrayPush(aOrder, &bi); + + taosArrayDestroy(pHandle->pSortInfo); + pHandle->pSortInfo = aOrder; + return; +} + +int32_t tsortSetSortByRowId(SSortHandle* pHandle, int32_t extRowsMemSize) { + pHandle->extRowBytes = blockDataGetRowSize(pHandle->pDataBlock) + taosArrayGetSize(pHandle->pDataBlock->pDataBlock) + sizeof(int32_t); + pHandle->extRowsMemSize = extRowsMemSize; + SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0); + pHandle->extRowsOrderInfo = *pOrder; + initRowIdSort(pHandle); + if (!osTempSpaceAvailable()) { + terrno = TSDB_CODE_NO_DISKSPACE; + qError("create sort mem file failed since %s, tempDir:%s", terrstr(), tsTempDir); + return terrno; + } + int32_t code = createSortMemFile(pHandle); + pHandle->bSortByRowId = true; + return code; +} + typedef struct SBlkMergeSupport { int64_t** aTs; int32_t* aRowIdx; @@ -925,7 +1350,7 @@ static int32_t getPageBufIncForRow(SSDataBlock* blk, int32_t row, int32_t rowIdx return sz; } -static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockOrderInfo* order, SArray* aExtSrc) { +static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray* aExtSrc) { int32_t code = TSDB_CODE_SUCCESS; int pgHeaderSz = sizeof(int32_t) + sizeof(int32_t) * taosArrayGetSize(pHandle->pDataBlock->pDataBlock); int32_t rowCap = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize, pgHeaderSz); @@ -933,13 +1358,15 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO blockDataCleanup(pHandle->pDataBlock); int32_t numBlks = taosArrayGetSize(aBlk); + SBlockOrderInfo* pOrigBlockOrder = (!pHandle->bSortByRowId) ? taosArrayGet(pHandle->pSortInfo, 0) : &pHandle->extRowsOrderInfo; + SBlockOrderInfo* pHandleBlockOrder = taosArrayGet(pHandle->pSortInfo, 0); SBlkMergeSupport sup; sup.aRowIdx = taosMemoryCalloc(numBlks, sizeof(int32_t)); sup.aTs = taosMemoryCalloc(numBlks, sizeof(int64_t*)); - sup.order = order->order; + sup.order = pOrigBlockOrder->order; for (int i = 0; i < numBlks; ++i) { SSDataBlock* blk = taosArrayGetP(aBlk, i); - SColumnInfoData* col = taosArrayGet(blk->pDataBlock, order->slotId); + SColumnInfoData* col = taosArrayGet(blk->pDataBlock, pOrigBlockOrder->slotId); sup.aTs[i] = (int64_t*)col->pData; sup.aRowIdx[i] = 0; } @@ -963,16 +1390,17 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO int32_t nMergedRows = 0; bool mergeLimitReached = false; size_t blkPgSz = pgHeaderSz; - int64_t lastPageBufTs = (order->order == TSDB_ORDER_ASC) ? INT64_MAX : INT64_MIN; - int64_t currTs = (order->order == TSDB_ORDER_ASC) ? INT64_MAX : INT64_MIN; + int64_t lastPageBufTs = (pHandleBlockOrder->order == TSDB_ORDER_ASC) ? INT64_MAX : INT64_MIN; + int64_t currTs = (pHandleBlockOrder->order == TSDB_ORDER_ASC) ? INT64_MAX : INT64_MIN; while (nRows < totalRows) { int32_t minIdx = tMergeTreeGetChosenIndex(pTree); SSDataBlock* minBlk = taosArrayGetP(aBlk, minIdx); int32_t minRow = sup.aRowIdx[minIdx]; - int32_t bufInc = getPageBufIncForRow(minBlk, minRow, pHandle->pDataBlock->info.rows); + SSDataBlock* incBlock = (pHandle->bSortByRowId) ? pHandle->pDataBlock : minBlk; + int32_t bufInc = getPageBufIncForRow(incBlock, minRow, pHandle->pDataBlock->info.rows); if (blkPgSz <= pHandle->pageSize && blkPgSz + bufInc > pHandle->pageSize) { - SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId); + SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, pHandleBlockOrder->slotId); lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; code = appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); if (code != TSDB_CODE_SUCCESS) { @@ -985,19 +1413,24 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO nMergedRows += pHandle->pDataBlock->info.rows; blockDataCleanup(pHandle->pDataBlock); blkPgSz = pgHeaderSz; - bufInc = getPageBufIncForRow(minBlk, minRow, 0); + incBlock = (pHandle->bSortByRowId) ? pHandle->pDataBlock : minBlk; + bufInc = getPageBufIncForRow(incBlock, minRow, 0); if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) { mergeLimitReached = true; - if ((lastPageBufTs < pHandle->currMergeLimitTs && order->order == TSDB_ORDER_ASC) || - (lastPageBufTs > pHandle->currMergeLimitTs && order->order == TSDB_ORDER_DESC)) { + if ((lastPageBufTs < pHandle->currMergeLimitTs && pHandleBlockOrder->order == TSDB_ORDER_ASC) || + (lastPageBufTs > pHandle->currMergeLimitTs && pHandleBlockOrder->order == TSDB_ORDER_DESC)) { pHandle->currMergeLimitTs = lastPageBufTs; } break; } } blockDataEnsureCapacity(pHandle->pDataBlock, pHandle->pDataBlock->info.rows + 1); - appendOneRowToDataBlock(pHandle->pDataBlock, minBlk, &minRow); + if (!pHandle->bSortByRowId) { + appendOneRowToDataBlock(pHandle->pDataBlock, minBlk, &minRow); + } else { + appendToRowIndexDataBlock(pHandle, minBlk, &minRow); + } blkPgSz += bufInc; ++nRows; @@ -1011,7 +1444,7 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO } if (pHandle->pDataBlock->info.rows > 0) { if (!mergeLimitReached) { - SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId); + SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, pHandleBlockOrder->slotId); lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; code = appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); if (code != TSDB_CODE_SUCCESS) { @@ -1024,14 +1457,15 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO nMergedRows += pHandle->pDataBlock->info.rows; if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) { mergeLimitReached = true; - if ((lastPageBufTs < pHandle->currMergeLimitTs && order->order == TSDB_ORDER_ASC) || - (lastPageBufTs > pHandle->currMergeLimitTs && order->order == TSDB_ORDER_DESC)) { + if ((lastPageBufTs < pHandle->currMergeLimitTs && pHandleBlockOrder->order == TSDB_ORDER_ASC) || + (lastPageBufTs > pHandle->currMergeLimitTs && pHandleBlockOrder->order == TSDB_ORDER_DESC)) { pHandle->currMergeLimitTs = lastPageBufTs; } } } blockDataCleanup(pHandle->pDataBlock); } + SSDataBlock* pMemSrcBlk = createOneDataBlock(pHandle->pDataBlock, false); doAddNewExternalMemSource(pHandle->pBuf, aExtSrc, pMemSrcBlk, &pHandle->sourceId, aPgId); @@ -1083,11 +1517,10 @@ static SSDataBlock* getRowsBlockWithinMergeLimit(const SSortHandle* pHandle, SSH } static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { - SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0); size_t nSrc = taosArrayGetSize(pHandle->pOrderedSource); SArray* aExtSrc = taosArrayInit(nSrc, POINTER_BYTES); - size_t maxBufSize = pHandle->numOfPages * pHandle->pageSize; + size_t maxBufSize = (pHandle->bSortByRowId) ? pHandle->extRowsMemSize : (pHandle->numOfPages * pHandle->pageSize); int32_t code = createPageBuf(pHandle); if (code != TSDB_CODE_SUCCESS) { @@ -1098,7 +1531,8 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { SSortSource* pSrc = taosArrayGetP(pHandle->pOrderedSource, 0); int32_t szSort = 0; - if (pOrder->order == TSDB_ORDER_ASC) { + SBlockOrderInfo* pOrigOrder = (!pHandle->bSortByRowId) ? taosArrayGet(pHandle->pSortInfo, 0) : &pHandle->extRowsOrderInfo; + if (pOrigOrder->order == TSDB_ORDER_ASC) { pHandle->currMergeLimitTs = INT64_MAX; } else { pHandle->currMergeLimitTs = INT64_MIN; @@ -1110,7 +1544,6 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { while (1) { SSDataBlock* pBlk = pHandle->fetchfp(pSrc->param); - int64_t p = taosGetTimestampUs(); bool bExtractedBlock = false; bool bSkipBlock = false; if (pBlk != NULL && pHandle->mergeLimit > 0) { @@ -1121,13 +1554,13 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { } if (pBlk != NULL) { - SColumnInfoData* tsCol = taosArrayGet(pBlk->pDataBlock, pOrder->slotId); + SColumnInfoData* tsCol = taosArrayGet(pBlk->pDataBlock, pOrigOrder->slotId); int64_t firstRowTs = *(int64_t*)tsCol->pData; - if ((pOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs) || - (pOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) { + if ((pOrigOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs) || + (pOrigOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) { if (bExtractedBlock) { blockDataDestroy(pBlk); - } + } continue; } } @@ -1150,7 +1583,13 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { if ((pBlk != NULL && szSort > maxBufSize) || (pBlk == NULL && szSort > 0)) { tSimpleHashClear(mUidBlk); - code = sortBlocksToExtSource(pHandle, aBlkSort, pOrder, aExtSrc); + + int64_t p = taosGetTimestampUs(); + if (pHandle->bSortByRowId) { + tsortOpenRegion(pHandle); + } + code = sortBlocksToExtSource(pHandle, aBlkSort, aExtSrc); + if (code != TSDB_CODE_SUCCESS) { for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) { blockDataDestroy(taosArrayGetP(aBlkSort, i)); @@ -1158,7 +1597,9 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { taosArrayClear(aBlkSort); break; } - + if (pHandle->bSortByRowId) { + tsortCloseRegion(pHandle); + } int64_t el = taosGetTimestampUs() - p; pHandle->sortElapsed += el; @@ -1195,10 +1636,25 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { } taosArrayDestroy(aExtSrc); tSimpleHashCleanup(mTableNumRows); + if (pHandle->bSortByRowId) { + tsortFinalizeRegions(pHandle); + } pHandle->type = SORT_SINGLESOURCE_SORT; return code; } +static void freeSSortSource(SSortSource* source) { + if (NULL == source) return; + if (source->param && !source->onlyRef) { + taosMemoryFree(source->param); + } + if (!source->onlyRef && source->src.pBlock) { + blockDataDestroy(source->src.pBlock); + source->src.pBlock = NULL; + } + taosMemoryFree(source); +} + static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { int32_t code = 0; size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize; @@ -1231,14 +1687,7 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { code = blockDataMerge(pHandle->pDataBlock, pBlock); if (code != TSDB_CODE_SUCCESS) { - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); - } - if (!source->onlyRef && source->src.pBlock) { - blockDataDestroy(source->src.pBlock); - source->src.pBlock = NULL; - } - taosMemoryFree(source); + freeSSortSource(source); return code; } @@ -1248,15 +1697,7 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { int64_t p = taosGetTimestampUs(); code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); if (code != 0) { - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); - } - if (!source->onlyRef && source->src.pBlock) { - blockDataDestroy(source->src.pBlock); - source->src.pBlock = NULL; - } - - taosMemoryFree(source); + freeSSortSource(source); return code; } @@ -1265,16 +1706,13 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { if (pHandle->pqMaxRows > 0) blockDataKeepFirstNRows(pHandle->pDataBlock, pHandle->pqMaxRows); code = doAddToBuf(pHandle->pDataBlock, pHandle); if (code != TSDB_CODE_SUCCESS) { + freeSSortSource(source); return code; } } } - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); - } - - taosMemoryFree(source); + freeSSortSource(source); if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) { size_t size = blockDataGetSize(pHandle->pDataBlock); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 3f5ffcae32..453d927378 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -457,6 +457,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_SCALAR_FIELD(isCountByTag); CLONE_OBJECT_FIELD(pFuncTypes, functParamClone); COPY_SCALAR_FIELD(paraTablesSort); + COPY_SCALAR_FIELD(smallDataTsSort); return TSDB_CODE_SUCCESS; } @@ -690,6 +691,7 @@ static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhy COPY_SCALAR_FIELD(filesetDelimited); COPY_SCALAR_FIELD(needCountEmptyTable); COPY_SCALAR_FIELD(paraTablesSort); + COPY_SCALAR_FIELD(smallDataTsSort); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 689886c366..019ef6f18b 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -699,6 +699,7 @@ static const char* jkScanLogicPlanGroupTags = "GroupTags"; static const char* jkScanLogicPlanOnlyMetaCtbIdx = "OnlyMetaCtbIdx"; static const char* jkScanLogicPlanFilesetDelimited = "FilesetDelimited"; static const char* jkScanLogicPlanParaTablesSort = "ParaTablesSort"; +static const char* jkScanLogicPlanSmallDataTsSort = "SmallDataTsSort"; static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; @@ -749,6 +750,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkScanLogicPlanParaTablesSort, pNode->paraTablesSort); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkScanLogicPlanSmallDataTsSort, pNode->paraTablesSort); + } return code; } @@ -800,7 +804,10 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { code = tjsonGetBoolValue(pJson, jkScanLogicPlanFilesetDelimited, &pNode->filesetDelimited); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkScanLogicPlanParaTablesSort, &pNode->paraTablesSort); + code = tjsonGetBoolValue(pJson, jkScanLogicPlanParaTablesSort, &pNode->smallDataTsSort); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkScanLogicPlanSmallDataTsSort, &pNode->smallDataTsSort); } return code; } @@ -1896,6 +1903,7 @@ static const char* jkTableScanPhysiPlanIgnoreUpdate = "IgnoreUpdate"; static const char* jkTableScanPhysiPlanFilesetDelimited = "FilesetDelimited"; static const char* jkTableScanPhysiPlanNeedCountEmptyTable = "NeedCountEmptyTable"; static const char* jkTableScanPhysiPlanParaTablesSort = "ParaTablesSort"; +static const char* jkTableScanPhysiPlanSmallDataTsSort = "SmallDataTsSort"; static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; @@ -1973,6 +1981,9 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanParaTablesSort, pNode->paraTablesSort); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanSmallDataTsSort, pNode->smallDataTsSort); + } return code; } @@ -2052,6 +2063,9 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanParaTablesSort, &pNode->paraTablesSort); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanSmallDataTsSort, &pNode->smallDataTsSort); + } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 357abc2858..95a5c2f51e 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2188,6 +2188,9 @@ static int32_t physiTableScanNodeInlineToMsg(const void* pObj, STlvEncoder* pEnc if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueBool(pEncoder, pNode->paraTablesSort); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueBool(pEncoder, pNode->smallDataTsSort); + } return code; } @@ -2275,6 +2278,9 @@ static int32_t msgToPhysiTableScanNodeInline(STlvDecoder* pDecoder, void* pObj) if (TSDB_CODE_SUCCESS == code) { code = tlvDecodeValueBool(pDecoder, &pNode->paraTablesSort); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueBool(pDecoder, &pNode->smallDataTsSort); + } return code; } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 1d6c5e800e..74a7934985 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -404,6 +404,9 @@ bool addHintNodeToList(SAstCreateContext* pCxt, SNodeList** ppHintList, EHintOpt case HINT_PARA_TABLES_SORT: if (paramNum > 0 || hasHint(*ppHintList, HINT_PARA_TABLES_SORT)) return true; break; + case HINT_SMALLDATA_TS_SORT: + if (paramNum > 0 || hasHint(*ppHintList, HINT_SMALLDATA_TS_SORT)) return true; + break; default: return true; } @@ -490,6 +493,14 @@ SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral) { } opt = HINT_PARA_TABLES_SORT; break; + case TK_SMALLDATA_TS_SORT: + lastComma = false; + if (0 != opt || inParamList) { + quit = true; + break; + } + opt = HINT_SMALLDATA_TS_SORT; + break; case TK_NK_LP: lastComma = false; if (0 == opt || inParamList) { diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index f1013d6157..0d943dd9db 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -213,6 +213,7 @@ static SKeyword keywordTable[] = { {"SLIDING", TK_SLIDING}, {"SLIMIT", TK_SLIMIT}, {"SMA", TK_SMA}, + {"SMALLDATA_TS_SORT", TK_SMALLDATA_TS_SORT}, {"SMALLINT", TK_SMALLINT}, {"SNODE", TK_SNODE}, {"SNODES", TK_SNODES}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index cc522c9b5f..38737d7a17 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4009,6 +4009,26 @@ static int32_t translateEventWindow(STranslateContext* pCxt, SSelectStmt* pSelec } static int32_t translateCountWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { + SCountWindowNode* pCountWin = (SCountWindowNode*)pSelect->pWindow; + if (pCountWin->windowCount <= 1) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Size of Count window must exceed 1."); + } + + if (pCountWin->windowSliding <= 0) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Size of Count window must exceed 0."); + } + + if (pCountWin->windowSliding > pCountWin->windowCount) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "sliding value no larger than the count value."); + } + + if (pCountWin->windowCount > INT32_MAX) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Size of Count window must less than 2147483647(INT32_MAX)."); + } if (QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TIMELINE_QUERY, @@ -4527,7 +4547,7 @@ static int32_t translateWhere(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) { code = getQueryTimeRange(pCxt, pSelect->pWhere, &pSelect->timeRange); } - if (pSelect->pWhere != NULL) { + if (pSelect->pWhere != NULL && pCxt->pParseCxt->topicQuery == false) { setTableVgroupsFromEqualTbnameCond(pCxt, pSelect); } return code; @@ -7828,29 +7848,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm if (pStmt->pOptions->ignoreExpired != 1) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Ignore expired data of Count window must be 1."); - } - - SCountWindowNode* pCountWin = (SCountWindowNode*)pSelect->pWindow; - if (pCountWin->windowCount <= 1) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Size of Count window must exceed 1."); } - - if (pCountWin->windowSliding <= 0) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Size of Count window must exceed 0."); - } - - if (pCountWin->windowSliding > pCountWin->windowCount) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "sliding value no larger than the count value."); - } - - if (pCountWin->windowCount > INT32_MAX) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Size of Count window must less than 2147483647(INT32_MAX)."); - } - } return TSDB_CODE_SUCCESS; @@ -8137,7 +8135,9 @@ static int32_t adjustTagsForCreateTable(STranslateContext* pCxt, SCreateStreamSt SColumnDefNode* pDef = (SColumnDefNode*)pTagDef; if (!dataTypeEqual(&pDef->dataType, &((SExprNode*)pTagExpr)->resType)) { SNode* pFunc = NULL; - int32_t code = createCastFunc(pCxt, pTagExpr, pDef->dataType, &pFunc); + SDataType defType = pDef->dataType; + defType.bytes = calcTypeBytes(defType); + int32_t code = createCastFunc(pCxt, pTagExpr, defType, &pFunc); if (TSDB_CODE_SUCCESS != code) { return code; } diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index fcccdcf23e..3f1cb0fbd3 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -47,7 +47,8 @@ int32_t validateQueryPlan(SPlanContext* pCxt, SQueryPlan* pPlan); bool getBatchScanOptionFromHint(SNodeList* pList); bool getSortForGroupOptHint(SNodeList* pList); -bool getparaTablesSortOptHint(SNodeList* pList); +bool getParaTablesSortOptHint(SNodeList* pList); +bool getSmallDataTsSortOptHint(SNodeList* pList); bool getOptHint(SNodeList* pList, EHintOption hint); SLogicNode* getLogicNodeRootNode(SLogicNode* pCurr); int32_t collectTableAliasFromNodes(SNode* pNode, SSHashObj** ppRes); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 37cdc31ae0..c34d8ac64f 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -502,7 +502,8 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } else { nodesDestroyNode((SNode*)pScan); } - pScan->paraTablesSort = getparaTablesSortOptHint(pSelect->pHint); + pScan->paraTablesSort = getParaTablesSortOptHint(pSelect->pHint); + pScan->smallDataTsSort = getSmallDataTsSortOptHint(pSelect->pHint); pCxt->hasScan = true; return code; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index baef39144c..8748cc7c17 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -527,7 +527,10 @@ static int32_t createSimpleScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSub if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } - vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); + + if (pScanLogicNode->pVgroupList) { + vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); + } return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, pScan, pPhyNode); } @@ -538,8 +541,9 @@ static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubpla if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } - vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); - + if (pScanLogicNode->pVgroupList) { + vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); + } pScan->onlyMetaCtbIdx = pScanLogicNode->onlyMetaCtbIdx; return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); @@ -563,8 +567,9 @@ static int32_t createLastRowScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSu pScan->groupSort = pScanLogicNode->groupSort; pScan->ignoreNull = pScanLogicNode->igLastNull; - vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); - + if (pScanLogicNode->pVgroupList) { + vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); + } int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); if (TSDB_CODE_SUCCESS == code && pScanLogicNode->pFuncTypes != NULL) { @@ -609,8 +614,9 @@ static int32_t createTableCountScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* } pScan->groupSort = pScanLogicNode->groupSort; - vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); - + if (pScanLogicNode->pVgroupList) { + vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); + } return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } @@ -652,6 +658,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp pTableScan->filesetDelimited = pScanLogicNode->filesetDelimited; pTableScan->needCountEmptyTable = pScanLogicNode->isCountByTag; pTableScan->paraTablesSort = pScanLogicNode->paraTablesSort; + pTableScan->smallDataTsSort = pScanLogicNode->smallDataTsSort; int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); if (TSDB_CODE_SUCCESS == code) { @@ -680,7 +687,9 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TABLES) || 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TAGS) || 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_COLS)) { - vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); + if (pScanLogicNode->pVgroupList) { + vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); + } } else { pSubplan->execNode.nodeId = MNODE_HANDLE; pSubplan->execNode.epSet = pCxt->pPlanCxt->mgmtEpSet; @@ -2216,11 +2225,12 @@ static int32_t createQueryInserter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNod pInserter->stableId = pModify->stableId; pInserter->tableType = pModify->tableType; strcpy(pInserter->tableName, pModify->tableName); - pInserter->vgId = pModify->pVgroupList->vgroups[0].vgId; - pInserter->epSet = pModify->pVgroupList->vgroups[0].epSet; pInserter->explain = (QUERY_NODE_EXPLAIN_STMT == nodeType(pCxt->pPlanCxt->pAstRoot) ? true : false); - vgroupInfoToNodeAddr(pModify->pVgroupList->vgroups, &pSubplan->execNode); - + if (pModify->pVgroupList) { + pInserter->vgId = pModify->pVgroupList->vgroups[0].vgId; + pInserter->epSet = pModify->pVgroupList->vgroups[0].epSet; + vgroupInfoToNodeAddr(pModify->pVgroupList->vgroups, &pSubplan->execNode); + } int32_t code = setListSlotId(pCxt, pSubplan->pNode->pOutputDataBlockDesc->dataBlockId, -1, pModify->pInsertCols, &pInserter->pCols); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index d9a7475f59..34be489333 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -274,7 +274,7 @@ static bool stbSplNeedSplitWindow(bool streamQuery, SLogicNode* pNode) { } } - if (WINDOW_TYPE_STATE == pWindow->winType) { + if (WINDOW_TYPE_STATE == pWindow->winType || WINDOW_TYPE_COUNT == pWindow->winType) { if (!streamQuery) { return stbSplHasMultiTbScan(streamQuery, pNode); } else { @@ -1320,7 +1320,11 @@ static int32_t stbSplSplitScanNodeWithPartTags(SSplitContext* pCxt, SStableSplit SLogicNode* pSplitNode = NULL; int32_t code = stbSplGetSplitNodeForScan(pInfo, &pSplitNode); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pSplitNode, NULL, pSplitNode, true, true); + bool needSort = true; + if (QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pSplitNode) && !pSplitNode->pLimit && !pSplitNode->pSlimit) { + needSort = !((SProjectLogicNode*)pSplitNode)->ignoreGroupId; + } + code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pSplitNode, NULL, pSplitNode, needSort, needSort); } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index f31bf23bc9..a6109cdacb 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -466,7 +466,7 @@ bool getOptHint(SNodeList* pList, EHintOption hint) { return false; } -bool getparaTablesSortOptHint(SNodeList* pList) { +bool getParaTablesSortOptHint(SNodeList* pList) { if (!pList) return false; SNode* pNode; FOREACH(pNode, pList) { @@ -478,6 +478,18 @@ bool getparaTablesSortOptHint(SNodeList* pList) { return false; } +bool getSmallDataTsSortOptHint(SNodeList* pList) { + if (!pList) return false; + SNode* pNode; + FOREACH(pNode, pList) { + SHintNode* pHint = (SHintNode*)pNode; + if (pHint->option == HINT_SMALLDATA_TS_SORT) { + return true; + } + } + return false; +} + int32_t collectTableAliasFromNodes(SNode* pNode, SSHashObj** ppRes) { int32_t code = TSDB_CODE_SUCCESS; SLogicNode* pCurr = (SLogicNode*)pNode; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 66ec460861..faa90dcbf8 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -715,6 +715,7 @@ int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SD uint64_t tId = req.taskId; int64_t rId = 0; int32_t eId = -1; + pRes->source = req.source; SQWMsg qwMsg = {.node = node, .msg = req.msg, .msgLen = req.phyLen, .connInfo = pMsg->info}; QW_SCH_TASK_DLOG("processDelete start, node:%p, handle:%p, sql:%s", node, pMsg->info.handle, req.sql); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index b1cd10eac9..5f43ae9f3c 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -808,7 +808,11 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o complete = false; continue; } - char *p = colDataGetData(params[m].columnData, i); + + // 1=1 and tag_column = 1 + int32_t ind = (i >= params[m].numOfRows)? (params[m].numOfRows - 1):i; + char* p = colDataGetData(params[m].columnData, ind); + GET_TYPED_DATA(value, bool, params[m].columnData->info.type, p); if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) { diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 1aa21e73a3..d129b0024f 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -304,6 +304,7 @@ typedef struct SSchJob { SSchResInfo userRes; char *sql; SQueryProfileSummary summary; + int8_t source; } SSchJob; typedef struct SSchTaskCtx { diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index e50ec64d54..48aab63ba3 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -746,6 +746,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->chkKillParam = pReq->chkKillParam; pJob->userRes.execFp = pReq->execFp; pJob->userRes.cbParam = pReq->cbParam; + pJob->source = pReq->source; if (pReq->pNodeList == NULL || taosArrayGetSize(pReq->pNodeList) <= 0) { qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 1c0b31109e..aa33d6f2fc 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -892,7 +892,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe int32_t schCloneSMsgSendInfo(void *src, void **dst) { SMsgSendInfo *pSrc = src; int32_t code = 0; - SMsgSendInfo *pDst = taosMemoryMalloc(sizeof(*pSrc)); + SMsgSendInfo *pDst = taosMemoryCalloc(1, sizeof(*pSrc)); if (NULL == pDst) { qError("malloc SMsgSendInfo for rpcCtx failed, len:%d", (int32_t)sizeof(*pSrc)); SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -940,6 +940,10 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery SCH_ERR_JRET(schGenerateCallBackInfo(pJob, pTask, msg, msgSize, msgType, trans, isHb, &pMsgSendInfo)); SCH_ERR_JRET(schUpdateSendTargetInfo(pMsgSendInfo, addr, pTask)); + if (isHb && persistHandle && trans->pHandle == 0) { + trans->pHandle = rpcAllocHandle(); + } + if (pJob && pTask) { SCH_TASK_DLOG("start to send %s msg to node[%d,%s,%d], pTrans:%p, pHandle:%p", TMSG_INFO(msgType), addr->nodeId, epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, trans->pTrans, trans->pHandle); @@ -1086,6 +1090,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.sqlLen = strlen(pJob->sql); req.sql = (char *)pJob->sql; req.msg = pTask->msg; + req.source = pJob->source; msgSize = tSerializeSVDeleteReq(NULL, 0, &req); msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 87f63b48ed..d0055d5400 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -99,6 +99,7 @@ void streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration); int32_t streamDispatchStreamBlock(SStreamTask* pTask); void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups); int32_t getNumOfDispatchBranch(SStreamTask* pTask); +void clearBufferedDispatchMsg(SStreamTask* pTask); int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBlock); SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg); diff --git a/source/libs/stream/inc/streamsm.h b/source/libs/stream/inc/streamsm.h index 22e1c4497b..47e0ce1b55 100644 --- a/source/libs/stream/inc/streamsm.h +++ b/source/libs/stream/inc/streamsm.h @@ -26,21 +26,21 @@ extern "C" { typedef int32_t (*__state_trans_fn)(SStreamTask*); typedef int32_t (*__state_trans_succ_fn)(SStreamTask*); -typedef struct SAttachedEventInfo { +typedef struct SFutureHandleEventInfo { ETaskStatus status; // required status that this event can be handled EStreamTaskEvent event; // the delayed handled event void* pParam; - void* pFn; -} SAttachedEventInfo; + __state_trans_user_fn callBackFn; +} SFutureHandleEventInfo; typedef struct STaskStateTrans { - bool autoInvokeEndFn; - SStreamTaskState state; - EStreamTaskEvent event; - SStreamTaskState next; - __state_trans_fn pAction; - __state_trans_succ_fn pSuccAction; - SAttachedEventInfo attachEvent; + bool autoInvokeEndFn; + SStreamTaskState state; + EStreamTaskEvent event; + SStreamTaskState next; + __state_trans_fn pAction; + __state_trans_succ_fn pSuccAction; + SFutureHandleEventInfo attachEvent; } STaskStateTrans; struct SStreamTaskSM { diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 910fd93989..5dafad8951 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -906,6 +906,7 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { return 0; } +#ifdef BUILD_NO_CALL static int32_t chkpIdComp(const void* a, const void* b) { int64_t x = *(int64_t*)a; int64_t y = *(int64_t*)b; @@ -964,6 +965,7 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) { taosMemoryFree(chkpPath); return 0; } +#endif #ifdef BUILD_NO_CALL int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t*** ppHandle, SArray* refs) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 78b914c3db..dc790b5b2d 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -315,6 +315,16 @@ int32_t getNumOfDispatchBranch(SStreamTask* pTask) { : taosArrayGetSize(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos); } +void clearBufferedDispatchMsg(SStreamTask* pTask) { + SDispatchMsgInfo* pMsgInfo = &pTask->msgInfo; + if (pMsgInfo->pData != NULL) { + destroyDispatchMsg(pMsgInfo->pData, getNumOfDispatchBranch(pTask)); + } + + pMsgInfo->pData = NULL; + pMsgInfo->dispatchMsgType = 0; +} + static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pData) { int32_t code = 0; int32_t numOfBlocks = taosArrayGetSize(pData->blocks); @@ -678,8 +688,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { // todo deal with only partially success dispatch case atomic_store_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 0); if (terrno == TSDB_CODE_APP_IS_STOPPING) { // in case of this error, do not retry anymore - destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); - pTask->msgInfo.pData = NULL; + clearBufferedDispatchMsg(pTask); return code; } @@ -740,6 +749,8 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) { int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + ASSERT(dataStrLen > 0); + void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) return -1; @@ -936,15 +947,12 @@ void streamClearChkptReadyMsg(SStreamTask* pTask) { // this message has been sent successfully, let's try next one. static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId) { stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData); - destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); - bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); if (delayDispatch) { pTask->chkInfo.dispatchCheckpointTrigger = true; } - pTask->msgInfo.pData = NULL; - pTask->msgInfo.dispatchMsgType = 0; + clearBufferedDispatchMsg(pTask); int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs; @@ -1084,7 +1092,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i } else { // this message has been sent successfully, let's try next one. pTask->msgInfo.retryCount = 0; - // transtate msg has been sent to downstream successfully. let's transfer the fill-history task state + // trans-state msg has been sent to downstream successfully. let's transfer the fill-history task state if (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__TRANS_STATE) { stDebug("s-task:%s dispatch transtate msgId:%d to downstream successfully, start to transfer state", id, msgId); ASSERT(pTask->info.fillHistory == 1); @@ -1093,6 +1101,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i if (code != TSDB_CODE_SUCCESS) { // todo: do nothing if error happens } + clearBufferedDispatchMsg(pTask); + // now ready for next data output atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL); } else { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index bac6022834..cb51d90fba 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -353,7 +353,8 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) { ASSERT(status == TASK_STATUS__HALT || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP); } else { - ASSERT(status == TASK_STATUS__READY || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP); + ASSERT(status == TASK_STATUS__READY || status == TASK_STATUS__PAUSE || status == TASK_STATUS__DROPPING || + status == TASK_STATUS__STOP); int32_t code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s halt stream task:%s failed, code:%s not transfer state to stream task", id, @@ -410,6 +411,12 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { return TSDB_CODE_SUCCESS; } +static int32_t haltCallback(SStreamTask* pTask, void* param) { + streamTaskOpenAllUpstreamInput(pTask); + streamTaskSendCheckpointReq(pTask); + return TSDB_CODE_SUCCESS; +} + int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { int32_t code = TSDB_CODE_SUCCESS; SStreamMeta* pMeta = pTask->pMeta; @@ -419,11 +426,12 @@ int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { int32_t level = pTask->info.taskLevel; if (level == TASK_LEVEL__AGG || level == TASK_LEVEL__SOURCE) { // do transfer task operator states. code = streamDoTransferStateToStreamTask(pTask); - } else { // no state transfer for sink tasks, and drop fill-history task, followed by opening inputQ of sink task. + } else { + // no state transfer for sink tasks, and drop fill-history task, followed by opening inputQ of sink task. SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); if (pStreamTask != NULL) { // halt the related stream sink task - code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); + code = streamTaskHandleEventAsync(pStreamTask->status.pSM, TASK_EVENT_HALT, haltCallback, NULL); if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s halt stream task:%s failed, code:%s not transfer state to stream task", pTask->id.idStr, pStreamTask->id.idStr, tstrerror(code)); @@ -432,9 +440,6 @@ int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { } else { stDebug("s-task:%s halt by related fill-history task:%s", pStreamTask->id.idStr, pTask->id.idStr); } - - streamTaskOpenAllUpstreamInput(pStreamTask); - streamTaskSendCheckpointReq(pStreamTask); streamMetaReleaseTask(pMeta, pStreamTask); } } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index a09b940a19..a072ee1f6f 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -669,6 +669,13 @@ static void doRemoveIdFromList(SStreamMeta* pMeta, int32_t num, SStreamTaskId* i } } +static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + streamTaskSendCheckpointSourceRsp(pTask); + } + return 0; +} + int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { SStreamTask* pTask = NULL; @@ -687,7 +694,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t } // handle the dropping event - streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_DROPPING); + streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL); } else { stDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId); streamMetaWUnLock(pMeta); diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 3d0241df75..723f04c499 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -156,6 +156,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pKey = *pDestWinKey; goto _end; } @@ -167,6 +168,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pKey = *pDestWinKey; goto _end; } @@ -380,6 +382,14 @@ static SStreamStateCur* seekKeyCurrentPrev_buff(SStreamFileState* pFileState, co (*pWins) = pWinStates; } + if (size > 0 && index == -1) { + SRowBuffPos* pPos = taosArrayGetP(pWinStates, 0); + SSessionKey* pWin = (SSessionKey*)pPos->pKey; + if (pWinKey->win.skey == pWin->win.skey) { + index = 0; + } + } + if (index >= 0) { pCur = createSessionStateCursor(pFileState); pCur->buffIndex = index; @@ -387,6 +397,7 @@ static SStreamStateCur* seekKeyCurrentPrev_buff(SStreamFileState* pFileState, co *pIndex = index; } } + return pCur; } @@ -666,6 +677,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *key = *pDestWinKey; goto _end; } @@ -679,6 +691,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *key = *pDestWinKey; goto _end; } @@ -771,6 +784,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pWinKey = *pDestWinKey; goto _end; } @@ -799,6 +813,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pWinKey = *pDestWinKey; goto _end; } diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 6112a208c6..cb340ade32 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -385,7 +385,7 @@ int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) { void doProcessDownstreamReadyRsp(SStreamTask* pTask) { EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST; - streamTaskOnHandleEventSuccess(pTask->status.pSM, event); + streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); int64_t initTs = pTask->execInfo.init; int64_t startTs = pTask->execInfo.start; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 9639921c77..45a9a68d3d 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -39,7 +39,7 @@ static int32_t doUpdateTaskEpset(SStreamTask* pTask, int32_t nodeId, SEpSet* pEp stDebug("s-task:0x%x (vgId:%d) self node epset is updated %s", pTask->id.taskId, nodeId, buf); } - // check for the dispath info and the upstream task info + // check for the dispatch info and the upstream task info int32_t level = pTask->info.taskLevel; if (level == TASK_LEVEL__SOURCE) { streamTaskUpdateDownstreamInfo(pTask, nodeId, pEpSet); @@ -412,9 +412,7 @@ void tFreeStreamTask(SStreamTask* pTask) { pTask->pReadyMsgList = taosArrayDestroy(pTask->pReadyMsgList); if (pTask->msgInfo.pData != NULL) { - destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); - pTask->msgInfo.pData = NULL; - pTask->msgInfo.dispatchMsgType = 0; + clearBufferedDispatchMsg(pTask); } if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { @@ -624,6 +622,7 @@ void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDo void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet) { char buf[512] = {0}; EPSET_TO_STR(pEpSet, buf); + int32_t id = pTask->id.taskId; int8_t type = pTask->outputInfo.type; if (type == TASK_OUTPUT__SHUFFLE_DISPATCH) { @@ -635,8 +634,8 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE if (pVgInfo->vgId == nodeId) { epsetAssign(&pVgInfo->epSet, pEpSet); - stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", pTask->id.taskId, - pVgInfo->taskId, nodeId, buf); + stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", id, pVgInfo->taskId, nodeId, + buf); break; } } @@ -644,11 +643,9 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE STaskDispatcherFixed* pDispatcher = &pTask->outputInfo.fixedDispatcher; if (pDispatcher->nodeId == nodeId) { epsetAssign(&pDispatcher->epSet, pEpSet); - stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpSet:%s", pTask->id.taskId, - pDispatcher->taskId, nodeId, buf); + stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", id, pDispatcher->taskId, nodeId, + buf); } - } else { - // do nothing } } @@ -766,21 +763,13 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) { return status; } -int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt, bool metaLock) { - if (pTask == NULL) { - return TSDB_CODE_SUCCESS; - } - +int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) { SStreamMeta* pMeta = pTask->pMeta; STaskId sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId}; if (pTask->info.fillHistory == 0) { return TSDB_CODE_SUCCESS; } - if (metaLock) { - streamMetaWLock(pMeta); - } - SStreamTask** ppStreamTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &sTaskId, sizeof(sTaskId)); if (ppStreamTask != NULL) { stDebug("s-task:%s clear the related stream task:0x%x attr to fill-history task", pTask->id.idStr, @@ -798,10 +787,6 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt, bool taosThreadMutexUnlock(&(*ppStreamTask)->lock); } - if (metaLock) { - streamMetaWUnLock(pMeta); - } - return TSDB_CODE_SUCCESS; } @@ -869,8 +854,8 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc) pDst->chkpointTransId = pSrc->chkpointTransId; } -void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask) { - streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_PAUSE); +static int32_t taskPauseCallback(SStreamTask* pTask, void* param) { + SStreamMeta* pMeta = pTask->pMeta; int32_t num = atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1); stInfo("vgId:%d s-task:%s pause stream task. pause task num:%d", pMeta->vgId, pTask->id.idStr, num); @@ -882,24 +867,24 @@ void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask) { } stDebug("vgId:%d s-task:%s set pause flag and pause task", pMeta->vgId, pTask->id.idStr); + return TSDB_CODE_SUCCESS; +} + +void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask) { + streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_PAUSE, taskPauseCallback, NULL); } void streamTaskResume(SStreamTask* pTask) { SStreamTaskState prevState = *streamTaskGetStatus(pTask); - SStreamMeta* pMeta = pTask->pMeta; - if (prevState.state == TASK_STATUS__PAUSE || prevState.state == TASK_STATUS__HALT) { - streamTaskRestoreStatus(pTask); - - char* pNew = streamTaskGetStatus(pTask)->name; - if (prevState.state == TASK_STATUS__PAUSE) { - int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1); - stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, prevState.name, num); - } else { - stInfo("s-task:%s status:%s resume from %s", pTask->id.idStr, pNew, prevState.name); - } + SStreamMeta* pMeta = pTask->pMeta; + int32_t code = streamTaskRestoreStatus(pTask); + if (code == TSDB_CODE_SUCCESS) { + char* pNew = streamTaskGetStatus(pTask)->name; + int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1); + stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, prevState.name, num); } else { - stDebug("s-task:%s status:%s not in pause/halt status, no need to resume", pTask->id.idStr, prevState.name); + stInfo("s-task:%s status:%s no need to resume, paused task(s):%d", pTask->id.idStr, prevState.name, pMeta->numOfPausedTasks); } } diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 83e71c42bc..6aa215586a 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -59,20 +59,23 @@ static int32_t streamTaskInitStatus(SStreamTask* pTask); static int32_t streamTaskKeepCurrentVerInWal(SStreamTask* pTask); static int32_t initStateTransferTable(); static void doInitStateTransferTable(void); -static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask); static STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStreamTaskEvent event, __state_trans_fn fn, __state_trans_succ_fn succFn, - SAttachedEventInfo* pEventInfo, bool autoInvoke); + SFutureHandleEventInfo* pEventInfo); static int32_t dummyFn(SStreamTask* UNUSED_PARAM(p)) { return TSDB_CODE_SUCCESS; } -static int32_t attachEvent(SStreamTask* pTask, SAttachedEventInfo* pEvtInfo) { +static int32_t attachWaitedEvent(SStreamTask* pTask, SFutureHandleEventInfo* pEvtInfo) { char* p = streamTaskGetStatus(pTask)->name; stDebug("s-task:%s status:%s attach event:%s required status:%s, since not allowed to handle it", pTask->id.idStr, p, GET_EVT_NAME(pEvtInfo->event), StreamTaskStatusList[pEvtInfo->status].name); - taosArrayPush(pTask->status.pSM->pWaitingEventList, pEvtInfo); + + SArray* pList = pTask->status.pSM->pWaitingEventList; + taosArrayPush(pList, pEvtInfo); + + stDebug("s-task:%s add into waiting list, total waiting events:%d", pTask->id.idStr, (int32_t)taosArrayGetSize(pList)); return 0; } @@ -85,18 +88,6 @@ int32_t streamTaskInitStatus(SStreamTask* pTask) { return 0; } -static int32_t streamTaskDoCheckpoint(SStreamTask* pTask) { - stDebug("s-task:%s start to do checkpoint", pTask->id.idStr); - return 0; -} - -int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask) { - if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - streamTaskSendCheckpointSourceRsp(pTask); - } - return 0; -} - int32_t streamTaskKeepCurrentVerInWal(SStreamTask* pTask) { if (!HAS_RELATED_FILLHISTORY_TASK(pTask)) { stError("s-task:%s no related fill-history task, since it may have been dropped already", pTask->id.idStr); @@ -170,9 +161,11 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr, pEventName, el, pSM->prev.state.name, pSM->current.name); - SAttachedEventInfo* pEvtInfo = taosArrayGet(pSM->pWaitingEventList, 0); + ASSERT(taosArrayGetSize(pSM->pWaitingEventList) == 1); - // OK, let's handle the attached event, since the task has reached the required status now + SFutureHandleEventInfo* pEvtInfo = taosArrayGet(pSM->pWaitingEventList, 0); + + // OK, let's handle the waiting event, since the task has reached the required status now if (pSM->current.state == pEvtInfo->status) { stDebug("s-task:%s handle the event:%s in waiting list, state:%s", pTask->id.idStr, GET_EVT_NAME(pEvtInfo->event), pSM->current.name); @@ -189,7 +182,7 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, code = pNextTrans->pAction(pSM->pTask); if (pNextTrans->autoInvokeEndFn) { - return streamTaskOnHandleEventSuccess(pSM, pNextTrans->event); + return streamTaskOnHandleEventSuccess(pSM, pNextTrans->event, pEvtInfo->callBackFn, pEvtInfo->pParam); } else { return code; } @@ -203,30 +196,61 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, return code; } -void streamTaskRestoreStatus(SStreamTask* pTask) { +static int32_t removeEventInWaitingList(SStreamTask* pTask, EStreamTaskEvent event) { SStreamTaskSM* pSM = pTask->status.pSM; + bool removed = false; taosThreadMutexLock(&pTask->lock); - ASSERT(pSM->pActiveTrans == NULL); - ASSERT(pSM->current.state == TASK_STATUS__PAUSE || pSM->current.state == TASK_STATUS__HALT); + int32_t num = taosArrayGetSize(pSM->pWaitingEventList); + for (int32_t i = 0; i < num; ++i) { + SFutureHandleEventInfo* pInfo = taosArrayGet(pSM->pWaitingEventList, i); + if (pInfo->event == event) { + taosArrayRemove(pSM->pWaitingEventList, i); + stDebug("s-task:%s pause event in waiting list not be handled yet, remove it from waiting list, remaining:%d", + pTask->id.idStr, pInfo->event); + removed = true; + break; + } + } - SStreamTaskState state = pSM->current; - pSM->current = pSM->prev.state; - - pSM->prev.state = state; - pSM->prev.evt = 0; - - pSM->startTs = taosGetTimestampMs(); - - if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { - stDebug("s-task:%s restore status, %s -> %s, and then handle waiting event", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); - doHandleWaitingEvent(pSM, "restore-pause/halt", pTask); - } else { - stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); + if (!removed) { + stDebug("s-task:%s failed to remove event:%s in waiting list", pTask->id.idStr, StreamTaskEventList[event].name); } taosThreadMutexUnlock(&pTask->lock); + return TSDB_CODE_SUCCESS; +} + +int32_t streamTaskRestoreStatus(SStreamTask* pTask) { + SStreamTaskSM* pSM = pTask->status.pSM; + int32_t code = 0; + + taosThreadMutexLock(&pTask->lock); + + if (pSM->current.state == TASK_STATUS__PAUSE && pSM->pActiveTrans == NULL) { + SStreamTaskState state = pSM->current; + pSM->current = pSM->prev.state; + + pSM->prev.state = state; + pSM->prev.evt = 0; + + pSM->startTs = taosGetTimestampMs(); + + if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { + stDebug("s-task:%s restore status, %s -> %s, and then handle waiting event", pTask->id.idStr, + pSM->prev.state.name, pSM->current.name); + doHandleWaitingEvent(pSM, "restore-pause/halt", pTask); + } else { + stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); + } + } else { + removeEventInWaitingList(pTask, TASK_EVENT_PAUSE); + code = -1; // failed to restore the status + } + + taosThreadMutexUnlock(&pTask->lock); + return code; } SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask) { @@ -242,7 +266,7 @@ SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask) { } pSM->pTask = pTask; - pSM->pWaitingEventList = taosArrayInit(4, sizeof(SAttachedEventInfo)); + pSM->pWaitingEventList = taosArrayInit(4, sizeof(SFutureHandleEventInfo)); if (pSM->pWaitingEventList == NULL) { taosMemoryFree(pSM); @@ -273,7 +297,7 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt const char* id = pTask->id.idStr; if (pTrans->attachEvent.event != 0) { - attachEvent(pTask, &pTrans->attachEvent); + attachWaitedEvent(pTask, &pTrans->attachEvent); taosThreadMutexUnlock(&pTask->lock); while (1) { @@ -303,7 +327,32 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt // todo handle error code; if (pTrans->autoInvokeEndFn) { - streamTaskOnHandleEventSuccess(pSM, event); + streamTaskOnHandleEventSuccess(pSM, event, NULL, NULL); + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t doHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskStateTrans* pTrans, __state_trans_user_fn callbackFn, void* param) { + SStreamTask* pTask = pSM->pTask; + if (pTrans->attachEvent.event != 0) { + SFutureHandleEventInfo info = pTrans->attachEvent; + info.pParam = param; + info.callBackFn = callbackFn; + + attachWaitedEvent(pTask, &info); + taosThreadMutexUnlock(&pTask->lock); + } else { // override current active trans + pSM->pActiveTrans = pTrans; + pSM->startTs = taosGetTimestampMs(); + taosThreadMutexUnlock(&pTask->lock); + + int32_t code = pTrans->pAction(pTask); + // todo handle error code; + + if (pTrans->autoInvokeEndFn) { + streamTaskOnHandleEventSuccess(pSM, event, callbackFn, param); } } @@ -349,6 +398,45 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) { return code; } +int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param) { + int32_t code = TSDB_CODE_SUCCESS; + SStreamTask* pTask = pSM->pTask; + STaskStateTrans* pTrans = NULL; + + while (1) { + taosThreadMutexLock(&pTask->lock); + + if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) { + EStreamTaskEvent evt = pSM->pActiveTrans->event; + taosThreadMutexUnlock(&pTask->lock); + + stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed", + pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt)); + taosMsleep(100); + } else { + // no active event trans exists, handle this event directly + pTrans = streamTaskFindTransform(pSM->current.state, event); + if (pTrans == NULL) { + stDebug("s-task:%s failed to handle event:%s, status:%s", pTask->id.idStr, GET_EVT_NAME(event), pSM->current.name); + taosThreadMutexUnlock(&pTask->lock); + return TSDB_CODE_STREAM_INVALID_STATETRANS; + } + + if (pSM->pActiveTrans != NULL) { + // currently in some state transfer procedure, not auto invoke transfer, quit from this procedure + stDebug("s-task:%s event:%s handle procedure quit, status %s -> %s failed, handle event %s now", + pTask->id.idStr, GET_EVT_NAME(pSM->pActiveTrans->event), pSM->current.name, + pSM->pActiveTrans->next.name, GET_EVT_NAME(event)); + } + + code = doHandleEventAsync(pSM, event, pTrans, callbackFn, param); + break; + } + } + + return code; +} + static void keepPrevInfo(SStreamTaskSM* pSM) { STaskStateTrans* pTrans = pSM->pActiveTrans; @@ -356,8 +444,9 @@ static void keepPrevInfo(SStreamTaskSM* pSM) { pSM->prev.evt = pTrans->event; } -int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event) { +int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param) { SStreamTask* pTask = pSM->pTask; + const char* id = pTask->id.idStr; // do update the task status taosThreadMutexLock(&pTask->lock); @@ -369,16 +458,16 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even s == TASK_STATUS__UNINIT || s == TASK_STATUS__READY); // the pSM->prev.evt may be 0, so print string is not appropriate. - stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", pTask->id.idStr, - GET_EVT_NAME(event), pSM->current.name, GET_EVT_NAME(pSM->prev.evt)); + stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", id, GET_EVT_NAME(event), + pSM->current.name, GET_EVT_NAME(pSM->prev.evt)); taosThreadMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INVALID_STATETRANS; } if (pTrans->event != event) { - stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", pTask->id.idStr, - GET_EVT_NAME(event), pSM->current.name, GET_EVT_NAME(pTrans->event)); + stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", id, GET_EVT_NAME(event), + pSM->current.name, GET_EVT_NAME(pTrans->event)); taosThreadMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INVALID_STATETRANS; } @@ -388,16 +477,31 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even pSM->current = pTrans->next; pSM->pActiveTrans = NULL; + // todo remove it // on success callback, add into lock if necessary, or maybe we should add an option for this? pTrans->pSuccAction(pTask); + taosThreadMutexUnlock(&pTask->lock); + + // todo: add parameter to control lock + // after handling the callback function assigned by invoker, go on handling the waiting tasks + if (callbackFn != NULL) { + stDebug("s-task:%s start to handle user-specified callback fn for event:%s", id, GET_EVT_NAME(pTrans->event)); + callbackFn(pSM->pTask, param); + + stDebug("s-task:%s handle user-specified callback fn for event:%s completed", id, GET_EVT_NAME(pTrans->event)); + } + + taosThreadMutexLock(&pTask->lock); + + // tasks in waiting list if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask); } else { taosThreadMutexUnlock(&pTask->lock); int64_t el = (taosGetTimestampMs() - pSM->startTs); - stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr, + stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", id, GET_EVT_NAME(pTrans->event), el, pSM->prev.state.name, pSM->current.name); } @@ -453,7 +557,7 @@ void streamTaskSetStatusReady(SStreamTask* pTask) { } STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStreamTaskEvent event, __state_trans_fn fn, - __state_trans_succ_fn succFn, SAttachedEventInfo* pEventInfo, bool autoInvoke) { + __state_trans_succ_fn succFn, SFutureHandleEventInfo* pEventInfo) { STaskStateTrans trans = {0}; trans.state = StreamTaskStatusList[current]; trans.next = StreamTaskStatusList[next]; @@ -468,7 +572,7 @@ STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStr trans.pAction = (fn != NULL) ? fn : dummyFn; trans.pSuccAction = (succFn != NULL) ? succFn : dummyFn; - trans.autoInvokeEndFn = autoInvoke; + trans.autoInvokeEndFn = (fn == NULL); return trans; } @@ -482,93 +586,93 @@ void doInitStateTransferTable(void) { streamTaskSMTrans = taosArrayInit(8, sizeof(STaskStateTrans)); // initialization event handle - STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT, streamTaskInitStatus, streamTaskOnNormalTaskReady, false, false); + STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT, streamTaskInitStatus, streamTaskOnNormalTaskReady, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, false, false); + trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, NULL); taosArrayPush(streamTaskSMTrans, &trans); // scan-history related event - trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); // halt stream task, from other task status - trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true); + trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true); + trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL); taosArrayPush(streamTaskSMTrans, &trans); - SAttachedEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT}; + SFutureHandleEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT}; - trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info, true); + trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true); + trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL); taosArrayPush(streamTaskSMTrans, &trans); // checkpoint related event - trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, streamTaskDoCheckpoint, NULL, true); + trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, streamTaskDoCheckpoint, NULL, true); + trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__READY, TASK_EVENT_CHECKPOINT_DONE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__READY, TASK_EVENT_CHECKPOINT_DONE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); // pause & resume related event handle - trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - info = (SAttachedEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE}; - trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); + info = (SFutureHandleEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE}; + trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); + trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_PAUSE, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_PAUSE, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); // resume is completed by restore status of state-machine // stop related event - trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); // dropping related event - trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL, NULL, true); + trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); } //clang-format on \ No newline at end of file diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp index a949748eb5..c9b981e5f9 100644 --- a/source/libs/stream/test/backendTest.cpp +++ b/source/libs/stream/test/backendTest.cpp @@ -29,7 +29,7 @@ class BackendEnv : public ::testing::Test { void *backendCreate() { const char *streamPath = "/tmp"; - void *p = NULL; + void * p = NULL; // char *absPath = NULL; // // SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(streamPath, -1, 2); @@ -52,7 +52,7 @@ SStreamState *stateCreate(const char *path) { } void *backendOpen() { streamMetaInit(); - const char *path = "/tmp/backend"; + const char * path = "/tmp/backend"; SStreamState *p = stateCreate(path); ASSERT(p != NULL); @@ -79,7 +79,7 @@ void *backendOpen() { const char *val = "value data"; int32_t len = 0; - char *newVal = NULL; + char * newVal = NULL; streamStateGet_rocksdb(p, &key, (void **)&newVal, &len); ASSERT(len == strlen(val)); } @@ -100,7 +100,7 @@ void *backendOpen() { const char *val = "value data"; int32_t len = 0; - char *newVal = NULL; + char * newVal = NULL; int32_t code = streamStateGet_rocksdb(p, &key, (void **)&newVal, &len); ASSERT(code != 0); } @@ -130,7 +130,7 @@ void *backendOpen() { winkey.groupId = 0; winkey.ts = tsArray[0]; - char *val = NULL; + char * val = NULL; int32_t len = 0; pCurr = streamStateSeekKeyNext_rocksdb(p, &winkey); @@ -157,7 +157,7 @@ void *backendOpen() { key.ts = tsArray[i]; key.exprIdx = i; - char *val = NULL; + char * val = NULL; int32_t len = 0; streamStateFuncGet_rocksdb(p, &key, (void **)&val, &len); ASSERT(len == strlen("Value")); @@ -168,7 +168,7 @@ void *backendOpen() { key.ts = tsArray[i]; key.exprIdx = i; - char *val = NULL; + char * val = NULL; int32_t len = 0; streamStateFuncDel_rocksdb(p, &key); } @@ -213,7 +213,7 @@ void *backendOpen() { { SSessionKey key; memset(&key, 0, sizeof(key)); - char *val = NULL; + char * val = NULL; int32_t vlen = 0; code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen); ASSERT(code == 0); @@ -260,7 +260,7 @@ void *backendOpen() { SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]}; key.groupId = (uint64_t)(i); key.ts = tsArray[i]; - char *val = NULL; + char * val = NULL; int32_t vlen = 0; ASSERT(streamStateFillGet_rocksdb(p, &key, (void **)&val, &vlen) == 0); taosMemoryFreeClear(val); @@ -272,7 +272,7 @@ void *backendOpen() { SStreamStateCur *pCurr = streamStateFillGetCur_rocksdb(p, &key); ASSERT(pCurr != NULL); - char *val = NULL; + char * val = NULL; int32_t vlen = 0; ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen)); ASSERT(vlen == strlen("Value")); @@ -296,7 +296,7 @@ void *backendOpen() { SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]}; key.groupId = (uint64_t)(i); key.ts = tsArray[i]; - char *val = NULL; + char * val = NULL; int32_t vlen = 0; ASSERT(streamStateFillDel_rocksdb(p, &key) == 0); taosMemoryFreeClear(val); @@ -338,7 +338,7 @@ void *backendOpen() { char key[128] = {0}; sprintf(key, "tbname_%d", i); - char *val = NULL; + char * val = NULL; int32_t len = 0; code = streamDefaultGet_rocksdb(p, key, (void **)&val, &len); ASSERT(code == 0); @@ -354,7 +354,7 @@ TEST_F(BackendEnv, checkOpen) { SStreamState *p = (SStreamState *)backendOpen(); int64_t tsStart = taosGetTimestampMs(); { - void *pBatch = streamStateCreateBatch(); + void * pBatch = streamStateCreateBatch(); int32_t size = 0; for (int i = 0; i < size; i++) { char key[128] = {0}; @@ -368,7 +368,7 @@ TEST_F(BackendEnv, checkOpen) { streamStateDestroyBatch(pBatch); } { - void *pBatch = streamStateCreateBatch(); + void * pBatch = streamStateCreateBatch(); int32_t size = 0; char valBuf[256] = {0}; for (int i = 0; i < size; i++) { @@ -385,7 +385,7 @@ TEST_F(BackendEnv, checkOpen) { // do checkpoint 2 taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 2); { - void *pBatch = streamStateCreateBatch(); + void * pBatch = streamStateCreateBatch(); int32_t size = 0; char valBuf[256] = {0}; for (int i = 0; i < size; i++) { @@ -407,12 +407,22 @@ TEST_F(BackendEnv, checkOpen) { // taosMkDir(dump); taosMulMkDir(dump); SBkdMgt *mgt = bkdMgtCreate((char *)path); - SArray *result = taosArrayInit(4, sizeof(void *)); + SArray * result = taosArrayInit(4, sizeof(void *)); bkdMgtGetDelta(mgt, p->pTdbState->idstr, 3, result, (char *)dump); + taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 4); + + taosArrayClear(result); + bkdMgtGetDelta(mgt, p->pTdbState->idstr, 4, result, (char *)dump); bkdMgtDestroy(mgt); streamStateClose((SStreamState *)p, true); + // { + // taosRemoveDir("/tmp/backend"); + // const char * path = "/tmp/backend"; + // SStreamState *p = stateCreate(path); + // } taosRemoveDir(path); + // streamStateClose((SStreamState *)p, true); } TEST_F(BackendEnv, backendChkp) { const char *path = "/tmp"; } @@ -430,6 +440,20 @@ TEST_F(BackendEnv, backendUtil) { ASSERT_EQ(nextPow2((uint32_t)(kvDict[i].k)), kvDict[i].v); } } +TEST_F(BackendEnv, oldBackendInit) { + const char *path = "/tmp/backend1"; + taosMulMkDir(path); + { + SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(path, 10, 10); + streamBackendCleanup((void *)p); + } + { + SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(path, 10, 10); + streamBackendCleanup((void *)p); + } + + taosRemoveDir(path); +} int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 6ae72eac14..ac45f1eef6 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -322,7 +322,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) { if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) { conn->ctx.freeFunc(msg->ctx->ahandle); - } else if (msg->ctx->ahandle != NULL && pThrd->destroyAhandleFp != NULL) { + } else if (msg->msg.info.notFreeAhandle == 0 && msg->ctx->ahandle != NULL && pThrd->destroyAhandleFp != NULL) { tDebug("%s conn %p destroy unfinished ahandle %p", CONN_GET_INST_LABEL(conn), conn, msg->ctx->ahandle); pThrd->destroyAhandleFp(msg->ctx->ahandle); } @@ -584,8 +584,8 @@ void* destroyConnPool(SCliThrd* pThrd) { static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) { void* pool = pThrd->pool; - SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key) + 1); STrans* pTranInst = pThrd->pTransInst; + SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key) + 1); if (plist == NULL) { SConnList list = {0}; taosHashPut((SHashObj*)pool, key, strlen(key) + 1, (void*)&list, sizeof(list)); @@ -867,17 +867,18 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { QUEUE_INIT(&conn->q); conn->broken = true; + if (conn->list == NULL) { + conn->list = taosHashGet((SHashObj*)pThrd->pool, conn->dstAddr, strlen(conn->dstAddr)); + } - if (conn->list != NULL) { - SConnList* connList = conn->list; - connList->list->numOfConn--; - connList->size--; - } else { - if (pThrd->pool) { - SConnList* connList = taosHashGet((SHashObj*)pThrd->pool, conn->dstAddr, strlen(conn->dstAddr) + 1); - if (connList != NULL) connList->list->numOfConn--; + if (conn->list) { + SConnList* list = conn->list; + list->list->numOfConn--; + if (conn->status == ConnInPool) { + list->size--; } } + conn->list = NULL; pThrd->newConnCount--; diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 5a1ef31b7d..f47a688e6f 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -159,7 +159,7 @@ static void uvStartSendResp(SSvrMsg* msg); static void uvNotifyLinkBrokenToApp(SSvrConn* conn); -static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); +static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); static FORCE_INLINE SSvrConn* createConn(void* hThrd); static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/); static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn); @@ -527,6 +527,10 @@ void uvOnSendCb(uv_write_t* req, int status) { if (!transQueueEmpty(&conn->srvMsgs)) { msg = (SSvrMsg*)transQueueGet(&conn->srvMsgs, 0); if (msg->type == Register && conn->status == ConnAcquire) { + if (conn->regArg.init) { + transFreeMsg(conn->regArg.msg.pCont); + conn->regArg.init = 0; + } conn->regArg.notifyCount = 0; conn->regArg.init = 1; conn->regArg.msg = msg->msg; @@ -1350,6 +1354,11 @@ void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd) { return; } transQueuePop(&conn->srvMsgs); + + if (conn->regArg.init) { + transFreeMsg(conn->regArg.msg.pCont); + conn->regArg.init = 0; + } conn->regArg.notifyCount = 0; conn->regArg.init = 1; conn->regArg.msg = msg->msg; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index e6491639dc..bdd43fe9fa 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -1404,3 +1404,35 @@ int32_t taosLinkFile(char *src, char *dst) { #endif return 0; } + +FILE* taosOpenCFile(const char* filename, const char* mode) { + return fopen(filename, mode); +} + +int taosSeekCFile(FILE* file, int64_t offset, int whence) { +#ifdef WINDOWS + return _fseeki64(file, offset, whence); +#else + return fseeko(file, offset, whence); +#endif +} + +size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream ) { + return fread(buffer, size, count, stream); +} + +size_t taosWriteToCFile(const void* ptr, size_t size, size_t nitems, FILE* stream) { + return fwrite(ptr, size, nitems, stream); +} + +int taosCloseCFile(FILE *f) { + return fclose(f); +} + +int taosSetAutoDelFile(char* path) { +#ifdef WINDOWS + return SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY); +#else + return unlink(path); +#endif +} \ No newline at end of file diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index ad3c766510..8fdc2654c5 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -449,6 +449,15 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { return -1; } + int size = pCfg->array->size; + for (int32_t i = 0; i < size; ++i) { + SConfigItem *existItem = taosArrayGet(pCfg->array, i); + if (existItem != NULL && strcmp(existItem->name, pItem->name) == 0) { + taosMemoryFree(pItem->name); + return TSDB_CODE_INVALID_CFG; + } + } + int32_t len = strlen(name); char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); @@ -457,6 +466,7 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { if (pItem->dtype == CFG_DTYPE_STRING) { taosMemoryFree(pItem->str); } + taosMemoryFree(pItem->name); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index c32790022a..a2348bdedd 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -115,7 +115,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/ins_topics_test.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxTopic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -R +#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqClientConsLog.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 3b3d275a07..c6bd78336c 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -113,11 +113,11 @@ echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG echo "fqdn ${HOSTNAME}" >> $TAOS_CFG echo "serverPort ${NODE}" >> $TAOS_CFG -echo "supportVnodes 1024" >> $TAOS_CFG +echo "supportVnodes 1024" >> $TAOS_CFG echo "statusInterval 1" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG -echo "debugFlag 0" >> $TAOS_CFG +echo "debugFlag 135" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "uDebugFlag 143" >> $TAOS_CFG echo "rpcDebugFlag 143" >> $TAOS_CFG @@ -143,4 +143,5 @@ echo "asyncLog 0" >> $TAOS_CFG echo "locale en_US.UTF-8" >> $TAOS_CFG echo "telemetryReporting 0" >> $TAOS_CFG echo "querySmaOptimize 1" >> $TAOS_CFG +echo "checkpointInterval 60" >> $TAOS_CFG echo " " >> $TAOS_CFG diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index c63d6daf8a..b447a7325e 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -15,42 +15,42 @@ fi PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` while [ -n "$PID" ]; do - echo kill -9 $PID - #pkill -9 taosd - kill -9 $PID + echo kill -15 $PID + #pkill -15 taosd + kill -15 $PID echo "Killing taosd processes" if [ "$OS_TYPE" != "Darwin" ]; then fuser -k -n tcp 6030 else - lsof -nti:6030 | xargs kill -9 + lsof -nti:6030 | xargs kill -15 fi PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` done PID=`ps -ef|grep -w taos | grep -v grep | awk '{print $2}'` while [ -n "$PID" ]; do - echo kill -9 $PID + echo kill -15 $PID #pkill -9 taos - kill -9 $PID + kill -15 $PID echo "Killing taos processes" if [ "$OS_TYPE" != "Darwin" ]; then fuser -k -n tcp 6030 else - lsof -nti:6030 | xargs kill -9 + lsof -nti:6030 | xargs kill -15 fi PID=`ps -ef|grep -w taos | grep -v grep | awk '{print $2}'` done PID=`ps -ef|grep -w tmq_sim | grep -v grep | awk '{print $2}'` while [ -n "$PID" ]; do - echo kill -9 $PID - #pkill -9 tmq_sim - kill -9 $PID + echo kill -15 $PID + #pkill -15 tmq_sim + kill -15 $PID echo "Killing tmq_sim processes" if [ "$OS_TYPE" != "Darwin" ]; then fuser -k -n tcp 6030 else - lsof -nti:6030 | xargs kill -9 + lsof -nti:6030 | xargs kill -15 fi PID=`ps -ef|grep -w tmq_sim | grep -v grep | awk '{print $2}'` -done +done \ No newline at end of file diff --git a/tests/script/tsim/parser/last_both.sim b/tests/script/tsim/parser/last_both.sim index d7daf4d333..e01a966744 100644 --- a/tests/script/tsim/parser/last_both.sim +++ b/tests/script/tsim/parser/last_both.sim @@ -6,7 +6,7 @@ sql connect print ======================== dnode1 start $db = testdb sql drop database if exists $db -sql create database $db cachemodel 'both' minrows 10 stt_trigger 1 +sql create database $db cachemodel 'none' minrows 10 stt_trigger 1 sql use $db sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int) @@ -69,13 +69,13 @@ sql drop table tbf; sql alter table st2 add column c1 int; sql alter table st2 drop column c1; -run tsim/parser/last_cache_query.sim +run tsim/parser/last_both_query.sim sql flush database $db system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -run tsim/parser/last_cache_query.sim +run tsim/parser/last_both_query.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start @@ -145,6 +145,6 @@ sql alter database $db cachemodel 'both' sql alter database $db cachesize 2 sleep 11000 -run tsim/parser/last_cache_query.sim +run tsim/parser/last_both_query.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/last_both_query.sim b/tests/script/tsim/parser/last_both_query.sim new file mode 100644 index 0000000000..5f86412199 --- /dev/null +++ b/tests/script/tsim/parser/last_both_query.sim @@ -0,0 +1,496 @@ + +sql connect + +$db = testdb +sql use $db +print "test tb1" + +sql select last(ts) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi + +sql select last(f1) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != 6 then + print $data00 + return -1 +endi + +sql select last(*) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 3 then + print expect 3, actual: $data03 + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi + +sql select last(tb1.*,ts,f4) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 3 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data06 != @70-01-01 07:59:57.000@ then + return -1 +endi + +print "test tb2" +sql select last(ts) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi + +sql select last(f1) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != -6 then + print $data00 + return -1 +endi + +sql select last(*) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi +if $data01 != -6 then + return -1 +endi +if $data02 != -7.000000000 then + print $data02 + return -1 +endi +if $data03 != -8 then + return -1 +endi +if $data04 != @70-01-01 07:59:56.999@ then + if $data04 != @70-01-01 07:59:57.-01@ then + return -1 + endi +endi + +sql select last(tb2.*,ts,f4) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi +if $data01 != -6 then + return -1 +endi +if $data02 != -7.000000000 then + print $data02 + return -1 +endi +if $data03 != -8 then + return -1 +endi +if $data04 != @70-01-01 07:59:56.999@ then + if $data04 != @70-01-01 07:59:57.-01@ then + return -1 + endi +endi +if $data05 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi +if $data06 != @70-01-01 07:59:56.999@ then + if $data04 != @70-01-01 07:59:57.-01@ then + return -1 + endi +endi + +print "test tbd" +sql select last(*) from tbd +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:12:29.000@ then + print $data00 + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != NULL then + print $data02 + return -1 +endi +if $data03 != NULL then + return -1 +endi +if $data04 != NULL then + return -1 +endi + +print "test tbe" +sql select last(*) from tbe +if $rows != 0 then + return -1 +endi + +print "test stable" +sql select last(ts) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi + +sql select last(f1) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != 6 then + print $data00 + return -1 +endi + +sql select last(*) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 37.000000000 then + print expect 37.000000000 actual: $data02 + return -1 +endi +if $data03 != 27 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi + + +sql select last(st2.*,ts,f4) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 37.000000000 then + print expect 37.000000000, acutal: $data02 + return -1 +endi +if $data03 != 27 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data06 != @70-01-01 07:59:57.000@ then + return -1 +endi + +sql select last(*), id from st2 group by id order by id +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49 + +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 21 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data10 != @21-05-11 10:12:23.000@ then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data12 != 23.000000000 then + print $data02 + return -1 +endi +if $data13 != -8 then + return -1 +endi +if $data14 != @70-01-01 07:59:58.-04@ then + return -1 +endi +if $data15 != 2 then + return -1 +endi +if $data20 != @21-05-10 10:12:24.000@ then + return -1 +endi +if $data21 != 24 then + return -1 +endi +if $data22 != 11.000000000 then + print expect 11.000000000 actual: $data22 + return -1 +endi +if $data23 != 25 then + return -1 +endi +if $data24 != @70-01-01 07:59:57.-04@ then = + return -1 +endi +if $data25 != 3 then + return -1 +endi +if $data30 != @21-05-11 10:12:25.000@ then + return -1 +endi +if $data31 != 26 then + return -1 +endi +if $data32 != 17.000000000 then + print $data02 + return -1 +endi +if $data33 != 27 then + return -1 +endi +if $data34 != @70-01-01 07:59:56.-04@ then + return -1 +endi +if $data35 != 4 then + return -1 +endi +if $data40 != @21-05-11 10:12:29.000@ then + return -1 +endi +if $data41 != 36 then + return -1 +endi +if $data42 != 37.000000000 then + print $data02 + return -1 +endi +if $data43 != 35 then + return -1 +endi +if $data44 != @70-01-01 07:59:56.-05@ then + return -1 +endi +if $data45 != 5 then + return -1 +endi + +sql select last_row(*), id from st2 group by id order by id +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49 + +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != NULL then + print $data02 + return -1 +endi +if $data03 != NULL then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data10 != @21-05-11 10:12:23.000@ then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data12 != 23.000000000 then + print $data02 + return -1 +endi +if $data13 != NULL then + return -1 +endi +if $data14 != @70-01-01 07:59:58.-04@ then + return -1 +endi +if $data15 != 2 then + return -1 +endi +if $data20 != @21-05-10 10:12:24.000@ then + return -1 +endi +if $data21 != 24 then + return -1 +endi +if $data22 != NULL then + print expect NULL actual: $data22 + return -1 +endi +if $data23 != 25 then + return -1 +endi +if $data24 != @70-01-01 07:59:57.-04@ then = + return -1 +endi +if $data25 != 3 then + return -1 +endi +if $data30 != @21-05-11 10:12:25.000@ then + return -1 +endi +if $data31 != 26 then + return -1 +endi +if $data32 != NULL then + print $data02 + return -1 +endi +if $data33 != 27 then + return -1 +endi +if $data34 != @70-01-01 07:59:56.-04@ then + return -1 +endi +if $data35 != 4 then + return -1 +endi +if $data40 != @21-05-11 10:12:29.000@ then + return -1 +endi +#if $data41 != NULL then +# return -1 +#endi +#if $data42 != NULL then +# print $data02 +# return -1 +#endi +if $data43 != NULL then + return -1 +endi +#if $data44 != NULL then +# return -1 +#endi +if $data45 != 5 then + return -1 +endi + +print "test tbn" +sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) +sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) +sql insert into tbn values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000) +sql insert into tbn values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000) +sql insert into tbn values ("2021-05-13 10:10:12", NULL,NULL, NULL,NULL) + +sql select last(*) from tbn; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-13 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 3 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi + +sql alter table tbn add column c1 int; +sql alter table tbn drop column c1; diff --git a/tests/script/tsim/parser/last_cache_query.sim b/tests/script/tsim/parser/last_cache_query.sim index c5961f2183..30196e0b62 100644 --- a/tests/script/tsim/parser/last_cache_query.sim +++ b/tests/script/tsim/parser/last_cache_query.sim @@ -357,112 +357,6 @@ if $data45 != 5 then return -1 endi -sql select last_row(*), id from st2 group by id order by id -print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 -print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 -print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 -print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49 - -if $rows != 5 then - return -1 -endi -if $data00 != @21-05-12 10:10:12.000@ then - return -1 -endi -if $data01 != 6 then - return -1 -endi -if $data02 != NULL then - print $data02 - return -1 -endi -if $data03 != NULL then - return -1 -endi -if $data04 != @70-01-01 07:59:57.000@ then - return -1 -endi -if $data05 != 1 then - return -1 -endi -if $data10 != @21-05-11 10:12:23.000@ then - return -1 -endi -if $data11 != 22 then - return -1 -endi -if $data12 != 23.000000000 then - print $data02 - return -1 -endi -if $data13 != NULL then - return -1 -endi -if $data14 != @70-01-01 07:59:58.-04@ then - return -1 -endi -if $data15 != 2 then - return -1 -endi -if $data20 != @21-05-10 10:12:24.000@ then - return -1 -endi -if $data21 != 24 then - return -1 -endi -if $data22 != NULL then - print expect NULL actual: $data22 - return -1 -endi -if $data23 != 25 then - return -1 -endi -if $data24 != @70-01-01 07:59:57.-04@ then = - return -1 -endi -if $data25 != 3 then - return -1 -endi -if $data30 != @21-05-11 10:12:25.000@ then - return -1 -endi -if $data31 != 26 then - return -1 -endi -if $data32 != NULL then - print $data02 - return -1 -endi -if $data33 != 27 then - return -1 -endi -if $data34 != @70-01-01 07:59:56.-04@ then - return -1 -endi -if $data35 != 4 then - return -1 -endi -if $data40 != @21-05-11 10:12:29.000@ then - return -1 -endi -if $data41 != 36 then - return -1 -endi -if $data42 != 37.000000000 then - print $data02 - return -1 -endi -if $data43 != NULL then - return -1 -endi -if $data44 != @70-01-01 07:59:56.-05@ then - return -1 -endi -if $data45 != 5 then - return -1 -endi - print "test tbn" sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) diff --git a/tests/script/tsim/query/interval.sim b/tests/script/tsim/query/interval.sim index 135fcc8591..7f950ea69c 100644 --- a/tests/script/tsim/query/interval.sim +++ b/tests/script/tsim/query/interval.sim @@ -226,4 +226,135 @@ print =============== clear # return -1 #endi +print ================= step12 + +sql create database test2 vgroups 4; +sql use test2; +sql create stable stb (ts timestamp, c1 int) tags (t1 int); +sql create table t1 using stb tags (1); +sql create table t2 using stb tags (2); +sql create table t3 using stb tags (3); +sql create table t4 using stb tags (4); +sql create table t5 using stb tags (4); +sql create table t6 using stb tags (4); +sql insert into t1 values ("2024-03-01 14:29:07.051", 11); +sql insert into t2 values ("2024-03-01 14:29:07.051", 21); +sql insert into t3 values ("2024-03-01 14:29:07.051", 31); +sql insert into t4 values ("2024-03-01 14:29:07.051", 41); +sql insert into t5 values ("2024-03-01 14:29:07.051", 51); +sql insert into t6 values ("2024-03-01 14:29:07.051", 61); +sql insert into t1 values ("2024-03-01 14:30:07.051", 12); +sql insert into t2 values ("2024-03-01 14:30:07.051", 22); +sql insert into t3 values ("2024-03-01 14:30:07.051", 32); +sql insert into t4 values ("2024-03-01 14:30:07.051", 42); +sql insert into t5 values ("2024-03-01 14:30:07.051", 52); +sql insert into t6 values ("2024-03-01 14:30:07.051", 62); +sql insert into t1 values ("2024-03-01 14:31:07.051", 13); +sql insert into t2 values ("2024-03-01 14:31:07.051", 23); +sql insert into t3 values ("2024-03-01 14:31:07.051", 33); +sql insert into t4 values ("2024-03-01 14:31:07.051", 43); +sql insert into t5 values ("2024-03-01 14:31:07.051", 53); +sql insert into t6 values ("2024-03-01 14:31:07.051", 63); +sql insert into t1 values ("2024-03-01 14:32:07.051", 14); +sql insert into t2 values ("2024-03-01 14:32:07.051", 24); +sql insert into t3 values ("2024-03-01 14:32:07.051", 34); +sql insert into t4 values ("2024-03-01 14:32:07.051", 44); +sql insert into t5 values ("2024-03-01 14:32:07.051", 54); +sql insert into t6 values ("2024-03-01 14:32:07.051", 64); +sql insert into t1 values ("2024-03-01 14:33:07.051", 15); +sql insert into t2 values ("2024-03-01 14:33:07.051", 25); +sql insert into t3 values ("2024-03-01 14:33:07.051", 35); +sql insert into t4 values ("2024-03-01 14:33:07.051", 45); +sql insert into t5 values ("2024-03-01 14:33:07.051", 55); +sql insert into t6 values ("2024-03-01 14:33:07.051", 65); +sql insert into t1 values ("2024-03-01 14:34:07.051", 16); +sql insert into t2 values ("2024-03-01 14:34:07.051", 26); +sql insert into t3 values ("2024-03-01 14:34:07.051", 36); +sql insert into t4 values ("2024-03-01 14:34:07.051", 46); +sql insert into t5 values ("2024-03-01 14:34:07.051", 56); +sql insert into t6 values ("2024-03-01 14:34:07.051", 66); + +sleep 300 + +sql select _wstart, count(*) from (select * from stb partition by tbname) interval(2s); + +print $data00,$data01 +print $data10,$data11 +print $data20,$data21 +print $data30,$data31 +print $data40,$data41 +print $data50,$data51 +print $data60,$data61 +print $data70,$data71 + +if $rows != 6 then + print $rows + return -1 +endi + +if $data01 != 6 then + print $data01 +endi + +if $data11 != 6 then + print $data11 +endi + +if $data21 != 6 then + print $data21 +endi + +if $data31 != 6 then + print $data31 +endi + +if $data41 != 6 then + print $data41 +endi + +if $data51 != 6 then + print $data51 +endi + + +sql select _wstart, count(*) from (select * from stb partition by tbname slimit 2) interval(2s); + +print $data00,$data01 +print $data10,$data11 +print $data20,$data21 +print $data30,$data31 +print $data40,$data41 +print $data50,$data51 +print $data60,$data61 +print $data70,$data71 + +if $rows != 6 then + print $rows then + return -1 +endi + +if $data01 != 2 then + print $data01 +endi + +if $data11 != 2 then + print $data11 +endi + +if $data21 != 2 then + print $data21 +endi + +if $data31 != 2 then + print $data31 +endi + +if $data41 != 2 then + print $data41 +endi + +if $data51 != 2 then + print $data51 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/query_count1.sim b/tests/script/tsim/query/query_count1.sim index 0694ab062a..043b604263 100644 --- a/tests/script/tsim/query/query_count1.sim +++ b/tests/script/tsim/query/query_count1.sim @@ -79,5 +79,27 @@ if $data22 != 4 then goto loop3 endi + +print step2 +print =============== create database +sql create database test1 vgroups 1; +sql use test1; + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +#2~INT32_MAX +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(-1); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(0); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(1); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(2147483648); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(10, 0); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(10, -1); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(10, 11); + +sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(2); +sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(2147483647); + print query_count0 end system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/pauseAndResume.sim b/tests/script/tsim/stream/pauseAndResume.sim index 7c62eaed81..9a05e645c2 100644 --- a/tests/script/tsim/stream/pauseAndResume.sim +++ b/tests/script/tsim/stream/pauseAndResume.sim @@ -8,16 +8,19 @@ sql connect print ===== step1 sql drop stream if exists streams1; sql drop database if exists test; -sql create database test vgroups 10; +sql create database test vgroups 3; sql use test; sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); + sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt1 as select _wstart, count(*) c1, sum(a) c3 from st interval(10s); sleep 2000 +sql_error create stream stream1_same_dst into streamt1 as select _wstart, count(*) c1, sum(a) c3 from st interval(10s); + sql pause stream streams1; sql insert into ts1 values(1648791213001,1,12,3,1.0); @@ -102,6 +105,11 @@ if $data11 != 4 then goto loop1 endi +print ===== idle for 70 sec for checkpoint gen +sleep 70000 + +print ===== idle 60 sec completed , continue + print ===== step 1 over print ===== step2 @@ -113,6 +121,12 @@ sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt2 as select _wstart, count(*) c1, sum(a) c3 from t1 interval(10s); +# duplicate stream +sql_error create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt2 as select _wstart, count(*) c1, sum(a) c3 from t1 interval(10s); +sql create stream if not exists streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt2 as select _wstart, count(*) c1, sum(a) c3 from t1 interval(10s); + +sleep 1000 + sql pause stream streams2; sql insert into t1 values(1648791213001,1,12,3,1.0); @@ -237,8 +251,9 @@ print ===== step 2 over print ===== step3 sql drop stream if exists streams3; sql drop database if exists test3; -sql create database test3 vgroups 10; +sql create database test3 vgroups 3; sql use test3; + sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim index 0d96d0841b..37081ad1ae 100644 --- a/tests/script/tsim/stream/state0.sim +++ b/tests/script/tsim/stream/state0.sim @@ -21,6 +21,8 @@ print create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); +sleep 1000 + sql insert into t1 values(1648791213000,1,2,3,1.0,1); sql insert into t1 values(1648791213000,1,2,3,1.0,2); $loop_count = 0 @@ -457,6 +459,8 @@ print create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); +sleep 1000 + sql insert into t1 values(1648791212000,2,2,3,1.0,1); sql insert into t1 values(1648791213000,1,2,3,1.0,1); sql insert into t1 values(1648791213000,1,2,4,1.0,2); @@ -504,6 +508,9 @@ sql create table t1(ts timestamp, a int, b int , c int, d double, id int); print create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart, count(*) c1, sum(b) c3 from t1 state_window(a); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart, count(*) c1, sum(b) c3 from t1 state_window(a); + +sleep 1000 + sql insert into t1 values(1648791212000,1,2,3,1.0,1); sql insert into t1 values(1648791213000,2,2,3,1.0,1); sql insert into t1 values(1648791214000,3,2,4,1.0,2); @@ -557,6 +564,8 @@ print create stream if not exists streams4 trigger window_close IGNORE EXPIRED 0 sql create stream if not exists streams4 trigger window_close IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart AS startts, min(c1),count(c1) from t1 state_window(c1); +sleep 1000 + sql insert into t1 (ts, c1) values (1668073288209, 11); sql insert into t1 (ts, c1) values (1668073288210, 11); sql insert into t1 (ts, c1) values (1668073288211, 11); @@ -745,6 +754,9 @@ sql create table b (c timestamp, d int, e int , f int, g double); print create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a); + +sleep 1000 + sql insert into b values(1648791213000,NULL,NULL,NULL,NULL); sql select * from streamt order by c1, c2, c3; diff --git a/tests/system-test/0-others/com_alltypedata.json b/tests/system-test/0-others/com_alltypedata.json index 0e6d8e3a07..1499ca7670 100644 --- a/tests/system-test/0-others/com_alltypedata.json +++ b/tests/system-test/0-others/com_alltypedata.json @@ -22,7 +22,7 @@ "vgroups": 2, "replica": 1, "precision": "ms", - "stt_trigger": 8, + "stt_trigger": 1, "minRows": 100, "maxRows": 4096 }, diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index c936cf1ae4..8163177a3b 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -1,11 +1,13 @@ from urllib.parse import uses_relative import taos +import taosws import sys import os import time import platform import inspect from taos.tmq import Consumer +from taos.tmq import * from pathlib import Path from util.log import * @@ -17,7 +19,7 @@ from util.dnodes import TDDnode from util.cluster import * import subprocess -BASEVERSION = "3.0.2.3" +BASEVERSION = "3.2.0.0" class TDTestCase: def caseDescription(self): f''' @@ -30,7 +32,7 @@ class TDTestCase: self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 4; ;use deldata; + self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 1; ;use deldata; create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int); create table deldata.ct1 using deldata.stb1 tags ( 1 ); insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); @@ -104,8 +106,19 @@ class TDTestCase: print(f"{packageName} has been exists") os.system(f" cd {packagePath} && tar xvf {packageName} && cd {packageTPath} && ./install.sh -e no " ) tdDnodes.stop(1) - print(f"start taosd: rm -rf {dataPath}/* && nohup taosd -c {cPath} & ") - os.system(f"rm -rf {dataPath}/* && nohup taosd -c {cPath} & " ) + print(f"start taosd: rm -rf {dataPath}/* && nohup /usr/bin/taosd -c {cPath} & ") + os.system(f"rm -rf {dataPath}/* && nohup /usr/bin/taosd -c {cPath} & " ) + os.system(f"killall taosadapter" ) + os.system(f"cp /etc/taos/taosadapter.toml {cPath}/taosadapter.toml " ) + taosadapter_cfg = cPath + "/taosadapter.toml" + taosadapter_log_path = cPath + "/../log/" + print(f"taosadapter_cfg:{taosadapter_cfg},taosadapter_log_path:{taosadapter_log_path} ") + self.alter_string_in_file(taosadapter_cfg,"#path = \"/var/log/taos\"",f"path = \"{taosadapter_log_path}\"") + self.alter_string_in_file(taosadapter_cfg,"taosConfigDir = \"\"",f"taosConfigDir = \"{cPath}\"") + print("/usr/bin/taosadapter --version") + os.system(f" /usr/bin/taosadapter --version" ) + print(f" LD_LIBRARY_PATH=/usr/lib -c {taosadapter_cfg} 2>&1 & ") + os.system(f" LD_LIBRARY_PATH=/usr/lib /usr/bin/taosadapter -c {taosadapter_cfg} 2>&1 & " ) sleep(5) @@ -116,7 +129,24 @@ class TDTestCase: def is_list_same_as_ordered_list(self,unordered_list, ordered_list): sorted_list = sorted(unordered_list) return sorted_list == ordered_list - + + def alter_string_in_file(self,file,old_str,new_str): + """ + replace str in file + :param file + :param old_str + :param new_str + :return: + """ + file_data = "" + with open(file, "r", encoding="utf-8") as f: + for line in f: + if old_str in line: + line = line.replace(old_str,new_str) + file_data += line + with open(file,"w",encoding="utf-8") as f: + f.write(file_data) + def run(self): scriptsPath = os.path.dirname(os.path.realpath(__file__)) distro_id = distro.id() @@ -131,7 +161,7 @@ class TDTestCase: dbname = "test" stb = f"{dbname}.meters" self.installTaosd(bPath,cPath) - os.system("echo 'debugFlag 143' > /etc/taos/taos.cfg ") + # os.system(f"echo 'debugFlag 143' >> {cPath}/taos.cfg ") tableNumbers=100 recordNumbers1=100 recordNumbers2=1000 @@ -163,11 +193,46 @@ class TDTestCase: # os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ") # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ') # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show streams;" ') - os.system(f"sed -i 's/\/etc\/taos/{cPath}/' 0-others/tmqBasic.json ") + self.alter_string_in_file("0-others/tmqBasic.json", "/etc/taos/", cPath) # os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/tmqBasic.json -y ") os.system('LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists tmq_test_topic as select current,voltage,phase from test.meters where voltage <= 106 and current <= 5;" ') os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show topics;" ') + os.system(f" /usr/bin/taosadapter --version " ) + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + } + consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) + try: + consumer.subscribe(["tmq_test_topic"]) + except TmqError: + tdLog.exit(f"subscribe error") + + while True: + message = consumer.poll(timeout=1.0) + if message: + print("message") + id = message.vgroup() + topic = message.topic() + database = message.database() + + for block in message: + nrows = block.nrows() + ncols = block.ncols() + for row in block: + print(row) + values = block.fetchall() + print(nrows, ncols) + + consumer.commit(message) + else: + print("break") + break + + consumer.close() tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") @@ -184,7 +249,8 @@ class TDTestCase: os.system("pkill taosd") # make sure all the data are saved in disk. self.checkProcessPid("taosd") - + os.system("pkill taosadapter") # make sure all the data are saved in disk. + self.checkProcessPid("taosadapter") tdLog.printNoPrefix("==========step2:update new version ") self.buildTaosd(bPath) @@ -193,6 +259,7 @@ class TDTestCase: tdsql=tdCom.newTdSql() print(tdsql) cmd = f" LD_LIBRARY_PATH=/usr/lib taos -h localhost ;" + print(os.system(cmd)) if os.system(cmd) == 0: raise Exception("failed to execute system command. cmd: %s" % cmd) diff --git a/tests/system-test/7-tmq/tmqParamsTest.py b/tests/system-test/7-tmq/tmqParamsTest.py index 82a5d42b47..b25f23ef11 100644 --- a/tests/system-test/7-tmq/tmqParamsTest.py +++ b/tests/system-test/7-tmq/tmqParamsTest.py @@ -164,7 +164,7 @@ class TDTestCase: offset_value_list = list(map(lambda x: (x[-2].replace("wal:", "").replace("earliest", "0").replace("latest", "0").replace(offset_value, "0")), subscription_info)) offset_value_list1 = list(map(lambda x: int(x.split("/")[0]), offset_value_list)) offset_value_list2 = list(map(lambda x: int(x.split("/")[1]), offset_value_list)) - tdSql.checkEqual(offset_value_list1 == offset_value_list2, True) + tdSql.checkEqual(offset_value_list1 <= offset_value_list2, True) tdSql.checkEqual(sum(offset_value_list1) >= 0, True) rows_value_list = list(map(lambda x: int(x[-1]), subscription_info)) tdSql.checkEqual(sum(rows_value_list), expected_res) @@ -187,4 +187,4 @@ class TDTestCase: event = threading.Event() tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index c547385d70..0330454b73 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -359,8 +359,101 @@ class TDTestCase: finally: consumer.close() + def consume_TS_4540_Test(self): + tdSql.execute(f'create database if not exists test') + tdSql.execute(f'use test') + tdSql.execute(f'CREATE STABLE `test`.`b` ( `time` TIMESTAMP , `task_id` NCHAR(1000) ) TAGS( `key` NCHAR(1000))') + tdSql.execute(f"insert into `test`.b1 using `test`.`b`(`key`) tags('1') (time, task_id) values ('2024-03-04 12:50:01.000', '32') `test`.b2 using `test`.`b`(`key`) tags('2') (time, task_id) values ('2024-03-04 12:50:01.000', '43') `test`.b3 using `test`.`b`(`key`) tags('3') (time, task_id) values ('2024-03-04 12:50:01.000', '123456')") + + tdSql.execute(f'create topic tt as select tbname,task_id,`key` from b') + + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + } + consumer = Consumer(consumer_dict) + + try: + consumer.subscribe(["tt"]) + except TmqError: + tdLog.exit(f"subscribe error") + + try: + while True: + res = consumer.poll(1) + if not res: + break + val = res.value() + if val is None: + continue + for block in val: + data = block.fetchall() + print(data) + if data != [('b1', '32', '1')] and data != [('b2', '43', '2')] and data != [('b3', '123456', '3')]: + tdLog.exit(f"index = 0 table b1 error") + + finally: + consumer.close() + + def consume_ts_4544(self): + tdSql.execute(f'create database if not exists d1') + tdSql.execute(f'use d1') + tdSql.execute(f'create table stt(ts timestamp, i int) tags(t int)') + tdSql.execute(f'insert into tt1 using stt tags(1) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into tt2 using stt tags(2) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into tt3 using stt tags(3) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into tt1 using stt tags(1) values(now+5s, 11) (now+10s, 12)') + + tdSql.execute(f'create topic topic_in as select * from stt where tbname in ("tt2")') + + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + } + consumer = Consumer(consumer_dict) + + try: + consumer.subscribe(["topic_in"]) + except TmqError: + tdLog.exit(f"subscribe error") + + consumer.close() + + def consume_ts_4551(self): + tdSql.execute(f'use d1') + + tdSql.execute(f'create topic topic_stable as stable stt where tbname like "t%"') + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + } + consumer = Consumer(consumer_dict) + + try: + consumer.subscribe(["topic_stable"]) + except TmqError: + tdLog.exit(f"subscribe error") + + try: + while True: + res = consumer.poll(1) + if not res: + break + finally: + consumer.close() + print("consume_ts_4551 ok") + def run(self): self.consumeTest() + self.consume_ts_4544() + self.consume_ts_4551() + self.consume_TS_4540_Test() tdSql.prepare() self.checkWal1VgroupOnlyMeta() diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 2257089f06..5012e50bab 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -574,6 +574,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "msg.consume.excluded", "1"); if (g_conf.snapShot) { tmq_conf_set(conf, "experimental.snapshot.enable", "true");