xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(stream): add new trigger continuous_window_close (#30125) 

* opt stream build twa result

* opt force window close memory

* feat(stream):optimize new interval and scan operator

* adj log

* opt code

* opt code

* fill history

* fix issue for fill history

* add ci

* feat(stream): add new stream nonblock interval operator

* adjust code

* use new scan operator

* use new scan operator

* add log

* fix issue

* recover stream scan next function

* fix issue

* fix issue

* fix issue

* ignore disorder data

* fix issue

* fix issue for interval sliding

* fix issue

* fix ci issue

* fix ci issue

* fix ci issue

* add semi && final nonblock interval operator

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* refactor(stream): track the msgId for each upstream tasks.

* fix(stream): fix race condition.

* fix(stream): update the task last msgId when putting into input queue succ.

* fix issue

* fix issue

* put recalculate data to rocksdb

* fix issue

* fix issue

* enh(query)[TD-33071]: add support for saving and restoring tsdbReader scan progress

- Implement functionality to save scan progress during tsdbReader operations
- Enable resuming scans from the last saved position

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* add rec interval check

* enh(stream):add recalculate tasks.

* enh(stream): support the re-calculate the tasks.

* fix issue && do refactor

* do refactor

* fix issue

* fix issue

* update backend opt

* add new interface

* add new session operator

* support blob

* add new session operator

* fix issue

* add rec state for rec task

* fix invalid read

* add new session window

* enh(stream): update the stream tasks backend.

* new session operator

* add pull data

* fix(stream): fix error in expand stream backend.

* fix issue

* fix issue

* fix issue

* merge code

* fix issue

* fix(stream): check for null ptr.

* fix(stream): add more check.

* fix issue

* fix issue

* fix issue

* add debug code

* fix issue

* fix issue

* fix issue

* set rec end flag

* fix(stream): opt re-calculate stream tasks.

* fix issue

* fix issue

* add new operator

* enh(stream): dispatch recalculate block to agg tasks.

* fix issue

* fix issue

* fix(stream): adjust filter.

* fix issue

* refactor

* refactor(stream): adjust the recalculate end block.

* fix issue

* fix(stream): set correct create trigger block.

* fix issue

* fix(stream): fix error in build recalculate end block.

* fix(stream): check null ptr.

* add stream client && fix issue

* fix mem leak

* fix(stream): free msg.

* add stream client

* fix(stream): fix error.

* add stream client && fix issue

* add stream client

* refactor(stream): set the recalculate task complete.

* add wend and group_id for session window dest table

* feat(stream): refactor and set the recalcul agg task complete.

* add cfg for adapter

* fix issue

* add state && event operator

* feat(stream): support fill-history task.

* add new fill operator

* fix(stream): set correct backend when updating fill-history task to recalculate task.

* add new fill operator

* fix(stream): s2025-03-06 11:10:31.272
et ops always open in scan tsdb

* fix(stream):set the correct taskType for sink task.

* new fill operator

* adj stream fill operator

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* adj test

* fix issue

* fix(stream): fix issue

* fix(steam): fix issue

* fix(steam): fix issue

* fix(steam): fix issue

* fix(steam): fix issue

* fix(stream): fix issue

* fix(stream): fix issue

* fix(stream): fix issue

* fix: ut com error

* fix(stream): fix mem leak and adjust operator type check rule

* fix(stream): fix mem leak and adjust test case

* refactor code

* fix(stream): free items.

* fix(stream): free fix memory leak.

* fix(stream): fix syntax error.

* fix: ignore unexpect block

* fix: adjust op type

---------

Co-authored-by: Haojun Liao <hjliao@taosdata.com>
Co-authored-by: Jinqing Kuang <kuangjinqingcn@gmail.com>
Co-authored-by: yihaoDeng <luomoxyz@126.com>
This commit is contained in:
liuyao 2025-03-14 20:14:01 +08:00 committed by GitHub
parent f50fcb85b8
commit 2de6736160
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
102 changed files with 10538 additions and 956 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/common/streamMsg.h
View File

@ -40,6 +40,7 @@ typedef struct SStreamUpstreamEpInfo {
SEpSet epSet;
bool dataAllowed; // denote if the data from this upstream task is allowed to put into inputQ, not serialize it
int64_t stage; // upstream task stage value, to denote if the upstream node has restart/replica changed/transfer
int64_t lastMsgId;
} SStreamUpstreamEpInfo;
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo);

6
include/common/tcommon.h
View File

@ -132,7 +132,7 @@ enum {
STREAM_INPUT__DATA_SUBMIT = 1,
STREAM_INPUT__DATA_BLOCK,
STREAM_INPUT__MERGED_SUBMIT,
STREAM_INPUT__TQ_SCAN,
STREAM_INPUT__RECALCULATE,
STREAM_INPUT__DATA_RETRIEVE,
STREAM_INPUT__GET_RES,
STREAM_INPUT__CHECKPOINT,
@ -162,6 +162,10 @@ typedef enum EStreamType {
STREAM_GET_RESULT,
STREAM_DROP_CHILD_TABLE,
STREAM_NOTIFY_EVENT,
STREAM_RECALCULATE_DATA,
STREAM_RECALCULATE_DELETE,
STREAM_RECALCULATE_START,
STREAM_RECALCULATE_END,
} EStreamType;
#pragma pack(push, 1)

4
include/common/tglobal.h
View File

@ -303,6 +303,10 @@ extern int32_t tsStreamNotifyMessageSize;
extern int32_t tsStreamNotifyFrameSize;
extern bool tsCompareAsStrInGreatest;
extern char tsAdapterFqdn[];
extern uint16_t tsAdapterPort;
extern char tsAdapterToken[];
extern bool tsExperimental;
// #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)

14
include/common/tmsg.h
View File

@ -485,6 +485,14 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC,
QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERP_FUNC,
QUERY_NODE_RESET_STREAM_STMT,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_STATE,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_EVENT,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_COUNT,
} ENodeType;
typedef struct {
@ -2961,6 +2969,7 @@ typedef struct {
#define STREAM_TRIGGER_WINDOW_CLOSE 2
#define STREAM_TRIGGER_MAX_DELAY 3
#define STREAM_TRIGGER_FORCE_WINDOW_CLOSE 4
#define STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE 5
#define STREAM_DEFAULT_IGNORE_EXPIRED 1
#define STREAM_FILL_HISTORY_ON 1
@ -3013,6 +3022,11 @@ typedef struct {
int32_t notifyEventTypes;
int32_t notifyErrorHandle;
int8_t notifyHistory;
int64_t recalculateInterval;
char pWstartName[TSDB_COL_NAME_LEN];
char pWendName[TSDB_COL_NAME_LEN];
char pGroupIdName[TSDB_COL_NAME_LEN];
char pIsWindowFilledName[TSDB_COL_NAME_LEN];
} SCMCreateStreamReq;
typedef struct STaskNotifyEventStat {

4
include/libs/executor/executor.h
View File

@ -53,6 +53,7 @@ typedef struct {
int32_t numOfVgroups;
void* sContext; // SSnapContext*
void* pStateBackend;
void* pOtherBackend;
int8_t fillHistory;
STimeWindow winRange;
@ -219,7 +220,6 @@ const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo);
const char* qExtractTbnameFromTask(qTaskInfo_t tinfo);
void* qExtractReaderFromStreamScanner(void* scanner);
void qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner);
int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo);
@ -229,7 +229,7 @@ int32_t qStreamRecoverFinish(qTaskInfo_t tinfo);
bool qStreamScanhistoryFinished(qTaskInfo_t tinfo);
int32_t qStreamInfoResetTimewindowFilter(qTaskInfo_t tinfo);
void qResetTaskInfoCode(qTaskInfo_t tinfo);
int32_t qGetStreamIntervalExecInfo(qTaskInfo_t tinfo, int64_t* pWaterMark, SInterval* pInterval, STimeWindow* pLastWindow);
int32_t qGetStreamIntervalExecInfo(qTaskInfo_t tinfo, int64_t* pWaterMark, SInterval* pInterval, STimeWindow* pLastWindow, TSKEY* pRecInteral);
int32_t qStreamOperatorReleaseState(qTaskInfo_t tInfo);
int32_t qStreamOperatorReloadState(qTaskInfo_t tInfo);

81
include/libs/executor/storageapi.h
View File

@ -127,6 +127,7 @@ typedef struct SRowBuffPos {
bool beUsed;
bool needFree;
bool beUpdated;
bool invalid;
} SRowBuffPos;
// tq
@ -208,6 +209,8 @@ typedef struct TsdReader {
int32_t (*fileSetGetEntryField)(struct SFileSetReader *, const char *, void *);
void (*fileSetReaderClose)(struct SFileSetReader **);
int32_t (*getProgress)(const void* pReader, void** pBuf, uint64_t* pLen);
int32_t (*setProgress)(void *pReader, const void *pBuf, uint64_t len);
} TsdReader;
typedef struct SStoreCacheReader {
@ -326,7 +329,7 @@ typedef struct SUpdateInfo {
TSKEY minTS;
SScalableBf* pCloseWinSBF;
SHashObj* pMap;
uint64_t maxDataVersion;
int64_t maxDataVersion;
int8_t pkColType;
int32_t pkColLen;
char* pKeyBuff;
@ -336,6 +339,27 @@ typedef struct SUpdateInfo {
__compar_fn_t comparePkCol;
} SUpdateInfo;
typedef struct SRecDataInfo {
STimeWindow calWin;
uint64_t tableUid;
int64_t dataVersion;
EStreamType mode;
char pPkColData[];
} SRecDataInfo;
typedef struct SScanRange {
STimeWindow win;
STimeWindow calWin;
SSHashObj* pGroupIds;
SSHashObj* pUIds;
} SScanRange;
typedef struct SResultWindowInfo {
SRowBuffPos* pStatePos;
SSessionKey sessionWin;
bool isOutput;
} SResultWindowInfo;
typedef struct {
void* iter; // rocksdb_iterator_t* iter;
void* snapshot; // rocksdb_snapshot_t* snapshot;
@ -350,23 +374,44 @@ typedef struct {
int64_t minGpId;
} SStreamStateCur;
typedef struct STableTsDataState {
SSHashObj* pTableTsDataMap;
__compar_fn_t comparePkColFn;
void* pPkValBuff;
int32_t pkValLen;
SStreamState* pState;
int32_t curRecId;
void* pStreamTaskState;
SArray* pScanRanges;
SRecDataInfo* pRecValueBuff;
int32_t recValueLen;
SStreamStateCur* pRecCur;
int32_t cfgIndex;
void* pBatch;
int32_t batchBufflen;
void* pBatchBuff;
} STableTsDataState;
typedef struct SStateStore {
int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache,
int32_t* pWinCode);
int32_t (*streamStateDeleteParName)(SStreamState* pState, int64_t groupId);
void (*streamStateSetParNameInvalid)(SStreamState* pState);
int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* pKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
void (*streamStateReleaseBuf)(SStreamState* pState, void* pVal, bool used);
void (*streamStateClearBuff)(SStreamState* pState, void* pVal);
void (*streamStateFreeVal)(void* val);
int32_t (*streamStateGetPrev)(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal,
int32_t* pVLen, int32_t* pWinCode);
int32_t (*streamStateGetAllPrev)(SStreamState* pState, const SWinKey* pKey, SArray* pResArray, int32_t maxNum);
int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key);
bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key, bool hasLimit, bool* pIsLast);
bool (*streamStateCheckSessionState)(SStreamState* pState, SSessionKey* pKey, TSKEY gap, bool* pIsLast);
int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal);
void (*streamStateDel)(SStreamState* pState, const SWinKey* key);
void (*streamStateDelByGroupId)(SStreamState* pState, uint64_t groupId);
@ -374,6 +419,8 @@ typedef struct SStateStore {
void (*streamStateSetNumber)(SStreamState* pState, int32_t number, int32_t tsIdex);
void (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
int32_t (*streamStateGetNumber)(SStreamState* pState);
int32_t (*streamStateDeleteInfo)(SStreamState* pState, void* pKey, int32_t keyLen);
int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
@ -398,7 +445,10 @@ typedef struct SStateStore {
int32_t (*streamStateFillGetGroupKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
void (*streamStateClearExpiredState)(SStreamState* pState);
void (*streamStateClearExpiredState)(SStreamState* pState, int32_t numOfKeep, TSKEY minTs);
void (*streamStateClearExpiredSessionState)(SStreamState* pState, int32_t numOfKeep, TSKEY minTs, SSHashObj* pFlushGroup);
int32_t (*streamStateSetRecFlag)(SStreamState* pState, const void* pKey, int32_t keyLen, int32_t mode);
int32_t (*streamStateGetRecFlag)(SStreamState* pState, const void* pKey, int32_t keyLen, int32_t* pMode);
int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal,
int32_t* pVLen, int32_t* pWinCode);
@ -415,6 +465,9 @@ typedef struct SStateStore {
int32_t (*streamStateCountGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t (*streamStateSessionAllocWinBuffByNextPosition)(SStreamState* pState, SStreamStateCur* pCur,
const SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*streamStateSessionSaveToDisk)(STableTsDataState* pTblState, SSessionKey* pKey, SRecDataInfo* pVal, int32_t vLen);
int32_t (*streamStateFlushReaminInfoToDisk)(STableTsDataState* pTblState);
int32_t (*streamStateSessionDeleteAll)(SStreamState* pState);
int32_t (*streamStateCountWinAddIfNotExist)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount,
void** ppVal, int32_t* pVLen, int32_t* pWinCode);
@ -460,12 +513,32 @@ typedef struct SStateStore {
bool (*needClearDiskBuff)(struct SStreamFileState* pFileState);
SStreamState* (*streamStateOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId);
SStreamState* (*streamStateRecalatedOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void (*streamStateClose)(SStreamState* pState, bool remove);
int32_t (*streamStateBegin)(SStreamState* pState);
void (*streamStateCommit)(SStreamState* pState);
void (*streamStateDestroy)(SStreamState* pState, bool remove);
void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts);
void (*streamStateCopyBackend)(SStreamState* src, SStreamState* dst);
int32_t (*streamStateGetAndSetTsData)(STableTsDataState* pState, uint64_t tableUid, TSKEY* pCurTs, void** ppCurPkVal,
TSKEY lastTs, void* pLastPkVal, int32_t lastPkLen, int32_t* pWinCode);
int32_t (*streamStateTsDataCommit)(STableTsDataState* pState);
int32_t (*streamStateInitTsDataState)(STableTsDataState** ppTsDataState, int8_t pkType, int32_t pkLen, void* pState, void* pOtherState);
void (*streamStateDestroyTsDataState)(STableTsDataState* pTsDataState);
int32_t (*streamStateRecoverTsData)(STableTsDataState* pTsDataState);
int32_t (*streamStateReloadTsDataState)(STableTsDataState* pTsDataState);
int32_t (*streamStateMergeAndSaveScanRange)(STableTsDataState* pTsDataState, STimeWindow* pWin, uint64_t gpId,
SRecDataInfo* pRecData, int32_t len);
int32_t (*streamStateMergeAllScanRange)(STableTsDataState* pTsDataState);
int32_t (*streamStatePopScanRange)(STableTsDataState* pTsDataState, SScanRange* pRange);
SStreamStateCur* (*streamStateGetLastStateCur)(SStreamState* pState);
void (*streamStateLastStateCurNext)(SStreamStateCur* pCur);
int32_t (*streamStateNLastStateGetKVByCur)(SStreamStateCur* pCur, int32_t num, SArray* pRes);
SStreamStateCur* (*streamStateGetLastSessionStateCur)(SStreamState* pState);
void (*streamStateLastSessionStateCurNext)(SStreamStateCur* pCur);
int32_t (*streamStateNLastSessionStateGetKVByCur)(SStreamStateCur* pCur, int32_t num, SArray* pRes);
} SStateStore;
typedef struct SStorageAPI {

13
include/libs/function/function.h
View File

@ -20,10 +20,10 @@
extern "C" {
#endif
#include "functionResInfo.h"
#include "tcommon.h"
#include "tsimplehash.h"
#include "tvariant.h"
#include "functionResInfo.h"
struct SqlFunctionCtx;
struct SResultRowEntryInfo;
@ -44,7 +44,8 @@ typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock);
typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
typedef int32_t (*FExecCombine)(struct SqlFunctionCtx *pDestCtx, struct SqlFunctionCtx *pSourceCtx);
typedef int32_t (*FExecDecode)(struct SqlFunctionCtx *pCtx, const char *buf, struct SResultRowEntryInfo *pResultCellInfo, int32_t version);
typedef int32_t (*FExecDecode)(struct SqlFunctionCtx *pCtx, const char *buf,
struct SResultRowEntryInfo *pResultCellInfo, int32_t version);
typedef int32_t (*processFuncByRow)(SArray *pCtx); // array of SqlFunctionCtx
typedef struct SScalarFuncExecFuncs {
@ -168,11 +169,14 @@ typedef struct STdbState {
void *pParNameDb;
void *pParTagDb;
void *txn;
int8_t recalc;
} STdbState;
typedef struct SResultRowStore {
int32_t (*resultRowPut)(struct SExprSupp *pSup, const char* inBuf, size_t inBufSize, char **outBuf, size_t *outBufSize);
int32_t (*resultRowGet)(struct SExprSupp *pSup, const char* inBuf, size_t inBufSize, char **outBuf, size_t *outBufSize);
int32_t (*resultRowPut)(struct SExprSupp *pSup, const char *inBuf, size_t inBufSize, char **outBuf,
size_t *outBufSize);
int32_t (*resultRowGet)(struct SExprSupp *pSup, const char *inBuf, size_t inBufSize, char **outBuf,
size_t *outBufSize);
} SResultRowStore;
struct SStreamState {
@ -187,6 +191,7 @@ struct SStreamState {
int32_t tsIndex;
SResultRowStore pResultRowStore;
struct SExprSupp *pExprSupp;
char pTaskIdStr[65];
};
typedef struct SFunctionStateStore {

3
include/libs/function/functionMgt.h
View File

@ -159,6 +159,8 @@ typedef enum EFunctionType {
FUNCTION_TYPE_FORECAST_ROWTS,
FUNCTION_TYPE_COLS,
FUNCTION_TYPE_IROWTS_ORIGIN,
FUNCTION_TYPE_GROUP_ID,
FUNCTION_TYPE_IS_WINDOW_FILLED,
// internal function
FUNCTION_TYPE_SELECT_VALUE = 3750,
@ -300,6 +302,7 @@ bool fmIsElapsedFunc(int32_t funcId);
bool fmIsDBUsageFunc(int32_t funcId);
bool fmIsRowTsOriginFunc(int32_t funcId);
bool fmIsSelectColsFunc(int32_t funcId);
bool fmIsGroupIdFunc(int32_t funcId);
void getLastCacheDataType(SDataType* pType, int32_t pkBytes);
int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc);

2
include/libs/nodes/cmdnodes.h
View File

@ -57,6 +57,7 @@ extern "C" {
#define BIT_FLAG_MASK(n) (1 << n)
#define BIT_FLAG_SET_MASK(val, mask) ((val) |= (mask))
#define BIT_FLAG_UNSET_MASK(val, mask) ((val) &= ~(mask))
#define BIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0)
#define PRIVILEGE_TYPE_ALL BIT_FLAG_MASK(0)
@ -565,6 +566,7 @@ typedef struct SStreamOptions {
SNode* pDelay;
SNode* pWatermark;
SNode* pDeleteMark;
SNode* pRecInterval;
int8_t fillHistory;
int8_t ignoreExpired;
int8_t ignoreUpdate;

13
include/libs/nodes/plannodes.h
View File

@ -316,6 +316,12 @@ typedef enum EWindowAlgorithm {
SESSION_ALGO_STREAM_SINGLE,
SESSION_ALGO_MERGE,
INTERVAL_ALGO_STREAM_MID,
INTERVAL_ALGO_STREAM_CONTINUE_SINGLE,
INTERVAL_ALGO_STREAM_CONTINUE_FINAL,
INTERVAL_ALGO_STREAM_CONTINUE_SEMI,
SESSION_ALGO_STREAM_CONTINUE_SINGLE,
SESSION_ALGO_STREAM_CONTINUE_FINAL,
SESSION_ALGO_STREAM_CONTINUE_SEMI,
} EWindowAlgorithm;
typedef struct SWindowLogicNode {
@ -347,6 +353,7 @@ typedef struct SWindowLogicNode {
SNodeList* pTsmaSubplans;
SNode* pAnomalyExpr;
char anomalyOpt[TSDB_ANALYTIC_ALGO_OPTION_LEN];
int64_t recalculateInterval;
} SWindowLogicNode;
typedef struct SFillLogicNode {
@ -508,6 +515,11 @@ typedef struct STableScanPhysiNode {
bool needCountEmptyTable;
bool paraTablesSort;
bool smallDataTsSort;
char pStbFullName[TSDB_TABLE_FNAME_LEN];
char pWstartName[TSDB_COL_NAME_LEN];
char pWendName[TSDB_COL_NAME_LEN];
char pGroupIdName[TSDB_COL_NAME_LEN];
char pIsWindowFilledName[TSDB_COL_NAME_LEN];
} STableScanPhysiNode;
typedef STableScanPhysiNode STableSeqScanPhysiNode;
@ -683,6 +695,7 @@ typedef struct SWindowPhysiNode {
int8_t igExpired;
int8_t destHasPrimaryKey;
bool mergeDataBlock;
int64_t recalculateInterval;
} SWindowPhysiNode;
typedef struct SIntervalPhysiNode {

3
include/libs/nodes/querynodes.h
View File

@ -70,7 +70,8 @@ typedef enum EColumnType {
COLUMN_TYPE_WINDOW_START,
COLUMN_TYPE_WINDOW_END,
COLUMN_TYPE_WINDOW_DURATION,
COLUMN_TYPE_GROUP_KEY
COLUMN_TYPE_GROUP_KEY,
COLUMN_TYPE_IS_WINDOW_FILLED,
} EColumnType;
typedef struct SColumnNode {

6
include/libs/planner/planner.h
View File

@ -47,6 +47,12 @@ typedef struct SPlanContext {
bool destHasPrimaryKey;
bool sourceHasPrimaryKey;
void* timezone;
int64_t recalculateInterval;
char pStbFullName[TSDB_TABLE_FNAME_LEN];
char pWstartName[TSDB_COL_NAME_LEN];
char pWendName[TSDB_COL_NAME_LEN];
char pGroupIdName[TSDB_COL_NAME_LEN];
char pIsWindowFilledName[TSDB_COL_NAME_LEN];
} SPlanContext;
// Create the physical plan for the query, according to the AST.

1
include/libs/scalar/scalar.h
View File

@ -123,6 +123,7 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t isWinFilledFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qPseudoTagFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

42
include/libs/stream/streamState.h
View File

@ -32,6 +32,7 @@ extern "C" {
#include "storageapi.h"
SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId);
SStreamState* streamStateRecalatedOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void streamStateClose(SStreamState* pState, bool remove);
int32_t streamStateBegin(SStreamState* pState);
void streamStateCommit(SStreamState* pState);
@ -43,7 +44,7 @@ int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVa
int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
bool streamStateCheck(SStreamState* pState, const SWinKey* key);
bool streamStateCheck(SStreamState* pState, const SWinKey* key, bool hasLimit, bool* pIsLast);
int32_t streamStateGetByPos(SStreamState* pState, void* pos, void** pVal);
void streamStateDel(SStreamState* pState, const SWinKey* key);
void streamStateDelByGroupId(SStreamState* pState, uint64_t groupId);
@ -51,8 +52,11 @@ void streamStateClear(SStreamState* pState);
void streamStateSetNumber(SStreamState* pState, int32_t number, int32_t tsIdex);
void streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
int32_t streamStateGetNumber(SStreamState* pState);
int32_t streamStateDeleteInfo(SStreamState* pState, void* pKey, int32_t keyLen);
int32_t streamStateGetPrev(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t streamStateGetAllPrev(SStreamState* pState, const SWinKey* pKey, SArray* pResArray, int32_t maxNum);
// session window
int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen,
@ -67,6 +71,9 @@ int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey*
int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t streamStateSessionAllocWinBuffByNextPosition(SStreamState* pState, SStreamStateCur* pCur,
const SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t streamStateSessionSaveToDisk(STableTsDataState* pTblState, SSessionKey* pKey, SRecDataInfo* pVal, int32_t vLen);
int32_t streamStateFlushReaminInfoToDisk(STableTsDataState* pTblState);
int32_t streamStateSessionDeleteAll(SStreamState* pState);
SStreamStateCur *streamStateSessionSeekKeyPrev(SStreamState *pState, const SSessionKey *key);
SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key);
@ -89,7 +96,7 @@ int32_t streamStateFillGetNext(SStreamState* pState, const SWinKey* pKey, SWinKe
int32_t streamStateFillGetPrev(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* pKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
void streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used);
void streamStateClearBuff(SStreamState* pState, void* pVal);
@ -98,7 +105,8 @@ void streamStateFreeVal(void* val);
// count window
int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal,
int32_t* pVLen, int32_t* pWinCode);
int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen);
int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal,
int32_t* pVLen);
SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key);
SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key);
@ -111,7 +119,10 @@ int32_t streamStateFillGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, con
int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
// twa
void streamStateClearExpiredState(SStreamState* pState);
void streamStateClearExpiredState(SStreamState* pState, int32_t numOfKeep, TSKEY minTs);
void streamStateClearExpiredSessionState(SStreamState* pState, int32_t numOfKeep, TSKEY minTs, SSHashObj* pFlushGroup);
int32_t streamStateSetRecFlag(SStreamState* pState, const void* pKey, int32_t keyLen, int32_t mode);
int32_t streamStateGetRecFlag(SStreamState* pState, const void* pKey, int32_t keyLen, int32_t* pMode);
void streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur);
void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
@ -119,6 +130,7 @@ void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode);
int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId);
void streamStateSetParNameInvalid(SStreamState* pState);
// group id
int32_t streamStateGroupPut(SStreamState* pState, int64_t groupId, void* value, int32_t vLen);
@ -126,6 +138,28 @@ SStreamStateCur* streamStateGroupGetCur(SStreamState* pState);
void streamStateGroupCurNext(SStreamStateCur* pCur);
int32_t streamStateGroupGetKVByCur(SStreamStateCur* pCur, int64_t* pKey, void** pVal, int32_t* pVLen);
// ts data
int32_t streamStateGetAndSetTsData(STableTsDataState* pState, uint64_t tableUid, TSKEY* pCurTs, void** ppCurPkVal,
TSKEY lastTs, void* pLastPkVal, int32_t lastPkLen, int32_t* pWinCode);
int32_t streamStateTsDataCommit(STableTsDataState* pState);
int32_t streamStateInitTsDataState(STableTsDataState** ppTsDataState, int8_t pkType, int32_t pkLen, void* pState, void* pOtherState);
void streamStateDestroyTsDataState(STableTsDataState* pTsDataState);
int32_t streamStateRecoverTsData(STableTsDataState* pTsDataState);
int32_t streamStateReloadTsDataState(STableTsDataState* pTsDataState);
int32_t streamStateMergeAndSaveScanRange(STableTsDataState* pTsDataState, STimeWindow* pWin, uint64_t gpId,
SRecDataInfo* pRecData, int32_t len);
int32_t streamStateMergeAllScanRange(STableTsDataState* pTsDataState);
int32_t streamStatePopScanRange(STableTsDataState* pTsDataState, SScanRange* pRange);
// continuous
bool streamStateCheckSessionState(SStreamState* pState, SSessionKey* pKey, TSKEY gap, bool* pIsLast);
SStreamStateCur* streamStateGetLastStateCur(SStreamState* pState);
void streamStateLastStateCurNext(SStreamStateCur* pCur);
int32_t streamStateNLastStateGetKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes);
SStreamStateCur* streamStateGetLastSessionStateCur(SStreamState* pState);
void streamStateLastSessionStateCurNext(SStreamStateCur* pCur);
int32_t streamStateNLastSessionStateGetKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes);
void streamStateReloadInfo(SStreamState* pState, TSKEY ts);
void streamStateCopyBackend(SStreamState* src, SStreamState* dst);

32
include/libs/stream/tstream.h
View File

@ -318,9 +318,10 @@ typedef struct SSTaskBasicInfo {
int32_t selfChildId;
int32_t trigger;
int8_t taskLevel;
int8_t fillHistory; // is fill history task or not
int8_t fillHistory; // enum , 1. is fill history task or not 2. recal
int64_t delaySchedParam; // in msec
int64_t watermark; // extracted from operators
int8_t hasAggTasks; // has agg tasks in current stream
SInterval interval;
} SSTaskBasicInfo;
@ -461,7 +462,6 @@ struct SStreamTask {
STaskExecStatisInfo execInfo;
TdThreadMutex lock; // secure the operation of set task status and puting data into inputQ
SMsgCb* pMsgCb; // msg handle
SStreamState* pState; // state backend
SUpstreamInfo upstreamInfo;
STaskCheckInfo taskCheckInfo;
SNotifyInfo notifyInfo;
@ -470,12 +470,18 @@ struct SStreamTask {
// the followings attributes don't be serialized
SScanhistorySchedInfo schedHistoryInfo;
int32_t transferStateAlignCnt;
int32_t recalculateAlignCnt;
struct SStreamMeta* pMeta;
SSHashObj* pNameMap;
void* pBackend;
int8_t subtableWithoutMd5; // only for tsma stream tasks
char reserve[256];
char* backendPath;
void* pBackend;
void* pRecalBackend;
SStreamState* pRecalState;
SStreamState* pState; // state backend
};
typedef int32_t (*startComplete_fn_t)(struct SStreamMeta*);
@ -558,16 +564,23 @@ typedef struct STaskUpdateEntry {
int32_t transId;
} STaskUpdateEntry;
typedef enum {
STREAM_NORMAL_TASK = 0,
STREAM_HISTORY_TASK = 1,
STREAM_RECALCUL_TASK = 2,
} EStreamTaskType;
typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param);
int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int32_t trigger,
int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, EStreamTaskType type, int32_t trigger,
int64_t triggerParam, SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5,
SStreamTask** pTask);
int8_t hasAggTasks, SStreamTask** pTask);
void tFreeStreamTask(void* pTask);
int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver);
void streamFreeTaskState(SStreamTask* pTask, int8_t remove);
int32_t streamCreateAddRecalculateEndBlock(SStreamTask* pTask);
int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo);
int32_t tDecodeStreamTaskId(SDecoder* pDecoder, STaskId* pTaskId);
@ -640,12 +653,6 @@ typedef struct STaskStatusEntry {
STaskNotifyEventStat notifyEventStat;
} STaskStatusEntry;
//typedef struct SNodeUpdateInfo {
// int32_t nodeId;
// SEpSet prevEp;
// SEpSet newEp;
//} SNodeUpdateInfo;
typedef struct SStreamTaskState {
ETaskStatus state;
char* name;
@ -753,9 +760,10 @@ int32_t streamTaskReloadState(SStreamTask* pTask);
void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId);
void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId);
void streamTaskOpenAllUpstreamInput(SStreamTask* pTask);
int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key);
int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key, uint8_t recalated);
bool streamTaskIsSinkTask(const SStreamTask* pTask);
void streamTaskSetRemoveBackendFiles(SStreamTask* pTask);
int8_t streamTaskShouldRecalated(SStreamTask* pTask);
void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask);
void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc);

40
include/libs/stream/tstreamFileState.h
View File

@ -56,12 +56,14 @@ void streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuf
int32_t addRowBuffIfNotExist(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t createRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen);
int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen);
void deleteRowBuffByGroupId(SStreamFileState* pFileState, uint64_t groupId);
int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** pVal);
bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen);
bool hasRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, bool hasLimit, bool* pIsLast);
int32_t putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
SStreamSnapshot* getSnapshot(SStreamFileState* pFileState);
@ -80,6 +82,7 @@ bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap);
TSKEY getFlushMark(SStreamFileState* pFileState);
SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState);
int32_t getRowStateRowSize(SStreamFileState* pFileState);
void freeArrayPtr(void* ptr);
// session window
int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen,
@ -147,12 +150,43 @@ SSHashObj* getGroupIdCache(SStreamFileState* pFileState);
int fillStateKeyCompare(const void* pWin1, const void* pDatas, int pos);
int32_t getRowStatePrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SWinKey* pResKey, void** ppVal,
int32_t* pVLen, int32_t* pWinCode);
int32_t addSearchItem(SStreamFileState* pFileState, SArray* pWinStates, const SWinKey* pKey);
int32_t addSearchItem(SStreamFileState* pFileState, SArray* pWinStates, const SWinKey* pKey, bool* pIsEnd);
int32_t getRowStateAllPrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SArray* pResArray, int32_t maxNum);
//twa
void setFillInfo(SStreamFileState* pFileState);
void clearExpiredState(SStreamFileState* pFileState);
void clearExpiredState(SStreamFileState* pFileState, int32_t numOfKeep, TSKEY minTs);
int32_t addArrayBuffIfNotExist(SSHashObj* pSearchBuff, uint64_t groupId, SArray** ppResStates);
int32_t recoverHashSortBuff(SStreamFileState* pFileState, SArray* pWinStates, uint64_t groupId);
int32_t getAndSetTsData(STableTsDataState* pTsDataState, uint64_t tableUid, TSKEY* pCurTs, void** ppCurPkVal,
TSKEY lastTs, void* pLastPkVal, int32_t lastPkLen, int32_t* pWinCode);
int32_t doTsDataCommit(STableTsDataState* pTsDataState);
int32_t doRangeDataCommit(STableTsDataState* pTsDataState);
int32_t initTsDataState(STableTsDataState** ppTsDataState, int8_t pkType, int32_t pkLen, void* pState,
void* pOtherState);
void destroyTsDataState(STableTsDataState* pTsDataState);
int32_t recoverTsData(STableTsDataState* pTsDataState);
int32_t mergeAndSaveScanRange(STableTsDataState* pTsDataState, STimeWindow* pWin, uint64_t gpId, SRecDataInfo* pRecData,
int32_t len);
int32_t mergeAllScanRange(STableTsDataState* pTsDataState);
int32_t popScanRange(STableTsDataState* pTsDataState, SScanRange* pRange);
// continuous
typedef void* (*getStateBuffFn)(SStreamFileState* pFileState);
SStreamStateCur* getLastStateCur(SStreamFileState* pFileState, getStateBuffFn fn);
void moveLastStateCurNext(SStreamStateCur* pCur, getStateBuffFn fn);
void moveOneStateCurNext(SStreamStateCur* pCur);
int32_t getNLastStateKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes);
int32_t getNLastSessionStateKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes);
int32_t reloadTsDataState(STableTsDataState* pTsDataState);
int32_t setStateRecFlag(SStreamFileState* pState, const void* pKey, int32_t keyLen, int32_t mode);
int32_t getStateRecFlag(SStreamFileState* pFileState, const void* pKey, int32_t keyLen, int32_t* pMode);
void clearExpiredSessionState(struct SStreamFileState* pFileState, int32_t numOfKeep, TSKEY minTs,
SSHashObj* pFlushGroup);
bool hasSessionState(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, bool* pIsLast);
int32_t saveRecInfoToDisk(STableTsDataState* pTsDataState, SSessionKey* pKey, SRecDataInfo* pVal, int32_t vLen);
int32_t flushRemainRecInfoToDisk(STableTsDataState* pTsDataState);
#ifdef __cplusplus
}

14
source/common/src/msg/tmsg.c
View File

@ -10178,6 +10178,12 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->notifyEventTypes));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->notifyErrorHandle));
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->notifyHistory));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->recalculateInterval));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->pWstartName));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->pWendName));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->pGroupIdName));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->pIsWindowFilledName));
tEndEncode(&encoder);
_exit:
@ -10336,6 +10342,14 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->notifyHistory));
}
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->recalculateInterval));
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->pWstartName));
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->pWendName));
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->pGroupIdName));
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->pIsWindowFilledName));
}
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);

19
source/common/src/tglobal.c
View File

@ -353,6 +353,10 @@ int tsStreamAggCnt = 100000;
bool tsStreamCoverage = false;
bool tsStreamRunHistoryAsync = false;
char tsAdapterFqdn[TSDB_FQDN_LEN] = "localhost";
uint16_t tsAdapterPort = 6041;
char tsAdapterToken[512] = "cm9vdDp0YW9zZGF0YQ==";
bool tsUpdateCacheBatch = true;
int8_t tsS3EpNum = 0;
@ -1003,6 +1007,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "streamNotifyMessageSize", tsStreamNotifyMessageSize, 8, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "streamNotifyFrameSize", tsStreamNotifyFrameSize, 8, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "adapterFqdn", tsAdapterFqdn, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "adapterPort", tsAdapterPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "adapterToken", tsAdapterToken, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL));
// clang-format on
// GRANT_CFG_ADD;
@ -1909,6 +1917,17 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "streamNotifyFrameSize");
tsStreamNotifyFrameSize = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "adapterFqdn");
TAOS_CHECK_RETURN(taosCheckCfgStrValueLen(pItem->name, pItem->str, TSDB_FQDN_LEN));
tstrncpy(tsAdapterFqdn, pItem->str, TSDB_FQDN_LEN);
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "adapterPort");
tsAdapterPort = (uint16_t)pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "adapterToken");
TAOS_CHECK_RETURN(taosCheckCfgStrValueLen(pItem->name, pItem->str, tListLen(tsAdapterToken)));
tstrncpy(tsAdapterToken, pItem->str, tListLen(tsAdapterToken));
// GRANT_CFG_GET;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}

4
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -817,9 +817,9 @@ typedef struct {
char* sql;
char* ast;
char* physicalPlan;
SArray* tasks; // SArray<SArray<SStreamTask>>
SArray* pHTasksList; // generate the results for already stored ts data
SArray* pTaskList; // SArray<SArray<SStreamTask>>
SArray* pHTaskList; // generate the results for already stored ts data
int64_t hTaskUid; // stream task for history ts data
SSchemaWrapper outputSchema;

18
source/dnode/mnode/impl/src/mndDef.c
View File

@ -66,10 +66,10 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, ""));
}
int32_t sz = taosArrayGetSize(pObj->tasks);
int32_t sz = taosArrayGetSize(pObj->pTaskList);
TAOS_CHECK_RETURN(tEncodeI32(pEncoder, sz));
for (int32_t i = 0; i < sz; i++) {
SArray *pArray = taosArrayGetP(pObj->tasks, i);
SArray *pArray = taosArrayGetP(pObj->pTaskList, i);
int32_t innerSz = taosArrayGetSize(pArray);
TAOS_CHECK_RETURN(tEncodeI32(pEncoder, innerSz));
for (int32_t j = 0; j < innerSz; j++) {
@ -129,16 +129,16 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->ast));
TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan));
if (pObj->tasks != NULL) {
pObj->tasks = freeStreamTasks(pObj->tasks);
if (pObj->pTaskList != NULL) {
pObj->pTaskList = freeStreamTasks(pObj->pTaskList);
}
int32_t sz;
TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &sz));
if (sz != 0) {
pObj->tasks = taosArrayInit(sz, sizeof(void *));
if (pObj->tasks == NULL) {
pObj->pTaskList = taosArrayInit(sz, sizeof(void *));
if (pObj->pTaskList == NULL) {
code = terrno;
TAOS_RETURN(code);
}
@ -168,7 +168,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
}
}
}
if (taosArrayPush(pObj->tasks, &pArray) == NULL) {
if (taosArrayPush(pObj->pTaskList, &pArray) == NULL) {
taosArrayDestroy(pArray);
code = terrno;
TAOS_RETURN(code);
@ -227,8 +227,8 @@ void tFreeStreamObj(SStreamObj *pStream) {
taosMemoryFree(pStream->outputSchema.pSchema);
}
pStream->tasks = freeStreamTasks(pStream->tasks);
pStream->pHTasksList = freeStreamTasks(pStream->pHTasksList);
pStream->pTaskList = freeStreamTasks(pStream->pTaskList);
pStream->pHTaskList = freeStreamTasks(pStream->pHTaskList);
// tagSchema.pSchema
if (pStream->tagSchema.nCols > 0) {

174
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -237,31 +237,49 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, SStreamObj* pStream) {
return pVgroup;
}
static int32_t doAddSinkTask(SStreamObj* pStream, SMnode* pMnode, SVgObj* pVgroup, SEpSet* pEpset, bool isFillhistory) {
int64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
static void streamGetUidTaskList(SStreamObj* pStream, EStreamTaskType type, uint64_t* pUid, SArray*** pTaskList) {
if (type == STREAM_NORMAL_TASK) {
*pUid = pStream->uid;
*pTaskList = taosArrayGetLast(pStream->pTaskList);
} else if (type == STREAM_HISTORY_TASK || type == STREAM_RECALCUL_TASK) {
*pUid = pStream->hTaskUid;
*pTaskList = taosArrayGetLast(pStream->pHTaskList);
}
}
static int32_t doAddSinkTask(SStreamObj* pStream, SMnode* pMnode, SVgObj* pVgroup, SEpSet* pEpset, EStreamTaskType type) {
uint64_t uid = 0;
SArray** pTaskList = NULL;
streamGetUidTaskList(pStream, type, &uid, &pTaskList);
SStreamTask* pTask = NULL;
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, 0, *pTaskList, pStream->conf.fillHistory,
pStream->subTableWithoutMd5, &pTask);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, type, pStream->conf.trigger, 0, *pTaskList, pStream->conf.fillHistory,
pStream->subTableWithoutMd5, 1, &pTask);
if (code != 0) {
return code;
}
mDebug("doAddSinkTask taskId:%s, %p vgId:%d, isFillHistory:%d", pTask->id.idStr, pTask, pVgroup->vgId, isFillhistory);
mDebug("doAddSinkTask taskId:%s, %p vgId:%d, isFillHistory:%d", pTask->id.idStr, pTask, pVgroup->vgId, (type == STREAM_HISTORY_TASK));
pTask->info.nodeId = pVgroup->vgId;
pTask->info.epSet = mndGetVgroupEpset(pMnode, pVgroup);
return mndSetSinkTaskInfo(pStream, pTask);
}
bool needHistoryTask(SStreamObj* pStream) {
return (pStream->conf.fillHistory) || (pStream->conf.trigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE);
}
static int32_t doAddSinkTaskToVg(SMnode* pMnode, SStreamObj* pStream, SEpSet* pEpset, SVgObj* vgObj) {
int32_t code = doAddSinkTask(pStream, pMnode, vgObj, pEpset, false);
int32_t code = doAddSinkTask(pStream, pMnode, vgObj, pEpset, STREAM_NORMAL_TASK);
if (code != 0) {
return code;
}
if (pStream->conf.fillHistory) {
code = doAddSinkTask(pStream, pMnode, vgObj, pEpset, true);
if (needHistoryTask(pStream)) {
EStreamTaskType type = (pStream->conf.trigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) ? STREAM_RECALCUL_TASK
: STREAM_HISTORY_TASK;
code = doAddSinkTask(pStream, pMnode, vgObj, pEpset, type);
if (code != 0) {
return code;
}
@ -321,7 +339,7 @@ static void streamTaskSetDataRange(SStreamTask* pTask, int64_t skey, SArray* pVe
SDataRange* pRange = &pTask->dataRange;
STimeWindow* pWindow = &pRange->window;
if (pTask->info.fillHistory) {
if (pTask->info.fillHistory == STREAM_HISTORY_TASK) {
pWindow->skey = INT64_MIN;
pWindow->ekey = skey - 1;
@ -352,13 +370,25 @@ static void haltInitialTaskStatus(SStreamTask* pTask, SSubplan* pPlan, bool isFi
}
}
static int32_t buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam, SStreamTask** pTask) {
uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
static int32_t buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, EStreamTaskType type, bool useTriggerParam,
int8_t hasAggTasks, SStreamTask** pTask) {
uint64_t uid = 0;
SArray** pTaskList = NULL;
streamGetUidTaskList(pStream, type, &uid, &pTaskList);
int32_t trigger = 0;
if (type == STREAM_RECALCUL_TASK) {
trigger = STREAM_TRIGGER_WINDOW_CLOSE;
} else {
trigger = pStream->conf.trigger;
}
int32_t triggerParam = useTriggerParam ? pStream->conf.triggerParam : 0;
int32_t code =
tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, type, trigger, triggerParam,
*pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5,
hasAggTasks, pTask);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, isFillhistory, pStream->conf.trigger,
useTriggerParam ? pStream->conf.triggerParam : 0, *pTaskList, pStream->conf.fillHistory,
pStream->subTableWithoutMd5, pTask);
return code;
}
@ -369,19 +399,19 @@ static int32_t addNewTaskList(SStreamObj* pStream) {
return terrno;
}
if (taosArrayPush(pStream->tasks, &pTaskList) == NULL) {
if (taosArrayPush(pStream->pTaskList, &pTaskList) == NULL) {
mError("failed to put into array, code:%s", tstrerror(terrno));
return terrno;
}
if (pStream->conf.fillHistory) {
if (needHistoryTask(pStream)) {
pTaskList = taosArrayInit(0, POINTER_BYTES);
if (pTaskList == NULL) {
mError("failed init task list, code:%s", tstrerror(terrno));
mError("failed init history task list, code:%s", tstrerror(terrno));
return terrno;
}
if (taosArrayPush(pStream->pHTasksList, &pTaskList) == NULL) {
if (taosArrayPush(pStream->pHTaskList, &pTaskList) == NULL) {
mError("failed to put into array, code:%s", tstrerror(terrno));
return terrno;
}
@ -392,8 +422,8 @@ static int32_t addNewTaskList(SStreamObj* pStream) {
// set the history task id
static void setHTasksId(SStreamObj* pStream) {
SArray* pTaskList = *(SArray**)taosArrayGetLast(pStream->tasks);
SArray* pHTaskList = *(SArray**)taosArrayGetLast(pStream->pHTasksList);
SArray* pTaskList = *(SArray**)taosArrayGetLast(pStream->pTaskList);
SArray* pHTaskList = *(SArray**)taosArrayGetLast(pStream->pHTaskList);
for (int32_t i = 0; i < taosArrayGetSize(pTaskList); ++i) {
SStreamTask** pStreamTask = taosArrayGet(pTaskList, i);
@ -411,17 +441,17 @@ static void setHTasksId(SStreamObj* pStream) {
}
static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset, int64_t skey,
SArray* pVerList, SVgObj* pVgroup, bool isHistoryTask, bool useTriggerParam) {
SArray* pVerList, SVgObj* pVgroup, EStreamTaskType type, bool useTriggerParam, int8_t hasAggTasks) {
SStreamTask* pTask = NULL;
int32_t code = buildSourceTask(pStream, pEpset, isHistoryTask, useTriggerParam, &pTask);
int32_t code = buildSourceTask(pStream, pEpset, type, useTriggerParam, hasAggTasks, &pTask);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
mDebug("doAddSourceTask taskId:%s, %p vgId:%d, historyTask:%d", pTask->id.idStr, pTask, pVgroup->vgId, isHistoryTask);
mDebug("doAddSourceTask taskId:%s, %p vgId:%d, historyTask:%d", pTask->id.idStr, pTask, pVgroup->vgId, (type == STREAM_HISTORY_TASK));
if (pStream->conf.fillHistory) {
haltInitialTaskStatus(pTask, plan, isHistoryTask);
if (needHistoryTask(pStream)) {
haltInitialTaskStatus(pTask, plan, (type == STREAM_HISTORY_TASK));
}
streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId);
@ -466,7 +496,7 @@ static SSubplan* getAggSubPlan(const SQueryPlan* pPlan, int index) {
}
static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset,
int64_t nextWindowSkey, SArray* pVerList, bool useTriggerParam) {
int64_t nextWindowSkey, SArray* pVerList, bool useTriggerParam, bool hasAggTasks) {
void* pIter = NULL;
SSdb* pSdb = pMnode->pSdb;
int32_t code = addNewTaskList(pStream);
@ -486,7 +516,8 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream
continue;
}
code = doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, false, useTriggerParam);
code = doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, STREAM_NORMAL_TASK,
useTriggerParam, hasAggTasks);
if (code != 0) {
mError("failed to create stream task, code:%s", tstrerror(code));
@ -495,8 +526,16 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream
return code;
}
if (pStream->conf.fillHistory) {
code = doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, true, useTriggerParam);
if (needHistoryTask(pStream)) {
EStreamTaskType type = 0;
if (pStream->conf.trigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE && (pStream->conf.fillHistory == 0)) {
type = STREAM_RECALCUL_TASK; // only the recalculating task
} else {
type = STREAM_HISTORY_TASK; // set the fill-history option
}
code = doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, type,
useTriggerParam, hasAggTasks);
if (code != 0) {
sdbRelease(pSdb, pVgroup);
return code;
@ -506,33 +545,35 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream
sdbRelease(pSdb, pVgroup);
}
if (pStream->conf.fillHistory) {
if (needHistoryTask(pStream)) {
setHTasksId(pStream);
}
return TSDB_CODE_SUCCESS;
}
static int32_t buildAggTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam,
static int32_t buildAggTask(SStreamObj* pStream, SEpSet* pEpset, EStreamTaskType type, bool useTriggerParam,
SStreamTask** pAggTask) {
*pAggTask = NULL;
uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
uint64_t uid = 0;
SArray** pTaskList = NULL;
streamGetUidTaskList(pStream, type, &uid, &pTaskList);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, isFillhistory, pStream->conf.trigger,
useTriggerParam ? pStream->conf.triggerParam : 0, *pTaskList, pStream->conf.fillHistory,
pStream->subTableWithoutMd5, pAggTask);
int64_t triggerParam = useTriggerParam? pStream->conf.triggerParam:0;
int32_t code = tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, type, pStream->conf.trigger,
triggerParam, *pTaskList, pStream->conf.fillHistory,
pStream->subTableWithoutMd5, 1, pAggTask);
return code;
}
static int32_t doAddAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, SEpSet* pEpset, SVgObj* pVgroup,
SSnodeObj* pSnode, bool isFillhistory, bool useTriggerParam) {
SSnodeObj* pSnode, EStreamTaskType type, bool useTriggerParam) {
int32_t code = 0;
SStreamTask* pTask = NULL;
const char* id = NULL;
code = buildAggTask(pStream, pEpset, isFillhistory, useTriggerParam, &pTask);
code = buildAggTask(pStream, pEpset, type, useTriggerParam, &pTask);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -540,10 +581,10 @@ static int32_t doAddAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan,
id = pTask->id.idStr;
if (pSnode != NULL) {
code = mndAssignStreamTaskToSnode(pMnode, pTask, plan, pSnode);
mDebug("doAddAggTask taskId:%s, %p snode id:%d, isFillHistory:%d", id, pTask, pSnode->id, isFillhistory);
mDebug("doAddAggTask taskId:%s, %p snode id:%d, isFillHistory:%d", id, pTask, pSnode->id, (type == STREAM_HISTORY_TASK));
} else {
code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup);
mDebug("doAddAggTask taskId:%s, %p vgId:%d, isFillHistory:%d", id, pTask, pVgroup->vgId, isFillhistory);
mDebug("doAddAggTask taskId:%s, %p vgId:%d, isFillHistory:%d", id, pTask, pVgroup->vgId, (type == STREAM_HISTORY_TASK));
}
return code;
}
@ -561,13 +602,15 @@ static int32_t addAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, S
pVgroup = mndSchedFetchOneVg(pMnode, pStream);
}
code = doAddAggTask(pStream, pMnode, plan, pEpset, pVgroup, pSnode, false, useTriggerParam);
code = doAddAggTask(pStream, pMnode, plan, pEpset, pVgroup, pSnode, STREAM_NORMAL_TASK, useTriggerParam);
if (code != 0) {
goto END;
}
if (pStream->conf.fillHistory) {
code = doAddAggTask(pStream, pMnode, plan, pEpset, pVgroup, pSnode, true, useTriggerParam);
if (needHistoryTask(pStream)) {
EStreamTaskType type = (pStream->conf.trigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) ? STREAM_RECALCUL_TASK
: STREAM_HISTORY_TASK;
code = doAddAggTask(pStream, pMnode, plan, pEpset, pVgroup, pSnode, type, useTriggerParam);
if (code != 0) {
goto END;
}
@ -602,9 +645,10 @@ static int32_t addSinkTask(SMnode* pMnode, SStreamObj* pStream, SEpSet* pEpset)
}
}
if (pStream->conf.fillHistory) {
if (needHistoryTask(pStream)) {
setHTasksId(pStream);
}
return TDB_CODE_SUCCESS;
}
@ -683,6 +727,7 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
bool hasExtraSink = false;
bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0;
SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb);
if (pDbObj == NULL) {
code = TSDB_CODE_QRY_INVALID_INPUT;
TAOS_RETURN(code);
@ -694,9 +739,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
mDebug("doScheduleStream numOfPlanLevel:%d, exDb:%d, multiTarget:%d, fix vgId:%d, physicalPlan:%s", numOfPlanLevel,
externalTargetDB, multiTarget, pStream->fixedSinkVgId, pStream->physicalPlan);
pStream->tasks = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES);
pStream->pHTasksList = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES);
if (pStream->tasks == NULL || pStream->pHTasksList == NULL) {
pStream->pTaskList = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES);
pStream->pHTaskList = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES);
if (pStream->pTaskList == NULL || pStream->pHTaskList == NULL) {
mError("failed to create stream obj, code:%s", tstrerror(terrno));
return terrno;
}
@ -719,15 +764,16 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
TAOS_RETURN(code);
}
code = addSourceTask(pMnode, plan, pStream, pEpset, skey, pVerList, (numOfPlanLevel == 1));
int8_t hasAggTasks = (numOfPlanLevel > 1) ? 1 : 0; // task level is greater than 1, which means agg existing
code = addSourceTask(pMnode, plan, pStream, pEpset, skey, pVerList, (numOfPlanLevel == 1), hasAggTasks);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (numOfPlanLevel == 1) {
bindSourceSink(pStream, pMnode, pStream->tasks, hasExtraSink);
if (pStream->conf.fillHistory) {
bindSourceSink(pStream, pMnode, pStream->pHTasksList, hasExtraSink);
bindSourceSink(pStream, pMnode, pStream->pTaskList, hasExtraSink);
if (needHistoryTask(pStream)) {
bindSourceSink(pStream, pMnode, pStream->pHTaskList, hasExtraSink);
}
return TDB_CODE_SUCCESS;
}
@ -741,7 +787,7 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
}
do {
SArray** list = taosArrayGetLast(pStream->tasks);
SArray** list = taosArrayGetLast(pStream->pTaskList);
float size = (float)taosArrayGetSize(*list);
size_t cnt = (size_t)ceil(size / tsStreamAggCnt);
if (cnt <= 1) break;
@ -758,9 +804,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
return code;
}
bindTwoLevel(pStream->tasks, j * tsStreamAggCnt, (j + 1) * tsStreamAggCnt);
if (pStream->conf.fillHistory) {
bindTwoLevel(pStream->pHTasksList, j * tsStreamAggCnt, (j + 1) * tsStreamAggCnt);
bindTwoLevel(pStream->pTaskList, j * tsStreamAggCnt, (j + 1) * tsStreamAggCnt);
if (needHistoryTask(pStream)) {
bindTwoLevel(pStream->pHTaskList, j * tsStreamAggCnt, (j + 1) * tsStreamAggCnt);
}
}
} while (1);
@ -774,7 +820,7 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
}
mDebug("doScheduleStream add final agg");
SArray** list = taosArrayGetLast(pStream->tasks);
SArray** list = taosArrayGetLast(pStream->pTaskList);
size_t size = taosArrayGetSize(*list);
code = addNewTaskList(pStream);
@ -786,14 +832,14 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
if (code != TSDB_CODE_SUCCESS) {
TAOS_RETURN(code);
}
bindTwoLevel(pStream->tasks, 0, size);
if (pStream->conf.fillHistory) {
bindTwoLevel(pStream->pHTasksList, 0, size);
bindTwoLevel(pStream->pTaskList, 0, size);
if (needHistoryTask(pStream)) {
bindTwoLevel(pStream->pHTaskList, 0, size);
}
bindAggSink(pStream, pMnode, pStream->tasks);
if (pStream->conf.fillHistory) {
bindAggSink(pStream, pMnode, pStream->pHTasksList);
bindAggSink(pStream, pMnode, pStream->pTaskList);
if (needHistoryTask(pStream)) {
bindAggSink(pStream, pMnode, pStream->pHTaskList);
}
TAOS_RETURN(code);
}

42
source/dnode/mnode/impl/src/mndStream.c
View File

@ -250,13 +250,13 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream
pOldStream->updateTime = pNewStream->updateTime;
pOldStream->checkpointId = pNewStream->checkpointId;
pOldStream->checkpointFreq = pNewStream->checkpointFreq;
if (pOldStream->tasks == NULL){
pOldStream->tasks = pNewStream->tasks;
pNewStream->tasks = NULL;
if (pOldStream->pTaskList == NULL) {
pOldStream->pTaskList = pNewStream->pTaskList;
pNewStream->pTaskList = NULL;
}
if (pOldStream->pHTasksList == NULL){
pOldStream->pHTasksList = pNewStream->pHTasksList;
pNewStream->pHTasksList = NULL;
if (pOldStream->pHTaskList == NULL) {
pOldStream->pHTaskList = pNewStream->pHTaskList;
pNewStream->pHTaskList = NULL;
}
taosWUnLockLatch(&pOldStream->lock);
return 0;
@ -478,7 +478,17 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
.deleteMark = pObj->deleteMark,
.igCheckUpdate = pObj->igCheckUpdate,
.destHasPrimaryKey = hasDestPrimaryKey(&pObj->outputSchema),
.recalculateInterval = pCreate->recalculateInterval,
};
char *pTargetFStable = strchr(pCreate->targetStbFullName, '.');
if (pTargetFStable != NULL) {
pTargetFStable = pTargetFStable + 1;
}
tstrncpy(cxt.pStbFullName, pTargetFStable, TSDB_TABLE_FNAME_LEN);
tstrncpy(cxt.pWstartName, pCreate->pWstartName, TSDB_COL_NAME_LEN);
tstrncpy(cxt.pWendName, pCreate->pWendName, TSDB_COL_NAME_LEN);
tstrncpy(cxt.pGroupIdName, pCreate->pGroupIdName, TSDB_COL_NAME_LEN);
tstrncpy(cxt.pIsWindowFilledName, pCreate->pIsWindowFilledName, TSDB_COL_NAME_LEN);
// using ast and param to build physical plan
if ((code = qCreateQueryPlan(&cxt, &pPlan, NULL)) < 0) {
@ -590,11 +600,11 @@ int32_t mndPersistStreamTasks(STrans *pTrans, SStreamObj *pStream) {
destroyStreamTaskIter(pIter);
// persistent stream task for already stored ts data
if (pStream->conf.fillHistory) {
int32_t level = taosArrayGetSize(pStream->pHTasksList);
if (pStream->conf.fillHistory || (pStream->conf.trigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE)) {
int32_t level = taosArrayGetSize(pStream->pHTaskList);
for (int32_t i = 0; i < level; i++) {
SArray *pLevel = taosArrayGetP(pStream->pHTasksList, i);
SArray *pLevel = taosArrayGetP(pStream->pHTaskList, i);
int32_t numOfTasks = taosArrayGetSize(pLevel);
for (int32_t j = 0; j < numOfTasks; j++) {
@ -812,9 +822,9 @@ static int32_t addStreamNotifyInfo(SCMCreateStreamReq *createReq, SStreamObj *pS
goto _end;
}
level = taosArrayGetSize(pStream->tasks);
level = taosArrayGetSize(pStream->pTaskList);
for (int32_t i = 0; i < level; ++i) {
pLevel = taosArrayGetP(pStream->tasks, i);
pLevel = taosArrayGetP(pStream->pTaskList, i);
nTasks = taosArrayGetSize(pLevel);
for (int32_t j = 0; j < nTasks; ++j) {
code = addStreamTaskNotifyInfo(createReq, pStream, taosArrayGetP(pLevel, j));
@ -823,9 +833,9 @@ static int32_t addStreamNotifyInfo(SCMCreateStreamReq *createReq, SStreamObj *pS
}
if (pStream->conf.fillHistory && createReq->notifyHistory) {
level = taosArrayGetSize(pStream->pHTasksList);
level = taosArrayGetSize(pStream->pHTaskList);
for (int32_t i = 0; i < level; ++i) {
pLevel = taosArrayGetP(pStream->pHTasksList, i);
pLevel = taosArrayGetP(pStream->pHTaskList, i);
nTasks = taosArrayGetSize(pLevel);
for (int32_t j = 0; j < nTasks; ++j) {
code = addStreamTaskNotifyInfo(createReq, pStream, taosArrayGetP(pLevel, j));
@ -921,7 +931,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
code = mndAcquireStream(pMnode, createReq.name, &pStream);
if (pStream != NULL && code == 0) {
if (pStream->tasks != NULL){
if (pStream->pTaskList != NULL){
if (createReq.igExists) {
mInfo("stream:%s, already exist, ignore exist is set", createReq.name);
mndReleaseStream(pMnode, pStream);
@ -1250,9 +1260,9 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
pStream->currentTick = 1;
// 1. redo action: broadcast checkpoint source msg for all source vg
int32_t totalLevel = taosArrayGetSize(pStream->tasks);
int32_t totalLevel = taosArrayGetSize(pStream->pTaskList);
for (int32_t i = 0; i < totalLevel; i++) {
SArray *pLevel = taosArrayGetP(pStream->tasks, i);
SArray *pLevel = taosArrayGetP(pStream->pTaskList, i);
SStreamTask *p = taosArrayGetP(pLevel, 0);
if (p->info.taskLevel == TASK_LEVEL__SOURCE) {

10
source/dnode/mnode/impl/src/mndStreamUtil.c
View File

@ -40,7 +40,7 @@ int32_t createStreamTaskIter(SStreamObj *pStream, SStreamTaskIter **pIter) {
(*pIter)->level = -1;
(*pIter)->ordinalIndex = 0;
(*pIter)->pStream = pStream;
(*pIter)->totalLevel = taosArrayGetSize(pStream->tasks);
(*pIter)->totalLevel = taosArrayGetSize(pStream->pTaskList);
(*pIter)->pTask = NULL;
return 0;
@ -57,7 +57,7 @@ bool streamTaskIterNextTask(SStreamTaskIter *pIter) {
}
while (pIter->level < pIter->totalLevel) {
SArray *pList = taosArrayGetP(pIter->pStream->tasks, pIter->level);
SArray *pList = taosArrayGetP(pIter->pStream->pTaskList, pIter->level);
if (pIter->ordinalIndex >= taosArrayGetSize(pList)) {
pIter->level += 1;
pIter->ordinalIndex = 0;
@ -394,8 +394,8 @@ int32_t mndGetStreamTask(STaskId *pId, SStreamObj *pStream, SStreamTask **pTask)
int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) {
int32_t num = 0;
for (int32_t i = 0; i < taosArrayGetSize(pStream->tasks); ++i) {
SArray *pLevel = taosArrayGetP(pStream->tasks, i);
for (int32_t i = 0; i < taosArrayGetSize(pStream->pTaskList); ++i) {
SArray *pLevel = taosArrayGetP(pStream->pTaskList, i);
num += taosArrayGetSize(pLevel);
}
@ -1110,7 +1110,7 @@ int32_t setStreamAttrInResBlock(SStreamObj *pStream, SSDataBlock *pBlock, int32_
TSDB_CHECK_CODE(code, lino, _end);
int8_t streamStatus = atomic_load_8(&pStream->status);
if (isPaused && pStream->tasks != NULL) {
if (isPaused && pStream->pTaskList != NULL) {
streamStatus = STREAM_STATUS__PAUSE;
}
mndShowStreamStatus(status2, streamStatus);

6
source/dnode/mnode/impl/test/stream/stream.cpp
View File

@ -229,8 +229,8 @@ TEST_F(StreamTest, kill_checkpoint_trans) {
pStream->uid = defStreamId;
pStream->lock = 0;
pStream->tasks = taosArrayInit(1, POINTER_BYTES);
pStream->pHTasksList = taosArrayInit(1, POINTER_BYTES);
pStream->pTaskList = taosArrayInit(1, POINTER_BYTES);
pStream->pHTaskList = taosArrayInit(1, POINTER_BYTES);
SArray* pLevel = taosArrayInit(1, POINTER_BYTES);
SStreamTask* pTask = static_cast<SStreamTask*>(taosMemoryCalloc(1, sizeof(SStreamTask)));
@ -243,7 +243,7 @@ TEST_F(StreamTest, kill_checkpoint_trans) {
void* px = taosArrayPush(pLevel, &pTask);
ASSERT(px != NULL);
px = taosArrayPush(pStream->tasks, &pLevel);
px = taosArrayPush(pStream->pTaskList, &pLevel);
ASSERT(px != NULL);
code = mndCreateStreamResetStatusTrans(pMnode, pStream, 1);

6
source/dnode/snode/src/snode.c
View File

@ -43,17 +43,17 @@ int32_t sndBuildStreamTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProce
tqSetRestoreVersionInfo(pTask);
char *p = streamTaskGetStatus(pTask).name;
if (pTask->info.fillHistory) {
if (pTask->info.fillHistory == STREAM_HISTORY_TASK) {
sndInfo("vgId:%d build stream task, s-task:%s, %p checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
" child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms",
" child id:%d, level:%d, status:%s taskType:%d, related stream task:0x%x trigger:%" PRId64 " ms",
SNODE_HANDLE, pTask->id.idStr, pTask, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
(int32_t)pTask->streamTaskId.taskId, pTask->info.delaySchedParam);
} else {
sndInfo("vgId:%d build stream task, s-task:%s, %p checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
" child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms",
" child id:%d, level:%d, status:%s taskType:%d, related helper-task:0x%x trigger:%" PRId64 " ms",
SNODE_HANDLE, pTask->id.idStr, pTask, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
(int32_t)pTask->hTaskInfo.id.taskId, pTask->info.delaySchedParam);

29
source/dnode/snode/src/snodeInitApi.c
View File

@ -32,6 +32,7 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStatePutParName = streamStatePutParName;
pStore->streamStateGetParName = streamStateGetParName;
pStore->streamStateDeleteParName = streamStateDeleteParName;
pStore->streamStateSetParNameInvalid = streamStateSetParNameInvalid;
pStore->streamStateAddIfNotExist = streamStateAddIfNotExist;
pStore->streamStateReleaseBuf = streamStateReleaseBuf;
@ -47,8 +48,11 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateClear = streamStateClear;
pStore->streamStateSaveInfo = streamStateSaveInfo;
pStore->streamStateGetInfo = streamStateGetInfo;
pStore->streamStateGetNumber = streamStateGetNumber;
pStore->streamStateDeleteInfo = streamStateDeleteInfo;
pStore->streamStateSetNumber = streamStateSetNumber;
pStore->streamStateGetPrev = streamStateGetPrev;
pStore->streamStateGetAllPrev = streamStateGetAllPrev;
pStore->streamStateFillPut = streamStateFillPut;
pStore->streamStateFillGet = streamStateFillGet;
@ -70,6 +74,9 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateGetKVByCur = streamStateGetKVByCur;
pStore->streamStateClearExpiredState = streamStateClearExpiredState;
pStore->streamStateClearExpiredSessionState = streamStateClearExpiredSessionState;
pStore->streamStateSetRecFlag = streamStateSetRecFlag;
pStore->streamStateGetRecFlag = streamStateGetRecFlag;
pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist;
pStore->streamStateSessionPut = streamStateSessionPut;
@ -82,6 +89,9 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange;
pStore->streamStateCountGetKeyByRange = streamStateCountGetKeyByRange;
pStore->streamStateSessionAllocWinBuffByNextPosition = streamStateSessionAllocWinBuffByNextPosition;
pStore->streamStateSessionSaveToDisk = streamStateSessionSaveToDisk;
pStore->streamStateFlushReaminInfoToDisk = streamStateFlushReaminInfoToDisk;
pStore->streamStateSessionDeleteAll = streamStateSessionDeleteAll;
pStore->updateInfoInit = updateInfoInit;
pStore->updateInfoFillBlockData = updateInfoFillBlockData;
@ -113,7 +123,26 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamFileStateClear = streamFileStateClear;
pStore->needClearDiskBuff = needClearDiskBuff;
pStore->streamStateGetAndSetTsData = streamStateGetAndSetTsData;
pStore->streamStateTsDataCommit = streamStateTsDataCommit;
pStore->streamStateInitTsDataState = streamStateInitTsDataState;
pStore->streamStateDestroyTsDataState = streamStateDestroyTsDataState;
pStore->streamStateRecoverTsData = streamStateRecoverTsData;
pStore->streamStateReloadTsDataState = streamStateReloadTsDataState;
pStore->streamStateMergeAndSaveScanRange = streamStateMergeAndSaveScanRange;
pStore->streamStateMergeAllScanRange = streamStateMergeAllScanRange;
pStore->streamStatePopScanRange = streamStatePopScanRange;
pStore->streamStateCheckSessionState = streamStateCheckSessionState;
pStore->streamStateGetLastStateCur = streamStateGetLastStateCur;
pStore->streamStateLastStateCurNext = streamStateLastStateCurNext;
pStore->streamStateNLastStateGetKVByCur = streamStateNLastStateGetKVByCur;
pStore->streamStateGetLastSessionStateCur = streamStateGetLastSessionStateCur;
pStore->streamStateLastSessionStateCurNext = streamStateLastSessionStateCurNext;
pStore->streamStateNLastSessionStateGetKVByCur = streamStateNLastSessionStateGetKVByCur;
pStore->streamStateOpen = streamStateOpen;
pStore->streamStateRecalatedOpen = streamStateRecalatedOpen;
pStore->streamStateClose = streamStateClose;
pStore->streamStateBegin = streamStateBegin;
pStore->streamStateCommit = streamStateCommit;

2
source/dnode/vnode/inc/vnode.h
View File

@ -185,6 +185,8 @@ void tsdbReaderSetCloseFlag(STsdbReader *pReader);
int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr);
void tsdbSetFilesetDelimited(STsdbReader *pReader);
void tsdbReaderSetNotifyCb(STsdbReader *pReader, TsdReaderNotifyCbFn notifyFn, void *param);
int32_t tsdbReaderGetProgress(const STsdbReader *pReader, void **pBuf, uint64_t *pLen);
int32_t tsdbReaderSetProgress(STsdbReader *pReader, const void *buf, uint64_t len);
int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables);
int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,

34
source/dnode/vnode/src/tq/tq.c
View File

@ -849,7 +849,7 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV
if (pTask->info.fillHistory) {
tqInfo("vgId:%d build stream task, s-task:%s, %p checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
" child id:%d, level:%d, cur-status:%s, next-status:%s fill-history:%d, related stream task:0x%x "
" child id:%d, level:%d, cur-status:%s, next-status:%s taskType:%d, related stream task:0x%x "
"delaySched:%" PRId64 " ms, inputVer:%" PRId64,
vgId, pTask->id.idStr, pTask, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory,
@ -857,7 +857,7 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV
} else {
tqInfo("vgId:%d build stream task, s-task:%s, %p checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
" child id:%d, level:%d, cur-status:%s next-status:%s fill-history:%d, related fill-task:0x%x "
" child id:%d, level:%d, cur-status:%s next-status:%s taskType:%d, related helper-task:0x%x "
"delaySched:%" PRId64 " ms, inputVer:%" PRId64,
vgId, pTask->id.idStr, pTask, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory,
@ -949,7 +949,7 @@ int32_t handleStep2Async(SStreamTask* pStreamTask, void* param) {
return TSDB_CODE_SUCCESS;
}
// this function should be executed by only one thread, so we set an sentinel to protect this function
// this function should be executed by only one thread, so we set a sentinel to protect this function
int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
SStreamScanHistoryReq* pReq = (SStreamScanHistoryReq*)pMsg->pCont;
SStreamMeta* pMeta = pTq->pStreamMeta;
@ -957,6 +957,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL;
SStreamTask* pStreamTask = NULL;
char* pStatus = NULL;
int32_t taskType = 0;
code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) {
@ -971,8 +972,11 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
SStreamTaskState s = streamTaskGetStatus(pTask);
pStatus = s.name;
taskType = pTask->info.fillHistory;
if ((s.state != TASK_STATUS__SCAN_HISTORY) || (pTask->status.downstreamReady == 0)) {
if ((s.state != TASK_STATUS__SCAN_HISTORY && taskType == STREAM_HISTORY_TASK) ||
(s.state != TASK_STATUS__READY && taskType == STREAM_RECALCUL_TASK) ||
(pTask->status.downstreamReady == 0)) {
tqError("s-task:%s vgId:%d status:%s downstreamReady:%d not allowed/ready for scan-history data, quit", id,
pMeta->vgId, s.name, pTask->status.downstreamReady);
@ -1046,12 +1050,12 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
streamExecScanHistoryInFuture(pTask, retInfo.idleTime);
} else {
SStreamTaskState p = streamTaskGetStatus(pTask);
ETaskStatus s = p.state;
ETaskStatus localStatus = p.state;
if (s == TASK_STATUS__PAUSE) {
if (localStatus == TASK_STATUS__PAUSE) {
tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs total:%.2fs, sched-status:%d", id, el,
pTask->execInfo.step1El, status);
} else if (s == TASK_STATUS__STOP || s == TASK_STATUS__DROPPING) {
} else if (localStatus == TASK_STATUS__STOP || localStatus == TASK_STATUS__DROPPING) {
tqDebug("s-task:%s status:%p not continue scan-history data, total elapsed time:%.2fs quit", id, p.name,
pTask->execInfo.step1El);
}
@ -1062,9 +1066,11 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
}
// the following procedure should be executed, no matter status is stop/pause or not
if (taskType == STREAM_HISTORY_TASK) {
tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, pTask->execInfo.step1El);
if (pTask->info.fillHistory != 1) {
} else if (taskType == STREAM_RECALCUL_TASK) {
tqDebug("s-task:%s recalculate ended, elapsed time:%.2fs", id, pTask->execInfo.step1El);
} else {
tqError("s-task:%s fill-history is disabled, unexpected", id);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
@ -1088,7 +1094,17 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
if (taskType == STREAM_HISTORY_TASK) {
code = streamTaskHandleEventAsync(pStreamTask->status.pSM, TASK_EVENT_HALT, handleStep2Async, pTq);
} else if (taskType == STREAM_RECALCUL_TASK) {
// send recalculate end block
code = streamCreateAddRecalculateEndBlock(pStreamTask);
if (code) {
tqError("s-task:%s failed to create-add recalculate end block, code:%s", id, tstrerror(code));
}
streamTaskSetSchedStatusInactive(pTask);
}
streamMetaReleaseTask(pMeta, pStreamTask);
atomic_store_32(&pTask->status.inScanHistorySentinel, 0);

10
source/dnode/vnode/src/tq/tqStreamTask.c
View File

@ -300,7 +300,11 @@ bool taskReadyForDataFromWal(SStreamTask* pTask) {
return false;
}
if (pInfo->taskLevel == TASK_LEVEL__SOURCE && pInfo->trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) {
if (pInfo->trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) {
return false;
}
if (pInfo->fillHistory == STREAM_RECALCUL_TASK) {
return false;
}
@ -311,8 +315,8 @@ bool taskReadyForDataFromWal(SStreamTask* pTask) {
return false;
}
// fill-history task has entered into the last phase, no need to anything
if ((pInfo->fillHistory == 1) && pTask->status.appendTranstateBlock) {
// fill-history task has entered into the last phase, no need to do anything
if ((pInfo->fillHistory == STREAM_HISTORY_TASK) && pTask->status.appendTranstateBlock) {
// the maximum version of data in the WAL has reached already, the step2 is done
tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr,
pTask->dataRange.range.maxVer);

27
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -40,7 +40,7 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) {
tqDebug("s-task:%s vgId:%d start to expand stream task", pTask->id.idStr, vgId);
if (pTask->info.fillHistory) {
if (pTask->info.fillHistory != STREAM_NORMAL_TASK) {
streamId = pTask->streamTaskId.streamId;
taskId = pTask->streamTaskId.taskId;
} else {
@ -50,20 +50,31 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) {
// sink task does not need the pState
if (pTask->info.taskLevel != TASK_LEVEL__SINK) {
if (pTask->info.fillHistory == STREAM_RECALCUL_TASK) {
pTask->pRecalState = streamStateRecalatedOpen(pMeta->path, pTask, pTask->id.streamId, pTask->id.taskId);
if (pTask->pRecalState == NULL) {
tqError("s-task:%s (vgId:%d) failed to open state for task, expand task failed", pTask->id.idStr, vgId);
return terrno;
} else {
tqDebug("s-task:%s recal state:%p", pTask->id.idStr, pTask->pRecalState);
}
}
pTask->pState = streamStateOpen(pMeta->path, pTask, streamId, taskId);
if (pTask->pState == NULL) {
tqError("s-task:%s (vgId:%d) failed to open state for task, expand task failed", pTask->id.idStr, vgId);
return terrno;
} else {
tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState);
tqDebug("s-task:%s stream state:%p", pTask->id.idStr, pTask->pState);
}
}
SReadHandle handle = {
.checkpointId = pTask->chkInfo.checkpointId,
.pStateBackend = pTask->pState,
.pStateBackend = NULL,
.fillHistory = pTask->info.fillHistory,
.winRange = pTask->dataRange.window,
.pOtherBackend = NULL,
};
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
@ -76,6 +87,14 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) {
initStorageAPI(&handle.api);
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE || pTask->info.taskLevel == TASK_LEVEL__AGG) {
if (pTask->info.fillHistory == STREAM_RECALCUL_TASK) {
handle.pStateBackend = pTask->pRecalState;
handle.pOtherBackend = pTask->pState;
} else {
handle.pStateBackend = pTask->pState;
handle.pOtherBackend = NULL;
}
code = qCreateStreamExecTaskInfo(&pTask->exec.pExecutor, pTask->exec.qmsg, &handle, vgId, pTask->id.taskId);
if (code) {
tqError("s-task:%s failed to expand task, code:%s", pTask->id.idStr, tstrerror(code));
@ -808,13 +827,11 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
}
pMeta->startInfo.startAllTasks = 1;
streamMetaWUnLock(pMeta);
terrno = 0;
tqInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d, ts:%" PRId64, vgId,
pMeta->updateInfo.completeTransId, pMeta->updateInfo.completeTs);
streamMetaWLock(pMeta);
streamMetaClear(pMeta);
int64_t el = taosGetTimestampMs() - st;

157
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -6962,3 +6962,160 @@ void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, v
pReader->notifyFn = notifyFn;
pReader->notifyParam = param;
}
#if 0
static int32_t tsdbEncodeLastProcKeys(const SSHashObj* pTableMap, void** buf) {
void* p = NULL;
int32_t iter = 0;
int32_t len = 0;
len += taosEncodeFixedI32(buf, tSimpleHashGetSize(pTableMap)); // number of tables
while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) {
uint64_t uid = *(uint64_t*)tSimpleHashGetKey(p, NULL);
SRowKey* pKey = &(*(STableBlockScanInfo**)p)->lastProcKey;
len += taosEncodeFixedU64(buf, uid); // table uid
len += taosEncodeFixedI64(buf, pKey->ts); // last read ts
len += taosEncodeFixedU8(buf, pKey->numOfPKs); // number of pks
if (pKey->numOfPKs > 0) {
int8_t type = pKey->pks[0].type;
len += taosEncodeFixedI8(buf, type); // pk data type
uint8_t* pData = 0;
uint32_t nData = 0;
if (IS_VAR_DATA_TYPE(type)) {
pData = pKey->pks[0].pData;
nData = pKey->pks[0].nData;
len += taosEncodeFixedU32(buf, nData); // data length
} else {
pData = (uint8_t*)&pKey->pks[0].val;
nData = tDataTypes[pKey->pks[0].type].bytes;
}
len += taosEncodeBinary(buf, pData, nData); // data value
}
}
return len;
}
static int32_t tsdbDecodeLastProcKeys(const void* buf, int32_t len, SSHashObj* pTableMap) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
const void *begin = buf;
TSDB_CHECK_NULL(buf, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(pTableMap, code, lino, _end, TSDB_CODE_INVALID_PARA);
int32_t numOfTables = 0;
buf = taosDecodeFixedI32(buf, &numOfTables);
TSDB_CHECK_CONDITION(numOfTables == tSimpleHashGetSize(pTableMap), code, lino, _end, TSDB_CODE_INVALID_PARA);
for (int32_t i = 0; i < numOfTables; ++i) {
uint64_t uid = 0;
buf = taosDecodeFixedU64(buf, &uid);
STableBlockScanInfo** pScanInfo = tSimpleHashGet(pTableMap, &uid, sizeof(uid));
if (pScanInfo == NULL) {
tsdbError("failed to find scan info of uid: %" PRIu64 " when decoding last proc keys", uid);
TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
}
SRowKey* pKey = &(*pScanInfo)->lastProcKey;
buf = taosDecodeFixedI64(buf, &pKey->ts);
uint8_t numOfPks = 0;
buf = taosDecodeFixedU8(buf, &numOfPks);
if (numOfPks != pKey->numOfPKs) {
tsdbError("numOfPks of uid %" PRIu64 " mismatch: [%u/%u]", uid, numOfPks, pKey->numOfPKs);
TSDB_CHECK_CONDITION(numOfPks == pKey->numOfPKs, code, lino, _end, TSDB_CODE_INVALID_PARA);
}
if (numOfPks > 0) {
int8_t type = 0;
buf = taosDecodeFixedI8(buf, &type);
if (type != pKey->pks[0].type) {
tsdbError("pk type of uid %" PRIu64 " mismatch: [%d/%d]", uid, type, pKey->pks[0].type);
TSDB_CHECK_CONDITION(type == pKey->pks[0].type, code, lino, _end, TSDB_CODE_INVALID_PARA);
}
uint8_t *pData = 0;
uint32_t nData = 0;
if (IS_VAR_DATA_TYPE(type)) {
pData = pKey->pks[0].pData;
buf = taosDecodeFixedU32(buf, &nData);
pKey->pks[0].nData = nData;
} else {
pKey->pks[0].val = 0;
pData = (uint8_t*)&pKey->pks[0].val;
nData = tDataTypes[type].bytes;
}
buf = taosDecodeBinary(buf, (void **)&pData, nData);
}
}
TSDB_CHECK_CONDITION(buf == begin + len , code, lino, _end, TSDB_CODE_INVALID_PARA);
_end:
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
#endif
int32_t tsdbReaderGetProgress(const STsdbReader* pReader, void** pBuf, uint64_t* pLen) {
int32_t code = TSDB_CODE_SUCCESS;
#if 0
int32_t lino = 0;
void* buf = NULL;
int32_t len = 0;
TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(pBuf, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(pLen, code, lino, _end, TSDB_CODE_INVALID_PARA);
*pBuf = NULL;
*pLen = tsdbEncodeLastProcKeys(pReader->status.pTableMap, NULL);
buf = taosMemoryMalloc(*pLen);
TSDB_CHECK_NULL(buf, code, lino, _end, terrno);
*pBuf = buf;
len = tsdbEncodeLastProcKeys(pReader->status.pTableMap, &buf);
TSDB_CHECK_CONDITION(len == *pLen, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
buf = NULL;
_end:
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (buf != NULL) {
taosMemoryFreeClear(buf);
}
#endif
return code;
}
int32_t tsdbReaderSetProgress(STsdbReader* pReader, const void* buf, uint64_t len) {
int32_t code = TSDB_CODE_SUCCESS;
#if 0
int32_t lino = 0;
if (len == 0) {
goto _end;
}
TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(buf, code, lino, _end, TSDB_CODE_INVALID_PARA);
code = tsdbDecodeLastProcKeys(buf, len, pReader->status.pTableMap);
TSDB_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
#endif
return code;
}

31
source/dnode/vnode/src/vnd/vnodeInitApi.c
View File

@ -69,6 +69,9 @@ void initTsdbReaderAPI(TsdReader* pReader) {
pReader->fileSetReadNext = tsdbFileSetReaderNext;
pReader->fileSetGetEntryField = tsdbFileSetGetEntryField;
pReader->fileSetReaderClose = tsdbFileSetReaderClose;
pReader->getProgress = (int32_t (*)(const void*, void**, uint64_t*))tsdbReaderGetProgress;
pReader->setProgress = (int32_t (*)(void*, const void*, uint64_t))tsdbReaderSetProgress;
}
void initMetadataAPI(SStoreMeta* pMeta) {
@ -158,6 +161,7 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStatePutParName = streamStatePutParName;
pStore->streamStateGetParName = streamStateGetParName;
pStore->streamStateDeleteParName = streamStateDeleteParName;
pStore->streamStateSetParNameInvalid = streamStateSetParNameInvalid;
pStore->streamStateAddIfNotExist = streamStateAddIfNotExist;
pStore->streamStateReleaseBuf = streamStateReleaseBuf;
@ -173,8 +177,11 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateClear = streamStateClear;
pStore->streamStateSaveInfo = streamStateSaveInfo;
pStore->streamStateGetInfo = streamStateGetInfo;
pStore->streamStateGetNumber = streamStateGetNumber;
pStore->streamStateDeleteInfo = streamStateDeleteInfo;
pStore->streamStateSetNumber = streamStateSetNumber;
pStore->streamStateGetPrev = streamStateGetPrev;
pStore->streamStateGetAllPrev = streamStateGetAllPrev;
pStore->streamStateFillPut = streamStateFillPut;
pStore->streamStateFillGet = streamStateFillGet;
@ -196,6 +203,9 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateGetKVByCur = streamStateGetKVByCur;
pStore->streamStateClearExpiredState = streamStateClearExpiredState;
pStore->streamStateClearExpiredSessionState = streamStateClearExpiredSessionState;
pStore->streamStateSetRecFlag = streamStateSetRecFlag;
pStore->streamStateGetRecFlag = streamStateGetRecFlag;
pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist;
pStore->streamStateSessionPut = streamStateSessionPut;
@ -208,6 +218,9 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange;
pStore->streamStateCountGetKeyByRange = streamStateCountGetKeyByRange;
pStore->streamStateSessionAllocWinBuffByNextPosition = streamStateSessionAllocWinBuffByNextPosition;
pStore->streamStateSessionSaveToDisk = streamStateSessionSaveToDisk;
pStore->streamStateFlushReaminInfoToDisk = streamStateFlushReaminInfoToDisk;
pStore->streamStateSessionDeleteAll = streamStateSessionDeleteAll;
pStore->streamStateCountWinAddIfNotExist = streamStateCountWinAddIfNotExist;
pStore->streamStateCountWinAdd = streamStateCountWinAdd;
@ -242,6 +255,24 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamFileStateClear = streamFileStateClear;
pStore->needClearDiskBuff = needClearDiskBuff;
pStore->streamStateGetAndSetTsData = streamStateGetAndSetTsData;
pStore->streamStateTsDataCommit = streamStateTsDataCommit;
pStore->streamStateInitTsDataState = streamStateInitTsDataState;
pStore->streamStateDestroyTsDataState = streamStateDestroyTsDataState;
pStore->streamStateRecoverTsData = streamStateRecoverTsData;
pStore->streamStateReloadTsDataState = streamStateReloadTsDataState;
pStore->streamStateMergeAndSaveScanRange = streamStateMergeAndSaveScanRange;
pStore->streamStateMergeAllScanRange = streamStateMergeAllScanRange;
pStore->streamStatePopScanRange = streamStatePopScanRange;
pStore->streamStateCheckSessionState = streamStateCheckSessionState;
pStore->streamStateGetLastStateCur = streamStateGetLastStateCur;
pStore->streamStateLastStateCurNext = streamStateLastStateCurNext;
pStore->streamStateNLastStateGetKVByCur = streamStateNLastStateGetKVByCur;
pStore->streamStateGetLastSessionStateCur = streamStateGetLastSessionStateCur;
pStore->streamStateLastSessionStateCurNext = streamStateLastSessionStateCurNext;
pStore->streamStateNLastSessionStateGetKVByCur = streamStateNLastSessionStateGetKVByCur;
pStore->streamStateOpen = streamStateOpen;
pStore->streamStateClose = streamStateClose;
pStore->streamStateBegin = streamStateBegin;

70
source/libs/executor/inc/executorInt.h
View File

@ -393,6 +393,7 @@ typedef enum EStreamScanMode {
STREAM_SCAN_FROM_DELETE_DATA,
STREAM_SCAN_FROM_DATAREADER_RETRIEVE,
STREAM_SCAN_FROM_DATAREADER_RANGE,
STREAM_SCAN_FROM_CREATE_TABLERES,
} EStreamScanMode;
enum {
@ -416,6 +417,7 @@ typedef struct SStreamAggSupporter {
struct SUpdateInfo* pUpdateInfo;
int32_t windowCount;
int32_t windowSliding;
SStreamStateCur* pCur;
} SStreamAggSupporter;
typedef struct SWindowSupporter {
@ -459,8 +461,17 @@ typedef struct SStreamNotifyEventSupp {
typedef struct SSteamOpBasicInfo {
int32_t primaryPkIndex;
bool updateOperatorInfo;
int16_t operatorFlag;
SStreamNotifyEventSupp notifyEventSup;
bool recvCkBlock;
SSDataBlock* pCheckpointRes;
SSHashObj* pSeDeleted;
void* pDelIterator;
SSDataBlock* pDelRes;
SArray* pUpdated;
bool destHasPrimaryKey;
STableTsDataState* pTsDataState;
int32_t numOfRecv;
} SSteamOpBasicInfo;
typedef struct SStreamFillSupporter {
@ -485,8 +496,26 @@ typedef struct SStreamFillSupporter {
int32_t pkColBytes;
__compar_fn_t comparePkColFn;
int32_t* pOffsetInfo;
bool normalFill;
void* pEmptyRow;
SArray* pResultRange;
} SStreamFillSupporter;
typedef struct SStreamRecParam {
char pSql[2048];
int32_t sqlCapcity;
char pUrl[TSDB_EP_LEN + 17]; // "http://localhost:6041/rest/sql"
char pAuth[512 + 22]; // Authorization: Basic token
char pStbFullName[TSDB_TABLE_FNAME_LEN];
char pWstartName[TSDB_COL_NAME_LEN];
char pWendName[TSDB_COL_NAME_LEN];
char pGroupIdName[TSDB_COL_NAME_LEN];
char pIsWindowFilledName[TSDB_COL_NAME_LEN];
void* pIteData;
int32_t iter;
TSKEY gap;
} SStreamRecParam;
typedef struct SStreamScanInfo {
SSteamOpBasicInfo basic;
SExprInfo* pPseudoExpr;
@ -549,6 +578,17 @@ typedef struct SStreamScanInfo {
bool useGetResultRange;
STimeWindow lastScanRange;
SSDataBlock* pRangeScanRes; // update SSDataBlock
bool hasPart;
//nonblock data scan
TSKEY recalculateInterval;
__compar_fn_t comparePkColFn;
SScanRange curRange;
struct SOperatorInfo* pRecTableScanOp;
bool scanAllTables;
SSHashObj* pRecRangeMap;
SArray* pRecRangeRes;
SStreamRecParam recParam;
} SStreamScanInfo;
typedef struct {
@ -680,11 +720,20 @@ typedef struct SWindowRowsSup {
uint64_t groupId;
} SWindowRowsSup;
typedef struct SResultWindowInfo {
SRowBuffPos* pStatePos;
SSessionKey sessionWin;
bool isOutput;
} SResultWindowInfo;
typedef int32_t (*AggImplFn)(struct SOperatorInfo* pOperator, SSDataBlock* pBlock);
typedef struct SNonBlockAggSupporter {
int32_t numOfKeep;
TSKEY tsOfKeep;
AggImplFn pWindowAggFn;
SArray* pPullWins;
int32_t pullIndex;
SSDataBlock* pPullDataRes;
SSHashObj* pHistoryGroup;
SSHashObj* pPullDataMap;
int32_t numOfChild;
SStreamRecParam recParam;
} SNonBlockAggSupporter;
typedef struct SSessionAggOperatorInfo {
SOptrBasicInfo binfo;
@ -745,7 +794,7 @@ typedef struct SStreamSessionAggOperatorInfo {
int32_t order; // current SSDataBlock scan order
STimeWindowAggSupp twAggSup;
SSDataBlock* pWinBlock; // window result
SSDataBlock* pDelRes; // delete result
SSDataBlock* pDelRes; // delete result, refactor
SSDataBlock* pUpdateRes; // update window
bool returnUpdate;
SSHashObj* pStDeleted;
@ -766,6 +815,7 @@ typedef struct SStreamSessionAggOperatorInfo {
bool destHasPrimaryKey;
SSHashObj* pPkDeleted;
struct SOperatorInfo* pOperator;
SNonBlockAggSupporter nbSup;
} SStreamSessionAggOperatorInfo;
typedef struct SStreamStateAggOperatorInfo {
@ -795,6 +845,7 @@ typedef struct SStreamStateAggOperatorInfo {
bool destHasPrimaryKey;
struct SOperatorInfo* pOperator;
int64_t trueForLimit;
SNonBlockAggSupporter nbSup;
} SStreamStateAggOperatorInfo;
typedef struct SStreamEventAggOperatorInfo {
@ -828,6 +879,7 @@ typedef struct SStreamEventAggOperatorInfo {
SNodeList* pStartCondCols;
SNodeList* pEndCondCols;
int64_t trueForLimit;
SNonBlockAggSupporter nbSup;
} SStreamEventAggOperatorInfo;
typedef struct SStreamCountAggOperatorInfo {
@ -889,6 +941,7 @@ typedef struct SStreamFillOperatorInfo {
SGroupResInfo groupResInfo;
SStreamState* pState;
SStateStore stateStore;
SNonBlockAggSupporter nbSup;
} SStreamFillOperatorInfo;
typedef struct SStreamTimeSliceOperatorInfo {
@ -948,6 +1001,7 @@ typedef struct SStreamIntervalSliceOperatorInfo {
bool hasFill;
bool hasInterpoFunc;
int32_t* pOffsetInfo;
SNonBlockAggSupporter nbSup;
} SStreamIntervalSliceOperatorInfo;
#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED)
@ -1083,7 +1137,7 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, in
SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr,
SStorageAPI* pApi, int32_t tsIndex, int8_t stateType, int32_t ratio);
int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type,
int32_t tsColIndex, STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic);
int32_t tsColIndex, STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic, int64_t recalculateInterval);
int32_t getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins);
void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList);
void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey);

18
source/libs/executor/inc/operator.h
View File

@ -116,6 +116,10 @@ int32_t createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMerge
int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createSemiIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createFinalIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
int32_t createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
@ -146,7 +150,9 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode
int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createStreamIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** ppOptInfo);
int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);
@ -170,6 +176,16 @@ int32_t createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32_t numO
int32_t createStreamTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** ppOptInfo);
int32_t createSessionNonblockOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** ppOptInfo);
int32_t createSemiSessionNonblockOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** ppOptInfo);
int32_t createFinalSessionNonblockOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** ppOptInfo);
int32_t createStateNonblockOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** ppOptInfo);
int32_t createEventNonblockOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** ppOptInfo);
// clang-format on
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup,

1
source/libs/executor/inc/querytask.h
View File

@ -71,6 +71,7 @@ typedef struct {
SVersionRange fillHistoryVer;
STimeWindow fillHistoryWindow;
SStreamState* pState;
SStreamState* pOtherState;
int32_t eventTypes; // event types to notify
SSchemaWrapper* notifyResultSchema; // agg result to notify
char* stbFullName; // used to generate dest child table name

107
source/libs/executor/inc/streamexecutorInt.h
View File

@ -56,6 +56,15 @@ extern "C" {
#define IS_FILL_CONST_VALUE(type) \
((type == TSDB_FILL_NULL || type == TSDB_FILL_NULL_F || type == TSDB_FILL_SET_VALUE || type == TSDB_FILL_SET_VALUE_F))
#define IS_INVALID_RANGE(range) (range.pGroupIds == NULL)
//
enum {
STREAM_NORMAL_OPERATOR = 0,
STREAM_HISTORY_OPERATOR = 1,
STREAM_RECALCUL_OPERATOR = 2,
};
typedef struct SSliceRowData {
TSKEY key;
char pRowVal[];
@ -68,6 +77,34 @@ typedef struct SSlicePoint {
SRowBuffPos* pResPos;
} SSlicePoint;
typedef struct SInervalSlicePoint {
SSessionKey winKey;
bool *pFinished;
SSliceRowData* pLastRow;
SRowBuffPos* pResPos;
} SInervalSlicePoint;
typedef enum SIntervalSliceType {
INTERVAL_SLICE_START = 1,
INTERVAL_SLICE_END = 2,
} SIntervalSliceType;
typedef struct SStateWindowInfo {
SResultWindowInfo winInfo;
SStateKeys* pStateKey;
} SStateWindowInfo;
typedef struct SEventWinfowFlag {
bool startFlag;
bool endFlag;
} SEventWinfowFlag;
typedef struct SEventWindowInfo {
SResultWindowInfo winInfo;
SEventWinfowFlag* pWinFlag;
} SEventWindowInfo;
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo);
@ -75,6 +112,18 @@ int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo, const struct SOperato
void destroyStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo);
int32_t encodeStreamBasicInfo(void** buf, const SSteamOpBasicInfo* pBasicInfo);
int32_t decodeStreamBasicInfo(void** buf, SSteamOpBasicInfo* pBasicInfo);
void setFillHistoryOperatorFlag(SSteamOpBasicInfo* pBasicInfo);
bool isHistoryOperator(SSteamOpBasicInfo* pBasicInfo);
void setFinalOperatorFlag(SSteamOpBasicInfo* pBasicInfo);
bool isFinalOperator(SSteamOpBasicInfo* pBasicInfo);
bool needBuildAllResult(SSteamOpBasicInfo* pBasicInfo);
void setSemiOperatorFlag(SSteamOpBasicInfo* pBasicInfo);
bool isSemiOperator(SSteamOpBasicInfo* pBasicInfo);
void setRecalculateOperatorFlag(SSteamOpBasicInfo* pBasicInfo);
void unsetRecalculateOperatorFlag(SSteamOpBasicInfo* pBasicInfo);
bool isRecalculateOperator(SSteamOpBasicInfo* pBasicInfo);
void setSingleOperatorFlag(SSteamOpBasicInfo* pBasicInfo);
bool isSingleOperator(SSteamOpBasicInfo* pBasicInfo);
int64_t getDeleteMarkFromOption(SStreamNodeOption* pOption);
void removeDeleteResults(SSHashObj* pUpdatedMap, SArray* pDelWins);
@ -116,9 +165,6 @@ void transBlockToSliceResultRow(const SSDataBlock* pBlock, int32_t r
int32_t rowSize, void* pPkData, SColumnInfoData* pPkCol, int32_t* pCellOffsetInfo);
int32_t getQualifiedRowNumDesc(SExprSupp* pExprSup, SSDataBlock* pBlock, TSKEY* tsCols, int32_t rowId, bool ignoreNull);
int32_t createStreamIntervalSliceOperatorInfo(struct SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
struct SOperatorInfo** ppOptInfo);
int32_t buildAllResultKey(SStateStore* pStateStore, SStreamState* pState, TSKEY ts, SArray* pUpdated);
int32_t initOffsetInfo(int32_t** ppOffset, SSDataBlock* pRes);
TSKEY compareTs(void* pKey);
@ -144,6 +190,61 @@ int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEvent
STaskNotifyEventStat* pNotifyEventStat);
int32_t removeOutdatedNotifyEvents(STimeWindowAggSupp* pTwSup, SStreamNotifyEventSupp* sup,
STaskNotifyEventStat* pNotifyEventStat);
void doStreamIntervalSaveCheckpoint(struct SOperatorInfo* pOperator);
int32_t getIntervalSliceCurStateBuf(SStreamAggSupporter* pAggSup, SInterval* pInterval, bool needPrev, STimeWindow* pTWin, int64_t groupId,
SInervalSlicePoint* pCurPoint, SInervalSlicePoint* pPrevPoint, int32_t* pWinCode);
int32_t getIntervalSlicePrevStateBuf(SStreamAggSupporter* pAggSup, SInterval* pInterval, SWinKey* pCurKey,
SInervalSlicePoint* pPrevPoint);
bool isInterpoWindowFinished(SInervalSlicePoint* pPoint);
void resetIntervalSliceFunctionKey(SqlFunctionCtx* pCtx, int32_t numOfOutput);
int32_t setIntervalSliceOutputBuf(SStreamAggSupporter* pAggSup, SInervalSlicePoint* pPoint, SqlFunctionCtx* pCtx,
int32_t numOfOutput, int32_t* rowEntryInfoOffset);
void doSetElapsedEndKey(TSKEY winKey, SExprSupp* pSup);
void doStreamSliceInterpolation(SSliceRowData* pPrevWinVal, TSKEY winKey, TSKEY curTs, SSDataBlock* pDataBlock,
int32_t curRowIndex, SExprSupp* pSup, SIntervalSliceType type, int32_t* pOffsetInfo);
void setInterpoWindowFinished(SInervalSlicePoint* pPoint);
int32_t doStreamIntervalNonblockAggNext(struct SOperatorInfo* pOperator, SSDataBlock** ppRes);
void streamIntervalNonblockReleaseState(struct SOperatorInfo* pOperator);
void streamIntervalNonblockReloadState(struct SOperatorInfo* pOperator);
int32_t compareWinKey(void* pKey, void* data, int32_t index);
int32_t buildIntervalSliceResult(struct SOperatorInfo* pOperator, SSDataBlock** ppRes);
int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, STqReader* pReader, SStoreTqReader* pReaderFn);
int32_t rebuildDeleteBlockData(struct SSDataBlock* pBlock, STimeWindow* pWindow, const char* id);
int32_t deletePartName(SStateStore* pStore, SStreamState* pState, SSDataBlock* pBlock, int32_t *deleteNum);
int32_t doTableScanNext(struct SOperatorInfo* pOperator, SSDataBlock** ppRes);
void streamScanOperatorSaveCheckpoint(struct SStreamScanInfo* pInfo);
int32_t doStreamDataScanNext(struct SOperatorInfo* pOperator, SSDataBlock** ppRes);
void streamDataScanReleaseState(struct SOperatorInfo* pOperator);
void streamDataScanReloadState(struct SOperatorInfo* pOperator);
int32_t extractTableIdList(const STableListInfo* pTableListInfo, SArray** ppArrayRes);
int32_t colIdComparFn(const void* param1, const void* param2);
int32_t doBlockDataWindowFilter(SSDataBlock* pBlock, int32_t tsIndex, STimeWindow* pWindow, const char* id);
STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval,
SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup);
int32_t appendPkToSpecialBlock(SSDataBlock* pBlock, TSKEY* pTsArray, SColumnInfoData* pPkCol, int32_t rowId,
uint64_t* pUid, uint64_t* pGp, void* pTbName);
int32_t appendOneRowToSpecialBlockImpl(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, TSKEY* pCalStartTs,
TSKEY* pCalEndTs, uint64_t* pUid, uint64_t* pGp, void* pTbName, void* pPkData);
SSDataBlock* readPreVersionData(struct SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs,
int64_t maxVersion);
bool comparePrimaryKey(SColumnInfoData* pCol, int32_t rowId, void* pVal);
void releaseFlusedPos(void* pRes);
typedef int32_t (*__compare_fn_t)(void* pKey, void* data, int32_t index);
int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn);
void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, int64_t ver);
int32_t calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowId);
int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock, STimeWindow* pTimeWindow,
bool filter);
int32_t setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock);
int32_t createStreamDataScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond,
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo,
struct SOperatorInfo** pOptrInfo);
int32_t saveRecalculateData(SStateStore* pStateStore, STableTsDataState* pTsDataState, SSDataBlock* pSrcBlock, EStreamType mode);
void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock);
int32_t doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex,
SSDataBlock** ppRes);
void prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex, bool* pRes);
#ifdef __cplusplus
}

79
source/libs/executor/inc/streaminterval.h Normal file
View File

@ -0,0 +1,79 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef STREAM_INTERVAL_H
#define STREAM_INTERVAL_H
#ifdef __cplusplus
extern "C" {
#endif
#include "executorInt.h"
#include "operator.h"
typedef struct SPullWindowInfo {
STimeWindow window;
uint64_t groupId;
STimeWindow calWin;
} SPullWindowInfo;
typedef struct STimeFillRange {
TSKEY skey;
TSKEY ekey;
uint64_t groupId;
void* pStartRow;
void* pEndRow;
} STimeFillRange;
int32_t doStreamIntervalNonblockAggImpl(struct SOperatorInfo* pOperator, SSDataBlock* pBlock);
STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval);
int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
TSKEY* primaryKeys, int32_t prevPosition);
int32_t copyNewResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar);
int32_t copyRecDataToBuff(TSKEY calStart, TSKEY calEnd, uint64_t uid, uint64_t version, EStreamType mode,
const SColumnInfoData* pPkColDataInfo, int32_t rowId, SRecDataInfo* pValueBuff,
int32_t buffLen);
int32_t saveRecWindowToDisc(SSessionKey* pWinKey, uint64_t uid, EStreamType mode, STableTsDataState* pTsDataState,
SStreamAggSupporter* pAggSup);
int32_t initNonBlockAggSupptor(SNonBlockAggSupporter* pNbSup, SInterval* pInterval, SOperatorInfo* downstream);
void destroyNonBlockAggSupptor(SNonBlockAggSupporter* pNbSup);
int32_t buildRetriveRequest(SExecTaskInfo* pTaskInfo, SStreamAggSupporter* pAggSup, STableTsDataState* pTsDataState,
SNonBlockAggSupporter* pNbSup);
int32_t getChildIndex(SSDataBlock* pBlock);
void adjustDownstreamBasicInfo(SOperatorInfo* downstream, struct SSteamOpBasicInfo* pBasic);
int32_t processDataPullOver(SSDataBlock* pBlock, SSHashObj* pPullMap, SExecTaskInfo* pTaskInfo);
int32_t doStreamNonblockFillNext(SOperatorInfo* pOperator, SSDataBlock** ppRes);
int32_t doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock, SSDataBlock* pDstBlock);
void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo);
void removeDuplicateResult(SArray* pTsArrray, __compar_fn_t fn);
int32_t keepBlockRowInStateBuf(SStreamFillOperatorInfo* pInfo, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock,
TSKEY* tsCol, int32_t rowId, uint64_t groupId, int32_t rowSize);
void setTimeSliceFillRule(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, TSKEY ts);
void doStreamTimeSliceFillRange(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pRes);
void resetTimeSlicePrevAndNextWindow(SStreamFillSupporter* pFillSup);
TSKEY adustPrevTsKey(TSKEY pointTs, TSKEY rowTs, SInterval* pInterval);
TSKEY adustEndTsKey(TSKEY pointTs, TSKEY rowTs, SInterval* pInterval);
void destroyStreamFillOperatorInfo(void* param);
void destroyStreamNonblockFillOperatorInfo(void* param);
int32_t buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId, SSDataBlock* delRes);
void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo);
void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes);
int32_t initFillSupRowInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pRes);
void getStateKeepInfo(SNonBlockAggSupporter* pNbSup, bool isRecOp, int32_t* pNumRes, TSKEY* pTsRes);
#ifdef __cplusplus
}
#endif
#endif // STREAM_INTERVAL_H

89
source/libs/executor/inc/streamsession.h Normal file
View File

@ -0,0 +1,89 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef STREAM_SESSION_H
#define STREAM_SESSION_H
#ifdef __cplusplus
extern "C" {
#endif
#include "executorInt.h"
#include "operator.h"
#include "streamexecutorInt.h"
int32_t doStreamSessionNonblockAggNext(SOperatorInfo* pOperator, SSDataBlock** ppBlock);
void streamSessionNonblockReleaseState(SOperatorInfo* pOperator);
void streamSessionNonblockReloadState(SOperatorInfo* pOperator);
void streamStateNonblockReleaseState(SOperatorInfo* pOperator);
void streamStateNonblockReloadState(SOperatorInfo* pOperator);
void streamEventNonblockReleaseState(SOperatorInfo* pOperator);
void streamEventNonblockReloadState(SOperatorInfo* pOperator);
int32_t doStreamSessionNonblockAggNextImpl(SOperatorInfo* pOperator, SOptrBasicInfo* pBInfo, SSteamOpBasicInfo* pBasic,
SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup,
SGroupResInfo* pGroupResInfo, SNonBlockAggSupporter* pNbSup,
SExprSupp* pScalarSupp, SArray* pHistoryWins, SSDataBlock** ppRes);
int32_t updateSessionWindowInfo(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo, TSKEY* pStartTs,
TSKEY* pEndTs, uint64_t groupId, int32_t rows, int32_t start, int64_t gap,
SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted,
int32_t* pWinRos);
int32_t setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
SResultWindowInfo* pCurWin, int32_t* pWinCode);
void streamSessionReleaseState(SOperatorInfo* pOperator);
void streamStateReleaseState(SOperatorInfo* pOperator);
void streamEventReleaseState(SOperatorInfo* pOperator);
int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool addGap, int32_t* pWinNum, bool* pIsEnd);
int32_t compactStateWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin,
SSHashObj* pStUpdated, SSHashObj* pStDeleted);
int32_t getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SStateWindowInfo* pCurWin,
SStateWindowInfo* pNextWin);
bool compareWinStateKey(SStateKeys* left, SStateKeys* right);
void getNextStateWin(const SStreamAggSupporter* pAggSup, SStateWindowInfo* pNextWin, bool asc);
int32_t updateStateWindowInfo(SStreamAggSupporter* pAggSup, SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin,
TSKEY* pTs, uint64_t groupId, SColumnInfoData* pKeyCol, int32_t rows, int32_t start,
bool* allEqual, SSHashObj* pResultRows, SSHashObj* pSeUpdated, SSHashObj* pSeDeleted,
int32_t* pWinRows);
int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,
SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin, int32_t* pWinCode);
int32_t doStreamStateNonblockAggNext(SOperatorInfo* pOperator, SSDataBlock** ppBlock);
int32_t doStreamEventNonblockAggNext(SOperatorInfo* pOperator, SSDataBlock** ppBlock);
int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool* pIsEnd);
void setEventWindowFlag(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo);
int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey,
TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start,
SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild,
int32_t* pWinRow);
int32_t setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupId, bool* pStart, bool* pEnd,
int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey,
int32_t* pWinCode);
bool isWindowIncomplete(SEventWindowInfo* pWinInfo);
void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos,
SEventWindowInfo* pWinInfo);
// stream client
int32_t streamClientGetResultRange(SStreamRecParam* pParam, SSHashObj* pRangeMap, SArray* pRangeRes);
int32_t streamClientGetFillRange(SStreamRecParam* pParam, SWinKey* pKey, SArray* pRangeRes, void* pEmptyRow, int32_t size, int32_t* pOffsetInfo, int32_t numOfCols);
#ifdef __cplusplus
}
#endif
#endif // STREAM_SESSION_H

1
source/libs/executor/inc/tfill.h
View File

@ -80,6 +80,7 @@ typedef struct SFillInfo {
SFillTagColInfo* pTags; // tags value for filling gap
const char* id;
SExecTaskInfo* pTaskInfo;
int8_t isFilled;
} SFillInfo;
typedef struct SResultCellData {

18
source/libs/executor/src/executil.c
View File

@ -2973,8 +2973,24 @@ char* getStreamOpName(uint16_t opType) {
return "stream interp";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL:
return "interval continue";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL:
return "interval continue semi";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL:
return "interval continue final";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SESSION:
return "session continue";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION:
return "session continue semi";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_SESSION:
return "session continue final";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_STATE:
return "state continue";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_EVENT:
return "event continue";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_COUNT:
return "count continue";
}
return "";
return "error name";
}
void printDataBlock(SSDataBlock* pBlock, const char* flag, const char* taskIdStr) {

14
source/libs/executor/src/executor.c
View File

@ -183,12 +183,13 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
}
pInfo->blockType = STREAM_INPUT__DATA_BLOCK;
} else if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
} else if (type == STREAM_INPUT__CHECKPOINT_TRIGGER || type == STREAM_INPUT__RECALCULATE) {
SPackedData tmp = {.pDataBlock = input};
void* tmpItem = taosArrayPush(pInfo->pBlockLists, &tmp);
QUERY_CHECK_NULL(tmpItem, code, lino, _end, terrno);
pInfo->blockType = STREAM_INPUT__CHECKPOINT;
pInfo->blockType =
(type == STREAM_INPUT__CHECKPOINT_TRIGGER) ? STREAM_INPUT__CHECKPOINT : STREAM_INPUT__RECALCULATE;
} else {
code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
QUERY_CHECK_CODE(code, lino, _end);
@ -1161,21 +1162,22 @@ _end:
return code;
}
static int32_t getOpratorIntervalInfo(SOperatorInfo* pOperator, int64_t* pWaterMark, SInterval* pInterval, STimeWindow* pLastWindow) {
static int32_t getOpratorIntervalInfo(SOperatorInfo* pOperator, int64_t* pWaterMark, SInterval* pInterval, STimeWindow* pLastWindow, TSKEY* pRecInteral) {
if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
return getOpratorIntervalInfo(pOperator->pDownstream[0], pWaterMark, pInterval, pLastWindow);
return getOpratorIntervalInfo(pOperator->pDownstream[0], pWaterMark, pInterval, pLastWindow, pRecInteral);
}
SStreamScanInfo* pScanOp = (SStreamScanInfo*) pOperator->info;
*pWaterMark = pScanOp->twAggSup.waterMark;
*pInterval = pScanOp->interval;
*pLastWindow = pScanOp->lastScanRange;
*pRecInteral = pScanOp->recalculateInterval;
return TSDB_CODE_SUCCESS;
}
int32_t qGetStreamIntervalExecInfo(qTaskInfo_t tinfo, int64_t* pWaterMark, SInterval* pInterval, STimeWindow* pLastWindow) {
int32_t qGetStreamIntervalExecInfo(qTaskInfo_t tinfo, int64_t* pWaterMark, SInterval* pInterval, STimeWindow* pLastWindow, TSKEY* pRecInteral) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
SOperatorInfo* pOperator = pTaskInfo->pRoot;
return getOpratorIntervalInfo(pOperator, pWaterMark, pInterval, pLastWindow);
return getOpratorIntervalInfo(pOperator, pWaterMark, pInterval, pLastWindow, pRecInteral);
}
int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {

15
source/libs/executor/src/groupoperator.c
View File

@ -1393,10 +1393,14 @@ static int32_t buildStreamCreateTableResult(SOperatorInfo* pOperator) {
SExecTaskInfo* pTask = pOperator->pTaskInfo;
SStreamPartitionOperatorInfo* pInfo = pOperator->info;
SSDataBlock* pSrc = pInfo->pInputDataBlock;
if ((pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) ||
taosHashGetSize(pInfo->pPartitions) == 0) {
if ((pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0)) {
pTask->storageAPI.stateStore.streamStateSetParNameInvalid(pTask->streamInfo.pState);
goto _end;
}
if (taosHashGetSize(pInfo->pPartitions) == 0) {
goto _end;
}
blockDataCleanup(pInfo->pCreateTbRes);
code = blockDataEnsureCapacity(pInfo->pCreateTbRes, taosHashGetSize(pInfo->pPartitions));
QUERY_CHECK_CODE(code, lino, _end);
@ -1503,6 +1507,10 @@ static int32_t doStreamHashPartitionNext(SOperatorInfo* pOperator, SSDataBlock**
(*ppRes) = pInfo->pDelRes;
return code;
} break;
case STREAM_RECALCULATE_DATA:
case STREAM_RECALCULATE_DELETE:
case STREAM_RECALCULATE_START:
case STREAM_RECALCULATE_END:
case STREAM_CREATE_CHILD_TABLE:
case STREAM_RETRIEVE:
case STREAM_CHECKPOINT:
@ -1587,12 +1595,13 @@ int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pPar
pScanInfo->partitionSup = *pParSup;
pScanInfo->pPartScalarSup = pExpr;
pScanInfo->pPartTbnameSup = pTbnameExpr;
pScanInfo->hasPart = true;
for (int32_t j = 0; j < pResExprSupp->numOfExprs; j++) {
if (pScanInfo->primaryKeyIndex == pResExprSupp->pExprInfo[j].base.pParam[0].pCol->slotId) {
*pPkColIndex = j;
}
}
if (!pScanInfo->pUpdateInfo) {
if (!pScanInfo->pUpdateInfo && pScanInfo->twAggSup.calTrigger != STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
code = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate,
pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);
}

20
source/libs/executor/src/operator.c
View File

@ -560,7 +560,9 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand
SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
code = createIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) {
code = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle, &pOptr);
code = createStreamSingleIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL == type) {
code = createStreamIntervalSliceOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) {
SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode;
code = createMergeAlignedIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo, &pOptr);
@ -570,12 +572,16 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL == type) {
int32_t children = 0;
code = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL == type) {
code = createSemiIntervalSliceOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL == type) {
int32_t children = pHandle->numOfVgroups;
code = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL == type) {
int32_t children = pHandle->numOfVgroups;
code = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL == type) {
code = createFinalIntervalSliceOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) {
code = createSortOperatorInfo(ops[0], (SSortPhysiNode*)pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT == type) {
@ -633,6 +639,18 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand
code = createStreamTimeSliceOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY == type) {
code = createAnomalywindowOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SESSION == type) {
code = createSessionNonblockOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION == type) {
code = createSemiSessionNonblockOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_SESSION == type) {
code = createFinalSessionNonblockOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_STATE == type) {
code = createStateNonblockOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_EVENT == type) {
code = createEventNonblockOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_COUNT == type) {
//todo (liuyao) add
} else {
code = TSDB_CODE_INVALID_PARA;
pTaskInfo->code = code;

7
source/libs/executor/src/projectoperator.c
View File

@ -1014,7 +1014,12 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
} else if (pExpr[k].pExpr->nodeType == QUERY_NODE_FUNCTION) {
// _rowts/_c0, not tbname column
if (fmIsPseudoColumnFunc(pfCtx->functionId) && (!fmIsScanPseudoColumnFunc(pfCtx->functionId))) {
// do nothing
if (fmIsGroupIdFunc(pfCtx->functionId)) {
SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId);
TSDB_CHECK_NULL(pColInfoData, code, lino, _exit, terrno);
code = colDataSetVal(pColInfoData, pResult->info.rows, (const char*)&pSrcBlock->info.id.groupId, false);
TSDB_CHECK_CODE(code, lino, _exit);
}
} else if (fmIsIndefiniteRowsFunc(pfCtx->functionId)) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pfCtx);
code = pfCtx->fpSet.init(pfCtx, pResInfo);

1
source/libs/executor/src/querytask.c
View File

@ -108,6 +108,7 @@ int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHand
if (pHandle) {
if (pHandle->pStateBackend) {
(*pTaskInfo)->streamInfo.pState = pHandle->pStateBackend;
(*pTaskInfo)->streamInfo.pOtherState = pHandle->pOtherBackend;
}
}

83
source/libs/executor/src/scanoperator.c
View File

@ -1326,7 +1326,7 @@ _end:
return code;
}
static int32_t doTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t doTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
STableScanInfo* pInfo = pOperator->info;
@ -1619,7 +1619,9 @@ FORCE_INLINE void doClearBufferedBlocks(SStreamScanInfo* pInfo) {
}
static bool isSessionWindow(SStreamScanInfo* pInfo) {
return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION ||
pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION;
}
static bool isStateWindow(SStreamScanInfo* pInfo) {
@ -1631,7 +1633,10 @@ static bool isIntervalWindow(SStreamScanInfo* pInfo) {
return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL ||
pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL ||
pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL;
pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL ||
pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL ||
pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL ||
pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL;
}
static bool isSignleIntervalWindow(SStreamScanInfo* pInfo) {
@ -1665,7 +1670,7 @@ static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t grou
}
}
void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint64_t ver) {
void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, int64_t ver) {
pTableScanInfo->base.cond.twindows = *pWin;
pTableScanInfo->base.cond.startVersion = 0;
pTableScanInfo->base.cond.endVersion = ver;
@ -1677,7 +1682,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint6
pTableScanInfo->scanMode = TABLE_SCAN__BLOCK_ORDER;
}
static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs,
SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs,
int64_t maxVersion) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
@ -1754,7 +1759,7 @@ bool comparePrimaryKey(SColumnInfoData* pCol, int32_t rowId, void* pVal) {
return false;
}
bool hasPrimaryKeyCol(SStreamScanInfo* pInfo) { return pInfo->primaryKeyIndex != -1; }
static bool hasPrimaryKeyCol(SStreamScanInfo* pInfo) { return pInfo->primaryKeyIndex != -1; }
static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion, void* pVal) {
SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, uid, ts, ts, maxVersion);
@ -1794,7 +1799,7 @@ static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts,
return getGroupIdByUid(pInfo, uid);
}
static void prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex, bool* pRes) {
void prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex, bool* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pBlock->info.rows == 0) {
@ -1845,12 +1850,17 @@ static void prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
}
STableScanInfo* pTScanInfo = pInfo->pTableScanOp->info;
int64_t maxVer = -1;
if (!isRecalculateOperator(&pInfo->basic)) {
// coverity scan
QUERY_CHECK_NULL(pInfo->pUpdateInfo, code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRIu64, win.skey, win.ekey,
qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRId64, win.skey, win.ekey,
pInfo->pUpdateInfo->maxDataVersion);
resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion);
maxVer = pInfo->pUpdateInfo->maxDataVersion;
}
resetTableScanInfo(pInfo->pTableScanOp->info, &win, maxVer);
pInfo->pTableScanOp->status = OP_OPENED;
if (pRes) {
(*pRes) = true;
@ -1862,7 +1872,7 @@ _end:
}
}
static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval,
STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval,
SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) {
SResultRowInfo dumyInfo = {0};
dumyInfo.cur.pageId = -1;
@ -1896,7 +1906,7 @@ static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t
}
}
static int32_t doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex,
int32_t doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex,
SSDataBlock** ppRes) {
qDebug("do stream range scan. windows index:%d", *pRowIndex);
@ -2532,12 +2542,15 @@ _end:
return code;
}
static int32_t calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowId) {
int32_t calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowId) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
blockDataCleanup(pInfo->pCreateTbRes);
if (pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) {
pBlock->info.parTbName[0] = 0;
if (pInfo->hasPart == false) {
pInfo->stateStore.streamStateSetParNameInvalid(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState);
}
} else {
code = appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup,
&pInfo->tagCalSup, pBlock->info.id.groupId, pBlock, rowId, pInfo->pCreateTbRes,
@ -2806,7 +2819,7 @@ _end:
return code;
}
static int32_t doBlockDataWindowFilter(SSDataBlock* pBlock, int32_t tsIndex, STimeWindow* pWindow, const char* id) {
int32_t doBlockDataWindowFilter(SSDataBlock* pBlock, int32_t tsIndex, STimeWindow* pWindow, const char* id) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
bool* p = NULL;
@ -2908,7 +2921,7 @@ _end:
}
// re-build the delete block, ONLY according to the split timestamp
static int32_t rebuildDeleteBlockData(SSDataBlock* pBlock, STimeWindow* pWindow, const char* id) {
int32_t rebuildDeleteBlockData(SSDataBlock* pBlock, STimeWindow* pWindow, const char* id) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t numOfRows = pBlock->info.rows;
@ -2967,7 +2980,7 @@ _end:
return code;
}
static int32_t colIdComparFn(const void* param1, const void* param2) {
int32_t colIdComparFn(const void* param1, const void* param2) {
int32_t p1 = *(int32_t*)param1;
int32_t p2 = *(int32_t*)param2;
@ -2978,7 +2991,7 @@ static int32_t colIdComparFn(const void* param1, const void* param2) {
}
}
static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock, STimeWindow* pTimeWindow,
int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock, STimeWindow* pTimeWindow,
bool filter) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
@ -3242,10 +3255,9 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
return pRes;
}
static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, SStreamScanInfo* pInfo) {
int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, STqReader* pReader, SStoreTqReader* pReaderFn) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
STqReader* pReader = pInfo->tqReader;
int32_t rows = pSrc->info.rows;
code = blockDataEnsureCapacity(pDst, rows);
QUERY_CHECK_CODE(code, lino, _end);
@ -3263,7 +3275,7 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S
int32_t j = 0;
for (int32_t i = 0; i < rows; i++) {
if (pInfo->readerFn.tqReaderIsQueriedTable(pReader, uidCol[i])) {
if (pReaderFn->tqReaderIsQueriedTable(pReader, uidCol[i])) {
code = colDataSetVal(pDstStartCol, j, (const char*)&startCol[i], false);
QUERY_CHECK_CODE(code, lino, _end);
@ -3294,7 +3306,7 @@ _end:
}
// for partition by tag
static int32_t setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
int32_t setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
@ -3536,7 +3548,7 @@ static int32_t copyGetResultBlock(SSDataBlock* dest, TSKEY start, TSKEY end) {
return appendDataToSpecialBlock(dest, &start, &end, NULL, NULL, NULL);
}
static int32_t deletePartName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t *deleteNum) {
int32_t deletePartName(SStateStore* pStore, SStreamState* pState, SSDataBlock* pBlock, int32_t *deleteNum) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
@ -3546,7 +3558,7 @@ static int32_t deletePartName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32
int64_t* gpIdCol = (int64_t*)pGpIdCol->pData;
void* pParName = NULL;
int32_t winCode = 0;
code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, gpIdCol[i],
code = pStore->streamStateGetParName(pState, gpIdCol[i],
&pParName, false, &winCode);
if (TSDB_CODE_SUCCESS == code && winCode != 0) {
qDebug("delete stream part Name for:%"PRId64 " not found", gpIdCol[i]);
@ -3560,9 +3572,9 @@ static int32_t deletePartName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32
int64_t len = tsnprintf(varTbName + VARSTR_HEADER_SIZE, TSDB_TABLE_NAME_LEN + 1, "%s", pParName);
code = colDataSetVal(pTbnameCol, i, varTbName, false);
qDebug("delete stream part for:%"PRId64 " res tb: %s", gpIdCol[i], (char*)pParName);
pInfo->stateStore.streamStateFreeVal(pParName);
pStore->streamStateFreeVal(pParName);
QUERY_CHECK_CODE(code, lino, _end);
code = pInfo->stateStore.streamStateDeleteParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, gpIdCol[i]);
code = pStore->streamStateDeleteParName(pState, gpIdCol[i]);
QUERY_CHECK_CODE(code, lino, _end);
pBlock->info.id.groupId = gpIdCol[i];
// currently, only one valid row in pBlock
@ -3739,7 +3751,7 @@ FETCH_NEXT_BLOCK:
code = createSpecialDataBlock(STREAM_DELETE_DATA, &pDelBlock);
QUERY_CHECK_CODE(code, lino, _end);
code = filterDelBlockByUid(pDelBlock, pBlock, pInfo);
code = filterDelBlockByUid(pDelBlock, pBlock, pInfo->tqReader, &pInfo->readerFn);
QUERY_CHECK_CODE(code, lino, _end);
} else {
pDelBlock = pBlock;
@ -3817,7 +3829,7 @@ FETCH_NEXT_BLOCK:
} break;
case STREAM_DROP_CHILD_TABLE: {
int32_t deleteNum = 0;
code = deletePartName(pInfo, pBlock, &deleteNum);
code = deletePartName(&pInfo->stateStore, pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, pBlock, &deleteNum);
QUERY_CHECK_CODE(code, lino, _end);
if (deleteNum == 0) {
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "block recv", GET_TASKID(pTaskInfo));
@ -4079,7 +4091,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
int32_t code = doStreamScanNext(pOperator, &pRes);
return pRes;
}
static int32_t extractTableIdList(const STableListInfo* pTableListInfo, SArray** ppArrayRes) {
int32_t extractTableIdList(const STableListInfo* pTableListInfo, SArray** ppArrayRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SArray* tableIdList = taosArrayInit(4, sizeof(uint64_t));
@ -4501,9 +4513,9 @@ _end:
return code;
}
int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond,
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo,
SOperatorInfo** pOptrInfo) {
static int32_t createStreamNormalScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode,
SNode* pTagCond, STableListInfo* pTableListInfo,
SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) {
QRY_PARAM_CHECK(pOptrInfo);
int32_t code = TSDB_CODE_SUCCESS;
@ -4630,6 +4642,7 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode*
pTaskInfo->streamInfo.snapshotVer = pHandle->version;
pInfo->pCreateTbRes = buildCreateTableBlock(&pInfo->tbnameCalSup, &pInfo->tagCalSup);
QUERY_CHECK_NULL(pInfo->pCreateTbRes, code, lino, _error, terrno);
pInfo->hasPart = false;
code = blockDataEnsureCapacity(pInfo->pCreateTbRes, 8);
QUERY_CHECK_CODE(code, lino, _error);
@ -4748,6 +4761,16 @@ _error:
return code;
}
int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond,
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo,
SOperatorInfo** pOptrInfo) {
if (pTableScanNode->triggerType == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
return createStreamDataScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTableListInfo, pTaskInfo, pOptrInfo);
} else {
return createStreamNormalScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTableListInfo, pTaskInfo, pOptrInfo);
}
}
static int32_t doTagScanOneTable(SOperatorInfo* pOperator, SSDataBlock* pRes, SMetaReader* mr, SStorageAPI* pAPI) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;

300
source/libs/executor/src/streamclient.c Normal file
View File

@ -0,0 +1,300 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef WINDOWS
#include <curl/curl.h>
#endif
#include "executorInt.h"
#include "streamsession.h"
#include "tjson.h"
#ifndef WINDOWS
static int32_t buildSessionResultSql(SSHashObj* pRangeMap, SStreamRecParam* pParam, bool* pEnd) {
int64_t prevLen = 0;
int64_t len = 0;
(void)memset(pParam->pSql, 0, pParam->sqlCapcity);
*pEnd = true;
while ((pParam->pIteData = tSimpleHashIterate(pRangeMap, pParam->pIteData, &pParam->iter)) != NULL) {
SSessionKey* pKey = tSimpleHashGetKey(pParam->pIteData, NULL);
if (prevLen > 0) {
len += tsnprintf(pParam->pSql + len, pParam->sqlCapcity - len, " union all ");
if (len >= pParam->sqlCapcity - 1) {
*pEnd = false;
break;
}
}
len += tsnprintf(pParam->pSql + len, pParam->sqlCapcity - len,
"select %" PRId64 ", %" PRId64 ", `%s` , cast(`%s` as bigint), cast( (`%s` - %" PRId64
") as bigint) from %s where `%s` == %" PRIu64 " and ( (`%s` - %" PRId64 ") >= %" PRId64
" and `%s` <= %" PRId64 " )",
pKey->win.skey, pKey->win.ekey, pParam->pGroupIdName, pParam->pWstartName, pParam->pWendName,
pParam->gap, pParam->pStbFullName, pParam->pGroupIdName, pKey->groupId, pParam->pWendName,
pParam->gap, pKey->win.skey, pParam->pWstartName, pKey->win.ekey);
if (len >= pParam->sqlCapcity - 1) {
*pEnd = false;
break;
} else {
prevLen = len;
}
}
pParam->pSql[prevLen + 1] = '\0';
return TSDB_CODE_SUCCESS;
}
static size_t parseResult(char* pCont, size_t contLen, size_t nmemb, void* userdata) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
qDebug("stream client response is received, contLen:%d, nmemb:%d, pCont:%p", (int32_t)contLen, (int32_t)nmemb, pCont);
QUERY_CHECK_CONDITION(contLen > 0, code, lino, _end, TSDB_CODE_FAILED);
QUERY_CHECK_CONDITION(nmemb > CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
QUERY_CHECK_NULL(pCont, code, lino, _end, TSDB_CODE_FAILED);
qTrace("===stream=== result:%s", pCont);
(*(SJson**)userdata) = tjsonParse(pCont);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return 0;
}
return contLen * nmemb;
}
static int32_t doProcessSql(SStreamRecParam* pParam, SJson** ppJsonResult) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
CURL* pCurl = curl_easy_init();
QUERY_CHECK_NULL(pCurl, code, lino, _end, TSDB_CODE_FAILED);
CURLcode curlRes = curl_easy_setopt(pCurl, CURLOPT_URL, pParam->pUrl);
QUERY_CHECK_CONDITION(curlRes == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
struct curl_slist* pHeaders = NULL;
pHeaders = curl_slist_append(pHeaders, "Content-Type:application/json;charset=UTF-8");
QUERY_CHECK_NULL(pHeaders, code, lino, _end, TSDB_CODE_FAILED);
pHeaders = curl_slist_append(pHeaders, pParam->pAuth);
QUERY_CHECK_NULL(pHeaders, code, lino, _end, TSDB_CODE_FAILED);
curlRes = curl_easy_setopt(pCurl, CURLOPT_HTTPHEADER, pHeaders);
QUERY_CHECK_CONDITION(curlRes == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
curlRes = curl_easy_setopt(pCurl, CURLOPT_POSTFIELDS, pParam->pSql);
QUERY_CHECK_CONDITION(curlRes == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
qTrace("===stream=== sql:%s", pParam->pSql);
curlRes = curl_easy_setopt(pCurl, CURLOPT_FOLLOWLOCATION, 1L);
QUERY_CHECK_CONDITION(curlRes == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
curlRes = curl_easy_setopt(pCurl, CURLOPT_WRITEFUNCTION, parseResult);
QUERY_CHECK_CONDITION(curlRes == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
curlRes = curl_easy_setopt(pCurl, CURLOPT_WRITEDATA, ppJsonResult);
QUERY_CHECK_CONDITION(curlRes == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
curlRes = curl_easy_perform(pCurl);
QUERY_CHECK_CONDITION(curlRes == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
_end:
if (pHeaders != NULL) {
curl_slist_free_all(pHeaders);
}
if (pCurl != NULL) {
curl_easy_cleanup(pCurl);
}
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. error code:%d", __func__, lino, tstrerror(code), curlRes);
}
return code;
}
static int32_t doTransformResult(const SJson* pJsonResult, SArray* pRangeRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SJson* jArray = tjsonGetObjectItem(pJsonResult, "data");
QUERY_CHECK_NULL(jArray, code, lino, _end, TSDB_CODE_FAILED);
int32_t rows = tjsonGetArraySize(jArray);
if (rows > 0) {
for (int32_t i = 0; i < rows; ++i) {
SJson* pRow = tjsonGetArrayItem(jArray, i);
QUERY_CHECK_NULL(jArray, code, lino, _end, TSDB_CODE_FAILED);
int32_t cols = tjsonGetArraySize(pRow);
if (cols > 0) {
SArray* pRowArray = taosArrayInit(cols, sizeof(int64_t));
QUERY_CHECK_NULL(pRowArray, code, lino, _end, terrno);
void* tmpRes = taosArrayPush(pRangeRes, &pRowArray);
QUERY_CHECK_NULL(tmpRes, code, lino, _end, terrno);
for (int32_t j = 0; j < cols; ++j) {
SJson* pCell = tjsonGetArrayItem(pRow, j);
int64_t data = 0;
tjsonGetObjectValueBigInt(pCell, &data);
tmpRes = taosArrayPush(pRowArray, &data);
QUERY_CHECK_NULL(tmpRes, code, lino, _end, terrno);
}
}
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t streamClientGetResultRange(SStreamRecParam* pParam, SSHashObj* pRangeMap, SArray* pRangeRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
pParam->pIteData = NULL;
pParam->iter = 0;
bool isEnd = false;
while (!isEnd) {
code = buildSessionResultSql(pRangeMap, pParam, &isEnd);
QUERY_CHECK_CODE(code, lino, _end);
SJson* pJsRes = NULL;
code = doProcessSql(pParam, &pJsRes);
QUERY_CHECK_CODE(code, lino, _end);
code = doTransformResult(pJsRes, pRangeRes);
QUERY_CHECK_CODE(code, lino, _end);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static int32_t buildFillResultSql(SWinKey* pKey, SStreamRecParam* pParam) {
(void)memset(pParam->pSql, 0, pParam->sqlCapcity);
(void)tsnprintf(
pParam->pSql, pParam->sqlCapcity,
"(select *, cast(`%s` as bigint) from %s where `%s` == %" PRIu64 " and `%s` < %" PRId64
" and `%s` ==0 order by 1 desc limit 1) union all (select *, cast(`%s` as bigint) from %s where `%s` == %" PRIu64
" and `%s` > %" PRId64 " and `%s` ==0 order by 1 asc limit 1)",
pParam->pWstartName, pParam->pStbFullName, pParam->pGroupIdName, pKey->groupId, pParam->pWstartName, pKey->ts,
pParam->pIsWindowFilledName, pParam->pWstartName, pParam->pStbFullName, pParam->pGroupIdName, pKey->groupId,
pParam->pWstartName, pKey->ts, pParam->pIsWindowFilledName);
return TSDB_CODE_SUCCESS;
}
static int32_t jsonToDataCell(const SJson* pJson, SResultCellData* pCell) {
int32_t code = TSDB_CODE_SUCCESS;
if (IS_INTEGER_TYPE(pCell->type)) {
int64_t data = 0;
tjsonGetObjectValueBigInt(pJson, &data);
SET_TYPED_DATA(pCell->pData, pCell->type, data);
} else if (IS_FLOAT_TYPE(pCell->type)) {
double data = 0;
tjsonGetObjectValueDouble(pJson, &data);
SET_TYPED_DATA(pCell->pData, pCell->type, data);
} else if (IS_TIMESTAMP_TYPE(pCell->type)) {
} else if (IS_BOOLEAN_TYPE(pCell->type)) {
bool data = cJSON_IsTrue(pJson) ? true : false;
SET_TYPED_DATA(pCell->pData, pCell->type, data);
} else {
char* pStr = cJSON_GetStringValue(pJson);
STR_TO_VARSTR(pCell->pData, pStr);
}
return code;
}
static int32_t doTransformFillResult(const SJson* pJsonResult, SArray* pRangeRes, void* pEmptyRow, int32_t size,
int32_t* pOffsetInfo, int32_t numOfCols) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SJson* jArray = tjsonGetObjectItem(pJsonResult, "data");
QUERY_CHECK_NULL(jArray, code, lino, _end, TSDB_CODE_FAILED);
int32_t rows = tjsonGetArraySize(jArray);
if (rows > 0) {
for (int32_t i = 0; i < rows; ++i) {
SJson* pRow = tjsonGetArrayItem(jArray, i);
QUERY_CHECK_NULL(jArray, code, lino, _end, TSDB_CODE_FAILED);
int32_t cols = tjsonGetArraySize(pRow);
SSliceRowData* pRowData = taosMemoryCalloc(1, sizeof(TSKEY) + size);
pRowData->key = INT64_MIN;
memcpy(pRowData->pRowVal, pEmptyRow, size);
for (int32_t j = 0; j < cols && j < numOfCols; ++j) {
SJson* pJsonCell = tjsonGetArrayItem(pRow, j);
QUERY_CHECK_NULL(pJsonCell, code, lino, _end, TSDB_CODE_FAILED);
SResultCellData* pDataCell = getSliceResultCell((SResultCellData*)pRowData->pRowVal, j, pOffsetInfo);
QUERY_CHECK_NULL(pDataCell, code, lino, _end, TSDB_CODE_FAILED);
code = jsonToDataCell(pJsonCell, pDataCell);
QUERY_CHECK_CODE(code, lino, _end);
}
if (cols > 1) {
SJson* pJsonSKey = tjsonGetArrayItem(pRow, cols - 1);
QUERY_CHECK_NULL(pJsonSKey, code, lino, _end, TSDB_CODE_FAILED);
tjsonGetObjectValueBigInt(pJsonSKey, &pRowData->key);
}
void* pTempRes = taosArrayPush(pRangeRes, &pRowData);
QUERY_CHECK_NULL(pTempRes, code, lino, _end, terrno);
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t streamClientGetFillRange(SStreamRecParam* pParam, SWinKey* pKey, SArray* pRangeRes, void* pEmptyRow, int32_t size,
int32_t* pOffsetInfo, int32_t numOfCols) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
code = buildFillResultSql(pKey, pParam);
QUERY_CHECK_CODE(code, lino, _end);
SJson* pJsRes = NULL;
code = doProcessSql(pParam, &pJsRes);
QUERY_CHECK_CODE(code, lino, _end);
code = doTransformFillResult(pJsRes, pRangeRes, pEmptyRow, size, pOffsetInfo, numOfCols);
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
#else
int32_t streamClientGetResultRange(SStreamRecParam* pParam, SSHashObj* pRangeMap, SArray* pRangeRes) {
return TSDB_CODE_FAILED;
}
int32_t streamClientGetFillRange(SStreamRecParam* pParam, SWinKey* pKey, SArray* pRangeRes, void* pEmptyRow, int32_t size, int32_t* pOffsetInfo, int32_t numOfCols) {
return TSDB_CODE_FAILED;
}
#endif

2
source/libs/executor/src/streamcountwindowoperator.c
View File

@ -984,7 +984,7 @@ int32_t createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
if (downstream) {
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
&pInfo->twAggSup, &pInfo->basic, 0);
QUERY_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);

262
source/libs/executor/src/streameventnonblockoperator.c Normal file
View File

@ -0,0 +1,262 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorInt.h"
#include "filter.h"
#include "function.h"
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
#include "streamexecutorInt.h"
#include "streaminterval.h"
#include "streamsession.h"
#include "tchecksum.h"
#include "tcommon.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "tglobal.h"
#include "tlog.h"
#include "ttime.h"
#define STREAM_EVENT_NONBLOCK_OP_STATE_NAME "StreamEventNonblockHistoryState"
void streamEventNonblockReleaseState(SOperatorInfo* pOperator) {
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
pAggSup->stateStore.streamStateClearExpiredSessionState(pAggSup->pState, pInfo->nbSup.numOfKeep,
pInfo->nbSup.tsOfKeep, NULL);
streamEventReleaseState(pOperator);
qDebug("===stream===%s streamEventNonblockReleaseState:%" PRId64, GET_TASKID(pOperator->pTaskInfo),
pInfo->twAggSup.maxTs);
}
void streamEventNonblockReloadState(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
int32_t size = 0;
void* pBuf = NULL;
resetWinRange(&pAggSup->winRange);
code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_EVENT_NONBLOCK_OP_STATE_NAME,
strlen(STREAM_EVENT_NONBLOCK_OP_STATE_NAME), &pBuf, &size);
QUERY_CHECK_CODE(code, lino, _end);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
qDebug("===stream===%s event window operator reload state. get result count:%d", GET_TASKID(pOperator->pTaskInfo),
num);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);
pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts);
for (int32_t i = 0; i < num; i++) {
SEventWindowInfo curInfo = {0};
qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey,
pSeKeyBuf[i].groupId, i);
code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo);
QUERY_CHECK_CODE(code, lino, _end);
// event window has been deleted
if (!IS_VALID_SESSION_WIN(curInfo.winInfo)) {
continue;
}
setEventWindowFlag(pAggSup, &curInfo);
if (!curInfo.pWinFlag->startFlag || curInfo.pWinFlag->endFlag) {
code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo);
QUERY_CHECK_CODE(code, lino, _end);
continue;
}
bool isEnd = true;
code = compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, &isEnd);
QUERY_CHECK_CODE(code, lino, _end);
qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,
curInfo.winInfo.sessionWin.groupId);
if (IS_VALID_SESSION_WIN(curInfo.winInfo)) {
code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo);
QUERY_CHECK_CODE(code, lino, _end);
}
if (!curInfo.pWinFlag->endFlag) {
continue;
}
if (isEnd) {
code = saveDeleteRes(pInfo->basic.pSeDeleted, curInfo.winInfo.sessionWin);
QUERY_CHECK_CODE(code, lino, _end);
qDebug("===stream=== reload state. save delete result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,
curInfo.winInfo.sessionWin.groupId);
} else {
void* pResPtr = taosArrayPush(pInfo->basic.pUpdated, &curInfo.winInfo);
QUERY_CHECK_NULL(pResPtr, code, lino, _end, terrno);
reuseOutputBuf(pAggSup->pState, curInfo.winInfo.pStatePos, &pAggSup->stateStore);
qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,
curInfo.winInfo.sessionWin.groupId);
}
}
taosMemoryFreeClear(pBuf);
SOperatorInfo* downstream = pOperator->pDownstream[0];
if (downstream->fpSet.reloadStreamStateFn) {
downstream->fpSet.reloadStreamStateFn(downstream);
}
qDebug("===stream===%s streamEventNonblockReloadState", GET_TASKID(pOperator->pTaskInfo));
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
}
int32_t doStreamEventNonblockAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamEventAggOperatorInfo* pInfo = (SStreamEventAggOperatorInfo*)pOperator->info;
SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo);
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SExprSupp* pSup = &pOperator->exprSupp;
int32_t numOfOutput = pSup->numOfExprs;
int64_t groupId = pBlock->info.id.groupId;
SColumnInfoData* pColStart = NULL;
SColumnInfoData* pColEnd = NULL;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SResultRow* pResult = NULL;
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
if (pAggSup->winRange.ekey <= 0) {
pAggSup->winRange.ekey = INT64_MAX;
}
if (pAggSup->winRange.skey != INT64_MIN && pInfo->nbSup.pHistoryGroup == NULL) {
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->nbSup.pHistoryGroup = tSimpleHashInit(1024, hashFn);
}
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
TSKEY* tsCols = (int64_t*)pStartTsCol->pData;
SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock};
code = filterSetDataFromSlotId(pInfo->pStartCondInfo, &paramStart);
if (code != TSDB_CODE_SUCCESS) {
qError("set data from start slotId error.");
goto _end;
}
int32_t statusStart = 0;
code = filterExecute(pInfo->pStartCondInfo, pBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart);
QUERY_CHECK_CODE(code, lino, _end);
SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock};
code = filterSetDataFromSlotId(pInfo->pEndCondInfo, &paramEnd);
if (code != TSDB_CODE_SUCCESS) {
qError("set data from end slotId error.");
goto _end;
}
int32_t statusEnd = 0;
code = filterExecute(pInfo->pEndCondInfo, pBlock, &pColEnd, NULL, paramEnd.numOfCols, &statusEnd);
QUERY_CHECK_CODE(code, lino, _end);
int32_t rows = pBlock->info.rows;
int32_t winRows = 0;
for (int32_t i = 0; i < rows; i += winRows) {
int32_t winIndex = 0;
bool allEqual = true;
SEventWindowInfo curWin = {0};
SSessionKey nextWinKey = {0};
int32_t winCode = TSDB_CODE_SUCCESS;
code = setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin,
&nextWinKey, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS) {
SStreamStateCur* pCur =
pAggSup->stateStore.streamStateSessionSeekKeyPrev(pAggSup->pState, &curWin.winInfo.sessionWin);
int32_t size = 0;
SResultWindowInfo prevWinInfo = {.sessionWin.groupId = groupId};
int32_t tmpWinCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &prevWinInfo.sessionWin,
(void**)&prevWinInfo.pStatePos, &size);
if (tmpWinCode == TSDB_CODE_SUCCESS) {
SEventWindowInfo prevWin = {0};
setEventWindowInfo(pAggSup, &prevWinInfo.sessionWin, prevWinInfo.pStatePos, &prevWin);
if (isWindowIncomplete(&prevWin)) {
continue;
}
void* pResPtr = taosArrayPush(pInfo->basic.pUpdated, &prevWinInfo);
QUERY_CHECK_NULL(pResPtr, code, lino, _end, terrno);
reuseOutputBuf(pAggSup->pState, prevWinInfo.pStatePos, &pAggSup->stateStore);
int32_t mode = 0;
int32_t winRes = pAggSup->stateStore.streamStateGetRecFlag(pAggSup->pState, &prevWinInfo.sessionWin,
sizeof(SSessionKey), &mode);
if (winRes == TSDB_CODE_SUCCESS) {
code = saveRecWindowToDisc(&prevWinInfo.sessionWin, pBlock->info.id.uid, mode, pInfo->basic.pTsDataState,
pAggSup);
QUERY_CHECK_CODE(code, lino, _end);
}
}
}
bool rebuild = false;
code = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData,
rows, i, pAggSup->pResultRows, NULL, NULL, &rebuild, &winRows);
QUERY_CHECK_CODE(code, lino, _end);
code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,
pOperator, 0);
QUERY_CHECK_CODE(code, lino, _end);
code = saveSessionOutputBuf(pAggSup, &curWin.winInfo);
QUERY_CHECK_CODE(code, lino, _end);
releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAggSup->stateStore);
}
_end:
colDataDestroy(pColStart);
taosMemoryFree(pColStart);
colDataDestroy(pColEnd);
taosMemoryFree(pColEnd);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}
int32_t doStreamEventNonblockAggNext(SOperatorInfo* pOperator, SSDataBlock** ppBlock) {
SStreamEventAggOperatorInfo* pInfo = (SStreamEventAggOperatorInfo*)pOperator->info;
return doStreamSessionNonblockAggNextImpl(pOperator, &pInfo->binfo, &pInfo->basic, &pInfo->streamAggSup,
&pInfo->twAggSup, &pInfo->groupResInfo, &pInfo->nbSup, &pInfo->scalarSupp,
pInfo->historyWins, ppBlock);
}
int32_t createEventNonblockOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,
SReadHandle* pHandle, SOperatorInfo** ppOptInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
code = createStreamEventAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, ppOptInfo);
QUERY_CHECK_CODE(code, lino, _end);
SStreamEventAggOperatorInfo* pInfo = (SStreamEventAggOperatorInfo*)(*ppOptInfo)->info;
pInfo->nbSup.pWindowAggFn = doStreamEventNonblockAggImpl;
pInfo->nbSup.numOfKeep = 1;
pInfo->nbSup.tsOfKeep = INT64_MIN;
setSingleOperatorFlag(&pInfo->basic);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}

57
source/libs/executor/src/streameventwindowoperator.c
View File

@ -21,6 +21,8 @@
#include "operator.h"
#include "querytask.h"
#include "streamexecutorInt.h"
#include "streaminterval.h"
#include "streamsession.h"
#include "tchecksum.h"
#include "tcommon.h"
#include "tcompare.h"
@ -33,16 +35,6 @@
#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState"
#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint"
typedef struct SEventWinfowFlag {
bool startFlag;
bool endFlag;
} SEventWinfowFlag;
typedef struct SEventWindowInfo {
SResultWindowInfo winInfo;
SEventWinfowFlag* pWinFlag;
} SEventWindowInfo;
void destroyStreamEventOperatorInfo(void* param) {
if (param == NULL) {
return;
@ -101,6 +93,7 @@ void destroyStreamEventOperatorInfo(void* param) {
nodesDestroyList(pInfo->pEndCondCols);
pInfo->pEndCondCols = NULL;
}
destroyNonBlockAggSupptor(&pInfo->nbSup);
taosMemoryFreeClear(param);
}
@ -126,7 +119,7 @@ int32_t getEndCondIndex(bool* pEnd, int32_t start, int32_t rows) {
return -1;
}
static bool isWindowIncomplete(SEventWindowInfo* pWinInfo) {
bool isWindowIncomplete(SEventWindowInfo* pWinInfo) {
return !(pWinInfo->pWinFlag->startFlag && pWinInfo->pWinFlag->endFlag);
}
void reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) {
@ -283,8 +276,8 @@ _end:
return code;
}
static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool addGap) {
int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool* pIsEnd) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExprSupp* pSup = &pOperator->exprSupp;
@ -294,6 +287,10 @@ static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pC
SResultRow* pCurResult = NULL;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
if (pIsEnd != NULL) {
(*pIsEnd) = false;
}
while (1) {
if (!pCurWin->pWinFlag->startFlag || pCurWin->pWinFlag->endFlag) {
break;
@ -302,6 +299,9 @@ static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pC
getNextSessionWinInfo(pAggSup, pStUpdated, &pCurWin->winInfo, &nextWinInfo.winInfo);
if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) ||
!inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) {
if (pIsEnd != NULL && !IS_VALID_SESSION_WIN(nextWinInfo.winInfo)) {
(*pIsEnd) = true;
}
releaseOutputBuf(pAggSup->pState, nextWinInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);
break;
}
@ -443,7 +443,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
pOperator, 0);
QUERY_CHECK_CODE(code, lino, _end);
code = compactEventWindow(pOperator, &curWin, pInfo->pSeUpdated, pInfo->pSeDeleted, false);
code = compactEventWindow(pOperator, &curWin, pInfo->pSeUpdated, pInfo->pSeDeleted, NULL);
QUERY_CHECK_CODE(code, lino, _end);
code = saveSessionOutputBuf(pAggSup, &curWin.winInfo);
@ -900,7 +900,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
continue;
}
code = compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false);
code = compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, NULL);
qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,
curInfo.winInfo.sessionWin.groupId);
QUERY_CHECK_CODE(code, lino, _end);
@ -1020,7 +1020,7 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
goto _error;
}
if (pHandle) {
pInfo->isHistoryOp = pHandle->fillHistory;
pInfo->isHistoryOp = (pHandle->fillHistory == STREAM_HISTORY_OPERATOR);
}
if (pInfo->isHistoryOp) {
@ -1040,7 +1040,7 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
pInfo->destHasPrimaryKey = pEventNode->window.destHasPrimaryKey;
pInfo->pOperator = pOperator;
setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED,
setOperatorInfo(pOperator, "StreamEventAggOperator", nodeType(pPhyNode), true, OP_NOT_OPENED,
pInfo, pTaskInfo);
// for stream
void* buff = NULL;
@ -1054,15 +1054,28 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
QUERY_CHECK_CODE(code, lino, _error);
}
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAggNext, NULL, destroyStreamEventOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState);
initNonBlockAggSupptor(&pInfo->nbSup, NULL, NULL);
code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
if (pEventNode->window.triggerType == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
if (pHandle->fillHistory == STREAM_HISTORY_OPERATOR) {
setFillHistoryOperatorFlag(&pInfo->basic);
} else if (pHandle->fillHistory == STREAM_RECALCUL_OPERATOR) {
setRecalculateOperatorFlag(&pInfo->basic);
}
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doStreamEventNonblockAggNext, NULL, destroyStreamEventOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamEventNonblockReleaseState, streamEventNonblockReloadState);
} else {
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAggNext, NULL, destroyStreamEventOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState);
}
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
&pInfo->twAggSup, &pInfo->basic, 0);
QUERY_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);

99
source/libs/executor/src/streamexecutorInt.c
View File

@ -17,8 +17,16 @@
#include "executorInt.h"
#include "operator.h"
#include "cmdnodes.h"
#include "tdatablock.h"
#define UPDATE_OPERATOR_INFO BIT_FLAG_MASK(0)
#define FILL_HISTORY_OPERATOR BIT_FLAG_MASK(1)
#define RECALCULATE_OPERATOR BIT_FLAG_MASK(2)
#define SEMI_OPERATOR BIT_FLAG_MASK(3)
#define FINAL_OPERATOR BIT_FLAG_MASK(4)
#define SINGLE_OPERATOR BIT_FLAG_MASK(5)
#define NOTIFY_EVENT_NAME_CACHE_LIMIT_MB 16
typedef struct SStreamNotifyEvent {
@ -34,13 +42,13 @@ typedef struct SStreamNotifyEvent {
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type) {
if (type != STREAM_GET_ALL && type != STREAM_CHECKPOINT) {
pBasicInfo->updateOperatorInfo = true;
BIT_FLAG_SET_MASK(pBasicInfo->operatorFlag, UPDATE_OPERATOR_INFO);
}
}
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo) { return pBasicInfo->updateOperatorInfo; }
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo) { return BIT_FLAG_TEST_MASK(pBasicInfo->operatorFlag, UPDATE_OPERATOR_INFO); }
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo) { pBasicInfo->updateOperatorInfo = false; }
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo) { BIT_FLAG_UNSET_MASK(pBasicInfo->operatorFlag, UPDATE_OPERATOR_INFO); }
static void destroyStreamWindowEvent(void* ptr) {
SStreamNotifyEvent* pEvent = (SStreamNotifyEvent*)ptr;
@ -109,8 +117,20 @@ _end:
}
int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo, const struct SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
pBasicInfo->primaryPkIndex = -1;
pBasicInfo->updateOperatorInfo = false;
pBasicInfo->operatorFlag = 0;
code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pBasicInfo->pDelRes);
QUERY_CHECK_CODE(code, lino, _end);
pBasicInfo->pUpdated = taosArrayInit(1024, sizeof(SResultWindowInfo));
QUERY_CHECK_NULL(pBasicInfo->pUpdated, code, lino, _end, terrno);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pBasicInfo->pSeDeleted = tSimpleHashInit(32, hashFn);
const char* windowType = NULL;
if (IS_NORMAL_INTERVAL_OP(pOperator)) {
windowType = "Time";
@ -125,10 +145,50 @@ int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo, const struct SOperato
} else {
return TSDB_CODE_SUCCESS;
}
return initStreamNotifyEventSupp(&pBasicInfo->notifyEventSup, windowType, pOperator->resultInfo.capacity);
code = initStreamNotifyEventSupp(&pBasicInfo->notifyEventSup, windowType, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
void setFillHistoryOperatorFlag(SSteamOpBasicInfo* pBasicInfo) {
BIT_FLAG_SET_MASK(pBasicInfo->operatorFlag, FILL_HISTORY_OPERATOR);
}
bool isHistoryOperator(SSteamOpBasicInfo* pBasicInfo) {
return BIT_FLAG_TEST_MASK(pBasicInfo->operatorFlag, FILL_HISTORY_OPERATOR);
}
bool needBuildAllResult(SSteamOpBasicInfo* pBasicInfo) {
return BIT_FLAG_TEST_MASK(pBasicInfo->operatorFlag, FILL_HISTORY_OPERATOR) || BIT_FLAG_TEST_MASK(pBasicInfo->operatorFlag, SEMI_OPERATOR);
}
void setSemiOperatorFlag(SSteamOpBasicInfo* pBasicInfo) {
BIT_FLAG_SET_MASK(pBasicInfo->operatorFlag, SEMI_OPERATOR);
}
bool isSemiOperator(SSteamOpBasicInfo* pBasicInfo) {
return BIT_FLAG_TEST_MASK(pBasicInfo->operatorFlag, SEMI_OPERATOR);
}
void destroyStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo) {
blockDataDestroy(pBasicInfo->pCheckpointRes);
pBasicInfo->pCheckpointRes = NULL;
tSimpleHashCleanup(pBasicInfo->pSeDeleted);
pBasicInfo->pSeDeleted = NULL;
blockDataDestroy(pBasicInfo->pDelRes);
pBasicInfo->pDelRes = NULL;
taosArrayDestroyP(pBasicInfo->pUpdated, destroyFlusedPos);
pBasicInfo->pUpdated = NULL;
pBasicInfo->pTsDataState = NULL;
destroyStreamNotifyEventSupp(&pBasicInfo->notifyEventSup);
}
@ -853,3 +913,32 @@ _end:
}
return code;
}
void setFinalOperatorFlag(SSteamOpBasicInfo* pBasicInfo) {
BIT_FLAG_SET_MASK(pBasicInfo->operatorFlag, FINAL_OPERATOR);
}
bool isFinalOperator(SSteamOpBasicInfo* pBasicInfo) {
return BIT_FLAG_TEST_MASK(pBasicInfo->operatorFlag, FINAL_OPERATOR);
}
void setRecalculateOperatorFlag(SSteamOpBasicInfo* pBasicInfo) {
BIT_FLAG_SET_MASK(pBasicInfo->operatorFlag, RECALCULATE_OPERATOR);
}
void unsetRecalculateOperatorFlag(SSteamOpBasicInfo* pBasicInfo) {
BIT_FLAG_UNSET_MASK(pBasicInfo->operatorFlag, RECALCULATE_OPERATOR);
}
bool isRecalculateOperator(SSteamOpBasicInfo* pBasicInfo) {
return BIT_FLAG_TEST_MASK(pBasicInfo->operatorFlag, RECALCULATE_OPERATOR);
}
void setSingleOperatorFlag(SSteamOpBasicInfo* pBasicInfo) {
BIT_FLAG_SET_MASK(pBasicInfo->operatorFlag, SINGLE_OPERATOR);
}
bool isSingleOperator(SSteamOpBasicInfo* pBasicInfo) {
return BIT_FLAG_TEST_MASK(pBasicInfo->operatorFlag, SINGLE_OPERATOR);
}

479
source/libs/executor/src/streamfillnonblockoperator.c Normal file
View File

@ -0,0 +1,479 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "filter.h"
#include "os.h"
#include "query.h"
#include "taosdef.h"
#include "tmsg.h"
#include "ttypes.h"
#include "executorInt.h"
#include "streamexecutorInt.h"
#include "streaminterval.h"
#include "streamsession.h"
#include "tcommon.h"
#include "thash.h"
#include "ttime.h"
#include "function.h"
#include "operator.h"
#include "querynodes.h"
#include "querytask.h"
#include "tdatablock.h"
#include "tfill.h"
static int32_t doStreamNonblockFillImpl(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamFillOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamFillSupporter* pFillSup = pInfo->pFillSup;
SStreamFillInfo* pFillInfo = pInfo->pFillInfo;
SSDataBlock* pBlock = pInfo->pSrcBlock;
uint64_t groupId = pBlock->info.id.groupId;
SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol);
TSKEY* tsCol = (TSKEY*)pTsCol->pData;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
code = keepBlockRowInStateBuf(pInfo, pFillInfo, pBlock, tsCol, i, groupId, pFillSup->rowSize);
QUERY_CHECK_CODE(code, lino, _end);
SWinKey key = {.groupId = groupId, .ts = tsCol[i]};
void* pPushRes = taosArrayPush(pInfo->pUpdated, &key);
QUERY_CHECK_NULL(pPushRes, code, lino, _end, terrno);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}
static int32_t getResultInfoFromState(SStateStore* pStateStore, SStreamState* pState, SStreamFillSupporter* pFillSup,
SWinKey* pKey) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
resetTimeSlicePrevAndNextWindow(pFillSup);
int32_t tmpRes = TSDB_CODE_SUCCESS;
SRowBuffPos* pStatePos = NULL;
int32_t curVLen = 0;
code = pStateStore->streamStateFillGet(pState, pKey, (void**)&pStatePos, &curVLen, &tmpRes);
QUERY_CHECK_CODE(code, lino, _end);
bool hasCurKey = true;
if (tmpRes == TSDB_CODE_SUCCESS) {
pFillSup->cur.key = pKey->ts;
pFillSup->cur.pRowVal = (SResultCellData*)pStatePos->pRowBuff;
} else {
qDebug("streamStateFillGet key failed, Data may be deleted. ts:%" PRId64 ", groupId:%" PRId64, pKey->ts,
pKey->groupId);
pFillSup->cur.key = pKey->ts;
pFillSup->cur.pRowVal = NULL;
hasCurKey = false;
}
SWinKey preKey = {.groupId = pKey->groupId};
SRowBuffPos* pPrevStatePos = NULL;
int32_t preVLen = 0;
code = pStateStore->streamStateFillGetPrev(pState, pKey, &preKey, (void**)&pPrevStatePos, &preVLen, &tmpRes);
QUERY_CHECK_CODE(code, lino, _end);
qDebug("===stream=== set stream prev buf.ts:%" PRId64 ", groupId:%" PRIu64 ", res:%d", preKey.ts, preKey.groupId,
tmpRes);
if (tmpRes == TSDB_CODE_SUCCESS) {
pFillSup->prevOriginKey = preKey.ts;
pFillSup->prev.key = adustPrevTsKey(preKey.ts, preKey.ts, &pFillSup->interval);
pFillSup->prev.pRowVal = (SResultCellData*)pPrevStatePos->pRowBuff;
}
SWinKey nextKey = {.groupId = pKey->groupId};
SRowBuffPos* pNextStatePos = NULL;
int32_t nextVLen = 0;
code = pStateStore->streamStateFillGetNext(pState, pKey, &nextKey, (void**)&pNextStatePos, &nextVLen, &tmpRes);
QUERY_CHECK_CODE(code, lino, _end);
qDebug("===stream=== set stream next buf.ts:%" PRId64 ", groupId:%" PRIu64 ", res:%d", nextKey.ts, nextKey.groupId,
tmpRes);
if (tmpRes == TSDB_CODE_SUCCESS) {
pFillSup->nextOriginKey = nextKey.ts;
pFillSup->next.key = adustEndTsKey(nextKey.ts, nextKey.ts, &pFillSup->interval);
pFillSup->next.pRowVal = (SResultCellData*)pNextStatePos->pRowBuff;
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static int32_t getResultInfoFromResult(SStreamRecParam* pParam, SStreamFillSupporter* pFillSup, SWinKey* pKey) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
code = streamClientGetFillRange(pParam, pKey, pFillSup->pResultRange, pFillSup->pEmptyRow, pFillSup->rowSize, pFillSup->pOffsetInfo, pFillSup->numOfAllCols);
QUERY_CHECK_CODE(code, lino, _end);
for (int32_t i = 0; i < taosArrayGetSize(pFillSup->pResultRange); i++) {
SSliceRowData* pSRdata = taosArrayGetP(pFillSup->pResultRange, i);
if (pSRdata->key < pKey->ts) {
if (pFillSup->prev.key < pSRdata->key) {
pFillSup->prevOriginKey = pSRdata->key;
pFillSup->prev.key = adustPrevTsKey(pSRdata->key, pSRdata->key, &pFillSup->interval);
pFillSup->prev.pRowVal = (SResultCellData*)pSRdata->pRowVal;
}
} else if (pSRdata->key > pKey->ts) {
if ((pFillSup->next.key > pSRdata->key) || (!hasNextWindow(pFillSup) && pSRdata->key != INT64_MIN)) {
pFillSup->nextOriginKey = pSRdata->key;
pFillSup->next.key = adustEndTsKey(pSRdata->key, pSRdata->key, &pFillSup->interval);
pFillSup->next.pRowVal = (SResultCellData*)pSRdata->pRowVal;
}
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
void doBuildNonblockFillResult(SOperatorInfo* pOperator, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo,
SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamFillOperatorInfo* pInfo = pOperator->info;
blockDataCleanup(pBlock);
if (!hasRemainResults(pGroupResInfo)) {
return;
}
int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);
while (pGroupResInfo->index < numOfRows) {
SWinKey* pKey = (SWinKey*)taosArrayGet(pGroupResInfo->pRows, pGroupResInfo->index);
if (pBlock->info.id.groupId == 0) {
pBlock->info.id.groupId = pKey->groupId;
} else if (pBlock->info.id.groupId != pKey->groupId) {
break;
}
code = getResultInfoFromState(&pInfo->stateStore, pInfo->pState, pInfo->pFillSup, pKey);
QUERY_CHECK_CODE(code, lino, _end);
if (isRecalculateOperator(&pInfo->basic)) {
taosArrayClearP(pFillSup->pResultRange, NULL);
getResultInfoFromResult(&pInfo->nbSup.recParam, pFillSup, pKey);
}
setTimeSliceFillRule(pFillSup, pFillInfo, pKey->ts);
doStreamTimeSliceFillRange(pFillSup, pFillInfo, pBlock);
pGroupResInfo->index++;
if (pBlock->info.rows >= pBlock->info.capacity) {
break;
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
}
static int32_t buildNonblockFillResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamFillOperatorInfo* pInfo = pOperator->info;
uint16_t opType = pOperator->operatorType;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
doBuildNonblockFillResult(pOperator, pInfo->pFillSup, pInfo->pFillInfo, pInfo->pRes, &pInfo->groupResInfo);
if (pInfo->pRes->info.rows != 0) {
printDataBlock(pInfo->pRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
(*ppRes) = pInfo->pRes;
goto _end;
}
(*ppRes) = NULL;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static int32_t doDeleteNonblockFillResult(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SStreamFillOperatorInfo* pInfo = pOperator->info;
SStreamFillInfo* pFillInfo = pInfo->pFillInfo;
SStreamFillSupporter* pFillSup = pInfo->pFillSup;
SSDataBlock* pBlock = pInfo->pSrcDelBlock;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* tsStarts = (TSKEY*)pStartCol->pData;
SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* groupIds = (uint64_t*)pGroupCol->pData;
SResultRowData prev = {0};
SResultRowData next = {0};
taosArrayClearP(pFillSup->pResultRange, NULL);
while (pInfo->srcDelRowIndex < pBlock->info.rows) {
TSKEY curDelTs = tsStarts[pInfo->srcDelRowIndex];
uint64_t curDelGroupId = groupIds[pInfo->srcDelRowIndex];
SWinKey curDelKey = {.ts = curDelTs, .groupId = curDelGroupId};
resetTimeSlicePrevAndNextWindow(pFillSup);
getResultInfoFromResult(&pInfo->nbSup.recParam, pFillSup, &curDelKey);
prev = pFillSup->prev;
next = pFillSup->next;
pInfo->srcDelRowIndex++;
while (pInfo->srcDelRowIndex < pBlock->info.rows) {
TSKEY nextDelTs = tsStarts[pInfo->srcDelRowIndex];
uint64_t nextGroupId = groupIds[pInfo->srcDelRowIndex];
if (curDelGroupId != nextGroupId || next.key < nextDelTs) {
break;
}
taosArrayClear(pFillSup->pResultRange);
curDelKey.ts = nextDelTs;
getResultInfoFromResult(&pInfo->nbSup.recParam, pFillSup, &curDelKey);
void* pPrevRowData = taosArrayGetP(pFillSup->pResultRange, 0);
taosMemFreeClear(pPrevRowData);
if (hasNextWindow(pFillSup)) {
next = pFillSup->next;
}
pInfo->srcDelRowIndex++;
}
if (hasNextWindow(pFillSup)) {
STimeFillRange tw = {
.skey = prev.key,
.ekey = next.key,
.groupId = curDelGroupId,
.pStartRow = prev.pRowVal,
.pEndRow = next.pRowVal,
};
void* tmpRes = taosArrayPush(pInfo->pFillInfo->delRanges, &tw);
QUERY_CHECK_NULL(tmpRes, code, lino, _end, terrno);
} else {
code = buildDeleteResult(pOperator, prev.key, next.key, curDelGroupId, pInfo->pDelRes);
QUERY_CHECK_CODE(code, lino, _end);
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}
static void doFillDeleteRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pBlock) {
int32_t size = taosArrayGetSize(pFillInfo->delRanges);
while (pFillInfo->delIndex < size) {
STimeFillRange* range = taosArrayGet(pFillInfo->delRanges, pFillInfo->delIndex);
if (pBlock->info.id.groupId != 0 && pBlock->info.id.groupId != range->groupId) {
return;
}
pFillSup->prev.key = range->skey;
pFillSup->prev.pRowVal = range->pStartRow;
pFillSup->next.key = range->ekey;
pFillSup->next.pRowVal = range->pEndRow;
setDeleteFillValueInfo(range->skey, range->ekey, pFillSup, pFillInfo);
pFillInfo->delIndex++;
if (pFillInfo->needFill) {
doStreamFillRange(pFillInfo, pFillSup, pBlock);
pBlock->info.id.groupId = range->groupId;
}
}
}
int32_t initFillSupRowInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
code = initOffsetInfo(&pFillSup->pOffsetInfo, pRes);
QUERY_CHECK_CODE(code, lino, _end);
int32_t numOfCol = taosArrayGetSize(pRes->pDataBlock);
pFillSup->pEmptyRow = taosMemoryCalloc(1, pFillSup->rowSize);
QUERY_CHECK_NULL(pFillSup->pEmptyRow, code, lino, _end, lino);
for (int32_t i = 0; i < numOfCol; i++) {
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, i);
int32_t bytes = 1;
int32_t type = 1;
if (pColInfo != NULL) {
bytes = pColInfo->info.bytes;
type = pColInfo->info.type;
}
SResultCellData* pCell = getSliceResultCell(pFillSup->pEmptyRow, i, pFillSup->pOffsetInfo);
pCell->bytes = bytes;
pCell->type = type;
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t doStreamNonblockFillNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamFillOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
if (pOperator->status == OP_EXEC_DONE) {
(*ppRes) = NULL;
return code;
}
if (pOperator->status == OP_RES_TO_RETURN) {
if (isRecalculateOperator(&pInfo->basic)) {
blockDataCleanup(pInfo->pRes);
code = blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _end);
doFillDeleteRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes);
if (pInfo->pRes->info.rows > 0) {
printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pInfo->pRes;
return code;
}
}
if (hasRemainCalc(pInfo->pFillInfo) ||
(pInfo->pFillInfo->pos != FILL_POS_INVALID && pInfo->pFillInfo->needFill == true)) {
blockDataCleanup(pInfo->pRes);
doStreamTimeSliceFillRange(pInfo->pFillSup, pInfo->pFillInfo, pInfo->pRes);
if (pInfo->pRes->info.rows > 0) {
printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pInfo->pRes;
goto _end;
}
}
SSDataBlock* resBlock = NULL;
code = buildNonblockFillResult(pOperator, &resBlock);
QUERY_CHECK_CODE(code, lino, _end);
if (resBlock != NULL) {
(*ppRes) = resBlock;
goto _end;
}
pInfo->stateStore.streamStateClearExpiredState(pInfo->pState, pInfo->nbSup.numOfKeep, pInfo->nbSup.tsOfKeep);
resetStreamFillInfo(pInfo);
setStreamOperatorCompleted(pOperator);
(*ppRes) = NULL;
goto _end;
}
SSDataBlock* fillResult = NULL;
SOperatorInfo* downstream = pOperator->pDownstream[0];
while (1) {
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
pOperator->status = OP_RES_TO_RETURN;
qDebug("===stream===%s return data:%s.", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType));
break;
}
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(&pInfo->basic, pBlock->info.type);
switch (pBlock->info.type) {
case STREAM_PULL_DATA:
case STREAM_NORMAL:
case STREAM_INVALID: {
code = doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock);
QUERY_CHECK_CODE(code, lino, _end);
memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
pInfo->srcRowIndex = -1;
} break;
case STREAM_CHECKPOINT: {
pInfo->stateStore.streamStateCommit(pInfo->pState);
(*ppRes) = pBlock;
goto _end;
} break;
case STREAM_DELETE_RESULT: {
if (!isRecalculateOperator(&pInfo->basic)) {
qDebug("===stream===%s ignore recv block. type:%d", GET_TASKID(pTaskInfo), pBlock->info.type);
continue;
}
pInfo->pSrcDelBlock = pBlock;
pInfo->srcDelRowIndex = 0;
blockDataCleanup(pInfo->pDelRes);
pInfo->pFillSup->hasDelete = true;
code = doDeleteNonblockFillResult(pOperator);
QUERY_CHECK_CODE(code, lino, _end);
if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pInfo->pDelRes;
return code;
}
continue;
} break;
case STREAM_RETRIEVE:
case STREAM_CREATE_CHILD_TABLE: {
(*ppRes) = pBlock;
goto _end;
} break;
default:
qDebug("===stream===%s ignore recv block. type:%d", GET_TASKID(pTaskInfo), pBlock->info.type);
continue;
}
code = doStreamNonblockFillImpl(pOperator);
QUERY_CHECK_CODE(code, lino, _end);
}
removeDuplicateResult(pInfo->pUpdated, winKeyCmprImpl);
initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);
pInfo->groupResInfo.freeItem = false;
pInfo->pUpdated = taosArrayInit(1024, sizeof(SWinKey));
QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);
code = blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _end);
code = buildNonblockFillResult(pOperator, ppRes);
QUERY_CHECK_CODE(code, lino, _end);
if ((*ppRes) == NULL) {
pInfo->stateStore.streamStateClearExpiredState(pInfo->pState, pInfo->nbSup.numOfKeep, pInfo->nbSup.tsOfKeep);
resetStreamFillInfo(pInfo);
setStreamOperatorCompleted(pOperator);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
pTaskInfo->code = code;
}
return code;
}
void destroyStreamNonblockFillOperatorInfo(void* param) {
SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param;
resetTimeSlicePrevAndNextWindow(pInfo->pFillSup);
destroyStreamFillOperatorInfo(param);
}

80
source/libs/executor/src/streamfilloperator.c
View File

@ -22,6 +22,7 @@
#include "executorInt.h"
#include "streamexecutorInt.h"
#include "streaminterval.h"
#include "tcommon.h"
#include "thash.h"
#include "ttime.h"
@ -38,12 +39,6 @@
#define FILL_POS_MID 2
#define FILL_POS_END 3
typedef struct STimeRange {
TSKEY skey;
TSKEY ekey;
uint64_t groupId;
} STimeRange;
TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) {
STimeWindow win = {.skey = ts, .ekey = ts};
getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);
@ -101,6 +96,8 @@ void destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) {
taosMemoryFree(pFillSup->nextNext.pRowVal);
taosMemoryFree(pFillSup->pOffsetInfo);
taosArrayDestroy(pFillSup->pResultRange);
pFillSup->pResultRange = NULL;
taosMemoryFree(pFillSup);
}
@ -135,7 +132,7 @@ void destroyStreamFillInfo(SStreamFillInfo* pFillInfo) {
taosMemoryFree(pFillInfo);
}
static void destroyStreamFillOperatorInfo(void* param) {
void destroyStreamFillOperatorInfo(void* param) {
SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param;
destroyStreamFillInfo(pInfo->pFillInfo);
destroyStreamFillSupporter(pInfo->pFillSup);
@ -158,6 +155,8 @@ static void destroyStreamFillOperatorInfo(void* param) {
if (pInfo->pState != NULL) {
taosMemoryFreeClear(pInfo->pState);
}
destroyStreamBasicInfo(&pInfo->basic);
destroyNonBlockAggSupptor(&pInfo->nbSup);
taosMemoryFree(pInfo);
}
@ -482,7 +481,7 @@ _end:
}
static int32_t buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFillSup, TSKEY ts, SSDataBlock* pBlock,
bool* pRes) {
bool* pRes, bool isFilld) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pBlock->info.rows >= pBlock->info.capacity) {
@ -506,6 +505,7 @@ static int32_t buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pF
.currentKey = ts,
.order = TSDB_ORDER_ASC,
.interval = pFillSup->interval,
.isFilled = isFilld,
};
bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pColData, pBlock->info.rows);
if (!filled) {
@ -538,7 +538,7 @@ static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo*
STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current};
if (inWinRange(&pFillSup->winRange, &st)) {
bool res = true;
code = buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock, &res);
code = buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock, &res, true);
QUERY_CHECK_CODE(code, lino, _end);
}
pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
@ -575,6 +575,7 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo*
.currentKey = pFillInfo->current,
.order = TSDB_ORDER_ASC,
.interval = pFillSup->interval,
.isFilled = true,
};
int32_t slotId = GET_DEST_SLOT_ID(pFillCol);
@ -631,18 +632,18 @@ static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SRes
}
}
static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) {
void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
bool res = false;
if (pFillInfo->needFill == false) {
code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res);
code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res, false);
QUERY_CHECK_CODE(code, lino, _end);
return;
}
if (pFillInfo->pos == FILL_POS_START) {
code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res);
code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res, false);
QUERY_CHECK_CODE(code, lino, _end);
if (res) {
pFillInfo->pos = FILL_POS_INVALID;
@ -654,7 +655,7 @@ static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter*
doStreamFillLinear(pFillSup, pFillInfo, pRes);
if (pFillInfo->pos == FILL_POS_MID) {
code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res);
code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res, false);
QUERY_CHECK_CODE(code, lino, _end);
if (res) {
pFillInfo->pos = FILL_POS_INVALID;
@ -671,7 +672,7 @@ static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter*
}
}
if (pFillInfo->pos == FILL_POS_END) {
code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res);
code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res, false);
QUERY_CHECK_CODE(code, lino, _end);
if (res) {
pFillInfo->pos = FILL_POS_INVALID;
@ -810,8 +811,7 @@ _end:
return code;
}
static int32_t buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId,
SSDataBlock* delRes) {
int32_t buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId, SSDataBlock* delRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamFillOperatorInfo* pInfo = pOperator->info;
@ -851,7 +851,7 @@ static int32_t doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, T
code = buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes);
QUERY_CHECK_CODE(code, lino, _end);
} else {
STimeRange tw = {
STimeFillRange tw = {
.skey = startTs,
.ekey = endTs,
.groupId = groupId,
@ -898,7 +898,7 @@ static void doDeleteFillFinalize(SOperatorInfo* pOperator) {
SStreamFillInfo* pFillInfo = pInfo->pFillInfo;
int32_t size = taosArrayGetSize(pFillInfo->delRanges);
while (pFillInfo->delIndex < size) {
STimeRange* range = taosArrayGet(pFillInfo->delRanges, pFillInfo->delIndex);
STimeFillRange* range = taosArrayGet(pFillInfo->delRanges, pFillInfo->delIndex);
if (pInfo->pRes->info.id.groupId != 0 && pInfo->pRes->info.id.groupId != range->groupId) {
return;
}
@ -1003,7 +1003,7 @@ void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) {
pInfo->pFillInfo->delIndex = 0;
}
static int32_t doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock,
int32_t doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock,
SSDataBlock* pDstBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
@ -1205,7 +1205,7 @@ void doBuildForceFillResultImpl(SOperatorInfo* pOperator, SStreamFillSupporter*
if (winCode == TSDB_CODE_SUCCESS) {
pFillSup->cur.key = pKey->ts;
pFillSup->cur.pRowVal = pValPos->pRowBuff;
code = buildFillResult(&pFillSup->cur, pFillSup, pKey->ts, pBlock, &res);
code = buildFillResult(&pFillSup->cur, pFillSup, pKey->ts, pBlock, &res, false);
QUERY_CHECK_CODE(code, lino, _end);
resetForceFillWindow(&pFillSup->cur);
releaseOutputBuf(pInfo->pState, pValPos, &pInfo->stateStore);
@ -1220,12 +1220,12 @@ void doBuildForceFillResultImpl(SOperatorInfo* pOperator, SStreamFillSupporter*
pFillSup->cur.key = pKey->ts;
pFillSup->cur.pRowVal = prePos->pRowBuff;
if (pFillInfo->type == TSDB_FILL_PREV) {
code = buildFillResult(&pFillSup->cur, pFillSup, pKey->ts, pBlock, &res);
code = buildFillResult(&pFillSup->cur, pFillSup, pKey->ts, pBlock, &res, true);
QUERY_CHECK_CODE(code, lino, _end);
} else {
copyNotFillExpData(pFillSup, pFillInfo);
pFillInfo->pResRow->key = pKey->ts;
code = buildFillResult(pFillInfo->pResRow, pFillSup, pKey->ts, pBlock, &res);
code = buildFillResult(pFillInfo->pResRow, pFillSup, pKey->ts, pBlock, &res, true);
QUERY_CHECK_CODE(code, lino, _end);
}
resetForceFillWindow(&pFillSup->cur);
@ -1385,7 +1385,7 @@ _end:
return code;
}
static void removeDuplicateResult(SArray* pTsArrray, __compar_fn_t fn) {
void removeDuplicateResult(SArray* pTsArrray, __compar_fn_t fn) {
taosArraySort(pTsArrray, fn);
taosArrayRemoveDuplicate(pTsArrray, fn, NULL);
}
@ -1412,7 +1412,7 @@ static int32_t doStreamForceFillNext(SOperatorInfo* pOperator, SSDataBlock** ppR
goto _end;
}
pInfo->stateStore.streamStateClearExpiredState(pInfo->pState);
pInfo->stateStore.streamStateClearExpiredState(pInfo->pState, 1, INT64_MAX);
resetStreamFillInfo(pInfo);
setStreamOperatorCompleted(pOperator);
(*ppRes) = NULL;
@ -1484,7 +1484,7 @@ static int32_t doStreamForceFillNext(SOperatorInfo* pOperator, SSDataBlock** ppR
QUERY_CHECK_CODE(code, lino, _end);
if ((*ppRes) == NULL) {
pInfo->stateStore.streamStateClearExpiredState(pInfo->pState);
pInfo->stateStore.streamStateClearExpiredState(pInfo->pState, 1, INT64_MAX);
resetStreamFillInfo(pInfo);
setStreamOperatorCompleted(pOperator);
}
@ -1560,6 +1560,9 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod
pFillSup->pResMap = tSimpleHashInit(16, hashFn);
QUERY_CHECK_NULL(pFillSup->pResMap, code, lino, _end, terrno);
pFillSup->hasDelete = false;
pFillSup->normalFill = true;
pFillSup->pResultRange = taosArrayInit(2, POINTER_BYTES);
_end:
if (code != TSDB_CODE_SUCCESS) {
@ -1683,7 +1686,7 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
}
pFillInfo->type = pFillSup->type;
pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeRange));
pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeFillRange));
if (!pFillInfo->delRanges) {
code = terrno;
QUERY_CHECK_CODE(code, lino, _end);
@ -1736,18 +1739,20 @@ static void setValueForFillInfo(SStreamFillSupporter* pFillSup, SStreamFillInfo*
}
}
int32_t getDownStreamInfo(SOperatorInfo* downstream, int8_t* triggerType, SInterval* pInterval) {
int32_t getDownStreamInfo(SOperatorInfo* downstream, int8_t* triggerType, SInterval* pInterval, int16_t* pOperatorFlag) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (IS_NORMAL_INTERVAL_OP(downstream)) {
SStreamIntervalOperatorInfo* pInfo = downstream->info;
*triggerType = pInfo->twAggSup.calTrigger;
*pInterval = pInfo->interval;
*pOperatorFlag = pInfo->basic.operatorFlag;
} else if (IS_CONTINUE_INTERVAL_OP(downstream)) {
SStreamIntervalSliceOperatorInfo* pInfo = downstream->info;
*triggerType = pInfo->twAggSup.calTrigger;
*pInterval = pInfo->interval;
pInfo->hasFill = true;
*pOperatorFlag = pInfo->basic.operatorFlag;
} else {
code = TSDB_CODE_STREAM_INTERNAL_ERROR;
}
@ -1810,7 +1815,8 @@ int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysi
int8_t triggerType = 0;
SInterval interval = {0};
code = getDownStreamInfo(downstream, &triggerType, &interval);
int16_t opFlag = 0;
code = getDownStreamInfo(downstream, &triggerType, &interval, &opFlag);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->pFillSup = initStreamFillSup(pPhyFillNode, &interval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI,
@ -1861,7 +1867,7 @@ int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysi
QUERY_CHECK_CODE(code, lino, _error);
pInfo->srcRowIndex = -1;
setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo,
setOperatorInfo(pOperator, "StreamFillOperator", nodeType(pPhyFillNode), false, OP_NOT_OPENED, pInfo,
pTaskInfo);
if (triggerType == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) {
@ -1870,6 +1876,22 @@ int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysi
QUERY_CHECK_CODE(code, lino, _error);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamForceFillNext, NULL, destroyStreamFillOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
} else if (triggerType == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
code = initFillOperatorStateBuff(pInfo, pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.stateStore, pHandle,
GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI);
QUERY_CHECK_CODE(code, lino, _error);
initNonBlockAggSupptor(&pInfo->nbSup, &pInfo->pFillSup->interval, downstream);
code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->basic.operatorFlag = opFlag;
if (isFinalOperator(&pInfo->basic)) {
pInfo->nbSup.numOfKeep++;
}
code = initFillSupRowInfo(pInfo->pFillSup, pInfo->pRes);
QUERY_CHECK_CODE(code, lino, _error);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamNonblockFillNext, NULL, destroyStreamNonblockFillOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
} else {
pInfo->pState = NULL;
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamFillNext, NULL, destroyStreamFillOperatorInfo,

1088
source/libs/executor/src/streamintervalnonblockoperator.c Normal file
View File

File diff suppressed because it is too large Load Diff

217
source/libs/executor/src/streamintervalsliceoperator.c
View File

@ -18,6 +18,7 @@
#include "querytask.h"
#include "storageapi.h"
#include "streamexecutorInt.h"
#include "streaminterval.h"
#include "tcommon.h"
#include "tcompare.h"
#include "tdatablock.h"
@ -25,23 +26,9 @@
#define STREAM_INTERVAL_SLICE_OP_CHECKPOINT_NAME "StreamIntervalSliceOperator_Checkpoint"
typedef struct SInervalSlicePoint {
SSessionKey winKey;
bool *pFinished;
SSliceRowData* pLastRow;
SRowBuffPos* pResPos;
} SInervalSlicePoint;
void streamIntervalSliceReleaseState(SOperatorInfo* pOperator) {}
typedef enum SIntervalSliceType {
INTERVAL_SLICE_START = 1,
INTERVAL_SLICE_END = 2,
} SIntervalSliceType;
void streamIntervalSliceReleaseState(SOperatorInfo* pOperator) {
}
void streamIntervalSliceReloadState(SOperatorInfo* pOperator) {
}
void streamIntervalSliceReloadState(SOperatorInfo* pOperator) {}
void destroyStreamIntervalSliceOperatorInfo(void* param) {
SStreamIntervalSliceOperatorInfo* pInfo = (SStreamIntervalSliceOperatorInfo*)param;
@ -76,11 +63,12 @@ void destroyStreamIntervalSliceOperatorInfo(void* param) {
blockDataDestroy(pInfo->pCheckpointRes);
taosMemoryFreeClear(pInfo->pOffsetInfo);
destroyNonBlockAggSupptor(&pInfo->nbSup);
taosMemoryFreeClear(param);
}
static int32_t buildIntervalSliceResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t buildIntervalSliceResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamIntervalSliceOperatorInfo* pInfo = pOperator->info;
@ -92,10 +80,21 @@ static int32_t buildIntervalSliceResult(SOperatorInfo* pOperator, SSDataBlock**
bool addNotifyEvent = false;
addNotifyEvent = IS_NORMAL_INTERVAL_OP(pOperator) &&
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
doBuildDeleteResultImpl(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pState, pInfo->pDelWins, &pInfo->delIndex,
pInfo->pDelRes);
(*ppRes) = NULL;
if (isFinalOperator(&pInfo->basic)) {
doBuildPullDataBlock(pInfo->nbSup.pPullWins, &pInfo->nbSup.pullIndex, pInfo->nbSup.pPullDataRes);
if (pInfo->nbSup.pPullDataRes->info.rows != 0) {
printDataBlock(pInfo->nbSup.pPullDataRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
(*ppRes) = pInfo->nbSup.pPullDataRes;
return code;
}
}
if (pOperator->status == OP_RES_TO_RETURN) {
doBuildDeleteResultImpl(&pInfo->streamAggSup.stateStore, pTaskInfo->streamInfo.pState, pInfo->pDelWins,
&pInfo->delIndex, pInfo->pDelRes);
if (pInfo->pDelRes->info.rows != 0) {
// process the rest of the data
printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
if (addNotifyEvent) {
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
@ -104,6 +103,7 @@ static int32_t buildIntervalSliceResult(SOperatorInfo* pOperator, SSDataBlock**
(*ppRes) = pInfo->pDelRes;
return code;
}
}
doBuildStreamIntervalResult(pOperator, pInfo->streamAggSup.pState, pInfo->binfo.pRes, &pInfo->groupResInfo,
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
@ -139,7 +139,8 @@ _end:
// static void doStreamIntervalSliceSaveCheckpoint(SOperatorInfo* pOperator) {
// }
void initIntervalSlicePoint(SStreamAggSupporter* pAggSup, STimeWindow* pTWin, int64_t groupId, SInervalSlicePoint* pPoint) {
void initIntervalSlicePoint(SStreamAggSupporter* pAggSup, STimeWindow* pTWin, int64_t groupId,
SInervalSlicePoint* pPoint) {
pPoint->winKey.groupId = groupId;
pPoint->winKey.win = *pTWin;
pPoint->pFinished = POINTER_SHIFT(pPoint->pResPos->pRowBuff, pAggSup->resultRowSize - pAggSup->stateKeySize);
@ -176,18 +177,19 @@ _end:
return code;
}
static int32_t getIntervalSliceCurStateBuf(SStreamAggSupporter* pAggSup, SInterval* pInterval, bool needPrev, STimeWindow* pTWin, int64_t groupId,
SInervalSlicePoint* pCurPoint, SInervalSlicePoint* pPrevPoint, int32_t* pWinCode) {
int32_t getIntervalSliceCurStateBuf(SStreamAggSupporter* pAggSup, SInterval* pInterval, bool needPrev,
STimeWindow* pTWin, int64_t groupId, SInervalSlicePoint* pCurPoint,
SInervalSlicePoint* pPrevPoint, int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SWinKey curKey = {.ts = pTWin->skey, .groupId = groupId};
int32_t curVLen = 0;
code = pAggSup->stateStore.streamStateAddIfNotExist(pAggSup->pState, &curKey, (void**)&pCurPoint->pResPos,
&curVLen, pWinCode);
code = pAggSup->stateStore.streamStateAddIfNotExist(pAggSup->pState, &curKey, (void**)&pCurPoint->pResPos, &curVLen,
pWinCode);
QUERY_CHECK_CODE(code, lino, _end);
qDebug("===stream=== set stream twa cur point buf.ts:%" PRId64 ", groupId:%" PRIu64 ", res:%d",
curKey.ts, curKey.groupId, *pWinCode);
qDebug("===stream=== set stream twa cur point buf.ts:%" PRId64 ", groupId:%" PRIu64 ", res:%d", curKey.ts,
curKey.groupId, *pWinCode);
initIntervalSlicePoint(pAggSup, pTWin, groupId, pCurPoint);
@ -216,7 +218,8 @@ void doStreamSliceInterpolation(SSliceRowData* pPrevWinVal, TSKEY winKey, TSKEY
SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, pParam->pCol->slotId);
double prevVal = 0, curVal = 0, winVal = 0;
SResultCellData* pCell = getSliceResultCell((SResultCellData*)pPrevWinVal->pRowVal, pParam->pCol->slotId, pOffsetInfo);
SResultCellData* pCell =
getSliceResultCell((SResultCellData*)pPrevWinVal->pRowVal, pParam->pCol->slotId, pOffsetInfo);
GET_TYPED_DATA(prevVal, double, pCell->type, pCell->pData, typeGetTypeModFromColInfo(&pColInfo->info));
GET_TYPED_DATA(curVal, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex), typeGetTypeModFromColInfo(&pColInfo->info));
@ -248,17 +251,31 @@ void doSetElapsedEndKey(TSKEY winKey, SExprSupp* pSup) {
}
}
static void resetIntervalSliceFunctionKey(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
void resetIntervalSliceFunctionKey(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
for (int32_t k = 0; k < numOfOutput; ++k) {
pCtx[k].start.key = INT64_MIN;
pCtx[k].end.key = INT64_MIN;
}
}
int32_t setIntervalSliceOutputBuf(SInervalSlicePoint* pPoint, SqlFunctionCtx* pCtx, int32_t numOfOutput,
int32_t* rowEntryInfoOffset) {
static int32_t checkAndRecoverPointBuff(SStreamAggSupporter* pAggSup, SInervalSlicePoint* pPoint) {
int32_t code = TSDB_CODE_SUCCESS;
if (pPoint->pResPos->pRowBuff == NULL) {
void* pVal = NULL;
// recover curPoint.pResPos->pRowBuff
code = pAggSup->stateStore.streamStateGetByPos(pAggSup->pState, pPoint->pResPos, &pVal);
pPoint->pFinished = POINTER_SHIFT(pPoint->pResPos->pRowBuff, pAggSup->resultRowSize - pAggSup->stateKeySize);
pPoint->pLastRow = POINTER_SHIFT(pPoint->pFinished, sizeof(bool));
}
return code;
}
int32_t setIntervalSliceOutputBuf(SStreamAggSupporter* pAggSup, SInervalSlicePoint* pPoint, SqlFunctionCtx* pCtx,
int32_t numOfOutput, int32_t* rowEntryInfoOffset) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
checkAndRecoverPointBuff(pAggSup, pPoint);
SResultRow* res = pPoint->pResPos->pRowBuff;
// set time window for current result
@ -273,13 +290,9 @@ _end:
return code;
}
static void setInterpoWindowFinished(SInervalSlicePoint* pPoint) {
(*pPoint->pFinished) = true;
}
void setInterpoWindowFinished(SInervalSlicePoint* pPoint) { (*pPoint->pFinished) = true; }
static bool isInterpoWindowFinished(SInervalSlicePoint* pPoint) {
return *pPoint->pFinished;
}
bool isInterpoWindowFinished(SInervalSlicePoint* pPoint) { return *pPoint->pFinished; }
static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock, SSHashObj* pUpdatedMap,
SSHashObj* pDeletedMap) {
@ -302,12 +315,12 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc
TSKEY curTs = getStartTsKey(&pBlock->info.window, tsCols);
SInervalSlicePoint curPoint = {0};
SInervalSlicePoint prevPoint = {0};
STimeWindow curWin =
getActiveTimeWindow(NULL, pResultRowInfo, curTs, &pInfo->interval, TSDB_ORDER_ASC);
STimeWindow curWin = getActiveTimeWindow(NULL, pResultRowInfo, curTs, &pInfo->interval, TSDB_ORDER_ASC);
while (1) {
int32_t winCode = TSDB_CODE_SUCCESS;
if (curTs <= pInfo->endTs) {
code = getIntervalSliceCurStateBuf(&pInfo->streamAggSup, &pInfo->interval, pInfo->hasInterpoFunc, &curWin, groupId, &curPoint, &prevPoint, &winCode);
code = getIntervalSliceCurStateBuf(&pInfo->streamAggSup, &pInfo->interval, pInfo->hasInterpoFunc, &curWin,
groupId, &curPoint, &prevPoint, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
} else if (pInfo->hasInterpoFunc) {
SWinKey curKey = {.ts = curWin.skey, .groupId = groupId};
@ -315,16 +328,19 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc
QUERY_CHECK_CODE(code, lino, _end);
}
if (pInfo->hasInterpoFunc && IS_VALID_WIN_KEY(prevPoint.winKey.win.skey) && isInterpoWindowFinished(&prevPoint) == false) {
code = setIntervalSliceOutputBuf(&prevPoint, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);
if (pInfo->hasInterpoFunc && IS_VALID_WIN_KEY(prevPoint.winKey.win.skey) &&
isInterpoWindowFinished(&prevPoint) == false) {
code = setIntervalSliceOutputBuf(&pInfo->streamAggSup, &prevPoint, pSup->pCtx, numOfOutput,
pSup->rowEntryInfoOffset);
QUERY_CHECK_CODE(code, lino, _end);
resetIntervalSliceFunctionKey(pSup->pCtx, numOfOutput);
doSetElapsedEndKey(prevPoint.winKey.win.ekey, &pOperator->exprSupp);
doStreamSliceInterpolation(prevPoint.pLastRow, prevPoint.winKey.win.ekey, curTs, pBlock, startPos, &pOperator->exprSupp, INTERVAL_SLICE_END, pInfo->pOffsetInfo);
doStreamSliceInterpolation(prevPoint.pLastRow, prevPoint.winKey.win.ekey, curTs, pBlock, startPos,
&pOperator->exprSupp, INTERVAL_SLICE_END, pInfo->pOffsetInfo);
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &prevPoint.winKey.win, 1);
code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos,
0, pBlock->info.rows, numOfOutput);
code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0,
pBlock->info.rows, numOfOutput);
QUERY_CHECK_CODE(code, lino, _end);
SWinKey prevKey = {.ts = prevPoint.winKey.win.skey, .groupId = prevPoint.winKey.groupId};
code = saveWinResult(&prevKey, prevPoint.pResPos, pInfo->pUpdatedMap);
@ -338,7 +354,8 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc
break;
}
code = setIntervalSliceOutputBuf(&curPoint, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);
code =
setIntervalSliceOutputBuf(&pInfo->streamAggSup, &curPoint, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator) &&
@ -351,7 +368,8 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc
resetIntervalSliceFunctionKey(pSup->pCtx, numOfOutput);
if (pInfo->hasInterpoFunc && IS_VALID_WIN_KEY(prevPoint.winKey.win.skey) && curPoint.winKey.win.skey != curTs) {
doStreamSliceInterpolation(prevPoint.pLastRow, curPoint.winKey.win.skey, curTs, pBlock, startPos, &pOperator->exprSupp, INTERVAL_SLICE_START, pInfo->pOffsetInfo);
doStreamSliceInterpolation(prevPoint.pLastRow, curPoint.winKey.win.skey, curTs, pBlock, startPos,
&pOperator->exprSupp, INTERVAL_SLICE_START, pInfo->pOffsetInfo);
}
forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, curWin.ekey, binarySearchForKey, NULL,
TSDB_ORDER_ASC);
@ -423,7 +441,7 @@ static int32_t doStreamIntervalSliceNext(SOperatorInfo* pOperator, SSDataBlock**
return code;
}
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState);
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState, 1, INT64_MAX);
setStreamOperatorCompleted(pOperator);
(*ppRes) = NULL;
return code;
@ -480,7 +498,6 @@ static int32_t doStreamIntervalSliceNext(SOperatorInfo* pOperator, SSDataBlock**
QUERY_CHECK_CODE(code, lino, _end);
code = doStreamIntervalSliceAggImpl(pOperator, pBlock, pInfo->pUpdatedMap, NULL);
QUERY_CHECK_CODE(code, lino, _end);
}
if (!pInfo->destHasPrimaryKey) {
@ -506,7 +523,6 @@ static int32_t doStreamIntervalSliceNext(SOperatorInfo* pOperator, SSDataBlock**
code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _end);
(*ppRes) = NULL;
code = buildIntervalSliceResult(pOperator, ppRes);
QUERY_CHECK_CODE(code, lino, _end);
@ -517,7 +533,7 @@ static int32_t doStreamIntervalSliceNext(SOperatorInfo* pOperator, SSDataBlock**
(*ppRes) = pInfo->pCheckpointRes;
return code;
}
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState);
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState, 1, INT64_MAX);
setStreamOperatorCompleted(pOperator);
}
@ -530,7 +546,7 @@ _end:
int32_t initIntervalSliceDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type,
int32_t tsColIndex, STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic,
SInterval* pInterval, bool hasInterpoFunc) {
SInterval* pInterval, bool hasInterpoFunc, int64_t recalculateInterval) {
SExecTaskInfo* pTaskInfo = downstream->pTaskInfo;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
@ -541,8 +557,8 @@ int32_t initIntervalSliceDownStream(SOperatorInfo* downstream, SStreamAggSupport
}
if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
code =
initIntervalSliceDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic, pInterval, hasInterpoFunc);
code = initIntervalSliceDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic, pInterval,
hasInterpoFunc, recalculateInterval);
return code;
}
SStreamScanInfo* pScanInfo = downstream->info;
@ -550,7 +566,7 @@ int32_t initIntervalSliceDownStream(SOperatorInfo* downstream, SStreamAggSupport
pScanInfo->igCheckUpdate = true;
pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};
pScanInfo->pState = pAggSup->pState;
if (!pScanInfo->pUpdateInfo) {
if (!pScanInfo->pUpdateInfo && pTwSup->calTrigger != STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
code = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark,
pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen,
&pScanInfo->pUpdateInfo);
@ -562,6 +578,12 @@ int32_t initIntervalSliceDownStream(SOperatorInfo* downstream, SStreamAggSupport
if (!hasSrcPrimaryKeyCol(pBasic)) {
pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;
}
pBasic->pTsDataState = pScanInfo->basic.pTsDataState;
if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL) {
pScanInfo->scanAllTables = true;
}
pScanInfo->recalculateInterval = recalculateInterval;
_end:
if (code != TSDB_CODE_SUCCESS) {
@ -582,6 +604,55 @@ static bool windowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols) {
return needed;
}
int32_t initNonBlockAggSupptor(SNonBlockAggSupporter* pNbSup, SInterval* pInterval, SOperatorInfo* downstream) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pInterval != NULL) {
pNbSup->numOfKeep = ceil(((double)pInterval->interval) / pInterval->sliding);
} else {
pNbSup->numOfKeep = 1;
}
pNbSup->tsOfKeep = INT64_MAX;
pNbSup->pullIndex = 0;
pNbSup->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));
QUERY_CHECK_NULL(pNbSup->pPullWins, code, lino, _end, terrno);
code = createSpecialDataBlock(STREAM_RETRIEVE, &pNbSup->pPullDataRes);
QUERY_CHECK_CODE(code, lino, _end);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pNbSup->pPullDataMap = tSimpleHashInit(64, hashFn);
pNbSup->numOfChild = 0;
while (downstream != NULL && downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
downstream = downstream->pDownstream[0];
}
if (downstream != NULL) {
SStreamScanInfo* pInfo = (SStreamScanInfo*)downstream->info;
pNbSup->recParam = pInfo->recParam;
} else {
pNbSup->recParam = (SStreamRecParam){0};
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
void destroyNonBlockAggSupptor(SNonBlockAggSupporter* pNbSup) {
blockDataDestroy(pNbSup->pPullDataRes);
pNbSup->pPullDataRes = NULL;
tSimpleHashCleanup(pNbSup->pHistoryGroup);
pNbSup->pHistoryGroup = NULL;
taosArrayDestroy(pNbSup->pPullWins);
pNbSup->pPullWins = NULL;
tSimpleHashCleanup(pNbSup->pPullDataMap);
pNbSup->pPullDataMap = NULL;
}
int32_t createStreamIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,
SReadHandle* pHandle, SOperatorInfo** ppOptInfo) {
int32_t code = TSDB_CODE_SUCCESS;
@ -646,6 +717,7 @@ int32_t createStreamIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiN
.deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)};
code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
if (pIntervalPhyNode->window.pExprs != NULL) {
int32_t numOfScalar = 0;
@ -666,7 +738,8 @@ int32_t createStreamIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiN
QUERY_CHECK_CODE(code, lino, _error);
int32_t keyBytes = sizeof(TSKEY);
keyBytes += blockDataGetRowSize(pDownRes) + sizeof(SResultCellData) * taosArrayGetSize(pDownRes->pDataBlock) + sizeof(bool);
keyBytes +=
blockDataGetRowSize(pDownRes) + sizeof(SResultCellData) * taosArrayGetSize(pDownRes->pDataBlock) + sizeof(bool);
if (pPkCol) {
keyBytes += pPkCol->bytes;
}
@ -678,19 +751,37 @@ int32_t createStreamIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiN
pInfo->pOperator = pOperator;
pInfo->hasFill = false;
pInfo->hasInterpoFunc = windowinterpNeeded(pExpSup->pCtx, numOfExprs);
initNonBlockAggSupptor(&pInfo->nbSup, &pInfo->interval, NULL);
setOperatorInfo(pOperator, "StreamIntervalSliceOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL, true, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalSliceNext, NULL, destroyStreamIntervalSliceOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamIntervalSliceReleaseState, streamIntervalSliceReloadState);
setOperatorInfo(pOperator, "StreamIntervalSliceOperator", pPhyNode->type, true, OP_NOT_OPENED, pInfo, pTaskInfo);
code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
if (pIntervalPhyNode->window.triggerType == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
if (pHandle->fillHistory == STREAM_HISTORY_OPERATOR) {
setFillHistoryOperatorFlag(&pInfo->basic);
} else if (pHandle->fillHistory == STREAM_RECALCUL_OPERATOR) {
setRecalculateOperatorFlag(&pInfo->basic);
}
pInfo->nbSup.pWindowAggFn = doStreamIntervalNonblockAggImpl;
if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL) {
setSingleOperatorFlag(&pInfo->basic);
}
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalNonblockAggNext, NULL,
destroyStreamIntervalSliceOperatorInfo, optrDefaultBufFn, NULL,
optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamIntervalNonblockReleaseState, streamIntervalNonblockReloadState);
} else {
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doStreamIntervalSliceNext, NULL, destroyStreamIntervalSliceOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamIntervalSliceReleaseState, streamIntervalSliceReloadState);
}
if (downstream) {
code = initIntervalSliceDownStream(downstream, &pInfo->streamAggSup, pPhyNode->type, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic, &pInfo->interval, pInfo->hasInterpoFunc);
&pInfo->twAggSup, &pInfo->basic, &pInfo->interval, pInfo->hasInterpoFunc,
pIntervalPhyNode->window.recalculateInterval);
QUERY_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);

1865
source/libs/executor/src/streamscanoperator.c Normal file
View File

File diff suppressed because it is too large Load Diff

879
source/libs/executor/src/streamsessionnonblockoperator.c Normal file
View File

@ -0,0 +1,879 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorInt.h"
#include "filter.h"
#include "function.h"
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
#include "streamexecutorInt.h"
#include "streaminterval.h"
#include "streamsession.h"
#include "tchecksum.h"
#include "tcommon.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "tglobal.h"
#include "tlog.h"
#include "ttime.h"
#define STREAM_SESSION_NONBLOCK_OP_STATE_NAME "StreamSessionNonblockHistoryState"
void streamSessionNonblockReleaseState(SOperatorInfo* pOperator) {
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
pAggSup->stateStore.streamStateClearExpiredSessionState(pAggSup->pState, pInfo->nbSup.numOfKeep,
pInfo->nbSup.tsOfKeep, NULL);
streamSessionReleaseState(pOperator);
qDebug("%s===stream===streamSessionNonblockReleaseState:%" PRId64, GET_TASKID(pOperator->pTaskInfo),
pInfo->twAggSup.maxTs);
}
void streamSessionNonblockReloadState(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
int32_t size = 0;
void* pBuf = NULL;
resetWinRange(&pAggSup->winRange);
code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_NONBLOCK_OP_STATE_NAME,
strlen(STREAM_SESSION_NONBLOCK_OP_STATE_NAME), &pBuf, &size);
QUERY_CHECK_CODE(code, lino, _end);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);
pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts);
for (int32_t i = 0; i < num; i++) {
SResultWindowInfo winInfo = {0};
code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo);
QUERY_CHECK_CODE(code, lino, _end);
if (!IS_VALID_SESSION_WIN(winInfo)) {
continue;
}
int32_t winNum = 0;
bool isEnd = false;
code = compactSessionWindow(pOperator, &winInfo, pInfo->pStUpdated, pInfo->basic.pSeDeleted, true, &winNum, &isEnd);
QUERY_CHECK_CODE(code, lino, _end);
if (winNum > 0) {
if (isEnd) {
code = saveDeleteRes(pInfo->basic.pSeDeleted, winInfo.sessionWin);
QUERY_CHECK_CODE(code, lino, _end);
qDebug("===stream=== reload state. save delete result %" PRId64 ", %" PRIu64, winInfo.sessionWin.win.skey,
winInfo.sessionWin.groupId);
} else {
void* pResPtr = taosArrayPush(pInfo->basic.pUpdated, &winInfo);
QUERY_CHECK_NULL(pResPtr, code, lino, _end, terrno);
reuseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->stateStore);
qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, winInfo.sessionWin.win.skey,
winInfo.sessionWin.groupId);
}
}
code = saveSessionOutputBuf(pAggSup, &winInfo);
QUERY_CHECK_CODE(code, lino, _end);
}
taosMemoryFree(pBuf);
SOperatorInfo* downstream = pOperator->pDownstream[0];
if (downstream->fpSet.reloadStreamStateFn) {
downstream->fpSet.reloadStreamStateFn(downstream);
}
qDebug("%s===stream===streamSessionNonblockReloadState", GET_TASKID(pOperator->pTaskInfo));
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
}
int32_t doStreamSessionNonblockAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)pOperator->info;
SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo);
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SResultRow* pResult = NULL;
int64_t groupId = pBlock->info.id.groupId;
int64_t rows = pBlock->info.rows;
int32_t winRows = 0;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
TSKEY* startTsCols = (int64_t*)pStartTsCol->pData;
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
if (pAggSup->winRange.ekey <= 0) {
pAggSup->winRange.ekey = INT64_MAX;
}
if (pAggSup->winRange.skey != INT64_MIN && pInfo->nbSup.pHistoryGroup == NULL) {
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->nbSup.pHistoryGroup = tSimpleHashInit(1024, hashFn);
}
for (int32_t i = 0; i < rows;) {
SResultWindowInfo curWinInfo = {0};
int32_t winCode = TSDB_CODE_SUCCESS;
code = setSessionOutputBuf(pAggSup, startTsCols[i], startTsCols[i], groupId, &curWinInfo, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS) {
SStreamStateCur* pCur =
pAggSup->stateStore.streamStateSessionSeekKeyPrev(pAggSup->pState, &curWinInfo.sessionWin);
int32_t size = 0;
SResultWindowInfo prevWinInfo = {.sessionWin.groupId = groupId};
int32_t tmpWinCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &prevWinInfo.sessionWin,
(void**)&prevWinInfo.pStatePos, &size);
if (tmpWinCode == TSDB_CODE_SUCCESS) {
void* pResPtr = taosArrayPush(pInfo->basic.pUpdated, &prevWinInfo);
QUERY_CHECK_NULL(pResPtr, code, lino, _end, terrno);
reuseOutputBuf(pAggSup->pState, prevWinInfo.pStatePos, &pAggSup->stateStore);
int32_t mode = 0;
int32_t winRes = pAggSup->stateStore.streamStateGetRecFlag(pAggSup->pState, &prevWinInfo.sessionWin,
sizeof(SSessionKey), &mode);
if (winRes == TSDB_CODE_SUCCESS) {
code = saveRecWindowToDisc(&prevWinInfo.sessionWin, pBlock->info.id.uid, mode, pInfo->basic.pTsDataState,
pAggSup);
QUERY_CHECK_CODE(code, lino, _end);
}
}
}
code = updateSessionWindowInfo(pAggSup, &curWinInfo, startTsCols, startTsCols, groupId, rows, i, pAggSup->gap,
pAggSup->pResultRows, NULL, pInfo->basic.pSeDeleted, &winRows);
QUERY_CHECK_CODE(code, lino, _end);
code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWinInfo, &pResult, i, winRows, rows, numOfOutput,
pOperator, pAggSup->gap);
QUERY_CHECK_CODE(code, lino, _end);
code = saveSessionOutputBuf(pAggSup, &curWinInfo);
QUERY_CHECK_CODE(code, lino, _end);
releaseOutputBuf(pAggSup->pState, curWinInfo.pStatePos, &pAggSup->stateStore);
i += winRows;
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}
int32_t getSessionHistoryRemainResultInfo(SStreamAggSupporter* pAggSup, int32_t numOfState, SArray* pUpdated,
int32_t capacity) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t winCode = TSDB_CODE_SUCCESS;
if (pAggSup->pCur == NULL) {
goto _end;
}
int32_t num = capacity - taosArrayGetSize(pUpdated);
for (int32_t i = 0; i < num; i++) {
winCode = pAggSup->stateStore.streamStateNLastSessionStateGetKVByCur(pAggSup->pCur, numOfState, pUpdated);
if (winCode == TSDB_CODE_FAILED) {
pAggSup->stateStore.streamStateFreeCur(pAggSup->pCur);
pAggSup->pCur = NULL;
break;
}
pAggSup->stateStore.streamStateLastSessionStateCurNext(pAggSup->pCur);
num = capacity - taosArrayGetSize(pUpdated);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s.", __func__, lino, tstrerror(code));
}
return code;
}
void releaseSessionFlusedPos(void* pRes) {
SResultWindowInfo* pWinInfo = (SResultWindowInfo*)pRes;
SRowBuffPos* pPos = pWinInfo->pStatePos;
if (pPos != NULL && pPos->needFree) {
pPos->beUsed = false;
}
}
int32_t buildSessionHistoryResult(SOperatorInfo* pOperator, SOptrBasicInfo* pBinfo, SSteamOpBasicInfo* pBasic,
SStreamAggSupporter* pAggSup, SNonBlockAggSupporter* pNbSup,
SGroupResInfo* pGroupResInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamNotifyEventSupp* pNotifySup = &pBasic->notifyEventSup;
bool addNotifyEvent = false;
code =
getSessionHistoryRemainResultInfo(pAggSup, pNbSup->numOfKeep, pBasic->pUpdated, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _end);
if (taosArrayGetSize(pBasic->pUpdated) > 0) {
taosArraySort(pBasic->pUpdated, sessionKeyCompareAsc);
if (pNbSup->numOfKeep > 1) {
taosArrayRemoveDuplicate(pBasic->pUpdated, sessionKeyCompareAsc, releaseSessionFlusedPos);
}
initGroupResInfoFromArrayList(pGroupResInfo, pBasic->pUpdated);
pBasic->pUpdated = taosArrayInit(1024, sizeof(SResultWindowInfo));
QUERY_CHECK_NULL(pBasic->pUpdated, code, lino, _end, terrno);
doBuildSessionResult(pOperator, pAggSup->pState, pGroupResInfo, pBinfo->pRes, addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s.", __func__, lino, tstrerror(code));
}
return code;
}
static int32_t doDeleteSessionRecalculateWindows(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SSHashObj* pDeleteMap) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* startTsCols = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
TSKEY* endTsCols = (TSKEY*)pEndTsCol->pData;
SColumnInfoData* pCalStTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
TSKEY* calStTsCols = (TSKEY*)pCalStTsCol->pData;
SColumnInfoData* pCalEnTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
TSKEY* calEnTsCols = (TSKEY*)pCalEnTsCol->pData;
SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* pGpDatas = (uint64_t*)pGpCol->pData;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
SSessionKey winRes = {.win.skey = startTsCols[i], .win.ekey = endTsCols[i], .groupId = pGpDatas[i]};
code = tSimpleHashPut(pDeleteMap, &winRes, sizeof(SSessionKey), NULL, 0);
QUERY_CHECK_CODE(code, lino, _end);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}
static int32_t buildOtherResult(SOperatorInfo* pOperator, SOptrBasicInfo* pBinfo, SSteamOpBasicInfo* pBasic,
SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup,
SNonBlockAggSupporter* pNbSup, SGroupResInfo* pGroupResInfo, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
if (isHistoryOperator(pBasic) && isSingleOperator(pBasic)) {
code = buildSessionHistoryResult(pOperator, pBinfo, pBasic, pAggSup, pNbSup, pGroupResInfo);
QUERY_CHECK_CODE(code, lino, _end);
if (pBinfo->pRes->info.rows != 0) {
printDataBlock(pBinfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pBinfo->pRes;
return code;
}
}
if (pBasic->recvCkBlock) {
pBasic->recvCkBlock = false;
printDataBlock(pBasic->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pBasic->pCheckpointRes;
return code;
}
if (pTwAggSup->minTs != INT64_MAX) {
pNbSup->tsOfKeep = pTwAggSup->minTs;
}
if (!isHistoryOperator(pBasic) || !isFinalOperator(pBasic)) {
int32_t numOfKeep = 0;
TSKEY tsOfKeep = INT64_MAX;
getStateKeepInfo(pNbSup, isRecalculateOperator(pBasic), &numOfKeep, &tsOfKeep);
pAggSup->stateStore.streamStateClearExpiredSessionState(pAggSup->pState, numOfKeep, tsOfKeep,
pNbSup->pHistoryGroup);
}
pTwAggSup->minTs = INT64_MAX;
setStreamOperatorCompleted(pOperator);
if (isFinalOperator(pBasic) && isRecalculateOperator(pBasic) && tSimpleHashGetSize(pNbSup->pPullDataMap) == 0) {
qDebug("===stream===%s recalculate is finished.", GET_TASKID(pTaskInfo));
pTaskInfo->streamInfo.recoverScanFinished = true;
}
(*ppRes) = NULL;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}
static int32_t closeNonBlockSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pUpdated,
SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
void* pIte = NULL;
int32_t iter = 0;
while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
void* key = tSimpleHashGetKey(pIte, NULL);
SSessionKey* pWinKey = (SSessionKey*)key;
if (isCloseWindow(&pWinKey->win, pTwSup)) {
void* pTemp = taosArrayPush(pUpdated, pIte);
QUERY_CHECK_NULL(pTemp, code, lino, _end, terrno);
int32_t tmpRes = tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SSessionKey), &pIte, &iter);
qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}
int32_t buildNonBlockSessionResult(SOperatorInfo* pOperator, SStreamAggSupporter* pAggSup, SOptrBasicInfo* pBInfo,
SSteamOpBasicInfo* pBasic, SGroupResInfo* pGroupResInfo, SNonBlockAggSupporter* pNbSup, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamNotifyEventSupp* pNotifySup = &pBasic->notifyEventSup;
bool addNotifyEvent = false;
addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
if (isFinalOperator(pBasic)) {
doBuildPullDataBlock(pNbSup->pPullWins, &pNbSup->pullIndex, pNbSup->pPullDataRes);
if (pNbSup->pPullDataRes->info.rows != 0) {
printDataBlock(pNbSup->pPullDataRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pNbSup->pPullDataRes;
return code;
}
}
SSDataBlock* pDelRes = pBasic->pDelRes;
doBuildDeleteDataBlock(pOperator, pBasic->pSeDeleted, pDelRes, &pBasic->pDelIterator, pGroupResInfo);
if (pDelRes->info.rows > 0) {
printDataBlock(pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pDelRes;
return code;
}
doBuildSessionResult(pOperator, pAggSup->pState, pGroupResInfo, pBInfo->pRes, addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
if (pBInfo->pRes->info.rows > 0) {
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pBInfo->pRes;
return code;
}
(*ppRes) = NULL;
return code;
}
static int32_t doSetSessionWindowRecFlag(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* startTsCols = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
TSKEY* endTsCols = (TSKEY*)pEndTsCol->pData;
SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* pGpDatas = (uint64_t*)pGpCol->pData;
SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
uint64_t* pUidDatas = (uint64_t*)pUidCol->pData;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
SSessionKey key = {.win.skey = startTsCols[i], .win.ekey = endTsCols[i], .groupId = pGpDatas[i]};
bool isLastWin = false;
if (pAggSup->stateStore.streamStateCheckSessionState(pAggSup->pState, &key, pAggSup->gap, &isLastWin)) {
qDebug("===stream===%s set recalculate flag start ts:%" PRId64 ",end ts:%" PRId64 ", group id:%" PRIu64,
GET_TASKID(pTaskInfo), key.win.skey, key.win.ekey, key.groupId);
pAggSup->stateStore.streamStateSetRecFlag(pAggSup->pState, &key, sizeof(SSessionKey), pBlock->info.type);
if ((isFinalOperator(&pInfo->basic) && isCloseWindow(&key.win, &pInfo->twAggSup)) ||
(isSingleOperator(&pInfo->basic) && isLastWin == false)) {
code = saveRecWindowToDisc(&key, pUidDatas[i], pBlock->info.type, pInfo->basic.pTsDataState,
&pInfo->streamAggSup);
QUERY_CHECK_CODE(code, lino, _end);
}
} else {
code = saveRecWindowToDisc(&key, pUidDatas[i], pBlock->info.type, pInfo->basic.pTsDataState,
&pInfo->streamAggSup);
QUERY_CHECK_CODE(code, lino, _end);
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}
static int32_t checkAndSaveSessionStateToDisc(int32_t startIndex, SArray* pUpdated, uint64_t uid, STableTsDataState* pTsDataState,
SStreamAggSupporter* pAggSup) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t mode = 0;
int32_t size = taosArrayGetSize(pUpdated);
for (int32_t i = startIndex; i < size; i++) {
SResultWindowInfo* pWinInfo = taosArrayGet(pUpdated, i);
SSessionKey* pKey = &pWinInfo->sessionWin;
int32_t winRes = pAggSup->stateStore.streamStateGetRecFlag(pAggSup->pState, pKey, sizeof(SSessionKey), &mode);
if (winRes == TSDB_CODE_SUCCESS) {
code = saveRecWindowToDisc(pKey, uid, mode, pTsDataState, pAggSup);
QUERY_CHECK_CODE(code, lino, _end);
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s.", __func__, lino, tstrerror(code));
}
return code;
}
int32_t doStreamSessionNonblockAggNextImpl(SOperatorInfo* pOperator, SOptrBasicInfo* pBInfo, SSteamOpBasicInfo* pBasic,
SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup,
SGroupResInfo* pGroupResInfo, SNonBlockAggSupporter* pNbSup,
SExprSupp* pScalarSupp, SArray* pHistoryWins, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SExprSupp* pSup = &pOperator->exprSupp;
qDebug("stask:%s %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);
if (pOperator->status == OP_EXEC_DONE) {
(*ppRes) = NULL;
return code;
}
code = buildNonBlockSessionResult(pOperator, pAggSup, pBInfo, pBasic, pGroupResInfo, pNbSup, ppRes);
QUERY_CHECK_CODE(code, lino, _end);
if ((*ppRes) != NULL) {
return code;
}
if (isHistoryOperator(pBasic) && !isFinalOperator(pBasic)) {
int32_t numOfKeep = 0;
TSKEY tsOfKeep = INT64_MAX;
getStateKeepInfo(pNbSup, isRecalculateOperator(pBasic), &numOfKeep, &tsOfKeep);
pAggSup->stateStore.streamStateClearExpiredSessionState(pAggSup->pState, pNbSup->numOfKeep, pNbSup->tsOfKeep,
pNbSup->pHistoryGroup);
}
if (pOperator->status == OP_RES_TO_RETURN) {
return buildOtherResult(pOperator, pBInfo, pBasic, pAggSup, pTwAggSup, pNbSup, pGroupResInfo, ppRes);
}
SOperatorInfo* downstream = pOperator->pDownstream[0];
while (1) {
if (isTaskKilled(pTaskInfo)) {
qInfo("===stream=== %s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));
(*ppRes) = NULL;
return code;
}
SSDataBlock* pBlock = NULL;
code = downstream->fpSet.getNextFn(downstream, &pBlock);
QUERY_CHECK_CODE(code, lino, _end);
if (pBlock == NULL) {
qDebug("===stream===return data:%s.", getStreamOpName(pOperator->operatorType));
if (isFinalOperator(pBasic) && isRecalculateOperator(pBasic)) {
code = buildRetriveRequest(pTaskInfo, pAggSup, pBasic->pTsDataState, pNbSup);
}
pOperator->status = OP_RES_TO_RETURN;
break;
}
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
setStreamOperatorState(pBasic, pBlock->info.type);
switch (pBlock->info.type) {
case STREAM_NORMAL:
case STREAM_INVALID:
case STREAM_PULL_DATA: {
SExprSupp* pExprSup = pScalarSupp;
if (pExprSup->pExprInfo != NULL) {
code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
QUERY_CHECK_CODE(code, lino, _end);
}
} break;
case STREAM_CHECKPOINT: {
pBasic->recvCkBlock = true;
pAggSup->stateStore.streamStateCommit(pAggSup->pState);
code = copyDataBlock(pBasic->pCheckpointRes, pBlock);
QUERY_CHECK_CODE(code, lino, _end);
continue;
} break;
case STREAM_CREATE_CHILD_TABLE:
case STREAM_DROP_CHILD_TABLE: {
(*ppRes) = pBlock;
return code;
} break;
case STREAM_RECALCULATE_DATA:
case STREAM_RECALCULATE_DELETE: {
if (isRecalculateOperator(pBasic)) {
if (!isSemiOperator(pBasic)) {
code = doDeleteSessionRecalculateWindows(pTaskInfo, pBlock, pBasic->pSeDeleted);
QUERY_CHECK_CODE(code, lino, _end);
if (isFinalOperator(pBasic)) {
saveRecalculateData(&pAggSup->stateStore, pBasic->pTsDataState, pBlock, pBlock->info.type);
}
continue;
}
}
if (isSemiOperator(pBasic)) {
(*ppRes) = pBlock;
return code;
} else {
code = doSetSessionWindowRecFlag(pOperator, pBlock);
QUERY_CHECK_CODE(code, lino, _end);
}
continue;
} break;
case STREAM_PULL_OVER: {
code = processDataPullOver(pBlock, pNbSup->pPullDataMap, pTaskInfo);
QUERY_CHECK_CODE(code, lino, _end);
continue;
} break;
default:
qDebug("===stream===%s ignore recv block. type:%d", GET_TASKID(pTaskInfo), pBlock->info.type);
continue;
}
if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) {
// set input version
pTaskInfo->version = pBlock->info.version;
}
code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
QUERY_CHECK_CODE(code, lino, _end);
pTwAggSup->maxTs = TMAX(pTwAggSup->maxTs, pBlock->info.window.ekey);
code = pNbSup->pWindowAggFn(pOperator, pBlock);
if (code == TSDB_CODE_STREAM_INTERNAL_ERROR) {
pOperator->status = OP_RES_TO_RETURN;
code = TSDB_CODE_SUCCESS;
}
QUERY_CHECK_CODE(code, lino, _end);
if (pAggSup->pScanBlock->info.rows > 0) {
(*ppRes) = pAggSup->pScanBlock;
printDataBlock(pAggSup->pScanBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
return code;
}
if (taosArrayGetSize(pBasic->pUpdated) > 0) {
break;
}
}
if (pOperator->status == OP_RES_TO_RETURN &&
(isHistoryOperator(pBasic) || isRecalculateOperator(pBasic) || isSemiOperator(pBasic))) {
code = copyNewResult(&pAggSup->pResultRows, pBasic->pUpdated, sessionKeyCompareAsc);
QUERY_CHECK_CODE(code, lino, _end);
if (isSingleOperator(pBasic)) {
if (pAggSup->pCur == NULL) {
pAggSup->pCur = pAggSup->stateStore.streamStateGetLastSessionStateCur(pAggSup->pState);
}
code = getSessionHistoryRemainResultInfo(pAggSup, pNbSup->numOfKeep, pBasic->pUpdated,
pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _end);
}
}
if (pOperator->status == OP_RES_TO_RETURN && pBasic->destHasPrimaryKey && isFinalOperator(pBasic)) {
code = closeNonBlockSessionWindow(pAggSup->pResultRows, pTwAggSup, pBasic->pUpdated, pTaskInfo);
QUERY_CHECK_CODE(code, lino, _end);
if (!isHistoryOperator(pBasic)) {
checkAndSaveSessionStateToDisc(0, pBasic->pUpdated, 0, pBasic->pTsDataState, pAggSup);
}
}
taosArraySort(pBasic->pUpdated, sessionKeyCompareAsc);
if (pNbSup->numOfKeep > 1) {
taosArrayRemoveDuplicate(pBasic->pUpdated, sessionKeyCompareAsc, releaseSessionFlusedPos);
}
if (!isSemiOperator(pBasic) && !pBasic->destHasPrimaryKey) {
removeSessionDeleteResults(pBasic->pSeDeleted, pBasic->pUpdated);
}
if (isHistoryOperator(pBasic)) {
code = getMaxTsWins(pBasic->pUpdated, pHistoryWins);
QUERY_CHECK_CODE(code, lino, _end);
}
initGroupResInfoFromArrayList(pGroupResInfo, pBasic->pUpdated);
pBasic->pUpdated = taosArrayInit(1024, sizeof(SResultWindowInfo));
QUERY_CHECK_NULL(pBasic->pUpdated, code, lino, _end, terrno);
code = blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _end);
code = buildNonBlockSessionResult(pOperator, pAggSup, pBInfo, pBasic, pGroupResInfo, pNbSup, ppRes);
QUERY_CHECK_CODE(code, lino, _end);
if ((*ppRes) != NULL) {
return code;
}
return buildOtherResult(pOperator, pBInfo, pBasic, pAggSup, pTwAggSup, pNbSup, pGroupResInfo, ppRes);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
pTaskInfo->code = code;
}
return code;
}
int32_t doStreamSessionNonblockAggNext(SOperatorInfo* pOperator, SSDataBlock** ppBlock) {
SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)pOperator->info;
return doStreamSessionNonblockAggNextImpl(pOperator, &pInfo->binfo, &pInfo->basic, &pInfo->streamAggSup,
&pInfo->twAggSup, &pInfo->groupResInfo, &pInfo->nbSup, &pInfo->scalarSupp,
pInfo->historyWins, ppBlock);
}
int32_t doStreamSemiSessionNonblockAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SExprSupp* pSup = &pOperator->exprSupp;
int32_t numOfOutput = pSup->numOfExprs;
int64_t groupId = pBlock->info.id.groupId;
SResultRow* pResult = NULL;
int64_t rows = pBlock->info.rows;
int32_t winRows = 0;
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
TSKEY* startTsCols = (int64_t*)pStartTsCol->pData;
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
if (pAggSup->winRange.ekey <= 0) {
pAggSup->winRange.ekey = INT64_MAX;
}
for (int64_t i = 0; i < rows;) {
SResultWindowInfo curWinInfo = {0};
int32_t winCode = TSDB_CODE_SUCCESS;
code = setSessionOutputBuf(pAggSup, startTsCols[i], startTsCols[i], groupId, &curWinInfo, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS) {
code = tSimpleHashPut(pAggSup->pResultRows, &curWinInfo.sessionWin, sizeof(SSessionKey), &curWinInfo,
sizeof(SResultWindowInfo));
QUERY_CHECK_CODE(code, lino, _end);
}
code = updateSessionWindowInfo(pAggSup, &curWinInfo, startTsCols, startTsCols, groupId, rows, i, pAggSup->gap,
pAggSup->pResultRows, NULL, NULL, &winRows);
QUERY_CHECK_CODE(code, lino, _end);
code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWinInfo, &pResult, i, winRows, rows, numOfOutput,
pOperator, 0);
QUERY_CHECK_CODE(code, lino, _end);
code = saveSessionOutputBuf(pAggSup, &curWinInfo);
QUERY_CHECK_CODE(code, lino, _end);
i += winRows;
}
if (isHistoryOperator(&pInfo->basic) &&
tSimpleHashGetSize(pAggSup->pResultRows) > pOperator->resultInfo.capacity * 10) {
code = copyNewResult(&pAggSup->pResultRows, pInfo->basic.pUpdated, sessionKeyCompareAsc);
QUERY_CHECK_CODE(code, lino, _end);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}
int32_t createSessionNonblockOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,
SReadHandle* pHandle, SOperatorInfo** ppOptInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
code = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, ppOptInfo);
QUERY_CHECK_CODE(code, lino, _end);
SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)(*ppOptInfo)->info;
pInfo->nbSup.numOfKeep = 1;
pInfo->nbSup.pWindowAggFn = doStreamSessionNonblockAggImpl;
setSingleOperatorFlag(&pInfo->basic);
adjustDownstreamBasicInfo(downstream, &pInfo->basic);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}
int32_t createSemiSessionNonblockOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,
SReadHandle* pHandle, SOperatorInfo** ppOptInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
code = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, ppOptInfo);
QUERY_CHECK_CODE(code, lino, _end);
SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)(*ppOptInfo)->info;
pInfo->nbSup.numOfKeep = 0;
pInfo->nbSup.pWindowAggFn = doStreamSemiSessionNonblockAggImpl;
setSemiOperatorFlag(&pInfo->basic);
adjustDownstreamBasicInfo(downstream, &pInfo->basic);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}
bool isDataDeletedSessionWindow(SStreamAggSupporter* pAggSup, SNonBlockAggSupporter* pNbSup, TSKEY startTs, TSKEY endTs,
uint64_t groupId, TSKEY gap) {
if (startTs < pNbSup->tsOfKeep) {
SSessionKey key = {.win.skey = startTs, .win.ekey = endTs, .groupId = groupId};
return !(pAggSup->stateStore.streamStateCheckSessionState(pAggSup->pState, &key, gap, NULL));
}
return false;
}
static int32_t doStreamFinalSessionNonblockAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo);
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SExprSupp* pSup = &pOperator->exprSupp;
int32_t numOfOutput = pSup->numOfExprs;
SResultRow* pResult = NULL;
uint64_t groupId = pBlock->info.id.groupId;
int64_t rows = pBlock->info.rows;
int32_t winRows = 0;
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
TSKEY* startTsCols = (int64_t*)pStartTsCol->pData;
SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, pInfo->endTsIndex);
TSKEY* endTsCols = (int64_t*)pEndTsCol->pData;
int32_t startPos = 0;
if (pAggSup->pScanBlock->info.rows > 0) {
blockDataCleanup(pAggSup->pScanBlock);
}
pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.ekey);
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
if (pAggSup->winRange.ekey <= 0) {
pAggSup->winRange.ekey = INT64_MAX;
}
for (int32_t i = 0; i < rows; i++) {
if (!isHistoryOperator(&pInfo->basic) &&
isDataDeletedSessionWindow(pAggSup, &pInfo->nbSup, startTsCols[i], endTsCols[i], groupId, pAggSup->gap)) {
uint64_t uid = 0;
code = appendOneRowToSpecialBlockImpl(pAggSup->pScanBlock, startTsCols + i, endTsCols + i, startTsCols + i, startTsCols + i,
&uid, &groupId, NULL, NULL);
QUERY_CHECK_CODE(code, lino, _end);
continue;
}
startPos = i;
break;
}
for (int32_t i = startPos; i < rows;) {
SResultWindowInfo curWinInfo = {0};
int32_t winCode = TSDB_CODE_SUCCESS;
code = setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &curWinInfo, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS) {
if (tSimpleHashGet(pAggSup->pResultRows, &curWinInfo.sessionWin, sizeof(SSessionKey)) == NULL) {
code = saveDeleteRes(pInfo->basic.pSeDeleted, curWinInfo.sessionWin);
QUERY_CHECK_CODE(code, lino, _end);
}
}
curWinInfo.pStatePos->beUpdated = true;
code = tSimpleHashPut(pAggSup->pResultRows, &curWinInfo.sessionWin, sizeof(SSessionKey), &curWinInfo,
sizeof(SResultWindowInfo));
QUERY_CHECK_CODE(code, lino, _end);
code = updateSessionWindowInfo(pAggSup, &curWinInfo, startTsCols, endTsCols, groupId, rows, i, pAggSup->gap,
pAggSup->pResultRows, NULL, pInfo->basic.pSeDeleted, &winRows);
QUERY_CHECK_CODE(code, lino, _end);
code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWinInfo, &pResult, i, winRows, rows, numOfOutput,
pOperator, pAggSup->gap);
QUERY_CHECK_CODE(code, lino, _end);
code = saveSessionOutputBuf(pAggSup, &curWinInfo);
QUERY_CHECK_CODE(code, lino, _end);
i += winRows;
}
if (!pInfo->destHasPrimaryKey && !isHistoryOperator(&pInfo->basic)) {
code = closeNonBlockSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->basic.pUpdated, pTaskInfo);
QUERY_CHECK_CODE(code, lino, _end);
} else if ((isHistoryOperator(&pInfo->basic) || isRecalculateOperator(&pInfo->basic)) &&
tSimpleHashGetSize(pAggSup->pResultRows) > pOperator->resultInfo.capacity * 10) {
code = copyNewResult(&pAggSup->pResultRows, pInfo->basic.pUpdated, sessionKeyCompareAsc);
QUERY_CHECK_CODE(code, lino, _end);
}
if (!isHistoryOperator(&pInfo->basic)) {
checkAndSaveSessionStateToDisc(0, pInfo->basic.pUpdated, 0, pInfo->basic.pTsDataState, &pInfo->streamAggSup);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}
int32_t createFinalSessionNonblockOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
SOperatorInfo** ppOptInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
code = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, ppOptInfo);
QUERY_CHECK_CODE(code, lino, _end);
SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)(*ppOptInfo)->info;
pInfo->nbSup.pWindowAggFn = doStreamFinalSessionNonblockAggImpl;
pInfo->streamAggSup.pScanBlock->info.type = STREAM_RETRIEVE;
pInfo->nbSup.tsOfKeep = INT64_MIN;
pInfo->nbSup.numOfChild = pHandle->numOfVgroups;
pInfo->twAggSup.waterMark = 0;
setFinalOperatorFlag(&pInfo->basic);
adjustDownstreamBasicInfo(downstream, &pInfo->basic);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}

226
source/libs/executor/src/streamstatenonblockoperator.c Normal file
View File

@ -0,0 +1,226 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorInt.h"
#include "filter.h"
#include "function.h"
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
#include "streamexecutorInt.h"
#include "streaminterval.h"
#include "streamsession.h"
#include "tchecksum.h"
#include "tcommon.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "tglobal.h"
#include "tlog.h"
#include "ttime.h"
#define STREAM_STATE_NONBLOCK_OP_STATE_NAME "StreamStateNonblockHistoryState"
void streamStateNonblockReleaseState(SOperatorInfo* pOperator) {
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
pAggSup->stateStore.streamStateClearExpiredSessionState(pAggSup->pState, pInfo->nbSup.numOfKeep,
pInfo->nbSup.tsOfKeep, NULL);
streamStateReleaseState(pOperator);
qDebug("===stream===%s streamStateNonblockReleaseState:%" PRId64, GET_TASKID(pOperator->pTaskInfo),
pInfo->twAggSup.maxTs);
}
void streamStateNonblockReloadState(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
int32_t size = 0;
void* pBuf = NULL;
resetWinRange(&pAggSup->winRange);
code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_NONBLOCK_OP_STATE_NAME,
strlen(STREAM_STATE_NONBLOCK_OP_STATE_NAME), &pBuf, &size);
QUERY_CHECK_CODE(code, lino, _end);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);
pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts);
for (int32_t i = 0; i < num; i++) {
SStateWindowInfo curInfo = {0};
SStateWindowInfo nextInfo = {0};
qDebug("===stream===%s reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", GET_TASKID(pTaskInfo), pSeKeyBuf[i].win.skey,
pSeKeyBuf[i].groupId, i);
code = getStateWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo, &nextInfo);
QUERY_CHECK_CODE(code, lino, _end);
bool cpRes = compareWinStateKey(curInfo.pStateKey, nextInfo.pStateKey);
qDebug("===stream=== reload state. next window info %" PRId64 ", %" PRIu64 ", compare:%d",
nextInfo.winInfo.sessionWin.win.skey, nextInfo.winInfo.sessionWin.groupId, cpRes);
if (cpRes) {
code = compactStateWindow(pOperator, &curInfo.winInfo, &nextInfo.winInfo, pInfo->pSeUpdated, pInfo->pSeDeleted);
qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,
curInfo.winInfo.sessionWin.groupId);
QUERY_CHECK_CODE(code, lino, _end);
bool isEnd = true;
SStateWindowInfo nextNextInfo = nextInfo;
getNextStateWin(pAggSup, &nextNextInfo, true);
if (IS_VALID_SESSION_WIN(nextNextInfo.winInfo)) {
isEnd = false;
}
if (isEnd) {
code = saveDeleteRes(pInfo->basic.pSeDeleted, curInfo.winInfo.sessionWin);
QUERY_CHECK_CODE(code, lino, _end);
qDebug("===stream=== reload state. save delete result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,
curInfo.winInfo.sessionWin.groupId);
} else {
void* pResPtr = taosArrayPush(pInfo->basic.pUpdated, &curInfo.winInfo);
QUERY_CHECK_NULL(pResPtr, code, lino, _end, terrno);
reuseOutputBuf(pAggSup->pState, curInfo.winInfo.pStatePos, &pAggSup->stateStore);
qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,
curInfo.winInfo.sessionWin.groupId);
}
} else if (IS_VALID_SESSION_WIN(nextInfo.winInfo)) {
releaseOutputBuf(pAggSup->pState, nextInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);
}
if (IS_VALID_SESSION_WIN(curInfo.winInfo)) {
code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo);
QUERY_CHECK_CODE(code, lino, _end);
}
}
taosMemoryFreeClear(pBuf);
SOperatorInfo* downstream = pOperator->pDownstream[0];
if (downstream->fpSet.reloadStreamStateFn) {
downstream->fpSet.reloadStreamStateFn(downstream);
}
qDebug("===stream===%s streamStateNonblockReloadState", GET_TASKID(pOperator->pTaskInfo));
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
}
int32_t doStreamStateNonblockAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)pOperator->info;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
SResultRow* pResult = NULL;
int64_t groupId = pBlock->info.id.groupId;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
int64_t rows = pBlock->info.rows;
int32_t winRows = 0;
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
if (pAggSup->winRange.ekey <= 0) {
pAggSup->winRange.ekey = INT64_MAX;
}
if (pAggSup->winRange.skey != INT64_MIN && pInfo->nbSup.pHistoryGroup == NULL) {
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->nbSup.pHistoryGroup = tSimpleHashInit(1024, hashFn);
}
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
TSKEY* tsCols = (int64_t*)pStartTsCol->pData;
SColumnInfoData* pKeyColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId);
for (int32_t i = 0; i < rows; i += winRows) {
char* pKeyData = colDataGetData(pKeyColInfo, i);
int32_t winIndex = 0;
bool allEqual = true;
SStateWindowInfo curWin = {0};
SStateWindowInfo nextWin = {0};
int32_t winCode = TSDB_CODE_SUCCESS;
code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS) {
SStreamStateCur* pCur =
pAggSup->stateStore.streamStateSessionSeekKeyPrev(pAggSup->pState, &curWin.winInfo.sessionWin);
int32_t size = 0;
SResultWindowInfo prevWinInfo = {.sessionWin.groupId = groupId};
int32_t tmpWinCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &prevWinInfo.sessionWin,
(void**)&prevWinInfo.pStatePos, &size);
if (tmpWinCode == TSDB_CODE_SUCCESS) {
void* pResPtr = taosArrayPush(pInfo->basic.pUpdated, &prevWinInfo);
QUERY_CHECK_NULL(pResPtr, code, lino, _end, terrno);
reuseOutputBuf(pAggSup->pState, prevWinInfo.pStatePos, &pAggSup->stateStore);
int32_t mode = 0;
int32_t winRes = pAggSup->stateStore.streamStateGetRecFlag(pAggSup->pState, &prevWinInfo.sessionWin,
sizeof(SSessionKey), &mode);
if (winRes == TSDB_CODE_SUCCESS) {
code = saveRecWindowToDisc(&prevWinInfo.sessionWin, pBlock->info.id.uid, mode, pInfo->basic.pTsDataState,
pAggSup);
QUERY_CHECK_CODE(code, lino, _end);
}
}
}
code = updateStateWindowInfo(pAggSup, &curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual,
pAggSup->pResultRows, NULL, NULL, &winRows);
QUERY_CHECK_CODE(code, lino, _end);
code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,
pOperator, 0);
QUERY_CHECK_CODE(code, lino, _end);
code = saveSessionOutputBuf(pAggSup, &curWin.winInfo);
QUERY_CHECK_CODE(code, lino, _end);
releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAggSup->stateStore);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}
int32_t doStreamStateNonblockAggNext(SOperatorInfo* pOperator, SSDataBlock** ppBlock) {
SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)pOperator->info;
return doStreamSessionNonblockAggNextImpl(pOperator, &pInfo->binfo, &pInfo->basic, &pInfo->streamAggSup,
&pInfo->twAggSup, &pInfo->groupResInfo, &pInfo->nbSup, &pInfo->scalarSupp,
pInfo->historyWins, ppBlock);
}
int32_t createStateNonblockOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,
SReadHandle* pHandle, SOperatorInfo** ppOptInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
code = createStreamStateAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, ppOptInfo);
QUERY_CHECK_CODE(code, lino, _end);
SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)(*ppOptInfo)->info;
pInfo->nbSup.numOfKeep = 1;
pInfo->nbSup.pWindowAggFn = doStreamStateNonblockAggImpl;
pInfo->nbSup.tsOfKeep = INT64_MIN;
setSingleOperatorFlag(&pInfo->basic);
adjustDownstreamBasicInfo(downstream, &pInfo->basic);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
return code;
}

181
source/libs/executor/src/streamtimesliceoperator.c
View File

@ -30,7 +30,6 @@
#define STREAM_TIME_SLICE_OP_STATE_NAME "StreamTimeSliceHistoryState"
#define STREAM_TIME_SLICE_OP_CHECKPOINT_NAME "StreamTimeSliceOperator_Checkpoint"
int32_t saveTimeSliceWinResult(SWinKey* pKey, SSHashObj* pUpdatedMap) {
return tSimpleHashPut(pUpdatedMap, pKey, sizeof(SWinKey), NULL, 0);
}
@ -85,7 +84,8 @@ void streamTimeSliceReloadState(SOperatorInfo* pOperator) {
int32_t num = (size - sizeof(TSKEY)) / sizeof(SWinKey);
qDebug("===stream=== time slice operator reload state. get result count:%d", num);
SWinKey* pKeyBuf = (SWinKey*)pBuf;
QUERY_CHECK_CONDITION((size == num * sizeof(SWinKey) + sizeof(TSKEY)), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
QUERY_CHECK_CONDITION((size == num * sizeof(SWinKey) + sizeof(TSKEY)), code, lino, _end,
TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);
@ -136,7 +136,7 @@ static void resetFillWindow(SResultRowData* pRowData) {
pRowData->pRowVal = NULL;
}
static void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup) {
void resetTimeSlicePrevAndNextWindow(SStreamFillSupporter* pFillSup) {
resetFillWindow(&pFillSup->cur);
resetFillWindow(&pFillSup->prev);
resetFillWindow(&pFillSup->next);
@ -152,7 +152,7 @@ void destroyStreamTimeSliceOperatorInfo(void* param) {
}
colDataDestroy(&pInfo->twAggSup.timeWindowData);
resetPrevAndNextWindow(pInfo->pFillSup);
resetTimeSlicePrevAndNextWindow(pInfo->pFillSup);
destroyStreamFillSupporter(pInfo->pFillSup);
destroyStreamFillInfo(pInfo->pFillInfo);
blockDataDestroy(pInfo->pRes);
@ -181,6 +181,8 @@ void destroyStreamTimeSliceOperatorInfo(void* param) {
taosArrayDestroy(pInfo->pCloseTs);
destroyStreamAggSupporter(&pInfo->streamAggSup);
destroyStreamBasicInfo(&pInfo->basic);
taosMemoryFreeClear(param);
}
@ -317,8 +319,8 @@ static int32_t initTimeSliceFillSup(SStreamInterpFuncPhysiNode* pPhyFillNode, SE
pFillSup->numOfFillCols = numOfExprs;
int32_t numOfNotFillCols = 0;
pFillSup->pAllColInfo = createFillColInfo(pExprSup->pExprInfo, pFillSup->numOfFillCols, NULL, numOfNotFillCols, NULL, 0,
(const SNodeListNode*)(pPhyFillNode->pFillValues));
pFillSup->pAllColInfo = createFillColInfo(pExprSup->pExprInfo, pFillSup->numOfFillCols, NULL, numOfNotFillCols, NULL,
0, (const SNodeListNode*)(pPhyFillNode->pFillValues));
QUERY_CHECK_NULL(pFillSup->pAllColInfo, code, lino, _end, terrno);
pFillSup->type = convertFillType(pPhyFillNode->fillMode);
@ -349,6 +351,7 @@ static int32_t initTimeSliceFillSup(SStreamInterpFuncPhysiNode* pPhyFillNode, SE
code = initOffsetInfo(&pFillSup->pOffsetInfo, pInputRes);
QUERY_CHECK_CODE(code, lino, _end);
pFillSup->normalFill = false;
(*ppResFillSup) = pFillSup;
@ -406,8 +409,15 @@ static bool isSelectGroupConstValueFunc(SExprInfo* pExprInfo) {
return (functionType == FUNCTION_TYPE_GROUP_CONST_VALUE);
}
static int32_t fillPointResult(SStreamFillSupporter* pFillSup, SResultRowData* pResRow, SResultRowData* pNonFillRow, TSKEY ts, SSDataBlock* pBlock,
bool* pRes, bool isFilled) {
static bool isWindowFunction(SFillColInfo* pCol) {
return (pCol->pExpr->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_START ||
pCol->pExpr->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_END ||
pCol->pExpr->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_DURATION ||
pCol->pExpr->base.pParam[0].pCol->colType == COLUMN_TYPE_IS_WINDOW_FILLED);
}
static int32_t fillPointResult(SStreamFillSupporter* pFillSup, SResultRowData* pResRow, SResultRowData* pNonFillRow,
TSKEY ts, SSDataBlock* pBlock, bool* pRes, bool isFilled) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pBlock->info.rows >= pBlock->info.capacity) {
@ -434,10 +444,22 @@ static int32_t fillPointResult(SStreamFillSupporter* pFillSup, SResultRowData* p
} else if (isIsfilledPseudoColumn(pFillCol->pExpr)) {
code = colDataSetVal(pDstCol, pBlock->info.rows, (char*)&isFilled, false);
QUERY_CHECK_CODE(code, lino, _end);
} else if (pFillSup->normalFill && isWindowFunction(pFillCol)) {
SFillInfo tmpInfo = {
.currentKey = ts,
.order = TSDB_ORDER_ASC,
.interval = pFillSup->interval,
.isFilled = isFilled,
};
bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pDstCol, pBlock->info.rows);
if (!filled) {
qError("%s failed at line %d since fill errror", __func__, __LINE__);
}
} else {
int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId;
SResultCellData* pCell = NULL;
if (IS_FILL_CONST_VALUE(pFillSup->type) && (isGroupKeyFunc(pFillCol->pExpr) || isSelectGroupConstValueFunc(pFillCol->pExpr)) ) {
if (IS_FILL_CONST_VALUE(pFillSup->type) &&
(isGroupKeyFunc(pFillCol->pExpr) || isSelectGroupConstValueFunc(pFillCol->pExpr))) {
pCell = getSliceResultCell(pNonFillRow->pRowVal, srcSlot, pFillSup->pOffsetInfo);
} else {
pCell = getSliceResultCell(pResRow->pRowVal, srcSlot, pFillSup->pOffsetInfo);
@ -464,7 +486,8 @@ static void fillNormalRange(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFi
STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current};
// if (inWinRange(&pFillSup->winRange, &st)) {
bool res = true;
code = fillPointResult(pFillSup, pFillInfo->pResRow, pFillInfo->pNonFillRow, pFillInfo->current, pBlock, &res, true);
code =
fillPointResult(pFillSup, pFillInfo->pResRow, pFillInfo->pNonFillRow, pFillInfo->current, pBlock, &res, true);
QUERY_CHECK_CODE(code, lino, _end);
// }
pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
@ -497,7 +520,18 @@ static void fillLinearRange(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFi
bool isFilled = true;
code = colDataSetVal(pDstCol, pBlock->info.rows, (char*)&isFilled, false);
QUERY_CHECK_CODE(code, lino, _end);
} else if (isInterpFunc(pFillCol->pExpr)) {
} else if (pFillSup->normalFill && isWindowFunction(pFillCol)) {
SFillInfo tmpInfo = {
.currentKey = pFillInfo->current,
.order = TSDB_ORDER_ASC,
.interval = pFillSup->interval,
.isFilled = true,
};
bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pDstCol, pBlock->info.rows);
if (!filled) {
qError("%s failed at line %d since fill errror", __func__, lino);
}
} else if (isInterpFunc(pFillCol->pExpr) || pFillSup->normalFill) {
int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId;
SResultCellData* pCell = getSliceResultCell(pFillInfo->pResRow->pRowVal, srcSlot, pFillSup->pOffsetInfo);
if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) {
@ -546,14 +580,14 @@ static void setFillKeyInfo(TSKEY start, TSKEY end, SInterval* pInterval, SStream
pFillInfo->end = end;
}
static TSKEY adustPrevTsKey(TSKEY pointTs, TSKEY rowTs, SInterval* pInterval) {
TSKEY adustPrevTsKey(TSKEY pointTs, TSKEY rowTs, SInterval* pInterval) {
if (rowTs >= pointTs) {
pointTs = taosTimeAdd(pointTs, pInterval->sliding, pInterval->slidingUnit, pInterval->precision, NULL);
}
return pointTs;
}
static TSKEY adustEndTsKey(TSKEY pointTs, TSKEY rowTs, SInterval* pInterval) {
TSKEY adustEndTsKey(TSKEY pointTs, TSKEY rowTs, SInterval* pInterval) {
if (rowTs <= pointTs) {
pointTs = taosTimeAdd(pointTs, pInterval->sliding * -1, pInterval->slidingUnit, pInterval->precision, NULL);
}
@ -568,7 +602,7 @@ static void adjustFillResRow(SResultRowData** ppResRow, SStreamFillSupporter* pF
}
}
static void doStreamFillRange(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pRes) {
void doStreamTimeSliceFillRange(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
bool res = true;
@ -618,7 +652,8 @@ static void doStreamFillRange(SStreamFillSupporter* pFillSup, SStreamFillInfo* p
pFillInfo->pLinearInfo->hasNext = false;
taosArraySwap(pFillInfo->pLinearInfo->pEndPoints, pFillInfo->pLinearInfo->pNextEndPoints);
pFillInfo->pResRow = &pFillSup->cur;
setFillKeyInfo(pFillSup->cur.key, pFillInfo->pLinearInfo->nextEnd, &pFillSup->interval, pFillInfo);
TSKEY newStart = adustPrevTsKey(pFillSup->cur.key, pFillSup->cur.key, &pFillSup->interval);
setFillKeyInfo(newStart, pFillInfo->pLinearInfo->nextEnd, &pFillSup->interval, pFillInfo);
fillLinearRange(pFillSup, pFillInfo, pRes);
}
}
@ -694,7 +729,7 @@ static int32_t getLinearResultInfoFromState(SStreamAggSupporter* pAggSup, SStrea
int32_t lino = 0;
int32_t tmpRes = TSDB_CODE_SUCCESS;
void* pState = pAggSup->pState;
resetPrevAndNextWindow(pFillSup);
resetTimeSlicePrevAndNextWindow(pFillSup);
pCurPoint->pResPos = NULL;
pPrevPoint->pResPos = NULL;
pNextPoint->pResPos = NULL;
@ -718,7 +753,8 @@ static int32_t getLinearResultInfoFromState(SStreamAggSupporter* pAggSup, SStrea
(void**)&pPrevPoint->pResPos, &preVLen, &tmpRes);
QUERY_CHECK_CODE(code, lino, _end);
if (tmpRes == TSDB_CODE_SUCCESS) {
QUERY_CHECK_CONDITION(!IS_INVALID_WIN_KEY(pPrevPoint->key.ts), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
QUERY_CHECK_CONDITION(!IS_INVALID_WIN_KEY(pPrevPoint->key.ts), code, lino, _end,
TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
setPointBuff(pPrevPoint, pFillSup);
if (HAS_ROW_DATA(pPrevPoint->pRightRow)) {
pFillSup->prev.key = pPrevPoint->pRightRow->key;
@ -746,7 +782,8 @@ static int32_t getLinearResultInfoFromState(SStreamAggSupporter* pAggSup, SStrea
(void**)&pNextPoint->pResPos, &nextVLen, &tmpRes);
QUERY_CHECK_CODE(code, lino, _end);
if (tmpRes == TSDB_CODE_SUCCESS) {
QUERY_CHECK_CONDITION(!IS_INVALID_WIN_KEY(pNextPoint->key.ts), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
QUERY_CHECK_CONDITION(!IS_INVALID_WIN_KEY(pNextPoint->key.ts), code, lino, _end,
TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
setPointBuff(pNextPoint, pFillSup);
if (HAS_ROW_DATA(pNextPoint->pLeftRow)) {
pFillSup->next.key = pNextPoint->pLeftRow->key;
@ -777,7 +814,7 @@ static int32_t getResultInfoFromState(SStreamAggSupporter* pAggSup, SStreamFillS
int32_t lino = 0;
int32_t tmpRes = TSDB_CODE_SUCCESS;
void* pState = pAggSup->pState;
resetPrevAndNextWindow(pFillSup);
resetTimeSlicePrevAndNextWindow(pFillSup);
pCurPoint->pResPos = NULL;
pPrevPoint->pResPos = NULL;
pNextPoint->pResPos = NULL;
@ -806,10 +843,12 @@ static int32_t getResultInfoFromState(SStreamAggSupporter* pAggSup, SStreamFillS
code = pAggSup->stateStore.streamStateFillGetPrev(pState, &pCurPoint->key, &pPrevPoint->key,
(void**)&pPrevPoint->pResPos, &preVLen, &tmpRes);
QUERY_CHECK_CODE(code, lino, _end);
qDebug("===stream=== set stream interp resutl prev buf.ts:%" PRId64 ", groupId:%" PRId64 ", res:%d", pPrevPoint->key.ts, pPrevPoint->key.groupId, tmpRes);
qDebug("===stream=== set stream interp resutl prev buf.ts:%" PRId64 ", groupId:%" PRId64 ", res:%d",
pPrevPoint->key.ts, pPrevPoint->key.groupId, tmpRes);
if (tmpRes == TSDB_CODE_SUCCESS) {
QUERY_CHECK_CONDITION(!IS_INVALID_WIN_KEY(pPrevPoint->key.ts), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
QUERY_CHECK_CONDITION(!IS_INVALID_WIN_KEY(pPrevPoint->key.ts), code, lino, _end,
TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
setPointBuff(pPrevPoint, pFillSup);
if (HAS_ROW_DATA(pPrevPoint->pRightRow)) {
pFillSup->prev.key = pPrevPoint->pRightRow->key;
@ -826,9 +865,11 @@ static int32_t getResultInfoFromState(SStreamAggSupporter* pAggSup, SStreamFillS
code = pAggSup->stateStore.streamStateFillGetNext(pState, &pCurPoint->key, &pNextPoint->key,
(void**)&pNextPoint->pResPos, &nextVLen, &tmpRes);
QUERY_CHECK_CODE(code, lino, _end);
qDebug("===stream=== set stream interp resutl next buf.ts:%" PRId64 ", groupId:%" PRId64 ", res:%d", pNextPoint->key.ts, pNextPoint->key.groupId, tmpRes);
qDebug("===stream=== set stream interp resutl next buf.ts:%" PRId64 ", groupId:%" PRId64 ", res:%d",
pNextPoint->key.ts, pNextPoint->key.groupId, tmpRes);
if (tmpRes == TSDB_CODE_SUCCESS) {
QUERY_CHECK_CONDITION(!IS_INVALID_WIN_KEY(pNextPoint->key.ts), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
QUERY_CHECK_CONDITION(!IS_INVALID_WIN_KEY(pNextPoint->key.ts), code, lino, _end,
TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
setPointBuff(pNextPoint, pFillSup);
if (HAS_ROW_DATA(pNextPoint->pLeftRow)) {
pFillSup->next.key = pNextPoint->pLeftRow->key;
@ -900,7 +941,8 @@ static int32_t getPointInfoFromStateRight(SStreamAggSupporter* pAggSup, SStreamF
(void**)&pCurPoint->pResPos, &nextVLen, &tmpRes);
QUERY_CHECK_CODE(code, lino, _end);
qDebug("===stream=== set stream interp cur point buf.ts:%" PRId64 ", groupId:%" PRId64 ", res:%d", pCurPoint->key.ts, pCurPoint->key.groupId, tmpRes);
qDebug("===stream=== set stream interp cur point buf.ts:%" PRId64 ", groupId:%" PRId64 ", res:%d", pCurPoint->key.ts,
pCurPoint->key.groupId, tmpRes);
if (tmpRes == TSDB_CODE_SUCCESS) {
setPointBuff(pCurPoint, pFillSup);
@ -930,7 +972,8 @@ static int32_t getPointInfoFromState(SStreamAggSupporter* pAggSup, SStreamFillSu
&curVLen, pWinCode);
QUERY_CHECK_CODE(code, lino, _end);
qDebug("===stream=== set stream interp buf.ts:%" PRId64 ", groupId:%" PRId64, pCurPoint->key.ts, pCurPoint->key.groupId);
qDebug("===stream=== set stream interp buf.ts:%" PRId64 ", groupId:%" PRId64, pCurPoint->key.ts,
pCurPoint->key.groupId);
setPointBuff(pCurPoint, pFillSup);
@ -990,9 +1033,10 @@ static void copyNonFillValueInfo(SStreamFillSupporter* pFillSup, SStreamFillInfo
}
}
static void copyCalcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, int32_t numOfCol, int32_t* pOffsetInfo) {
static void copyCalcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, int32_t numOfCol,
int32_t* pOffsetInfo, bool normalFill) {
for (int32_t i = 0; i < numOfCol; i++) {
if (isInterpFunc(pFillCol[i].pExpr)) {
if (isInterpFunc(pFillCol[i].pExpr) || normalFill) {
int32_t slotId = pFillCol[i].pExpr->base.pParam[0].pCol->slotId;
SResultCellData* pECell = getSliceResultCell(pEndRow->pRowVal, slotId, pOffsetInfo);
SPoint* pPoint = taosArrayGet(pEndPoins, slotId);
@ -1045,7 +1089,7 @@ static void setForceWindowCloseFillRule(SStreamFillSupporter* pFillSup, SStreamF
}
}
static void setTimeSliceFillRule(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, TSKEY ts) {
void setTimeSliceFillRule(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, TSKEY ts) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (IS_FILL_CONST_VALUE(pFillInfo->type)) {
@ -1131,21 +1175,34 @@ static void setTimeSliceFillRule(SStreamFillSupporter* pFillSup, SStreamFillInfo
} break;
case TSDB_FILL_LINEAR: {
if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) {
if (pFillSup->normalFill) {
setFillKeyInfo(prevWKey, endTs, &pFillSup->interval, pFillInfo);
pFillInfo->pos = FILL_POS_MID;
copyCalcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
pFillSup->numOfAllCols, pFillSup->pOffsetInfo, pFillSup->normalFill);
pFillSup->next.key = pFillSup->nextOriginKey;
pFillInfo->pResRow = &pFillSup->prev;
copyCalcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pNextEndPoints, pFillSup->pAllColInfo,
pFillSup->numOfAllCols, pFillSup->pOffsetInfo, pFillSup->normalFill);
pFillInfo->pLinearInfo->nextEnd = nextWKey;
pFillInfo->pLinearInfo->hasNext = true;
} else {
setFillKeyInfo(prevWKey, nextWKey, &pFillSup->interval, pFillInfo);
pFillInfo->pos = FILL_POS_INVALID;
SET_WIN_KEY_INVALID(pFillInfo->pLinearInfo->nextEnd);
pFillSup->next.key = pFillSup->nextOriginKey;
copyCalcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
pFillSup->numOfAllCols, pFillSup->pOffsetInfo);
pFillSup->numOfAllCols, pFillSup->pOffsetInfo, pFillSup->normalFill);
pFillSup->prev.key = pFillSup->prevOriginKey;
pFillInfo->pResRow = &pFillSup->prev;
pFillInfo->pLinearInfo->hasNext = false;
}
} else if (hasPrevWindow(pFillSup)) {
setFillKeyInfo(prevWKey, endTs, &pFillSup->interval, pFillInfo);
pFillInfo->pos = FILL_POS_END;
SET_WIN_KEY_INVALID(pFillInfo->pLinearInfo->nextEnd);
copyCalcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
pFillSup->numOfAllCols, pFillSup->pOffsetInfo);
pFillSup->numOfAllCols, pFillSup->pOffsetInfo, pFillSup->normalFill);
pFillSup->prev.key = pFillSup->prevOriginKey;
pFillInfo->pResRow = &pFillSup->prev;
pFillInfo->pLinearInfo->hasNext = false;
@ -1156,7 +1213,7 @@ static void setTimeSliceFillRule(SStreamFillSupporter* pFillSup, SStreamFillInfo
SET_WIN_KEY_INVALID(pFillInfo->pLinearInfo->nextEnd);
pFillSup->next.key = pFillSup->nextOriginKey;
copyCalcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
pFillSup->numOfAllCols, pFillSup->pOffsetInfo);
pFillSup->numOfAllCols, pFillSup->pOffsetInfo, pFillSup->normalFill);
pFillInfo->pResRow = &pFillSup->cur;
pFillInfo->pLinearInfo->hasNext = false;
}
@ -1196,8 +1253,7 @@ static bool needAdjustValue(SSlicePoint* pPoint, TSKEY ts, void* pPkVal, SStream
return true;
} else {
if (pPoint->key.ts == ts) {
if (pFillSup->comparePkColFn == NULL ||
comparePkVal(pPoint->pRightRow, pPkVal, pFillSup) >= 0) {
if (pFillSup->comparePkColFn == NULL || comparePkVal(pPoint->pRightRow, pPkVal, pFillSup) >= 0) {
return true;
}
}
@ -1234,8 +1290,7 @@ static bool needAdjustValue(SSlicePoint* pPoint, TSKEY ts, void* pPkVal, SStream
if (pPoint->pRightRow->key > ts) {
return true;
} else if (pPoint->pRightRow->key == ts) {
if (pFillSup->comparePkColFn == NULL ||
comparePkVal(pPoint->pRightRow, pPkVal, pFillSup) >= 0) {
if (pFillSup->comparePkColFn == NULL || comparePkVal(pPoint->pRightRow, pPkVal, pFillSup) >= 0) {
return true;
}
}
@ -1262,8 +1317,7 @@ static bool needAdjustValue(SSlicePoint* pPoint, TSKEY ts, void* pPkVal, SStream
if (pPoint->pRightRow->key > ts) {
return true;
} else if (pPoint->pRightRow->key == ts) {
if (pFillSup->comparePkColFn == NULL ||
comparePkVal(pPoint->pRightRow, pPkVal, pFillSup) >= 0) {
if (pFillSup->comparePkColFn == NULL || comparePkVal(pPoint->pRightRow, pPkVal, pFillSup) >= 0) {
return true;
}
}
@ -1402,7 +1456,8 @@ static void doStreamTimeSliceImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock)
}
right = needAdjustValue(&curPoint, tsCols[startPos], pPkVal, pFillSup, false, pFillSup->type);
if (right) {
transBlockToSliceResultRow(pBlock, startPos, tsCols[startPos], curPoint.pRightRow, pFillSup->rowSize, pPkVal, pPkColDataInfo, pFillSup->pOffsetInfo);
transBlockToSliceResultRow(pBlock, startPos, tsCols[startPos], curPoint.pRightRow, pFillSup->rowSize, pPkVal,
pPkColDataInfo, pFillSup->pOffsetInfo);
bool needDel = pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS;
code = saveTimeSliceWinResultInfo(pAggSup, &pInfo->twAggSup, &curPoint.key, pInfo->pUpdatedMap, needDel,
pInfo->pDeletedMap);
@ -1421,10 +1476,11 @@ static void doStreamTimeSliceImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock)
}
left = needAdjustValue(&nextPoint, tsCols[leftRowId], pPkVal, pFillSup, true, pFillSup->type);
if (left) {
transBlockToSliceResultRow(pBlock, leftRowId, tsCols[leftRowId], nextPoint.pLeftRow, pFillSup->rowSize, pPkVal, pPkColDataInfo, pFillSup->pOffsetInfo);
transBlockToSliceResultRow(pBlock, leftRowId, tsCols[leftRowId], nextPoint.pLeftRow, pFillSup->rowSize, pPkVal,
pPkColDataInfo, pFillSup->pOffsetInfo);
bool needDel = pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS;
code = saveTimeSliceWinResultInfo(pAggSup, &pInfo->twAggSup, &nextPoint.key, pInfo->pUpdatedMap,
needDel, pInfo->pDeletedMap);
code = saveTimeSliceWinResultInfo(pAggSup, &pInfo->twAggSup, &nextPoint.key, pInfo->pUpdatedMap, needDel,
pInfo->pDeletedMap);
QUERY_CHECK_CODE(code, lino, _end);
}
releaseOutputBuf(pAggSup->pState, nextPoint.pResPos, &pAggSup->stateStore);
@ -1446,7 +1502,8 @@ static void doStreamTimeSliceImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock)
}
right = needAdjustValue(&curPoint, tsCols[startPos], pPkVal, pFillSup, false, pFillSup->type);
if (right) {
transBlockToSliceResultRow(pBlock, startPos, tsCols[startPos], curPoint.pRightRow, pFillSup->rowSize, pPkVal, pPkColDataInfo, pFillSup->pOffsetInfo);
transBlockToSliceResultRow(pBlock, startPos, tsCols[startPos], curPoint.pRightRow, pFillSup->rowSize, pPkVal,
pPkColDataInfo, pFillSup->pOffsetInfo);
bool needDel = pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS;
code = saveTimeSliceWinResultInfo(pAggSup, &pInfo->twAggSup, &curPoint.key, pInfo->pUpdatedMap, needDel,
pInfo->pDeletedMap);
@ -1485,8 +1542,9 @@ void getPrevResKey(int64_t curGroupId, SArray* pKeyArray, int32_t curIndex, TSKE
*pNextKey = INT64_MIN;
}
void doBuildTimeSlicePointResult(SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwSup, SStreamFillSupporter* pFillSup,
SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) {
void doBuildTimeSlicePointResult(SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwSup,
SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock,
SGroupResInfo* pGroupResInfo, SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
blockDataCleanup(pBlock);
@ -1512,8 +1570,8 @@ void doBuildTimeSlicePointResult(SStreamAggSupporter* pAggSup, STimeWindowAggSup
}
void* tbname = NULL;
int32_t winCode = TSDB_CODE_SUCCESS;
code =
pAggSup->stateStore.streamStateGetParName(pAggSup->pState, pBlock->info.id.groupId, &tbname, false, &winCode);
code = pAggSup->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname,
false, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS) {
pBlock->info.parTbName[0] = 0;
@ -1526,7 +1584,8 @@ void doBuildTimeSlicePointResult(SStreamAggSupporter* pAggSup, STimeWindowAggSup
SSlicePoint prevPoint = {0};
SSlicePoint nextPoint = {0};
if (pFillSup->type != TSDB_FILL_LINEAR) {
code = getResultInfoFromState(pAggSup, pFillSup, pKey->ts, pKey->groupId, &curPoint, &prevPoint, &nextPoint, isFwc);
code =
getResultInfoFromState(pAggSup, pFillSup, pKey->ts, pKey->groupId, &curPoint, &prevPoint, &nextPoint, isFwc);
} else {
code =
getLinearResultInfoFromState(pAggSup, pFillSup, pKey->ts, pKey->groupId, &curPoint, &prevPoint, &nextPoint);
@ -1550,7 +1609,7 @@ void doBuildTimeSlicePointResult(SStreamAggSupporter* pAggSup, STimeWindowAggSup
} else {
setTimeSliceFillRule(pFillSup, pFillInfo, pKey->ts);
}
doStreamFillRange(pFillSup, pFillInfo, pBlock);
doStreamTimeSliceFillRange(pFillSup, pFillInfo, pBlock);
releaseOutputBuf(pAggSup->pState, curPoint.pResPos, &pAggSup->stateStore);
releaseOutputBuf(pAggSup->pState, prevPoint.pResPos, &pAggSup->stateStore);
releaseOutputBuf(pAggSup->pState, nextPoint.pResPos, &pAggSup->stateStore);
@ -1566,7 +1625,8 @@ _end:
}
}
static void doBuildTimeSliceDeleteResult(SStreamAggSupporter* pAggSup, SStreamFillSupporter* pFillSup, SArray* pWins, int32_t* index, SSDataBlock* pBlock) {
static void doBuildTimeSliceDeleteResult(SStreamAggSupporter* pAggSup, SStreamFillSupporter* pFillSup, SArray* pWins,
int32_t* index, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
blockDataCleanup(pBlock);
@ -1587,7 +1647,8 @@ static void doBuildTimeSliceDeleteResult(SStreamAggSupporter* pAggSup, SStreamFi
SSlicePoint nextPoint = {0};
STimeWindow tw = {0};
if (pFillSup->type != TSDB_FILL_LINEAR) {
code = getResultInfoFromState(pAggSup, pFillSup, pKey->ts, pKey->groupId, &curPoint, &prevPoint, &nextPoint, false);
code =
getResultInfoFromState(pAggSup, pFillSup, pKey->ts, pKey->groupId, &curPoint, &prevPoint, &nextPoint, false);
} else {
code =
getLinearResultInfoFromState(pAggSup, pFillSup, pKey->ts, pKey->groupId, &curPoint, &prevPoint, &nextPoint);
@ -1627,7 +1688,8 @@ static void doBuildTimeSliceDeleteResult(SStreamAggSupporter* pAggSup, SStreamFi
void* tbname = NULL;
int32_t winCode = TSDB_CODE_SUCCESS;
code = pAggSup->stateStore.streamStateGetParName(pAggSup->pState, pKey->groupId, &tbname, false, &winCode);
code = pAggSup->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pKey->groupId, &tbname, false,
&winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS) {
@ -1658,8 +1720,7 @@ static int32_t buildTimeSliceResult(SOperatorInfo* pOperator, SSDataBlock** ppRe
uint16_t opType = pOperator->operatorType;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
doBuildTimeSliceDeleteResult(pAggSup, pInfo->pFillSup, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
doBuildTimeSliceDeleteResult(pAggSup, pInfo->pFillSup, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes, pTaskInfo);
if (pInfo->pDelRes->info.rows != 0) {
// process the rest of the data
printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
@ -1667,7 +1728,8 @@ static int32_t buildTimeSliceResult(SOperatorInfo* pOperator, SSDataBlock** ppRe
goto _end;
}
doBuildTimeSlicePointResult(pAggSup, &pInfo->twAggSup, pInfo->pFillSup, pInfo->pFillInfo, pInfo->pRes, &pInfo->groupResInfo);
doBuildTimeSlicePointResult(pAggSup, &pInfo->twAggSup, pInfo->pFillSup, pInfo->pFillInfo, pInfo->pRes,
&pInfo->groupResInfo, pTaskInfo);
if (pInfo->pRes->info.rows != 0) {
printDataBlock(pInfo->pRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
(*ppRes) = pInfo->pRes;
@ -1805,7 +1867,7 @@ static int32_t doStreamTimeSliceNext(SOperatorInfo* pOperator, SSDataBlock** ppR
if (hasRemainCalc(pInfo->pFillInfo) ||
(pInfo->pFillInfo->pos != FILL_POS_INVALID && pInfo->pFillInfo->needFill == true)) {
blockDataCleanup(pInfo->pRes);
doStreamFillRange(pInfo->pFillSup, pInfo->pFillInfo, pInfo->pRes);
doStreamTimeSliceFillRange(pInfo->pFillSup, pInfo->pFillInfo, pInfo->pRes);
if (pInfo->pRes->info.rows > 0) {
printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pInfo->pRes;
@ -1830,7 +1892,7 @@ static int32_t doStreamTimeSliceNext(SOperatorInfo* pOperator, SSDataBlock** ppR
}
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) {
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState);
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState, 1, INT64_MAX);
}
setStreamOperatorCompleted(pOperator);
resetStreamFillSup(pInfo->pFillSup);
@ -1953,7 +2015,7 @@ static int32_t doStreamTimeSliceNext(SOperatorInfo* pOperator, SSDataBlock** ppR
if (!(*ppRes)) {
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) {
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState);
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState, 1, INT64_MAX);
}
setStreamOperatorCompleted(pOperator);
resetStreamFillSup(pInfo->pFillSup);
@ -2176,7 +2238,7 @@ int32_t createStreamTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
QUERY_CHECK_NULL(pInfo->historyWins, code, lino, _error, terrno);
if (pHandle) {
pInfo->isHistoryOp = pHandle->fillHistory;
pInfo->isHistoryOp = (pHandle->fillHistory == STREAM_HISTORY_OPERATOR);
}
pInfo->pCloseTs = taosArrayInit(1024, sizeof(TSKEY));
@ -2198,7 +2260,8 @@ int32_t createStreamTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
taosMemoryFree(buff);
QUERY_CHECK_CODE(code, lino, _error);
}
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamTimeSliceNext, NULL, destroyStreamTimeSliceOperatorInfo,
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doStreamTimeSliceNext, NULL, destroyStreamTimeSliceOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamTimeSliceReleaseState, streamTimeSliceReloadState);

169
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -19,6 +19,8 @@
#include "operator.h"
#include "querytask.h"
#include "streamexecutorInt.h"
#include "streaminterval.h"
#include "streamsession.h"
#include "tchecksum.h"
#include "tcommon.h"
#include "tcompare.h"
@ -43,22 +45,9 @@
#define MAX_STREAM_HISTORY_RESULT 20000000
typedef struct SStateWindowInfo {
SResultWindowInfo winInfo;
SStateKeys* pStateKey;
} SStateWindowInfo;
typedef struct SPullWindowInfo {
STimeWindow window;
uint64_t groupId;
STimeWindow calWin;
} SPullWindowInfo;
static int32_t doStreamMidIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes);
typedef int32_t (*__compare_fn_t)(void* pKey, void* data, int32_t index);
static int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {
int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {
int firstPos = 0, lastPos = num - 1, midPos = -1;
int numOfRows = 0;
@ -190,7 +179,7 @@ static void removeResults(SArray* pWins, SSHashObj* pUpdatedMap) {
}
}
static int32_t compareWinKey(void* pKey, void* data, int32_t index) {
int32_t compareWinKey(void* pKey, void* data, int32_t index) {
void* pDataPos = taosArrayGet((SArray*)data, index);
return winKeyCmprImpl(pKey, pDataPos);
}
@ -230,7 +219,7 @@ static void doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId)
pAPI->stateStore.streamStateDel(pInfo->pState, &key);
}
static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }
int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }
static void doDeleteWindowByGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
@ -413,9 +402,9 @@ STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {
return w;
}
static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWins, int32_t* index,
static void doBuildDeleteResult(SExecTaskInfo* pTaskInfo, SArray* pWins, int32_t* index,
SSDataBlock* pBlock) {
doBuildDeleteResultImpl(&pInfo->stateStore, pInfo->pState, pWins, index, pBlock);
doBuildDeleteResultImpl(&pTaskInfo->storageAPI.stateStore, pTaskInfo->streamInfo.pState, pWins, index, pBlock);
}
void doBuildDeleteResultImpl(SStateStore* pAPI, SStreamState* pState, SArray* pWins, int32_t* index,
@ -655,7 +644,7 @@ _end:
}
bool hasIntervalWindow(void* pState, SWinKey* pKey, SStateStore* pStore) {
return pStore->streamStateCheck(pState, pKey);
return pStore->streamStateCheck(pState, pKey, false, NULL);
}
int32_t setIntervalOutputBuf(void* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId,
@ -741,7 +730,7 @@ static void clearSpecialDataBlock(SSDataBlock* pBlock) {
blockDataCleanup(pBlock);
}
static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {
void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
clearSpecialDataBlock(pBlock);
@ -1050,7 +1039,7 @@ void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataB
buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);
}
static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
TSKEY* primaryKeys, int32_t prevPosition) {
int32_t startPos = prevPosition + 1;
if (startPos == pDataBlockInfo->rows) {
@ -1577,7 +1566,7 @@ static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes
}
}
doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
doBuildDeleteResult(pTaskInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
if (pInfo->pDelRes->info.rows != 0) {
// process the rest of the data
printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
@ -1774,7 +1763,7 @@ static int32_t doStreamFinalIntervalAggNext(SOperatorInfo* pOperator, SSDataBloc
}
taosArrayDestroy(delWins);
doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
doBuildDeleteResult(pTaskInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
if (pInfo->pDelRes->info.rows != 0) {
// process the rest of the data
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
@ -2231,6 +2220,7 @@ void destroyStreamSessionAggOperatorInfo(void* param) {
taosArrayDestroy(pInfo->historyWins);
blockDataDestroy(pInfo->pCheckpointRes);
tSimpleHashCleanup(pInfo->pPkDeleted);
destroyNonBlockAggSupptor(&pInfo->nbSup);
taosMemoryFreeClear(param);
}
@ -2260,7 +2250,7 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num
}
int32_t initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) {
STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic, int64_t recalculateInterval) {
SExecTaskInfo* pTaskInfo = downstream->pTaskInfo;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
@ -2271,7 +2261,7 @@ int32_t initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup,
}
if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
code = initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic);
code = initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic, recalculateInterval);
return code;
}
SStreamScanInfo* pScanInfo = downstream->info;
@ -2289,6 +2279,15 @@ int32_t initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup,
pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;
}
pBasic->pTsDataState = pScanInfo->basic.pTsDataState;
if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION) {
pScanInfo->scanAllTables = true;
}
pScanInfo->recalculateInterval = recalculateInterval;
pScanInfo->windowSup.parentType = type;
pScanInfo->recParam.gap = pAggSup->gap;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
@ -2354,6 +2353,8 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, in
pExpSup->pCtx[i].saveHandle.pState = pSup->pState;
}
pSup->pCur = NULL;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
@ -2422,7 +2423,6 @@ int32_t setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY e
}
qDebug("===stream===set session window buff .start:%" PRId64 ",end:%" PRId64 ",groupid:%" PRIu64,
pCurWin->sessionWin.win.skey, pCurWin->sessionWin.win.ekey, pCurWin->sessionWin.groupId);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
@ -2537,6 +2537,7 @@ int32_t updateSessionWindowInfo(SStreamAggSupporter* pAggSup, SResultWindowInfo*
code = saveDeleteRes(pStDeleted, pWinInfo->sessionWin);
QUERY_CHECK_CODE(code, lino, _end);
}
removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->sessionWin);
pWinInfo->sessionWin.win.skey = pStartTs[i];
}
@ -2659,8 +2660,8 @@ _end:
return code;
}
static int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool addGap, int32_t* pWinNum) {
int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool addGap, int32_t* pWinNum, bool* pIsEnd) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExprSupp* pSup = &pOperator->exprSupp;
@ -2672,11 +2673,21 @@ static int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo*
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
if (pIsEnd != NULL) {
(*pIsEnd) = false;
}
// Just look for the window behind StartIndex
while (1) {
SResultWindowInfo winInfo = {0};
getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo);
if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap) ||
if (!IS_VALID_SESSION_WIN(winInfo)) {
if (pIsEnd != NULL) {
(*pIsEnd) = true;
}
releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);
break;
}
if (!isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap) ||
!inWinRange(&pAggSup->winRange, &winInfo.sessionWin.win)) {
releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);
break;
@ -2805,7 +2816,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
pOperator, winDelta);
QUERY_CHECK_CODE(code, lino, _end);
code = compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted, addGap, NULL);
code = compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted, addGap, NULL, NULL);
QUERY_CHECK_CODE(code, lino, _end);
code = saveSessionOutputBuf(pAggSup, &winInfo);
@ -3069,7 +3080,7 @@ static int32_t rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray,
&pInfo->twAggSup.timeWindowData);
QUERY_CHECK_CODE(code, lino, _end);
code = compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL, true, NULL);
code = compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL, true, NULL, NULL);
QUERY_CHECK_CODE(code, lino, _end);
releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore);
@ -3974,7 +3985,7 @@ void streamSessionReloadState(SOperatorInfo* pOperator) {
}
int32_t winNum = 0;
code = compactSessionWindow(pOperator, &winInfo, pInfo->pStUpdated, pInfo->pStDeleted, true, &winNum);
code = compactSessionWindow(pOperator, &winInfo, pInfo->pStUpdated, pInfo->pStDeleted, true, &winNum, NULL);
QUERY_CHECK_CODE(code, lino, _end);
if (winNum > 0) {
@ -4097,7 +4108,7 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode
goto _error;
}
if (pHandle) {
pInfo->isHistoryOp = pHandle->fillHistory;
pInfo->isHistoryOp = (pHandle->fillHistory == STREAM_HISTORY_OPERATOR);
}
code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);
@ -4109,9 +4120,9 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode
pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);
QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno);
pInfo->pOperator = pOperator;
initNonBlockAggSupptor(&pInfo->nbSup, NULL, NULL);
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,
setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), nodeType(pSessionNode), true,
OP_NOT_OPENED, pInfo, pTaskInfo);
if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
// for stream
@ -4126,18 +4137,31 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode
QUERY_CHECK_CODE(code, lino, _error);
}
}
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doStreamSessionAggNext, NULL, destroyStreamSessionAggOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);
code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
if (pHandle->fillHistory == STREAM_HISTORY_OPERATOR) {
setFillHistoryOperatorFlag(&pInfo->basic);
} else if (pHandle->fillHistory == STREAM_RECALCUL_OPERATOR) {
setRecalculateOperatorFlag(&pInfo->basic);
}
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doStreamSessionNonblockAggNext, NULL, destroyStreamSessionAggOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamSessionNonblockReleaseState, streamSessionNonblockReloadState);
} else {
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doStreamSessionAggNext, NULL, destroyStreamSessionAggOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);
}
if (downstream) {
pInfo->basic.primaryPkIndex = -1;
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
&pInfo->twAggSup, &pInfo->basic, pSessionNode->window.recalculateInterval);
QUERY_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
@ -4359,9 +4383,7 @@ int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhys
QUERY_CHECK_CODE(code, lino, _error);
}
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
pOperator->operatorType = pPhyNode->type;
pInfo->pOperator = pOperator;
if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
@ -4371,12 +4393,6 @@ int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhys
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionSemiReloadState);
}
code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), pPhyNode->type, false, OP_NOT_OPENED, pInfo,
pTaskInfo);
if (numOfChild > 0) {
pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));
QUERY_CHECK_NULL(pInfo->pChildren, code, lino, _error, terrno);
@ -4389,7 +4405,7 @@ int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhys
SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
pAPI->stateStore.streamStateSetNumber(pChInfo->streamAggSup.pState, i, pInfo->primaryTsIndex);
pTaskInfo->storageAPI.stateStore.streamStateSetNumber(pChInfo->streamAggSup.pState, i, pInfo->primaryTsIndex);
void* tmp = taosArrayPush(pInfo->pChildren, &pChildOp);
if (!tmp) {
code = terrno;
@ -4467,6 +4483,7 @@ void destroyStreamStateOperatorInfo(void* param) {
taosArrayDestroy(pInfo->historyWins);
blockDataDestroy(pInfo->pCheckpointRes);
tSimpleHashCleanup(pInfo->pPkDeleted);
destroyNonBlockAggSupptor(&pInfo->nbSup);
taosMemoryFreeClear(param);
}
@ -4498,7 +4515,7 @@ bool compareWinStateKey(SStateKeys* left, SStateKeys* right) {
return compareVal(left->pData, right);
}
static void getNextStateWin(const SStreamAggSupporter* pAggSup, SStateWindowInfo* pNextWin, bool asc) {
void getNextStateWin(const SStreamAggSupporter* pAggSup, SStateWindowInfo* pNextWin, bool asc) {
SStreamStateCur* pCur = NULL;
if (pAggSup == NULL || pNextWin == NULL) {
@ -5198,7 +5215,7 @@ void streamStateReleaseState(SOperatorInfo* pOperator) {
}
}
static int32_t compactStateWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin,
int32_t compactStateWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin,
SSHashObj* pStUpdated, SSHashObj* pStDeleted) {
SExprSupp* pSup = &pOperator->exprSupp;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -5378,7 +5395,7 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
}
if (pHandle) {
pInfo->isHistoryOp = pHandle->fillHistory;
pInfo->isHistoryOp = (pHandle->fillHistory == STREAM_HISTORY_OPERATOR);
}
code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);
@ -5390,7 +5407,7 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
pInfo->destHasPrimaryKey = pStateNode->window.destHasPrimaryKey;
pInfo->pOperator = pOperator;
setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED,
setOperatorInfo(pOperator, "StreamStateAggOperator", nodeType(pPhyNode), true, OP_NOT_OPENED,
pInfo, pTaskInfo);
// for stream
void* buff = NULL;
@ -5403,16 +5420,29 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
taosMemoryFree(buff);
QUERY_CHECK_CODE(code, lino, _error);
}
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAggNext, NULL, destroyStreamStateOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);
initNonBlockAggSupptor(&pInfo->nbSup, NULL, NULL);
code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
if (pStateNode->window.triggerType == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
if (pHandle->fillHistory == STREAM_HISTORY_OPERATOR) {
setFillHistoryOperatorFlag(&pInfo->basic);
} else if (pHandle->fillHistory == STREAM_RECALCUL_OPERATOR) {
setRecalculateOperatorFlag(&pInfo->basic);
}
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doStreamStateNonblockAggNext, NULL, destroyStreamStateOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamStateNonblockReleaseState, streamStateNonblockReloadState);
} else {
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAggNext, NULL, destroyStreamStateOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);
}
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
&pInfo->twAggSup, &pInfo->basic, 0);
QUERY_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
@ -5542,8 +5572,8 @@ static int32_t doStreamIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** p
continue;
} else {
if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {
code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
QUERY_CHECK_CODE(code, lino, _end);
qDebug("===stream===%s ignore recv block. type:%d", GET_TASKID(pTaskInfo), pBlock->info.type);
continue;
}
}
@ -5627,7 +5657,7 @@ _end:
return code;
}
static int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
SOperatorInfo** pOptrInfo) {
QRY_PARAM_CHECK(pOptrInfo);
@ -5792,17 +5822,6 @@ _error:
return code;
}
int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,
SReadHandle* pHandle, SOperatorInfo** pOptrInfo) {
SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;
if (pIntervalPhyNode->window.triggerType == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) {
return createStreamIntervalSliceOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, pOptrInfo);
} else {
return createStreamSingleIntervalOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, pOptrInfo);
}
return TSDB_CODE_SUCCESS;
}
static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pUpdatedMap) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
@ -5939,7 +5958,7 @@ static SSDataBlock* buildMidIntervalResult(SOperatorInfo* pOperator) {
}
qDebug("===stream=== build mid interval result");
doBuildDeleteResult(pInfo, pInfo->pMidPullDatas, &pInfo->midDelIndex, pInfo->pDelRes);
doBuildDeleteResult(pTaskInfo, pInfo->pMidPullDatas, &pInfo->midDelIndex, pInfo->pDelRes);
if (pInfo->pDelRes->info.rows != 0) {
// process the rest of the data
printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
@ -6059,7 +6078,7 @@ static int32_t doStreamMidIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock*
}
taosArrayDestroy(delWins);
doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
doBuildDeleteResult(pTaskInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
if (pInfo->pDelRes->info.rows != 0) {
// process the rest of the data
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

4
source/libs/executor/src/tfill.c
View File

@ -156,6 +156,10 @@ bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnI
code = colDataSetVal(pDstColInfoData, rowIndex, (const char*)&pFillInfo->interval.sliding, false);
QUERY_CHECK_CODE(code, lino, _end);
return true;
} else if (pCol->pExpr->base.pParam[0].pCol->colType == COLUMN_TYPE_IS_WINDOW_FILLED) {
code = colDataSetVal(pDstColInfoData, rowIndex, (const char*)&pFillInfo->isFilled, false);
QUERY_CHECK_CODE(code, lino, _end);
return true;
}
}

2
source/libs/executor/test/queryPlanTests.cpp
View File

@ -3096,7 +3096,7 @@ void qptExecPlan(SReadHandle* pReadHandle, SNode* pNode, SExecTaskInfo* pTaskInf
qptCtx.result.code = createMergeAlignedIntervalOperatorInfo(NULL, (SMergeAlignedIntervalPhysiNode*)pNode, pTaskInfo, ppOperaotr);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
qptCtx.result.code = createStreamIntervalOperatorInfo(NULL, (SPhysiNode*)pNode, pTaskInfo, pReadHandle, ppOperaotr);
qptCtx.result.code = createStreamSingleIntervalOperatorInfo(NULL, (SPhysiNode*)pNode, pTaskInfo, pReadHandle, ppOperaotr);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:

28
source/libs/function/src/builtins.c
View File

@ -5804,6 +5804,34 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = leastFunction,
.finalizeFunc = NULL
},
{
.name = "_group_id",
.type = FUNCTION_TYPE_GROUP_ID,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_SKIP_SCAN_CHECK_FUNC,
.parameters = {.minParamNum = 0,
.maxParamNum = 0,
.paramInfoPattern = 0,
.outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_BIGINT_TYPE}},
.translateFunc = translateOutBigInt,
.getEnvFunc = getTimePseudoFuncEnv,
.initFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL,
},
{
.name = "_iswindowfilled",
.type = FUNCTION_TYPE_IS_WINDOW_FILLED,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_SKIP_SCAN_CHECK_FUNC,
.parameters = {.minParamNum = 0,
.maxParamNum = 0,
.paramInfoPattern = 0,
.outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_BOOL_TYPE}},
.translateFunc = translateIsFilledPseudoColumn,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = isWinFilledFunction,
.finalizeFunc = NULL,
},
};
// clang-format on

7
source/libs/function/src/functionMgt.c
View File

@ -723,3 +723,10 @@ bool fmIsRowTsOriginFunc(int32_t funcId) {
}
return FUNCTION_TYPE_IROWTS_ORIGIN == funcMgtBuiltins[funcId].type;
}
bool fmIsGroupIdFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false;
}
return FUNCTION_TYPE_GROUP_ID == funcMgtBuiltins[funcId].type;
}

6
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -662,6 +662,7 @@ static int32_t logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* p
COPY_SCALAR_FIELD(windowSliding);
CLONE_NODE_FIELD(pAnomalyExpr);
COPY_CHAR_ARRAY_FIELD(anomalyOpt);
COPY_SCALAR_FIELD(recalculateInterval);
return TSDB_CODE_SUCCESS;
}
@ -1140,10 +1141,15 @@ int32_t nodesCloneNode(const SNode* pNode, SNode** ppNode) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL:
code = physiIntervalCopy((const SIntervalPhysiNode*)pNode, (SIntervalPhysiNode*)pDst);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_SESSION:
code = physiSessionCopy((const SSessionWinodwPhysiNode*)pNode, (SSessionWinodwPhysiNode*)pDst);
break;
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:

94
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -416,8 +416,14 @@ const char* nodesNodeName(ENodeType type) {
return "PhysiStreamInterval";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
return "PhysiStreamFinalInterval";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL:
return "PhysiStreamContinueInterval";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL:
return "PhysiStreamContinueFinalInterval";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
return "PhysiStreamSemiInterval";
return "PhysiStreamContinueSemiInterval";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL:
return "PhysiStreamContinueSemiInterval";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
return "PhysiStreamMidInterval";
case QUERY_NODE_PHYSICAL_PLAN_FILL:
@ -463,6 +469,18 @@ const char* nodesNodeName(ENodeType type) {
return "PhysiSubplan";
case QUERY_NODE_PHYSICAL_PLAN:
return "PhysiPlan";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SESSION:
return "PhysiStreamContinueSession";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_SESSION:
return "PhysiStreamContinueFinalSession";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION:
return "PhysiStreamContinueSemiSession";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_STATE:
return "PhysiStreamContinueState";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_EVENT:
return "PhysiStreamContinueEvent";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_COUNT:
return "PhysiStreamContinueCount";
default:
break;
}
@ -1032,6 +1050,7 @@ static const char* jkWindowLogicPlanStateExpr = "StateExpr";
static const char* jkWindowLogicPlanTriggerType = "TriggerType";
static const char* jkWindowLogicPlanWatermark = "Watermark";
static const char* jkWindowLogicPlanDeleteMark = "DeleteMark";
static const char* jkWindowLogicPlanRecalculateInterval = "RecalculateInterval";
static int32_t logicWindowNodeToJson(const void* pObj, SJson* pJson) {
const SWindowLogicNode* pNode = (const SWindowLogicNode*)pObj;
@ -1082,6 +1101,9 @@ static int32_t logicWindowNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkWindowLogicPlanDeleteMark, pNode->deleteMark);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkWindowLogicPlanRecalculateInterval, pNode->recalculateInterval);
}
return code;
}
@ -1135,6 +1157,9 @@ static int32_t jsonToLogicWindowNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkWindowLogicPlanDeleteMark, &pNode->deleteMark);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkWindowLogicPlanRecalculateInterval, &pNode->recalculateInterval);
}
return code;
}
@ -2102,6 +2127,11 @@ static const char* jkTableScanPhysiPlanFilesetDelimited = "FilesetDelimited";
static const char* jkTableScanPhysiPlanNeedCountEmptyTable = "NeedCountEmptyTable";
static const char* jkTableScanPhysiPlanParaTablesSort = "ParaTablesSort";
static const char* jkTableScanPhysiPlanSmallDataTsSort = "SmallDataTsSort";
static const char* jkTableScanPhysiPlanStreamResInfoStbFullName = "StreamResInfoStbFullName";
static const char* jkTableScanPhysiPlanStreamResInfoWstartName = "StreamResInfoWstartName";
static const char* jkTableScanPhysiPlanStreamResInfoWendName = "StreamResInfoWendName";
static const char* jkTableScanPhysiPlanStreamResInfoGroupIdName = "StreamResInfoGroupIdName";
static const char* jkTableScanPhysiPlanStreamResInfoIsWindowFilledName = "StreamResInfoIsWindowFilledName";
static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj;
@ -2182,6 +2212,21 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanSmallDataTsSort, pNode->smallDataTsSort);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkTableScanPhysiPlanStreamResInfoStbFullName, pNode->pStbFullName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkTableScanPhysiPlanStreamResInfoWstartName, pNode->pWstartName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkTableScanPhysiPlanStreamResInfoWendName, pNode->pWendName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkTableScanPhysiPlanStreamResInfoGroupIdName, pNode->pGroupIdName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkTableScanPhysiPlanStreamResInfoIsWindowFilledName, pNode->pIsWindowFilledName);
}
return code;
}
@ -2264,6 +2309,21 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanSmallDataTsSort, &pNode->smallDataTsSort);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkTableScanPhysiPlanStreamResInfoStbFullName, pNode->pStbFullName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkTableScanPhysiPlanStreamResInfoWstartName, pNode->pWstartName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkTableScanPhysiPlanStreamResInfoWendName, pNode->pWendName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkTableScanPhysiPlanStreamResInfoGroupIdName, pNode->pGroupIdName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkTableScanPhysiPlanStreamResInfoIsWindowFilledName, pNode->pIsWindowFilledName);
}
return code;
}
@ -2888,6 +2948,7 @@ static const char* jkWindowPhysiPlanIgnoreExpired = "IgnoreExpired";
static const char* jkWindowPhysiPlanInputTsOrder = "InputTsOrder";
static const char* jkWindowPhysiPlanMergeDataBlock = "MergeDataBlock";
static const char* jkWindowPhysiPlanDestHasPrimaryKey = "DestHasPrimaryKey";
static const char* jkWindowPhysiPlanRecalculateInterval = "RecalculateInterval";
static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) {
const SWindowPhysiNode* pNode = (const SWindowPhysiNode*)pObj;
@ -2923,6 +2984,9 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkWindowPhysiPlanDestHasPrimaryKey, pNode->destHasPrimaryKey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkWindowPhysiPlanRecalculateInterval, pNode->recalculateInterval);
}
return code;
}
@ -2961,6 +3025,9 @@ static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkWindowPhysiPlanDestHasPrimaryKey, &pNode->destHasPrimaryKey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkWindowPhysiPlanRecalculateInterval, &pNode->recalculateInterval);
}
return code;
}
@ -5804,6 +5871,7 @@ static const char* jkStreamOptionsWatermark = "Watermark";
static const char* jkStreamOptionsDeleteMark = "DeleteMark";
static const char* jkStreamOptionsFillHistory = "FillHistory";
static const char* jkStreamOptionsIgnoreExpired = "IgnoreExpired";
static const char* jkStreamOptionsRecInterval = "RecInterval";
static int32_t streamOptionsToJson(const void* pObj, SJson* pJson) {
const SStreamOptions* pNode = (const SStreamOptions*)pObj;
@ -5824,6 +5892,9 @@ static int32_t streamOptionsToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkStreamOptionsIgnoreExpired, pNode->ignoreExpired);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkStreamOptionsRecInterval, nodeToJson, pNode->pRecInterval);
}
return code;
}
@ -5847,6 +5918,9 @@ static int32_t jsonToStreamOptions(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkStreamOptionsIgnoreExpired, &pNode->ignoreExpired);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkStreamOptionsRecInterval, &pNode->pRecInterval);
}
return code;
}
@ -8367,6 +8441,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL:
return physiIntervalNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
@ -8375,15 +8452,21 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_SESSION:
return physiSessionWindowNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_STATE:
return physiStateWindowNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_EVENT:
return physiEventWindowNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_COUNT:
return physiCountWindowNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY:
return physiAnomalyWindowNodeToJson(pObj, pJson);
@ -8746,6 +8829,9 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL:
return jsonToPhysiIntervalNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
@ -8754,15 +8840,21 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_SESSION:
return jsonToPhysiSessionWindowNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_STATE:
return jsonToPhysiStateWindowNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_EVENT:
return jsonToPhysiEventWindowNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_COUNT:
return jsonToPhysiCountWindowNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY:
return jsonToPhysiAnomalyWindowNode(pJson, pObj);

18
source/libs/nodes/src/nodesMsgFuncs.c
View File

@ -4692,6 +4692,9 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL:
code = physiIntervalNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_FILL:
@ -4702,18 +4705,24 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_SESSION:
code = physiSessionWindowNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_STATE:
code = physiStateWindowNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_EVENT:
code = physiEventWindowNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_COUNT:
code = physiCountWindowNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY:
@ -4861,6 +4870,9 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL:
code = msgToPhysiIntervalNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_FILL:
@ -4871,18 +4883,24 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_SESSION:
code = msgToPhysiSessionWindowNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_STATE:
code = msgToPhysiStateWindowNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_EVENT:
code = msgToPhysiEventWindowNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_COUNT:
code = msgToPhysiCountWindowNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ANOMALY:

25
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -891,12 +891,15 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) {
code = makeNode(type, sizeof(SMergeAlignedIntervalPhysiNode), &pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL:
code = makeNode(type, sizeof(SStreamIntervalPhysiNode), &pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL:
code = makeNode(type, sizeof(SStreamFinalIntervalPhysiNode), &pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL:
code = makeNode(type, sizeof(SStreamSemiIntervalPhysiNode), &pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
@ -910,24 +913,29 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) {
code = makeNode(type, sizeof(SSessionWinodwPhysiNode), &pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SESSION:
code = makeNode(type, sizeof(SStreamSessionWinodwPhysiNode), &pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION:
code = makeNode(type, sizeof(SStreamSemiSessionWinodwPhysiNode), &pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_SESSION:
code = makeNode(type, sizeof(SStreamFinalSessionWinodwPhysiNode), &pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE:
code = makeNode(type, sizeof(SStateWinodwPhysiNode), &pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_STATE:
code = makeNode(type, sizeof(SStreamStateWinodwPhysiNode), &pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT:
code = makeNode(type, sizeof(SEventWinodwPhysiNode), &pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_EVENT:
code = makeNode(type, sizeof(SStreamEventWinodwPhysiNode), &pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
@ -937,6 +945,7 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) {
code = makeNode(type, sizeof(SAnomalyWindowPhysiNode), &pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_COUNT:
code = makeNode(type, sizeof(SStreamCountWinodwPhysiNode), &pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
@ -1225,6 +1234,7 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pOptions->pDelay);
nodesDestroyNode(pOptions->pWatermark);
nodesDestroyNode(pOptions->pDeleteMark);
nodesDestroyNode(pOptions->pRecInterval);
break;
}
case QUERY_NODE_TSMA_OPTIONS: {
@ -1936,6 +1946,9 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL:
destroyWinodwPhysiNode((SWindowPhysiNode*)pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_FILL:
@ -1953,17 +1966,22 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION:
destroyWinodwPhysiNode((SWindowPhysiNode*)pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_STATE: {
SStateWinodwPhysiNode* pPhyNode = (SStateWinodwPhysiNode*)pNode;
destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode);
nodesDestroyNode(pPhyNode->pStateKey);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_EVENT: {
SEventWinodwPhysiNode* pPhyNode = (SEventWinodwPhysiNode*)pNode;
destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode);
nodesDestroyNode(pPhyNode->pStartCond);
@ -1971,7 +1989,8 @@ void nodesDestroyNode(SNode* pNode) {
break;
}
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT: {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_COUNT: {
SCountWinodwPhysiNode* pPhyNode = (SCountWinodwPhysiNode*)pNode;
destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode);
break;

5
source/libs/parser/inc/sql.y
View File

@ -833,6 +833,8 @@ stream_options(A) ::= .
stream_options(A) ::= stream_options(B) TRIGGER AT_ONCE(C). { A = setStreamOptions(pCxt, B, SOPT_TRIGGER_TYPE_SET, &C, NULL); }
stream_options(A) ::= stream_options(B) TRIGGER WINDOW_CLOSE(C). { A = setStreamOptions(pCxt, B, SOPT_TRIGGER_TYPE_SET, &C, NULL); }
stream_options(A) ::= stream_options(B) TRIGGER FORCE_WINDOW_CLOSE(C). { A = setStreamOptions(pCxt, B, SOPT_TRIGGER_TYPE_SET, &C, NULL); }
stream_options(A) ::= stream_options(B) TRIGGER CONTINUOUS_WINDOW_CLOSE(C)
recalculate_opt(D). { A = setStreamOptions(pCxt, B, SOPT_TRIGGER_TYPE_SET, &C, D); }
stream_options(A) ::= stream_options(B) TRIGGER MAX_DELAY(C) duration_literal(D). { A = setStreamOptions(pCxt, B, SOPT_TRIGGER_TYPE_SET, &C, releaseRawExprNode(pCxt, D)); }
stream_options(A) ::= stream_options(B) WATERMARK duration_literal(C). { A = setStreamOptions(pCxt, B, SOPT_WATERMARK_SET, NULL, releaseRawExprNode(pCxt, C)); }
stream_options(A) ::= stream_options(B) IGNORE EXPIRED NK_INTEGER(C). { A = setStreamOptions(pCxt, B, SOPT_IGNORE_EXPIRED_SET, &C, NULL); }
@ -840,6 +842,9 @@ stream_options(A) ::= stream_options(B) FILL_HISTORY NK_INTEGER(C).
stream_options(A) ::= stream_options(B) DELETE_MARK duration_literal(C). { A = setStreamOptions(pCxt, B, SOPT_DELETE_MARK_SET, NULL, releaseRawExprNode(pCxt, C)); }
stream_options(A) ::= stream_options(B) IGNORE UPDATE NK_INTEGER(C). { A = setStreamOptions(pCxt, B, SOPT_IGNORE_UPDATE_SET, &C, NULL); }
recalculate_opt(A) ::= . { A = NULL; }
recalculate_opt(A) ::= RECALCULATE duration_literal(B). { A = releaseRawExprNode(pCxt, B); }
subtable_opt(A) ::= . { A = NULL; }
subtable_opt(A) ::= SUBTABLE NK_LP expression(B) NK_RP. { A = releaseRawExprNode(pCxt, B); }

5
source/libs/parser/src/parAstCreater.c
View File

@ -3762,6 +3762,8 @@ static int8_t getTriggerType(uint32_t tokenType) {
return STREAM_TRIGGER_MAX_DELAY;
case TK_FORCE_WINDOW_CLOSE:
return STREAM_TRIGGER_FORCE_WINDOW_CLOSE;
case TK_CONTINUOUS_WINDOW_CLOSE:
return STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE;
default:
break;
}
@ -3783,6 +3785,9 @@ SNode* setStreamOptions(SAstCreateContext* pCxt, SNode* pOptions, EStreamOptions
if (STREAM_TRIGGER_MAX_DELAY == pStreamOptions->triggerType) {
pStreamOptions->pDelay = pNode;
}
if (STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE == pStreamOptions->triggerType) {
pStreamOptions->pRecInterval = pNode;
}
break;
case SOPT_WATERMARK_SET:
pStreamOptions->pWatermark = pNode;

4
source/libs/parser/src/parTokenizer.c
View File

@ -363,7 +363,9 @@ static SKeyword keywordTable[] = {
{"REGEXP", TK_REGEXP},
{"ASSIGN", TK_ASSIGN},
{"TRUE_FOR", TK_TRUE_FOR},
{"META_ONLY", TK_META_ONLY}
{"META_ONLY", TK_META_ONLY},
{"CONTINUOUS_WINDOW_CLOSE", TK_CONTINUOUS_WINDOW_CLOSE},
{"RECALCULATE", TK_RECALCULATE},
};
// clang-format on

284
source/libs/parser/src/parTranslater.c
View File

@ -11876,25 +11876,21 @@ static int32_t addIrowTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSelect
return code;
}
static int32_t addWstartTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSelectStmt* pSelect,
SHashObj* pUserAliasSet, SNodeList* pCols, SCMCreateStreamReq* pReq) {
SNode* pProj = nodesListGetNode(pSelect->pProjectionList, 0);
if (NULL == pSelect->pWindow ||
(QUERY_NODE_FUNCTION == nodeType(pProj) && 0 == strcmp("_wstart", ((SFunctionNode*)pProj)->functionName))) {
return TSDB_CODE_SUCCESS;
}
typedef int32_t (*insertFn)(SNodeList* pList, SNode* pNode);
static int32_t addFunctionToCreateStreamQueryImpl(STranslateContext* pCxt, SSelectStmt* pSelect,
SHashObj* pUserAliasSet, SNodeList* pCols, SCMCreateStreamReq* pReq,
char pFunName[], insertFn pFn, char* defaultNameList[]) {
SFunctionNode* pFunc = NULL;
int32_t code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc);
if (NULL == pFunc) {
return code;
}
tstrncpy(pFunc->functionName, "_wstart", TSDB_FUNC_NAME_LEN);
tstrncpy(pFunc->node.userAlias, "_irowts", TSDB_COL_NAME_LEN);
char* defaultName[] = {"_wstart", "ts", NULL};
getStreamQueryFirstProjectAliasName(pUserAliasSet, pFunc->node.aliasName, sizeof(pFunc->node.aliasName), defaultName);
tstrncpy(pFunc->functionName, pFunName, TSDB_FUNC_NAME_LEN);
tstrncpy(pFunc->node.userAlias, pFunName, TSDB_COL_NAME_LEN);
getStreamQueryFirstProjectAliasName(pUserAliasSet, pFunc->node.aliasName, sizeof(pFunc->node.aliasName), defaultNameList);
code = getFuncInfo(pCxt, pFunc);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListPushFront(pSelect->pProjectionList, (SNode*)pFunc);
code = pFn(pSelect->pProjectionList, (SNode*)pFunc);
}
if (TSDB_CODE_SUCCESS == code && STREAM_CREATE_STABLE_TRUE == pReq->createStb) {
@ -11906,7 +11902,7 @@ static int32_t addWstartTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSele
pColDef->sma = true;
code = setColumnDefNodePrimaryKey(pColDef, false);
}
if (TSDB_CODE_SUCCESS == code) code = nodesListPushFront(pCols, (SNode*)pColDef);
if (TSDB_CODE_SUCCESS == code) code = pFn(pCols, (SNode*)pColDef);
if (TSDB_CODE_SUCCESS != code) nodesDestroyNode((SNode*)pColDef);
}
if (TSDB_CODE_SUCCESS != code) {
@ -11915,9 +11911,112 @@ static int32_t addWstartTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSele
return code;
}
static int32_t addTsKeyToCreateStreamQuery(STranslateContext* pCxt, SNode* pStmt, SNodeList* pCols,
static int32_t addWstartTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSelectStmt* pSelect,
SHashObj* pUserAliasSet, SNodeList* pCols, SCMCreateStreamReq* pReq) {
SNode* pProj = nodesListGetNode(pSelect->pProjectionList, 0);
char defaultName[] = {"_wstart"};
if (NULL == pSelect->pWindow ||
(QUERY_NODE_FUNCTION == nodeType(pProj) && 0 == strcmp(defaultName, ((SFunctionNode*)pProj)->functionName))) {
tstrncpy(pReq->pWstartName, ((SFunctionNode*)pProj)->node.userAlias, TSDB_FUNC_NAME_LEN);
return TSDB_CODE_SUCCESS;
}
char* defaultNameList[] = {"_wstart", "ts", "_wstart_1",NULL};
int32_t code =
addFunctionToCreateStreamQueryImpl(pCxt, pSelect, pUserAliasSet, pCols, pReq, defaultName, nodesListPushFront, defaultNameList);
if (TSDB_CODE_SUCCESS == code) {
SNode* pFunc = nodesListGetNode(pSelect->pProjectionList, 0);
tstrncpy(pReq->pWstartName, ((SFunctionNode*)pFunc)->node.aliasName, TSDB_FUNC_NAME_LEN);
}
return code;
}
static int32_t addWendTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSelectStmt* pSelect, SHashObj* pUserAliasSet,
SNodeList* pCols, SCMCreateStreamReq* pReq) {
int32_t code = TSDB_CODE_SUCCESS;
char defaultName[] = {"_wend"};
if (NULL == pSelect->pWindow) {
return code;
}
SNode* pProj = NULL;
FOREACH(pProj, pSelect->pProjectionList) {
if (QUERY_NODE_FUNCTION == nodeType(pProj) && 0 == strcmp(defaultName, ((SFunctionNode*)pProj)->functionName)) {
tstrncpy(pReq->pWendName, ((SFunctionNode*)pProj)->node.aliasName, TSDB_FUNC_NAME_LEN);
return code;
}
}
char* defaultNameList[] = {"_wend", "_wend_1",NULL};
code = addFunctionToCreateStreamQueryImpl(pCxt, pSelect, pUserAliasSet, pCols, pReq, defaultName, nodesListAppend, defaultNameList);
if (TSDB_CODE_SUCCESS == code) {
SNode* pFunc = pSelect->pProjectionList->pTail->pNode;
tstrncpy(pReq->pWendName, ((SFunctionNode*)pFunc)->node.aliasName, TSDB_FUNC_NAME_LEN);
}
return code;
}
static int32_t addIsFilledToCreateStreamQueryImpl(STranslateContext* pCxt, SSelectStmt* pSelect, SHashObj* pUserAliasSet,
SNodeList* pCols, SCMCreateStreamReq* pReq) {
int32_t code = TSDB_CODE_SUCCESS;
char defaultName[] = {"_iswindowfilled"};
if (NULL == pSelect->pWindow) {
return code;
}
SNode* pProj = NULL;
FOREACH(pProj, pSelect->pProjectionList) {
if (QUERY_NODE_FUNCTION == nodeType(pProj) && 0 == strcmp(defaultName, ((SFunctionNode*)pProj)->functionName)) {
tstrncpy(pReq->pIsWindowFilledName, ((SFunctionNode*)pProj)->node.aliasName, TSDB_FUNC_NAME_LEN);
return code;
}
}
char* defaultNameList[] = {"_window_isfilled", "_window_isfilled_1",NULL};
code = addFunctionToCreateStreamQueryImpl(pCxt, pSelect, pUserAliasSet, pCols, pReq, defaultName, nodesListAppend, defaultNameList);
if (TSDB_CODE_SUCCESS == code) {
SNode* pFunc = pSelect->pProjectionList->pTail->pNode;
tstrncpy(pReq->pIsWindowFilledName, ((SFunctionNode*)pFunc)->node.aliasName, TSDB_FUNC_NAME_LEN);
}
return code;
}
static bool needAddWend(SCreateStreamStmt* pStmt) {
if (pStmt->pOptions->triggerType != STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
return false;
}
SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery;
if (pSelect->pWindow == NULL || pSelect->pWindow->type == QUERY_NODE_INTERVAL_WINDOW) {
return false;
}
return true;
}
static bool needAddGroupId(SCreateStreamStmt* pStmt) {
if (pStmt->pOptions->triggerType != STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
return false;
}
SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery;
if (pSelect->pWindow == NULL || (pSelect->pWindow->type == QUERY_NODE_INTERVAL_WINDOW && NULL == ((SIntervalWindowNode*)pSelect->pWindow)->pFill)) {
return false;
}
return true;
}
static bool needAddIsFilled(SCreateStreamStmt* pStmt) {
if (pStmt->pOptions->triggerType != STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
return false;
}
SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery;
if (pSelect->pWindow == NULL || pSelect->pWindow->type != QUERY_NODE_INTERVAL_WINDOW || NULL == ((SIntervalWindowNode*)pSelect->pWindow)->pFill) {
return false;
}
return true;
}
static int32_t addTsKeyToCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SNodeList* pCols,
SCMCreateStreamReq* pReq) {
SSelectStmt* pSelect = (SSelectStmt*)pStmt;
SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery;
SHashObj* pUserAliasSet = NULL;
int32_t code = checkProjectAlias(pCxt, pSelect->pProjectionList, &pUserAliasSet);
if (TSDB_CODE_SUCCESS == code) {
@ -11926,6 +12025,13 @@ static int32_t addTsKeyToCreateStreamQuery(STranslateContext* pCxt, SNode* pStmt
if (TSDB_CODE_SUCCESS == code) {
code = addIrowTsToCreateStreamQueryImpl(pCxt, pSelect, pUserAliasSet, pCols, pReq);
}
if (TSDB_CODE_SUCCESS == code && needAddWend(pStmt)) {
code = addWendTsToCreateStreamQueryImpl(pCxt, pSelect, pUserAliasSet, pCols, pReq);
}
if (TSDB_CODE_SUCCESS == code && needAddIsFilled(pStmt)) {
code = addIsFilledToCreateStreamQueryImpl(pCxt, pSelect, pUserAliasSet, pCols, pReq);
}
taosHashCleanup(pUserAliasSet);
return code;
}
@ -11937,6 +12043,14 @@ static const char* getTagNameForCreateStreamTag(SNode* pTag) {
return ((SColumnNode*)pTag)->colName;
}
static bool isTagDef(SNodeList* pTags) {
if (NULL == pTags) {
return false;
}
SColumnDefNode* pColDef = (SColumnDefNode*)nodesListGetNode(pTags, 0);
return TSDB_DATA_TYPE_NULL != pColDef->dataType.type;
}
static int32_t addTagsToCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SSelectStmt* pSelect) {
if (NULL == pStmt->pTags) {
return TSDB_CODE_SUCCESS;
@ -12340,11 +12454,25 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported Having");
}
if (tsStreamCoverage == false && NULL != pStmt->pOptions->pRecInterval) {
SValueNode* pVal = (SValueNode*)pStmt->pOptions->pRecInterval;
int64_t minDelay = 0;
char* str = "600s";
if (DEAL_RES_ERROR != translateValue(pCxt, pVal) &&
TSDB_CODE_SUCCESS ==
parseNatualDuration(str, strlen(str), &minDelay, &pVal->unit, pVal->node.resType.precision, false)) {
if (pVal->datum.i < minDelay) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"stream recalculate interval must be bigger than 10 minutes");
}
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t adjustDataTypeOfProjections(STranslateContext* pCxt, const STableMeta* pMeta, SNodeList* pProjections,
SNodeList** ppCols) {
SNodeList** ppCols, SCMCreateStreamReq* pReq, bool needQueryRes) {
if (getNumOfColumns(pMeta) != LIST_LENGTH(pProjections)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of columns");
}
@ -12355,6 +12483,13 @@ static int32_t adjustDataTypeOfProjections(STranslateContext* pCxt, const STable
SNode* pProj = NULL;
FOREACH(pProj, pProjections) {
SSchema* pSchema = pSchemas + index++;
if (needQueryRes) {
if (QUERY_NODE_FUNCTION == nodeType(pProj) && 0 == strcmp(pReq->pWendName, ((SFunctionNode*)pProj)->functionName)) {
if (pSchema->type != TSDB_DATA_TYPE_TIMESTAMP || pSchema->type != TSDB_DATA_TYPE_BIGINT) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, "A column with the timestamp data type is required to store _wend");
}
}
}
SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes};
if (IS_DECIMAL_TYPE(pSchema->type)) {
STypeMod typeMod = (pExtSchemas + (index - 1))->typeMod;
@ -12455,7 +12590,7 @@ static int32_t setFillNullCols(SArray* pProjColPos, const STableMeta* pMeta, SCM
}
static int32_t adjustOrderOfProjections(STranslateContext* pCxt, SNodeList** ppCols, const STableMeta* pMeta,
SNodeList** pProjections, SCMCreateStreamReq* pReq) {
SNodeList** pProjections, SCMCreateStreamReq* pReq, bool needQueryRes) {
if (LIST_LENGTH((*ppCols)) != LIST_LENGTH(*pProjections)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of columns");
}
@ -12488,6 +12623,14 @@ static int32_t adjustOrderOfProjections(STranslateContext* pCxt, SNodeList** ppC
if (pSchema->flags & COL_IS_KEY) {
hasPrimaryKey = true;
}
if (needQueryRes) {
if (QUERY_NODE_FUNCTION == nodeType(pProj) && 0 == strcmp(pReq->pWendName, ((SFunctionNode*)pProj)->functionName)) {
if (pSchema->type != TSDB_DATA_TYPE_TIMESTAMP || pSchema->type != TSDB_DATA_TYPE_BIGINT) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, "A column with the timestamp data type is required to store _wend");
}
}
}
}
if (TSDB_CODE_SUCCESS == code && !hasTsKey) {
@ -12540,18 +12683,22 @@ static int32_t adjustOrderOfProjections(STranslateContext* pCxt, SNodeList** ppC
static int32_t adjustProjectionsForExistTable(STranslateContext* pCxt, SCreateStreamStmt* pStmt,
const STableMeta* pMeta, SCMCreateStreamReq* pReq) {
SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery;
bool needQueryRes = needAddWend(pStmt);
if (NULL == pStmt->pCols) {
return adjustDataTypeOfProjections(pCxt, pMeta, pSelect->pProjectionList, &pStmt->pCols);
return adjustDataTypeOfProjections(pCxt, pMeta, pSelect->pProjectionList, &pStmt->pCols, pReq, needQueryRes);
}
return adjustOrderOfProjections(pCxt, &pStmt->pCols, pMeta, &pSelect->pProjectionList, pReq);
return adjustOrderOfProjections(pCxt, &pStmt->pCols, pMeta, &pSelect->pProjectionList, pReq, needQueryRes);
}
static bool isGroupIdTagStream(const STableMeta* pMeta, SNodeList* pTags) {
return (NULL == pTags && 1 == pMeta->tableInfo.numOfTags && TSDB_DATA_TYPE_UBIGINT == getTableTagSchema(pMeta)->type);
}
static int32_t adjustDataTypeOfTags(STranslateContext* pCxt, const STableMeta* pMeta, SNodeList* pTags) {
static int32_t adjustDataTypeOfTags(STranslateContext* pCxt, const STableMeta* pMeta, SNodeList* pTags,
SCMCreateStreamReq* pReq) {
if (isGroupIdTagStream(pMeta, pTags)) {
SSchema* pSchema = getTableTagSchema(pMeta);
tstrncpy(pReq->pGroupIdName, pSchema->name, tListLen(pSchema->name));
return TSDB_CODE_SUCCESS;
}
@ -12578,8 +12725,27 @@ static int32_t adjustDataTypeOfTags(STranslateContext* pCxt, const STableMeta* p
return TSDB_CODE_SUCCESS;
}
static int32_t adjustOrderOfTags(STranslateContext* pCxt, SNodeList* pTags, const STableMeta* pMeta,
static int32_t addGroupIdTagForExistDestTable(STranslateContext* pCxt, const SSchema* pSchema, SNodeList* pTagExprs, SCMCreateStreamReq* pReq) {
int32_t code = TSDB_CODE_SUCCESS;
SFunctionNode* pFunc = NULL;
code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc);
if (NULL == pFunc) {
return code;
}
tstrncpy(pFunc->functionName, "_group_id", tListLen(pFunc->functionName));
tstrncpy(pFunc->node.userAlias, pSchema->name, tListLen(pSchema->name));
code = getFuncInfo(pCxt, pFunc);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
tstrncpy(pReq->pGroupIdName, pFunc->node.userAlias, tListLen(pFunc->node.userAlias));
code = nodesListStrictAppend(pTagExprs, (SNode*)pFunc);
return code;
}
static int32_t adjustOrderOfTags(STranslateContext* pCxt, SCreateStreamStmt* pStmt, const STableMeta* pMeta,
SNodeList** pTagExprs, SCMCreateStreamReq* pReq) {
SNodeList* pTags = pStmt->pTags;
if (LIST_LENGTH(pTags) != LIST_LENGTH(*pTagExprs)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of tags");
}
@ -12606,6 +12772,7 @@ static int32_t adjustOrderOfTags(STranslateContext* pCxt, SNodeList* pTags, cons
}
SNodeList* pNewTagExprs = NULL;
bool needAddTag = true;
if (TSDB_CODE_SUCCESS == code) {
taosArraySort(pTagPos, projColPosCompar);
int32_t indexOfBoundTags = 0;
@ -12624,12 +12791,24 @@ static int32_t adjustOrderOfTags(STranslateContext* pCxt, SNodeList* pTags, cons
continue;
}
}
if (needAddGroupId(pStmt) && needAddTag && pTagSchema->type == TSDB_DATA_TYPE_UBIGINT) {
needAddTag = false;
code = addGroupIdTagForExistDestTable(pCxt, pTagSchema, pNewTagExprs, pReq);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
continue;
}
SNode* pNull = NULL;
code = createNullValue(&pNull);
if (TSDB_CODE_SUCCESS == code) code = nodesListStrictAppend(pNewTagExprs, pNull);
}
}
if (needAddGroupId(pStmt) && needAddTag) {
code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_TAGS_NOT_MATCHED, "a bigint unsigned type tag is required to store the group id");
}
if (TSDB_CODE_SUCCESS == code) {
taosArrayDestroy(pTagPos);
nodesDestroyList(*pTagExprs);
@ -12649,9 +12828,9 @@ static int32_t adjustTagsForExistTable(STranslateContext* pCxt, SCreateStreamStm
return TSDB_CODE_SUCCESS;
}
if (NULL == pStmt->pTags) {
return adjustDataTypeOfTags(pCxt, pMeta, pSelect->pTags);
return adjustDataTypeOfTags(pCxt, pMeta, pSelect->pTags, pReq);
}
return adjustOrderOfTags(pCxt, pStmt->pTags, pMeta, &pSelect->pTags, pReq);
return adjustOrderOfTags(pCxt, pStmt, pMeta, &pSelect->pTags, pReq);
}
static int32_t adjustTagsForCreateTable(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) {
@ -12679,22 +12858,53 @@ static int32_t adjustTagsForCreateTable(STranslateContext* pCxt, SCreateStreamSt
return TSDB_CODE_SUCCESS;
}
static int32_t addGroupIdTagForCreateDestTable(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) {
int32_t code = TSDB_CODE_SUCCESS;
SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery;
if (!needAddGroupId(pStmt)) {
return code;
}
char alias [] = {"group_id"};
if (NULL == pSelect->pPartitionByList || NULL == pSelect->pTags || NULL == pStmt->pTags) {
tstrncpy(pReq->pGroupIdName, alias, tListLen(pReq->pGroupIdName));
return code;
}
SFunctionNode* pFunc = NULL;
code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc);
if (NULL == pFunc) {
return code;
}
tstrncpy(pFunc->functionName, "_group_id", tListLen(pFunc->functionName));
tstrncpy(pFunc->node.userAlias, alias, tListLen(pFunc->node.userAlias));
code = getFuncInfo(pCxt, pFunc);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = nodesListStrictAppend(pSelect->pTags, (SNode*)pFunc);
if (code == TSDB_CODE_SUCCESS) {
if (isTagDef(pStmt->pTags)) {
code = addColDefNodeByProj(&pStmt->pTags, (const SNode*)pFunc, 0);
tstrncpy(pReq->pGroupIdName, pFunc->node.userAlias, tListLen(pFunc->node.userAlias));
}
}
return code;
}
static int32_t adjustTags(STranslateContext* pCxt, SCreateStreamStmt* pStmt, const STableMeta* pMeta,
SCMCreateStreamReq* pReq) {
if (NULL == pMeta) {
int32_t code = addGroupIdTagForCreateDestTable(pCxt, pStmt, pReq);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
return adjustTagsForCreateTable(pCxt, pStmt, pReq);
}
return adjustTagsForExistTable(pCxt, pStmt, pMeta, pReq);
}
static bool isTagDef(SNodeList* pTags) {
if (NULL == pTags) {
return false;
}
SColumnDefNode* pColDef = (SColumnDefNode*)nodesListGetNode(pTags, 0);
return TSDB_DATA_TYPE_NULL != pColDef->dataType.type;
}
static bool isTagBound(SNodeList* pTags) {
if (NULL == pTags) {
return false;
@ -12922,7 +13132,7 @@ static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt
code = addColsToCreateStreamQuery(pCxt, pStmt, pReq);
}
if (TSDB_CODE_SUCCESS == code) {
code = addTsKeyToCreateStreamQuery(pCxt, pStmt->pQuery, pStmt->pCols, pReq);
code = addTsKeyToCreateStreamQuery(pCxt, pStmt, pStmt->pCols, pReq);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkStreamQuery(pCxt, pStmt);
@ -12962,9 +13172,11 @@ static int32_t translateStreamOptions(STranslateContext* pCxt, SCreateStreamStmt
SStreamOptions* pOptions = pStmt->pOptions;
if ((NULL != pOptions->pWatermark && (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pOptions->pWatermark))) ||
(NULL != pOptions->pDeleteMark && (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pOptions->pDeleteMark))) ||
(NULL != pOptions->pDelay && (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pOptions->pDelay)))) {
(NULL != pOptions->pDelay && (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pOptions->pDelay))) ||
(NULL != pOptions->pRecInterval && (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pOptions->pRecInterval))) ) {
return pCxt->errCode;
}
return TSDB_CODE_SUCCESS;
}
@ -13051,6 +13263,12 @@ static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt*
if (TSDB_CODE_SUCCESS == code) {
code = columnDefNodeToField(pStmt->pCols, &pReq->pCols, false);
}
pReq->recalculateInterval = 0;
if (NULL != pStmt->pOptions->pRecInterval) {
SValueNode* pValueNode = ((SValueNode*)pStmt->pOptions->pRecInterval);
pReq->recalculateInterval =
convertTimePrecision(pValueNode->datum.i, pValueNode->node.resType.precision, TSDB_TIME_PRECISION_MILLI);
}
}
if (TSDB_CODE_SUCCESS == code) {

28
source/libs/planner/src/planLogicCreater.c
View File

@ -76,6 +76,9 @@ static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol, bool isPartit
case FUNCTION_TYPE_GROUP_KEY:
pCol->colType = COLUMN_TYPE_GROUP_KEY;
break;
case FUNCTION_TYPE_IS_WINDOW_FILLED:
pCol->colType = COLUMN_TYPE_IS_WINDOW_FILLED;
break;
default:
break;
}
@ -1111,6 +1114,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm
pWindow->deleteMark = pCxt->pPlanCxt->deleteMark;
pWindow->igExpired = pCxt->pPlanCxt->igExpired;
pWindow->igCheckUpdate = pCxt->pPlanCxt->igCheckUpdate;
pWindow->recalculateInterval = pCxt->pPlanCxt->recalculateInterval;
}
pWindow->node.inputTsOrder = ORDER_ASC;
pWindow->node.outputTsOrder = ORDER_ASC;
@ -1188,7 +1192,17 @@ static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionW
pWindow->winType = WINDOW_TYPE_SESSION;
pWindow->sessionGap = ((SValueNode*)pSession->pGap)->datum.i;
pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? SESSION_ALGO_STREAM_SINGLE : SESSION_ALGO_MERGE;
if (pCxt->pPlanCxt->streamQuery) {
if (pCxt->pPlanCxt->triggerType == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
pWindow->windowAlgo = SESSION_ALGO_STREAM_CONTINUE_SINGLE;
} else {
pWindow->windowAlgo = SESSION_ALGO_STREAM_SINGLE;
}
} else {
pWindow->windowAlgo = SESSION_ALGO_MERGE;
}
pWindow->node.groupAction = getGroupAction(pCxt, pSelect);
pWindow->node.requireDataOrder =
pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : getRequireDataOrder(true, pSelect);
@ -1226,7 +1240,15 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva
pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval);
pWindow->slidingUnit =
(NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit);
pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH;
if (pCxt->pPlanCxt->streamQuery) {
if (pCxt->pPlanCxt->triggerType == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE || pCxt->pPlanCxt->triggerType == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) {
pWindow->windowAlgo = INTERVAL_ALGO_STREAM_CONTINUE_SINGLE;
} else {
pWindow->windowAlgo = INTERVAL_ALGO_STREAM_SINGLE;
}
} else {
pWindow->windowAlgo = INTERVAL_ALGO_HASH;
}
pWindow->node.groupAction = (NULL != pInterval->pFill ? GROUP_ACTION_KEEP : getGroupAction(pCxt, pSelect));
pWindow->node.requireDataOrder =
pCxt->pPlanCxt->streamQuery
@ -1409,7 +1431,7 @@ static EDealRes needFillValueImpl(SNode* pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
SColumnNode* pCol = (SColumnNode*)pNode;
if (COLUMN_TYPE_WINDOW_START == pCol->colType || COLUMN_TYPE_WINDOW_END == pCol->colType ||
COLUMN_TYPE_WINDOW_DURATION == pCol->colType) {
COLUMN_TYPE_WINDOW_DURATION == pCol->colType || COLUMN_TYPE_IS_WINDOW_FILLED == pCol->colType) {
pCtx->hasPseudoWinCol = true;
pCtx->code =
taosHashPut(pCtx->pCollectFillCtx->pPseudoCols, pCol->colName, TSDB_COL_NAME_LEN, &pNode, POINTER_BYTES);

62
source/libs/planner/src/planPhysiCreater.c
View File

@ -771,6 +771,11 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp
pTableScan->needCountEmptyTable = pScanLogicNode->isCountByTag;
pTableScan->paraTablesSort = pScanLogicNode->paraTablesSort;
pTableScan->smallDataTsSort = pScanLogicNode->smallDataTsSort;
tstrncpy(pTableScan->pStbFullName, pCxt->pPlanCxt->pStbFullName, TSDB_TABLE_FNAME_LEN);
tstrncpy(pTableScan->pWstartName, pCxt->pPlanCxt->pWstartName, TSDB_COL_NAME_LEN);
tstrncpy(pTableScan->pWendName, pCxt->pPlanCxt->pWendName, TSDB_COL_NAME_LEN);
tstrncpy(pTableScan->pGroupIdName, pCxt->pPlanCxt->pGroupIdName, TSDB_COL_NAME_LEN);
tstrncpy(pTableScan->pIsWindowFilledName, pCxt->pPlanCxt->pIsWindowFilledName, TSDB_COL_NAME_LEN);
code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode);
if (TSDB_CODE_SUCCESS == code) {
@ -2216,6 +2221,15 @@ static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExc
if (TSDB_CODE_SUCCESS == code) {
code = setConditionsSlotId(pCxt, (const SLogicNode*)pExchangeLogicNode, (SPhysiNode*)pScan);
}
if (TSDB_CODE_SUCCESS == code) {
SStreamScanPhysiNode* pTableScan = (SStreamScanPhysiNode*)pScan;
pTableScan->triggerType = pCxt->pPlanCxt->triggerType;
tstrncpy(pTableScan->pStbFullName, pCxt->pPlanCxt->pStbFullName, TSDB_TABLE_FNAME_LEN);
tstrncpy(pTableScan->pWstartName, pCxt->pPlanCxt->pWstartName, TSDB_COL_NAME_LEN);
tstrncpy(pTableScan->pWendName, pCxt->pPlanCxt->pWendName, TSDB_COL_NAME_LEN);
tstrncpy(pTableScan->pGroupIdName, pCxt->pPlanCxt->pGroupIdName, TSDB_COL_NAME_LEN);
tstrncpy(pTableScan->pIsWindowFilledName, pCxt->pPlanCxt->pIsWindowFilledName, TSDB_COL_NAME_LEN);
}
if (TSDB_CODE_SUCCESS == code) {
*pPhyNode = (SPhysiNode*)pScan;
@ -2250,6 +2264,7 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList*
if (nodeType(pWindow) == QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL) {
pWindow->node.inputTsOrder = pWindowLogicNode->node.outputTsOrder;
}
pWindow->recalculateInterval = pWindowLogicNode->recalculateInterval;
SNodeList* pPrecalcExprs = NULL;
SNodeList* pFuncs = NULL;
@ -2318,6 +2333,18 @@ static ENodeType getIntervalOperatorType(EWindowAlgorithm windowAlgo) {
return QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
case SESSION_ALGO_MERGE:
return QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION;
case INTERVAL_ALGO_STREAM_CONTINUE_SINGLE:
return QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL;
case INTERVAL_ALGO_STREAM_CONTINUE_FINAL:
return QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL;
case INTERVAL_ALGO_STREAM_CONTINUE_SEMI:
return QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL;
case SESSION_ALGO_STREAM_CONTINUE_SINGLE:
return QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SESSION;
case SESSION_ALGO_STREAM_CONTINUE_FINAL:
return QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_SESSION;
case SESSION_ALGO_STREAM_CONTINUE_SEMI:
return QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION;
default:
break;
}
@ -2371,9 +2398,16 @@ static int32_t createSessionWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList*
static int32_t createStateWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) {
ENodeType type = QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE;
if (pCxt->pPlanCxt->streamQuery) {
if (pCxt->pPlanCxt->triggerType == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
type = QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_STATE;
} else {
type = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
}
}
SStateWinodwPhysiNode* pState = (SStateWinodwPhysiNode*)makePhysiNode(
pCxt, (SLogicNode*)pWindowLogicNode,
(pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE : QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE));
pCxt, (SLogicNode*)pWindowLogicNode, type);
if (NULL == pState) {
return terrno;
}
@ -2418,9 +2452,17 @@ static int32_t createStateWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC
static int32_t createEventWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) {
ENodeType type = QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT;
if (pCxt->pPlanCxt->streamQuery) {
if (pCxt->pPlanCxt->triggerType == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
type = QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_EVENT;
} else {
type = QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT;
}
}
SEventWinodwPhysiNode* pEvent = (SEventWinodwPhysiNode*)makePhysiNode(
pCxt, (SLogicNode*)pWindowLogicNode,
(pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT : QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT));
pCxt, (SLogicNode*)pWindowLogicNode, type);
if (NULL == pEvent) {
return terrno;
}
@ -2446,9 +2488,17 @@ static int32_t createEventWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC
static int32_t createCountWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) {
ENodeType type = QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT;
if (pCxt->pPlanCxt->streamQuery) {
if (pCxt->pPlanCxt->triggerType == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
type = QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_COUNT;
} else {
type = QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT;
}
}
SCountWinodwPhysiNode* pCount = (SCountWinodwPhysiNode*)makePhysiNode(
pCxt, (SLogicNode*)pWindowLogicNode,
(pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT : QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT));
pCxt, (SLogicNode*)pWindowLogicNode, type);
if (NULL == pCount) {
return terrno;
}

11
source/libs/planner/src/planSpliter.c
View File

@ -795,8 +795,13 @@ static int32_t stbSplSplitIntervalForStreamMultiAgg(SSplitContext* pCxt, SStable
int32_t code = stbSplCreatePartMidWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow, &pMidWindow);
if (TSDB_CODE_SUCCESS == code) {
((SWindowLogicNode*)pMidWindow)->windowAlgo = INTERVAL_ALGO_STREAM_MID;
if (pCxt->pPlanCxt->triggerType == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
((SWindowLogicNode*)pInfo->pSplitNode)->windowAlgo = INTERVAL_ALGO_STREAM_CONTINUE_FINAL;
((SWindowLogicNode*)pPartWindow)->windowAlgo = INTERVAL_ALGO_STREAM_CONTINUE_SEMI;
} else {
((SWindowLogicNode*)pInfo->pSplitNode)->windowAlgo = INTERVAL_ALGO_STREAM_FINAL;
((SWindowLogicNode*)pPartWindow)->windowAlgo = INTERVAL_ALGO_STREAM_SEMI;
}
code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pMidWindow);
if (TSDB_CODE_SUCCESS == code) {
code = stbSplCreateExchangeNode(pCxt, pMidWindow, pPartWindow);
@ -836,8 +841,14 @@ static int32_t stbSplSplitSessionForStream(SSplitContext* pCxt, SStableSplitInfo
if (TSDB_CODE_SUCCESS == code) {
SWindowLogicNode* pPartWin = (SWindowLogicNode*)pPartWindow;
SWindowLogicNode* pMergeWin = (SWindowLogicNode*)pInfo->pSplitNode;
if (pCxt->pPlanCxt->triggerType == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
pPartWin->windowAlgo = SESSION_ALGO_STREAM_CONTINUE_SEMI;
pMergeWin->windowAlgo = SESSION_ALGO_STREAM_CONTINUE_FINAL;
} else {
pPartWin->windowAlgo = SESSION_ALGO_STREAM_SEMI;
pMergeWin->windowAlgo = SESSION_ALGO_STREAM_FINAL;
}
int32_t index = 0;
int32_t code = stbSplAppendWEnd(pPartWin, &index);
if (TSDB_CODE_SUCCESS == code) {

9
source/libs/planner/src/planValidator.c
View File

@ -89,6 +89,9 @@ int32_t doValidatePhysiNode(SValidatePlanContext* pCxt, SNode* pNode) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
@ -113,6 +116,12 @@ int32_t doValidatePhysiNode(SValidatePlanContext* pCxt, SNode* pNode) {
case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE:
case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERP_FUNC:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_STATE:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_EVENT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_COUNT:
break;
case QUERY_NODE_PHYSICAL_SUBPLAN:
return validateSubplanNode(pCxt, (SSubplan*)pNode);

6
source/libs/scalar/src/sclfunc.c
View File

@ -3250,6 +3250,12 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
return TSDB_CODE_SUCCESS;
}
int32_t isWinFilledFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
int8_t data = 0;
colDataSetInt8(pOutput->columnData, pOutput->numOfRows, &data);
return TSDB_CODE_SUCCESS;
}
int32_t qPseudoTagFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
char *p = colDataGetData(pInput->columnData, 0);
int32_t code = colDataSetNItems(pOutput->columnData, pOutput->numOfRows, p, pInput->numOfRows, true);

4
source/libs/stream/inc/streamBackendRocksdb.h
View File

@ -189,7 +189,7 @@ int32_t streamStateSessionPut_rocksdb(SStreamState* pState, const SSessionKey* k
int32_t streamStateSessionGet_rocksdb(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateSessionDel_rocksdb(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, SSessionKey* key);
SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* streamStateSessionSeekKeyPrev_rocksdb(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* streamStateSessionSeekToLast_rocksdb(SStreamState* pState, int64_t groupId);
@ -221,7 +221,7 @@ int32_t streamStateFillGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* p
int32_t streamStatePutParTag_rocksdb(SStreamState* pState, int64_t groupId, const void* tag, int32_t tagLen);
int32_t streamStateGetParTag_rocksdb(SStreamState* pState, int64_t groupId, void** tagVal, int32_t* tagLen);
void streamStateParTagSeekKeyNext_rocksdb(SStreamState* pState, const int64_t groupId, SStreamStateCur* pCur);
int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGroupId, const void** pVal,
int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGroupId, void** pVal,
int32_t* pVLen);
// parname cf

5
source/libs/stream/inc/streamInt.h
View File

@ -243,11 +243,10 @@ int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32
int64_t checkpointId, SRpcMsg* pMsg);
int32_t flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock);
int32_t streamCreateSinkResTrigger(SStreamTrigger** pTrigger);
int32_t streamCreateTriggerBlock(SStreamTrigger** pTrigger, int32_t type, int32_t blockType);
int32_t streamCreateForcewindowTrigger(SStreamTrigger** pTrigger, int32_t trigger, SInterval* pInterval,
STimeWindow* pLatestWindow, const char* id);
// inject stream errors
void chkptFailedByRetrieveReqToSource(SStreamTask* pTask, int64_t checkpointId);
int32_t streamCreateRecalculateBlock(SStreamTask* pTask, SStreamDataBlock** pBlock, int32_t type);
// inject stream errors
void chkptFailedByRetrieveReqToSource(SStreamTask* pTask, int64_t checkpointId);

101
source/libs/stream/src/streamBackendRocksdb.c
View File

@ -1665,7 +1665,7 @@ _EXIT:
// clear checkpoint dir if failed
if (code != 0 && pChkpDir != NULL) {
if (taosDirExist(pChkpIdDir)) {
TAOS_UNUSED(taosRemoveDir(pChkpIdDir));
taosRemoveDir(pChkpIdDir);
}
}
taosMemoryFree(pChkpIdDir);
@ -3042,6 +3042,9 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) {
}
if (pState != NULL && idx != -1) {
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
if (wrapper == NULL) {
return -1;
}
@ -3084,6 +3087,9 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe
*readOpt = rocksdb_readoptions_create();
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
if (snapshot != NULL) {
*snapshot = (rocksdb_snapshot_t*)rocksdb_create_snapshot(wrapper->db);
rocksdb_readoptions_set_snapshot(*readOpt, *snapshot);
@ -3105,6 +3111,9 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe
break; \
} \
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \
if (pState->pTdbState->recalc) { \
wrapper = pState->pTdbState->pOwner->pRecalBackend; \
} \
TAOS_UNUSED(atomic_add_fetch_64(&wrapper->dataWritten, 1)); \
char toString[128] = {0}; \
if (stDebugFlag & DEBUG_TRACE) TAOS_UNUSED((ginitDict[i].toStrFunc((void*)key, toString))); \
@ -3138,6 +3147,9 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe
break; \
} \
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \
if (pState->pTdbState->recalc) { \
wrapper = pState->pTdbState->pOwner->pRecalBackend; \
} \
char toString[128] = {0}; \
if (stDebugFlag & DEBUG_TRACE) TAOS_UNUSED((ginitDict[i].toStrFunc((void*)key, toString))); \
int32_t klen = ginitDict[i].enFunc((void*)key, buf); \
@ -3182,6 +3194,9 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe
break; \
} \
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \
if (pState->pTdbState->recalc) { \
wrapper = pState->pTdbState->pOwner->pRecalBackend; \
} \
TAOS_UNUSED(atomic_add_fetch_64(&wrapper->dataWritten, 1)); \
char toString[128] = {0}; \
if (stDebugFlag & DEBUG_TRACE) TAOS_UNUSED(ginitDict[i].toStrFunc((void*)key, toString)); \
@ -3250,6 +3265,10 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) {
stDebug("streamStateClear_rocksdb");
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
TAOS_UNUSED(atomic_add_fetch_64(&wrapper->dataWritten, 1));
char sKeyStr[128] = {0};
@ -3407,6 +3426,9 @@ SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWin
return NULL;
}
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
pCur->number = pState->number;
pCur->db = wrapper->db;
pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot,
@ -3458,7 +3480,13 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) {
if (pCur == NULL) return NULL;
pCur->number = pState->number;
pCur->db = ((STaskDbWrapper*)pState->pTdbState->pOwner->pBackend)->db;
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
pCur->db = ((STaskDbWrapper*)wrapper)->db;
pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot,
(rocksdb_readoptions_t**)&pCur->readOpt);
@ -3481,7 +3509,11 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) {
SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) {
stDebug("streamStateGetCur_rocksdb");
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
SStreamStateCur* pCur = createStreamStateCursor();
if (pCur == NULL) return NULL;
@ -3628,6 +3660,9 @@ SStreamStateCur* streamStateSessionSeekToLast_rocksdb(SStreamState* pState, int6
return NULL;
}
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
SStreamStateCur* pCur = createStreamStateCursor();
pCur->number = pState->number;
@ -3664,6 +3699,9 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta
stDebug("streamStateSessionSeekKeyCurrentPrev_rocksdb");
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
SStreamStateCur* pCur = createStreamStateCursor();
if (pCur == NULL) {
return NULL;
@ -3711,9 +3749,13 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta
}
return pCur;
}
SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, SSessionKey* key) {
SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, const SSessionKey* key) {
stDebug("streamStateSessionSeekKeyCurrentNext_rocksdb");
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
SStreamStateCur* pCur = createStreamStateCursor();
if (pCur == NULL) {
return NULL;
@ -3752,6 +3794,10 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pSta
SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, const SSessionKey* key) {
stDebug("streamStateSessionSeekKeyNext_rocksdb");
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
SStreamStateCur* pCur = createStreamStateCursor();
if (pCur == NULL) {
return NULL;
@ -3792,6 +3838,9 @@ SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, con
SStreamStateCur* streamStateSessionSeekKeyPrev_rocksdb(SStreamState* pState, const SSessionKey* key) {
stDebug("streamStateSessionSeekKeyPrev_rocksdb");
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
SStreamStateCur* pCur = createStreamStateCursor();
if (pCur == NULL) {
return NULL;
@ -3912,6 +3961,9 @@ SStreamStateCur* streamStateFillGetCur_rocksdb(SStreamState* pState, const SWinK
stDebug("streamStateFillGetCur_rocksdb");
SStreamStateCur* pCur = createStreamStateCursor();
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
if (pCur == NULL) return NULL;
@ -3970,6 +4022,9 @@ int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey,
SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key) {
stDebug("streamStateFillSeekKeyNext_rocksdb");
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
SStreamStateCur* pCur = createStreamStateCursor();
if (!pCur) {
return NULL;
@ -4008,6 +4063,9 @@ SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const
SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey* key) {
stDebug("streamStateFillSeekKeyPrev_rocksdb");
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
SStreamStateCur* pCur = createStreamStateCursor();
if (pCur == NULL) {
return NULL;
@ -4053,6 +4111,9 @@ SStreamStateCur* streamStateFillSeekToLast_rocksdb(SStreamState* pState) {
int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) {
stDebug("streamStateSessionGetKeyByRange_rocksdb");
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
SStreamStateCur* pCur = createStreamStateCursor();
if (pCur == NULL) {
return -1;
@ -4262,6 +4323,10 @@ void streamStateParTagSeekKeyNext_rocksdb(SStreamState* pState, const int64_t gr
return;
}
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
pCur->number = pState->number;
pCur->db = wrapper->db;
pCur->iter = streamStateIterCreate(pState, "partag", (rocksdb_snapshot_t**)&pCur->snapshot,
@ -4293,8 +4358,7 @@ void streamStateParTagSeekKeyNext_rocksdb(SStreamState* pState, const int64_t gr
}
}
int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGroupId, const void** pVal,
int32_t* pVLen) {
int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGroupId, void** pVal, int32_t* pVLen) {
stDebug("streamStateFillGetKVByCur_rocksdb");
if (!pCur) {
return -1;
@ -4320,7 +4384,6 @@ int32_t streamStateParTagGetKVByCur_rocksdb(SStreamStateCur* pCur, int64_t* pGro
return 0;
}
#ifdef BUILD_NO_CALL
int32_t streamStateGetParTag_rocksdb(SStreamState* pState, int64_t groupId, void** tagVal, int32_t* tagLen) {
int code = 0;
char* tVal;
@ -4335,7 +4398,7 @@ int32_t streamStateGetParTag_rocksdb(SStreamState* pState, int64_t groupId, void
return code;
}
#endif
// parname cfg
int32_t streamStatePutParName_rocksdb(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) {
int code = 0;
@ -4376,6 +4439,9 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co
char* err = NULL;
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
rocksdb_snapshot_t* snapshot = NULL;
rocksdb_readoptions_t* readopts = NULL;
rocksdb_iterator_t* pIter = streamStateIterCreate(pState, "default", &snapshot, &readopts);
@ -4420,6 +4486,9 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co
void* streamDefaultIterCreate_rocksdb(SStreamState* pState) {
SStreamStateCur* pCur = createStreamStateCursor();
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
pCur->db = wrapper->db;
pCur->iter = streamStateIterCreate(pState, "default", (rocksdb_snapshot_t**)&pCur->snapshot,
@ -4477,6 +4546,9 @@ int32_t streamStatePutBatch(SStreamState* pState, const char* cfKeyName, rocksdb
void* val, int32_t vlen, int64_t ttl) {
int32_t code = 0;
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
TAOS_UNUSED(atomic_add_fetch_64(&wrapper->dataWritten, 1));
int i = streamStateGetCfIdx(pState, cfKeyName);
@ -4524,6 +4596,9 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb
int32_t ttlVLen = ginitDict[cfIdx].enValueFunc(dst, size, ttl, &ttlV);
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
TAOS_UNUSED(atomic_add_fetch_64(&wrapper->dataWritten, 1));
@ -4548,6 +4623,9 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb
int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch) {
char* err = NULL;
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
TAOS_UNUSED(atomic_add_fetch_64(&wrapper->dataWritten, 1));
rocksdb_write(wrapper->db, wrapper->writeOpt, (rocksdb_writebatch_t*)pBatch, &err);
if (err != NULL) {
@ -5272,6 +5350,10 @@ int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname) {
SStreamStateCur* streamStateSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey* key) {
stDebug("streamStateSeekKeyPrev_rocksdb");
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
if (pState->pTdbState->recalc) {
wrapper = pState->pTdbState->pOwner->pRecalBackend;
}
SStreamStateCur* pCur = createStreamStateCursor();
if (pCur == NULL) {
return NULL;
@ -5282,8 +5364,9 @@ SStreamStateCur* streamStateSeekKeyPrev_rocksdb(SStreamState* pState, const SWin
(rocksdb_readoptions_t**)&pCur->readOpt);
pCur->number = pState->number;
SStateKey sKey = {.key = *key, .opNum = pState->number};
char buf[128] = {0};
int len = winKeyEncode((void*)key, buf);
int len = stateKeyEncode((void*)&sKey, buf);
if (!streamStateIterSeekAndValid(pCur->iter, buf, len)) {
streamStateFreeCur(pCur);
return NULL;
@ -5297,7 +5380,7 @@ SStreamStateCur* streamStateSeekKeyPrev_rocksdb(SStreamState* pState, const SWin
size_t kLen = 0;
char* keyStr = (char*)rocksdb_iter_key(pCur->iter, &kLen);
TAOS_UNUSED(winKeyDecode((void*)&curKey, keyStr));
if (winKeyCmpr(key, sizeof(*key), &curKey, sizeof(curKey)) > 0) {
if (winKeyCmpr(&sKey, sizeof(sKey), &curKey, sizeof(curKey)) > 0) {
return pCur;
}
rocksdb_iter_prev(pCur->iter);

6
source/libs/stream/src/streamCheckStatus.c
View File

@ -330,7 +330,7 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) {
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void processDownstreamReadyRsp(SStreamTask* pTask) {
EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST;
EStreamTaskEvent event = (pTask->info.fillHistory != STREAM_HISTORY_TASK) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST;
int32_t code = streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL);
if (code) {
stError("s-task:%s failed to set event succ, code:%s", pTask->id.idStr, tstrerror(code));
@ -362,10 +362,10 @@ void processDownstreamReadyRsp(SStreamTask* pTask) {
// not invoke in success callback due to the deadlock.
// todo: let's retry
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
stDebug("s-task:%s try to launch related fill-history task", pTask->id.idStr);
stDebug("s-task:%s try to launch related task", pTask->id.idStr);
code = streamLaunchFillHistoryTask(pTask);
if (code) {
stError("s-task:%s failed to launch history task, code:%s", pTask->id.idStr, tstrerror(code));
stError("s-task:%s failed to launch related task, code:%s", pTask->id.idStr, tstrerror(code));
}
}
}

49
source/libs/stream/src/streamCheckpoint.c
View File

@ -626,6 +626,7 @@ static int32_t doUpdateCheckpointInfoCheck(SStreamTask* pTask, bool restored, SV
{ // destroy the related fill-history tasks
if (pReq->dropRelHTask) {
if (pTask->info.trigger != STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
code = streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId);
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
@ -636,6 +637,7 @@ static int32_t doUpdateCheckpointInfoCheck(SStreamTask* pTask, bool restored, SV
code = streamMetaCommit(pMeta);
}
}
}
*pContinue = false;
// always return true
@ -724,9 +726,14 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
streamTaskClearCheckInfo(pTask, true);
if (pReq->dropRelHTask) {
stDebug("s-task:0x%x vgId:%d drop the related fill-history task:0x%" PRIx64 " after update checkpoint",
if (pTask->info.trigger != STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
stInfo("s-task:0x%x vgId:%d drop the related fill-history task:0x%" PRIx64 " after update checkpoint",
pReq->taskId, vgId, pReq->hTaskId);
CLEAR_RELATED_FILLHISTORY_TASK(pTask);
} else {
stInfo("s-task:0x%x vgId:%d update the related fill-history task:0x%" PRIx64" to be recalculate task",
pReq->taskId, vgId, pReq->hTaskId);
}
}
stDebug("s-task:0x%x set the persistent status attr to be ready, prev:%s, status in sm:%s", pReq->taskId,
@ -739,16 +746,54 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s vgId:%d failed to save task info after do checkpoint, checkpointId:%" PRId64 ", since %s", id,
vgId, pReq->checkpointId, terrstr());
vgId, pReq->checkpointId, tstrerror(code));
return TSDB_CODE_SUCCESS;
}
// drop task should not in the meta-lock, and drop the related fill-history task now
if (pReq->dropRelHTask) {
if (pTask->info.trigger != STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
code = streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId);
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped, remain tasks:%d", id, vgId,
(int32_t)pReq->hTaskId, numOfTasks);
} else {
STaskId hTaskId = {.streamId = pReq->hStreamId, .taskId = pReq->hTaskId};
SStreamTask* pHTask = NULL;
code = streamMetaAcquireTaskUnsafe(pMeta, &hTaskId, &pHTask);
if (code == 0 && pHTask != NULL) {
stInfo("s-task:0x%x fill-history updated to recalculate task, reset step2Start ts, stream task:0x%x",
(int32_t)hTaskId.taskId, pReq->taskId);
streamMutexLock(&pHTask->lock);
pHTask->info.fillHistory = STREAM_RECALCUL_TASK;
pHTask->execInfo.step2Start = 0; // clear the step2start timestamp
SStreamTaskState status = streamTaskGetStatus(pHTask);
if (status.state == TASK_STATUS__SCAN_HISTORY) {
code = streamTaskHandleEvent(pHTask->status.pSM, TASK_EVENT_SCANHIST_DONE);
}
if (pHTask->pBackend != NULL) {
streamFreeTaskState(pHTask, TASK_STATUS__READY);
pHTask->pBackend = NULL;
}
if (pHTask->exec.pExecutor != NULL) {
qDestroyTask(pHTask->exec.pExecutor);
pHTask->exec.pExecutor = NULL;
}
pMeta->expandTaskFn(pHTask);
streamMutexUnlock(&pHTask->lock);
code = streamMetaSaveTaskInMeta(pMeta, pHTask);
streamMetaReleaseTask(pMeta, pHTask);
}
}
}
code = streamMetaCommit(pMeta);

63
source/libs/stream/src/streamData.c
View File

@ -274,6 +274,10 @@ int32_t streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem
}
void streamFreeQitem(SStreamQueueItem* data) {
if (data == NULL) {
return;
}
int8_t type = data->type;
if (type == STREAM_INPUT__GET_RES) {
blockDataDestroy(((SStreamTrigger*)data)->pBlock);
@ -301,7 +305,7 @@ void streamFreeQitem(SStreamQueueItem* data) {
blockDataDestroy(pRefBlock->pBlock);
taosFreeQitem(pRefBlock);
} else if (type == STREAM_INPUT__CHECKPOINT || type == STREAM_INPUT__CHECKPOINT_TRIGGER ||
type == STREAM_INPUT__TRANS_STATE) {
type == STREAM_INPUT__TRANS_STATE || type == STREAM_INPUT__RECALCULATE) {
SStreamDataBlock* pBlock = (SStreamDataBlock*)data;
taosArrayDestroyEx(pBlock->blocks, freeItems);
taosFreeQitem(pBlock);
@ -340,23 +344,72 @@ int32_t streamCreateForcewindowTrigger(SStreamTrigger** pTrigger, int32_t interv
return code;
}
int32_t streamCreateSinkResTrigger(SStreamTrigger** pTrigger) {
int32_t streamCreateTriggerBlock(SStreamTrigger** pTrigger, int32_t type, int32_t blockType) {
QRY_PARAM_CHECK(pTrigger);
SStreamTrigger* p = NULL;
SStreamTrigger* p = NULL;
int32_t code = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM, 0, (void**)&p);
if (code) {
return code;
}
p->type = STREAM_INPUT__GET_RES;
p->type = (int8_t) type;
p->pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
if (p->pBlock == NULL) {
taosFreeQitem(p);
return terrno;
}
p->pBlock->info.type = STREAM_GET_ALL;
p->pBlock->info.type = blockType;
*pTrigger = p;
return code;
}
int32_t streamCreateRecalculateBlock(SStreamTask* pTask, SStreamDataBlock** pBlock, int32_t type) {
int32_t code = 0;
SSDataBlock* p = NULL;
SStreamDataBlock* pRecalc = NULL;
if (pBlock != NULL) {
*pBlock = NULL;
}
code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock), (void**)&pRecalc);
if (code) {
return code;
}
p = taosMemoryCalloc(1, sizeof(SSDataBlock));
if (p == NULL) {
code = terrno;
goto _err;
}
pRecalc->type = STREAM_INPUT__RECALCULATE;
p->info.type = type;
p->info.rows = 1;
p->info.childId = pTask->info.selfChildId;
pRecalc->blocks = taosArrayInit(4, sizeof(SSDataBlock)); // pBlock;
if (pRecalc->blocks == NULL) {
code = terrno;
goto _err;
}
void* px = taosArrayPush(pRecalc->blocks, p);
if (px == NULL) {
code = terrno;
goto _err;
}
taosMemoryFree(p);
*pBlock = pRecalc;
return code;
_err:
taosMemoryFree(p);
taosFreeQitem(pRecalc);
return code;
}

161
source/libs/stream/src/streamDispatch.c
View File

@ -367,8 +367,10 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD
}
// TODO: do not use broadcast
if (pDataBlock->info.type == STREAM_DELETE_RESULT || pDataBlock->info.type == STREAM_CHECKPOINT ||
pDataBlock->info.type == STREAM_TRANS_STATE) {
EStreamType type = pDataBlock->info.type;
if (type == STREAM_DELETE_RESULT || type == STREAM_CHECKPOINT ||
type == STREAM_TRANS_STATE || type == STREAM_RECALCULATE_START) {
for (int32_t j = 0; j < numOfVgroups; j++) {
code = streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]);
if (code != 0) {
@ -527,6 +529,76 @@ static void cleanupInMonitor(int32_t taskId, int64_t taskRefId, void* param) {
streamTaskFreeRefId(param);
}
static int32_t sendFailedDispatchData(SStreamTask* pTask, int64_t now) {
int32_t code = 0;
const char* id = pTask->id.idStr;
SDispatchMsgInfo* pMsgInfo = &pTask->msgInfo;
streamMutexLock(&pMsgInfo->lock);
int32_t msgId = pMsgInfo->msgId;
SStreamDispatchReq* pReq = pTask->msgInfo.pData;
if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
stDebug("s-task:%s (child taskId:%d) retry shuffle-dispatch to down streams, msgId:%d", id, pTask->info.selfChildId,
msgId);
int32_t numOfRetry = 0;
for (int32_t i = 0; i < taosArrayGetSize(pTask->msgInfo.pSendInfo); ++i) {
SDispatchEntry* pEntry = taosArrayGet(pTask->msgInfo.pSendInfo, i);
if (pEntry == NULL) {
continue;
}
if (pEntry->status == TSDB_CODE_SUCCESS && pEntry->rspTs > 0) {
continue;
}
// downstream not rsp yet beyond threshold that is 10s
if (isDispatchRspTimeout(pEntry, now)) { // not respond yet beyonds 30s, re-send data
doSendFailedDispatch(pTask, pEntry, now, "timeout");
numOfRetry += 1;
continue;
}
// downstream inputQ is closed
if (pEntry->status == TASK_INPUT_STATUS__BLOCKED) {
doSendFailedDispatch(pTask, pEntry, now, "downstream inputQ blocked");
numOfRetry += 1;
continue;
}
// handle other errors
if (pEntry->status != TSDB_CODE_SUCCESS) {
doSendFailedDispatch(pTask, pEntry, now, "downstream error");
numOfRetry += 1;
}
}
stDebug("s-task:%s complete retry shuffle-dispatch blocks to all %d vnodes, msgId:%d", pTask->id.idStr, numOfRetry,
msgId);
} else {
int32_t dstVgId = pTask->outputInfo.fixedDispatcher.nodeId;
SEpSet* pEpSet = &pTask->outputInfo.fixedDispatcher.epSet;
int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId;
int32_t s = taosArrayGetSize(pTask->msgInfo.pSendInfo);
SDispatchEntry* pEntry = taosArrayGet(pTask->msgInfo.pSendInfo, 0);
if (pEntry != NULL) {
setResendInfo(pEntry, now);
code = doSendDispatchMsg(pTask, pReq, dstVgId, pEpSet);
stDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to s-task:0x%x (vgId:%d), msgId:%d, code:%s", id,
pTask->info.selfChildId, 1, downstreamTaskId, dstVgId, msgId, tstrerror(code));
} else {
stError("s-task:%s invalid index 0, size:%d", id, s);
}
}
streamMutexUnlock(&pMsgInfo->lock);
return code;
}
static void doMonitorDispatchData(void* param, void* tmrId) {
int32_t code = 0;
int64_t now = taosGetTimestampMs();
@ -590,65 +662,7 @@ static void doMonitorDispatchData(void* param, void* tmrId) {
return;
}
{
SStreamDispatchReq* pReq = pTask->msgInfo.pData;
if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
stDebug("s-task:%s (child taskId:%d) retry shuffle-dispatch to down streams, msgId:%d", id,
pTask->info.selfChildId, msgId);
int32_t numOfRetry = 0;
for (int32_t i = 0; i < taosArrayGetSize(pTask->msgInfo.pSendInfo); ++i) {
SDispatchEntry* pEntry = taosArrayGet(pTask->msgInfo.pSendInfo, i);
if (pEntry == NULL) {
continue;
}
if (pEntry->status == TSDB_CODE_SUCCESS && pEntry->rspTs > 0) {
continue;
}
// downstream not rsp yet beyond threshold that is 10s
if (isDispatchRspTimeout(pEntry, now)) { // not respond yet beyonds 30s, re-send data
doSendFailedDispatch(pTask, pEntry, now, "timeout");
numOfRetry += 1;
continue;
}
// downstream inputQ is closed
if (pEntry->status == TASK_INPUT_STATUS__BLOCKED) {
doSendFailedDispatch(pTask, pEntry, now, "downstream inputQ blocked");
numOfRetry += 1;
continue;
}
// handle other errors
if (pEntry->status != TSDB_CODE_SUCCESS) {
doSendFailedDispatch(pTask, pEntry, now, "downstream error");
numOfRetry += 1;
}
}
stDebug("s-task:%s complete retry shuffle-dispatch blocks to all %d vnodes, msgId:%d", pTask->id.idStr,
numOfRetry, msgId);
} else {
int32_t dstVgId = pTask->outputInfo.fixedDispatcher.nodeId;
SEpSet* pEpSet = &pTask->outputInfo.fixedDispatcher.epSet;
int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId;
int32_t s = taosArrayGetSize(pTask->msgInfo.pSendInfo);
SDispatchEntry* pEntry = taosArrayGet(pTask->msgInfo.pSendInfo, 0);
if (pEntry != NULL) {
setResendInfo(pEntry, now);
code = doSendDispatchMsg(pTask, pReq, dstVgId, pEpSet);
stDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to s-task:0x%x (vgId:%d), msgId:%d, code:%s", id,
pTask->info.selfChildId, 1, downstreamTaskId, dstVgId, msgId, tstrerror(code));
} else {
stError("s-task:%s invalid index 0, size:%d", id, s);
}
}
}
code = sendFailedDispatchData(pTask, now);
if (streamTaskShouldStop(pTask)) {
stDebug("s-task:%s should stop, abort from timer", pTask->id.idStr);
@ -833,7 +847,8 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
int32_t type = pBlock->type;
if (!(type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__CHECKPOINT_TRIGGER ||
type == STREAM_INPUT__TRANS_STATE)) {
type == STREAM_INPUT__TRANS_STATE || type == STREAM_INPUT__RECALCULATE)) {
atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL);
stError("s-task:%s invalid dispatch block type:%d", id, type);
return TSDB_CODE_INTERNAL_ERROR;
}
@ -880,7 +895,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
code = sendDispatchMsg(pTask, pTask->msgInfo.pData);
// todo: secure the timerActive and start timer in after lock pTask->lock
// todo: start timer in after lock pTask->lock
streamMutexLock(&pTask->lock);
bool shouldStop = streamTaskShouldStop(pTask);
streamMutexUnlock(&pTask->lock);
@ -890,7 +905,6 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
} else {
streamMutexLock(&pTask->msgInfo.lock);
if (pTask->msgInfo.inMonitor == 0) {
// int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s start dispatch monitor tmr in %dms, dispatch code:%s", id, DISPATCH_RETRY_INTERVAL_MS,
tstrerror(code));
streamStartMonitorDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS);
@ -1845,6 +1859,9 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
return TSDB_CODE_STREAM_TASK_NOT_EXIST;
}
stDebug("s-task:%s lastMsgId:%"PRId64 " for upstream taskId:0x%x(vgId:%d)", id, pInfo->lastMsgId, pReq->upstreamTaskId,
pReq->upstreamNodeId);
if (pMeta->role == NODE_ROLE_FOLLOWER) {
stError("s-task:%s task on follower received dispatch msgs, dispatch msg rejected", id);
status = TASK_INPUT_STATUS__REFUSED;
@ -1868,9 +1885,25 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
stDebug("s-task:%s close inputQ for upstream:0x%x, msgId:%d", id, pReq->upstreamTaskId, pReq->msgId);
} else if (pReq->type == STREAM_INPUT__TRANS_STATE) {
stDebug("s-task:%s recv trans-state msgId:%d from upstream:0x%x", id, pReq->msgId, pReq->upstreamTaskId);
} else if (pReq->type == STREAM_INPUT__RECALCULATE) {
stDebug("s-task:%s recv recalculate msgId:%d from upstream:0x%x", id, pReq->msgId, pReq->upstreamTaskId);
}
if (pReq->msgId > pInfo->lastMsgId) {
status = streamTaskAppendInputBlocks(pTask, pReq);
if (status == TASK_INPUT_STATUS__NORMAL) {
stDebug("s-task:%s update the lastMsgId from %" PRId64 " to %d", id, pInfo->lastMsgId, pReq->msgId);
pInfo->lastMsgId = pReq->msgId;
} else {
stDebug("s-task:%s not update the lastMsgId, remain:%" PRId64, id, pInfo->lastMsgId);
}
} else {
stWarn(
"s-task:%s duplicate msgId:%d from upstream:0x%x discard and return succ, from vgId:%d already recv "
"msgId:%" PRId64,
id, pReq->msgId, pReq->upstreamTaskId, pReq->upstreamNodeId, pInfo->lastMsgId);
status = TASK_INPUT_STATUS__NORMAL; // still return success
}
}
}
}

195
source/libs/stream/src/streamExec.c
View File

@ -23,6 +23,8 @@
#define MIN_INVOKE_INTERVAL 50 // 50ms
#define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec
static int32_t streamAlignRecalculateStart(SStreamTask* pTask);
static int32_t continueDispatchRecalculateStart(SStreamDataBlock* pBlock, SStreamTask* pTask);
static int32_t streamTransferStateDoPrepare(SStreamTask* pTask);
static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize,
int32_t* totalBlocks);
@ -129,6 +131,25 @@ static int32_t doAppendPullOverBlock(SStreamTask* pTask, int32_t* pNumOfBlocks,
return code;
}
static int32_t doAppendRecalBlock(SStreamTask* pTask, int32_t* pNumOfBlocks, SStreamTrigger* pRecalculateBlock,
SArray* pRes) {
int32_t code = 0;
SSDataBlock block = {0};
void* p = taosArrayPush(pRes, pRecalculateBlock->pBlock);
if (p != NULL) {
(*pNumOfBlocks) += 1;
stDebug("s-task:%s(child %d) recalculate from upstream completed, QID:0x%" PRIx64, pTask->id.idStr,
pTask->info.selfChildId, /*pRecalculateBlock->reqId*/ (int64_t)0);
} else {
code = terrno;
stError("s-task:%s failed to append recalculate block for downstream, QID:0x%" PRIx64" code:%s", pTask->id.idStr,
/*pRecalculateBlock->reqId*/(int64_t)0, tstrerror(code));
}
return code;
}
int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) {
int32_t size = 0;
int32_t numOfBlocks = 0;
@ -168,7 +189,7 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t*
}
if (output == NULL) {
if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
if (pItem != NULL && (pItem->type == STREAM_INPUT__DATA_RETRIEVE)) {
code = doAppendPullOverBlock(pTask, &numOfBlocks, (SStreamDataBlock*)pItem, pRes);
if (code) {
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
@ -179,6 +200,10 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t*
break;
}
if (pTask->info.fillHistory == STREAM_RECALCUL_TASK && pTask->info.taskLevel == TASK_LEVEL__AGG) {
stDebug("s-task:%s exec output type:%d", pTask->id.idStr, output->info.type);
}
if (output->info.type == STREAM_RETRIEVE) {
if (streamBroadcastToUpTasks(pTask, output) < 0) {
// TODO
@ -320,7 +345,7 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) {
return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0);
}
if (!pTask->hTaskInfo.operatorOpen) {
if ((!pTask->hTaskInfo.operatorOpen) || (pTask->info.fillHistory == STREAM_RECALCUL_TASK)) {
int32_t code = qSetStreamOpOpen(exec);
pTask->hTaskInfo.operatorOpen = true;
}
@ -523,8 +548,9 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int
const SStreamTrigger* pTrigger = (const SStreamTrigger*)pInput;
code = qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
if (pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) {
stDebug("s-task:%s set force_window_close as source block, skey:%"PRId64, id, pTrigger->pBlock->info.window.skey);
(*pVer) = pTrigger->pBlock->info.window.skey;
TSKEY k = pTrigger->pBlock->info.window.skey;
stDebug("s-task:%s set force_window_close as source block, skey:%" PRId64, id, k);
(*pVer) = k;
}
} else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput;
@ -565,10 +591,20 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int
const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)pInput;
code = qSetMultiStreamInput(pExecutor, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
} else if (pItem->type == STREAM_INPUT__CHECKPOINT || pItem->type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
} else if (pItem->type == STREAM_INPUT__CHECKPOINT || pItem->type == STREAM_INPUT__CHECKPOINT_TRIGGER ||
pItem->type == STREAM_INPUT__RECALCULATE) {
const SStreamDataBlock* pCheckpoint = (const SStreamDataBlock*)pInput;
code = qSetMultiStreamInput(pExecutor, pCheckpoint->blocks, 1, pItem->type);
if (pItem->type == STREAM_INPUT__RECALCULATE) {
int32_t t = ((SStreamDataBlock*) pCheckpoint)->type;
int32_t tId = (int32_t)pTask->hTaskInfo.id.taskId;
if (t == STREAM_RECALCULATE_START) {
stDebug("s-task:%s set recalculate block to start related recalculate task:0x%x", id, tId);
} else {
stDebug("s-task:%s set recalculate block:%d, task:0x%x", id, t, tId);
}
}
} else {
stError("s-task:%s invalid input block type:%d, discard", id, pItem->type);
code = TSDB_CODE_STREAM_INTERNAL_ERROR;
@ -581,6 +617,7 @@ void streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock)
const char* id = pTask->id.idStr;
int32_t code = TSDB_CODE_SUCCESS;
int32_t level = pTask->info.taskLevel;
// dispatch the tran-state block to downstream task immediately
int32_t type = pTask->outputInfo.type;
@ -660,7 +697,6 @@ static int32_t doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock
int32_t code = 0;
stDebug("s-task:%s start to process batch blocks, num:%d, type:%s", id, num, streamQueueItemGetTypeStr(pBlock->type));
code = doSetStreamInputBlock(pTask, pBlock, &ver, id);
if (code) {
stError("s-task:%s failed to set input block, not exec for these blocks", id);
@ -777,6 +813,9 @@ int32_t flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpoi
static int32_t doStreamExecTask(SStreamTask* pTask) {
const char* id = pTask->id.idStr;
int32_t code = 0;
int32_t vgId = pTask->pMeta->vgId;
int32_t taskLevel = pTask->info.taskLevel;
int32_t taskType = pTask->info.fillHistory;
// merge multiple input data if possible in the input queue.
int64_t st = taosGetTimestampMs();
@ -842,8 +881,14 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
continue;
}
if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
if (type != STREAM_INPUT__DATA_BLOCK && type != STREAM_INPUT__CHECKPOINT) {
if (type == STREAM_INPUT__CHECKPOINT) {
code = doHandleChkptBlock(pTask);
streamFreeQitem(pInput);
return code;
}
if (taskLevel == TASK_LEVEL__SINK) {
if (type != STREAM_INPUT__DATA_BLOCK && type != STREAM_INPUT__RECALCULATE) {
stError("s-task:%s invalid block type:%d for sink task, discard", id, type);
continue;
}
@ -858,26 +903,118 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
}
double el = (taosGetTimestampMs() - st) / 1000.0;
if (fabs(el - 0.0) <= DBL_EPSILON) {
pTask->execInfo.procsThroughput = 0;
pTask->execInfo.procsThroughput = (fabs(el - 0.0) <= DBL_EPSILON) ? 0 : (blockSize / el);
} else {
pTask->execInfo.procsThroughput = (blockSize / el);
streamFreeQitem((SStreamQueueItem*)pInput);
}
continue;
}
if (type == STREAM_INPUT__RECALCULATE) {
if (taskType == STREAM_NORMAL_TASK && taskLevel == TASK_LEVEL__AGG) {
int32_t remain = streamAlignRecalculateStart(pTask);
if (remain > 0) {
streamFreeQitem((SStreamQueueItem*)pInput);
stDebug("s-task:%s receive upstream recalculate msg, not sent remain:%d", id, remain);
return code;
}
if (type == STREAM_INPUT__CHECKPOINT) {
code = doHandleChkptBlock(pTask);
streamFreeQitem(pInput);
return code;
} else {
stDebug("s-task:%s all upstream send recalculate msg, continue", id);
}
// 1. generate the recalculating snapshot for related recalculate tasks.
if ((taskType == STREAM_NORMAL_TASK) &&
((taskLevel == TASK_LEVEL__AGG) || (taskLevel == TASK_LEVEL__SOURCE && (!pTask->info.hasAggTasks)))) {
code = doStreamTaskExecImpl(pTask, pInput, numOfBlocks);
} else if (taskType == STREAM_RECALCUL_TASK && taskLevel == TASK_LEVEL__AGG) {
// send retrieve to upstream tasks (source tasks, to start to recalculate procedure.
stDebug("s-task:%s recalculate agg task send retrieve to upstream source tasks", id);
code = doStreamTaskExecImpl(pTask, pInput, numOfBlocks);
}
}
if (type != STREAM_INPUT__RECALCULATE) {
code = doStreamTaskExecImpl(pTask, pInput, numOfBlocks);
streamFreeQitem(pInput);
if (code) {
return code;
}
}
// for stream with only 1 task, start related re-calculate stream task directly.
// We only start the re-calculate agg task here, and do NOT start the source task, for streams with agg-tasks.
if ((type == STREAM_INPUT__RECALCULATE) && (taskType == STREAM_NORMAL_TASK)) {
SSDataBlock* pb = taosArrayGet(((SStreamDataBlock*)pInput)->blocks, 0);
if ((taskLevel == TASK_LEVEL__AGG) || ((taskLevel == TASK_LEVEL__SOURCE) && (!pTask->info.hasAggTasks))) {
EStreamType blockType = pb->info.type;
if (pTask->hTaskInfo.id.streamId == 0) {
stError("s-task:%s related re-calculate stream task is dropping, failed to start re-calculate", id);
streamFreeQitem(pInput);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
if (pTask->info.trigger != STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
stError("s-task:%s invalid trigger model, expect:%d, actually:%d, not exec tasks", id,
STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE, pTask->info.trigger);
streamFreeQitem(pInput);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
SStreamTask* pHTask = NULL;
code = streamMetaAcquireTask(pTask->pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId, &pHTask);
if (code != 0) {
stError("s-task:%s failed to acquire related recalculate task:0x%x, not start the recalculation, code:%s", id,
(int32_t)pTask->hTaskInfo.id.taskId, tstrerror(code));
streamFreeQitem(pInput);
return code;
}
if (blockType == STREAM_RECALCULATE_START) {
// start the related recalculate task to do recalculate
stDebug("s-task:%s start recalculate task to do recalculate:0x%x", id, pHTask->id.taskId);
if (taskLevel == TASK_LEVEL__SOURCE) {
code = streamStartScanHistoryAsync(pHTask, 0);
} else { // for agg task, in normal stream queue to execute
SStreamDataBlock* pRecalBlock = NULL;
code = streamCreateRecalculateBlock(pTask, &pRecalBlock, STREAM_RECALCULATE_START);
if (code) {
stError("s-task:%s failed to generate recalculate block, code:%s", id, tstrerror(code));
} else {
code = streamTaskPutDataIntoInputQ(pHTask, (SStreamQueueItem*)pRecalBlock);
if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed to put recalculate block into q, code:%s", pTask->id.idStr, tstrerror(code));
} else {
stDebug("s-task:%s put recalculate block into inputQ", pHTask->id.idStr);
}
code = streamTrySchedExec(pHTask, false);
}
}
}
streamMetaReleaseTask(pTask->pMeta, pHTask);
} else if ((taskLevel == TASK_LEVEL__SOURCE) && pTask->info.hasAggTasks) {
code = continueDispatchRecalculateStart((SStreamDataBlock*)pInput, pTask);
}
}
if (type == STREAM_INPUT__RECALCULATE) {
streamFreeQitem(pInput);
}
if (code) {
return code;
}
if (taskType == STREAM_RECALCUL_TASK && taskLevel == TASK_LEVEL__AGG && type != STREAM_INPUT__RECALCULATE) {
bool complete = qStreamScanhistoryFinished(pTask->exec.pExecutor);
if (complete) {
stDebug("s-task:%s recalculate agg task complete recalculate procedure", id);
return 0;
}
}
double el = (taosGetTimestampMs() - st) / 1000.0;
if (el > 2.0) { // elapsed more than 5 sec, not occupy the CPU anymore
@ -887,7 +1024,6 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
}
}
}
}
// the task may be set dropping/stopping, while it is still in the task queue, therefore, the sched-status can not
// be updated by tryExec function, therefore, the schedStatus will always be the TASK_SCHED_STATUS__WAITING.
@ -1038,3 +1174,28 @@ int32_t streamAlignTransferState(SStreamTask* pTask) {
return atomic_sub_fetch_32(&pTask->transferStateAlignCnt, 1);
}
int32_t streamAlignRecalculateStart(SStreamTask* pTask) {
int32_t numOfUpstream = taosArrayGetSize(pTask->upstreamInfo.pList);
int32_t old = atomic_val_compare_exchange_32(&pTask->recalculateAlignCnt, 0, numOfUpstream);
if (old == 0) {
stDebug("s-task:%s set start recalculate state aligncnt %d", pTask->id.idStr, numOfUpstream);
}
return atomic_sub_fetch_32(&pTask->recalculateAlignCnt, 1);
}
int32_t continueDispatchRecalculateStart(SStreamDataBlock* pBlock, SStreamTask* pTask) {
pBlock->srcTaskId = pTask->id.taskId;
pBlock->srcVgId = pTask->pMeta->vgId;
int32_t code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock);
if (code == 0) {
code = streamDispatchStreamBlock(pTask);
} else {
stError("s-task:%s failed to put recalculate start block into outputQ, code:%s", pTask->id.idStr, tstrerror(code));
streamFreeQitem((SStreamQueueItem*)pBlock);
}
return code;
}

2
source/libs/stream/src/streamHb.c
View File

@ -238,7 +238,7 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) {
}
// not report the status of fill-history task
if (pTask->info.fillHistory == 1) {
if (pTask->info.fillHistory != STREAM_NORMAL_TASK) {
streamMetaReleaseTask(pMeta, pTask);
continue;
}

19
source/libs/stream/src/streamMeta.c
View File

@ -284,11 +284,16 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) {
return 0;
}
int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key) {
int8_t streamTaskShouldRecalated(SStreamTask* pTask) { return pTask->info.fillHistory == 2 ? 1 : 0; }
int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key, uint8_t recalated) {
int32_t code = 0;
int64_t chkpId = pTask->chkInfo.checkpointId;
// int8_t recalated = streamTaskShouldRecalated(pTask);
streamMutexLock(&pMeta->backendMutex);
// streamId--taskId
void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key));
if ((ppBackend != NULL) && (*ppBackend != NULL)) {
void* p = taskDbAddRef(*ppBackend);
@ -300,7 +305,11 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
STaskDbWrapper* pBackend = *ppBackend;
pBackend->pMeta = pMeta;
if (recalated) {
pTask->pRecalBackend = pBackend;
} else {
pTask->pBackend = pBackend;
}
streamMutexUnlock(&pMeta->backendMutex);
stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend);
@ -323,7 +332,11 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
}
int64_t tref = taosAddRef(taskDbWrapperId, pBackend);
if (recalated) {
pTask->pRecalBackend = pBackend;
} else {
pTask->pBackend = pBackend;
}
pBackend->refId = tref;
pBackend->pTask = pTask;
pBackend->pMeta = pMeta;
@ -348,7 +361,11 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
}
streamMutexUnlock(&pMeta->backendMutex);
if (recalated) {
stDebug("s-task:0x%x set recalated backend %p", pTask->id.taskId, pBackend);
} else {
stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend);
}
return 0;
}

25
source/libs/stream/src/streamQueue.c
View File

@ -144,20 +144,13 @@ void streamQueueNextItemInSourceQ(SStreamQueue* pQueue, SStreamQueueItem** pItem
// let's try the ordinary input q
pQueue->qItem = NULL;
int32_t code = taosGetQitem(pQueue->qall, &pQueue->qItem);
if (code) {
stError("s-task:%s failed to extract data from inputQ, code:%s", id, tstrerror(code));
}
int32_t num = taosGetQitem(pQueue->qall, &pQueue->qItem);
TAOS_UNUSED(num);
if (pQueue->qItem == NULL) {
code = taosReadAllQitems(pQueue->pQueue, pQueue->qall);
if (code) {
stError("s-task:%s failed to read qitem into qall, code:%s", id, tstrerror(code));
}
code = taosGetQitem(pQueue->qall, &pQueue->qItem);
if (code) {
stError("s-task:%s failed to extract data from inputQ(qall), code:%s", id, tstrerror(code));
}
num = taosReadAllQitems(pQueue->pQueue, pQueue->qall);
num = taosGetQitem(pQueue->qall, &pQueue->qItem);
TAOS_UNUSED(num);
}
*pItem = streamQueueCurItem(pQueue);
@ -237,6 +230,8 @@ const char* streamQueueItemGetTypeStr(int32_t type) {
return "trans-state";
case STREAM_INPUT__REF_DATA_BLOCK:
return "ref-block";
case STREAM_INPUT__RECALCULATE:
return "recalculate";
default:
return "datablock";
}
@ -289,7 +284,7 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIte
// do not merge blocks for sink node and check point data block
int8_t type = qItem->type;
if (type == STREAM_INPUT__CHECKPOINT || type == STREAM_INPUT__CHECKPOINT_TRIGGER ||
type == STREAM_INPUT__TRANS_STATE || type == STREAM_INPUT__REF_DATA_BLOCK) {
type == STREAM_INPUT__TRANS_STATE || type == STREAM_INPUT__REF_DATA_BLOCK || type == STREAM_INPUT__RECALCULATE) {
const char* p = streamQueueItemGetTypeStr(type);
if (*pInput == NULL) {
@ -407,7 +402,7 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem)
double size = SIZE_IN_MiB(taosQueueMemorySize(pQueue));
stDebug("s-task:%s blockdata enqueue, total in queue:%d, size:%.2fMiB", pTask->id.idStr, total, size);
} else if (type == STREAM_INPUT__CHECKPOINT || type == STREAM_INPUT__CHECKPOINT_TRIGGER ||
type == STREAM_INPUT__TRANS_STATE || type == STREAM_INPUT__DATA_RETRIEVE) {
type == STREAM_INPUT__TRANS_STATE || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__RECALCULATE) {
int32_t code = 0;
if ((type == STREAM_INPUT__CHECKPOINT_TRIGGER || type == STREAM_INPUT__CHECKPOINT) && (level == TASK_LEVEL__SOURCE)) {
@ -446,7 +441,7 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem)
}
if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && type != STREAM_INPUT__CHECKPOINT_TRIGGER &&
(pTask->info.delaySchedParam != 0)) {
type != STREAM_INPUT__RECALCULATE && (pTask->info.delaySchedParam != 0)) {
(void)atomic_val_compare_exchange_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE,
TASK_TRIGGER_STATUS__MAY_ACTIVE);
stDebug("s-task:%s new data arrived, active the sched-trigger, triggerStatus:%d", pTask->id.idStr,

122
source/libs/stream/src/streamSched.c
View File

@ -29,16 +29,33 @@ void streamSetupScheduleTrigger(SStreamTask* pTask) {
const char* id = pTask->id.idStr;
int64_t* pTaskRefId = NULL;
if (pTask->info.fillHistory == 1) {
if (pTask->info.fillHistory != STREAM_NORMAL_TASK) {
return;
}
// dynamic set the trigger & triggerParam for STREAM_TRIGGER_FORCE_WINDOW_CLOSE
if ((pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) && (pTask->info.taskLevel == TASK_LEVEL__SOURCE)) {
if ((pTask->info.trigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) && (pTask->info.taskLevel == TASK_LEVEL__SOURCE)) {
int64_t waterMark = 0;
SInterval interval = {0};
STimeWindow lastTimeWindow = {0};
code = qGetStreamIntervalExecInfo(pTask->exec.pExecutor, &waterMark, &interval, &lastTimeWindow);
TSKEY recInterval = 0;
code = qGetStreamIntervalExecInfo(pTask->exec.pExecutor, &waterMark, &interval, &lastTimeWindow, &recInterval);
if (code) {
stError("s-task:%s failed to init scheduler info, code:%s", id, tstrerror(code));
return;
}
pTask->info.delaySchedParam = 60000;//recInterval;
stInfo("s-task:%s cont-window-close extract re-calculate delay:%" PRId64, id, pTask->info.delaySchedParam);
delay = pTask->info.delaySchedParam;
} else if ((pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) && (pTask->info.taskLevel == TASK_LEVEL__SOURCE)) {
// dynamic set the trigger & triggerParam for STREAM_TRIGGER_FORCE_WINDOW_CLOSE
int64_t waterMark = 0;
SInterval interval = {0};
STimeWindow lastTimeWindow = {0};
TSKEY recInterval = 0;
code = qGetStreamIntervalExecInfo(pTask->exec.pExecutor, &waterMark, &interval, &lastTimeWindow, &recInterval);
if (code) {
stError("s-task:%s failed to init scheduler info, code:%s", id, tstrerror(code));
return;
@ -84,8 +101,7 @@ void streamSetupScheduleTrigger(SStreamTask* pTask) {
code = streamTaskAllocRefId(pTask, &pTaskRefId);
if (code == 0) {
stDebug("s-task:%s refId:%" PRId64 " enable the scheduler trigger, delay:%" PRId64, pTask->id.idStr,
pTask->id.refId, delay);
stDebug("s-task:%s refId:%" PRId64 " enable the scheduler trigger, delay:%" PRId64, id, pTask->id.refId, delay);
streamTmrStart(streamTaskSchedHelper, (int32_t)delay, pTaskRefId, streamTimer, &pTask->schedInfo.pDelayTimer,
pTask->pMeta->vgId, "sched-tmr");
@ -281,9 +297,34 @@ static int32_t doCreateForceWindowTrigger(SStreamTask* pTask, int32_t* pNextTrig
}
}
int32_t streamCreateAddRecalculateEndBlock(SStreamTask* pTask) {
const char* id = pTask->id.idStr;
SStreamDataBlock* pBlock = NULL;
int32_t code = streamCreateRecalculateBlock(pTask, &pBlock, STREAM_RECALCULATE_END);
if (code) {
stError("s-task:%s failed to create recalculate end trigger, code:%s, try again in ms", id, tstrerror(code));
return code;
}
code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pBlock);
if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed to put recalculate end block into q, code:%s", pTask->id.idStr, tstrerror(code));
} else {
stDebug("s-task:%s add the recalculate end block in inputQ", pTask->id.idStr);
}
return code;
}
void streamTaskSchedHelper(void* param, void* tmrId) {
int32_t code = 0;
int64_t taskRefId = *(int64_t*)param;
int32_t trigger = 0;
int32_t vgId = 0;
int32_t nextTrigger = 0;
int32_t level = 0;
const char* id = NULL;
SStreamTask* pTask = taosAcquireRef(streamTaskRefPool, taskRefId);
if (pTask == NULL) {
stError("invalid task rid:%" PRId64 " failed to acquired stream-task at %s", taskRefId, __func__);
@ -293,13 +334,17 @@ void streamTaskSchedHelper(void* param, void* tmrId) {
stDebug("s-task:%s acquire task, refId:%" PRId64, pTask->id.idStr, pTask->id.refId);
const char* id = pTask->id.idStr;
int32_t nextTrigger = (int32_t)pTask->info.delaySchedParam;
int32_t vgId = pTask->pMeta->vgId;
id = pTask->id.idStr;
nextTrigger = (int32_t)pTask->info.delaySchedParam;
vgId = pTask->pMeta->vgId;
trigger = pTask->info.trigger;
level = pTask->info.taskLevel;
int8_t status = atomic_load_8(&pTask->schedInfo.status);
if (pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE && pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
if (trigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE && level == TASK_LEVEL__SOURCE) {
stTrace("s-task:%s in scheduler to recalculate the update time window, trigger status:%d", id, status);
} else if (trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE && level == TASK_LEVEL__SOURCE) {
int32_t next = convertTimePrecision(nextTrigger, pTask->info.interval.precision, TSDB_TIME_PRECISION_MILLI);
stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, next);
} else {
@ -314,33 +359,66 @@ void streamTaskSchedHelper(void* param, void* tmrId) {
}
if (streamTaskShouldPause(pTask)) {
nextTrigger = TRIGGER_RECHECK_INTERVAL; // retry in 5 sec
stDebug("s-task:%s is paused, recheck in %.2fs", id, TRIGGER_RECHECK_INTERVAL / 1000.0);
streamTmrStart(streamTaskSchedHelper, TRIGGER_RECHECK_INTERVAL, param, streamTimer, &pTask->schedInfo.pDelayTimer,
vgId, "sched-run-tmr");
streamMetaReleaseTask(pTask->pMeta, pTask);
return;
goto _end;
}
if (pTask->status.downstreamReady == 0) {
nextTrigger = TRIGGER_RECHECK_INTERVAL; // retry in 5 sec
stDebug("s-task:%s downstream not ready, recheck in %.2fs", id, TRIGGER_RECHECK_INTERVAL / 1000.0);
streamTmrStart(streamTaskSchedHelper, TRIGGER_RECHECK_INTERVAL, param, streamTimer, &pTask->schedInfo.pDelayTimer,
vgId, "sched-run-tmr");
streamMetaReleaseTask(pTask->pMeta, pTask);
return;
goto _end;
}
if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK) {
nextTrigger = TRIGGER_RECHECK_INTERVAL; // retry in 10 sec
nextTrigger = TRIGGER_RECHECK_INTERVAL; // retry in 5 sec
stDebug("s-task:%s in checkpoint procedure, not retrieve result, next:%dms", id, TRIGGER_RECHECK_INTERVAL);
} else {
if (pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE && pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
if (trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE && level == TASK_LEVEL__SOURCE) {
code = doCreateForceWindowTrigger(pTask, &nextTrigger);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
}
} else if (trigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE && level == TASK_LEVEL__SOURCE) {
// we need to make sure fill-history process is completed
STaskId hId = pTask->hTaskInfo.id;
{
SStreamTask* pHTask = NULL;
int32_t ret = streamMetaAcquireTaskUnsafe(pTask->pMeta, &hId, &pHTask);
if (ret == 0 && pHTask != NULL) {
if (pHTask->info.fillHistory == STREAM_RECALCUL_TASK) {
SStreamDataBlock* pTrigger = NULL;
code = streamCreateRecalculateBlock(pTask, &pTrigger, STREAM_RECALCULATE_START);
if (code) {
stError("s-task:%s failed to prepare recalculate data trigger, code:%s, try again in %dms", id,
tstrerror(code), nextTrigger);
streamMetaReleaseTask(pTask->pMeta, pHTask);
goto _end;
}
atomic_store_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE);
code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger);
if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed to put recalculate block into q, code:%s", pTask->id.idStr, tstrerror(code));
streamMetaReleaseTask(pTask->pMeta, pHTask);
goto _end;
} else {
stDebug("s-task:%s put recalculate block into inputQ", pTask->id.idStr);
}
} else {
stDebug("s-task:%s related task:0x%" PRIx64 " in fill-history model, not start the recalculate procedure",
pTask->id.idStr, hId.taskId);
}
streamMetaReleaseTask(pTask->pMeta, pHTask);
}
}
} else if (status == TASK_TRIGGER_STATUS__MAY_ACTIVE) {
SStreamTrigger* pTrigger = NULL;
code = streamCreateSinkResTrigger(&pTrigger);
code = streamCreateTriggerBlock(&pTrigger, STREAM_INPUT__GET_RES, STREAM_GET_ALL);
if (code) {
stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, tstrerror(code),
nextTrigger);
@ -351,7 +429,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) {
code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger);
if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed to put retrieve aggRes block into q, code:%s", pTask->id.idStr, tstrerror(code));
stError("s-task:%s failed to put retrieve data trigger into q, code:%s", pTask->id.idStr, tstrerror(code));
goto _end;
}
}

344
source/libs/stream/src/streamSessionState.c
View File

@ -20,6 +20,8 @@
#include "tcommon.h"
#include "tsimplehash.h"
#define MAX_SCAN_RANGE_SIZE 102400
int sessionStateKeyCompare(const void* pWin1, const void* pDatas, int pos) {
SRowBuffPos* pPos2 = taosArrayGetP(pDatas, pos);
SSessionKey* pWin2 = (SSessionKey*)pPos2->pKey;
@ -496,14 +498,16 @@ void sessionWinStateClear(SStreamFileState* pFileState) {
}
}
void sessionWinStateCleanup(void* pBuff) {
void* pIte = NULL;
size_t keyLen = 0;
int32_t iter = 0;
while ((pIte = tSimpleHashIterate(pBuff, pIte, &iter)) != NULL) {
SArray* pWinStates = (SArray*)(*(void**)pIte);
taosArrayDestroy(pWinStates);
void freeArrayPtr(void* ptr) {
SArray* pArray = *(void**)ptr;
taosArrayDestroy(pArray);
}
void sessionWinStateCleanup(void* pBuff) {
if (pBuff == NULL) {
return;
}
tSimpleHashSetFreeFp(pBuff, freeArrayPtr);
tSimpleHashCleanup(pBuff);
}
@ -1152,3 +1156,329 @@ _end:
}
return code;
}
static int compareRangeKey(const void* pKey, const void* data, int index) {
SScanRange* pRange1 = (SScanRange*) pKey;
SScanRange* pRange2 = taosArrayGet((SArray*)data, index);
if (pRange1->win.skey > pRange2->win.skey) {
return 1;
} else if (pRange1->win.skey < pRange2->win.skey) {
return -1;
}
if (pRange1->win.ekey > pRange2->win.ekey) {
return 1;
} else if (pRange1->win.ekey < pRange2->win.ekey) {
return -1;
}
return 0;
}
static int32_t scanRangeKeyCmpr(const SScanRange* pRange1, const SScanRange* pRange2) {
if (pRange1->win.skey > pRange2->win.ekey) {
return 1;
} else if (pRange1->win.ekey < pRange2->win.skey) {
return -1;
}
return 0;
}
static int32_t putRangeIdInfo(SScanRange* pRangeKey, uint64_t gpId, uint64_t uId) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
code = tSimpleHashPut(pRangeKey->pUIds, &uId, sizeof(uint64_t), NULL, 0);
QUERY_CHECK_CODE(code, lino, _end);
code = tSimpleHashPut(pRangeKey->pGroupIds, &gpId, sizeof(uint64_t), NULL, 0);
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
void mergeRangeKey(SScanRange* pRangeDest, SScanRange* pRangeSrc) {
pRangeDest->win.skey = TMIN(pRangeDest->win.skey, pRangeSrc->win.skey);
pRangeDest->win.ekey = TMAX(pRangeDest->win.ekey, pRangeSrc->win.ekey);
pRangeDest->calWin.skey = TMIN(pRangeDest->calWin.skey, pRangeSrc->calWin.skey);
pRangeDest->calWin.ekey = TMAX(pRangeDest->calWin.ekey, pRangeSrc->calWin.ekey);
}
int32_t mergeScanRange(SArray* pRangeArray, SScanRange* pRangeKey, uint64_t gpId, uint64_t uId, int32_t* pIndex, bool* pRes, char* idStr) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t size = taosArrayGetSize(pRangeArray);
int32_t index = binarySearch(pRangeArray, size, pRangeKey, compareRangeKey);
if (index >= 0) {
SScanRange* pFindRangeKey = (SScanRange*) taosArrayGet(pRangeArray, index);
if (scanRangeKeyCmpr(pFindRangeKey, pRangeKey) == 0) {
mergeRangeKey(pFindRangeKey, pRangeKey);
code = putRangeIdInfo(pFindRangeKey, gpId, uId);
QUERY_CHECK_CODE(code, lino, _end);
*pRes = true;
goto _end;
}
}
if (index + 1 < size) {
SScanRange* pFindRangeKey = (SScanRange*) taosArrayGet(pRangeArray, index + 1);
if (scanRangeKeyCmpr(pFindRangeKey, pRangeKey) == 0) {
mergeRangeKey(pFindRangeKey, pRangeKey);
code = putRangeIdInfo(pFindRangeKey, gpId, uId);
QUERY_CHECK_CODE(code, lino, _end);
*pRes = true;
goto _end;
}
}
(*pIndex) = index;
*pRes = false;
_end:
qDebug("===stream===%s mergeScanRange start ts:%" PRId64 ",end ts:%" PRId64 ",group id:%" PRIu64 ", uid:%" PRIu64
", res:%d", idStr, pRangeKey->win.skey, pRangeKey->win.ekey, gpId, uId, *pRes);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t mergeAndSaveScanRange(STableTsDataState* pTsDataState, STimeWindow* pWin, uint64_t gpId, SRecDataInfo* pRecData, int32_t recLen) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SArray* pRangeArray = pTsDataState->pScanRanges;
uint64_t uId = pRecData->tableUid;
int32_t index = 0;
bool merge = false;
SScanRange rangeKey = {.win = *pWin, .calWin = pRecData->calWin, .pUIds = NULL, .pGroupIds = NULL};
code = mergeScanRange(pRangeArray, &rangeKey, gpId, uId, &index, &merge, pTsDataState->pState->pTaskIdStr);
QUERY_CHECK_CODE(code, lino, _end);
if (merge == true) {
goto _end;
}
int32_t size = taosArrayGetSize(pRangeArray);
if (size > MAX_SCAN_RANGE_SIZE) {
SSessionKey sesKey = {.win = *pWin, .groupId = gpId};
void* pVal = NULL;
int32_t len = 0;
int32_t winCode = streamStateSessionGet_rocksdb(pTsDataState->pState, &sesKey, &pVal, &len);
if (winCode != TSDB_CODE_SUCCESS) {
code = streamStateSessionPut_rocksdb(pTsDataState->pState, &sesKey, &uId, sizeof(uint64_t));
} else {
char* pTempBuf = taosMemoryRealloc(pVal, len + sizeof(uint64_t));
QUERY_CHECK_NULL(pTempBuf, code, lino, _end, terrno);
memcpy(pTempBuf+len, &uId, sizeof(uint64_t));
code = streamStateSessionPut_rocksdb(pTsDataState->pState, &sesKey, pTempBuf, len + sizeof(uint64_t));
taosMemFreeClear(pTempBuf);
}
QUERY_CHECK_CODE(code, lino, _end);
goto _end;
}
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
rangeKey.pGroupIds = tSimpleHashInit(8, hashFn);
rangeKey.pUIds = tSimpleHashInit(8, hashFn);
code = putRangeIdInfo(&rangeKey, gpId, uId);
QUERY_CHECK_CODE(code, lino, _end);
if (index < 0) {
index = 0;
}
taosArrayInsert(pRangeArray, index, &rangeKey);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t mergeSimpleHashMap(SSHashObj* pDest, SSHashObj* pSource) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
void* pIte = NULL;
int32_t iter = 0;
while ((pIte = tSimpleHashIterate(pSource, pIte, &iter)) != NULL) {
size_t keyLen = 0;
void* pKey = tSimpleHashGetKey(pIte, &keyLen);
code = tSimpleHashPut(pDest, pKey, keyLen, pIte, sizeof(uint64_t));
QUERY_CHECK_CODE(code, lino, _end);
}
tSimpleHashCleanup(pSource);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t mergeAllScanRange(STableTsDataState* pTsDataState) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamStateCur* pCur = NULL;
SArray* pRangeArray = pTsDataState->pScanRanges;
if (taosArrayGetSize(pRangeArray) < 2) {
return code;
}
for (int32_t i = 0; i < taosArrayGetSize(pRangeArray) - 1;) {
SScanRange* pCurRange = taosArrayGet(pRangeArray, i);
SScanRange* pNextRange = taosArrayGet(pRangeArray, i + 1);
if (scanRangeKeyCmpr(pCurRange, pNextRange) == 0) {
pCurRange->win.skey = TMIN(pCurRange->win.skey, pNextRange->win.skey);
pCurRange->win.ekey = TMAX(pCurRange->win.ekey, pNextRange->win.ekey);
code = mergeSimpleHashMap(pCurRange->pGroupIds, pNextRange->pGroupIds);
QUERY_CHECK_CODE(code, lino, _end);
code = mergeSimpleHashMap(pCurRange->pUIds, pNextRange->pUIds);
QUERY_CHECK_CODE(code, lino, _end);
taosArrayRemove(pRangeArray, i+1);
continue;
}
i++;
}
int32_t winRes = TSDB_CODE_SUCCESS;
pCur = streamStateSessionSeekToLast_rocksdb(pTsDataState->pState, INT64_MAX);
while (winRes == TSDB_CODE_SUCCESS) {
void* pVal = NULL;
int32_t vlen = 0;
SSessionKey key = {0};
winRes = streamStateSessionGetKVByCur_rocksdb(pTsDataState->pState, pCur, &key, &pVal, &vlen);
if (winRes != TSDB_CODE_SUCCESS) {
break;
}
int32_t index = 0;
bool merge = false;
SScanRange tmpRange = {.win = key.win, .pUIds = NULL, .pGroupIds = NULL};
int32_t num = vlen / sizeof(uint64_t);
uint64_t* pUids = (uint64_t*) pVal;
for (int32_t i = 0; i < num; i++) {
code = mergeScanRange(pRangeArray, &tmpRange, key.groupId, pUids[i], &index, &merge, pTsDataState->pState->pTaskIdStr);
QUERY_CHECK_CODE(code, lino, _end);
}
if (merge == true) {
code = streamStateSessionDel_rocksdb(pTsDataState->pState, &key);
QUERY_CHECK_CODE(code, lino, _end);
}
}
_end:
streamStateFreeCur(pCur);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t popScanRange(STableTsDataState* pTsDataState, SScanRange* pRange) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamStateCur* pCur = NULL;
SArray* pRangeArray = pTsDataState->pScanRanges;
if (taosArrayGetSize(pRangeArray) > 0) {
(*pRange) = *(SScanRange*) taosArrayPop(pRangeArray);
goto _end;
}
{
pCur = streamStateSessionSeekToLast_rocksdb(pTsDataState->pState, INT64_MAX);
SRecDataInfo* pRecVal = NULL;
int32_t vlen = 0;
SSessionKey key = {0};
int32_t winRes = streamStateSessionGetKVByCur_rocksdb(pTsDataState->pState, pCur, &key, (void**)&pRecVal, &vlen);
if (winRes != TSDB_CODE_SUCCESS) {
goto _end;
}
qDebug("===stream===get scan range from disc start ts:%" PRId64 ",end ts:%" PRId64 ",group id:%" PRIu64,
key.win.skey, key.win.ekey, key.groupId);
pRange->win = key.win;
pRange->calWin = pRecVal->calWin;
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
if (pRange->pGroupIds == NULL) {
pRange->pGroupIds = tSimpleHashInit(8, hashFn);
}
if (pRange->pUIds == NULL) {
pRange->pUIds = tSimpleHashInit(8, hashFn);
}
code = putRangeIdInfo(pRange, key.groupId, pRecVal->tableUid);
QUERY_CHECK_CODE(code, lino, _end);
code = streamStateSessionDel_rocksdb(pTsDataState->pState, &key);
QUERY_CHECK_CODE(code, lino, _end);
}
_end:
streamStateFreeCur(pCur);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t getNLastSessionStateKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pCur->pHashData == NULL) {
return TSDB_CODE_FAILED;
}
SArray* pWinStates = *((void**)pCur->pHashData);
int32_t size = taosArrayGetSize(pWinStates);
if (size == 0) {
return TSDB_CODE_FAILED;
}
int32_t i = TMAX(size - num, 0);
for ( ; i < size; i++) {
SRowBuffPos* pPos = taosArrayGetP(pWinStates, i);
QUERY_CHECK_NULL(pPos, code, lino, _end, terrno);
SResultWindowInfo winInfo = {0};
winInfo.pStatePos = pPos;
winInfo.sessionWin = *(SSessionKey*)pPos->pKey;
void* pTempRes = taosArrayPush(pRes, &winInfo);
QUERY_CHECK_NULL(pTempRes, code, lino, _end, terrno);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
bool hasSessionState(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, bool* pIsLast) {
SSHashObj* pRowStateBuff = getRowStateBuff(pFileState);
void** ppBuff = (void**)tSimpleHashGet(pRowStateBuff, &pKey->groupId, sizeof(uint64_t));
if (ppBuff == NULL) {
return false;
}
SArray* pWinStates = (SArray*)(*ppBuff);
int32_t size = taosArrayGetSize(pWinStates);
int32_t index = binarySearch(pWinStates, size, pKey, sessionStateKeyCompare);
SRowBuffPos* pPos = NULL;
if (index >= 0) {
pPos = taosArrayGetP(pWinStates, index);
if (inSessionWindow(pPos->pKey, pKey->win.skey, gap)) {
*pKey = *((SSessionKey*)pPos->pKey);
*pIsLast = (index == size - 1);
return true;
}
}
if (index + 1 < size) {
pPos = taosArrayGetP(pWinStates, index + 1);
if (inSessionWindow(pPos->pKey, pKey->win.skey, gap) ||
(pKey->win.ekey != INT64_MIN && inSessionWindow(pPos->pKey, pKey->win.ekey, gap))) {
*pKey = *((SSessionKey*)pPos->pKey);
*pIsLast = (index + 1 == size - 1);
return true;
}
}
return false;
}

3
source/libs/stream/src/streamSliceState.c
View File

@ -69,7 +69,8 @@ int32_t getHashSortRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, vo
QUERY_CHECK_CODE(code, lino, _end);
}
code = addSearchItem(pFileState, pWinStates, pKey);
bool isEnd = false;
code = addSearchItem(pFileState, pWinStates, pKey, &isEnd);
QUERY_CHECK_CODE(code, lino, _end);
_end:

31
source/libs/stream/src/streamStartHistory.c
View File

@ -157,6 +157,10 @@ int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) {
int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask) {
// set the state to be ready
int32_t code = streamTaskSetReady(pTask);
if (pTask->info.fillHistory == STREAM_RECALCUL_TASK) {
// if this task is used as the recalculate task, do nothing
} else {
if (code == 0) {
code = streamTaskSetRangeStreamCalc(pTask);
}
@ -166,6 +170,7 @@ int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask) {
stDebug("s-task:%s fill-history task enters into scan-history data stage, status:%s", pTask->id.idStr, p.name);
code = streamTaskStartScanHistory(pTask);
}
}
// NOTE: there will be a deadlock if launch fill history here.
// start the related fill-history task, when current task is ready
@ -191,7 +196,7 @@ int32_t streamSetParamForStreamScannerStep2(SStreamTask* pTask, SVersionRange* p
return qStreamSourceScanParamForHistoryScanStep2(pTask->exec.pExecutor, pVerRange, pWindow);
}
// an fill history task needs to be started.
// A fill history task needs to be started.
int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) {
SStreamMeta* pMeta = pTask->pMeta;
STaskExecStatisInfo* pExecInfo = &pTask->execInfo;
@ -261,6 +266,7 @@ void checkFillhistoryTaskStatus(SStreamTask* pTask, SStreamTask* pHTask) {
// the query version range should be limited to the already processed data
pHTask->execInfo.checkTs = taosGetTimestampMs();
if (pHTask->info.fillHistory == STREAM_HISTORY_TASK) {
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
stDebug("s-task:%s set the launch condition for fill-history s-task:%s, window:%" PRId64 " - %" PRId64
" verRange:%" PRId64 " - %" PRId64 ", init:%" PRId64,
@ -270,10 +276,16 @@ void checkFillhistoryTaskStatus(SStreamTask* pTask, SStreamTask* pHTask) {
stDebug("s-task:%s no fill-history condition for non-source task:%s", pTask->id.idStr, pHTask->id.idStr);
}
// check if downstream tasks have been ready
int32_t code = streamTaskHandleEvent(pHTask->status.pSM, TASK_EVENT_INIT_SCANHIST);
if (code) {
stError("s-task:%s handle event init_scanhist failed", pTask->id.idStr);
stError("s-task:%s rel hist task:%s handle event init_scanhist failed", pTask->id.idStr, pHTask->id.idStr);
}
} else { // check if downstream tasks have been ready
stDebug("s-task:%s start rel recalculate task:%s, handle event init", pTask->id.idStr, pHTask->id.idStr);
int32_t code = streamTaskHandleEvent(pHTask->status.pSM, TASK_EVENT_INIT);
if (code) {
stError("s-task:%s rel hist task:%s handle event init failed", pTask->id.idStr, pHTask->id.idStr);
}
}
}
@ -281,7 +293,6 @@ void notRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo,
SStreamMeta* pMeta = pTask->pMeta;
SHistoryTaskInfo* pHTaskInfo = &pTask->hTaskInfo;
// int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
int32_t code = streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false);
if (code) {
@ -300,7 +311,6 @@ void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, i
SHistoryTaskInfo* pHTaskInfo = &pTask->hTaskInfo;
if (streamTaskShouldStop(pTask)) { // record the failure
// int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:0x%" PRIx64 " stopped, not launch rel history task:0x%" PRIx64, pInfo->id.taskId,
pInfo->hTaskId.taskId);
@ -537,7 +547,7 @@ int32_t streamTaskSetRangeStreamCalc(SStreamTask* pTask) {
SDataRange* pRange = &pTask->dataRange;
if (!HAS_RELATED_FILLHISTORY_TASK(pTask)) {
if (pTask->info.fillHistory == 1) {
if (pTask->info.fillHistory == STREAM_HISTORY_TASK) {
stDebug("s-task:%s fill-history task, time window:%" PRId64 "-%" PRId64 ", verRange:%" PRId64 "-%" PRId64,
pTask->id.idStr, pRange->window.skey, pRange->window.ekey, pRange->range.minVer, pRange->range.maxVer);
} else {
@ -548,8 +558,8 @@ int32_t streamTaskSetRangeStreamCalc(SStreamTask* pTask) {
}
return TSDB_CODE_SUCCESS;
} else {
if (pTask->info.fillHistory != 0) {
} else { // has related helper tasks
if (pTask->info.fillHistory != STREAM_NORMAL_TASK) {
stError("s-task:%s task should not be fill-history task, internal error", pTask->id.idStr);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
@ -558,6 +568,10 @@ int32_t streamTaskSetRangeStreamCalc(SStreamTask* pTask) {
return TSDB_CODE_SUCCESS;
}
if (pTask->info.trigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
stDebug("s-task:%s level:%d related recalculate task exist, do nothing", pTask->id.idStr, pTask->info.taskLevel);
return 0;
} else {
stDebug("s-task:%s level:%d related fill-history task exists, stream task timeWindow:%" PRId64 " - %" PRId64
", verRang:%" PRId64 " - %" PRId64,
pTask->id.idStr, pTask->info.taskLevel, pRange->window.skey, pRange->window.ekey, pRange->range.minVer,
@ -568,6 +582,7 @@ int32_t streamTaskSetRangeStreamCalc(SStreamTask* pTask) {
return streamSetParamForStreamScannerStep2(pTask, &verRange, &win);
}
}
}
void doExecScanhistoryInFuture(void* param, void* tmrId) {
int64_t taskRefId = *(int64_t*) param;

2
source/libs/stream/src/streamStartTask.c
View File

@ -147,7 +147,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) {
// negotiate the consensus checkpoint id for current task
code = streamTaskSendNegotiateChkptIdMsg(pTask);
// this task may has no checkpoint, but others tasks may generate checkpoint already?
// this task may have no checkpoint, but others tasks may generate checkpoint already?
streamMetaReleaseTask(pMeta, pTask);
}

248
source/libs/stream/src/streamState.c
View File

@ -98,6 +98,59 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) {
return winKeyCmprImpl(&pWin1->key, &pWin2->key);
}
SStreamState* streamStateRecalatedOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamTask* pStreamTask = pTask;
if (!streamTaskShouldRecalated(pStreamTask)) {
stError("failed to open recalation stream state %s", path);
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState));
stDebug("open stream state %p, %s", pState, path);
if (pState == NULL) {
code = terrno;
QUERY_CHECK_CODE(code, lino, _end);
}
pState->pTdbState = taosMemoryCalloc(1, sizeof(STdbState));
if (pState->pTdbState == NULL) {
streamStateDestroy(pState, true);
code = terrno;
QUERY_CHECK_CODE(code, lino, _end);
}
pState->streamId = streamId;
pState->taskId = taskId;
pState->pTdbState->recalc = 1;
TAOS_UNUSED(tsnprintf(pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr), "0x%" PRIx64 "-0x%x-%s",
pState->streamId, pState->taskId, "recalc"));
code = streamTaskSetDb(pStreamTask->pMeta, pTask, pState->pTdbState->idstr, 1);
QUERY_CHECK_CODE(code, lino, _end);
SStreamMeta* pMeta = pStreamTask->pMeta;
pState->pTdbState->pOwner = pTask;
pState->pFileState = NULL;
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT);
pState->parNameMap = tSimpleHashInit(1024, hashFn);
QUERY_CHECK_NULL(pState->parNameMap, code, lino, _end, terrno);
stInfo("open state %p on backend %p 0x%" PRIx64 "-%d succ", pState, pMeta->streamBackend, pState->streamId,
pState->taskId);
return pState;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("0x%x %s recalated failed at line %d since %s", taskId, __func__, lino, tstrerror(code));
}
return NULL;
}
SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
@ -105,6 +158,9 @@ SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, i
SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState));
stDebug("open stream state %p, %s", pState, path);
TAOS_UNUSED(tsnprintf(pState->pTaskIdStr, sizeof(pState->pTaskIdStr), "TID:0x%x QID:0x%" PRIx64,
taskId, streamId));
if (pState == NULL) {
code = terrno;
QUERY_CHECK_CODE(code, lino, _end);
@ -122,8 +178,8 @@ SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, i
pState->taskId = taskId;
TAOS_UNUSED(tsnprintf(pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr), "0x%" PRIx64 "-0x%x",
pState->streamId, pState->taskId));
code = streamTaskSetDb(pStreamTask->pMeta, pTask, pState->pTdbState->idstr);
// recal id + cal
code = streamTaskSetDb(pStreamTask->pMeta, pTask, pState->pTdbState->idstr, 0);
QUERY_CHECK_CODE(code, lino, _end);
SStreamMeta* pMeta = pStreamTask->pMeta;
@ -210,8 +266,8 @@ int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, in
return addRowBuffIfNotExist(pState->pFileState, (void*)key, sizeof(SWinKey), pVal, pVLen, pWinCode);
}
bool streamStateCheck(SStreamState* pState, const SWinKey* key) {
return hasRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey));
bool streamStateCheck(SStreamState* pState, const SWinKey* pKey, bool hasLimit, bool* pIsLast) {
return hasRowBuff(pState->pFileState, pKey, hasLimit, pIsLast);
}
int32_t streamStateGetByPos(SStreamState* pState, void* pos, void** pVal) {
@ -291,21 +347,41 @@ int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, voi
return code;
}
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
int32_t streamStateCreate(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) {
return createRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), pVal, pVLen);
}
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* pKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
code = streamStateGet(pState, key, pVal, pVLen, pWinCode);
QUERY_CHECK_CODE(code, lino, _end);
bool isEnd = false;
SSHashObj* pSearchBuff = getSearchBuff(pState->pFileState);
if (pSearchBuff != NULL) {
SArray* pWinStates = NULL;
code = addArrayBuffIfNotExist(pSearchBuff, key->groupId, &pWinStates);
code = addArrayBuffIfNotExist(pSearchBuff, pKey->groupId, &pWinStates);
QUERY_CHECK_CODE(code, lino, _end);
code = addSearchItem(pState->pFileState, pWinStates, key);
// recover
if (taosArrayGetSize(pWinStates) == 0 && needClearDiskBuff(pState->pFileState)) {
code = recoverHashSortBuff(pState->pFileState, pWinStates, pKey->groupId);
QUERY_CHECK_CODE(code, lino, _end);
}
code = addSearchItem(pState->pFileState, pWinStates, pKey, &isEnd);
QUERY_CHECK_CODE(code, lino, _end);
}
if (isEnd) {
code = streamStateCreate(pState, pKey, pVal, pVLen);
QUERY_CHECK_CODE(code, lino, _end);
(*pWinCode) = TSDB_CODE_FAILED;
} else {
code = streamStateGet(pState, pKey, pVal, pVLen, pWinCode);
QUERY_CHECK_CODE(code, lino, _end);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
@ -441,8 +517,11 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, cons
}
SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, const SSessionKey* key) {
if (pState->pFileState != NULL) {
return sessionWinStateSeekKeyCurrentNext(pState->pFileState, key);
}
return streamStateSessionSeekKeyCurrentNext_rocksdb(pState, key);
}
SStreamStateCur *streamStateSessionSeekKeyPrev(SStreamState *pState, const SSessionKey *key) {
return sessionWinStateSeekKeyPrev(pState->pFileState, key);
@ -457,8 +536,11 @@ SStreamStateCur* streamStateCountSeekKeyPrev(SStreamState* pState, const SSessio
}
int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen) {
if (pCur != NULL && pCur->pStreamFileState != NULL) {
return sessionWinStateGetKVByCur(pCur, pKey, pVal, pVLen);
}
return streamStateSessionGetKVByCur_rocksdb(NULL, pCur, pKey, pVal, pVLen);
}
void streamStateSessionClear(SStreamState* pState) {
sessionWinStateClear(pState->pFileState);
@ -486,6 +568,10 @@ int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, ch
int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pState->parNameMap == NULL) {
return TSDB_CODE_SUCCESS;
}
if (tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t)) == NULL) {
if (tSimpleHashGetSize(pState->parNameMap) < MAX_TABLE_NAME_NUM) {
code = tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN);
@ -505,6 +591,11 @@ _end:
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pState->parNameMap == NULL) {
(*pWinCode) = TSDB_CODE_FAILED;
return code;
}
void* pStr = tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t));
if (!pStr) {
if (onlyCache && tSimpleHashGetSize(pState->parNameMap) < MAX_TABLE_NAME_NUM) {
@ -535,6 +626,10 @@ _end:
}
int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId) {
if (pState->parNameMap == NULL) {
return TSDB_CODE_INVALID_PARA;
}
int32_t code = tSimpleHashRemove(pState->parNameMap, &groupId, sizeof(int64_t));
if (TSDB_CODE_SUCCESS != code) {
qWarn("failed to remove parname from cache, code:%d", code);
@ -546,6 +641,13 @@ int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId) {
return TSDB_CODE_SUCCESS;
}
void streamStateSetParNameInvalid(SStreamState* pState) {
if (pState != NULL) {
tSimpleHashCleanup(pState->parNameMap);
pState->parNameMap = NULL;
}
}
void streamStateDestroy(SStreamState* pState, bool remove) {
streamFileStateDestroy(pState->pFileState);
tSimpleHashCleanup(pState->parNameMap);
@ -619,9 +721,135 @@ int32_t streamStateGroupGetKVByCur(SStreamStateCur* pCur, int64_t* pKey, void**
return streamFileStateGroupGetKVByCur(pCur, pKey, pVal, pVLen);
}
void streamStateClearExpiredState(SStreamState* pState) { clearExpiredState(pState->pFileState); }
void streamStateClearExpiredState(SStreamState* pState, int32_t numOfKeep, TSKEY minTs) {
qDebug("===stream=== clear stream state. keep:%d, ts:%" PRId64, numOfKeep, minTs);
if (numOfKeep == 0) {
streamFileStateClear(pState->pFileState);
SSHashObj* pSearchBuff = getSearchBuff(pState->pFileState);
void* pIte = NULL;
int32_t iter = 0;
while ((pIte = tSimpleHashIterate(pSearchBuff, pIte, &iter)) != NULL) {
SArray* pWinStates = *((void**)pIte);
taosArrayClear(pWinStates);
}
return;
}
clearExpiredState(pState->pFileState, numOfKeep, minTs);
}
int32_t streamStateGetPrev(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode) {
return getRowStatePrevRow(pState->pFileState, pKey, pResKey, pVal, pVLen, pWinCode);
}
int32_t streamStateGetAllPrev(SStreamState* pState, const SWinKey* pKey, SArray* pResArray, int32_t maxNum) {
return getRowStateAllPrevRow(pState->pFileState, pKey, pResArray, maxNum);
}
int32_t streamStateGetAndSetTsData(STableTsDataState* pState, uint64_t tableUid, TSKEY* pCurTs, void** ppCurPkVal,
TSKEY lastTs, void* pLastPkVal, int32_t lastPkLen, int32_t* pWinCode) {
return getAndSetTsData(pState, tableUid, pCurTs, ppCurPkVal, lastTs, pLastPkVal, lastPkLen, pWinCode);
}
int32_t streamStateTsDataCommit(STableTsDataState* pState) {
int32_t code = doTsDataCommit(pState);
if (code != TSDB_CODE_SUCCESS) return code;
return doRangeDataCommit(pState);
}
int32_t streamStateInitTsDataState(STableTsDataState** ppTsDataState, int8_t pkType, int32_t pkLen, void* pState,
void* pOtherState) {
return initTsDataState(ppTsDataState, pkType, pkLen, pState, pOtherState);
}
void streamStateDestroyTsDataState(STableTsDataState* pTsDataState) { destroyTsDataState(pTsDataState); }
int32_t streamStateRecoverTsData(STableTsDataState* pTsDataState) { return recoverTsData(pTsDataState); }
SStreamStateCur* streamStateGetLastStateCur(SStreamState* pState) { return getLastStateCur(pState->pFileState, getSearchBuff); }
void streamStateLastStateCurNext(SStreamStateCur* pCur) { moveLastStateCurNext(pCur, getSearchBuff); }
int32_t streamStateNLastStateGetKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes) {
return getNLastStateKVByCur(pCur, num, pRes);
}
SStreamStateCur* streamStateGetLastSessionStateCur(SStreamState* pState) { return getLastStateCur(pState->pFileState, getRowStateBuff); }
void streamStateLastSessionStateCurNext(SStreamStateCur* pCur) { moveLastStateCurNext(pCur, getRowStateBuff); }
int32_t streamStateNLastSessionStateGetKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes) {
return getNLastSessionStateKVByCur(pCur, num, pRes);
}
int32_t streamStateReloadTsDataState(STableTsDataState* pTsDataState) { return reloadTsDataState(pTsDataState); }
int32_t streamStateMergeAndSaveScanRange(STableTsDataState* pTsDataState, STimeWindow* pWin, uint64_t gpId,
SRecDataInfo* pRecData, int32_t len) {
return mergeAndSaveScanRange(pTsDataState, pWin, gpId, pRecData, len);
}
int32_t streamStateMergeAllScanRange(STableTsDataState* pTsDataState) { return mergeAllScanRange(pTsDataState); }
int32_t streamStatePopScanRange(STableTsDataState* pTsDataState, SScanRange* pRange) {
return popScanRange(pTsDataState, pRange);
}
int32_t streamStateGetNumber(SStreamState* pState) { return pState->number; }
int32_t streamStateDeleteInfo(SStreamState* pState, void* pKey, int32_t keyLen) {
return streamDefaultDel_rocksdb(pState, pKey);
}
int32_t streamStateSessionSaveToDisk(STableTsDataState* pTblState, SSessionKey* pKey, SRecDataInfo* pVal,
int32_t vLen) {
SStreamState* pState = pTblState->pState;
qDebug("===stream===%s save recalculate range.recId:%d. start:%" PRId64 ",end:%" PRId64 ",groupId:%" PRIu64
". cal start:%" PRId64 ",cal end:%" PRId64 ",tbl uid:%" PRIu64 ",data version:%" PRId64 ",mode:%d",
pState->pTaskIdStr, pState->number, pKey->win.skey, pKey->win.ekey, pKey->groupId, pVal->calWin.skey,
pVal->calWin.ekey, pVal->tableUid, pVal->dataVersion, pVal->mode);
return saveRecInfoToDisk(pTblState, pKey, pVal, vLen);
}
int32_t streamStateFlushReaminInfoToDisk(STableTsDataState* pTblState) {
return flushRemainRecInfoToDisk(pTblState);
}
int32_t streamStateSessionDeleteAll(SStreamState* pState) {
SSessionKey key = {.win.skey = INT64_MIN, .win.ekey = INT64_MIN, .groupId = 0};
while (1) {
SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext_rocksdb(pState, &key);
SSessionKey delKey = {0};
int32_t winRes = streamStateSessionGetKVByCur_rocksdb(pState, pCur, &delKey, NULL, 0);
if (winRes != TSDB_CODE_SUCCESS) {
streamStateFreeCur(pCur);
break;
}
streamStateSessionDel_rocksdb(pState, &delKey);
streamStateFreeCur(pCur);
}
return TSDB_CODE_SUCCESS;
}
int32_t streamStateSetRecFlag(SStreamState* pState, const void* pKey, int32_t keyLen, int32_t mode) {
return setStateRecFlag(pState->pFileState, pKey, keyLen, mode);
}
int32_t streamStateGetRecFlag(SStreamState* pState, const void* pKey, int32_t keyLen, int32_t* pMode) {
return getStateRecFlag(pState->pFileState, pKey, keyLen, pMode);
}
void streamStateClearExpiredSessionState(SStreamState* pState, int32_t numOfKeep, TSKEY minTs, SSHashObj* pFlushGroup) {
if (numOfKeep == 0) {
void* pBuff = getRowStateBuff(pState->pFileState);
tSimpleHashSetFreeFp(pBuff, freeArrayPtr);
streamFileStateClear(pState->pFileState);
return;
}
clearExpiredSessionState(pState->pFileState, numOfKeep, minTs, pFlushGroup);
}
bool streamStateCheckSessionState(SStreamState* pState, SSessionKey* pKey, TSKEY gap, bool* pIsLast) {
return hasSessionState(pState->pFileState, pKey, gap, pIsLast);
}

43
source/libs/stream/src/streamTask.c
View File

@ -17,12 +17,12 @@
#include "osDir.h"
#include "osMemory.h"
#include "streamInt.h"
#include "streamMsg.h"
#include "streamsm.h"
#include "tmisce.h"
#include "tstream.h"
#include "ttimer.h"
#include "wal.h"
#include "streamMsg.h"
static void streamTaskDestroyUpstreamInfo(SUpstreamInfo* pUpstreamInfo);
static int32_t streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdated);
@ -100,13 +100,14 @@ static SStreamUpstreamEpInfo* createStreamTaskEpInfo(const SStreamTask* pTask) {
pEpInfo->nodeId = pTask->info.nodeId;
pEpInfo->taskId = pTask->id.taskId;
pEpInfo->stage = -1;
pEpInfo->lastMsgId = -1;
return pEpInfo;
}
int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int32_t trigger,
int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, EStreamTaskType type, int32_t trigger,
int64_t triggerParam, SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5,
SStreamTask** p) {
int8_t hasAggTasks, SStreamTask** p) {
*p = NULL;
SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask));
@ -121,8 +122,9 @@ int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool
pTask->id.streamId = streamId;
pTask->info.taskLevel = taskLevel;
pTask->info.fillHistory = fillHistory;
pTask->info.fillHistory = type;
pTask->info.trigger = trigger;
pTask->info.hasAggTasks = hasAggTasks;
pTask->info.delaySchedParam = triggerParam;
pTask->subtableWithoutMd5 = subtableWithoutMd5;
@ -146,13 +148,14 @@ int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool
}
pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
pTask->status.taskStatus = fillHistory ? TASK_STATUS__SCAN_HISTORY : TASK_STATUS__READY;
pTask->status.taskStatus =
(pTask->info.fillHistory == STREAM_HISTORY_TASK) ? TASK_STATUS__SCAN_HISTORY : TASK_STATUS__READY;
pTask->inputq.status = TASK_INPUT_STATUS__NORMAL;
pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL;
pTask->taskCheckInfo.pList = taosArrayInit(4, sizeof(SDownstreamStatusInfo));
if (fillHistory && !hasFillhistory) {
if ((pTask->info.fillHistory == STREAM_HISTORY_TASK) && !hasFillhistory) {
stError("s-task:0x%x create task failed, due to inconsistent fill-history flag", pTask->id.taskId);
return TSDB_CODE_INVALID_PARA;
}
@ -347,6 +350,7 @@ void streamFreeTaskState(SStreamTask* pTask, int8_t remove) {
taskDbRemoveRef(pTask->pBackend);
pTask->pBackend = NULL;
pTask->pState = NULL;
} else {
stDebug("s-task:0x%x task state is NULL, may del backend:%s", pTask->id.taskId,
pTask->backendPath ? pTask->backendPath : "NULL");
@ -362,6 +366,31 @@ void streamFreeTaskState(SStreamTask* pTask, int8_t remove) {
taosMemoryFree(pTask->backendPath);
pTask->backendPath = NULL;
}
// clear recal backend
if (pTask->pRecalState != NULL) {
stDebug("s-task:0x%x start to free recal task state", pTask->id.taskId);
streamStateClose(pTask->pRecalState, remove);
if (remove) taskDbSetClearFileFlag(pTask->pRecalBackend);
taskDbRemoveRef(pTask->pRecalBackend);
pTask->pRecalBackend = NULL;
pTask->pRecalState = NULL;
} // else {
// stDebug("s-task:0x%x task state is NULL, may del backend:%s", pTask->id.taskId,
// pTask->backendPath ? pTask->backendPath : "NULL");
// if (remove) {
// if (pTask->backendPath != NULL) {
// stDebug("s-task:0x%x task state is NULL, do del backend:%s", pTask->id.taskId, pTask->backendPath);
// taosRemoveDir(pTask->backendPath);
// }
// }
//}
// if (pTask->backendPath != NULL) {
// taosMemoryFree(pTask->backendPath);
// pTask->backendPath = NULL;
// }
}
static void setInitialVersionInfo(SStreamTask* pTask, int64_t ver) {
@ -1477,6 +1506,7 @@ int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
TAOS_CHECK_EXIT(tEncodeCStrWithLen(pEncoder, pTask->reserve, sizeof(pTask->reserve) - 1));
if (pTask->ver >= SSTREAM_TASK_ADD_NOTIFY_VER) {
TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->info.hasAggTasks));
TAOS_CHECK_EXIT(tEncodeStreamNotifyInfo(pEncoder, &pTask->notifyInfo));
}
@ -1579,6 +1609,7 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->reserve));
if (pTask->ver >= SSTREAM_TASK_ADD_NOTIFY_VER) {
TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->info.hasAggTasks));
TAOS_CHECK_EXIT(tDecodeStreamNotifyInfo(pDecoder, &pTask->notifyInfo));
}

2
source/libs/stream/src/streamUpdate.c
View File

@ -58,7 +58,7 @@ int32_t getKeyBuff(TSKEY ts, int64_t tbUid, void* pVal, int32_t len, char* buff)
return sizeof(TSKEY) + sizeof(int64_t) + len;
}
int32_t getValueBuff(TSKEY ts, char* pVal, int32_t len, char* buff) {
static int32_t getValueBuff(TSKEY ts, char* pVal, int32_t len, char* buff) {
*(TSKEY*)buff = ts;
if (len == 0) {
return sizeof(TSKEY);

748
source/libs/stream/src/tstreamFileState.c
View File

@ -19,6 +19,7 @@
#include "streamBackendRocksdb.h"
#include "taos.h"
#include "tcommon.h"
#include "tcompare.h"
#include "thash.h"
#include "tsimplehash.h"
@ -32,6 +33,15 @@
#define MAX_GROUP_ID_NUM 200000
#define NUM_OF_CACHE_WIN 64
#define MAX_NUM_OF_CACHE_WIN 128
#define MIN_NUM_OF_SORT_CACHE_WIN 40960
#define BATCH_LIMIT 256
#define DEFAULT_STATE_MAP_CAPACITY 10240
#define MAX_STATE_MAP_SIZE 10240000
#define SET_TSDATA_FLAG(ptr, len) ((*(char*)POINTER_SHIFT(ptr, (len - 1))) |= 1)
#define UNSET_TSDATA_FLAG(ptr, len) ((*(char*)POINTER_SHIFT(ptr, (len - 1))) &= 0)
#define HAS_TSDATA_FLAG(ptr, len) ((*(char*)POINTER_SHIFT(ptr, (len - 1))) & 1)
#define TASK_KEY "streamFileState"
#define STREAM_STATE_INFO_NAME "StreamStateCheckPoint"
@ -57,6 +67,7 @@ struct SStreamFileState {
void* searchBuff;
SSHashObj* pGroupIdMap;
bool hasFillCatch;
SSHashObj* pRecFlagMap;
_state_buff_cleanup_fn stateBuffCleanupFn;
_state_buff_remove_fn stateBuffRemoveFn;
@ -76,10 +87,23 @@ int fillStateKeyCompare(const void* pWin1, const void* pDatas, int pos) {
return winKeyCmprImpl((SWinKey*)pWin1, pWin2);
}
int fillTSKeyCompare(const void* pKey1, const void* pDatas, int pos) {
SWinKey* pWin1 = (SWinKey*)pKey1;
SWinKey* pWin2 = taosArrayGet(pDatas, pos);
if (pWin1->ts > pWin2->ts) {
return 1;
} else if (pWin1->ts < pWin2->ts) {
return -1;
}
return 0;
}
int32_t stateHashBuffRemoveFn(void* pBuff, const void* pKey, size_t keyLen) {
SRowBuffPos** pos = tSimpleHashGet(pBuff, pKey, keyLen);
if (pos) {
(*pos)->beFlushed = true;
(*pos)->invalid = true;
}
return tSimpleHashRemove(pBuff, pKey, keyLen);
}
@ -254,6 +278,10 @@ int32_t streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize,
pFileState->pGroupIdMap = tSimpleHashInit(1024, hashFn);
QUERY_CHECK_NULL(pFileState->pGroupIdMap, code, lino, _end, terrno);
pFileState->pRecFlagMap = tSimpleHashInit(1024, hashFn);
QUERY_CHECK_NULL(pFileState->pRecFlagMap, code, lino, _end, terrno);
pFileState->hasFillCatch = true;
if (type == STREAM_STATE_BUFF_HASH || type == STREAM_STATE_BUFF_HASH_SEARCH) {
@ -327,6 +355,7 @@ void streamFileStateDestroy(SStreamFileState* pFileState) {
pFileState->stateBuffCleanupFn(pFileState->rowStateBuff);
sessionWinStateCleanup(pFileState->searchBuff);
tSimpleHashCleanup(pFileState->pGroupIdMap);
tSimpleHashCleanup(pFileState->pRecFlagMap);
taosMemoryFree(pFileState);
}
@ -453,7 +482,7 @@ int32_t popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList,
}
}
qInfo("stream state flush %d rows to disk. is used:%d", listNEles(pFlushList), used);
qInfo("%s stream state flush %d rows to disk. is used:%d", pFileState->id, listNEles(pFlushList), used);
_end:
if (code != TSDB_CODE_SUCCESS) {
@ -524,6 +553,57 @@ int32_t clearRowBuff(SStreamFileState* pFileState) {
return TSDB_CODE_SUCCESS;
}
int32_t clearFlushedRowBuffByFlag(SStreamFileState* pFileState, uint64_t max) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
uint64_t i = 0;
SListIter iter = {0};
tdListInitIter(pFileState->usedBuffs, &iter, TD_LIST_FORWARD);
SListNode* pNode = NULL;
while ((pNode = tdListNext(&iter)) != NULL && i < max) {
SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data;
if (pPos->invalid) {
if (!pPos->beUsed) {
SListNode* tmp = tdListPopNode(pFileState->usedBuffs, pNode);
taosMemoryFreeClear(tmp);
if (pPos->pRowBuff) {
i++;
}
code = putFreeBuff(pFileState, pPos);
QUERY_CHECK_CODE(code, lino, _end);
destroyRowBuffPos(pPos);
}
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t clearRowBuffNonFlush(SStreamFileState* pFileState) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pFileState->deleteMark != INT64_MAX) {
clearExpiredRowBuff(pFileState, pFileState->maxTs - pFileState->deleteMark, false);
}
uint64_t num = (uint64_t)(pFileState->curRowCount * FLUSH_RATIO);
num = TMAX(num, FLUSH_NUM);
code = clearFlushedRowBuffByFlag(pFileState, num);
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
void* getFreeBuff(SStreamFileState* pFileState) {
SList* lists = pFileState->freeBuffs;
int32_t buffSize = pFileState->rowSize;
@ -603,6 +683,7 @@ SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState) {
newPos->beFlushed = false;
newPos->needFree = false;
newPos->beUpdated = true;
newPos->invalid = false;
return newPos;
_error:
@ -664,6 +745,35 @@ _end:
return code;
}
int32_t createRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
pFileState->maxTs = TMAX(pFileState->maxTs, pFileState->getTs(pKey));
SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState);
if (!pNewPos || !pNewPos->pRowBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
QUERY_CHECK_CODE(code, lino, _end);
}
memcpy(pNewPos->pKey, pKey, keyLen);
code = tSimpleHashPut(pFileState->rowStateBuff, pKey, keyLen, &pNewPos, POINTER_BYTES);
QUERY_CHECK_CODE(code, lino, _end);
if (pVal) {
*pVLen = pFileState->rowSize;
*pVal = pNewPos;
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) {
int32_t code_buff = pFileState->stateBuffRemoveFn(pFileState->rowStateBuff, pKey, keyLen);
qTrace("%s at line %d res:%d", __func__, __LINE__, code_buff);
@ -775,12 +885,39 @@ _end:
return code;
}
bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen) {
SRowBuffPos** pos = tSimpleHashGet(pFileState->rowStateBuff, pKey, keyLen);
if (pos) {
return true;
bool hasRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, bool hasLimit, bool* pIsLast) {
bool res = false;
if (pIsLast != NULL) {
(*pIsLast) = false;
}
return false;
SRowBuffPos** pos = tSimpleHashGet(pFileState->rowStateBuff, pKey, sizeof(SWinKey));
if (pos) {
res = true;
}
void* pSearchBuff = getSearchBuff(pFileState);
if (pSearchBuff != NULL) {
void** ppBuff = (void**)tSimpleHashGet(pSearchBuff, &pKey->groupId, sizeof(uint64_t));
if (ppBuff != NULL) {
SArray* pWinStates = (SArray*)(*ppBuff);
if (pIsLast != NULL) {
SWinKey* pLastKey = (SWinKey*) taosArrayGetLast(pWinStates);
*pIsLast = (winKeyCmprImpl(pKey, pLastKey) == 0);
}
if (hasLimit && taosArrayGetSize(pWinStates) <= MIN_NUM_OF_SORT_CACHE_WIN) {
res = true;
}
if (qDebugFlag & DEBUG_DEBUG) {
SWinKey* fistKey = (SWinKey*)taosArrayGet(pWinStates, 0);
qDebug("===stream===check window state. buff min ts:%" PRId64 ",groupId:%" PRIu64 ".key ts:%" PRId64
",groupId:%" PRIu64,
fistKey->ts, fistKey->groupId, pKey->ts, pKey->groupId);
}
} else {
res = true;
}
}
return res;
}
SStreamSnapshot* getSnapshot(SStreamFileState* pFileState) {
@ -797,8 +934,6 @@ void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, boo
SListIter iter = {0};
tdListInitIter(pSnapshot, &iter, TD_LIST_FORWARD);
const int32_t BATCH_LIMIT = 256;
int64_t st = taosGetTimestampMs();
SListNode* pNode = NULL;
@ -1008,7 +1143,8 @@ int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) {
QUERY_CHECK_CODE(code, lino, _end);
}
winCode = streamStateGetKVByCur_rocksdb(getStateFileStore(pFileState), pCur, pNewPos->pKey, (const void**)&pVal, &vlen);
winCode =
streamStateGetKVByCur_rocksdb(getStateFileStore(pFileState), pCur, pNewPos->pKey, (const void**)&pVal, &vlen);
qDebug("===stream=== get state by cur winres:%d. %s", winCode, __func__);
if (winCode != TSDB_CODE_SUCCESS || pFileState->getTs(pNewPos->pKey) < pFileState->flushMark) {
destroyRowBuffPos(pNewPos);
@ -1244,7 +1380,7 @@ SSHashObj* getGroupIdCache(SStreamFileState* pFileState) {
return pFileState->pGroupIdMap;
}
void clearExpiredState(SStreamFileState* pFileState) {
void clearExpiredState(SStreamFileState* pFileState, int32_t numOfKeep, TSKEY minTs) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SSHashObj* pSearchBuff = pFileState->searchBuff;
@ -1252,20 +1388,33 @@ void clearExpiredState(SStreamFileState* pFileState) {
int32_t iter = 0;
while ((pIte = tSimpleHashIterate(pSearchBuff, pIte, &iter)) != NULL) {
SArray* pWinStates = *((void**)pIte);
int32_t size = taosArrayGetSize(pWinStates);
for (int32_t i = 0; i < size - 1; i++) {
int32_t arraySize = TARRAY_SIZE(pWinStates);
if (minTs != INT64_MAX && arraySize > numOfKeep) {
SWinKey key = {.ts = minTs};
key.groupId = *(uint64_t*)tSimpleHashGetKey(pIte, NULL);
int32_t index = binarySearch(pWinStates, arraySize, &key, fillStateKeyCompare);
numOfKeep = TMAX(arraySize - index, MIN_NUM_OF_SORT_CACHE_WIN);
qDebug("modify numOfKeep, numOfKeep:%d. %s at line %d", numOfKeep, __func__, __LINE__);
}
int32_t size = arraySize - numOfKeep;
for (int32_t i = 0; i < size; i++) {
SWinKey* pKey = taosArrayGet(pWinStates, i);
int32_t code_buff = pFileState->stateBuffRemoveFn(pFileState->rowStateBuff, pKey, sizeof(SWinKey));
qTrace("clear expired buff, ts:%" PRId64 ". %s at line %d res:%d", pKey->ts, __func__, __LINE__, code_buff);
qTrace("clear expired buff, ts:%" PRId64 ",groupid:%" PRIu64 ". %s at line %d res:%d", pKey->ts, pKey->groupId, __func__, __LINE__, code_buff);
if (isFlushedState(pFileState, pKey->ts, 0)) {
int32_t code_file = pFileState->stateFileRemoveFn(pFileState, pKey);
qTrace("clear expired file, ts:%" PRId64 ". %s at line %d res:%d", pKey->ts, __func__, __LINE__, code_file);
}
if (tSimpleHashGetSize(pFileState->pRecFlagMap) > 0) {
tSimpleHashRemove(pFileState->pRecFlagMap, pKey, sizeof(SWinKey));
}
taosArrayRemoveBatch(pWinStates, 0, size - 1, NULL);
}
code = clearRowBuff(pFileState);
taosArrayRemoveBatch(pWinStates, 0, size, NULL);
}
code = clearRowBuffNonFlush(pFileState);
QUERY_CHECK_CODE(code, lino, _end);
_end:
@ -1298,22 +1447,7 @@ int32_t getStateSearchRowBuff(SStreamFileState* pFileState, const SWinKey* pKey,
// recover
if (taosArrayGetSize(pWinStates) == 0 && needClearDiskBuff(pFileState)) {
TSKEY ts = getFlushMark(pFileState);
SWinKey start = {.groupId = pKey->groupId, .ts = INT64_MAX};
void* pState = getStateFileStore(pFileState);
SStreamStateCur* pCur = streamStateSeekKeyPrev_rocksdb(pState, &start);
for (int32_t i = 0; i < NUM_OF_CACHE_WIN; i++) {
SWinKey tmpKey = {.groupId = pKey->groupId};
int32_t tmpRes = streamStateGetGroupKVByCur_rocksdb(pState, pCur, &tmpKey, NULL, 0);
if (tmpRes != TSDB_CODE_SUCCESS) {
break;
}
void* tmp = taosArrayPush(pWinStates, &tmpKey);
QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
streamStateCurPrev_rocksdb(pCur);
}
taosArraySort(pWinStates, winKeyCmprImpl);
streamStateFreeCur(pCur);
recoverHashSortBuff(pFileState, pWinStates, pKey->groupId);
}
int32_t size = taosArrayGetSize(pWinStates);
@ -1354,7 +1488,7 @@ int32_t getRowStatePrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SW
void** ppBuff = (void**)tSimpleHashGet(pSearchBuff, &pKey->groupId, sizeof(uint64_t));
if (ppBuff) {
pWinStates = (SArray*)(*ppBuff);
} else {
} else if (needClearDiskBuff(pFileState)) {
qDebug("===stream=== search buff is empty.group id:%" PRId64, pKey->groupId);
SStreamStateCur* pCur = streamStateSeekKeyPrev_rocksdb(pState, pKey);
void* tmpVal = NULL;
@ -1373,6 +1507,9 @@ int32_t getRowStatePrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SW
}
streamStateFreeCur(pCur);
return code;
} else {
(*pWinCode) = TSDB_CODE_FAILED;
return code;
}
int32_t size = taosArrayGetSize(pWinStates);
int32_t index = binarySearch(pWinStates, size, pKey, fillStateKeyCompare);
@ -1381,7 +1518,8 @@ int32_t getRowStatePrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SW
if (winKeyCmprImpl(pCurKey, pKey) == 0) {
index--;
} else {
qDebug("%s failed at line %d since do not find cur SWinKey. trigger may be force window close", __func__, __LINE__);
qDebug("%s failed at line %d since do not find cur SWinKey. trigger may be force window close", __func__,
__LINE__);
}
}
if (index == -1) {
@ -1416,11 +1554,11 @@ _end:
return code;
}
int32_t addSearchItem(SStreamFileState* pFileState, SArray* pWinStates, const SWinKey* pKey) {
int32_t addSearchItem(SStreamFileState* pFileState, SArray* pWinStates, const SWinKey* pKey, bool* pIsEnd) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t size = taosArrayGetSize(pWinStates);
int32_t index = binarySearch(pWinStates, size, pKey, fillStateKeyCompare);
int32_t index = binarySearch(pWinStates, size, pKey, fillTSKeyCompare);
if (!isFlushedState(pFileState, pKey->ts, 0) || index >= 0 || size == 0) {
if (index >= 0) {
SWinKey* pTmpKey = taosArrayGet(pWinStates, index);
@ -1429,14 +1567,11 @@ int32_t addSearchItem(SStreamFileState* pFileState, SArray* pWinStates, const SW
}
}
index++;
(*pIsEnd) = (index >= size);
void* tmp = taosArrayInsert(pWinStates, index, pKey);
QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
}
if (size >= MAX_NUM_OF_CACHE_WIN) {
int32_t num = size - NUM_OF_CACHE_WIN;
taosArrayRemoveBatch(pWinStates, 0, num, NULL);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
@ -1467,3 +1602,538 @@ _end:
}
return code;
}
static void setValueBuff(TSKEY ts, char* pVal, int32_t len, char* pBuff, int32_t buffLen) {
SET_TSDATA_FLAG(pBuff, buffLen);
if (len == 0) {
*(TSKEY*)pBuff = ts;
return;
}
memset(pBuff, 0, buffLen - 1);
*(TSKEY*)pBuff = ts;
memcpy(pBuff + sizeof(TSKEY), pVal, len);
}
int32_t getAndSetTsData(STableTsDataState* pTsDataState, uint64_t tableUid, TSKEY* pCurTs, void** ppCurPkVal,
TSKEY lastTs, void* pLastPkVal, int32_t lastPkLen, int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
bool hasPk = (lastPkLen != 0);
TSKEY* pDataVal = tSimpleHashGet(pTsDataState->pTableTsDataMap, &tableUid, sizeof(uint64_t));
if (pDataVal != NULL) {
(*pWinCode) = TSDB_CODE_SUCCESS;
*pCurTs = *pDataVal;
if ((*pCurTs) < lastTs) {
setValueBuff(lastTs, pLastPkVal, lastPkLen, (char*)pDataVal, pTsDataState->pkValLen);
} else {
if (hasPk) {
(*ppCurPkVal) = POINTER_SHIFT(pDataVal, sizeof(TSKEY));
if ((*pCurTs) == lastTs && pTsDataState->comparePkColFn((*ppCurPkVal), pLastPkVal) < 0) {
setValueBuff(lastTs, pLastPkVal, lastPkLen, (char*)pDataVal, pTsDataState->pkValLen);
}
}
}
} else {
setValueBuff(lastTs, pLastPkVal, lastPkLen, pTsDataState->pPkValBuff, pTsDataState->pkValLen);
int32_t size = tSimpleHashGetSize(pTsDataState->pTableTsDataMap);
if (size < MAX_STATE_MAP_SIZE) {
(*pWinCode) = TSDB_CODE_FAILED;
code = tSimpleHashPut(pTsDataState->pTableTsDataMap, &tableUid, sizeof(uint64_t), pTsDataState->pPkValBuff,
pTsDataState->pkValLen);
QUERY_CHECK_CODE(code, lino, _end);
} else {
(*pWinCode) = streamStateGetParTag_rocksdb(pTsDataState->pState, tableUid, &pTsDataState->pPkValBuff,
&pTsDataState->pkValLen);
if ((*pWinCode) == TSDB_CODE_SUCCESS) {
*pCurTs = *(TSKEY*)pTsDataState->pPkValBuff;
if (hasPk) {
(*ppCurPkVal) = POINTER_SHIFT(pTsDataState->pPkValBuff, sizeof(TSKEY));
}
}
int32_t tmpCode = streamStatePutParTag_rocksdb(pTsDataState->pState, tableUid, pTsDataState->pPkValBuff,
pTsDataState->pkValLen);
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t doTsDataCommit(STableTsDataState* pTsDataState) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
void* batch = NULL;
char* pTempBuf = NULL;
batch = streamStateCreateBatch();
QUERY_CHECK_NULL(batch, code, lino, _end, terrno);
int idx = streamStateGetCfIdx(pTsDataState->pState, "partag");
int32_t len = (pTsDataState->pkValLen + sizeof(uint64_t) + sizeof(int32_t) + 64) * 2;
pTempBuf = taosMemoryCalloc(1, len);
QUERY_CHECK_NULL(pTempBuf, code, lino, _end, terrno);
void* pIte = NULL;
int32_t iter = 0;
while ((pIte = tSimpleHashIterate(pTsDataState->pTableTsDataMap, pIte, &iter)) != NULL) {
if (streamStateGetBatchSize(batch) >= BATCH_LIMIT) {
code = streamStatePutBatch_rocksdb(pTsDataState->pState, batch);
streamStateClearBatch(batch);
QUERY_CHECK_CODE(code, lino, _end);
}
if (HAS_TSDATA_FLAG(pIte, pTsDataState->pkValLen)) {
void* pKey = tSimpleHashGetKey(pIte, NULL);
UNSET_TSDATA_FLAG(pIte, pTsDataState->pkValLen);
code = streamStatePutBatchOptimize(pTsDataState->pState, idx, batch, pKey, pIte, pTsDataState->pkValLen, 0,
pTempBuf);
QUERY_CHECK_CODE(code, lino, _end);
memset(pTempBuf, 0, len);
qDebug("flush ts data,table id:%" PRIu64 , *(uint64_t*)pKey);
}
}
int32_t numOfElems = streamStateGetBatchSize(batch);
if (numOfElems > 0) {
code = streamStatePutBatch_rocksdb(pTsDataState->pState, batch);
QUERY_CHECK_CODE(code, lino, _end);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
taosMemoryFree(pTempBuf);
streamStateDestroyBatch(batch);
return code;
}
int32_t doRangeDataCommit(STableTsDataState* pTsDataState) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
void* batch = NULL;
batch = streamStateCreateBatch();
QUERY_CHECK_NULL(batch, code, lino, _end, terrno);
int idx = streamStateGetCfIdx(pTsDataState->pState, "sess");
int32_t len = (pTsDataState->pkValLen + sizeof(uint64_t) + sizeof(int32_t) + 64) * 2;
int32_t size = taosArrayGetSize(pTsDataState->pScanRanges);
for (int32_t i = 0; i < size; i++) {
SScanRange* pRange = taosArrayGet(pTsDataState->pScanRanges, i);
if (streamStateGetBatchSize(batch) >= BATCH_LIMIT) {
code = streamStatePutBatch_rocksdb(pTsDataState->pState, batch);
streamStateClearBatch(batch);
QUERY_CHECK_CODE(code, lino, _end);
}
SSessionKey key = {.win = pRange->win, .groupId = 0};
int32_t uidSize = tSimpleHashGetSize(pRange->pUIds);
int32_t gpIdSize = tSimpleHashGetSize(pRange->pGroupIds);
int32_t size = uidSize + gpIdSize;
uint64_t* pIdBuf = (uint64_t*)taosMemoryCalloc(1, size);
void* pIte = NULL;
int32_t iter = 0;
int32_t i = 0;
while ((pIte = tSimpleHashIterate(pTsDataState->pTableTsDataMap, pIte, &iter)) != NULL) {
void* pTempKey = tSimpleHashGetKey(pIte, NULL);
pIdBuf[i] = *(uint64_t*)pTempKey;
i++;
}
code = streamStatePutBatchOptimize(pTsDataState->pState, idx, batch, &key, (void*)pIdBuf, size, 0,
NULL);
QUERY_CHECK_CODE(code, lino, _end);
}
int32_t numOfElems = streamStateGetBatchSize(batch);
if (numOfElems > 0) {
code = streamStatePutBatch_rocksdb(pTsDataState->pState, batch);
QUERY_CHECK_CODE(code, lino, _end);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
streamStateDestroyBatch(batch);
return code;
}
int32_t initTsDataState(STableTsDataState** ppTsDataState, int8_t pkType, int32_t pkLen, void* pState, void* pOtherState) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
STableTsDataState* pTsDataState = taosMemoryCalloc(1, sizeof(STableTsDataState));
QUERY_CHECK_NULL(pTsDataState, code, lino, _end, terrno);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT);
pTsDataState->pTableTsDataMap = tSimpleHashInit(DEFAULT_STATE_MAP_CAPACITY, hashFn);
QUERY_CHECK_NULL(pTsDataState->pTableTsDataMap, code, lino, _end, terrno);
pTsDataState->pkValLen = sizeof(TSKEY) + pkLen + sizeof(char);
pTsDataState->pPkValBuff = taosMemoryCalloc(1, pTsDataState->pkValLen);
QUERY_CHECK_NULL(pTsDataState->pPkValBuff, code, lino, _end, terrno);
if (pkLen != 0) {
pTsDataState->comparePkColFn = getKeyComparFunc(pkType, TSDB_ORDER_ASC);
} else {
pTsDataState->comparePkColFn = NULL;
}
pTsDataState->pScanRanges = taosArrayInit(64, sizeof(SScanRange));
QUERY_CHECK_NULL(pTsDataState->pScanRanges, code, lino, _end, terrno);
pTsDataState->pState = pState;
pTsDataState->recValueLen = sizeof(SRecDataInfo) + pkLen;
pTsDataState->pRecValueBuff = taosMemoryCalloc(1, pTsDataState->recValueLen);
QUERY_CHECK_NULL(pTsDataState->pRecValueBuff, code, lino, _end, terrno);
pTsDataState->curRecId = -1;
pTsDataState->pStreamTaskState = pOtherState;
pTsDataState->cfgIndex = streamStateGetCfIdx(pTsDataState->pState, "sess");
pTsDataState->pBatch = streamStateCreateBatch();
QUERY_CHECK_NULL(pTsDataState->pBatch, code, lino, _end, TSDB_CODE_FAILED);
pTsDataState->batchBufflen = (pTsDataState->recValueLen + sizeof(uint64_t) + sizeof(int32_t) + 64) * 2;
pTsDataState->pBatchBuff = taosMemoryCalloc(1, pTsDataState->batchBufflen);
(*ppTsDataState) = pTsDataState;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static void destroyScanRange(SScanRange* pRange) {
pRange->win.skey = INT64_MIN;
pRange->win.ekey = INT64_MIN;
tSimpleHashCleanup(pRange->pUIds);
pRange->pUIds = NULL;
tSimpleHashCleanup(pRange->pGroupIds);
pRange->pGroupIds = NULL;
}
void destroyTsDataState(STableTsDataState* pTsDataState) {
SArray* pScanRanges = pTsDataState->pScanRanges;
int32_t size = taosArrayGetSize(pScanRanges);
for (int32_t i = 0; i < size; i++) {
SScanRange* pRange = taosArrayGet(pScanRanges, i);
destroyScanRange(pRange);
}
taosArrayDestroy(pTsDataState->pScanRanges);
tSimpleHashCleanup(pTsDataState->pTableTsDataMap);
taosMemoryFreeClear(pTsDataState->pPkValBuff);
taosMemoryFreeClear(pTsDataState->pState);
taosMemoryFreeClear(pTsDataState->pRecValueBuff);
pTsDataState->pStreamTaskState = NULL;
streamStateClearBatch(pTsDataState->pBatch);
streamStateDestroyBatch(pTsDataState->pBatch);
pTsDataState->pBatch = NULL;
taosMemoryFreeClear(pTsDataState->pBatchBuff);
taosMemoryFreeClear(pTsDataState);
}
int32_t recoverTsData(STableTsDataState* pTsDataState) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamStateCur* pCur = createStateCursor(NULL);
streamStateParTagSeekKeyNext_rocksdb(pTsDataState->pState, INT64_MIN, pCur);
while (1) {
uint64_t tableUid = 0;
void* pVal = NULL;
int32_t len = 0;
int32_t winCode = streamStateParTagGetKVByCur_rocksdb(pCur, &tableUid, &pVal, &len);
if (winCode != TSDB_CODE_SUCCESS) {
break;
}
if (pTsDataState->pkValLen != len) {
taosMemoryFree(pVal);
streamStateCurNext_rocksdb(pCur);
continue;
}
UNSET_TSDATA_FLAG(pVal, len);
code = tSimpleHashPut(pTsDataState->pTableTsDataMap, &tableUid, sizeof(uint64_t), pVal, len);
taosMemoryFree(pVal);
QUERY_CHECK_CODE(code, lino, _end);
streamStateCurNext_rocksdb(pCur);
}
_end:
streamStateFreeCur(pCur);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
SStreamStateCur* getLastStateCur(SStreamFileState* pFileState, getStateBuffFn fn) {
SStreamStateCur* pCur = createStateCursor(pFileState);
if (pCur == NULL) {
return NULL;
}
SSHashObj* pSearchBuff = fn(pFileState);
pCur->buffIndex = 0;
pCur->hashIter = 0;
pCur->pHashData = NULL;
pCur->pHashData = tSimpleHashIterate(pSearchBuff, pCur->pHashData, &pCur->hashIter);
return pCur;
}
void moveLastStateCurNext(SStreamStateCur* pCur, getStateBuffFn fn) {
SSHashObj* pSearchBuff = fn(pCur->pStreamFileState);
pCur->pHashData = tSimpleHashIterate(pSearchBuff, pCur->pHashData, &pCur->hashIter);
}
int32_t getNLastStateKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SArray* pWinStates = NULL;
int32_t size = 0;
while(1) {
if (pCur->pHashData == NULL) {
return TSDB_CODE_FAILED;
}
pWinStates = *((void**)pCur->pHashData);
size = taosArrayGetSize(pWinStates);
if (size > 0) {
break;
}
moveLastStateCurNext(pCur, getSearchBuff);
}
int32_t i = TMAX(size - num, 0);
for ( ; i < size; i++) {
SWinKey* pKey = taosArrayGet(pWinStates, i);
int32_t len = 0;
void* pVal = NULL;
int32_t winCode = TSDB_CODE_SUCCESS;
code = addRowBuffIfNotExist(pCur->pStreamFileState, (void*)pKey, sizeof(SWinKey), &pVal, &len, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since window not exist. ts:%" PRId64 ",groupId:%" PRIu64, __func__, __LINE__,
pKey->ts, pKey->groupId);
}
void* pTempRes = taosArrayPush(pRes, &pVal);
QUERY_CHECK_NULL(pTempRes, code, lino, _end, terrno);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t reloadTsDataState(STableTsDataState* pTsDataState) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
STableTsDataState tmpState = *pTsDataState;
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT);
tmpState.pTableTsDataMap = tSimpleHashInit(DEFAULT_STATE_MAP_CAPACITY, hashFn);
QUERY_CHECK_NULL(tmpState.pTableTsDataMap, code, lino, _end, terrno);
code = recoverTsData(&tmpState);
QUERY_CHECK_CODE(code, lino, _end);
void* pIte = NULL;
int32_t iter = 0;
while ((pIte = tSimpleHashIterate(pTsDataState->pTableTsDataMap, pIte, &iter)) != NULL) {
size_t keyLen = 0;
void* pKey = tSimpleHashGetKey(pIte, &keyLen);
code = tSimpleHashPut(tmpState.pTableTsDataMap, pKey, keyLen, pIte, pTsDataState->pkValLen);
QUERY_CHECK_CODE(code, lino, _end);
}
tSimpleHashCleanup(pTsDataState->pTableTsDataMap);
pTsDataState->pTableTsDataMap = tmpState.pTableTsDataMap;
_end:
if (code != TSDB_CODE_SUCCESS) {
tSimpleHashCleanup(tmpState.pTableTsDataMap);
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t saveRecInfoToDisk(STableTsDataState* pTsDataState, SSessionKey* pKey, SRecDataInfo* pVal, int32_t vLen) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStateSessionKey stateKey = {.key = *pKey, .opNum = ((SStreamState*)pTsDataState->pState)->number};
code = streamStatePutBatchOptimize(pTsDataState->pState, pTsDataState->cfgIndex, pTsDataState->pBatch, &stateKey, pVal, vLen, 0,
pTsDataState->pBatchBuff);
QUERY_CHECK_CODE(code, lino, _end);
memset(pTsDataState->pBatchBuff, 0, pTsDataState->batchBufflen);
if (streamStateGetBatchSize(pTsDataState->pBatch) >= BATCH_LIMIT) {
code = streamStatePutBatch_rocksdb(pTsDataState->pState, pTsDataState->pBatch);
streamStateClearBatch(pTsDataState->pBatch);
QUERY_CHECK_CODE(code, lino, _end);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t flushRemainRecInfoToDisk(STableTsDataState* pTsDataState) {
int32_t code = streamStatePutBatch_rocksdb(pTsDataState->pState, pTsDataState->pBatch);
streamStateClearBatch(pTsDataState->pBatch);
return code;
}
int32_t recoverHashSortBuff(SStreamFileState* pFileState, SArray* pWinStates, uint64_t groupId) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SWinKey start = {.groupId = groupId, .ts = INT64_MAX};
void* pState = getStateFileStore(pFileState);
SStreamStateCur* pCur = streamStateSeekKeyPrev_rocksdb(pState, &start);
for (int32_t i = 0; i < NUM_OF_CACHE_WIN; i++) {
SWinKey tmpKey = {.groupId = groupId};
int32_t tmpRes = streamStateGetGroupKVByCur_rocksdb(pState, pCur, &tmpKey, NULL, 0);
if (tmpRes != TSDB_CODE_SUCCESS) {
break;
}
void* tmp = taosArrayPush(pWinStates, &tmpKey);
QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
streamStateCurPrev_rocksdb(pCur);
}
taosArraySort(pWinStates, winKeyCmprImpl);
streamStateFreeCur(pCur);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t getRowStateAllPrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SArray* pResArray, int32_t maxNum) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SWinKey* pPrevKey = NULL;
SSHashObj* pSearchBuff = getSearchBuff(pFileState);
void* pState = getStateFileStore(pFileState);
void** ppBuff = (void**)tSimpleHashGet(pSearchBuff, &pKey->groupId, sizeof(uint64_t));
int32_t num = 0;
if (ppBuff) {
SArray* pWinStates = (SArray*)(*ppBuff);
int32_t size = taosArrayGetSize(pWinStates);
int32_t index = binarySearch(pWinStates, size, pKey, fillStateKeyCompare);
for (; index >= 0 && num <= maxNum; index--) {
pPrevKey = taosArrayGet(pWinStates, index);
if (winKeyCmprImpl(pPrevKey, pKey) == 0) {
continue;
}
void* pVal = NULL;
int32_t len = 0;
int32_t winCode = TSDB_CODE_SUCCESS;
code = addRowBuffIfNotExist(pFileState, (void*)pPrevKey, sizeof(SWinKey), &pVal, &len, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
void* tempRes = taosArrayPush(pResArray, &pVal);
QUERY_CHECK_NULL(tempRes, code, lino, _end, terrno);
num++;
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t setStateRecFlag(SStreamFileState* pFileState, const void* pKey, int32_t keyLen, int32_t mode) {
return tSimpleHashPut(pFileState->pRecFlagMap, pKey, keyLen, &mode, sizeof(int32_t));
}
int32_t getStateRecFlag(SStreamFileState* pFileState, const void* pKey, int32_t keyLen, int32_t* pMode) {
void* pVal = tSimpleHashGet(pFileState->pRecFlagMap, pKey, keyLen);
if (pVal == NULL) {
return TSDB_CODE_FAILED;
}
*pMode = *(int32_t*) pVal;
return TSDB_CODE_SUCCESS;
}
void clearExpiredSessionState(SStreamFileState* pFileState, int32_t numOfKeep, TSKEY minTs, SSHashObj* pFlushGroup) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SSHashObj* pSessionBuff = pFileState->rowStateBuff;
SStreamSnapshot* pFlushList = NULL;
if (pFlushGroup != NULL) {
pFlushList = tdListNew(POINTER_BYTES);
}
void* pIte = NULL;
int32_t iter = 0;
while ((pIte = tSimpleHashIterate(pSessionBuff, pIte, &iter)) != NULL) {
SArray* pWinStates = *((void**)pIte);
int32_t arraySize = TARRAY_SIZE(pWinStates);
if (minTs != INT64_MAX && arraySize > numOfKeep) {
SSessionKey key = {.win.skey = minTs, .win.ekey = minTs};
key.groupId = *(uint64_t*)tSimpleHashGetKey(pIte, NULL);
int32_t index = binarySearch(pWinStates, arraySize, &key, fillStateKeyCompare);
numOfKeep = TMAX(arraySize - index, MIN_NUM_OF_SORT_CACHE_WIN);
qDebug("modify numOfKeep, numOfKeep:%d. %s at line %d", numOfKeep, __func__, __LINE__);
}
int32_t size = arraySize - numOfKeep;
for (int32_t i = 0; i < size; i++) {
SRowBuffPos* pPos = taosArrayGetP(pWinStates, i);
SSessionKey* pKey = pPos->pKey;
if (tSimpleHashGetSize(pFileState->pRecFlagMap) > 0) {
tSimpleHashRemove(pFileState->pRecFlagMap, pKey, sizeof(SSessionKey));
}
pPos->invalid = true;
if (i == 0 && pFlushGroup != NULL) {
void* pGpVal = tSimpleHashGet(pFlushGroup, &pKey->groupId, sizeof(uint64_t));
if (pGpVal == NULL) {
code = tdListAppend(pFlushList, &pPos);
QUERY_CHECK_CODE(code, lino, _end);
code = tSimpleHashPut(pFlushGroup, &pKey->groupId, sizeof(uint64_t), NULL, 0);
QUERY_CHECK_CODE(code, lino, _end);
continue;
}
}
pPos->beFlushed = true;
qTrace("clear expired session buff, ts:%" PRId64 ",groupid:%" PRIu64 ". %s at line %d", pKey->win.skey, pKey->groupId, __func__, __LINE__);
if (isFlushedState(pFileState, pKey->win.skey, 0)) {
int32_t code_file = pFileState->stateFileRemoveFn(pFileState, pKey);
qTrace("clear expired file, ts:%" PRId64 ". %s at line %d res:%d", pKey->win.skey, __func__, __LINE__, code_file);
}
}
taosArrayRemoveBatch(pWinStates, 0, size, NULL);
}
if (pFlushList != NULL) {
flushSnapshot(pFileState, pFlushList, false);
code = clearRowBuffNonFlush(pFileState);
QUERY_CHECK_CODE(code, lino, _end);
tdListFreeP(pFlushList, destroyRowBuffPosPtr);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
}

40
source/libs/stream/test/streamCheckPointTest.cpp
View File

@ -33,8 +33,8 @@ TEST(streamCheckpointTest, StreamTaskProcessCheckpointTriggerRsp) {
SStreamTask* pTask = NULL;
int64_t uid = 1111111111111111;
SArray* array = taosArrayInit(4, POINTER_BYTES);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array,
false, 1, &pTask);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, STREAM_NORMAL_TASK, 0, 0, array,
false, 1, false, &pTask);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
initTaskLock(pTask);
@ -71,8 +71,8 @@ TEST(streamCheckpointTest, StreamTaskSetFailedCheckpointId) {
SStreamTask* pTask = NULL;
int64_t uid = 1111111111111111;
SArray* array = taosArrayInit(4, POINTER_BYTES);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array,
false, 1, &pTask);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, STREAM_NORMAL_TASK, 0, 0, array,
false, 1, false, &pTask);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
initTaskLock(pTask);
@ -102,8 +102,8 @@ TEST(UploadCheckpointDataTest, UploadSuccess) {
SStreamTask* pTask = NULL;
int64_t uid = 1111111111111111;
SArray* array = taosArrayInit(4, POINTER_BYTES);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array,
false, 1, &pTask);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, STREAM_NORMAL_TASK, 0, 0, array,
false, 1, false, &pTask);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
initTaskLock(pTask);
@ -142,8 +142,8 @@ TEST(UploadCheckpointDataTest, UploadDisabled) {
SStreamTask* pTask = NULL;
int64_t uid = 2222222222222;
SArray* array = taosArrayInit(4, POINTER_BYTES);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array,
false, 1, &pTask);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, STREAM_NORMAL_TASK, 0, 0, array,
false, 1, false, &pTask);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
initTaskLock(pTask);
@ -183,8 +183,8 @@ TEST(StreamTaskAlreadySendTriggerTest, AlreadySendTrigger) {
SStreamTask* pTask = NULL;
int64_t uid = 2222222222222;
SArray* array = taosArrayInit(4, POINTER_BYTES);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array,
false, 1, &pTask);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, STREAM_NORMAL_TASK, 0, 0, array,
false, 1, false, &pTask);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
initTaskLock(pTask);
@ -227,8 +227,8 @@ TEST(ChkptTriggerRecvMonitorHelperTest, chkptTriggerRecvMonitorHelper) {
SStreamTask* pTask = NULL;
int64_t uid = 2222222222222;
SArray* array = taosArrayInit(4, POINTER_BYTES);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array,
false, 1, &pTask);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, STREAM_NORMAL_TASK, 0, 0, array,
false, 1, false, &pTask);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
initTaskLock(pTask);
@ -298,8 +298,8 @@ TEST(StreamTaskSendCheckpointTriggerMsgTest, SendCheckpointTriggerMsgSuccessTest
SStreamTask* pTask = NULL;
int64_t uid = 2222222222222;
SArray* array = taosArrayInit(4, POINTER_BYTES);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array,
false, 1, &pTask);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, STREAM_NORMAL_TASK, 0, 0, array,
false, 1, false, &pTask);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
initTaskLock(pTask);
@ -322,8 +322,8 @@ TEST(streamTaskBuildCheckpointTest, streamTaskBuildCheckpointFnTest) {
SStreamTask* pTask = NULL;
int64_t uid = 2222222222222;
SArray* array = taosArrayInit(4, POINTER_BYTES);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array,
false, 1, &pTask);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, STREAM_NORMAL_TASK, 0, 0, array,
false, 1, false, &pTask);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
initTaskLock(pTask);
@ -356,8 +356,8 @@ TEST(sstreamTaskGetTriggerRecvStatusTest, streamTaskGetTriggerRecvStatusFnTest)
SStreamTask* pTask = NULL;
int64_t uid = 2222222222222;
SArray* array = taosArrayInit(4, POINTER_BYTES);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array,
false, 1, &pTask);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, STREAM_NORMAL_TASK, 0, 0, array,
false, 1, false, &pTask);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
initTaskLock(pTask);
@ -404,8 +404,8 @@ TEST(doCheckBeforeHandleChkptTriggerTest, doCheckBeforeHandleChkptTriggerFnTest)
SStreamTask* pTask = NULL;
int64_t uid = 2222222222222;
SArray* array = taosArrayInit(4, POINTER_BYTES);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array,
false, 1, &pTask);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, STREAM_NORMAL_TASK, 0, 0, array,
false, 1, false, &pTask);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
initTaskLock(pTask);

4
tests/parallel_test/cases.task
View File

@ -1562,7 +1562,7 @@
,,y,script,./test.sh -f tsim/table/table.sim
,,y,script,./test.sh -f tsim/table/tinyint.sim
,,y,script,./test.sh -f tsim/table/vgroup.sim
,,n,script,./test.sh -f tsim/stream/basic0.sim -g
,,y,script,./test.sh -f tsim/stream/basic0.sim
,,y,script,./test.sh -f tsim/stream/basic1.sim
,,y,script,./test.sh -f tsim/stream/basic2.sim
,,y,script,./test.sh -f tsim/stream/basic3.sim
@ -1606,6 +1606,8 @@
,,y,script,./test.sh -f tsim/stream/fillIntervalValue.sim
,,y,script,./test.sh -f tsim/stream/ignoreCheckUpdate.sim
,,y,script,./test.sh -f tsim/stream/ignoreExpiredData.sim
,,y,script,./test.sh -f tsim/stream/nonblockIntervalBasic.sim
,,y,script,./test.sh -f tsim/stream/nonblockIntervalHistory.sim
,,y,script,./test.sh -f tsim/stream/partitionby1.sim
,,y,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim
,,y,script,./test.sh -f tsim/stream/partitionbyColumnOther.sim

20
tests/perf-test/stream.py
View File

@ -100,7 +100,7 @@ def do_multi_insert(index, total, host, user, passwd, conf, tz):
class StreamStarter:
def __init__(self) -> None:
self.sql = None
self.host='ubuntu'
self.host='localhost'
self.user = 'root'
self.passwd = 'taosdata'
self.conf = '/etc/taos/taos.cfg'
@ -110,7 +110,7 @@ class StreamStarter:
json_data = {
"filetype": "insert",
"cfgdir": "/etc/taos/cfg",
"host": "ubuntu",
"host": "localhost",
"port": 6030,
"rest_port": 6041,
"user": "root",
@ -151,13 +151,13 @@ class StreamStarter:
"insert_mode": "taosc",
"interlace_rows": 400,
"tcp_transfer": "no",
"insert_rows": 1000,
"insert_rows": 50000,
"partial_col_num": 0,
"childtable_limit": 0,
"childtable_offset": 0,
"rows_per_tbl": 0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_ratio": 30,
"disorder_range": 1000,
"keep_trying": -1,
"timestamp_step": 1000,
@ -320,20 +320,22 @@ class StreamStarter:
host=self.host, user=self.user, password=self.passwd, config=self.conf, timezone=self.tz
)
time.sleep(10)
time.sleep(20)
print("start to connect db")
cursor = conn.cursor()
cursor.execute('use stream_test')
sql = "create stream str1 ignore update 0 into str1_dst as select _wstart as wstart, min(c1),max(c2), count(c3) from stream_test.stb partition by cast(t1 as int) t1,tbname interval(5s)"
# sql = "create stream str1 trigger continuous_window_close ignore expired 0 ignore update 0 into str1_dst as select _wstart as wstart, min(c1),max(c2), count(c3), tbname from stream_test.stb partition by tbname interval(5s)"
sql = "create stream str1 trigger continuous_window_close ignore expired 0 ignore update 0 into str1_dst as select _wstart as wstart, min(c1),max(c2), count(c3) from stream_test.stb interval(5s)"
cursor.execute(sql)
print("create stream completed, start to monitor system load")
conn.close()
loader = MonitorSystemLoad('taosd', 80)
loader = MonitorSystemLoad('taosd', 600)
loader.get_proc_status()
def do_query_then_insert(self):
@ -408,6 +410,6 @@ class StreamStarter:
loader.get_proc_status()
if __name__ == "__main__":
# StreamStarter().do_start()
StreamStarter().do_start()
# StreamStarter().do_query_then_insert()
StreamStarter().multi_insert()
# StreamStarter().multi_insert()

Some files were not shown because too many files have changed in this diff Show More