xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #24538 from taosdata/feat/TD-22023 

stream count window
This commit is contained in:
Haojun Liao 2024-02-20 08:50:25 +08:00 committed by GitHub
parent f95fb4f27e af1893d109
commit 46b1fe520f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
60 changed files with 7230 additions and 3953 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/tcommon.h
View File

@ -54,6 +54,8 @@ typedef struct SSessionKey {
uint64_t groupId;
} SSessionKey;
typedef int64_t COUNT_TYPE;
typedef struct SVersionRange {
int64_t minVer;
int64_t maxVer;

14
include/common/tmsg.h
View File

@ -247,6 +247,7 @@ typedef enum ENodeType {
QUERY_NODE_EVENT_WINDOW,
QUERY_NODE_HINT,
QUERY_NODE_VIEW,
QUERY_NODE_COUNT_WINDOW,
// Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR = 100,
@ -432,7 +433,9 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN,
QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE,
QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL
QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT,
QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT,
QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL,
} ENodeType;
typedef struct {
@ -3182,18 +3185,11 @@ typedef struct {
typedef struct {
SMsgHead head;
int64_t leftForVer;
int64_t resetRelHalt; // reset related stream task halt status
int64_t streamId;
int32_t taskId;
} SVDropStreamTaskReq;
typedef struct {
SMsgHead head;
int64_t streamId;
int32_t taskId;
int64_t dataVer;
} SVStreamTaskVerUpdateReq;
typedef struct {
int8_t reserved;
} SVDropStreamTaskRsp;

140
include/common/ttokendef.h
View File

@ -297,75 +297,77 @@
#define TK_SESSION 278
#define TK_STATE_WINDOW 279
#define TK_EVENT_WINDOW 280
#define TK_SLIDING 281
#define TK_FILL 282
#define TK_VALUE 283
#define TK_VALUE_F 284
#define TK_NONE 285
#define TK_PREV 286
#define TK_NULL_F 287
#define TK_LINEAR 288
#define TK_NEXT 289
#define TK_HAVING 290
#define TK_RANGE 291
#define TK_EVERY 292
#define TK_ORDER 293
#define TK_SLIMIT 294
#define TK_SOFFSET 295
#define TK_LIMIT 296
#define TK_OFFSET 297
#define TK_ASC 298
#define TK_NULLS 299
#define TK_ABORT 300
#define TK_AFTER 301
#define TK_ATTACH 302
#define TK_BEFORE 303
#define TK_BEGIN 304
#define TK_BITAND 305
#define TK_BITNOT 306
#define TK_BITOR 307
#define TK_BLOCKS 308
#define TK_CHANGE 309
#define TK_COMMA 310
#define TK_CONCAT 311
#define TK_CONFLICT 312
#define TK_COPY 313
#define TK_DEFERRED 314
#define TK_DELIMITERS 315
#define TK_DETACH 316
#define TK_DIVIDE 317
#define TK_DOT 318
#define TK_EACH 319
#define TK_FAIL 320
#define TK_FILE 321
#define TK_FOR 322
#define TK_GLOB 323
#define TK_ID 324
#define TK_IMMEDIATE 325
#define TK_IMPORT 326
#define TK_INITIALLY 327
#define TK_INSTEAD 328
#define TK_ISNULL 329
#define TK_KEY 330
#define TK_MODULES 331
#define TK_NK_BITNOT 332
#define TK_NK_SEMI 333
#define TK_NOTNULL 334
#define TK_OF 335
#define TK_PLUS 336
#define TK_PRIVILEGE 337
#define TK_RAISE 338
#define TK_RESTRICT 339
#define TK_ROW 340
#define TK_SEMI 341
#define TK_STAR 342
#define TK_STATEMENT 343
#define TK_STRICT 344
#define TK_STRING 345
#define TK_TIMES 346
#define TK_VALUES 347
#define TK_VARIABLE 348
#define TK_WAL 349
#define TK_COUNT_WINDOW 281
#define TK_SLIDING 282
#define TK_FILL 283
#define TK_VALUE 284
#define TK_VALUE_F 285
#define TK_NONE 286
#define TK_PREV 287
#define TK_NULL_F 288
#define TK_LINEAR 289
#define TK_NEXT 290
#define TK_HAVING 291
#define TK_RANGE 292
#define TK_EVERY 293
#define TK_ORDER 294
#define TK_SLIMIT 295
#define TK_SOFFSET 296
#define TK_LIMIT 297
#define TK_OFFSET 298
#define TK_ASC 299
#define TK_NULLS 300
#define TK_ABORT 301
#define TK_AFTER 302
#define TK_ATTACH 303
#define TK_BEFORE 304
#define TK_BEGIN 305
#define TK_BITAND 306
#define TK_BITNOT 307
#define TK_BITOR 308
#define TK_BLOCKS 309
#define TK_CHANGE 310
#define TK_COMMA 311
#define TK_CONCAT 312
#define TK_CONFLICT 313
#define TK_COPY 314
#define TK_DEFERRED 315
#define TK_DELIMITERS 316
#define TK_DETACH 317
#define TK_DIVIDE 318
#define TK_DOT 319
#define TK_EACH 320
#define TK_FAIL 321
#define TK_FILE 322
#define TK_FOR 323
#define TK_GLOB 324
#define TK_ID 325
#define TK_IMMEDIATE 326
#define TK_IMPORT 327
#define TK_INITIALLY 328
#define TK_INSTEAD 329
#define TK_ISNULL 330
#define TK_KEY 331
#define TK_MODULES 332
#define TK_NK_BITNOT 333
#define TK_NK_SEMI 334
#define TK_NOTNULL 335
#define TK_OF 336
#define TK_PLUS 337
#define TK_PRIVILEGE 338
#define TK_RAISE 339
#define TK_RESTRICT 340
#define TK_ROW 341
#define TK_SEMI 342
#define TK_STAR 343
#define TK_STATEMENT 344
#define TK_STRICT 345
#define TK_STRING 346
#define TK_TIMES 347
#define TK_VALUES 348
#define TK_VARIABLE 349
#define TK_WAL 350
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601

6
include/libs/executor/storageapi.h
View File

@ -352,14 +352,19 @@ typedef struct SStateStore {
int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen);
int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen);
int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key);
int32_t (*streamStateSessionReset)(SStreamState* pState, void* pVal);
int32_t (*streamStateSessionClear)(SStreamState* pState);
int32_t (*streamStateSessionGetKVByCur)(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen);
int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t (*streamStateCountGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t (*streamStateSessionAllocWinBuffByNextPosition)(SStreamState* pState, SStreamStateCur* pCur,
const SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*streamStateCountWinAddIfNotExist)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal, int32_t* pVLen);
int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp);
TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol);
bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts);
@ -377,6 +382,7 @@ typedef struct SStateStore {
int32_t (*updateInfoDeserialize)(void* buf, int32_t bufLen, SUpdateInfo* pInfo);
SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* (*streamStateCountSeekKeyPrev)(SStreamState* pState, const SSessionKey* pKey, COUNT_TYPE count);
SStreamStateCur* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* (*streamStateSessionSeekKeyCurrentNext)(SStreamState* pState, const SSessionKey* key);

13
include/libs/nodes/plannodes.h
View File

@ -244,7 +244,8 @@ typedef enum EWindowType {
WINDOW_TYPE_INTERVAL = 1,
WINDOW_TYPE_SESSION,
WINDOW_TYPE_STATE,
WINDOW_TYPE_EVENT
WINDOW_TYPE_EVENT,
WINDOW_TYPE_COUNT
} EWindowType;
typedef enum EWindowAlgorithm {
@ -282,6 +283,8 @@ typedef struct SWindowLogicNode {
int8_t igCheckUpdate;
EWindowAlgorithm windowAlgo;
bool isPartTb;
int64_t windowCount;
int64_t windowSliding;
} SWindowLogicNode;
typedef struct SFillLogicNode {
@ -631,6 +634,14 @@ typedef struct SEventWinodwPhysiNode {
typedef SEventWinodwPhysiNode SStreamEventWinodwPhysiNode;
typedef struct SCountWinodwPhysiNode {
SWindowPhysiNode window;
int64_t windowCount;
int64_t windowSliding;
} SCountWinodwPhysiNode;
typedef SCountWinodwPhysiNode SStreamCountWinodwPhysiNode;
typedef struct SSortPhysiNode {
SPhysiNode node;
SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function

7
include/libs/nodes/querynodes.h
View File

@ -278,6 +278,13 @@ typedef struct SEventWindowNode {
SNode* pEndCond;
} SEventWindowNode;
typedef struct SCountWindowNode {
ENodeType type; // QUERY_NODE_EVENT_WINDOW
SNode* pCol; // timestamp primary key
int64_t windowCount;
int64_t windowSliding;
} SCountWindowNode;
typedef enum EFillMode {
FILL_MODE_NONE = 1,
FILL_MODE_VALUE,

11
include/libs/stream/streamState.h
View File

@ -55,13 +55,16 @@ int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key,
int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen);
int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key);
int32_t streamStateSessionReset(SStreamState* pState, void* pVal);
int32_t streamStateSessionClear(SStreamState* pState);
int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t streamStateSessionAllocWinBuffByNextPosition(SStreamState* pState, SStreamStateCur* pCur,
const SSessionKey* pKey, void** pVal, int32_t* pVLen);
SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* streamStateCountSeekKeyPrev(SStreamState* pState, const SSessionKey* pKey, COUNT_TYPE count);
SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, const SSessionKey* key);
@ -79,6 +82,10 @@ int32_t streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used);
int32_t streamStateClearBuff(SStreamState* pState, void* pVal);
void streamStateFreeVal(void* val);
// count window
int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal, int32_t* pVLen);
int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key);
SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key);
SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key);
@ -128,10 +135,6 @@ int sessionRangeKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2);
int sessionWinKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2);
int stateSessionKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2);
int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2);
#if 0
char* streamStateSessionDump(SStreamState* pState);
char* streamStateIntervalDump(SStreamState* pState);
#endif
#ifdef __cplusplus
}

4
include/libs/stream/tstream.h
View File

@ -784,7 +784,7 @@ bool streamTaskIsAllUpstreamClosed(SStreamTask* pTask);
bool streamTaskSetSchedStatusWait(SStreamTask* pTask);
int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask);
int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask);
int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, bool metaLock);
int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt, bool metaLock);
int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event);
int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event);
@ -870,7 +870,7 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask);
int32_t streamTaskBuildCheckpoint(SStreamTask* pTask);
void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg);
int32_t streamAlignTransferState(SStreamTask* pTask);
int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId);
int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId, int64_t resetRelHalt);
int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask,
int8_t isSucceed);
int32_t buildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg,

9
include/libs/stream/tstreamFileState.h
View File

@ -41,6 +41,8 @@ typedef int32_t (*_state_file_remove_fn)(SStreamFileState* pFileState, const voi
typedef int32_t (*_state_file_get_fn)(SStreamFileState* pFileState, void* pKey, void* data, int32_t* pDataLen);
typedef int32_t (*_state_file_clear_fn)(SStreamState* pState);
typedef int32_t (*range_cmpr_fn)(const SSessionKey* pWin1, const SSessionKey* pWin2);
SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize,
GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId,
int64_t checkpointId, int8_t type);
@ -90,14 +92,19 @@ void sessionWinStateCleanup(void* pBuff);
SStreamStateCur* sessionWinStateSeekKeyCurrentPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey);
SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey, COUNT_TYPE count);
int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t sessionWinStateMoveToNext(SStreamStateCur* pCur);
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey);
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey, range_cmpr_fn cmpFn);
// state window
int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen);
// count window
int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen);
int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
#ifdef __cplusplus
}
#endif

35
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -27,6 +27,28 @@
#define SINK_NODE_LEVEL (0)
extern bool tsDeployOnSnode;
static bool hasCountWindowNode(SPhysiNode* pNode) {
if (nodeType(pNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT) {
return true;
} else {
size_t size = LIST_LENGTH(pNode->pChildren);
for (int32_t i = 0; i < size; ++i) {
SPhysiNode* pChild = (SPhysiNode*)nodesListGetNode(pNode->pChildren, i);
if (hasCountWindowNode(pChild)) {
return true;
}
}
return false;
}
}
static bool countWindowStreamTask(SSubplan* pPlan) {
SPhysiNode* pNode = pPlan->pNode;
return hasCountWindowNode(pNode);
}
int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType,
int64_t watermark, int64_t deleteMark) {
SNode* pAst = NULL;
@ -320,6 +342,17 @@ static void streamTaskSetDataRange(SStreamTask* pTask, int64_t skey, SArray* pVe
}
}
static void haltInitialTaskStatus(SStreamTask* pTask, SSubplan* pPlan) {
bool hasCountWindowNode = countWindowStreamTask(pPlan);
bool isRelStreamTask = (pTask->hTaskInfo.id.taskId != 0);
if (hasCountWindowNode && isRelStreamTask) {
SStreamStatus* pStatus = &pTask->status;
mDebug("s-task:0x%x status is set to %s from %s for count window agg task with fill-history option set",
pTask->id.taskId, streamTaskGetStatusStr(pStatus->taskStatus), streamTaskGetStatusStr(TASK_STATUS__HALT));
pStatus->taskStatus = TASK_STATUS__HALT;
}
}
static SStreamTask* buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam) {
uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
@ -373,6 +406,8 @@ static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStre
}
mDebug("doAddSourceTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory);
haltInitialTaskStatus(pTask, plan);
streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId);
int32_t code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup);

6
source/dnode/snode/src/snodeInitApi.c
View File

@ -67,12 +67,17 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateSessionPut = streamStateSessionPut;
pStore->streamStateSessionGet = streamStateSessionGet;
pStore->streamStateSessionDel = streamStateSessionDel;
pStore->streamStateSessionReset = streamStateSessionReset;
pStore->streamStateSessionClear = streamStateSessionClear;
pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur;
pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist;
pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange;
pStore->streamStateCountGetKeyByRange = streamStateCountGetKeyByRange;
pStore->streamStateSessionAllocWinBuffByNextPosition = streamStateSessionAllocWinBuffByNextPosition;
pStore->streamStateCountWinAddIfNotExist = streamStateCountWinAddIfNotExist;
pStore->streamStateCountWinAdd = streamStateCountWinAdd;
pStore->updateInfoInit = updateInfoInit;
pStore->updateInfoFillBlockData = updateInfoFillBlockData;
pStore->updateInfoIsUpdated = updateInfoIsUpdated;
@ -89,6 +94,7 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->updateInfoDeserialize = updateInfoDeserialize;
pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext;
pStore->streamStateCountSeekKeyPrev = streamStateCountSeekKeyPrev;
pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev;
pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext;

18
source/dnode/vnode/src/tq/tq.c
View File

@ -841,22 +841,24 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
}
char* p = streamTaskGetStatus(pTask)->name;
const char* pNext = streamTaskGetStatusStr(pTask->status.taskStatus);
if (pTask->info.fillHistory) {
tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
" child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64
" ms, inputVer:%" PRId64,
" child id:%d, level:%d, cur-status:%s, next-status:%s fill-history:%d, related stream task:0x%x "
"trigger:%" PRId64 " ms, inputVer:%" PRId64,
vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory,
(int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam, nextProcessVer);
} else {
tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
tqInfo(
"vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
" child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64
" child id:%d, level:%d, cur-status:%s next-status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64
" ms, inputVer:%" PRId64,
vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory,
(int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam, nextProcessVer);
}
@ -1016,8 +1018,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s",
pTask->streamTaskId.taskId, pTask->id.idStr);
tqDebug("s-task:%s fill-history task set status to be dropping", id);
streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id);
tqDebug("s-task:%s fill-history task set status to be dropping and drop it", id);
streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0);
atomic_store_32(&pTask->status.inScanHistorySentinel, 0);
streamMetaReleaseTask(pMeta, pTask);

4
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -612,6 +612,10 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
streamMetaReleaseTask(pMeta, pTask);
}
streamMetaWLock(pMeta);
streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt, false);
streamMetaWUnLock(pMeta);
// drop the stream task now
streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId);

6
source/dnode/vnode/src/vnd/vnodeInitApi.c
View File

@ -181,12 +181,17 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateSessionPut = streamStateSessionPut;
pStore->streamStateSessionGet = streamStateSessionGet;
pStore->streamStateSessionDel = streamStateSessionDel;
pStore->streamStateSessionReset = streamStateSessionReset;
pStore->streamStateSessionClear = streamStateSessionClear;
pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur;
pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist;
pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange;
pStore->streamStateCountGetKeyByRange = streamStateCountGetKeyByRange;
pStore->streamStateSessionAllocWinBuffByNextPosition = streamStateSessionAllocWinBuffByNextPosition;
pStore->streamStateCountWinAddIfNotExist = streamStateCountWinAddIfNotExist;
pStore->streamStateCountWinAdd = streamStateCountWinAdd;
pStore->updateInfoInit = updateInfoInit;
pStore->updateInfoFillBlockData = updateInfoFillBlockData;
pStore->updateInfoIsUpdated = updateInfoIsUpdated;
@ -203,6 +208,7 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->updateInfoDeserialize = updateInfoDeserialize;
pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext;
pStore->streamStateCountSeekKeyPrev = streamStateCountSeekKeyPrev;
pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev;
pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext;

4
source/libs/command/inc/commandInt.h
View File

@ -69,6 +69,10 @@ extern "C" {
#define EXPLAIN_EVENT_END_FORMAT "End Cond: "
#define EXPLAIN_GROUP_CACHE_FORMAT "Group Cache"
#define EXPLAIN_DYN_QRY_CTRL_FORMAT "Dynamic Query Control for %s"
#define EXPLAIN_COUNT_FORMAT "Count"
#define EXPLAIN_COUNT_INFO_FORMAT "Window Count Info"
#define EXPLAIN_COUNT_NUM_FORMAT "Window Count=%" PRId64
#define EXPLAIN_COUNT_SLIDING_FORMAT "Window Sliding=%" PRId64
#define EXPLAIN_PLANNING_TIME_FORMAT "Planning Time: %.3f ms"
#define EXPLAIN_EXEC_TIME_FORMAT "Execution Time: %.3f ms"

25
source/libs/command/src/explain.c
View File

@ -1735,6 +1735,31 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT: {
SCountWinodwPhysiNode *pCountNode = (SCountWinodwPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_COUNT_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pCountNode->window.pFuncs->length);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pCountNode->window.node.pOutputDataBlockDesc->totalRowSize);
EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
if (verbose) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_COUNT_NUM_FORMAT, pCountNode->windowCount);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_COUNT_SLIDING_FORMAT, pCountNode->windowSliding);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
break;
}
default:
qError("not supported physical node type %d", pNode->type);
return TSDB_CODE_APP_ERROR;

33
source/libs/executor/inc/executorInt.h
View File

@ -371,6 +371,8 @@ typedef struct SStreamAggSupporter {
STimeWindow winRange;
SStorageAPI* pSessionAPI;
struct SUpdateInfo* pUpdateInfo;
int32_t windowCount;
int32_t windowSliding;
} SStreamAggSupporter;
typedef struct SWindowSupporter {
@ -663,6 +665,27 @@ typedef struct SStreamEventAggOperatorInfo {
SFilterInfo* pEndCondInfo;
} SStreamEventAggOperatorInfo;
typedef struct SStreamCountAggOperatorInfo {
SOptrBasicInfo binfo;
SStreamAggSupporter streamAggSup;
SExprSupp scalarSupp; // supporter for perform scalar function
SGroupResInfo groupResInfo;
int32_t primaryTsIndex; // primary timestamp slot id
STimeWindowAggSupp twAggSup;
SSDataBlock* pDelRes;
SSHashObj* pStDeleted;
void* pDelIterator;
bool ignoreExpiredData;
bool ignoreExpiredDataSaved;
SArray* pUpdated;
SSHashObj* pStUpdated;
int64_t dataVersion;
SArray* historyWins;
bool reCkBlock;
bool recvGetAll;
SSDataBlock* pCheckpointRes;
} SStreamCountAggOperatorInfo;
typedef struct SStreamPartitionOperatorInfo {
SOptrBasicInfo binfo;
SPartitionBySupporter partitionSup;
@ -799,6 +822,9 @@ bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, ST
SStateStore* pStore);
void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid,
uint64_t* pGp, void* pTbName);
void appendAllColumnToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, TSKEY* pCalStartTs,
TSKEY* pCalEndTs, uint64_t* pUid, uint64_t* pGp, void* pTbName);
uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId);
int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup,
@ -880,11 +906,18 @@ void freeExchangeGetBasicOperatorParam(void* pParam);
void freeOperatorParam(SOperatorParam* pParam, SOperatorParamType type);
void freeResetOperatorParams(struct SOperatorInfo* pOperator, SOperatorParamType type, bool allFree);
SSDataBlock* getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int32_t idx, bool clearParam);
void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, EStreamType mode, SSessionKey* pDelRange);
bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey);
void saveDeleteInfo(SArray* pWins, SSessionKey key);
void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins);
void copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted);
bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo);
bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd, EStreamType blockType);
bool compareVal(const char* v, const SStateKeys* pKey);
bool inWinRange(STimeWindow* range, STimeWindow* cur);
void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result);
int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
TSKEY* primaryKeys, int32_t prevPosition, int32_t order);

4
source/libs/executor/inc/operator.h
View File

@ -154,10 +154,14 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle);
SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle);
SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createGroupCacheOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SGroupCachePhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SDynQueryCtrlPhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo);

271
source/libs/executor/src/countwindowoperator.c Normal file
View File

@ -0,0 +1,271 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorInt.h"
#include "filter.h"
#include "function.h"
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
#include "tcommon.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "ttime.h"
typedef struct SCountWindowResult {
int32_t winRows;
SResultRow row;
} SCountWindowResult;
typedef struct SCountWindowSupp {
SArray* pWinStates;
int32_t stateIndex;
} SCountWindowSupp;
typedef struct SCountWindowOperatorInfo {
SOptrBasicInfo binfo;
SAggSupporter aggSup;
SExprSupp scalarSup;
int32_t tsSlotId; // primary timestamp column slot id
STimeWindowAggSupp twAggSup;
uint64_t groupId; // current group id, used to identify the data block from different groups
SResultRow* pRow;
int32_t windowCount;
int32_t windowSliding;
SCountWindowSupp countSup;
} SCountWindowOperatorInfo;
void destroyCountWindowOperatorInfo(void* param) {
SCountWindowOperatorInfo* pInfo = (SCountWindowOperatorInfo*)param;
if (pInfo == NULL) {
return;
}
cleanupBasicInfo(&pInfo->binfo);
colDataDestroy(&pInfo->twAggSup.timeWindowData);
cleanupAggSup(&pInfo->aggSup);
cleanupExprSupp(&pInfo->scalarSup);
taosArrayDestroy(pInfo->countSup.pWinStates);
taosMemoryFreeClear(param);
}
static void clearWinStateBuff(SCountWindowResult* pBuff) {
pBuff->winRows = 0;
}
static SCountWindowResult* getCountWinStateInfo(SCountWindowSupp* pCountSup) {
SCountWindowResult* pBuffInfo = taosArrayGet(pCountSup->pWinStates, pCountSup->stateIndex);
pCountSup->stateIndex = (pCountSup->stateIndex + 1) % taosArrayGetSize(pCountSup->pWinStates);
return pBuffInfo;
}
static SCountWindowResult* setCountWindowOutputBuff(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SResultRow** pResult) {
SCountWindowResult* pBuff = getCountWinStateInfo(pCountSup);
(*pResult) = &pBuff->row;
setResultRowInitCtx(*pResult, pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset);
return pBuff;
}
static int32_t updateCountWindowInfo(int32_t start, int32_t blockRows, int32_t countWinRows, int32_t* pCurrentRows) {
int32_t rows = TMIN(countWinRows - (*pCurrentRows), blockRows - start);
(*pCurrentRows) += rows;
return rows;
}
int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SExprSupp* pExprSup = &pOperator->exprSupp;
SCountWindowOperatorInfo* pInfo = pOperator->info;
SSDataBlock* pRes = pInfo->binfo.pRes;
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
TSKEY* tsCols = (TSKEY*)pColInfoData->pData;
int32_t code = TSDB_CODE_SUCCESS;
for (int32_t i = 0; i < pBlock->info.rows;) {
int32_t step = pInfo->windowSliding;
SCountWindowResult* pBuffInfo = setCountWindowOutputBuff(pExprSup, &pInfo->countSup, &pInfo->pRow);
int32_t prevRows = pBuffInfo->winRows;
int32_t num = updateCountWindowInfo(i, pBlock->info.rows, pInfo->windowCount, &pBuffInfo->winRows);
if (prevRows == 0) {
pInfo->pRow->win.skey = tsCols[i];
}
pInfo->pRow->win.ekey = tsCols[num + i - 1];
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->pRow->win, 0);
applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &pInfo->twAggSup.timeWindowData, i, num,
pBlock->info.rows, pExprSup->numOfExprs);
if (pBuffInfo->winRows == pInfo->windowCount) {
doUpdateNumOfRows(pExprSup->pCtx, pInfo->pRow, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset);
copyResultrowToDataBlock(pExprSup->pExprInfo, pExprSup->numOfExprs, pInfo->pRow, pExprSup->pCtx, pRes,
pExprSup->rowEntryInfoOffset, pTaskInfo);
pRes->info.rows += pInfo->pRow->numOfRows;
clearWinStateBuff(pBuffInfo);
clearResultRowInitFlag(pExprSup->pCtx, pExprSup->numOfExprs);
}
if (pInfo->windowCount != pInfo->windowSliding) {
if (prevRows <= pInfo->windowSliding) {
if (pBuffInfo->winRows > pInfo->windowSliding) {
step = pInfo->windowSliding - prevRows;
}
} else {
step = 0;
}
}
i += step;
}
return code;
}
static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock) {
SResultRow* pResultRow = NULL;
for (int32_t i = 0; i < taosArrayGetSize(pCountSup->pWinStates); i++) {
SCountWindowResult* pBuff = setCountWindowOutputBuff(pExprSup, pCountSup, &pResultRow);
if (pBuff->winRows == 0) {
continue;;
}
doUpdateNumOfRows(pExprSup->pCtx, pResultRow, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset);
copyResultrowToDataBlock(pExprSup->pExprInfo, pExprSup->numOfExprs, pResultRow, pExprSup->pCtx, pBlock,
pExprSup->rowEntryInfoOffset, pTaskInfo);
pBlock->info.rows += pResultRow->numOfRows;
clearWinStateBuff(pBuff);
clearResultRowInitFlag(pExprSup->pCtx, pExprSup->numOfExprs);
}
}
static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) {
SCountWindowOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SExprSupp* pExprSup = &pOperator->exprSupp;
int32_t order = pInfo->binfo.inputTsOrder;
SSDataBlock* pRes = pInfo->binfo.pRes;
SOperatorInfo* downstream = pOperator->pDownstream[0];
blockDataCleanup(pRes);
while (1) {
SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
if (pBlock == NULL) {
break;
}
pRes->info.scanFlag = pBlock->info.scanFlag;
setInputDataBlock(pExprSup, pBlock, order, MAIN_SCAN, true);
blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);
// there is an scalar expression that needs to be calculated right before apply the group aggregation.
if (pInfo->scalarSup.pExprInfo != NULL) {
pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx,
pInfo->scalarSup.numOfExprs, NULL);
if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
}
if (pInfo->groupId == 0) {
pInfo->groupId = pBlock->info.id.groupId;
} else if (pInfo->groupId != pBlock->info.id.groupId) {
buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pRes);
pInfo->groupId = pBlock->info.id.groupId;
}
doCountWindowAggImpl(pOperator, pBlock);
if (pRes->info.rows >= pOperator->resultInfo.threshold) {
return pRes;
}
}
buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pRes);
return pRes->info.rows == 0 ? NULL : pRes;
}
SOperatorInfo* createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode,
SExecTaskInfo* pTaskInfo) {
SCountWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SCountWindowOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
int32_t code = TSDB_CODE_SUCCESS;
SCountWinodwPhysiNode* pCountWindowNode = (SCountWinodwPhysiNode*)physiNode;
pInfo->tsSlotId = ((SColumnNode*)pCountWindowNode->window.pTspk)->slotId;
if (pCountWindowNode->window.pExprs != NULL) {
int32_t numOfScalarExpr = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pCountWindowNode->window.pExprs, NULL, &numOfScalarExpr);
code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
}
size_t keyBufSize = 0;
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pCountWindowNode->window.pFuncs, NULL, &num);
initResultSizeInfo(&pOperator->resultInfo, 4096);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
SSDataBlock* pResBlock = createDataBlockFromDescNode(pCountWindowNode->window.node.pOutputDataBlockDesc);
blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity);
initBasicInfo(&pInfo->binfo, pResBlock);
initResultRowInfo(&pInfo->binfo.resultRowInfo);
pInfo->binfo.inputTsOrder = physiNode->inputTsOrder;
pInfo->binfo.outputTsOrder = physiNode->outputTsOrder;
pInfo->windowCount = pCountWindowNode->windowCount;
pInfo->windowSliding = pCountWindowNode->windowSliding;
//sizeof(SCountWindowResult)
int32_t itemSize = sizeof(int32_t) + pInfo->aggSup.resultRowSize;
int32_t numOfItem = 1;
if (pInfo->windowCount != pInfo->windowSliding) {
numOfItem = pInfo->windowCount / pInfo->windowSliding + 1;
}
pInfo->countSup.pWinStates = taosArrayInit_s(itemSize, numOfItem);
if (!pInfo->countSup.pWinStates) {
goto _error;
}
pInfo->countSup.stateIndex = 0;
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
setOperatorInfo(pOperator, "CountWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT, true, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, countWindowAggregate, NULL, destroyCountWindowOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
return pOperator;
_error:
if (pInfo != NULL) {
destroyCountWindowOperatorInfo(pInfo);
}
taosMemoryFreeClear(pOperator);
pTaskInfo->code = code;
return NULL;
}

2
source/libs/executor/src/executil.c
View File

@ -2245,6 +2245,8 @@ char* getStreamOpName(uint16_t opType) {
return "stream partitionby";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT:
return "stream event";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
return "stream count";
}
return "";
}

4
source/libs/executor/src/operator.c
View File

@ -542,6 +542,10 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
pOptr = createGroupCacheOperatorInfo(ops, size, (SGroupCachePhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL == type) {
pOptr = createDynQueryCtrlOperatorInfo(ops, size, (SDynQueryCtrlPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT == type) {
pOptr = createStreamCountAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT == type) {
pOptr = createCountwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo);
} else {
terrno = TSDB_CODE_INVALID_PARA;
pTaskInfo->code = terrno;

84
source/libs/executor/src/scanoperator.c
View File

@ -1284,6 +1284,14 @@ static bool isSlidingWindow(SStreamScanInfo* pInfo) {
return isIntervalWindow(pInfo) && pInfo->interval.interval != pInfo->interval.sliding;
}
static bool isCountSlidingWindow(SStreamScanInfo* pInfo) {
return pInfo->windowSup.pStreamAggSup && (pInfo->windowSup.pStreamAggSup->windowCount != pInfo->windowSup.pStreamAggSup->windowSliding);
}
static bool isCountWindow(SStreamScanInfo* pInfo) {
return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT;
}
static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex);
uint64_t* groupCol = (uint64_t*)pColInfo->pData;
@ -1394,7 +1402,7 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
TSKEY* calEndData = (TSKEY*)pCalEndTsCol->pData;
setGroupId(pInfo, pBlock, GROUPID_COLUMN_INDEX, *pRowIndex);
if (isSlidingWindow(pInfo)) {
if (isSlidingWindow(pInfo) || isCountSlidingWindow(pInfo)) {
pInfo->updateWin.skey = calStartData[*pRowIndex];
pInfo->updateWin.ekey = calEndData[*pRowIndex];
}
@ -1588,6 +1596,47 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr
return TSDB_CODE_SUCCESS;
}
static int32_t generateCountScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock, EStreamType mode) {
blockDataCleanup(pDestBlock);
if (pSrcBlock->info.rows == 0) {
return TSDB_CODE_SUCCESS;
}
int32_t code = blockDataEnsureCapacity(pDestBlock, pSrcBlock->info.rows);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
ASSERT(taosArrayGetSize(pSrcBlock->pDataBlock) >= 3);
SColumnInfoData* pStartTsCol = taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* startData = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pEndTsCol = taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX);
TSKEY* endData = (TSKEY*)pEndTsCol->pData;
SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
uint64_t* uidCol = (uint64_t*)pUidCol->pData;
SColumnInfoData* pDestStartCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX);
SColumnInfoData* pDestEndCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX);
SColumnInfoData* pDestUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX);
SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX);
SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
int64_t ver = pSrcBlock->info.version - 1;
for (int32_t i = 0; i < pSrcBlock->info.rows; i++) {
uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], ver);
SSessionKey startWin = {.win.skey = startData[i], .win.ekey = endData[i], .groupId = groupId};
SSessionKey range = {0};
getCountWinRange(pInfo->windowSup.pStreamAggSup, &startWin, mode, &range);
colDataSetVal(pDestStartCol, i, (const char*)&range.win.skey, false);
colDataSetVal(pDestEndCol, i, (const char*)&range.win.ekey, false);
colDataSetNULL(pDestUidCol, i);
colDataSetVal(pDestGpCol, i, (const char*)&groupId, false);
colDataSetVal(pDestCalStartTsCol, i, (const char*)&range.win.skey, false);
colDataSetVal(pDestCalEndTsCol, i, (const char*)&range.win.ekey, false);
pDestBlock->info.rows++;
}
return TSDB_CODE_SUCCESS;
}
static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
blockDataCleanup(pDestBlock);
int32_t rows = pSrcBlock->info.rows;
@ -1724,12 +1773,14 @@ static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pS
return TSDB_CODE_SUCCESS;
}
static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock, EStreamType type) {
int32_t code = TSDB_CODE_SUCCESS;
if (isIntervalWindow(pInfo)) {
code = generateIntervalScanRange(pInfo, pSrcBlock, pDestBlock);
} else if (isSessionWindow(pInfo) || isStateWindow(pInfo)) {
code = generateSessionScanRange(pInfo, pSrcBlock, pDestBlock);
} else if (isCountWindow(pInfo)) {
code = generateCountScanRange(pInfo, pSrcBlock, pDestBlock, type);
} else {
code = generateDeleteResultBlock(pInfo, pSrcBlock, pDestBlock);
}
@ -1742,6 +1793,11 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock,
void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid,
uint64_t* pGp, void* pTbName) {
appendAllColumnToStreamSpecialBlock(pBlock, pStartTs, pEndTs, pStartTs, pEndTs, pUid, pGp, pTbName);
}
void appendAllColumnToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, TSKEY* pCalStartTs,
TSKEY* pCalEndTs, uint64_t* pUid, uint64_t* pGp, void* pTbName) {
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
@ -1753,8 +1809,8 @@ void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKE
colDataSetVal(pEndTsCol, pBlock->info.rows, (const char*)pEndTs, false);
colDataSetVal(pUidCol, pBlock->info.rows, (const char*)pUid, false);
colDataSetVal(pGpCol, pBlock->info.rows, (const char*)pGp, false);
colDataSetVal(pCalStartCol, pBlock->info.rows, (const char*)pStartTs, false);
colDataSetVal(pCalEndCol, pBlock->info.rows, (const char*)pEndTs, false);
colDataSetVal(pCalStartCol, pBlock->info.rows, (const char*)pCalStartTs, false);
colDataSetVal(pCalEndCol, pBlock->info.rows, (const char*)pCalEndTs, false);
colDataSetVal(pTableCol, pBlock->info.rows, (const char*)pTbName, pTbName == NULL);
pBlock->info.rows++;
}
@ -2159,6 +2215,14 @@ void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo)
taosMemoryFree(pUpInfo);
}
}
static bool hasScanRange(SStreamScanInfo* pInfo) {
SStreamAggSupporter* pSup = pInfo->windowSup.pStreamAggSup;
return pSup && pSup->pScanBlock->info.rows > 0 && (isStateWindow(pInfo) || isCountWindow(pInfo));
}
static bool isStreamWindow(SStreamScanInfo* pInfo) {
return isIntervalWindow(pInfo) || isSessionWindow(pInfo) || isStateWindow(pInfo) || isCountWindow(pInfo);
}
static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
// NOTE: this operator does never check if current status is done or not
@ -2307,7 +2371,7 @@ FETCH_NEXT_BLOCK:
goto FETCH_NEXT_BLOCK;
}
if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) {
if (!isStreamWindow(pInfo)) {
generateDeleteResultBlock(pInfo, pDelBlock, pInfo->pDeleteDataRes);
pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT;
printSpecDataBlock(pDelBlock, getStreamOpName(pOperator->operatorType), "delete result", GET_TASKID(pTaskInfo));
@ -2321,7 +2385,7 @@ FETCH_NEXT_BLOCK:
} else {
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
pInfo->updateResIndex = 0;
generateScanRange(pInfo, pDelBlock, pInfo->pUpdateRes);
generateScanRange(pInfo, pDelBlock, pInfo->pUpdateRes, STREAM_DELETE_DATA);
prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes);
pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA;
@ -2359,7 +2423,7 @@ FETCH_NEXT_BLOCK:
}
} break;
case STREAM_SCAN_FROM_DELETE_DATA: {
generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes);
generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes, STREAM_DELETE_DATA);
prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes);
@ -2367,7 +2431,7 @@ FETCH_NEXT_BLOCK:
return pInfo->pDeleteDataRes;
} break;
case STREAM_SCAN_FROM_UPDATERES: {
generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes);
generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes, STREAM_CLEAR);
prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
return pInfo->pUpdateRes;
@ -2392,10 +2456,10 @@ FETCH_NEXT_BLOCK:
break;
}
SStreamAggSupporter* pSup = pInfo->windowSup.pStreamAggSup;
if (isStateWindow(pInfo) && pSup->pScanBlock->info.rows > 0) {
if (hasScanRange(pInfo)) {
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
pInfo->updateResIndex = 0;
SStreamAggSupporter* pSup = pInfo->windowSup.pStreamAggSup;
copyDataBlock(pInfo->pUpdateRes, pSup->pScanBlock);
blockDataCleanup(pSup->pScanBlock);
prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);

714
source/libs/executor/src/streamcountwindowoperator.c Normal file
View File

@ -0,0 +1,714 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorInt.h"
#include "function.h"
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
#include "tchecksum.h"
#include "tcommon.h"
#include "tdatablock.h"
#include "tglobal.h"
#include "tlog.h"
#include "ttime.h"
#define IS_FINAL_COUNT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_COUNT)
#define STREAM_COUNT_OP_STATE_NAME "StreamCountHistoryState"
#define STREAM_COUNT_OP_CHECKPOINT_NAME "StreamCountOperator_Checkpoint"
typedef struct SCountWindowInfo {
SResultWindowInfo winInfo;
COUNT_TYPE* pWindowCount;
} SCountWindowInfo;
typedef enum {
NONE_WINDOW = 0,
CREATE_NEW_WINDOW,
MOVE_NEXT_WINDOW,
} BuffOp;
typedef struct SBuffInfo {
bool rebuildWindow;
BuffOp winBuffOp;
SStreamStateCur* pCur;
} SBuffInfo;
void destroyStreamCountAggOperatorInfo(void* param) {
SStreamCountAggOperatorInfo* pInfo = (SStreamCountAggOperatorInfo*)param;
cleanupBasicInfo(&pInfo->binfo);
destroyStreamAggSupporter(&pInfo->streamAggSup);
cleanupExprSupp(&pInfo->scalarSupp);
clearGroupResInfo(&pInfo->groupResInfo);
colDataDestroy(&pInfo->twAggSup.timeWindowData);
blockDataDestroy(pInfo->pDelRes);
tSimpleHashCleanup(pInfo->pStUpdated);
tSimpleHashCleanup(pInfo->pStDeleted);
pInfo->pUpdated = taosArrayDestroy(pInfo->pUpdated);
cleanupGroupResInfo(&pInfo->groupResInfo);
taosArrayDestroy(pInfo->historyWins);
blockDataDestroy(pInfo->pCheckpointRes);
taosMemoryFreeClear(param);
}
bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) {
return pAggSup->windowCount != pAggSup->windowSliding;
}
void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin,
SBuffInfo* pBuffInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t size = pAggSup->resultRowSize;
pCurWin->winInfo.sessionWin.groupId = groupId;
pCurWin->winInfo.sessionWin.win.skey = ts;
pCurWin->winInfo.sessionWin.win.ekey = ts;
if (isSlidingCountWindow(pAggSup)) {
if (pBuffInfo->winBuffOp == CREATE_NEW_WINDOW) {
pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
code = TSDB_CODE_FAILED;
} else if (pBuffInfo->winBuffOp == MOVE_NEXT_WINDOW) {
ASSERT(pBuffInfo->pCur);
pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur);
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
if (code == TSDB_CODE_FAILED) {
pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
}
} else {
pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount);
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
if (code == TSDB_CODE_FAILED) {
pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
}
}
} else {
code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(
pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size);
}
if (code == TSDB_CODE_SUCCESS) {
pCurWin->winInfo.isOutput = true;
}
pCurWin->pWindowCount=
(COUNT_TYPE*) ((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE)));
if (*pCurWin->pWindowCount == pAggSup->windowCount) {
pBuffInfo->rebuildWindow = true;
}
}
static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowInfo* pWinInfo, TSKEY* pTs, int32_t start, int32_t rows, int32_t maxRows,
SSHashObj* pStDeleted, bool* pRebuild) {
SSessionKey sWinKey = pWinInfo->winInfo.sessionWin;
int32_t num = 0;
for (int32_t i = start; i < rows; i++) {
if (pTs[i] < pWinInfo->winInfo.sessionWin.win.ekey) {
num++;
} else {
break;
}
}
int32_t maxNum = TMIN(maxRows - *pWinInfo->pWindowCount, rows - start);
if (num > maxNum) {
*pRebuild = true;
}
*pWinInfo->pWindowCount += maxNum;
bool needDelState = false;
if (pWinInfo->winInfo.sessionWin.win.skey > pTs[start]) {
needDelState = true;
if (pStDeleted && pWinInfo->winInfo.isOutput) {
saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin);
}
pWinInfo->winInfo.sessionWin.win.skey = pTs[start];
}
if (pWinInfo->winInfo.sessionWin.win.ekey < pTs[maxNum + start - 1]) {
needDelState = true;
pWinInfo->winInfo.sessionWin.win.ekey = pTs[maxNum + start - 1];
}
if (needDelState) {
memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey));
if (pWinInfo->winInfo.pStatePos->needFree) {
pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &sWinKey);
}
}
return maxNum;
}
void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, EStreamType mode, SSessionKey* pDelRange) {
*pDelRange = *pKey;
SStreamStateCur* pCur = NULL;
if (isSlidingCountWindow(pAggSup)) {
pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, pKey, pAggSup->windowCount);
} else {
pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, pKey);
}
SSessionKey tmpKey = {0};
int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0);
if (code != TSDB_CODE_SUCCESS) {
pAggSup->stateStore.streamStateFreeCur(pCur);
return;
}
pDelRange->win = tmpKey.win;
while (mode == STREAM_DELETE_DATA) {
pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur);
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0);
if (code != TSDB_CODE_SUCCESS) {
break;
}
pDelRange->win.ekey = TMAX(pDelRange->win.ekey, tmpKey.win.ekey);
}
pAggSup->stateStore.streamStateFreeCur(pCur);
}
static void destroySBuffInfo(SStreamAggSupporter* pAggSup, SBuffInfo* pBuffInfo) {
pAggSup->stateStore.streamStateFreeCur(pBuffInfo->pCur);
}
bool inCountCalSlidingWindow(SStreamAggSupporter* pAggSup, STimeWindow* pWin, TSKEY sKey, TSKEY eKey) {
if (pAggSup->windowCount == pAggSup->windowSliding) {
return true;
}
if (sKey <= pWin->skey && pWin->ekey <= eKey) {
return true;
}
return false;
}
bool inCountSlidingWindow(SStreamAggSupporter* pAggSup, STimeWindow* pWin, SDataBlockInfo* pBlockInfo) {
return inCountCalSlidingWindow(pAggSup, pWin, pBlockInfo->calWin.skey, pBlockInfo->calWin.ekey);
}
static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pStUpdated,
SSHashObj* pStDeleted) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
uint64_t groupId = pSDataBlock->info.id.groupId;
int64_t code = TSDB_CODE_SUCCESS;
SResultRow* pResult = NULL;
int32_t rows = pSDataBlock->info.rows;
int32_t winRows = 0;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL};
pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
if (pAggSup->winRange.ekey <= 0) {
pAggSup->winRange.ekey = INT64_MAX;
}
SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
TSKEY* startTsCols = (int64_t*)pStartTsCol->pData;
blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2);
SStreamStateCur* pCur = NULL;
COUNT_TYPE slidingRows = 0;
for (int32_t i = 0; i < rows;) {
if (pInfo->ignoreExpiredData &&
checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, &pInfo->twAggSup,
pSDataBlock->info.id.uid, startTsCols[i])) {
i++;
continue;
}
SCountWindowInfo curWin = {0};
buffInfo.rebuildWindow = false;
setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo);
if (!inCountSlidingWindow(pAggSup, &curWin.winInfo.sessionWin.win, &pSDataBlock->info)) {
buffInfo.winBuffOp = MOVE_NEXT_WINDOW;
continue;
}
setSessionWinOutputInfo(pStUpdated, &curWin.winInfo);
slidingRows = *curWin.pWindowCount;
if (!buffInfo.rebuildWindow) {
winRows = updateCountWindowInfo(pAggSup, &curWin, startTsCols, i, rows, pAggSup->windowCount, pStDeleted, &buffInfo.rebuildWindow);
}
if (buffInfo.rebuildWindow) {
SSessionKey range = {0};
if (isSlidingCountWindow(pAggSup)) {
curWin.winInfo.sessionWin.win.skey = startTsCols[i];
curWin.winInfo.sessionWin.win.ekey = startTsCols[i];
}
getCountWinRange(pAggSup, &curWin.winInfo.sessionWin, STREAM_DELETE_DATA, &range);
range.win.skey = TMIN(startTsCols[i], range.win.skey);
range.win.ekey = TMAX(startTsCols[rows-1], range.win.ekey);
uint64_t uid = 0;
appendOneRowToStreamSpecialBlock(pAggSup->pScanBlock, &range.win.skey, &range.win.ekey, &uid, &range.groupId, NULL);
break;
}
code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,
pOperator, 0);
if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
qError("%s do stream count aggregate impl error, code %s", GET_TASKID(pTaskInfo), tstrerror(code));
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
}
saveSessionOutputBuf(pAggSup, &curWin.winInfo);
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
code = saveResult(curWin.winInfo, pStUpdated);
if (code != TSDB_CODE_SUCCESS) {
qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo),
tstrerror(code));
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
}
}
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
SSessionKey key = {0};
getSessionHashKey(&curWin.winInfo.sessionWin, &key);
tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
}
if (isSlidingCountWindow(pAggSup)) {
if (slidingRows <= pAggSup->windowSliding) {
if (slidingRows + winRows > pAggSup->windowSliding) {
buffInfo.winBuffOp = CREATE_NEW_WINDOW;
winRows = pAggSup->windowSliding - slidingRows;
ASSERT(i >= 0);
}
} else {
buffInfo.winBuffOp = MOVE_NEXT_WINDOW;
winRows = 0;
}
}
i += winRows;
}
destroySBuffInfo(pAggSup, &buffInfo);
}
static SSDataBlock* buildCountResult(SOperatorInfo* pOperator) {
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
return pInfo->pDelRes;
}
doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
if (pBInfo->pRes->info.rows > 0) {
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
return pBInfo->pRes;
}
return NULL;
}
int32_t doStreamCountEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator, bool isParent) {
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
if (!pInfo) {
return 0;
}
void* pData = (buf == NULL) ? NULL : *buf;
// 1.streamAggSup.pResultRows
int32_t tlen = 0;
int32_t mapSize = tSimpleHashGetSize(pInfo->streamAggSup.pResultRows);
tlen += taosEncodeFixedI32(buf, mapSize);
void* pIte = NULL;
size_t keyLen = 0;
int32_t iter = 0;
while ((pIte = tSimpleHashIterate(pInfo->streamAggSup.pResultRows, pIte, &iter)) != NULL) {
void* key = taosHashGetKey(pIte, &keyLen);
tlen += encodeSSessionKey(buf, key);
tlen += encodeSResultWindowInfo(buf, pIte, pInfo->streamAggSup.resultRowSize);
}
// 2.twAggSup
tlen += encodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);
// 3.dataVersion
tlen += taosEncodeFixedI32(buf, pInfo->dataVersion);
// 4.checksum
if (isParent) {
if (buf) {
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
tlen += taosEncodeFixedU32(buf, cksum);
} else {
tlen += sizeof(uint32_t);
}
}
return tlen;
}
void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent) {
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
if (!pInfo) {
return buf;
}
// 4.checksum
if (isParent) {
int32_t dataLen = len - sizeof(uint32_t);
void* pCksum = POINTER_SHIFT(buf, dataLen);
if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {
qError("stream count state is invalid");
return buf;
}
}
// 1.streamAggSup.pResultRows
int32_t mapSize = 0;
buf = taosDecodeFixedI32(buf, &mapSize);
for (int32_t i = 0; i < mapSize; i++) {
SSessionKey key = {0};
SResultWindowInfo winfo = {0};
buf = decodeSSessionKey(buf, &key);
buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);
tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo));
}
// 2.twAggSup
buf = decodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);
// 3.dataVersion
buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);
return buf;
}
void doStreamCountSaveCheckpoint(SOperatorInfo* pOperator) {
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
int32_t len = doStreamCountEncodeOpState(NULL, 0, pOperator, true);
void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf;
len = doStreamCountEncodeOpState(&pBuf, len, pOperator, true);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME,
strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), buf, len);
taosMemoryFree(buf);
}
void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* startDatas = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
TSKEY* endDatas = (TSKEY*)pEndTsCol->pData;
SColumnInfoData* pCalStartTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
TSKEY* calStartDatas = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pCalEndTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
TSKEY* calEndDatas = (TSKEY*)pEndTsCol->pData;
SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* gpDatas = (uint64_t*)pGroupCol->pData;
SRowBuffPos* pPos = NULL;
int32_t size = 0;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]};
SStreamStateCur* pCur = NULL;
if (isSlidingCountWindow(pAggSup)) {
pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &key, pAggSup->windowCount);
} else {
pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, &key);
}
while (1) {
SSessionKey tmpKey = {0};
int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void **)&pPos, &size);
if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) {
pAggSup->stateStore.streamStateFreeCur(pCur);
break;
}
if (!inCountCalSlidingWindow(pAggSup, &tmpKey.win, calStartDatas[i], calEndDatas[i])) {
pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur);
continue;
}
pAggSup->stateStore.streamStateSessionReset(pAggSup->pState, pPos->pRowBuff);
pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur);
}
}
}
void doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) {
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* startDatas = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
TSKEY* endDatas = (TSKEY*)pEndTsCol->pData;
SColumnInfoData* pCalStartTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
TSKEY* calStartDatas = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pCalEndTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
TSKEY* calEndDatas = (TSKEY*)pEndTsCol->pData;
SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* gpDatas = (uint64_t*)pGroupCol->pData;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
SSessionKey key = {.win.skey = startDatas[i], .win.ekey = endDatas[i], .groupId = gpDatas[i]};
while (1) {
SSessionKey curWin = {0};
int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin);
if (code == TSDB_CODE_FAILED) {
break;
}
doDeleteSessionWindow(pAggSup, &curWin);
if (result) {
saveDeleteInfo(result, curWin);
}
}
}
}
void deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate,
SSHashObj* pMapDelete) {
SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));
if (isSlidingCountWindow(pAggSup)) {
doDeleteCountWindows(pAggSup, pBlock, pWins);
} else {
doDeleteTimeWindows(pAggSup, pBlock, pWins);
}
removeSessionResults(pAggSup, pMapUpdate, pWins);
copyDeleteWindowInfo(pWins, pMapDelete);
taosArrayDestroy(pWins);
}
static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
SExprSupp* pSup = &pOperator->exprSupp;
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
qDebug("stask:%s %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
} else if (pOperator->status == OP_RES_TO_RETURN) {
SSDataBlock* opRes = buildCountResult(pOperator);
if (opRes) {
return opRes;
}
if (pInfo->recvGetAll) {
pInfo->recvGetAll = false;
resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows);
}
if (pInfo->reCkBlock) {
pInfo->reCkBlock = false;
printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
return pInfo->pCheckpointRes;
}
setStreamOperatorCompleted(pOperator);
return NULL;
}
SOperatorInfo* downstream = pOperator->pDownstream[0];
if (!pInfo->pUpdated) {
pInfo->pUpdated = taosArrayInit(16, sizeof(SResultWindowInfo));
}
if (!pInfo->pStUpdated) {
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pStUpdated = tSimpleHashInit(64, hashFn);
}
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) {
break;
}
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
deleteCountWinState(&pInfo->streamAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted);
continue;
} else if (pBlock->info.type == STREAM_CLEAR) {
doResetCountWindows(&pInfo->streamAggSup, pBlock);
continue;
} else if (pBlock->info.type == STREAM_GET_ALL) {
pInfo->recvGetAll = true;
getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated);
continue;
} else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {
return pBlock;
} else if (pBlock->info.type == STREAM_CHECKPOINT) {
pAggSup->stateStore.streamStateCommit(pAggSup->pState);
doStreamCountSaveCheckpoint(pOperator);
copyDataBlock(pInfo->pCheckpointRes, pBlock);
continue;
} else {
ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type");
}
if (pInfo->scalarSupp.pExprInfo != NULL) {
SExprSupp* pExprSup = &pInfo->scalarSupp;
projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
}
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
doStreamCountAggImpl(pOperator, pBlock, pInfo->pStUpdated, pInfo->pStDeleted);
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);
}
// restore the value
pOperator->status = OP_RES_TO_RETURN;
closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated);
copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc);
removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated);
initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);
pInfo->pUpdated = NULL;
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
SSDataBlock* opRes = buildCountResult(pOperator);
if (opRes) {
return opRes;
}
setStreamOperatorCompleted(pOperator);
return NULL;
}
void streamCountReleaseState(SOperatorInfo* pOperator) {
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
int32_t resSize = sizeof(TSKEY);
char* pBuff = taosMemoryCalloc(1, resSize);
memcpy(pBuff, &pInfo->twAggSup.maxTs, sizeof(TSKEY));
qDebug("===stream=== count window operator relase state. ");
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_STATE_NAME,
strlen(STREAM_COUNT_OP_STATE_NAME), pBuff, resSize);
pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);
taosMemoryFreeClear(pBuff);
SOperatorInfo* downstream = pOperator->pDownstream[0];
if (downstream->fpSet.releaseStreamStateFn) {
downstream->fpSet.releaseStreamStateFn(downstream);
}
}
void streamCountReloadState(SOperatorInfo* pOperator) {
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
int32_t size = 0;
void* pBuf = NULL;
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME,
strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size);
TSKEY ts = *(TSKEY*)pBuf;
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);
taosMemoryFree(pBuf);
SOperatorInfo* downstream = pOperator->pDownstream[0];
if (downstream->fpSet.reloadStreamStateFn) {
downstream->fpSet.reloadStreamStateFn(downstream);
}
reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup);
}
SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) {
SCountWinodwPhysiNode* pCountNode = (SCountWinodwPhysiNode*)pPhyNode;
int32_t numOfCols = 0;
int32_t code = TSDB_CODE_OUT_OF_MEMORY;
SStreamCountAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamCountAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
pOperator->pTaskInfo = pTaskInfo;
initResultSizeInfo(&pOperator->resultInfo, 4096);
if (pCountNode->window.pExprs != NULL) {
int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pCountNode->window.pExprs, NULL, &numOfScalar);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
}
SExprSupp* pExpSup = &pOperator->exprSupp;
SExprInfo* pExprInfo = createExprInfo(pCountNode->window.pFuncs, NULL, &numOfCols);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0,
pTaskInfo->streamInfo.pState, sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle,
&pInfo->twAggSup, GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
pInfo->streamAggSup.windowCount = pCountNode->windowCount;
pInfo->streamAggSup.windowSliding = pCountNode->windowSliding;
pInfo->twAggSup = (STimeWindowAggSupp){
.waterMark = pCountNode->window.watermark,
.calTrigger = pCountNode->window.triggerType,
.maxTs = INT64_MIN,
.minTs = INT64_MAX,
};
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
pInfo->primaryTsIndex = ((SColumnNode*)pCountNode->window.pTspk)->slotId;
pInfo->binfo.pRes = pResBlock;
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pStDeleted = tSimpleHashInit(64, hashFn);
pInfo->pDelIterator = NULL;
pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
pInfo->ignoreExpiredData = pCountNode->window.igExpired;
pInfo->ignoreExpiredDataSaved = false;
pInfo->pUpdated = NULL;
pInfo->pStUpdated = NULL;
pInfo->dataVersion = 0;
pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));
if (!pInfo->historyWins) {
goto _error;
}
pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
pInfo->recvGetAll = false;
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT;
// for stream
void* buff = NULL;
int32_t len = 0;
int32_t res =
pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME,
strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), &buff, &len);
if (res == TSDB_CODE_SUCCESS) {
doStreamCountDecodeOpState(buff, len, pOperator, true);
taosMemoryFree(buff);
}
setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, true,
OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamCountAgg, NULL, destroyStreamCountAggOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState);
if (downstream) {
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup);
code = appendDownstream(pOperator, &downstream, 1);
}
return pOperator;
_error:
if (pInfo != NULL) {
destroyStreamCountAggOperatorInfo(pInfo);
}
taosMemoryFreeClear(pOperator);
pTaskInfo->code = code;
return NULL;
}

4
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -1945,7 +1945,7 @@ int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo*
return TSDB_CODE_SUCCESS;
}
static bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) {
bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) {
pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey);
SSessionKey hashKey = {0};
getSessionHashKey(pKey, &hashKey);
@ -2343,7 +2343,7 @@ int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) {
return TSDB_CODE_SUCCESS;
}
static void copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) {
void copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) {
int32_t size = taosArrayGetSize(pResWins);
for (int32_t i = 0; i < size; i++) {
SSessionKey* pWinKey = taosArrayGet(pResWins, i);

12
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -329,6 +329,13 @@ static int32_t eventWindowNodeCopy(const SEventWindowNode* pSrc, SEventWindowNod
return TSDB_CODE_SUCCESS;
}
static int32_t countWindowNodeCopy(const SCountWindowNode* pSrc, SCountWindowNode* pDst) {
CLONE_NODE_FIELD(pCol);
COPY_SCALAR_FIELD(windowCount);
COPY_SCALAR_FIELD(windowSliding);
return TSDB_CODE_SUCCESS;
}
static int32_t sessionWindowNodeCopy(const SSessionWindowNode* pSrc, SSessionWindowNode* pDst) {
CLONE_NODE_FIELD_EX(pCol, SColumnNode*);
CLONE_NODE_FIELD_EX(pGap, SValueNode*);
@ -551,6 +558,8 @@ static int32_t logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* p
COPY_SCALAR_FIELD(igExpired);
COPY_SCALAR_FIELD(igCheckUpdate);
COPY_SCALAR_FIELD(windowAlgo);
COPY_SCALAR_FIELD(windowCount);
COPY_SCALAR_FIELD(windowSliding);
return TSDB_CODE_SUCCESS;
}
@ -853,6 +862,9 @@ SNode* nodesCloneNode(const SNode* pNode) {
case QUERY_NODE_EVENT_WINDOW:
code = eventWindowNodeCopy((const SEventWindowNode*)pNode, (SEventWindowNode*)pDst);
break;
case QUERY_NODE_COUNT_WINDOW:
code = countWindowNodeCopy((const SCountWindowNode*)pNode, (SCountWindowNode*)pDst);
break;
case QUERY_NODE_SESSION_WINDOW:
code = sessionWindowNodeCopy((const SSessionWindowNode*)pNode, (SSessionWindowNode*)pDst);
break;

76
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -91,6 +91,8 @@ const char* nodesNodeName(ENodeType type) {
return "CaseWhen";
case QUERY_NODE_EVENT_WINDOW:
return "EventWindow";
case QUERY_NODE_COUNT_WINDOW:
return "CountWindow";
case QUERY_NODE_SET_OPERATOR:
return "SetOperator";
case QUERY_NODE_SELECT_STMT:
@ -343,6 +345,10 @@ const char* nodesNodeName(ENodeType type) {
return "PhysiMergeEventWindow";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT:
return "PhysiStreamEventWindow";
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
return "PhysiMergeCountWindow";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
return "PhysiStreamCountWindow";
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return "PhysiProject";
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN:
@ -2745,6 +2751,36 @@ static int32_t jsonToPhysiEventWindowNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkCountWindowPhysiPlanWindowCount = "WindowCount";
static const char* jkCountWindowPhysiPlanWindowSliding = "WindowSliding";
static int32_t physiCountWindowNodeToJson(const void* pObj, SJson* pJson) {
const SCountWinodwPhysiNode* pNode = (const SCountWinodwPhysiNode*)pObj;
int32_t code = physiWindowNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkCountWindowPhysiPlanWindowCount, pNode->windowCount);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkCountWindowPhysiPlanWindowSliding, pNode->windowSliding);
}
return code;
}
static int32_t jsonToPhysiCountWindowNode(const SJson* pJson, void* pObj) {
SCountWinodwPhysiNode* pNode = (SCountWinodwPhysiNode*)pObj;
int32_t code = jsonToPhysiWindowNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkCountWindowPhysiPlanWindowCount, &pNode->windowCount);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkCountWindowPhysiPlanWindowSliding, &pNode->windowSliding);
}
return code;
}
static const char* jkPartitionPhysiPlanExprs = "Exprs";
static const char* jkPartitionPhysiPlanPartitionKeys = "PartitionKeys";
static const char* jkPartitionPhysiPlanTargets = "Targets";
@ -4408,6 +4444,36 @@ static int32_t jsonToEventWindowNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkCountWindowTsPrimaryKey = "CountTsPrimaryKey";
static const char* jkCountWindowCount = "CountWindowCount";
static const char* jkCountWindowSliding = "CountWindowSliding";
static int32_t countWindowNodeToJson(const void* pObj, SJson* pJson) {
const SCountWindowNode* pNode = (const SCountWindowNode*)pObj;
int32_t code = tjsonAddObject(pJson, jkCountWindowTsPrimaryKey, nodeToJson, pNode->pCol);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkCountWindowCount, pNode->windowCount);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkCountWindowSliding, pNode->windowSliding);
}
return code;
}
static int32_t jsonToCountWindowNode(const SJson* pJson, void* pObj) {
SCountWindowNode* pNode = (SCountWindowNode*)pObj;
int32_t code = jsonToNodeObject(pJson, jkCountWindowTsPrimaryKey, &pNode->pCol);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkCountWindowCount, &pNode->windowCount);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkCountWindowSliding, &pNode->windowSliding);
}
return code;
}
static const char* jkIntervalWindowInterval = "Interval";
static const char* jkIntervalWindowOffset = "Offset";
static const char* jkIntervalWindowSliding = "Sliding";
@ -6985,6 +7051,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return caseWhenNodeToJson(pObj, pJson);
case QUERY_NODE_EVENT_WINDOW:
return eventWindowNodeToJson(pObj, pJson);
case QUERY_NODE_COUNT_WINDOW:
return countWindowNodeToJson(pObj, pJson);
case QUERY_NODE_SET_OPERATOR:
return setOperatorToJson(pObj, pJson);
case QUERY_NODE_SELECT_STMT:
@ -7224,6 +7292,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT:
return physiEventWindowNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
return physiCountWindowNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
return physiPartitionNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
@ -7317,6 +7388,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToCaseWhenNode(pJson, pObj);
case QUERY_NODE_EVENT_WINDOW:
return jsonToEventWindowNode(pJson, pObj);
case QUERY_NODE_COUNT_WINDOW:
return jsonToCountWindowNode(pJson, pObj);
case QUERY_NODE_SET_OPERATOR:
return jsonToSetOperator(pJson, pObj);
case QUERY_NODE_SELECT_STMT:
@ -7564,6 +7637,9 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT:
return jsonToPhysiEventWindowNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
return jsonToPhysiCountWindowNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
return jsonToPhysiPartitionNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:

48
source/libs/nodes/src/nodesMsgFuncs.c
View File

@ -3210,6 +3210,46 @@ static int32_t msgToPhysiEventWindowNode(STlvDecoder* pDecoder, void* pObj) {
return code;
}
enum { PHY_COUNT_CODE_WINDOW = 1, PHY_COUNT_CODE_WINDOW_COUNT, PHY_COUNT_CODE_WINDOW_SLIDING };
static int32_t physiCountWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
const SCountWinodwPhysiNode* pNode = (const SCountWinodwPhysiNode*)pObj;
int32_t code = tlvEncodeObj(pEncoder, PHY_COUNT_CODE_WINDOW, physiWindowNodeToMsg, &pNode->window);
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI64(pEncoder, PHY_COUNT_CODE_WINDOW_COUNT, pNode->windowCount);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI64(pEncoder, PHY_COUNT_CODE_WINDOW_SLIDING, pNode->windowSliding);
}
return code;
}
static int32_t msgToPhysiCountWindowNode(STlvDecoder* pDecoder, void* pObj) {
SCountWinodwPhysiNode* pNode = (SCountWinodwPhysiNode*)pObj;
int32_t code = TSDB_CODE_SUCCESS;
STlv* pTlv = NULL;
tlvForEach(pDecoder, pTlv, code) {
switch (pTlv->type) {
case PHY_COUNT_CODE_WINDOW:
code = tlvDecodeObjFromTlv(pTlv, msgToPhysiWindowNode, &pNode->window);
break;
case PHY_COUNT_CODE_WINDOW_COUNT:
code = tlvDecodeI64(pTlv, &pNode->windowCount);
break;
case PHY_COUNT_CODE_WINDOW_SLIDING:
code = tlvDecodeI64(pTlv, &pNode->windowSliding);
break;
default:
break;
}
}
return code;
}
enum {
PHY_PARTITION_CODE_BASE_NODE = 1,
PHY_PARTITION_CODE_EXPR,
@ -4200,6 +4240,10 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT:
code = physiEventWindowNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
code = physiCountWindowNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
code = physiPartitionNodeToMsg(pObj, pEncoder);
break;
@ -4355,6 +4399,10 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT:
code = msgToPhysiEventWindowNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
code = msgToPhysiCountWindowNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
code = msgToPhysiPartitionNode(pDecoder, pObj);
break;

10
source/libs/nodes/src/nodesTraverseFuncs.c
View File

@ -176,6 +176,11 @@ static EDealRes dispatchExpr(SNode* pNode, ETraversalOrder order, FNodeWalker wa
}
break;
}
case QUERY_NODE_COUNT_WINDOW: {
SCountWindowNode* pEvent = (SCountWindowNode*)pNode;
res = walkExpr(pEvent->pCol, order, walker, pContext);
break;
}
default:
break;
}
@ -367,6 +372,11 @@ static EDealRes rewriteExpr(SNode** pRawNode, ETraversalOrder order, FNodeRewrit
}
break;
}
case QUERY_NODE_COUNT_WINDOW: {
SCountWindowNode* pEvent = (SCountWindowNode*)pNode;
res = rewriteExpr(&pEvent->pCol, order, rewriter, pContext);
break;
}
default:
break;
}

17
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -302,6 +302,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SCaseWhenNode));
case QUERY_NODE_EVENT_WINDOW:
return makeNode(type, sizeof(SEventWindowNode));
case QUERY_NODE_COUNT_WINDOW:
return makeNode(type, sizeof(SCountWindowNode));
case QUERY_NODE_HINT:
return makeNode(type, sizeof(SHintNode));
case QUERY_NODE_VIEW:
@ -585,6 +587,10 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SEventWinodwPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT:
return makeNode(type, sizeof(SStreamEventWinodwPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
return makeNode(type, sizeof(SCountWinodwPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT:
return makeNode(type, sizeof(SStreamCountWinodwPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
return makeNode(type, sizeof(SPartitionPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
@ -850,6 +856,11 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pEvent->pEndCond);
break;
}
case QUERY_NODE_COUNT_WINDOW: {
SCountWindowNode* pEvent = (SCountWindowNode*)pNode;
nodesDestroyNode(pEvent->pCol);
break;
}
case QUERY_NODE_HINT: {
SHintNode* pHint = (SHintNode*)pNode;
destroyHintValue(pHint->option, pHint->value);
@ -1427,6 +1438,12 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pPhyNode->pEndCond);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT: {
SCountWinodwPhysiNode* pPhyNode = (SCountWinodwPhysiNode*)pNode;
destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_PARTITION: {
destroyPartitionPhysiNode((SPartitionPhysiNode*)pNode);
break;

1
source/libs/parser/inc/parAst.h
View File

@ -131,6 +131,7 @@ SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order
SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap);
SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr);
SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond);
SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken);
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
SNode* pFill);
SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);

4
source/libs/parser/inc/sql.y
View File

@ -1159,6 +1159,10 @@ twindow_clause_opt(A) ::=
sliding_opt(D) fill_opt(E). { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), D, E); }
twindow_clause_opt(A) ::=
EVENT_WINDOW START WITH search_condition(B) END WITH search_condition(C). { A = createEventWindowNode(pCxt, B, C); }
twindow_clause_opt(A) ::=
COUNT_WINDOW NK_LP NK_INTEGER(B) NK_RP. { A = createCountWindowNode(pCxt, &B, &B); }
twindow_clause_opt(A) ::=
COUNT_WINDOW NK_LP NK_INTEGER(B) NK_COMMA NK_INTEGER(C) NK_RP. { A = createCountWindowNode(pCxt, &B, &C); }
sliding_opt(A) ::= . { A = NULL; }
sliding_opt(A) ::= SLIDING NK_LP interval_sliding_duration_literal(B) NK_RP. { A = releaseRawExprNode(pCxt, B); }

14
source/libs/parser/src/parAstCreater.c
View File

@ -883,6 +883,20 @@ SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode*
return (SNode*)pEvent;
}
SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken) {
CHECK_PARSER_STATUS(pCxt);
SCountWindowNode* pCount = (SCountWindowNode*)nodesMakeNode(QUERY_NODE_COUNT_WINDOW);
CHECK_OUT_OF_MEM(pCount);
pCount->pCol = createPrimaryKeyCol(pCxt, NULL);
if (NULL == pCount->pCol) {
nodesDestroyNode((SNode*)pCount);
CHECK_OUT_OF_MEM(NULL);
}
pCount->windowCount = taosStr2Int64(pCountToken->z, NULL, 10);
pCount->windowSliding = taosStr2Int64(pSlidingToken->z, NULL, 10);
return (SNode*)pCount;
}
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
SNode* pFill) {
CHECK_PARSER_STATUS(pCxt);

1
source/libs/parser/src/parTokenizer.c
View File

@ -72,6 +72,7 @@ static SKeyword keywordTable[] = {
{"CONSUMERS", TK_CONSUMERS},
{"CONTAINS", TK_CONTAINS},
{"COUNT", TK_COUNT},
{"COUNT_WINDOW", TK_COUNT_WINDOW},
{"CREATE", TK_CREATE},
{"CURRENT_USER", TK_CURRENT_USER},
{"DATABASE", TK_DATABASE},

58
source/libs/parser/src/parTranslater.c
View File

@ -4002,6 +4002,15 @@ static int32_t translateEventWindow(STranslateContext* pCxt, SSelectStmt* pSelec
return TSDB_CODE_SUCCESS;
}
static int32_t translateCountWindow(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) &&
!isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TIMELINE_QUERY,
"COUNT_WINDOW requires valid time series input");
}
return TSDB_CODE_SUCCESS;
}
static int32_t translateSpecificWindow(STranslateContext* pCxt, SSelectStmt* pSelect) {
switch (nodeType(pSelect->pWindow)) {
case QUERY_NODE_STATE_WINDOW:
@ -4012,6 +4021,8 @@ static int32_t translateSpecificWindow(STranslateContext* pCxt, SSelectStmt* pSe
return translateIntervalWindow(pCxt, pSelect);
case QUERY_NODE_EVENT_WINDOW:
return translateEventWindow(pCxt, pSelect);
case QUERY_NODE_COUNT_WINDOW:
return translateCountWindow(pCxt, pSelect);
default:
break;
}
@ -7787,6 +7798,53 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"The trigger mode of non window query can only be AT_ONCE");
}
if (pSelect->pWindow != NULL && pSelect->pWindow->type == QUERY_NODE_COUNT_WINDOW) {
if ( (SRealTableNode*)pSelect->pFromTable && ((SRealTableNode*)pSelect->pFromTable)->pMeta
&& TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType
&& !hasPartitionByTbname(pSelect->pPartitionByList) ) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"Count window for stream on super table must patitioned by table name");
}
int64_t watermark = 0;
if (pStmt->pOptions->pWatermark) {
translateValue(pCxt, (SValueNode*)pStmt->pOptions->pWatermark);
watermark =((SValueNode*)pStmt->pOptions->pWatermark)->datum.i;
}
if (watermark <= 0) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"Watermark of Count window must exceed 0.");
}
if (pStmt->pOptions->ignoreExpired != 1) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"Ignore expired data of Count window must be 1.");
}
SCountWindowNode* pCountWin = (SCountWindowNode*)pSelect->pWindow;
if (pCountWin->windowCount <= 1) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"Size of Count window must exceed 1.");
}
if (pCountWin->windowSliding <= 0) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"Size of Count window must exceed 0.");
}
if (pCountWin->windowSliding > pCountWin->windowCount) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"sliding value no larger than the count value.");
}
if (pCountWin->windowCount > INT32_MAX) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"Size of Count window must less than 2147483647(INT32_MAX).");
}
}
return TSDB_CODE_SUCCESS;
}

6528
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

25
source/libs/planner/src/planLogicCreater.c
View File

@ -1008,6 +1008,29 @@ static int32_t createWindowLogicNodeByEvent(SLogicPlanContext* pCxt, SEventWindo
return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode);
}
static int32_t createWindowLogicNodeByCount(SLogicPlanContext* pCxt, SCountWindowNode* pCount, SSelectStmt* pSelect,
SLogicNode** pLogicNode) {
SWindowLogicNode* pWindow = (SWindowLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW);
if (NULL == pWindow) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pWindow->winType = WINDOW_TYPE_COUNT;
pWindow->node.groupAction = getGroupAction(pCxt, pSelect);
pWindow->node.requireDataOrder =
pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : getRequireDataOrder(true, pSelect);
pWindow->node.resultDataOrder =
pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_GLOBAL : pWindow->node.requireDataOrder;
pWindow->windowCount = pCount->windowCount;
pWindow->windowSliding = pCount->windowSliding;
pWindow->pTspk = nodesCloneNode(pCount->pCol);
if (NULL == pWindow->pTspk) {
nodesDestroyNode((SNode*)pWindow);
return TSDB_CODE_OUT_OF_MEMORY;
}
return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode);
}
static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
if (NULL == pSelect->pWindow) {
return TSDB_CODE_SUCCESS;
@ -1021,6 +1044,8 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele
return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect, pLogicNode);
case QUERY_NODE_EVENT_WINDOW:
return createWindowLogicNodeByEvent(pCxt, (SEventWindowNode*)pSelect->pWindow, pSelect, pLogicNode);
case QUERY_NODE_COUNT_WINDOW:
return createWindowLogicNodeByCount(pCxt, (SCountWindowNode*)pSelect->pWindow, pSelect, pLogicNode);
default:
break;
}

23
source/libs/planner/src/planPhysiCreater.c
View File

@ -1751,6 +1751,27 @@ static int32_t createEventWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC
return code;
}
static int32_t createCountWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) {
SCountWinodwPhysiNode* pCount = (SCountWinodwPhysiNode*)makePhysiNode(
pCxt, (SLogicNode*)pWindowLogicNode,
(pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT : QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT));
if (NULL == pCount) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCount->windowCount = pWindowLogicNode->windowCount;
pCount->windowSliding = pWindowLogicNode->windowSliding;
int32_t code = createWindowPhysiNodeFinalize(pCxt, pChildren, &pCount->window, pWindowLogicNode);
if (TSDB_CODE_SUCCESS == code) {
*pPhyNode = (SPhysiNode*)pCount;
} else {
nodesDestroyNode((SNode*)pCount);
}
return code;
}
static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode,
SPhysiNode** pPhyNode) {
switch (pWindowLogicNode->winType) {
@ -1762,6 +1783,8 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr
return createStateWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode);
case WINDOW_TYPE_EVENT:
return createEventWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode);
case WINDOW_TYPE_COUNT:
return createCountWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode);
default:
break;
}

14
source/libs/planner/src/planSpliter.c
View File

@ -872,6 +872,18 @@ static int32_t stbSplSplitEvent(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
}
}
static int32_t stbSplSplitCountForStream(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
static int32_t stbSplSplitCount(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
if (pCxt->pPlanCxt->streamQuery) {
return stbSplSplitCountForStream(pCxt, pInfo);
} else {
return stbSplSplitSessionOrStateForBatch(pCxt, pInfo);
}
}
static int32_t stbSplSplitWindowForCrossTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
switch (((SWindowLogicNode*)pInfo->pSplitNode)->winType) {
case WINDOW_TYPE_INTERVAL:
@ -882,6 +894,8 @@ static int32_t stbSplSplitWindowForCrossTable(SSplitContext* pCxt, SStableSplitI
return stbSplSplitState(pCxt, pInfo);
case WINDOW_TYPE_EVENT:
return stbSplSplitEvent(pCxt, pInfo);
case WINDOW_TYPE_COUNT:
return stbSplSplitCount(pCxt, pInfo);
default:
break;
}

11
source/libs/planner/src/planUtil.c
View File

@ -237,6 +237,15 @@ static int32_t adjustEventDataRequirement(SWindowLogicNode* pWindow, EDataOrderL
return TSDB_CODE_SUCCESS;
}
static int32_t adjustCountDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) {
if (requirement <= pWindow->node.resultDataOrder) {
return TSDB_CODE_SUCCESS;
}
pWindow->node.resultDataOrder = requirement;
pWindow->node.requireDataOrder = requirement;
return TSDB_CODE_SUCCESS;
}
static int32_t adjustWindowDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) {
switch (pWindow->winType) {
case WINDOW_TYPE_INTERVAL:
@ -247,6 +256,8 @@ static int32_t adjustWindowDataRequirement(SWindowLogicNode* pWindow, EDataOrder
return adjustStateDataRequirement(pWindow, requirement);
case WINDOW_TYPE_EVENT:
return adjustEventDataRequirement(pWindow, requirement);
case WINDOW_TYPE_COUNT:
return adjustCountDataRequirement(pWindow, requirement);
default:
break;
}

3
source/libs/stream/inc/streamBackendRocksdb.h
View File

@ -179,7 +179,8 @@ int32_t streamStateSessionDel_rocksdb(SStreamState* pState, const SSessionKey* k
SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, SSessionKey* key);
SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* streamStateSessionSeekToLast_rocksdb(SStreamState* pState);
SStreamStateCur* streamStateSessionSeekKeyPrev_rocksdb(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* streamStateSessionSeekToLast_rocksdb(SStreamState* pState, int64_t groupId);
int32_t streamStateSessionCurPrev_rocksdb(SStreamStateCur* pCur);
int32_t streamStateSessionGetKVByCur_rocksdb(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);

46
source/libs/stream/src/streamBackendRocksdb.c
View File

@ -2883,13 +2883,13 @@ int32_t streamStateSessionDel_rocksdb(SStreamState* pState, const SSessionKey* k
return code;
}
SStreamStateCur* streamStateSessionSeekToLast_rocksdb(SStreamState* pState) {
SStreamStateCur* streamStateSessionSeekToLast_rocksdb(SStreamState* pState, int64_t groupId) {
stDebug("streamStateSessionSeekToLast_rocksdb");
int32_t code = 0;
SSessionKey maxSessionKey = {.groupId = UINT64_MAX, .win = {.skey = INT64_MAX, .ekey = INT64_MAX}};
SStateSessionKey maxKey = {.key = maxSessionKey, .opNum = INT64_MAX};
SSessionKey maxSessionKey = {.groupId = groupId, .win = {.skey = INT64_MAX, .ekey = INT64_MAX}};
SStateSessionKey maxKey = {.key = maxSessionKey, .opNum = pState->number};
STREAM_STATE_PUT_ROCKSDB(pState, "sess", &maxKey, "", 0);
if (code != 0) {
@ -3048,6 +3048,46 @@ SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, con
return pCur;
}
SStreamStateCur* streamStateSessionSeekKeyPrev_rocksdb(SStreamState* pState, const SSessionKey* key) {
stDebug("streamStateSessionSeekKeyPrev_rocksdb");
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
SStreamStateCur* pCur = createStreamStateCursor();
if (pCur == NULL) {
return NULL;
}
pCur->db = wrapper->db;
pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot,
(rocksdb_readoptions_t**)&pCur->readOpt);
pCur->number = pState->number;
SStateSessionKey sKey = {.key = *key, .opNum = pState->number};
char buf[128] = {0};
int len = stateSessionKeyEncode(&sKey, buf);
if (!streamStateIterSeekAndValid(pCur->iter, buf, len)) {
streamStateFreeCur(pCur);
return NULL;
}
while (rocksdb_iter_valid(pCur->iter) && iterValueIsStale(pCur->iter)) rocksdb_iter_prev(pCur->iter);
if (!rocksdb_iter_valid(pCur->iter)) {
streamStateFreeCur(pCur);
return NULL;
}
size_t klen;
const char* iKey = rocksdb_iter_key(pCur->iter, &klen);
SStateSessionKey curKey = {0};
stateSessionKeyDecode(&curKey, (char*)iKey);
if (stateSessionKeyCmpr(&sKey, sizeof(sKey), &curKey, sizeof(curKey)) > 0) return pCur;
rocksdb_iter_prev(pCur->iter);
if (!rocksdb_iter_valid(pCur->iter)) {
streamStateFreeCur(pCur);
return NULL;
}
return pCur;
}
int32_t streamStateSessionGetKVByCur_rocksdb(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen) {
stDebug("streamStateSessionGetKVByCur_rocksdb");
if (!pCur) {

2
source/libs/stream/src/streamCheckpoint.c
View File

@ -512,7 +512,7 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
SStreamTaskId hTaskId = {.streamId = pTask->hTaskInfo.id.streamId, .taskId = pTask->hTaskInfo.id.taskId};
stDebug("s-task:%s fill-history finish checkpoint done, drop related fill-history task:0x%x", id, hTaskId.taskId);
streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pTask->pMeta->vgId, &hTaskId);
streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pTask->pMeta->vgId, &hTaskId, 1);
} else {
stWarn("s-task:%s related fill-history task:0x%x is erased", id, (int32_t)pTask->hTaskInfo.id.taskId);
}

2
source/libs/stream/src/streamExec.c
View File

@ -328,7 +328,7 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {
id, (int32_t) pTask->streamTaskId.taskId);
// 1. free it and remove fill-history task from disk meta-store
streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id);
streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0);
// 2. save to disk
streamMetaWLock(pMeta);

3
source/libs/stream/src/streamMeta.c
View File

@ -725,9 +725,6 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
// it is an fill-history task, remove the related stream task's id that points to it
atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1);
if (pTask->info.fillHistory == 1) {
streamTaskClearHTaskAttr(pTask, false);
}
taosHashRemove(pMeta->pTasksMap, &id, sizeof(id));
doRemoveIdFromList(pMeta, (int32_t)taosArrayGetSize(pMeta->pTaskList), &pTask->id);

230
source/libs/stream/src/streamSessionState.c
View File

@ -28,6 +28,12 @@ int sessionStateKeyCompare(const SSessionKey* pWin1, const void* pDatas, int pos
return sessionWinKeyCmpr(pWin1, pWin2);
}
int sessionStateRangeKeyCompare(const SSessionKey* pWin1, const void* pDatas, int pos) {
SRowBuffPos* pPos2 = taosArrayGetP(pDatas, pos);
SSessionKey* pWin2 = (SSessionKey*)pPos2->pKey;
return sessionRangeKeyCmpr(pWin1, pWin2);
}
int32_t binarySearch(void* keyList, int num, const void* key, __session_compare_fn_t cmpFn) {
int firstPos = 0, lastPos = num - 1, midPos = -1;
int numOfRows = 0;
@ -69,6 +75,12 @@ bool inSessionWindow(SSessionKey* pKey, TSKEY ts, int64_t gap) {
return false;
}
SStreamStateCur* createSessionStateCursor(SStreamFileState* pFileState) {
SStreamStateCur* pCur = createStreamStateCursor();
pCur->pStreamFileState = pFileState;
return pCur;
}
static SRowBuffPos* addNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey) {
SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState);
ASSERT(pNewPos->pRowBuff);
@ -91,7 +103,12 @@ SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKe
memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey));
pNewPos->needFree = true;
pNewPos->beFlushed = true;
if(p) {
memcpy(pNewPos->pRowBuff, p, *pVLen);
} else {
int32_t len = getRowStateRowSize(pFileState);
memset(pNewPos->pRowBuff, 0, len);
}
taosMemoryFree(p);
return pNewPos;
}
@ -364,9 +381,8 @@ static SStreamStateCur* seekKeyCurrentPrev_buff(SStreamFileState* pFileState, co
}
if (index >= 0) {
pCur = createStreamStateCursor();
pCur = createSessionStateCursor(pFileState);
pCur->buffIndex = index;
pCur->pStreamFileState = pFileState;
if (pIndex) {
*pIndex = index;
}
@ -405,7 +421,7 @@ static void checkAndTransformCursor(SStreamFileState* pFileState, const uint64_t
if (taosArrayGetSize(pWinStates) > 0 &&
(code == TSDB_CODE_FAILED || sessionStateKeyCompare(&key, pWinStates, 0) >= 0)) {
if (!(*ppCur)) {
(*ppCur) = createStreamStateCursor();
(*ppCur) = createSessionStateCursor(pFileState);
}
transformCursor(pFileState, *ppCur);
} else if (*ppCur) {
@ -419,7 +435,7 @@ SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState,
int32_t index = -1;
SStreamStateCur* pCur = seekKeyCurrentPrev_buff(pFileState, pWinKey, &pWinStates, &index);
if (pCur) {
if (sessionStateKeyCompare(pWinKey, pWinStates, index) > 0) {
if (sessionStateRangeKeyCompare(pWinKey, pWinStates, index) > 0) {
sessionWinStateMoveToNext(pCur);
}
return pCur;
@ -446,6 +462,66 @@ SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const
return pCur;
}
SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey, COUNT_TYPE count) {
SArray* pWinStates = NULL;
int32_t index = -1;
SStreamStateCur* pBuffCur = seekKeyCurrentPrev_buff(pFileState, pWinKey, &pWinStates, &index);
int32_t resSize = getRowStateRowSize(pFileState);
COUNT_TYPE winCount = 0;
if (pBuffCur) {
while (index >= 0) {
SRowBuffPos* pPos = taosArrayGetP(pWinStates, index);
winCount = *((COUNT_TYPE*) ((char*)pPos->pRowBuff + (resSize - sizeof(COUNT_TYPE))));
if (sessionStateRangeKeyCompare(pWinKey, pWinStates, index) == 0 || winCount < count) {
index--;
} else if (index >= 0) {
pBuffCur->buffIndex = index + 1;
return pBuffCur;
}
}
pBuffCur->buffIndex = 0;
} else if (taosArrayGetSize(pWinStates) > 0) {
pBuffCur = createSessionStateCursor(pFileState);
pBuffCur->buffIndex = 0;
}
void* pFileStore = getStateFileStore(pFileState);
SStreamStateCur* pCur = streamStateSessionSeekKeyPrev_rocksdb(pFileStore, pWinKey);
if (pCur) {
SSessionKey key = {0};
void* pVal = NULL;
int len = 0;
int32_t code = streamStateSessionGetKVByCur_rocksdb(pCur, &key, &pVal, &len);
if (code == TSDB_CODE_FAILED) {
streamStateFreeCur(pCur);
return pBuffCur;
}
winCount = *((COUNT_TYPE*) ((char*)pVal + (resSize - sizeof(COUNT_TYPE))));
if (sessionRangeKeyCmpr(pWinKey, &key) != 0 && winCount == count) {
streamStateFreeCur(pCur);
return pBuffCur;
}
streamStateCurPrev(pFileStore, pCur);
while (1) {
code = streamStateSessionGetKVByCur_rocksdb(pCur, &key, &pVal, &len);
if (code == TSDB_CODE_FAILED) {
streamStateCurNext(pFileStore, pCur);
streamStateFreeCur(pBuffCur);
return pCur;
}
winCount = *((COUNT_TYPE*) ((char*)pVal + (resSize - sizeof(COUNT_TYPE))));
if (sessionRangeKeyCmpr(pWinKey, &key) == 0 || winCount < count) {
streamStateCurPrev(pFileStore, pCur);
} else {
streamStateCurNext(pFileStore, pCur);
streamStateFreeCur(pBuffCur);
return pCur;
}
}
}
return pBuffCur;
}
int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen) {
if (!pCur) {
return TSDB_CODE_FAILED;
@ -503,7 +579,7 @@ int32_t sessionWinStateMoveToNext(SStreamStateCur* pCur) {
return TSDB_CODE_SUCCESS;
}
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey) {
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey, range_cmpr_fn cmpFn) {
SStreamStateCur* pCur = sessionWinStateSeekKeyCurrentPrev(pFileState, key);
SSessionKey tmpKey = *key;
int32_t code = sessionWinStateGetKVByCur(pCur, &tmpKey, NULL, NULL);
@ -520,7 +596,7 @@ int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessio
goto _end;
}
if (sessionRangeKeyCmpr(key, &tmpKey) == 0) {
if (cmpFn(key, &tmpKey) == 0) {
*curKey = tmpKey;
goto _end;
} else if (!hasCurrentPrev) {
@ -530,7 +606,7 @@ int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessio
sessionWinStateMoveToNext(pCur);
code = sessionWinStateGetKVByCur(pCur, &tmpKey, NULL, NULL);
if (code == TSDB_CODE_SUCCESS && sessionRangeKeyCmpr(key, &tmpKey) == 0) {
if (code == TSDB_CODE_SUCCESS && cmpFn(key, &tmpKey) == 0) {
*curKey = tmpKey;
} else {
code = TSDB_CODE_FAILED;
@ -636,3 +712,143 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch
_end:
return code;
}
int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen) {
SSessionKey* pWinKey = pKey;
const TSKEY gap = 0;
int32_t code = TSDB_CODE_SUCCESS;
SSHashObj* pSessionBuff = getRowStateBuff(pFileState);
SArray* pWinStates = NULL;
void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t));
if (ppBuff) {
pWinStates = (SArray*)(*ppBuff);
} else {
pWinStates = taosArrayInit(16, POINTER_BYTES);
tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES);
}
TSKEY startTs = pWinKey->win.skey;
TSKEY endTs = pWinKey->win.ekey;
int32_t size = taosArrayGetSize(pWinStates);
if (size == 0) {
void* pFileStore = getStateFileStore(pFileState);
void* pRockVal = NULL;
SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileStore, pKey->groupId);
code = streamStateSessionGetKVByCur_rocksdb(pCur, pWinKey, &pRockVal, pVLen);
if (code == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) {
qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code);
if (code == TSDB_CODE_SUCCESS) {
int32_t valSize = *pVLen;
COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)( (char*)(pRockVal) + (valSize - sizeof(COUNT_TYPE)) );
if (inSessionWindow(pWinKey, startTs, gap) || (*pWinStateCout) < winCount) {
(*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen);
streamStateFreeCur(pCur);
goto _end;
}
}
pWinKey->win.skey = startTs;
pWinKey->win.ekey = endTs;
(*pVal) = createSessionWinBuff(pFileState, pWinKey, NULL, NULL);
taosMemoryFree(pRockVal);
streamStateFreeCur(pCur);
} else {
(*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey);
code = TSDB_CODE_FAILED;
}
goto _end;
}
// find the first position which is smaller than the pWinKey
int32_t index = binarySearch(pWinStates, size, pWinKey, sessionStateKeyCompare);
SRowBuffPos* pPos = NULL;
int32_t valSize = *pVLen;
if (index >= 0) {
pPos = taosArrayGetP(pWinStates, index);
COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)( (char*)(pPos->pRowBuff) + (valSize - sizeof(COUNT_TYPE)) );
if (inSessionWindow(pPos->pKey, startTs, gap) || (index == size - 1 && (*pWinStateCout) < winCount) ) {
(*pVal) = pPos;
SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey;
pPos->beUsed = true;
*pWinKey = *pDestWinKey;
goto _end;
}
}
if (index == -1) {
if (!isDeteled(pFileState, endTs)) {
void* p = NULL;
void* pFileStore = getStateFileStore(pFileState);
SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileStore, pKey->groupId);
int32_t code_file = streamStateSessionGetKVByCur_rocksdb(pCur, pWinKey, &p, pVLen);
if (code_file == TSDB_CODE_SUCCESS) {
(*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen);
code = code_file;
qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file);
streamStateFreeCur(pCur);
goto _end;
}
taosMemoryFree(p);
streamStateFreeCur(pCur);
}
}
if (index + 1 < size) {
pPos = taosArrayGetP(pWinStates, index + 1);
(*pVal) = pPos;
SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey;
pPos->beUsed = true;
*pWinKey = *pDestWinKey;
goto _end;
}
(*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey);
code = TSDB_CODE_FAILED;
_end:
return code;
}
int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen) {
SSessionKey* pWinKey = pKey;
const TSKEY gap = 0;
int32_t code = TSDB_CODE_SUCCESS;
SSHashObj* pSessionBuff = getRowStateBuff(pFileState);
SArray* pWinStates = NULL;
void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t));
if (ppBuff) {
pWinStates = (SArray*)(*ppBuff);
} else {
pWinStates = taosArrayInit(16, POINTER_BYTES);
tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES);
}
TSKEY startTs = pWinKey->win.skey;
TSKEY endTs = pWinKey->win.ekey;
int32_t size = taosArrayGetSize(pWinStates);
if (size == 0) {
void* pFileStore = getStateFileStore(pFileState);
void* p = NULL;
SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileStore, pKey->groupId);
int32_t code_file = streamStateSessionGetKVByCur_rocksdb(pCur, pWinKey, &p, pVLen);
if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) {
(*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen);
code = code_file;
qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file);
} else {
(*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey);
code = TSDB_CODE_FAILED;
taosMemoryFree(p);
}
streamStateFreeCur(pCur);
goto _end;
} else {
(*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey);
}
_end:
return code;
}

12
source/libs/stream/src/streamStart.c
View File

@ -391,6 +391,16 @@ void doProcessDownstreamReadyRsp(SStreamTask* pTask) {
int64_t startTs = pTask->execInfo.start;
streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, initTs, startTs, true);
if (pTask->status.taskStatus == TASK_STATUS__HALT) {
ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask) && (pTask->info.fillHistory == 0));
// halt it self for count window stream task until the related
// fill history task completd.
stDebug("s-task:%s level:%d initial status is %s from mnode, set it to be halt", pTask->id.idStr,
pTask->info.taskLevel, streamTaskGetStatusStr(pTask->status.taskStatus));
streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT);
}
// start the related fill-history task, when current task is ready
// not invoke in success callback due to the deadlock.
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
@ -799,7 +809,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) {
// check stream task status in the first place.
SStreamTaskState* pStatus = streamTaskGetStatus(pTask);
if (pStatus->state != TASK_STATUS__READY) {
if (pStatus->state != TASK_STATUS__READY && pStatus->state != TASK_STATUS__HALT) {
stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", idStr, hStreamId, hTaskId,
pStatus->name);

119
source/libs/stream/src/streamState.c
View File

@ -42,6 +42,14 @@ int sessionRangeKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2) {
return 0;
}
int countRangeKeyEqual(const SSessionKey* pWin1, const SSessionKey* pWin2) {
if (pWin1->groupId == pWin2->groupId && pWin1->win.skey <= pWin2->win.skey && pWin2->win.skey <= pWin1->win.ekey) {
return 0;
}
return 1;
}
int sessionWinKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2) {
if (pWin1->groupId > pWin2->groupId) {
return 1;
@ -752,6 +760,12 @@ int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) {
#endif
}
int32_t streamStateSessionReset(SStreamState* pState, void* pVal) {
int32_t len = getRowStateRowSize(pState->pFileState);
memset(pVal, 0, len);
return TSDB_CODE_SUCCESS;
}
SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key) {
#ifdef USE_ROCKSDB
return sessionWinStateSeekKeyCurrentPrev(pState->pFileState, key);
@ -846,6 +860,13 @@ SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSess
#endif
}
SStreamStateCur* streamStateCountSeekKeyPrev(SStreamState* pState, const SSessionKey* key, COUNT_TYPE count) {
#ifdef USE_ROCKSDB
return countWinStateSeekKeyPrev(pState->pFileState, key, count);
#else
#endif
}
int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen) {
#ifdef USE_ROCKSDB
return sessionWinStateGetKVByCur(pCur, pKey, pVal, pVLen);
@ -896,7 +917,7 @@ int32_t streamStateSessionClear(SStreamState* pState) {
int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) {
#ifdef USE_ROCKSDB
return sessionWinStateGetKeyByRange(pState->pFileState, key, curKey);
return sessionWinStateGetKeyByRange(pState->pFileState, key, curKey, sessionRangeKeyCmpr);
#else
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
if (pCur == NULL) {
@ -946,6 +967,13 @@ int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey*
#endif
}
int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) {
#ifdef USE_ROCKSDB
return sessionWinStateGetKeyByRange(pState->pFileState, key, curKey, countRangeKeyEqual);
#else
#endif
}
int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal,
int32_t* pVLen) {
#ifdef USE_ROCKSDB
@ -1135,90 +1163,11 @@ SStreamStateCur* createStreamStateCursor() {
return pCur;
}
#if 0
char* streamStateSessionDump(SStreamState* pState) {
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
if (pCur == NULL) {
return NULL;
}
pCur->number = pState->number;
if (tdbTbcOpen(pState->pTdbState->pSessionStateDb, &pCur->pCur, NULL) < 0) {
streamStateFreeCur(pCur);
return NULL;
}
tdbTbcMoveToFirst(pCur->pCur);
SSessionKey key = {0};
void* buf = NULL;
int32_t bufSize = 0;
int32_t code = streamStateSessionGetKVByCur(pCur, &key, &buf, &bufSize);
if (code != 0) {
streamStateFreeCur(pCur);
return NULL;
// count window
int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal, int32_t* pVLen) {
return getCountWinResultBuff(pState->pFileState, pKey, winCount, ppVal, pVLen);
}
int32_t size = 2048;
char* dumpBuf = taosMemoryCalloc(size, 1);
int64_t len = 0;
len += snprintf(dumpBuf + len, size - len, "||s:%15" PRId64 ",", key.win.skey);
len += snprintf(dumpBuf + len, size - len, "e:%15" PRId64 ",", key.win.ekey);
len += snprintf(dumpBuf + len, size - len, "g:%15" PRId64 "||", key.groupId);
while (1) {
tdbTbcMoveToNext(pCur->pCur);
key = (SSessionKey){0};
code = streamStateSessionGetKVByCur(pCur, &key, NULL, 0);
if (code != 0) {
streamStateFreeCur(pCur);
return dumpBuf;
int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen) {
return createCountWinResultBuff(pState->pFileState, pKey, pVal, pVLen);
}
len += snprintf(dumpBuf + len, size - len, "||s:%15" PRId64 ",", key.win.skey);
len += snprintf(dumpBuf + len, size - len, "e:%15" PRId64 ",", key.win.ekey);
len += snprintf(dumpBuf + len, size - len, "g:%15" PRId64 "||", key.groupId);
}
streamStateFreeCur(pCur);
return dumpBuf;
}
char* streamStateIntervalDump(SStreamState* pState) {
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
if (pCur == NULL) {
return NULL;
}
pCur->number = pState->number;
if (tdbTbcOpen(pState->pTdbState->pStateDb, &pCur->pCur, NULL) < 0) {
streamStateFreeCur(pCur);
return NULL;
}
tdbTbcMoveToFirst(pCur->pCur);
SWinKey key = {0};
void* buf = NULL;
int32_t bufSize = 0;
int32_t code = streamStateGetKVByCur(pCur, &key, (const void **)&buf, &bufSize);
if (code != 0) {
streamStateFreeCur(pCur);
return NULL;
}
int32_t size = 2048;
char* dumpBuf = taosMemoryCalloc(size, 1);
int64_t len = 0;
len += snprintf(dumpBuf + len, size - len, "||s:%15" PRId64 ",", key.ts);
// len += snprintf(dumpBuf + len, size - len, "e:%15" PRId64 ",", key.win.ekey);
len += snprintf(dumpBuf + len, size - len, "g:%15" PRId64 "||", key.groupId);
while (1) {
tdbTbcMoveToNext(pCur->pCur);
key = (SWinKey){0};
code = streamStateGetKVByCur(pCur, &key, NULL, 0);
if (code != 0) {
streamStateFreeCur(pCur);
return dumpBuf;
}
len += snprintf(dumpBuf + len, size - len, "||s:%15" PRId64 ",", key.ts);
// len += snprintf(dumpBuf + len, size - len, "e:%15" PRId64 ",", key.win.ekey);
len += snprintf(dumpBuf + len, size - len, "g:%15" PRId64 "||", key.groupId);
}
streamStateFreeCur(pCur);
return dumpBuf;
}
#endif

18
source/libs/stream/src/streamTask.c
View File

@ -109,7 +109,7 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset,
pTask->id.idStr = taosStrdup(buf);
pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
pTask->status.taskStatus = (fillHistory || hasFillhistory) ? TASK_STATUS__SCAN_HISTORY : TASK_STATUS__READY;
pTask->status.taskStatus = fillHistory? TASK_STATUS__SCAN_HISTORY : TASK_STATUS__READY;
pTask->inputq.status = TASK_INPUT_STATUS__NORMAL;
pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL;
@ -127,7 +127,6 @@ int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo)
if (tEncodeI32(pEncoder, pInfo->taskId) < 0) return -1;
if (tEncodeI32(pEncoder, pInfo->nodeId) < 0) return -1;
if (tEncodeI32(pEncoder, pInfo->childId) < 0) return -1;
/*if (tEncodeI64(pEncoder, pInfo->processedVer) < 0) return -1;*/
if (tEncodeSEpSet(pEncoder, &pInfo->epSet) < 0) return -1;
if (tEncodeI64(pEncoder, pInfo->stage) < 0) return -1;
return 0;
@ -137,7 +136,6 @@ int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) {
if (tDecodeI32(pDecoder, &pInfo->taskId) < 0) return -1;
if (tDecodeI32(pDecoder, &pInfo->nodeId) < 0) return -1;
if (tDecodeI32(pDecoder, &pInfo->childId) < 0) return -1;
/*if (tDecodeI64(pDecoder, &pInfo->processedVer) < 0) return -1;*/
if (tDecodeSEpSet(pDecoder, &pInfo->epSet) < 0) return -1;
if (tDecodeI64(pDecoder, &pInfo->stage) < 0) return -1;
return 0;
@ -297,7 +295,6 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
}
int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo) {
int64_t ver;
int64_t skip64;
int8_t skip8;
int32_t skip32;
@ -651,7 +648,7 @@ int32_t streamTaskStop(SStreamTask* pTask) {
streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_STOP);
qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS);
while (/*pTask->status.schedStatus != TASK_SCHED_STATUS__INACTIVE */ !streamTaskIsIdle(pTask)) {
while (!streamTaskIsIdle(pTask)) {
stDebug("s-task:%s level:%d wait for task to be idle and then close, check again in 100ms", id,
pTask->info.taskLevel);
taosMsleep(100);
@ -758,7 +755,7 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) {
return status;
}
int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, bool metaLock) {
int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt, bool metaLock) {
SStreamMeta* pMeta = pTask->pMeta;
STaskId sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId};
if (pTask->info.fillHistory == 0) {
@ -776,6 +773,12 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, bool metaLock) {
taosThreadMutexLock(&(*ppStreamTask)->lock);
CLEAR_RELATED_FILLHISTORY_TASK((*ppStreamTask));
if (resetRelHalt) {
(*ppStreamTask)->status.taskStatus = TASK_STATUS__READY;
stDebug("s-task:0x%" PRIx64 " set the status to be ready", sTaskId.taskId);
}
streamMetaSaveTask(pMeta, *ppStreamTask);
taosThreadMutexUnlock(&(*ppStreamTask)->lock);
}
@ -787,7 +790,7 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, bool metaLock) {
return TSDB_CODE_SUCCESS;
}
int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId) {
int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId, int64_t resetRelHalt) {
SVDropStreamTaskReq* pReq = rpcMallocCont(sizeof(SVDropStreamTaskReq));
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -797,6 +800,7 @@ int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskI
pReq->head.vgId = vgId;
pReq->taskId = pTaskId->taskId;
pReq->streamId = pTaskId->streamId;
pReq->resetRelHalt = resetRelHalt;
SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_DROP, .pCont = pReq, .contLen = sizeof(SVDropStreamTaskReq)};
int32_t code = tmsgPutToQueue(pMsgCb, WRITE_QUEUE, &msg);

11
source/libs/stream/src/tstreamFileState.c
View File

@ -480,6 +480,15 @@ int32_t deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t ke
return TSDB_CODE_FAILED;
}
int32_t resetRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) {
int32_t code_buff = pFileState->stateBuffRemoveFn(pFileState->rowStateBuff, pKey, keyLen);
int32_t code_file = pFileState->stateFileRemoveFn(pFileState, pKey);
if (code_buff == TSDB_CODE_SUCCESS || code_file == TSDB_CODE_SUCCESS) {
return TSDB_CODE_SUCCESS;
}
return TSDB_CODE_FAILED;
}
static void recoverSessionRowBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) {
int32_t len = 0;
void* pBuff = NULL;
@ -656,7 +665,7 @@ int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) {
deleteExpiredCheckPoint(pFileState, mark);
}
SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileState->pFileStore);
SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileState->pFileStore, INT64_MAX);
if (pCur == NULL) {
return -1;
}

9
tests/parallel_test/cases.task
View File

@ -1141,6 +1141,8 @@
,,y,script,./test.sh -f tsim/query/bug3398.sim
,,y,script,./test.sh -f tsim/query/explain_tsorder.sim
,,y,script,./test.sh -f tsim/query/apercentile.sim
,,y,script,./test.sh -f tsim/query/query_count0.sim
,,y,script,./test.sh -f tsim/query/query_count_sliding0.sim
,,y,script,./test.sh -f tsim/qnode/basic1.sim
,,y,script,./test.sh -f tsim/snode/basic1.sim
,,y,script,./test.sh -f tsim/mnode/basic1.sim
@ -1185,6 +1187,13 @@
,,y,script,./test.sh -f tsim/stream/checkpointInterval0.sim
,,y,script,./test.sh -f tsim/stream/checkStreamSTable1.sim
,,y,script,./test.sh -f tsim/stream/checkStreamSTable.sim
,,y,script,./test.sh -f tsim/stream/count0.sim
,,y,script,./test.sh -f tsim/stream/count1.sim
,,y,script,./test.sh -f tsim/stream/count2.sim
,,y,script,./test.sh -f tsim/stream/count3.sim
,,y,script,./test.sh -f tsim/stream/countSliding0.sim
,,y,script,./test.sh -f tsim/stream/countSliding1.sim
,,y,script,./test.sh -f tsim/stream/countSliding2.sim
,,y,script,./test.sh -f tsim/stream/deleteInterval.sim
,,y,script,./test.sh -f tsim/stream/deleteSession.sim
,,y,script,./test.sh -f tsim/stream/deleteState.sim

179
tests/script/tsim/query/query_count0.sim Normal file
View File

@ -0,0 +1,179 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print step1
print =============== create database
sql create database test vgroups 1;
sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double);
sql insert into t1 values(1648791213000,0,1,1,1.0);
sql insert into t1 values(1648791213001,9,2,2,1.1);
sql insert into t1 values(1648791213009,0,3,3,1.0);
sql insert into t1 values(1648791223000,0,1,1,1.0);
sql insert into t1 values(1648791223001,9,2,2,1.1);
sql insert into t1 values(1648791223009,0,3,3,1.0);
$loop_count = 0
loop2:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3);
sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $data01 != 3 then
print ======data01=$data01
goto loop2
endi
if $data02 != 6 then
print ======data02=$data02
goto loop2
endi
if $data03 != 3 then
print ======data03=$data03
goto loop2
endi
# row 1
if $data11 != 3 then
print ======data11=$data11
goto loop2
endi
if $data12 != 6 then
print ======data12=$data12
goto loop2
endi
if $data13 != 3 then
print ======data13=$data13
goto loop2
endi
print step2
print =============== create database
sql create database test2 vgroups 4;
sql use test2;
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql insert into t1 values(1648791213000,0,1,1,1.0);
sql insert into t1 values(1648791213001,9,2,2,1.1);
sql insert into t1 values(1648791213009,0,3,3,1.0);
sql insert into t2 values(1648791213000,0,1,1,1.0);
sql insert into t2 values(1648791213001,9,2,2,1.1);
sql insert into t2 values(1648791213009,0,3,3,1.0);
sql insert into t1 values(1648791223000,0,1,1,1.0);
sql insert into t1 values(1648791223001,9,2,2,1.1);
sql insert into t1 values(1648791223009,0,3,3,1.0);
sql insert into t2 values(1648791223000,0,1,1,1.0);
sql insert into t2 values(1648791223001,9,2,2,1.1);
sql insert into t2 values(1648791223009,0,3,3,1.0);
$loop_count = 0
loop3:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(3);
sql select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(3);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $data01 != 3 then
print ======data01=$data01
goto loop3
endi
if $data02 != 6 then
print ======data02=$data02
goto loop3
endi
if $data03 != 3 then
print ======data03=$data03
goto loop3
endi
# row 1
if $data11 != 3 then
print ======data11=$data11
goto loop3
endi
if $data12 != 6 then
print ======data12=$data12
goto loop3
endi
if $data13 != 3 then
print ======data13=$data13
goto loop3
endi
# row 2
if $data21 != 3 then
print ======data21=$data21
goto loop3
endi
if $data22 != 6 then
print ======data22=$data22
goto loop3
endi
if $data23 != 3 then
print ======data23=$data23
goto loop3
endi
# row 3
if $data31 != 3 then
print ======data31=$data31
goto loop3
endi
if $data32 != 6 then
print ======data32=$data32
goto loop3
endi
if $data33 != 3 then
print ======data33=$data33
goto loop3
endi
print query_count0 end
system sh/exec.sh -n dnode1 -s stop -x SIGINT

83
tests/script/tsim/query/query_count1.sim Normal file
View File

@ -0,0 +1,83 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print step1
print =============== create database
sql create database test vgroups 4;
sql use test;
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql insert into t1 values(1648791213000,0,1,1,1.0);
sql insert into t1 values(1648791213001,9,1,2,1.1);
sql insert into t1 values(1648791213009,0,1,3,1.0);
sql insert into t2 values(1648791213000,0,1,4,1.0);
sql insert into t2 values(1648791213001,9,1,5,1.1);
sql insert into t2 values(1648791213009,0,1,6,1.0);
sql insert into t1 values(1648791223000,0,1,7,1.0);
sql insert into t1 values(1648791223001,9,1,8,1.1);
sql insert into t1 values(1648791223009,0,1,9,1.0);
sql insert into t2 values(1648791223000,0,1,10,1.0);
sql insert into t2 values(1648791223001,9,1,11,1.1);
sql insert into t2 values(1648791223009,0,1,12,1.0);
$loop_count = 0
loop3:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from st count_window(4);
sql select _wstart as s, count(*) c1, sum(b), max(c) from st count_window(4);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop3
endi
if $data02 != 4 then
print ======data02=$data02
goto loop3
endi
# row 1
if $data11 != 4 then
print ======data11=$data11
goto loop3
endi
if $data12 != 4 then
print ======data12=$data12
goto loop3
endi
# row 2
if $data21 != 4 then
print ======data21=$data21
goto loop3
endi
if $data22 != 4 then
print ======data22=$data22
goto loop3
endi
print query_count0 end
system sh/exec.sh -n dnode1 -s stop -x SIGINT

670
tests/script/tsim/query/query_count_sliding0.sim Normal file
View File

@ -0,0 +1,670 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print step1
print =============== create database
sql create database test vgroups 1;
sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double);
sql insert into t1 values(1648791213000,0,1,1,1.0);
$loop_count = 0
loop00:
sleep 300
print 00 sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 1 then
print ======rows=$rows
goto loop00
endi
# row 0
if $data01 != 1 then
print ======data01=$data01
goto loop00
endi
sql insert into t1 values(1648791213001,9,2,2,1.1);
$loop_count = 0
loop01:
sleep 300
print 01 sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 1 then
print ======rows=$rows
goto loop01
endi
# row 0
if $data01 != 2 then
print ======data01=$data01
goto loop01
endi
sql insert into t1 values(1648791213002,0,3,3,1.0);
$loop_count = 0
loop02:
sleep 300
print 02 sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 2 then
print ======rows=$rows
goto loop02
endi
# row 0
if $data01 != 3 then
print ======data01=$data01
goto loop02
endi
# row 1
if $data11 != 1 then
print ======data01=$data01
goto loop02
endi
sql insert into t1 values(1648791213009,0,3,3,1.0);
$loop_count = 0
loop0:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 2 then
print ======rows=$rows
goto loop0
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop0
endi
# row 1
if $data11 != 2 then
print ======data11=$data11
goto loop0
endi
sql insert into t1 values(1648791223000,0,1,1,1.0);
sql insert into t1 values(1648791223001,9,2,2,1.1);
sql insert into t1 values(1648791223002,9,2,2,1.1);
sql insert into t1 values(1648791223009,0,3,3,1.0);
$loop_count = 0
loop2:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 4 then
print ======rows=$rows
goto loop2
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop2
endi
# row 1
if $data11 != 4 then
print ======data11=$data11
goto loop2
endi
# row 2
if $data21 != 4 then
print ======data21=$data21
goto loop2
endi
# row 3
if $data31 != 2 then
print ======data31=$data31
goto loop2
endi
sql insert into t1 values(1648791233000,0,1,1,1.0) (1648791233001,9,2,2,1.1) (1648791233002,9,2,2,1.1) (1648791233009,0,3,3,1.0);
$loop_count = 0
loop3:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 6 then
print ======rows=$rows
goto loop3
endi
sql insert into t1 values(1648791243000,0,1,1,1.0) (1648791243001,9,2,2,1.1);
$loop_count = 0
loop4:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 7 then
print ======rows=$rows
goto loop4
endi
sql insert into t1 values(1648791253000,0,1,1,1.0) (1648791253001,9,2,2,1.1) (1648791253002,9,2,2,1.1);
$loop_count = 0
loop5:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
print $data80 $data81 $data82 $data83
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 9 then
print ======rows=$rows
goto loop5
endi
sql insert into t1 values(1648791263000,0,1,1,1.0);
$loop_count = 0
loop6:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
print $data80 $data81 $data82 $data83
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 9 then
print ======rows=$rows
goto loop6
endi
print step2
print =============== create database
sql create database test2 vgroups 4;
sql use test2;
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql insert into t1 values(1648791213000,0,1,1,1.0);
sql insert into t1 values(1648791213001,9,2,2,1.1);
sql insert into t1 values(1648791213002,0,3,3,1.0);
sql insert into t1 values(1648791213009,0,3,3,1.0);
$loop_count = 0
loop7:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2);
sql select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 2 then
print ======rows=$rows
goto loop7
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop7
endi
# row 1
if $data11 != 2 then
print ======data11=$data11
goto loop7
endi
sql insert into t1 values(1648791223000,0,1,1,1.0);
sql insert into t1 values(1648791223001,9,2,2,1.1);
sql insert into t1 values(1648791223002,9,2,2,1.1);
sql insert into t1 values(1648791223009,0,3,3,1.0);
$loop_count = 0
loop8:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2);
sql select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 4 then
print ======rows=$rows
goto loop8
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop8
endi
# row 1
if $data11 != 4 then
print ======data11=$data11
goto loop8
endi
# row 2
if $data21 != 4 then
print ======data21=$data21
goto loop8
endi
# row 3
if $data31 != 2 then
print ======data31=$data31
goto loop8
endi
sql insert into t1 values(1648791233000,0,1,1,1.0) (1648791233001,9,2,2,1.1) (1648791233002,9,2,2,1.1) (1648791233009,0,3,3,1.0);
$loop_count = 0
loop9:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2);
sql select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 6 then
print ======rows=$rows
goto loop9
endi
sql insert into t1 values(1648791243000,0,1,1,1.0) (1648791243001,9,2,2,1.1);
$loop_count = 0
loop10:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2);
sql select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 7 then
print ======rows=$rows
goto loop10
endi
sql insert into t1 values(1648791253000,0,1,1,1.0) (1648791253001,9,2,2,1.1) (1648791253002,9,2,2,1.1);
$loop_count = 0
loop11:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2);
sql select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
print $data80 $data81 $data82 $data83
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 9 then
print ======rows=$rows
goto loop11
endi
sql insert into t1 values(1648791263000,0,1,1,1.0);
$loop_count = 0
loop12:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2);
sql select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
print $data80 $data81 $data82 $data83
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 9 then
print ======rows=$rows
goto loop12
endi
print step3
print =============== create database
sql create database test3 vgroups 4;
sql use test3;
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql insert into t1 values(1648791213000,0,1,1,1.0);
sql insert into t1 values(1648791213001,9,1,2,1.1);
sql insert into t1 values(1648791213009,0,1,3,1.0);
sql insert into t2 values(1648791213000,0,1,4,1.0);
sql insert into t2 values(1648791213001,9,1,5,1.1);
sql insert into t2 values(1648791213009,0,1,6,1.0);
sql insert into t1 values(1648791223000,0,1,7,1.0);
sql insert into t1 values(1648791223001,9,1,8,1.1);
sql insert into t1 values(1648791223009,0,1,9,1.0);
sql insert into t2 values(1648791223000,0,1,10,1.0);
sql insert into t2 values(1648791223001,9,1,11,1.1);
sql insert into t2 values(1648791223009,0,1,12,1.0);
$loop_count = 0
loop13:
sleep 300
print 1 sql select _wstart as s, count(*) c1, sum(b), max(c) from st count_window(4, 1);
sql select _wstart as s, count(*) c1, sum(b), max(c) from st count_window(4,1);
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
print $data80 $data81 $data82 $data83
print $data90 $data91 $data92 $data93
print $data[10][0] $data[10][1] $data[10][2] $data[10][3]
print $data[11][0] $data[11][1] $data[11][2] $data[11][3]
print $data[12][0] $data[12][1] $data[12][2] $data[12][3]
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# rows
if $rows != 12 then
print ======rows=$rows
goto loop13
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop13
endi
if $data02 != 4 then
print ======data02=$data02
goto loop13
endi
# row 1
if $data11 != 4 then
print ======data11=$data11
goto loop13
endi
if $data12 != 4 then
print ======data12=$data12
goto loop13
endi
# row 2
if $data21 != 4 then
print ======data21=$data21
goto loop13
endi
if $data22 != 4 then
print ======data22=$data22
goto loop13
endi
# row 9
if $data91 != 3 then
print ======data91=$data91
goto loop13
endi
if $data92 != 3 then
print ======data92=$data92
goto loop13
endi
# row 10
if $data[10][1] != 2 then
print ======data[10][1]=$data[10][1]
goto loop13
endi
if $data[10][2] != 2 then
print ======data[10][2]=$data[10][2]
goto loop13
endi
# row 11
if $data[11][1] != 1 then
print ======data[11][1]=$data[11][1]
goto loop13
endi
if $data[11][2] != 1 then
print ======data[11][2]=$data[11][2]
goto loop13
endi
print count sliding 0 end
system sh/exec.sh -n dnode1 -s stop -x SIGINT

249
tests/script/tsim/stream/count0.sim Normal file
View File

@ -0,0 +1,249 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print step1
print =============== create database
sql create database test vgroups 1;
sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3);
sleep 1000
sql insert into t1 values(1648791213000,0,1,1,1.0);
sql insert into t1 values(1648791213001,9,2,2,1.1);
sql insert into t1 values(1648791213009,0,3,3,1.0);
sql insert into t1 values(1648791223000,0,1,1,1.0);
sql insert into t1 values(1648791223001,9,2,2,1.1);
sql insert into t1 values(1648791223009,0,3,3,1.0);
$loop_count = 0
loop2:
sleep 300
print 1 sql select * from streamt;
sql select * from streamt;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $data01 != 3 then
print ======data01=$data01
goto loop2
endi
if $data02 != 6 then
print ======data02=$data02
goto loop2
endi
if $data03 != 3 then
print ======data03=$data03
goto loop2
endi
# row 1
if $data11 != 3 then
print ======data11=$data11
goto loop2
endi
if $data12 != 6 then
print ======data12=$data12
goto loop2
endi
if $data13 != 3 then
print ======data13=$data13
goto loop2
endi
print step2
print =============== create database
sql create database test2 vgroups 4;
sql use test2;
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create stream streams2 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(3)
sleep 1000
sql insert into t1 values(1648791213000,0,1,1,1.0);
sql insert into t1 values(1648791213001,9,2,2,1.1);
sql insert into t1 values(1648791213009,0,3,3,1.0);
sql insert into t2 values(1648791213000,0,1,1,1.0);
sql insert into t2 values(1648791213001,9,2,2,1.1);
sql insert into t2 values(1648791213009,0,3,3,1.0);
sql insert into t1 values(1648791223000,0,1,1,1.0);
sql insert into t1 values(1648791223001,9,2,2,1.1);
sql insert into t1 values(1648791223009,0,3,3,1.0);
sql insert into t2 values(1648791223000,0,1,1,1.0);
sql insert into t2 values(1648791223001,9,2,2,1.1);
sql insert into t2 values(1648791223009,0,3,3,1.0);
$loop_count = 0
loop3:
sleep 300
print 1 sql select * from streamt;
sql select * from streamt2 order by 1,2;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $data01 != 3 then
print ======data01=$data01
goto loop3
endi
if $data02 != 6 then
print ======data02=$data02
goto loop3
endi
if $data03 != 3 then
print ======data03=$data03
goto loop3
endi
# row 1
if $data11 != 3 then
print ======data11=$data11
goto loop3
endi
if $data12 != 6 then
print ======data12=$data12
goto loop3
endi
if $data13 != 3 then
print ======data13=$data13
goto loop3
endi
# row 2
if $data21 != 3 then
print ======data21=$data21
goto loop3
endi
if $data22 != 6 then
print ======data22=$data22
goto loop3
endi
if $data23 != 3 then
print ======data23=$data23
goto loop3
endi
# row 3
if $data31 != 3 then
print ======data31=$data31
goto loop3
endi
if $data32 != 6 then
print ======data32=$data32
goto loop3
endi
if $data33 != 3 then
print ======data33=$data33
goto loop3
endi
print step3
print =============== create database
sql create database test3 vgroups 1;
sql use test3;
sql create table t1(ts timestamp, a int, b int , c int, d double);
sql insert into t1 values(1648791213000,0,1,1,1.0);
sql insert into t1 values(1648791213001,9,2,2,1.1);
sql insert into t1 values(1648791213009,0,3,3,1.0);
sleep 500
sql create stream streams3 trigger at_once FILL_HISTORY 1 IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt3 as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3);
sleep 1000
sql insert into t1 values(1648791223000,0,1,1,1.0);
sql insert into t1 values(1648791223001,9,2,2,1.1);
sql insert into t1 values(1648791223009,0,3,3,1.0);
$loop_count = 0
loop4:
sleep 300
print 1 sql select * from streamt3;
sql select * from streamt3;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $data01 != 3 then
print ======data01=$data01
goto loop4
endi
if $data02 != 6 then
print ======data02=$data02
goto loop4
endi
if $data03 != 3 then
print ======data03=$data03
goto loop4
endi
# row 1
if $data11 != 3 then
print ======data11=$data11
goto loop4
endi
if $data12 != 6 then
print ======data12=$data12
goto loop4
endi
if $data13 != 3 then
print ======data13=$data13
goto loop4
endi
print count0 end
system sh/exec.sh -n dnode1 -s stop -x SIGINT

36
tests/script/tsim/stream/count1.sim Normal file
View File

@ -0,0 +1,36 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print step1
print =============== create database
sql create database test vgroups 1;
sql use test;
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
# stable
sql_error create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 10s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from st count_window(3);
# IGNORE EXPIRED 0
sql_error create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 WATERMARK 10s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3);
# WATERMARK 0
sql_error create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3);
# All
sql_error create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from st count_window(3);
#2~INT32_MAX
sql_error create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(1);
sql_error create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(2147483648);
sql create stream streams2 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 10s into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(2);
sql create stream streams3 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 10s into streamt3 as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(2147483647);
print count1 end
system sh/exec.sh -n dnode1 -s stop -x SIGINT

302
tests/script/tsim/stream/count2.sim Normal file
View File

@ -0,0 +1,302 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print step1
print =============== create database
sql create database test vgroups 1;
sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3);
sleep 1000
sql insert into t1 values(1648791213001,9,2,2,1.1);
sql insert into t1 values(1648791213009,0,3,3,1.0);
$loop_count = 0
loop0:
sleep 300
print 0 sql select * from streamt;
sql select * from streamt;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $data01 != 2 then
print ======data01=$data01
goto loop0
endi
sql insert into t1 values(1648791213000,0,1,1,1.0);
$loop_count = 0
loop1:
sleep 300
print 1 sql select * from streamt;
sql select * from streamt;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 1 then
print ======rows=$rows
goto loop1
endi
if $data01 != 3 then
print ======data01=$data01
goto loop1
endi
sql insert into t1 values(1648791223000,0,1,1,1.0);
sql insert into t1 values(1648791223001,9,2,2,1.1);
sql insert into t1 values(1648791223009,0,3,3,1.0);
$loop_count = 0
loop2:
sleep 300
print 2 sql select * from streamt order by 1;
sql select * from streamt order by 1;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 2 then
print ======rows=$rows
goto loop2
endi
sql insert into t1 values(1648791212000,0,1,1,1.0);
$loop_count = 0
loop3:
sleep 300
print 3 sql select * from streamt order by 1;
sql select * from streamt order by 1;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 3 then
print ======rows=$rows
goto loop3
endi
if $data01 != 3 then
print ======data01=$data01
goto loop3
endi
if $data11 != 3 then
print ======data11=$data11
goto loop3
endi
if $data21 != 1 then
print ======data21=$data21
goto loop3
endi
print step2
print =============== create database
sql create database test2 vgroups 1;
sql use test2;
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create stream streams2 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(3)
sleep 1000
sql insert into t1 values(1648791213001,9,2,2,1.1);
sql insert into t1 values(1648791213009,0,3,3,1.0);
sql insert into t2 values(1648791213001,9,2,2,1.1);
sql insert into t2 values(1648791213009,0,3,3,1.0);
$loop_count = 0
loop4:
sleep 300
print 0 sql select * from streamt2 order by 1;;
sql select * from streamt2 order by 1;;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $data01 != 2 then
print ======data01=$data01
goto loop4
endi
if $data11 != 2 then
print ======data11=$data11
goto loop4
endi
sql insert into t1 values(1648791213000,0,1,1,1.0);
sql insert into t2 values(1648791213000,0,1,1,1.0);
$loop_count = 0
loop5:
sleep 300
print 1 sql select * from streamt2 order by 1;;
sql select * from streamt2 order by 1;;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 2 then
print ======rows=$rows
goto loop5
endi
if $data01 != 3 then
print ======data01=$data01
goto loop5
endi
if $data11 != 3 then
print ======data11=$data11
goto loop5
endi
sql insert into t1 values(1648791223000,0,1,1,1.0);
sql insert into t1 values(1648791223001,9,2,2,1.1);
sql insert into t1 values(1648791223009,0,3,3,1.0);
sql insert into t2 values(1648791223000,0,1,1,1.0);
sql insert into t2 values(1648791223001,9,2,2,1.1);
sql insert into t2 values(1648791223009,0,3,3,1.0);
$loop_count = 0
loop6:
sleep 300
print 2 sql select * from streamt2 order by 1;
sql select * from streamt2 order by 1;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 4 then
print ======rows=$rows
goto loop6
endi
sql insert into t1 values(1648791212000,0,1,1,1.0);
sql insert into t2 values(1648791212000,0,1,1,1.0);
$loop_count = 0
loop7:
sleep 300
print 3 sql select * from streamt2 order by 1;
sql select * from streamt2 order by 1;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 6 then
print ======rows=$rows
goto loop7
endi
if $data01 != 3 then
print ======data01=$data01
goto loop7
endi
if $data11 != 3 then
print ======data11=$data11
goto loop7
endi
if $data21 != 3 then
print ======data21=$data21
goto loop7
endi
if $data31 != 3 then
print ======data31=$data31
goto loop7
endi
if $data41 != 1 then
print ======data41=$data41
goto loop7
endi
if $data51 != 1 then
print ======data51=$data51
goto loop7
endi
print count2 end
system sh/exec.sh -n dnode1 -s stop -x SIGINT

116
tests/script/tsim/stream/count3.sim Normal file
View File

@ -0,0 +1,116 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print step1
print =============== create database
sql create database test vgroups 1;
sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3);
sleep 1000
sql insert into t1 values(1648791213000,0,1,1,1.0);
sql insert into t1 values(1648791213001,9,2,2,1.1);
sql insert into t1 values(1648791213009,0,3,3,1.0);
sql insert into t1 values(1648791223000,0,1,1,1.0);
sql insert into t1 values(1648791223001,9,2,2,1.1);
sql insert into t1 values(1648791223009,0,3,3,1.0);
$loop_count = 0
loop2:
sleep 300
print 2 sql select * from streamt order by 1;
sql select * from streamt order by 1;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 2 then
print ======rows=$rows
goto loop2
endi
sql insert into t1 values(1648791213000,4,4,4,4.0);
$loop_count = 0
loop3:
sleep 300
print 3 sql select * from streamt order by 1;
sql select * from streamt order by 1;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 2 then
print ======rows=$rows
goto loop3
endi
if $data01 != 3 then
print ======data01=$data01
goto loop3
endi
if $data11 != 3 then
print ======data11=$data11
goto loop3
endi
sql delete from t1 where ts = 1648791223001;
$loop_count = 0
loop4:
sleep 300
print 3 sql select * from streamt order by 1;
sql select * from streamt order by 1;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 2 then
print ======rows=$rows
goto loop4
endi
if $data01 != 3 then
print ======data01=$data01
goto loop4
endi
if $data11 != 2 then
print ======data11=$data11
goto loop4
endi
print count3 end
system sh/exec.sh -n dnode1 -s stop -x SIGINT

463
tests/script/tsim/stream/countSliding0.sim Normal file
View File

@ -0,0 +1,463 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print step1
print =============== create database
sql create database test vgroups 1;
sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
sleep 1000
sql insert into t1 values(1648791213000,0,1,1,1.0);
sleep 100
sql insert into t1 values(1648791213001,9,2,2,1.1);
sleep 100
sql insert into t1 values(1648791213002,0,3,3,1.0);
sleep 100
sql insert into t1 values(1648791213009,0,3,3,1.0);
$loop_count = 0
loop0:
sleep 300
print 1 sql select * from streamt;
sql select * from streamt;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 2 then
print ======rows=$rows
goto loop0
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop0
endi
# row 1
if $data11 != 2 then
print ======data11=$data11
goto loop0
endi
sql insert into t1 values(1648791223000,0,1,1,1.0);
sleep 100
sql insert into t1 values(1648791223001,9,2,2,1.1);
sleep 100
sql insert into t1 values(1648791223002,9,2,2,1.1);
sleep 100
sql insert into t1 values(1648791223009,0,3,3,1.0);
$loop_count = 0
loop2:
sleep 300
print 1 sql select * from streamt;
sql select * from streamt;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 4 then
print ======rows=$rows
goto loop2
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop2
endi
# row 1
if $data11 != 4 then
print ======data11=$data11
goto loop2
endi
# row 2
if $data21 != 4 then
print ======data21=$data21
goto loop2
endi
# row 3
if $data31 != 2 then
print ======data31=$data31
goto loop2
endi
sql insert into t1 values(1648791233000,0,1,1,1.0) (1648791233001,9,2,2,1.1) (1648791233002,9,2,2,1.1) (1648791233009,0,3,3,1.0);
$loop_count = 0
loop3:
sleep 300
print 1 sql select * from streamt;
sql select * from streamt;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 6 then
print ======rows=$rows
goto loop3
endi
sql insert into t1 values(1648791243000,0,1,1,1.0) (1648791243001,9,2,2,1.1);
$loop_count = 0
loop4:
sleep 300
print 1 sql select * from streamt;
sql select * from streamt;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 7 then
print ======rows=$rows
goto loop4
endi
sql insert into t1 values(1648791253000,0,1,1,1.0) (1648791253001,9,2,2,1.1) (1648791253002,9,2,2,1.1);
$loop_count = 0
loop5:
sleep 300
print 1 sql select * from streamt;
sql select * from streamt;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
print $data80 $data81 $data82 $data83
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 9 then
print ======rows=$rows
goto loop5
endi
sql insert into t1 values(1648791263000,0,1,1,1.0);
$loop_count = 0
loop6:
sleep 300
print 1 sql select * from streamt;
sql select * from streamt;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
print $data80 $data81 $data82 $data83
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 9 then
print ======rows=$rows
goto loop6
endi
print step2
print =============== create database
sql create database test2 vgroups 4;
sql use test2;
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create stream streams2 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2);
sleep 1000
sql insert into t1 values(1648791213000,0,1,1,1.0);
sleep 100
sql insert into t1 values(1648791213001,9,2,2,1.1);
sleep 100
sql insert into t1 values(1648791213002,0,3,3,1.0);
sleep 100
sql insert into t1 values(1648791213009,0,3,3,1.0);
$loop_count = 0
loop7:
sleep 300
print 1 sql select * from streamt2;
sql select * from streamt2;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 2 then
print ======rows=$rows
goto loop7
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop7
endi
# row 1
if $data11 != 2 then
print ======data11=$data11
goto loop7
endi
sql insert into t1 values(1648791223000,0,1,1,1.0);
sleep 100
sql insert into t1 values(1648791223001,9,2,2,1.1);
sleep 100
sql insert into t1 values(1648791223002,9,2,2,1.1);
sleep 100
sql insert into t1 values(1648791223009,0,3,3,1.0);
$loop_count = 0
loop8:
sleep 300
print 1 sql select * from streamt2;
sql select * from streamt2;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 4 then
print ======rows=$rows
goto loop8
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop8
endi
# row 1
if $data11 != 4 then
print ======data11=$data11
goto loop8
endi
# row 2
if $data21 != 4 then
print ======data21=$data21
goto loop8
endi
# row 3
if $data31 != 2 then
print ======data31=$data31
goto loop8
endi
sql insert into t1 values(1648791233000,0,1,1,1.0) (1648791233001,9,2,2,1.1) (1648791233002,9,2,2,1.1) (1648791233009,0,3,3,1.0);
$loop_count = 0
loop9:
sleep 300
print 1 sql select * from streamt2;
sql select * from streamt2;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 6 then
print ======rows=$rows
goto loop9
endi
sql insert into t1 values(1648791243000,0,1,1,1.0) (1648791243001,9,2,2,1.1);
$loop_count = 0
loop10:
sleep 300
print 1 sql select * from streamt2;
sql select * from streamt2;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 7 then
print ======rows=$rows
goto loop10
endi
sql insert into t1 values(1648791253000,0,1,1,1.0) (1648791253001,9,2,2,1.1) (1648791253002,9,2,2,1.1);
$loop_count = 0
loop11:
sleep 300
print 1 sql select * from streamt2;
sql select * from streamt2;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
print $data80 $data81 $data82 $data83
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 9 then
print ======rows=$rows
goto loop11
endi
sql insert into t1 values(1648791263000,0,1,1,1.0);
$loop_count = 0
loop12:
sleep 300
print 1 sql select * from streamt2;
sql select * from streamt2;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
print $data40 $data41 $data42 $data43
print $data50 $data51 $data52 $data53
print $data60 $data61 $data62 $data63
print $data70 $data71 $data72 $data73
print $data80 $data81 $data82 $data83
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 9 then
print ======rows=$rows
goto loop12
endi
print count sliding 0 end
system sh/exec.sh -n dnode1 -s stop -x SIGINT

181
tests/script/tsim/stream/countSliding1.sim Normal file
View File

@ -0,0 +1,181 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print step1
print =============== create database
sql create database test vgroups 1;
sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
sleep 1000
sql insert into t1 values(1648791213000,0,1,1,1.0);
sleep 100
sql insert into t1 values(1648791213001,9,2,2,1.1);
sleep 100
sql insert into t1 values(1648791213002,0,3,3,1.0);
sleep 100
sql insert into t1 values(1648791213009,0,3,3,1.0);
sleep 100
sql insert into t1 values(1648791223000,0,1,1,1.0);
sleep 100
sql insert into t1 values(1648791223001,9,2,2,1.1);
sleep 100
sql insert into t1 values(1648791223002,9,2,2,1.1);
sleep 100
sql insert into t1 values(1648791223009,0,3,3,1.0);
$loop_count = 0
loop0:
sleep 300
print 1 sql select * from streamt;
sql select * from streamt;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 4 then
print ======rows=$rows
goto loop0
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop0
endi
# row 1
if $data11 != 4 then
print ======data11=$data11
goto loop0
endi
# row 2
if $data21 != 4 then
print ======data21=$data21
goto loop0
endi
# row 3
if $data31 != 2 then
print ======data31=$data31
goto loop0
endi
sql insert into t1 values(1648791213000,0,1,1,1.0);
$loop_count = 0
loop1:
sleep 300
print 1 sql select * from streamt;
sql select * from streamt;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 4 then
print ======rows=$rows
goto loop1
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop1
endi
# row 1
if $data11 != 4 then
print ======data11=$data11
goto loop1
endi
# row 2
if $data21 != 4 then
print ======data21=$data21
goto loop1
endi
# row 3
if $data31 != 2 then
print ======data31=$data31
goto loop1
endi
sleep 500
sql insert into t1 values(1648791223002,9,2,2,1.1);
$loop_count = 0
loop2:
sleep 300
print 1 sql select * from streamt;
sql select * from streamt;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 4 then
print ======rows=$rows
goto loop2
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop2
endi
# row 1
if $data11 != 4 then
print ======data11=$data11
goto loop2
endi
# row 2
if $data21 != 4 then
print ======data21=$data21
goto loop2
endi
# row 3
if $data31 != 2 then
print ======data31=$data31
goto loop2
endi
print count sliding 1 end
system sh/exec.sh -n dnode1 -s stop -x SIGINT

175
tests/script/tsim/stream/countSliding2.sim Normal file
View File

@ -0,0 +1,175 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print step1
print =============== create database
sql create database test vgroups 1;
sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
sleep 1000
sql insert into t1 values(1648791213000,0,1,1,1.0);
sleep 100
sql insert into t1 values(1648791213001,9,2,2,1.1);
sleep 100
sql insert into t1 values(1648791213002,0,3,3,1.0);
sleep 100
sql insert into t1 values(1648791213009,0,3,3,1.0);
sleep 100
sql insert into t1 values(1648791223000,0,1,1,1.0);
sleep 100
sql insert into t1 values(1648791223001,9,2,2,1.1);
sleep 100
sql insert into t1 values(1648791223002,9,2,2,1.1);
sleep 100
sql insert into t1 values(1648791223009,0,3,3,1.0);
$loop_count = 0
loop0:
sleep 300
print 1 sql select * from streamt;
sql select * from streamt;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 4 then
print ======rows=$rows
goto loop0
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop0
endi
# row 1
if $data11 != 4 then
print ======data11=$data11
goto loop0
endi
# row 2
if $data21 != 4 then
print ======data21=$data21
goto loop0
endi
# row 3
if $data31 != 2 then
print ======data31=$data31
goto loop0
endi
sql delete from t1 where ts = 1648791213000;
$loop_count = 0
loop1:
sleep 300
print 1 sql select * from streamt;
sql select * from streamt;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 4 then
print ======rows=$rows
goto loop1
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop1
endi
# row 1
if $data11 != 4 then
print ======data11=$data11
goto loop1
endi
# row 2
if $data21 != 3 then
print ======data21=$data21
goto loop1
endi
# row 3
if $data31 != 1 then
print ======data31=$data31
goto loop1
endi
sleep 500
sql delete from t1 where ts = 1648791223002;
$loop_count = 0
loop2:
sleep 300
print 1 sql select * from streamt;
sql select * from streamt;
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data30 $data31 $data32 $data33
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $rows != 3 then
print ======rows=$rows
goto loop2
endi
# row 0
if $data01 != 4 then
print ======data01=$data01
goto loop2
endi
# row 1
if $data11 != 4 then
print ======data11=$data11
goto loop2
endi
# row 2
if $data21 != 2 then
print ======data21=$data21
goto loop2
endi
print count sliding 1 end
system sh/exec.sh -n dnode1 -s stop -x SIGINT