xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(stream)[TS-5469]: add more stream event notifications 

- Add event notifications for all window types
- Implement window invalidation notifications
- Add statistics for event notification delivery
This commit is contained in:
Jinqing Kuang 2025-02-12 23:33:15 +08:00
parent c0aa91ff1e
commit fc2d2672b9
32 changed files with 1340 additions and 430 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/tdatablock.h
View File

@ -195,6 +195,8 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const
// For the VAR_DATA_TYPE type, if a row already has data before inserting it (judged by offset != -1),
// it will be inserted at the original position and the old data will be overwritten.
int32_t colDataSetValOrCover(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull);
int32_t varColSetVarData(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pVarData, int32_t varDataLen,
bool isNull);
int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData);
int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows,
bool trimValue);

16
include/common/tmsg.h
View File

@ -2966,6 +2966,22 @@ typedef struct {
int8_t notifyHistory;
} SCMCreateStreamReq;
typedef struct STaskNotifyEventStat {
int64_t notifyEventAddTimes; // call times of add function
int64_t notifyEventAddElems; // elements added by add function
double notifyEventAddCostSec; // time cost of add function
int64_t notifyEventPushTimes; // call times of push function
int64_t notifyEventPushElems; // elements pushed by push function
double notifyEventPushCostSec; // time cost of push function
int64_t notifyEventPackTimes; // call times of pack function
int64_t notifyEventPackElems; // elements packed by pack function
double notifyEventPackCostSec; // time cost of pack function
int64_t notifyEventSendTimes; // call times of send function
int64_t notifyEventSendElems; // elements sent by send function
double notifyEventSendCostSec; // time cost of send function
int64_t notifyEventHoldElems; // elements hold due to watermark
} STaskNotifyEventStat;
typedef struct {
int64_t streamId;
} SCMCreateStreamRsp;

2
include/libs/executor/executor.h
View File

@ -99,7 +99,7 @@ int32_t qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId);
int32_t qSetStreamOpOpen(qTaskInfo_t tinfo);
int32_t qSetStreamNotifyInfo(qTaskInfo_t tinfo, int32_t eventTypes, const SSchemaWrapper* pSchemaWrapper,
const char* stbFullName, bool newSubTableRule);
const char* stbFullName, bool newSubTableRule, STaskNotifyEventStat* pNotifyEventStat);
/**
* Set multiple input data blocks for the stream scan.

1
include/libs/executor/storageapi.h
View File

@ -431,6 +431,7 @@ typedef struct SStateStore {
int32_t (*updateInfoSerialize)(SEncoder* pEncoder, const SUpdateInfo* pInfo);
int32_t (*updateInfoDeserialize)(SDecoder* pDeCoder, SUpdateInfo* pInfo);
SStreamStateCur* (*streamStateSessionSeekKeyPrev)(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* (*streamStateCountSeekKeyPrev)(SStreamState* pState, const SSessionKey* pKey, COUNT_TYPE count);
SStreamStateCur* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key);

1
include/libs/nodes/cmdnodes.h
View File

@ -573,6 +573,7 @@ typedef enum EStreamNotifyOptionSetFlag {
} EStreamNotifyOptionSetFlag;
typedef enum EStreamNotifyEventType {
SNOTIFY_EVENT_WINDOW_INVALIDATION = 0,
SNOTIFY_EVENT_WINDOW_OPEN = BIT_FLAG_MASK(0),
SNOTIFY_EVENT_WINDOW_CLOSE = BIT_FLAG_MASK(1),
} EStreamNotifyEventType;

3
include/libs/stream/streamState.h
View File

@ -65,6 +65,7 @@ int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* r
int32_t streamStateSessionAllocWinBuffByNextPosition(SStreamState* pState, SStreamStateCur* pCur,
const SSessionKey* pKey, void** pVal, int32_t* pVLen);
SStreamStateCur *streamStateSessionSeekKeyPrev(SStreamState *pState, const SSessionKey *key);
SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key);
SStreamStateCur* streamStateCountSeekKeyPrev(SStreamState* pState, const SSessionKey* pKey, COUNT_TYPE count);
SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key);
@ -162,4 +163,4 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2);
}
#endif
#endif /* ifndef _STREAM_STATE_H_ */
#endif /* ifndef _STREAM_STATE_H_ */

2
include/libs/stream/tstream.h
View File

@ -463,6 +463,7 @@ struct SStreamTask {
SUpstreamInfo upstreamInfo;
STaskCheckInfo taskCheckInfo;
SNotifyInfo notifyInfo;
STaskNotifyEventStat notifyEventStat;
// the followings attributes don't be serialized
SScanhistorySchedInfo schedHistoryInfo;
@ -632,6 +633,7 @@ typedef struct STaskStatusEntry {
int64_t startCheckpointVer;
int64_t hTaskId;
STaskCkptInfo checkpointInfo;
STaskNotifyEventStat notifyEventStat;
} STaskStatusEntry;
//typedef struct SNodeUpdateInfo {

2
include/libs/stream/tstreamFileState.h
View File

@ -100,10 +100,12 @@ void sessionWinStateCleanup(void* pBuff);
SStreamStateCur* createStateCursor(SStreamFileState* pFileState);
SStreamStateCur* sessionWinStateSeekKeyCurrentPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey);
SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
SStreamStateCur* sessionWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey);
SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey, COUNT_TYPE count);
int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
void sessionWinStateMoveToNext(SStreamStateCur* pCur);
void sessionWinStateMoveToPrev(SStreamStateCur* pCur);
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey,
range_cmpr_fn cmpFn);

1
include/util/tdef.h
View File

@ -246,6 +246,7 @@ typedef enum ELogicConditionType {
#define TSDB_USER_CGROUP_LEN (TSDB_USER_LEN + TSDB_CGROUP_LEN) // it is a null-terminated string
#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string
#define TSDB_STREAM_NOTIFY_URL_LEN 128 // it includes the terminating '\0'
#define TSDB_STREAM_NOTIFY_STAT_LEN 350 // it includes the terminating '\0'
#define TSDB_DB_NAME_LEN 65
#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_PRIVILEDGE_CONDITION_LEN 48 * 1024

30
source/common/src/msg/streamMsg.c
View File

@ -56,6 +56,7 @@ typedef struct STaskStatusEntry {
int64_t startCheckpointVer;
int64_t hTaskId;
STaskCkptInfo checkpointInfo;
STaskNotifyEventStat notifyEventStat;
} STaskStatusEntry;
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo) {
@ -523,6 +524,19 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) {
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->startCheckpointId));
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->startCheckpointVer));
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->hTaskId));
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventAddTimes));
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventAddElems));
TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventAddCostSec));
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPushTimes));
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPushElems));
TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventPushCostSec));
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPackTimes));
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPackElems));
TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventPackCostSec));
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventSendTimes));
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventSendElems));
TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventSendCostSec));
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventHoldElems));
}
int32_t numOfVgs = taosArrayGetSize(pReq->pUpdateNodes);
@ -596,6 +610,20 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) {
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.startCheckpointVer));
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.hTaskId));
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventAddTimes));
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventAddElems));
TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventAddCostSec));
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPushTimes));
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPushElems));
TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventPushCostSec));
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPackTimes));
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPackElems));
TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventPackCostSec));
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventSendTimes));
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventSendElems));
TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventSendCostSec));
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventHoldElems));
entry.id.taskId = taskId;
if (taosArrayPush(pReq->pTaskStatus, &entry) == NULL) {
TAOS_CHECK_EXIT(terrno);
@ -837,4 +865,4 @@ int32_t tDecodeStreamTaskRunReq(SDecoder* pDecoder, SStreamTaskRunReq* pReq) {
_exit:
return code;
}
}

1
source/common/src/systable.c
View File

@ -212,6 +212,7 @@ static const SSysDbTableSchema streamTaskSchema[] = {
{.name = "extra_info", .bytes = 25 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "history_task_id", .bytes = 16 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "history_task_status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "notify_event_stat", .bytes = TSDB_STREAM_NOTIFY_STAT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
};
static const SSysDbTableSchema userTblsSchema[] = {

49
source/common/src/tdatablock.c
View File

@ -165,6 +165,55 @@ int32_t colDataSetValOrCover(SColumnInfoData* pColumnInfoData, uint32_t rowIndex
return colDataSetValHelp(pColumnInfoData, rowIndex, pData, isNull);
}
int32_t varColSetVarData(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pVarData, int32_t varDataLen,
bool isNull) {
if (!IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
return TSDB_CODE_INVALID_PARA;
}
if (isNull || pVarData == NULL) {
pColumnInfoData->varmeta.offset[rowIndex] = -1; // it is a null value of VAR type.
pColumnInfoData->hasNull = true;
return TSDB_CODE_SUCCESS;
}
int32_t dataLen = VARSTR_HEADER_SIZE + varDataLen;
if (pColumnInfoData->varmeta.offset[rowIndex] > 0) {
pColumnInfoData->varmeta.length = pColumnInfoData->varmeta.offset[rowIndex];
}
SVarColAttr* pAttr = &pColumnInfoData->varmeta;
if (pAttr->allocLen < pAttr->length + dataLen) {
uint32_t newSize = pAttr->allocLen;
if (newSize <= 1) {
newSize = 8;
}
while (newSize < pAttr->length + dataLen) {
newSize = newSize * 1.5;
if (newSize > UINT32_MAX) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
char* buf = taosMemoryRealloc(pColumnInfoData->pData, newSize);
if (buf == NULL) {
return terrno;
}
pColumnInfoData->pData = buf;
pAttr->allocLen = newSize;
}
uint32_t len = pColumnInfoData->varmeta.length;
pColumnInfoData->varmeta.offset[rowIndex] = len;
(void)memmove(varDataVal(pColumnInfoData->pData + len), pVarData, varDataLen);
varDataSetLen(pColumnInfoData->pData + len, varDataLen);
pColumnInfoData->varmeta.length += dataLen;
return TSDB_CODE_SUCCESS;
}
int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx,
const char* pData) {
int32_t type = pColumnInfoData->info.type;

2
source/dnode/mnode/impl/src/mndStream.c
View File

@ -765,7 +765,7 @@ static int32_t addStreamTaskNotifyInfo(const SCMCreateStreamReq *createReq, cons
TSDB_CHECK_NULL(pTask->notifyInfo.pNotifyAddrUrls, code, lino, _end, terrno);
pTask->notifyInfo.notifyEventTypes = createReq->notifyEventTypes;
pTask->notifyInfo.notifyErrorHandle = createReq->notifyErrorHandle;
pTask->notifyInfo.streamName = taosStrdup(createReq->name);
pTask->notifyInfo.streamName = taosStrdup(mndGetDbStr(createReq->name));
TSDB_CHECK_NULL(pTask->notifyInfo.streamName, code, lino, _end, terrno);
pTask->notifyInfo.stbFullName = taosStrdup(createReq->targetStbFullName);
TSDB_CHECK_NULL(pTask->notifyInfo.stbFullName, code, lino, _end, terrno);

47
source/dnode/mnode/impl/src/mndStreamUtil.c
View File

@ -1309,8 +1309,8 @@ int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDataBlo
TSDB_CHECK_CODE(code, lino, _end);
// input queue
char vbuf[40] = {0};
char buf[38] = {0};
char vbuf[TSDB_STREAM_NOTIFY_STAT_LEN + 2] = {0};
char buf[TSDB_STREAM_NOTIFY_STAT_LEN] = {0};
const char *queueInfoStr = "%4.2f MiB (%6.2f%)";
snprintf(buf, tListLen(buf), queueInfoStr, pe->inputQUsed, pe->inputRate);
STR_TO_VARSTR(vbuf, buf);
@ -1503,6 +1503,47 @@ int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDataBlo
code = colDataSetVal(pColInfo, numOfRows, 0, true);
TSDB_CHECK_CODE(code, lino, _end);
// notify_event_stat
int32_t offset =0;
if (pe->notifyEventStat.notifyEventAddTimes > 0) {
offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Add %" PRId64 "x, %" PRId64 " elems in %lfs; ",
pe->notifyEventStat.notifyEventAddTimes, pe->notifyEventStat.notifyEventAddElems,
pe->notifyEventStat.notifyEventAddCostSec);
}
if (pe->notifyEventStat.notifyEventPushTimes > 0) {
offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Push %" PRId64 "x, %" PRId64 " elems in %lfs; ",
pe->notifyEventStat.notifyEventPushTimes, pe->notifyEventStat.notifyEventPushElems,
pe->notifyEventStat.notifyEventPushCostSec);
}
if (pe->notifyEventStat.notifyEventPackTimes > 0) {
offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Pack %" PRId64 "x, %" PRId64 " elems in %lfs; ",
pe->notifyEventStat.notifyEventPackTimes, pe->notifyEventStat.notifyEventPackElems,
pe->notifyEventStat.notifyEventPackCostSec);
}
if (pe->notifyEventStat.notifyEventSendTimes > 0) {
offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Send %" PRId64 "x, %" PRId64 " elems in %lfs; ",
pe->notifyEventStat.notifyEventSendTimes, pe->notifyEventStat.notifyEventSendElems,
pe->notifyEventStat.notifyEventSendCostSec);
}
if (pe->notifyEventStat.notifyEventHoldElems > 0) {
offset += tsnprintf(buf + offset, sizeof(buf) - offset, "[Hold %" PRId64 " elems] ",
pe->notifyEventStat.notifyEventHoldElems);
}
TSDB_CHECK_CONDITION(offset < sizeof(buf), code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
buf[offset] = '\0';
STR_TO_VARSTR(vbuf, buf);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
TSDB_CHECK_NULL(pColInfo, code, lino, _end, terrno);
if (offset == 0) {
colDataSetNULL(pColInfo, numOfRows);
} else {
code = colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
TSDB_CHECK_CODE(code, lino, _end);
}
_end:
if (code) {
mError("error happens during build task attr result blocks, lino:%d, code:%s", lino, tstrerror(code));
@ -1689,4 +1730,4 @@ int32_t mndCheckForSnode(SMnode *pMnode, SDbObj *pSrcDb) {
mError("snode not existed when trying to create stream in db with multiple replica");
return TSDB_CODE_SNODE_NOT_DEPLOYED;
}
}
}

3
source/dnode/snode/src/snodeInitApi.c
View File

@ -97,6 +97,7 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->updateInfoSerialize = updateInfoSerialize;
pStore->updateInfoDeserialize = updateInfoDeserialize;
pStore->streamStateSessionSeekKeyPrev = streamStateSessionSeekKeyPrev;
pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext;
pStore->streamStateCountSeekKeyPrev = streamStateCountSeekKeyPrev;
pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev;
@ -123,4 +124,4 @@ void initStateStoreAPI(SStateStore* pStore) {
void initFunctionStateStore(SFunctionStateStore* pStore) {
pStore->streamStateFuncPut = streamStateFuncPut;
pStore->streamStateFuncGet = streamStateFuncGet;
}
}

140
source/dnode/vnode/src/tq/tqStreamNotify.c
View File

@ -20,14 +20,16 @@
#include "curl/curl.h"
#endif
#define STREAM_EVENT_NOTIFY_RETRY_MS 50 // 50ms
#define STREAM_EVENT_NOTIFY_RETRY_MS 50 // 50 ms
#define STREAM_EVENT_NOTIFY_MESSAAGE_SIZE_KB 8 * 1024 // 8 MB
#define STREAM_EVENT_NOTIFY_FRAME_SIZE 256 * 1024 // 256 KB
typedef struct SStreamNotifyHandle {
TdThreadMutex mutex;
#ifndef WINDOWS
CURL* curl;
CURL* curl;
#endif
char* url;
char* url;
} SStreamNotifyHandle;
struct SStreamNotifyHandleMap {
@ -49,6 +51,7 @@ static void stopStreamNotifyConn(SStreamNotifyHandle* pHandle) {
}
// TODO: add wait mechanism for peer connection close response
curl_easy_cleanup(pHandle->curl);
pHandle->curl = NULL;
#endif
}
@ -258,7 +261,8 @@ _end:
}
static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBlocks, char** pMsg,
int32_t* nNotifyEvents) {
int32_t* nNotifyEvents, STaskNotifyEventStat* pNotifyEventStat,
int32_t* pBlockIdx) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t numOfBlocks = 0;
@ -268,15 +272,20 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
char* msgHeader = NULL;
const char* msgTail = "]}]}";
char* msg = NULL;
int64_t startTime = 0;
int64_t endTime = 0;
int32_t nBlocks = 0;
TSDB_CHECK_NULL(pMsg, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
*pMsg = NULL;
numOfBlocks = taosArrayGetSize(pBlocks);
*nNotifyEvents = 0;
for (int32_t i = 0; i < numOfBlocks; ++i) {
for (int32_t i = *pBlockIdx; i < numOfBlocks; ++i) {
SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
nBlocks++;
if (pDataBlock == NULL || pDataBlock->info.type != STREAM_NOTIFY_EVENT) {
continue;
}
@ -287,13 +296,19 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
msgLen += varDataLen(val) + 1;
}
*nNotifyEvents += pDataBlock->info.rows;
if (msgLen >= STREAM_EVENT_NOTIFY_MESSAAGE_SIZE_KB * 1024) {
break;
}
}
*pBlockIdx += nBlocks;
if (msgLen == 0) {
// skip since no notification events found
goto _end;
}
startTime = taosGetMonoTimestampMs();
code = getStreamNotifyEventHeader(streamName, &msgHeader);
TSDB_CHECK_CODE(code, lino, _end);
msgHeaderLen = strlen(msgHeader);
@ -306,7 +321,7 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
TAOS_STRNCPY(p, msgHeader, msgHeaderLen);
p += msgHeaderLen - msgTailLen;
for (int32_t i = 0; i < numOfBlocks; ++i) {
for (int32_t i = *pBlockIdx - nBlocks; i < *pBlockIdx; ++i) {
SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
if (pDataBlock == NULL || pDataBlock->info.type != STREAM_NOTIFY_EVENT) {
continue;
@ -328,6 +343,11 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
*pMsg = msg;
msg = NULL;
endTime = taosGetMonoTimestampMs();
pNotifyEventStat->notifyEventPackTimes++;
pNotifyEventStat->notifyEventPackElems += *nNotifyEvents;
pNotifyEventStat->notifyEventPackCostSec += (endTime - startTime) / 1000.0;
_end:
if (code != TSDB_CODE_SUCCESS) {
tqError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
@ -354,9 +374,21 @@ static int32_t sendSingleStreamNotify(SStreamNotifyHandle* pHandle, char* msg) {
TSDB_CHECK_NULL(pHandle->curl, code, lino, _end, TSDB_CODE_INVALID_PARA);
totalLen = strlen(msg);
while (sentLen < totalLen) {
res = curl_ws_send(pHandle->curl, msg + sentLen, totalLen - sentLen, &nbytes, 0, CURLWS_TEXT);
if (totalLen > 0) {
// send PING frame to check if the connection is still alive
res = curl_ws_send(pHandle->curl, "", 0, (size_t*)&sentLen, 0, CURLWS_PING);
TSDB_CHECK_CONDITION(res == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
}
sentLen = 0;
while (sentLen < totalLen) {
size_t chunkSize = TMIN(totalLen - sentLen, STREAM_EVENT_NOTIFY_FRAME_SIZE);
if (sentLen == 0) {
res = curl_ws_send(pHandle->curl, msg, chunkSize, &nbytes, totalLen, CURLWS_TEXT | CURLWS_OFFSET);
TSDB_CHECK_CONDITION(res == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
} else {
res = curl_ws_send(pHandle->curl, msg + sentLen, chunkSize, &nbytes, 0, CURLWS_TEXT | CURLWS_OFFSET);
TSDB_CHECK_CONDITION(res == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
}
sentLen += nbytes;
}
@ -379,6 +411,9 @@ int32_t tqSendAllNotifyEvents(const SArray* pBlocks, SStreamTask* pTask, SVnode*
int32_t nNotifyAddr = 0;
int32_t nNotifyEvents = 0;
SStreamNotifyHandle* pHandle = NULL;
int64_t startTime = 0;
int64_t endTime = 0;
int32_t blockIdx = 0;
TSDB_CHECK_NULL(pTask, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(pVnode, code, lino, _end, TSDB_CODE_INVALID_PARA);
@ -388,50 +423,61 @@ int32_t tqSendAllNotifyEvents(const SArray* pBlocks, SStreamTask* pTask, SVnode*
goto _end;
}
code = packupStreamNotifyEvent(pTask->notifyInfo.streamName, pBlocks, &msg, &nNotifyEvents);
TSDB_CHECK_CODE(code, lino, _end);
if (msg == NULL) {
goto _end;
}
while (blockIdx < taosArrayGetSize(pBlocks)) {
code = packupStreamNotifyEvent(pTask->notifyInfo.streamName, pBlocks, &msg, &nNotifyEvents, &pTask->notifyEventStat,
&blockIdx);
TSDB_CHECK_CODE(code, lino, _end);
if (msg == NULL) {
continue;
}
tqDebug("stream task %s prepare to send %d notify events, total msg length: %" PRIu64, pTask->notifyInfo.streamName,
nNotifyEvents, (uint64_t)strlen(msg));
tqDebug("stream task %s prepare to send %d notify events, total msg length: %" PRIu64, pTask->notifyInfo.streamName,
nNotifyEvents, (uint64_t)strlen(msg));
for (int32_t i = 0; i < nNotifyAddr; ++i) {
if (streamTaskShouldStop(pTask)) {
break;
}
const char* url = taosArrayGetP(pTask->notifyInfo.pNotifyAddrUrls, i);
code = acquireStreamNotifyHandle(pVnode->pNotifyHandleMap, url, &pHandle);
if (code != TSDB_CODE_SUCCESS) {
tqError("failed to get stream notify handle of %s", url);
if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
// retry for event message sending in PAUSE error handling mode
taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
--i;
continue;
} else {
// simply ignore the failure in DROP error handling mode
code = TSDB_CODE_SUCCESS;
continue;
startTime = taosGetMonoTimestampMs();
for (int32_t i = 0; i < nNotifyAddr; ++i) {
if (streamTaskShouldStop(pTask)) {
break;
}
}
code = sendSingleStreamNotify(pHandle, msg);
if (code != TSDB_CODE_SUCCESS) {
tqError("failed to send stream notify handle to %s since %s", url, tstrerror(code));
if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
// retry for event message sending in PAUSE error handling mode
taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
--i;
} else {
// simply ignore the failure in DROP error handling mode
code = TSDB_CODE_SUCCESS;
const char* url = taosArrayGetP(pTask->notifyInfo.pNotifyAddrUrls, i);
code = acquireStreamNotifyHandle(pVnode->pNotifyHandleMap, url, &pHandle);
if (code != TSDB_CODE_SUCCESS) {
tqError("failed to get stream notify handle of %s", url);
if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
// retry for event message sending in PAUSE error handling mode
taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
--i;
continue;
} else {
// simply ignore the failure in DROP error handling mode
code = TSDB_CODE_SUCCESS;
continue;
}
}
} else {
tqDebug("stream task %s send %d notify events to %s successfully", pTask->notifyInfo.streamName, nNotifyEvents,
url);
code = sendSingleStreamNotify(pHandle, msg);
if (code != TSDB_CODE_SUCCESS) {
tqError("failed to send stream notify handle to %s since %s", url, tstrerror(code));
if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
// retry for event message sending in PAUSE error handling mode
taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
--i;
} else {
// simply ignore the failure in DROP error handling mode
code = TSDB_CODE_SUCCESS;
}
} else {
tqDebug("stream task %s send %d notify events to %s successfully", pTask->notifyInfo.streamName, nNotifyEvents,
url);
}
releaseStreamNotifyHandle(&pHandle);
}
releaseStreamNotifyHandle(&pHandle);
endTime = taosGetMonoTimestampMs();
pTask->notifyEventStat.notifyEventSendTimes++;
pTask->notifyEventStat.notifyEventSendElems += nNotifyEvents;
pTask->notifyEventStat.notifyEventSendCostSec += (endTime - startTime) / 1000.0;
taosMemoryFreeClear(msg);
}
_end:

6
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -87,9 +87,9 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) {
return code;
}
code =
qSetStreamNotifyInfo(pTask->exec.pExecutor, pTask->notifyInfo.notifyEventTypes,
pTask->notifyInfo.pSchemaWrapper, pTask->notifyInfo.stbFullName, IS_NEW_SUBTB_RULE(pTask));
code = qSetStreamNotifyInfo(pTask->exec.pExecutor, pTask->notifyInfo.notifyEventTypes,
pTask->notifyInfo.pSchemaWrapper, pTask->notifyInfo.stbFullName,
IS_NEW_SUBTB_RULE(pTask), &pTask->notifyEventStat);
if (code) {
tqError("s-task:%s failed to set stream notify info, code:%s", pTask->id.idStr, tstrerror(code));
return code;

1
source/dnode/vnode/src/vnd/vnodeInitApi.c
View File

@ -223,6 +223,7 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->updateInfoSerialize = updateInfoSerialize;
pStore->updateInfoDeserialize = updateInfoDeserialize;
pStore->streamStateSessionSeekKeyPrev = streamStateSessionSeekKeyPrev;
pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext;
pStore->streamStateCountSeekKeyPrev = streamStateCountSeekKeyPrev;
pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev;

15
source/libs/executor/inc/executorInt.h
View File

@ -450,16 +450,17 @@ typedef struct STimeWindowAggSupp {
} STimeWindowAggSupp;
typedef struct SStreamNotifyEventSupp {
SArray* pWindowEvents; // Array of SStreamNotifyEvent, storing window events and trigger values.
SHashObj* pTableNameHashMap; // Hash map from groupid to the dest child table name.
SHashObj* pResultHashMap; // Hash map from groupid+skey to the window agg result.
SSDataBlock* pEventBlock; // The datablock contains all window events and results.
SHashObj* pWindowEventHashMap; // Hash map from gorupid+skey+eventType to the list node of window event.
SHashObj* pTableNameHashMap; // Hash map from groupid to the dest child table name.
SSDataBlock* pEventBlock; // The datablock contains all window events and results.
SArray* pSessionKeys;
const char* windowType;
} SStreamNotifyEventSupp;
typedef struct SSteamOpBasicInfo {
int32_t primaryPkIndex;
bool updateOperatorInfo;
SStreamNotifyEventSupp windowEventSup;
SStreamNotifyEventSupp notifyEventSup;
} SSteamOpBasicInfo;
typedef struct SStreamFillSupporter {
@ -1053,7 +1054,7 @@ int32_t saveDeleteRes(SSHashObj* pStDelete, SSessionKey key);
void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey);
void doBuildDeleteDataBlock(struct SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite);
void doBuildSessionResult(struct SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo,
SSDataBlock* pBlock);
SSDataBlock* pBlock, SArray* pSessionKeys);
int32_t getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo);
void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin,
SResultWindowInfo* pNextWin);
@ -1090,7 +1091,7 @@ void freeResetOperatorParams(struct SOperatorInfo* pOperator, SOperatorParamT
int32_t getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int32_t idx, bool clearParam,
SSDataBlock** pResBlock);
void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, EStreamType mode, SSessionKey* pDelRange);
void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey);
void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey);
int32_t saveDeleteInfo(SArray* pWins, SSessionKey key);
void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins);

39
source/libs/executor/inc/querytask.h
View File

@ -18,9 +18,9 @@
#ifdef __cplusplus
extern "C" {
#endif
#include "executorInt.h"
#endif
#include "executorInt.h"
#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str)
@ -59,22 +59,23 @@ typedef struct STaskStopInfo {
} STaskStopInfo;
typedef struct {
STqOffsetVal currentOffset; // for tmq
SMqBatchMetaRsp btMetaRsp; // for tmq fetching meta
int8_t sourceExcluded;
int64_t snapshotVer;
SSchemaWrapper* schema;
char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor
int8_t recoverStep;
int8_t recoverScanFinished;
SQueryTableDataCond tableCond;
SVersionRange fillHistoryVer;
STimeWindow fillHistoryWindow;
SStreamState* pState;
int32_t eventTypes; // event types to notify
SSchemaWrapper* notifyResultSchema; // agg result to notify
char* stbFullName; // used to generate dest child table name
bool newSubTableRule; // used to generate dest child table name
STqOffsetVal currentOffset; // for tmq
SMqBatchMetaRsp btMetaRsp; // for tmq fetching meta
int8_t sourceExcluded;
int64_t snapshotVer;
SSchemaWrapper* schema;
char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor
int8_t recoverStep;
int8_t recoverScanFinished;
SQueryTableDataCond tableCond;
SVersionRange fillHistoryVer;
STimeWindow fillHistoryWindow;
SStreamState* pState;
int32_t eventTypes; // event types to notify
SSchemaWrapper* notifyResultSchema; // agg result to notify
char* stbFullName; // used to generate dest child table name
bool newSubTableRule; // used to generate dest child table name
STaskNotifyEventStat* pNotifyEventStat; // used to store notify event statistics
} SStreamTaskInfo;
struct SExecTaskInfo {

60
source/libs/executor/inc/streamexecutorInt.h
View File

@ -30,20 +30,31 @@ extern "C" {
#define FILL_POS_MID 2
#define FILL_POS_END 3
#define HAS_NON_ROW_DATA(pRowData) (pRowData->key == INT64_MIN)
#define HAS_ROW_DATA(pRowData) (pRowData && pRowData->key != INT64_MIN)
#define HAS_NON_ROW_DATA(pRowData) (pRowData->key == INT64_MIN)
#define HAS_ROW_DATA(pRowData) (pRowData && pRowData->key != INT64_MIN)
#define IS_INVALID_WIN_KEY(ts) ((ts) == INT64_MIN)
#define IS_VALID_WIN_KEY(ts) ((ts) != INT64_MIN)
#define SET_WIN_KEY_INVALID(ts) ((ts) = INT64_MIN)
#define IS_INVALID_WIN_KEY(ts) ((ts) == INT64_MIN)
#define IS_VALID_WIN_KEY(ts) ((ts) != INT64_MIN)
#define SET_WIN_KEY_INVALID(ts) ((ts) = INT64_MIN)
#define IS_NORMAL_INTERVAL_OP(op) \
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || \
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
#define IS_NORMAL_SESSION_OP(op) \
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE)
#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT)
#define IS_NORMAL_COUNT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT)
#define IS_CONTINUE_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL)
#define IS_FILL_CONST_VALUE(type) ((type == TSDB_FILL_NULL || type == TSDB_FILL_NULL_F || type == TSDB_FILL_SET_VALUE || type == TSDB_FILL_SET_VALUE_F))
#define IS_FILL_CONST_VALUE(type) \
((type == TSDB_FILL_NULL || type == TSDB_FILL_NULL_F || type == TSDB_FILL_SET_VALUE || type == TSDB_FILL_SET_VALUE_F))
typedef struct SSliceRowData {
TSKEY key;
@ -57,11 +68,13 @@ typedef struct SSlicePoint {
SRowBuffPos* pResPos;
} SSlicePoint;
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo);
int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo);
void destroyStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo);
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo);
int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo, const struct SOperatorInfo* pOperator);
void destroyStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo);
int32_t encodeStreamBasicInfo(void** buf, const SSteamOpBasicInfo* pBasicInfo);
int32_t decodeStreamBasicInfo(void** buf, SSteamOpBasicInfo* pBasicInfo);
int64_t getDeleteMarkFromOption(SStreamNodeOption* pOption);
void removeDeleteResults(SSHashObj* pUpdatedMap, SArray* pDelWins);
@ -98,7 +111,7 @@ SResultCellData* getSliceResultCell(SResultCellData* pRowVal, int32_t index, int
int32_t getDownstreamRes(struct SOperatorInfo* downstream, SSDataBlock** ppRes, SColumnInfo** ppPkCol);
void destroyFlusedppPos(void* ppRes);
void doBuildStreamIntervalResult(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
SGroupResInfo* pGroupResInfo);
SGroupResInfo* pGroupResInfo, SArray* pSessionKeys);
void transBlockToSliceResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SSliceRowData* pRowVal,
int32_t rowSize, void* pPkData, SColumnInfoData* pPkCol, int32_t* pCellOffsetInfo);
int32_t getQualifiedRowNumDesc(SExprSupp* pExprSup, SSDataBlock* pBlock, TSKEY* tsCols, int32_t rowId, bool ignoreNull);
@ -112,10 +125,25 @@ TSKEY compareTs(void* pKey);
int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
const SSDataBlock* pInputBlock, const SNodeList* pCondCols, int32_t ri,
SStreamNotifyEventSupp* sup);
int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SSchemaWrapper* pSchemaWrapper,
SStreamNotifyEventSupp* sup);
int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup);
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
int32_t addStateAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
const SStateKeys* pCurState, const SStateKeys* pAnotherState, bool onlyUpdate,
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
int32_t addIntervalAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
int32_t addSessionAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
int32_t addCountAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SArray* pSessionKeys,
const SSchemaWrapper* pSchemaWrapper, SStreamNotifyEventSupp* sup,
STaskNotifyEventStat* pNotifyEventStat);
int32_t addAggDeleteNotifyEvent(const SSDataBlock* pDeleteBlock, SStreamNotifyEventSupp* sup,
STaskNotifyEventStat* pNotifyEventStat);
int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup,
STaskNotifyEventStat* pNotifyEventStat);
int32_t removeOutdatedNotifyEvents(STimeWindowAggSupp* pTwSup, SStreamNotifyEventSupp* sup,
STaskNotifyEventStat* pNotifyEventStat);
#ifdef __cplusplus
}

3
source/libs/executor/src/executor.c
View File

@ -251,7 +251,7 @@ int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) {
}
int32_t qSetStreamNotifyInfo(qTaskInfo_t tinfo, int32_t eventTypes, const SSchemaWrapper* pSchemaWrapper,
const char* stbFullName, bool newSubTableRule) {
const char* stbFullName, bool newSubTableRule, STaskNotifyEventStat* pNotifyEventStat) {
int32_t code = TSDB_CODE_SUCCESS;
SStreamTaskInfo *pStreamInfo = NULL;
@ -267,6 +267,7 @@ int32_t qSetStreamNotifyInfo(qTaskInfo_t tinfo, int32_t eventTypes, const SSchem
}
pStreamInfo->stbFullName = taosStrdup(stbFullName);
pStreamInfo->newSubTableRule = newSubTableRule;
pStreamInfo->pNotifyEventStat = pNotifyEventStat;
_end:
return code;

105
source/libs/executor/src/streamcountwindowoperator.c
View File

@ -25,7 +25,6 @@
#include "tlog.h"
#include "ttime.h"
#define IS_NORMAL_COUNT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT)
#define STREAM_COUNT_OP_STATE_NAME "StreamCountHistoryState"
#define STREAM_COUNT_OP_CHECKPOINT_NAME "StreamCountOperator_Checkpoint"
@ -56,6 +55,8 @@ void destroyStreamCountAggOperatorInfo(void* param) {
&pInfo->groupResInfo);
pInfo->pOperator = NULL;
}
destroyStreamBasicInfo(&pInfo->basic);
destroyStreamAggSupporter(&pInfo->streamAggSup);
cleanupExprSupp(&pInfo->scalarSupp);
clearGroupResInfo(&pInfo->groupResInfo);
@ -79,10 +80,9 @@ void destroyStreamCountAggOperatorInfo(void* param) {
bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; }
int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin,
SBuffInfo* pBuffInfo) {
SBuffInfo* pBuffInfo, int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t winCode = TSDB_CODE_SUCCESS;
int32_t size = pAggSup->resultRowSize;
pCurWin->winInfo.sessionWin.groupId = groupId;
pCurWin->winInfo.sessionWin.win.skey = ts;
@ -90,19 +90,21 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
if (isSlidingCountWindow(pAggSup)) {
if (pBuffInfo->winBuffOp == CREATE_NEW_WINDOW) {
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount,
(void**)&pCurWin->winInfo.pStatePos, &size);
code =
pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size);
QUERY_CHECK_CODE(code, lino, _end);
winCode = TSDB_CODE_FAILED;
*pWinCode = TSDB_CODE_FAILED;
} else if (pBuffInfo->winBuffOp == MOVE_NEXT_WINDOW) {
QUERY_CHECK_NULL(pBuffInfo->pCur, code, lino, _end, terrno);
pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur);
winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
if (winCode == TSDB_CODE_FAILED) {
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount,
(void**)&pCurWin->winInfo.pStatePos, &size);
*pWinCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
if (*pWinCode == TSDB_CODE_FAILED) {
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos,
&size);
QUERY_CHECK_CODE(code, lino, _end);
} else {
reuseOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->stateStore);
@ -110,11 +112,12 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
} else {
pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount);
winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
if (winCode == TSDB_CODE_FAILED) {
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount,
(void**)&pCurWin->winInfo.pStatePos, &size);
*pWinCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
if (*pWinCode == TSDB_CODE_FAILED) {
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos,
&size);
QUERY_CHECK_CODE(code, lino, _end);
} else {
reuseOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->stateStore);
@ -126,11 +129,11 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
} else {
code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount,
(void**)&pCurWin->winInfo.pStatePos, &size, &winCode);
(void**)&pCurWin->winInfo.pStatePos, &size, pWinCode);
QUERY_CHECK_CODE(code, lino, _end);
}
if (winCode == TSDB_CODE_SUCCESS) {
if (*pWinCode == TSDB_CODE_SUCCESS) {
pCurWin->winInfo.isOutput = true;
}
pCurWin->pWindowCount =
@ -297,10 +300,18 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
continue;
}
SCountWindowInfo curWin = {0};
int32_t winCode = TSDB_CODE_SUCCESS;
buffInfo.rebuildWindow = false;
code = setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo);
code = setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS &&
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
code = addCountAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, &pInfo->basic.notifyEventSup,
pTaskInfo->streamInfo.pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
if (!inCountSlidingWindow(pAggSup, &curWin.winInfo.sessionWin.win, &pSDataBlock->info)) {
buffInfo.winBuffOp = MOVE_NEXT_WINDOW;
continue;
@ -375,23 +386,54 @@ _end:
static int32_t buildCountResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
bool addNotifyEvent = false;
addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
if (addNotifyEvent) {
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
(*ppRes) = pInfo->pDelRes;
return code;
}
doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes,
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
if (pBInfo->pRes->info.rows > 0) {
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
if (addNotifyEvent) {
code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
(*ppRes) = pBInfo->pRes;
return code;
}
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pNotifySup->pEventBlock;
return code;
}
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
(*ppRes) = NULL;
return code;
}
@ -423,7 +465,10 @@ int32_t doStreamCountEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
// 3.dataVersion
tlen += taosEncodeFixedI32(buf, pInfo->dataVersion);
// 4.checksum
// 4.basicInfo
tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
// 5.checksum
if (isParent) {
if (buf) {
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
@ -441,12 +486,13 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
int32_t lino = 0;
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
void* pDataEnd = POINTER_SHIFT(buf, len);
if (!pInfo) {
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
// 4.checksum
// 5.checksum
if (isParent) {
int32_t dataLen = len - sizeof(uint32_t);
void* pCksum = POINTER_SHIFT(buf, dataLen);
@ -454,6 +500,7 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
pDataEnd = pCksum;
}
// 1.streamAggSup.pResultRows
@ -462,9 +509,10 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
for (int32_t i = 0; i < mapSize; i++) {
SSessionKey key = {0};
SCountWindowInfo curWin = {0};
int32_t winCode = TSDB_CODE_SUCCESS;
buf = decodeSSessionKey(buf, &key);
SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL};
code = setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo);
code = setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize);
@ -479,6 +527,12 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
// 3.dataVersion
buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);
// 4.basicInfo
if (buf < pDataEnd) {
code = decodeStreamBasicInfo(&buf, &pInfo->basic);
QUERY_CHECK_CODE(code, lino, _end);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
@ -851,7 +905,7 @@ int32_t createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);
pInfo->binfo.pRes = pResBlock;
SExprInfo* pExprInfo = NULL;
SExprInfo* pExprInfo = NULL;
code = createExprInfo(pCountNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);
QUERY_CHECK_CODE(code, lino, _error);
@ -925,6 +979,9 @@ int32_t createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState);
code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
if (downstream) {
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);

78
source/libs/executor/src/streameventwindowoperator.c
View File

@ -30,7 +30,6 @@
#include "tlog.h"
#include "ttime.h"
#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT)
#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState"
#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint"
@ -135,7 +134,8 @@ void reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) {
}
int32_t setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupId, bool* pStart, bool* pEnd,
int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey, int32_t* pWinCode) {
int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey,
int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t winCode = TSDB_CODE_SUCCESS;
@ -179,7 +179,7 @@ int32_t setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t gro
SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId};
code = pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len);
QUERY_CHECK_CODE(code, lino, _error);
(*pWinCode) = TSDB_CODE_FAILED;
(*pWinCode) = TSDB_CODE_FAILED;
setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin);
pCurWin->pWinFlag->startFlag = start;
@ -335,6 +335,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SColumnInfoData* pColStart = NULL;
SColumnInfoData* pColEnd = NULL;
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
@ -395,17 +397,19 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
&nextWinKey, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN) &&
*(bool*)colDataGetNumData(pColStart, i) && winCode != TSDB_CODE_SUCCESS) {
if (winCode != TSDB_CODE_SUCCESS &&
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN) &&
*(bool*)colDataGetNumData(pColStart, i)) {
code = addEventAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, pSDataBlock,
pInfo->pStartCondCols, i, &pInfo->basic.windowEventSup);
pInfo->pStartCondCols, i, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);
bool rebuild = false;
code = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData,
rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild, &winRows);
code = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData,
(bool*)pColEnd->pData, rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild,
&winRows);
QUERY_CHECK_CODE(code, lino, _end);
if (rebuild) {
@ -471,7 +475,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {
code = addEventAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &curWin.winInfo.sessionWin, pSDataBlock,
pInfo->pEndCondCols, i + winRows - 1, &pInfo->basic.windowEventSup);
pInfo->pEndCondCols, i + winRows - 1, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
}
@ -513,7 +517,10 @@ int32_t doStreamEventEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
// 3.dataVersion
tlen += taosEncodeFixedI32(buf, pInfo->dataVersion);
// 4.checksum
// 4.basicInfo
tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
// 5.checksum
if (buf) {
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
tlen += taosEncodeFixedU32(buf, cksum);
@ -529,13 +536,14 @@ int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
int32_t lino = 0;
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
void* pDataEnd = POINTER_SHIFT(buf, len);
if (!pInfo) {
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
// 4.checksum
// 5.checksum
int32_t dataLen = len - sizeof(uint32_t);
void* pCksum = POINTER_SHIFT(buf, dataLen);
if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {
@ -543,6 +551,7 @@ int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
pDataEnd = pCksum;
// 1.streamAggSup.pResultRows
int32_t mapSize = 0;
@ -567,6 +576,12 @@ int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
// 3.dataVersion
buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);
// 4.basicInfo
if (buf < pDataEnd) {
code = decodeStreamBasicInfo(&buf, &pInfo->basic);
QUERY_CHECK_CODE(code, lino, _end);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
@ -598,33 +613,45 @@ static int32_t buildEventResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
bool addNotifyEvent = false;
addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
if (addNotifyEvent) {
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
(*ppRes) = pInfo->pDelRes;
return code;
}
doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes);
doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes,
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
if (pBInfo->pRes->info.rows > 0) {
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {
code = addAggResultNotifyEvent(pBInfo->pRes, pTaskInfo->streamInfo.notifyResultSchema, &pInfo->basic.windowEventSup);
if (addNotifyEvent) {
code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
(*ppRes) = pBInfo->pRes;
return code;
}
code = buildNotifyEventBlock(pTaskInfo, &pInfo->basic.windowEventSup);
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
if (pInfo->basic.windowEventSup.pEventBlock->info.rows > 0) {
printDataBlock(pInfo->basic.windowEventSup.pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pInfo->basic.windowEventSup.pEventBlock;
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pNotifySup->pEventBlock;
return code;
}
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
_end:
(*ppRes) = NULL;
if (code != TSDB_CODE_SUCCESS) {
@ -800,7 +827,7 @@ void streamEventReleaseState(SOperatorInfo* pOperator) {
int32_t resSize = winSize + sizeof(TSKEY);
char* pBuff = taosMemoryCalloc(1, resSize);
if (!pBuff) {
return ;
return;
}
memcpy(pBuff, pInfo->historyWins->pData, winSize);
memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));
@ -954,9 +981,9 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);
pInfo->binfo.pRes = pResBlock;
SExprSupp* pExpSup = &pOperator->exprSupp;
int32_t numOfCols = 0;
SExprInfo* pExprInfo = NULL;
SExprSupp* pExpSup = &pOperator->exprSupp;
int32_t numOfCols = 0;
SExprInfo* pExprInfo = NULL;
code = createExprInfo(pEventNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);
QUERY_CHECK_CODE(code, lino, _error);
@ -1006,7 +1033,6 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);
QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno);
pInfo->destHasPrimaryKey = pEventNode->window.destHasPrimaryKey;
initStreamBasicInfo(&pInfo->basic);
pInfo->pOperator = pOperator;
setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED,
@ -1026,6 +1052,10 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAggNext, NULL, destroyStreamEventOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState);
code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
QUERY_CHECK_CODE(code, lino, _error);

706
source/libs/executor/src/streamexecutorInt.c
View File

@ -16,53 +16,58 @@
#include "streamexecutorInt.h"
#include "executorInt.h"
#include "operator.h"
#include "tdatablock.h"
#define NOTIFY_EVENT_NAME_CACHE_LIMIT_MB 16
typedef struct SStreamNotifyEvent {
uint64_t gid;
TSKEY skey;
char* content;
bool isEnd;
int64_t eventType;
STimeWindow win;
cJSON* pJson;
} SStreamNotifyEvent;
#define NOTIFY_EVENT_KEY_SIZE \
((sizeof(((struct SStreamNotifyEvent*)0)->gid) + sizeof(((struct SStreamNotifyEvent*)0)->eventType)) + \
sizeof(((struct SStreamNotifyEvent*)0)->win.skey))
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type) {
if (type != STREAM_GET_ALL && type != STREAM_CHECKPOINT) {
pBasicInfo->updateOperatorInfo = true;
}
}
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo) {
return pBasicInfo->updateOperatorInfo;
}
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo) { return pBasicInfo->updateOperatorInfo; }
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo) {
pBasicInfo->updateOperatorInfo = false;
}
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo) { pBasicInfo->updateOperatorInfo = false; }
static void destroyStreamWindowEvent(void* ptr) {
SStreamNotifyEvent* pEvent = ptr;
if (pEvent == NULL || pEvent->content == NULL) return;
cJSON_free(pEvent->content);
SStreamNotifyEvent* pEvent = (SStreamNotifyEvent*)ptr;
if (pEvent) {
if (pEvent->pJson) {
cJSON_Delete(pEvent->pJson);
}
*pEvent = (SStreamNotifyEvent){0};
}
}
static void destroyStreamNotifyEventSupp(SStreamNotifyEventSupp* sup) {
if (sup == NULL) return;
taosArrayDestroyEx(sup->pWindowEvents, destroyStreamWindowEvent);
taosHashCleanup(sup->pWindowEventHashMap);
taosHashCleanup(sup->pTableNameHashMap);
taosHashCleanup(sup->pResultHashMap);
blockDataDestroy(sup->pEventBlock);
taosArrayDestroy(sup->pSessionKeys);
*sup = (SStreamNotifyEventSupp){0};
}
static int32_t initStreamNotifyEventSupp(SStreamNotifyEventSupp *sup) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SSDataBlock* pBlock = NULL;
static int32_t initStreamNotifyEventSupp(SStreamNotifyEventSupp* sup, const char* windowType, int32_t resCapacity) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SSDataBlock* pBlock = NULL;
SColumnInfoData infoData = {0};
if (sup == NULL) {
if (sup == NULL || sup->pWindowEventHashMap != NULL) {
goto _end;
}
@ -77,15 +82,18 @@ static int32_t initStreamNotifyEventSupp(SStreamNotifyEventSupp *sup) {
code = blockDataAppendColInfo(pBlock, &infoData);
QUERY_CHECK_CODE(code, lino, _end);
sup->pWindowEvents = taosArrayInit(0, sizeof(SStreamNotifyEvent));
QUERY_CHECK_NULL(sup->pWindowEvents, code, lino, _end, terrno);
sup->pWindowEventHashMap = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
QUERY_CHECK_NULL(sup->pWindowEventHashMap, code, lino, _end, terrno);
taosHashSetFreeFp(sup->pWindowEventHashMap, destroyStreamWindowEvent);
sup->pTableNameHashMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
QUERY_CHECK_NULL(sup->pTableNameHashMap, code, lino, _end, terrno);
sup->pResultHashMap = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
QUERY_CHECK_NULL(sup->pResultHashMap, code, lino, _end, terrno);
taosHashSetFreeFp(sup->pResultHashMap, destroyStreamWindowEvent);
sup->pEventBlock = pBlock;
pBlock = NULL;
code = blockDataEnsureCapacity(sup->pEventBlock, resCapacity);
QUERY_CHECK_CODE(code, lino, _end);
sup->windowType = windowType;
sup->pSessionKeys = taosArrayInit(resCapacity, sizeof(SSessionKey));
QUERY_CHECK_NULL(sup->pSessionKeys, code, lino, _end, terrno);
_end:
if (code != TSDB_CODE_SUCCESS) {
@ -100,17 +108,99 @@ _end:
return code;
}
int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo) {
int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo, const struct SOperatorInfo* pOperator) {
pBasicInfo->primaryPkIndex = -1;
pBasicInfo->updateOperatorInfo = false;
return initStreamNotifyEventSupp(&pBasicInfo->windowEventSup);
const char* windowType = NULL;
if (IS_NORMAL_INTERVAL_OP(pOperator)) {
windowType = "Time";
} else if (IS_NORMAL_SESSION_OP(pOperator)) {
windowType = "Session";
} else if (IS_NORMAL_STATE_OP(pOperator)) {
windowType = "State";
} else if (IS_NORMAL_EVENT_OP(pOperator)) {
windowType = "Event";
} else if (IS_NORMAL_COUNT_OP(pOperator)) {
windowType = "Count";
} else {
return TSDB_CODE_SUCCESS;
}
return initStreamNotifyEventSupp(&pBasicInfo->notifyEventSup, windowType, pOperator->resultInfo.capacity);
}
void destroyStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo) {
destroyStreamNotifyEventSupp(&pBasicInfo->windowEventSup);
destroyStreamNotifyEventSupp(&pBasicInfo->notifyEventSup);
}
static void streamNotifyGetEventWindowId(const SSessionKey* pSessionKey, char *buf) {
static int32_t encodeStreamNotifyEventSupp(void** buf, const SStreamNotifyEventSupp* sup) {
int32_t tlen = 0;
void* pIter = NULL;
char* str = NULL;
if (sup == NULL) {
return tlen;
}
tlen += taosEncodeFixedI32(buf, taosHashGetSize(sup->pWindowEventHashMap));
pIter = taosHashIterate(sup->pWindowEventHashMap, NULL);
while (pIter) {
const SStreamNotifyEvent* pEvent = (const SStreamNotifyEvent*)pIter;
str = cJSON_PrintUnformatted(pEvent->pJson);
tlen += taosEncodeFixedU64(buf, pEvent->gid);
tlen += taosEncodeFixedI64(buf, pEvent->eventType);
tlen += taosEncodeFixedI64(buf, pEvent->win.skey);
tlen += taosEncodeFixedI64(buf, pEvent->win.ekey);
tlen += taosEncodeString(buf, str);
cJSON_free(str);
pIter = taosHashIterate(sup->pWindowEventHashMap, pIter);
}
return tlen;
}
static int32_t decodeStreamNotifyEventSupp(void** buf, SStreamNotifyEventSupp* sup) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
void* p = *buf;
int32_t size = 0;
uint64_t len = 0;
SStreamNotifyEvent item = {0};
p = taosDecodeFixedI32(p, &size);
for (int32_t i = 0; i < size; i++) {
p = taosDecodeFixedU64(p, &item.gid);
p = taosDecodeFixedI64(p, &item.eventType);
p = taosDecodeFixedI64(p, &item.win.skey);
p = taosDecodeFixedI64(p, &item.win.ekey);
p = taosDecodeVariantU64(p, &len);
item.pJson = cJSON_Parse(p);
if (item.pJson == NULL) {
qWarn("failed to parse the json content since %s", cJSON_GetErrorPtr());
}
QUERY_CHECK_NULL(item.pJson, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
p = POINTER_SHIFT(p, len);
code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
QUERY_CHECK_CODE(code, lino, _end);
item.pJson = NULL;
}
*buf = p;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
destroyStreamWindowEvent(&item);
return code;
}
int32_t encodeStreamBasicInfo(void** buf, const SSteamOpBasicInfo* pBasicInfo) {
return encodeStreamNotifyEventSupp(buf, &pBasicInfo->notifyEventSup);
}
int32_t decodeStreamBasicInfo(void** buf, SSteamOpBasicInfo* pBasicInfo) {
return decodeStreamNotifyEventSupp(buf, &pBasicInfo->notifyEventSup);
}
static void streamNotifyGetEventWindowId(const SSessionKey* pSessionKey, char* buf) {
uint64_t hash = 0;
uint64_t ar[2];
@ -123,60 +213,60 @@ static void streamNotifyGetEventWindowId(const SSessionKey* pSessionKey, char *b
#define JSON_CHECK_ADD_ITEM(obj, str, item) \
QUERY_CHECK_CONDITION(cJSON_AddItemToObjectCS(obj, str, item), code, lino, _end, TSDB_CODE_OUT_OF_MEMORY)
static int32_t jsonAddColumnField(const char* colName, const SColumnInfoData* pColData, int32_t ri, cJSON* obj) {
static int32_t jsonAddColumnField(const char* colName, int16_t type, bool isNull, const char* pData, cJSON* obj) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
char* temp = NULL;
QUERY_CHECK_NULL(colName, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_CONDITION(isNull || (pData != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(obj, code, lino, _end, TSDB_CODE_INVALID_PARA);
if (colDataIsNull_s(pColData, ri)) {
if (isNull) {
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNull());
goto _end;
}
switch (pColData->info.type) {
switch (type) {
case TSDB_DATA_TYPE_BOOL: {
bool val = *(bool*)colDataGetNumData(pColData, ri);
bool val = *(const bool*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateBool(val));
break;
}
case TSDB_DATA_TYPE_TINYINT: {
int8_t val = *(int8_t*)colDataGetNumData(pColData, ri);
int8_t val = *(const int8_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
int16_t val = *(int16_t*)colDataGetNumData(pColData, ri);
int16_t val = *(const int16_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_INT: {
int32_t val = *(int32_t*)colDataGetNumData(pColData, ri);
int32_t val = *(const int32_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP: {
int64_t val = *(int64_t*)colDataGetNumData(pColData, ri);
int64_t val = *(const int64_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_FLOAT: {
float val = *(float*)colDataGetNumData(pColData, ri);
float val = *(const float*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
double val = *(double*)colDataGetNumData(pColData, ri);
double val = *(const double*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
@ -185,8 +275,8 @@ static int32_t jsonAddColumnField(const char* colName, const SColumnInfoData* pC
case TSDB_DATA_TYPE_NCHAR: {
// cJSON requires null-terminated strings, but this data is not null-terminated,
// so we need to manually copy the string and add null termination.
const char* src = varDataVal(colDataGetVarData(pColData, ri));
int32_t len = varDataLen(colDataGetVarData(pColData, ri));
const char* src = varDataVal(pData);
int32_t len = varDataLen(pData);
temp = cJSON_malloc(len + 1);
QUERY_CHECK_NULL(temp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
memcpy(temp, src, len);
@ -202,25 +292,25 @@ static int32_t jsonAddColumnField(const char* colName, const SColumnInfoData* pC
}
case TSDB_DATA_TYPE_UTINYINT: {
uint8_t val = *(uint8_t*)colDataGetNumData(pColData, ri);
uint8_t val = *(const uint8_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_USMALLINT: {
uint16_t val = *(uint16_t*)colDataGetNumData(pColData, ri);
uint16_t val = *(const uint16_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_UINT: {
uint32_t val = *(uint32_t*)colDataGetNumData(pColData, ri);
uint32_t val = *(const uint32_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
case TSDB_DATA_TYPE_UBIGINT: {
uint64_t val = *(uint64_t*)colDataGetNumData(pColData, ri);
uint64_t val = *(const uint64_t*)pData;
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
break;
}
@ -241,53 +331,91 @@ _end:
return code;
}
int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
const SSDataBlock* pInputBlock, const SNodeList* pCondCols, int32_t ri,
SStreamNotifyEventSupp* sup) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SNode* node = NULL;
cJSON* event = NULL;
cJSON* fields = NULL;
cJSON* cond = NULL;
SStreamNotifyEvent item = {0};
char windowId[32];
static cJSON* createBasicAggNotifyEvent(const char* windowType, EStreamNotifyEventType eventType,
const SSessionKey* pSessionKey) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
const char* eventTypeStr = NULL;
cJSON* event = NULL;
char windowId[32];
QUERY_CHECK_NULL(windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pSessionKey, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pInputBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pInputBlock->pDataBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pCondCols, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
qDebug("add stream notify event from event window, type: %s, start: %" PRId64 ", end: %" PRId64,
(eventType == SNOTIFY_EVENT_WINDOW_OPEN) ? "WINDOW_OPEN" : "WINDOW_CLOSE", pSessionKey->win.skey,
pSessionKey->win.ekey);
if (eventType == SNOTIFY_EVENT_WINDOW_OPEN) {
eventTypeStr = "WINDOW_OPEN";
} else if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
eventTypeStr = "WINDOW_CLOSE";
} else if (eventType == SNOTIFY_EVENT_WINDOW_INVALIDATION) {
eventTypeStr = "WINDOW_INVALIDATION";
} else {
QUERY_CHECK_CONDITION(false, code, lino, _end, TSDB_CODE_INVALID_PARA);
}
qDebug("add stream notify event from %s Window, type: %s, start: %" PRId64 ", end: %" PRId64, windowType,
eventTypeStr, pSessionKey->win.skey, pSessionKey->win.ekey);
event = cJSON_CreateObject();
QUERY_CHECK_NULL(event, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
// add basic info
streamNotifyGetEventWindowId(pSessionKey, windowId);
if (eventType == SNOTIFY_EVENT_WINDOW_OPEN) {
JSON_CHECK_ADD_ITEM(event, "eventType", cJSON_CreateStringReference("WINDOW_OPEN"));
} else if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
JSON_CHECK_ADD_ITEM(event, "eventType", cJSON_CreateStringReference("WINDOW_CLOSE"));
}
JSON_CHECK_ADD_ITEM(event, "eventType", cJSON_CreateStringReference(eventTypeStr));
JSON_CHECK_ADD_ITEM(event, "eventTime", cJSON_CreateNumber(taosGetTimestampMs()));
JSON_CHECK_ADD_ITEM(event, "windowId", cJSON_CreateStringReference(windowId));
JSON_CHECK_ADD_ITEM(event, "windowType", cJSON_CreateStringReference("Event"));
JSON_CHECK_ADD_ITEM(event, "windowId", cJSON_CreateString(windowId));
JSON_CHECK_ADD_ITEM(event, "windowType", cJSON_CreateStringReference(windowType));
JSON_CHECK_ADD_ITEM(event, "windowStart", cJSON_CreateNumber(pSessionKey->win.skey));
if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
JSON_CHECK_ADD_ITEM(event, "windowEnd", cJSON_CreateNumber(pSessionKey->win.ekey));
if (eventType != SNOTIFY_EVENT_WINDOW_OPEN) {
if (strcmp(windowType, "Time") == 0) {
JSON_CHECK_ADD_ITEM(event, "windowEnd", cJSON_CreateNumber(pSessionKey->win.ekey + 1));
} else {
JSON_CHECK_ADD_ITEM(event, "windowEnd", cJSON_CreateNumber(pSessionKey->win.ekey));
}
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
terrno = code;
cJSON_Delete(event);
event = NULL;
}
return event;
}
int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
const SSDataBlock* pInputBlock, const SNodeList* pCondCols, int32_t ri,
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
cJSON* event = NULL;
cJSON* fields = NULL;
cJSON* cond = NULL;
const SNode* pNode = NULL;
int32_t origSize = 0;
int64_t startTime = 0;
int64_t endTime = 0;
SStreamNotifyEvent item = {0};
QUERY_CHECK_NULL(pInputBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pInputBlock->pDataBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pCondCols, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(sup->windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
startTime = taosGetMonoTimestampMs();
event = createBasicAggNotifyEvent(sup->windowType, eventType, pSessionKey);
QUERY_CHECK_NULL(event, code, lino, _end, terrno);
// create fields object to store matched column values
fields = cJSON_CreateObject();
QUERY_CHECK_NULL(fields, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
FOREACH(node, pCondCols) {
SColumnNode* pColDef = (SColumnNode*)node;
SColumnInfoData* pColData = taosArrayGet(pInputBlock->pDataBlock, pColDef->slotId);
code = jsonAddColumnField(pColDef->colName, pColData, ri, fields);
FOREACH(pNode, pCondCols) {
const SColumnNode* pColDef = (const SColumnNode*)pNode;
const SColumnInfoData* pColData = taosArrayGet(pInputBlock->pDataBlock, pColDef->slotId);
code = jsonAddColumnField(pColDef->colName, pColData->info.type, colDataIsNull_s(pColData, ri),
colDataGetData(pColData, ri), fields);
QUERY_CHECK_CODE(code, lino, _end);
}
@ -297,16 +425,24 @@ int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionK
JSON_CHECK_ADD_ITEM(cond, "conditionIndex", cJSON_CreateNumber(0));
JSON_CHECK_ADD_ITEM(cond, "fieldValues", fields);
fields = NULL;
JSON_CHECK_ADD_ITEM(event, "triggerConditions", cond);
JSON_CHECK_ADD_ITEM(event, "triggerCondition", cond);
cond = NULL;
// convert json object to string value
item.gid = pSessionKey->groupId;
item.skey = pSessionKey->win.skey;
item.isEnd = (eventType == SNOTIFY_EVENT_WINDOW_CLOSE);
item.content = cJSON_PrintUnformatted(event);
QUERY_CHECK_NULL(taosArrayPush(sup->pWindowEvents, &item), code, lino, _end, terrno);
item.content = NULL;
item.win = pSessionKey->win;
item.eventType = eventType;
item.pJson = event;
event = NULL;
origSize = taosHashGetSize(sup->pWindowEventHashMap);
code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
QUERY_CHECK_CODE(code, lino, _end);
item.pJson = NULL;
endTime = taosGetMonoTimestampMs();
pNotifyEventStat->notifyEventAddTimes++;
pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
_end:
if (code != TSDB_CODE_SUCCESS) {
@ -325,50 +461,204 @@ _end:
return code;
}
int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SSchemaWrapper* pSchemaWrapper,
SStreamNotifyEventSupp* sup) {
int32_t addStateAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
const SStateKeys* pCurState, const SStateKeys* pAnotherState, bool onlyUpdate,
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SNode * node = NULL;
cJSON* event = NULL;
cJSON* result = NULL;
int32_t origSize = 0;
int64_t startTime = 0;
int64_t endTime = 0;
SStreamNotifyEvent item = {0};
QUERY_CHECK_NULL(pCurState, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(sup->windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
item.gid = pSessionKey->groupId;
item.win = pSessionKey->win;
item.eventType = eventType;
// Check if the notify event exists for update
if (onlyUpdate && taosHashGet(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE) == NULL) {
goto _end;
}
startTime = taosGetMonoTimestampMs();
event = createBasicAggNotifyEvent(sup->windowType, eventType, pSessionKey);
QUERY_CHECK_NULL(event, code, lino, _end, terrno);
// add state value
if (eventType == SNOTIFY_EVENT_WINDOW_OPEN) {
if (pAnotherState) {
code = jsonAddColumnField("prevState", pAnotherState->type, pAnotherState->isNull, pAnotherState->pData, event);
QUERY_CHECK_CODE(code, lino, _end);
} else {
code = jsonAddColumnField("prevState", pCurState->type, true, NULL, event);
QUERY_CHECK_CODE(code, lino, _end);
}
}
code = jsonAddColumnField("curState", pCurState->type, pCurState->isNull, pCurState->pData, event);
QUERY_CHECK_CODE(code, lino, _end);
if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
if (pAnotherState) {
code = jsonAddColumnField("nextState", pAnotherState->type, pAnotherState->isNull, pAnotherState->pData, event);
QUERY_CHECK_CODE(code, lino, _end);
} else {
code = jsonAddColumnField("nextState", pCurState->type, true, NULL, event);
QUERY_CHECK_CODE(code, lino, _end);
}
}
item.pJson = event;
event = NULL;
origSize = taosHashGetSize(sup->pWindowEventHashMap);
code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
QUERY_CHECK_CODE(code, lino, _end);
item.pJson = NULL;
endTime = taosGetMonoTimestampMs();
pNotifyEventStat->notifyEventAddTimes++;
pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
destroyStreamWindowEvent(&item);
if (event != NULL) {
cJSON_Delete(event);
}
return code;
}
static int32_t addNormalAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
cJSON* event = NULL;
int32_t origSize = 0;
int64_t startTime = 0;
int64_t endTime = 0;
SStreamNotifyEvent item = {0};
QUERY_CHECK_NULL(pSessionKey, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(sup->windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
startTime = taosGetMonoTimestampMs();
event = createBasicAggNotifyEvent(sup->windowType, eventType, pSessionKey);
QUERY_CHECK_NULL(event, code, lino, _end, terrno);
item.gid = pSessionKey->groupId;
item.win = pSessionKey->win;
item.eventType = eventType;
item.pJson = event;
event = NULL;
origSize = taosHashGetSize(sup->pWindowEventHashMap);
code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
QUERY_CHECK_CODE(code, lino, _end);
item.pJson = NULL;
endTime = taosGetMonoTimestampMs();
pNotifyEventStat->notifyEventAddTimes++;
pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
destroyStreamWindowEvent(&item);
if (event != NULL) {
cJSON_Delete(event);
}
return code;
}
int32_t addIntervalAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
return addNormalAggNotifyEvent(eventType, pSessionKey, sup, pNotifyEventStat);
}
int32_t addSessionAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
return addNormalAggNotifyEvent(eventType, pSessionKey, sup, pNotifyEventStat);
}
int32_t addCountAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
return addNormalAggNotifyEvent(eventType, pSessionKey, sup, pNotifyEventStat);
}
int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SArray* pSessionKeys,
const SSchemaWrapper* pSchemaWrapper, SStreamNotifyEventSupp* sup,
STaskNotifyEventStat* pNotifyEventStat) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
cJSON* result = NULL;
int32_t origSize = 0;
int64_t startTime = 0;
int64_t endTime = 0;
SStreamNotifyEvent item = {0};
SColumnInfoData* pWstartCol = NULL;
QUERY_CHECK_NULL(pResultBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pSessionKeys, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pSchemaWrapper, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
qDebug("add %" PRId64 " stream notify results from window agg", pResultBlock->info.rows);
startTime = taosGetMonoTimestampMs();
origSize = taosHashGetSize(sup->pWindowEventHashMap);
pWstartCol = taosArrayGet(pResultBlock->pDataBlock, 0);
for (int32_t i = 0; i< pResultBlock->info.rows; ++i) {
event = cJSON_CreateObject();
QUERY_CHECK_NULL(event, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
for (int32_t i = 0; i < pResultBlock->info.rows; ++i) {
const SSessionKey* pSessionKey = taosArrayGet(pSessionKeys, i);
item.gid = pSessionKey->groupId;
item.win = pSessionKey->win;
item.eventType = SNOTIFY_EVENT_WINDOW_CLOSE;
SStreamNotifyEvent* pItem = taosHashGet(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE);
if (pItem == NULL) {
item.pJson = createBasicAggNotifyEvent(sup->windowType, SNOTIFY_EVENT_WINDOW_CLOSE, pSessionKey);
QUERY_CHECK_NULL(item.pJson, code, lino, _end, terrno);
if (strcmp(sup->windowType, "Event") == 0) {
JSON_CHECK_ADD_ITEM(item.pJson, "triggerCondition", cJSON_CreateNull());
} else if (strcmp(sup->windowType, "State") == 0) {
JSON_CHECK_ADD_ITEM(item.pJson, "curState", cJSON_CreateNull());
JSON_CHECK_ADD_ITEM(item.pJson, "nextState", cJSON_CreateNull());
}
code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
QUERY_CHECK_CODE(code, lino, _end);
item.pJson = NULL;
pItem = taosHashGet(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE);
QUERY_CHECK_NULL(pItem, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
}
// convert the result row into json
result = cJSON_CreateObject();
QUERY_CHECK_NULL(result, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
for (int32_t j = 0; j < pSchemaWrapper->nCols; ++j) {
SSchema *pCol = pSchemaWrapper->pSchema + j;
SColumnInfoData *pColData = taosArrayGet(pResultBlock->pDataBlock, pCol->colId - 1);
code = jsonAddColumnField(pCol->name, pColData, i, result);
const SSchema* pCol = pSchemaWrapper->pSchema + j;
const SColumnInfoData* pColData = taosArrayGet(pResultBlock->pDataBlock, pCol->colId - 1);
code = jsonAddColumnField(pCol->name, pColData->info.type, colDataIsNull_s(pColData, i),
colDataGetData(pColData, i), result);
QUERY_CHECK_CODE(code, lino, _end);
}
JSON_CHECK_ADD_ITEM(event, "result", result);
JSON_CHECK_ADD_ITEM(pItem->pJson, "result", result);
result = NULL;
item.gid = pResultBlock->info.id.groupId;
item.skey = *(uint64_t*)colDataGetNumData(pWstartCol, i);
item.content = cJSON_PrintUnformatted(event);
code = taosHashPut(sup->pResultHashMap, &item.gid, sizeof(item.gid) + sizeof(item.skey), &item, sizeof(item));
TSDB_CHECK_CODE(code, lino, _end);
item.content = NULL;
cJSON_Delete(event);
event = NULL;
}
endTime = taosGetMonoTimestampMs();
pNotifyEventStat->notifyEventAddTimes++;
pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
@ -377,8 +667,38 @@ _end:
if (result != NULL) {
cJSON_Delete(result);
}
if (event != NULL) {
cJSON_Delete(event);
return code;
}
int32_t addAggDeleteNotifyEvent(const SSDataBlock* pDeleteBlock, SStreamNotifyEventSupp* sup,
STaskNotifyEventStat* pNotifyEventStat) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SSessionKey sessionKey = {0};
SColumnInfoData* pWstartCol = NULL;
SColumnInfoData* pWendCol = NULL;
SColumnInfoData* pGroupIdCol = NULL;
QUERY_CHECK_NULL(pDeleteBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
qDebug("add %" PRId64 " stream notify delete events from window agg", pDeleteBlock->info.rows);
pWstartCol = taosArrayGet(pDeleteBlock->pDataBlock, START_TS_COLUMN_INDEX);
pWendCol = taosArrayGet(pDeleteBlock->pDataBlock, END_TS_COLUMN_INDEX);
pGroupIdCol = taosArrayGet(pDeleteBlock->pDataBlock, GROUPID_COLUMN_INDEX);
for (int32_t i = 0; i < pDeleteBlock->info.rows; ++i) {
sessionKey.win.skey = *(int64_t*)colDataGetNumData(pWstartCol, i);
sessionKey.win.ekey = *(int64_t*)colDataGetNumData(pWendCol, i);
sessionKey.groupId = *(uint64_t*)colDataGetNumData(pGroupIdCol, i);
code = addNormalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_INVALIDATION, &sessionKey, sup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
@ -418,97 +738,42 @@ _end:
return code;
}
static int32_t streamNotifyFillTableName(const char* tableName, const SStreamNotifyEvent* pEvent,
const SStreamNotifyEvent* pResult, char** pVal) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
static const char* prefix = "{\"tableName\":\"";
uint64_t prefixLen = 0;
uint64_t nameLen = 0;
uint64_t eventLen = 0;
uint64_t resultLen = 0;
uint64_t valLen = 0;
char* val = NULL;
char* p = NULL;
QUERY_CHECK_NULL(tableName, code, lino, _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pEvent, code, lino , _end, TSDB_CODE_INVALID_PARA);
QUERY_CHECK_NULL(pVal, code, lino , _end, TSDB_CODE_INVALID_PARA);
*pVal = NULL;
prefixLen = strlen(prefix);
nameLen = strlen(tableName);
eventLen = strlen(pEvent->content);
if (pResult != NULL) {
resultLen = strlen(pResult->content);
valLen = VARSTR_HEADER_SIZE + prefixLen + nameLen + eventLen + resultLen;
} else {
valLen = VARSTR_HEADER_SIZE + prefixLen + nameLen + eventLen + 1;
}
val = taosMemoryMalloc(valLen);
QUERY_CHECK_NULL(val, code, lino, _end, terrno);
varDataSetLen(val, valLen - VARSTR_HEADER_SIZE);
p = varDataVal(val);
TAOS_STRNCPY(p, prefix, prefixLen);
p += prefixLen;
TAOS_STRNCPY(p, tableName, nameLen);
p += nameLen;
*(p++) = '\"';
TAOS_STRNCPY(p, pEvent->content, eventLen);
*p = ',';
if (pResult != NULL) {
p += eventLen - 1;
TAOS_STRNCPY(p, pResult->content, resultLen);
*p = ',';
}
*pVal = val;
val = NULL;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (val != NULL) {
taosMemoryFreeClear(val);
}
return code;
}
int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup) {
int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup,
STaskNotifyEventStat* pNotifyEventStat) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SColumnInfoData* pEventStrCol = NULL;
int32_t nWindowEvents = 0;
int32_t nWindowResults = 0;
char* val = NULL;
SColumnInfoData* pEventStrCol = NULL;
int64_t startTime = 0;
int64_t endTime = 0;
void* pIter = NULL;
if (pTaskInfo == NULL || sup == NULL) {
if (pTaskInfo == NULL || sup == NULL || sup->pEventBlock == NULL || pNotifyEventStat == NULL) {
goto _end;
}
QUERY_CHECK_NULL(sup->pEventBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
blockDataCleanup(sup->pEventBlock);
nWindowEvents = taosArrayGetSize(sup->pWindowEvents);
nWindowResults = taosHashGetSize(sup->pResultHashMap);
qDebug("start to build stream notify event block, nWindowEvents: %d, nWindowResults: %d", nWindowEvents,
nWindowResults);
if (nWindowEvents == 0) {
goto _end;
}
code = blockDataEnsureCapacity(sup->pEventBlock, nWindowEvents);
QUERY_CHECK_CODE(code, lino, _end);
startTime = taosGetMonoTimestampMs();
blockDataCleanup(sup->pEventBlock);
nWindowEvents = taosHashGetSize(sup->pWindowEventHashMap);
qDebug("start to build stream notify event block, nWindowEvents: %d", nWindowEvents);
pEventStrCol = taosArrayGet(sup->pEventBlock->pDataBlock, NOTIFY_EVENT_STR_COLUMN_INDEX);
QUERY_CHECK_NULL(pEventStrCol, code, lino, _end, terrno);
for (int32_t i = 0; i < nWindowEvents; ++i) {
SStreamNotifyEvent* pResult = NULL;
SStreamNotifyEvent* pEvent = taosArrayGet(sup->pWindowEvents, i);
char* tableName = taosHashGet(sup->pTableNameHashMap, &pEvent->gid, sizeof(pEvent->gid));
// Append all events content into data block.
pIter = taosHashIterate(sup->pWindowEventHashMap, NULL);
while (pIter) {
const SStreamNotifyEvent* pEvent = (const SStreamNotifyEvent*)pIter;
pIter = taosHashIterate(sup->pWindowEventHashMap, pIter);
if (pEvent->eventType == SNOTIFY_EVENT_WINDOW_CLOSE && !cJSON_HasObjectItem(pEvent->pJson, "result")) {
// current WINDOW_CLOSE event cannot be pushed yet due to watermark
continue;
}
// get name of the dest child table
char* tableName = taosHashGet(sup->pTableNameHashMap, &pEvent->gid, sizeof(&pEvent->gid));
if (tableName == NULL) {
code = streamNotifyGetDestTableName(pTaskInfo, pEvent->gid, &tableName);
QUERY_CHECK_CODE(code, lino, _end);
@ -518,32 +783,73 @@ int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEvent
tableName = taosHashGet(sup->pTableNameHashMap, &pEvent->gid, sizeof(pEvent->gid));
QUERY_CHECK_NULL(tableName, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
}
if (pEvent->isEnd) {
pResult = taosHashGet(sup->pResultHashMap, &pEvent->gid, sizeof(pEvent->gid) + sizeof(pEvent->skey));
QUERY_CHECK_NULL(pResult, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
}
code = streamNotifyFillTableName(tableName, pEvent, pResult, &val);
JSON_CHECK_ADD_ITEM(pEvent->pJson, "tableName", cJSON_CreateStringReference(tableName));
// convert the json object into string and append it into the block
char* str = cJSON_PrintUnformatted(pEvent->pJson);
QUERY_CHECK_NULL(str, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
int32_t len = strlen(str);
code = varColSetVarData(pEventStrCol, sup->pEventBlock->info.rows, str, len, false);
cJSON_free(str);
QUERY_CHECK_CODE(code, lino, _end);
code = colDataSetVal(pEventStrCol, i, val, false);
QUERY_CHECK_CODE(code, lino, _end);
taosMemoryFreeClear(val);
sup->pEventBlock->info.rows++;
code = taosHashRemove(sup->pWindowEventHashMap, pEvent, NOTIFY_EVENT_KEY_SIZE);
if (code == TSDB_CODE_NOT_FOUND) {
code = TSDB_CODE_SUCCESS;
}
QUERY_CHECK_CODE(code, lino, _end);
if (sup->pEventBlock->info.rows >= sup->pEventBlock->info.capacity) {
break;
}
}
if (taosHashGetMemSize(sup->pTableNameHashMap) >= NOTIFY_EVENT_NAME_CACHE_LIMIT_MB * 1024 * 1024) {
taosHashClear(sup->pTableNameHashMap);
}
endTime = taosGetMonoTimestampMs();
if (sup->pEventBlock->info.rows > 0) {
pNotifyEventStat->notifyEventPushTimes++;
pNotifyEventStat->notifyEventPushElems += sup->pEventBlock->info.rows;
pNotifyEventStat->notifyEventPushCostSec += (endTime - startTime) / 1000.0;
}
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (val != NULL) {
taosMemoryFreeClear(val);
}
if (sup != NULL) {
taosArrayClearEx(sup->pWindowEvents, destroyStreamWindowEvent);
taosHashClear(sup->pResultHashMap);
if (pIter) {
taosHashCancelIterate(sup->pWindowEventHashMap, pIter);
}
return code;
}
int32_t removeOutdatedNotifyEvents(STimeWindowAggSupp* pTwSup, SStreamNotifyEventSupp* sup,
STaskNotifyEventStat* pNotifyEventStat) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
void* pIter = NULL;
if (pTwSup || sup == NULL || pNotifyEventStat == NULL) {
goto _end;
}
pIter = taosHashIterate(sup->pWindowEventHashMap, NULL);
while (pIter) {
const SStreamNotifyEvent* pEvent = (const SStreamNotifyEvent*)pIter;
pIter = taosHashIterate(sup->pWindowEventHashMap, pIter);
if (isOverdue(pEvent->win.ekey, pTwSup)) {
code = taosHashRemove(sup->pWindowEventHashMap, pEvent, NOTIFY_EVENT_KEY_SIZE);
QUERY_CHECK_CODE(code, lino, _end);
}
}
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}

40
source/libs/executor/src/streamintervalsliceoperator.c
View File

@ -87,23 +87,48 @@ static int32_t buildIntervalSliceResult(SOperatorInfo* pOperator, SSDataBlock**
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
uint16_t opType = pOperator->operatorType;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
bool addNotifyEvent = false;
addNotifyEvent = IS_NORMAL_INTERVAL_OP(pOperator) &&
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
doBuildDeleteResultImpl(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pState, pInfo->pDelWins, &pInfo->delIndex,
pInfo->pDelRes);
if (pInfo->pDelRes->info.rows != 0) {
// process the rest of the data
printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
if (addNotifyEvent) {
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
(*ppRes) = pInfo->pDelRes;
return code;
}
doBuildStreamIntervalResult(pOperator, pInfo->streamAggSup.pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
doBuildStreamIntervalResult(pOperator, pInfo->streamAggSup.pState, pInfo->binfo.pRes, &pInfo->groupResInfo,
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
if (pInfo->binfo.pRes->info.rows != 0) {
printDataBlock(pInfo->binfo.pRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
if (addNotifyEvent) {
code = addAggResultNotifyEvent(pInfo->binfo.pRes, pNotifySup->pSessionKeys,
pTaskInfo->streamInfo.notifyResultSchema, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
(*ppRes) = pInfo->binfo.pRes;
goto _end;
}
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(opType), GET_TASKID(pTaskInfo));
(*ppRes) = pNotifySup->pEventBlock;
return code;
}
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
@ -316,6 +341,14 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc
code = setIntervalSliceOutputBuf(&curPoint, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator) &&
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
SSessionKey key = {.win = curWin, .groupId = groupId};
code = addIntervalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &key, &pInfo->basic.notifyEventSup,
pTaskInfo->streamInfo.pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
resetIntervalSliceFunctionKey(pSup->pCtx, numOfOutput);
if (pInfo->hasInterpoFunc && IS_VALID_WIN_KEY(prevPoint.winKey.win.skey) && curPoint.winKey.win.skey != curTs) {
doStreamSliceInterpolation(prevPoint.pLastRow, curPoint.winKey.win.skey, curTs, pBlock, startPos, &pOperator->exprSupp, INTERVAL_SLICE_START, pInfo->pOffsetInfo);
@ -652,8 +685,9 @@ int32_t createStreamIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiN
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamIntervalSliceReleaseState, streamIntervalSliceReloadState);
code = initStreamBasicInfo(&pInfo->basic);
code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
if (downstream) {
code = initIntervalSliceDownStream(downstream, &pInfo->streamAggSup, pPhyNode->type, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic, &pInfo->interval, pInfo->hasInterpoFunc);

3
source/libs/executor/src/streamtimesliceoperator.c
View File

@ -150,7 +150,6 @@ void destroyStreamTimeSliceOperatorInfo(void* param) {
&pInfo->groupResInfo);
pInfo->pOperator = NULL;
}
destroyStreamBasicInfo(&pInfo->basic);
colDataDestroy(&pInfo->twAggSup.timeWindowData);
destroyStreamAggSupporter(&pInfo->streamAggSup);
resetPrevAndNextWindow(pInfo->pFillSup);
@ -2202,7 +2201,7 @@ int32_t createStreamTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamTimeSliceReleaseState, streamTimeSliceReloadState);
code = initStreamBasicInfo(&pInfo->basic);
code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
if (downstream) {
code = initTimeSliceDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,

366
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -32,11 +32,6 @@
#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL)
#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
#define IS_NORMAL_SESSION_OP(op) \
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE)
#define DEAULT_DELETE_MARK INT64_MAX
#define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState"
@ -480,6 +475,8 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) {
false);
pInfo->pOperator = NULL;
}
destroyStreamBasicInfo(&pInfo->basic);
cleanupAggSup(&pInfo->aggSup);
clearGroupResInfo(&pInfo->groupResInfo);
taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);
@ -917,7 +914,7 @@ int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SSta
}
void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,
SGroupResInfo* pGroupResInfo) {
SGroupResInfo* pGroupResInfo, SArray* pSessionKeys) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -991,6 +988,14 @@ void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBl
}
}
if (pSessionKeys) {
SSessionKey key = {.groupId = groupId, .win = pRow->win};
for (int32_t j = 0; j < pRow->numOfRows; ++j) {
const void* px = taosArrayPush(pSessionKeys, &key);
QUERY_CHECK_NULL(px, code, lino, _end, terrno);
}
}
pBlock->info.rows += pRow->numOfRows;
}
@ -1005,19 +1010,20 @@ _end:
}
void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
SGroupResInfo* pGroupResInfo) {
SGroupResInfo* pGroupResInfo, SArray* pSessionKeys) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
// set output datablock version
pBlock->info.version = pTaskInfo->version;
blockDataCleanup(pBlock);
taosArrayClear(pSessionKeys);
if (!hasRemainResults(pGroupResInfo)) {
return;
}
// clear the existed group id
pBlock->info.id.groupId = 0;
buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo);
buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);
}
static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
@ -1150,6 +1156,14 @@ static int32_t doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS
pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->stateStore, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator) &&
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
SSessionKey key = {.win = nextWin, .groupId = groupId};
code = addIntervalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &key, &pInfo->basic.notifyEventSup,
pTaskInfo->streamInfo.pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
pResult = (SResultRow*)pResPos->pRowBuff;
if (IS_FINAL_INTERVAL_OP(pOperator)) {
@ -1371,7 +1385,10 @@ int32_t doStreamIntervalEncodeOpState(void** buf, int32_t len, SOperatorInfo* pO
// 5.dataVersion
tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);
// 6.checksum
// 6.basicInfo
tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
// 7.checksum
if (buf) {
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
tlen += taosEncodeFixedU32(buf, cksum);
@ -1387,18 +1404,20 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
int32_t lino = 0;
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
void* pDataEnd = POINTER_SHIFT(buf, len);
if (!pInfo) {
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
// 6.checksum
// 7.checksum
int32_t dataLen = len - sizeof(uint32_t);
void* pCksum = POINTER_SHIFT(buf, dataLen);
if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
pDataEnd = pCksum;
// 1.pResultRowHashTable
int32_t mapSize = 0;
@ -1454,6 +1473,12 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
// 5.dataVersion
buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);
// 6.basicInfo
if (buf < pDataEnd) {
code = decodeStreamBasicInfo(&buf, &pInfo->basic);
QUERY_CHECK_CODE(code, lino, _end);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
@ -1503,8 +1528,12 @@ _end:
static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
uint16_t opType = pOperator->operatorType;
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
bool addNotifyEvent = false;
// check if query task is closed or not
if (isTaskKilled(pTaskInfo)) {
@ -1512,6 +1541,8 @@ static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes
return code;
}
addNotifyEvent = IS_NORMAL_INTERVAL_OP(pOperator) &&
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
if (IS_FINAL_INTERVAL_OP(pOperator)) {
doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
if (pInfo->pPullDataRes->info.rows != 0) {
@ -1526,17 +1557,42 @@ static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes
if (pInfo->pDelRes->info.rows != 0) {
// process the rest of the data
printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
if (addNotifyEvent) {
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
(*ppRes) = pInfo->pDelRes;
return code;
}
doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo,
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
if (pInfo->binfo.pRes->info.rows != 0) {
printDataBlock(pInfo->binfo.pRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
if (addNotifyEvent) {
code = addAggResultNotifyEvent(pInfo->binfo.pRes, pNotifySup->pSessionKeys,
pTaskInfo->streamInfo.notifyResultSchema, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
(*ppRes) = pInfo->binfo.pRes;
return code;
}
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pNotifySup->pEventBlock;
return code;
}
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
(*ppRes) = NULL;
return code;
}
@ -2023,7 +2079,7 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN
pInfo->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;
pInfo->pState->pResultRowStore.resultRowPut = putResultRowToBuf;
pInfo->pState->pExprSupp = &pOperator->exprSupp;
code =
pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,
compareTs, pInfo->pState, pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo),
@ -2069,6 +2125,10 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
}
setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);
code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
pInfo->basic.primaryPkIndex = -1;
@ -2120,6 +2180,8 @@ void destroyStreamSessionAggOperatorInfo(void* param) {
&pInfo->groupResInfo);
pInfo->pOperator = NULL;
}
destroyStreamBasicInfo(&pInfo->basic);
destroyStreamAggSupporter(&pInfo->streamAggSup);
cleanupExprSupp(&pInfo->scalarSupp);
clearGroupResInfo(&pInfo->groupResInfo);
@ -2248,8 +2310,8 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, in
}
if (stateType == STREAM_STATE_BUFF_SORT) {
pSup->pState->pFileState = NULL;
code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize,
pSup->pState->pFileState = NULL;
code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize,
funResSize, sesionTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr,
pHandle->checkpointId, stateType, &pSup->pState->pFileState);
} else if (stateType == STREAM_STATE_BUFF_HASH_SORT || stateType == STREAM_STATE_BUFF_HASH_SEARCH) {
@ -2309,24 +2371,23 @@ bool inWinRange(STimeWindow* range, STimeWindow* cur) {
void clearOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { pAPI->streamStateClearBuff(pState, pPos); }
int32_t setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
SResultWindowInfo* pCurWin) {
SResultWindowInfo* pCurWin, int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
pCurWin->sessionWin.groupId = groupId;
pCurWin->sessionWin.win.skey = startTs;
pCurWin->sessionWin.win.ekey = endTs;
int32_t size = pAggSup->resultRowSize;
int32_t winCode = TSDB_CODE_SUCCESS;
code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap,
(void**)&pCurWin->pStatePos, &size, &winCode);
(void**)&pCurWin->pStatePos, &size, pWinCode);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) {
winCode = TSDB_CODE_FAILED;
if (*pWinCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) {
*pWinCode = TSDB_CODE_FAILED;
clearOutputBuf(pAggSup->pState, pCurWin->pStatePos, &pAggSup->pSessionAPI->stateStore);
}
if (winCode == TSDB_CODE_SUCCESS) {
if (*pWinCode == TSDB_CODE_SUCCESS) {
pCurWin->isOutput = true;
if (pCurWin->pStatePos->needFree) {
pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->sessionWin);
@ -2692,9 +2753,17 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
continue;
}
SResultWindowInfo winInfo = {0};
code = setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo);
int32_t winCode = TSDB_CODE_SUCCESS;
code = setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_SESSION_OP(pOperator) &&
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
code = addSessionAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &winInfo.sessionWin, &pInfo->basic.notifyEventSup,
pTaskInfo->streamInfo.pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
// coverity scan error
if (!winInfo.pStatePos) {
continue;
@ -2908,7 +2977,9 @@ static int32_t rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray,
if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pWinKey->win, &childWin.sessionWin.win)) {
if (num == 0) {
code = setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin);
int32_t winCode = TSDB_CODE_SUCCESS;
code = setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin,
&winCode);
QUERY_CHECK_CODE(code, lino, _end);
parentWin.sessionWin = childWin.sessionWin;
@ -3051,7 +3122,7 @@ void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayL
}
int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,
SGroupResInfo* pGroupResInfo) {
SGroupResInfo* pGroupResInfo, SArray* pSessionKeys) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -3131,6 +3202,13 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa
}
}
if (pSessionKeys) {
for (int32_t j = 0; j < pRow->numOfRows; ++j) {
const void* px = taosArrayPush(pSessionKeys, pKey);
QUERY_CHECK_NULL(px, code, lino, _end, terrno);
}
}
pBlock->info.dataLoad = 1;
pBlock->info.rows += pRow->numOfRows;
}
@ -3144,7 +3222,8 @@ _end:
return code;
}
void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock) {
void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock,
SArray* pSessionKeys) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -3152,6 +3231,7 @@ void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo*
pBlock->info.version = pTaskInfo->version;
blockDataCleanup(pBlock);
taosArrayClear(pSessionKeys);
if (!hasRemainResults(pGroupResInfo)) {
cleanupGroupResInfo(pGroupResInfo);
goto _end;
@ -3159,7 +3239,7 @@ void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo*
// clear the existed group id
pBlock->info.id.groupId = 0;
code = buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo);
code = buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);
QUERY_CHECK_CODE(code, lino, _end);
if (pBlock->info.rows == 0) {
@ -3174,23 +3254,60 @@ _end:
static int32_t buildSessionResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
bool addNotifyEvent = false;
addNotifyEvent = IS_NORMAL_SESSION_OP(pOperator) &&
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
if (addNotifyEvent) {
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
(*ppRes) = pInfo->pDelRes;
return code;
}
doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes,
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
if (pBInfo->pRes->info.rows > 0) {
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
if (addNotifyEvent) {
// Adjust the window end time based on the Session Window gap
for (int32_t i = 0; i < taosArrayGetSize(pNotifySup->pSessionKeys); ++i) {
SSessionKey* pKey = taosArrayGet(pNotifySup->pSessionKeys, i);
pKey->win.ekey += pAggSup->gap;
}
code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
(*ppRes) = pBInfo->pRes;
return code;
}
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pNotifySup->pEventBlock;
return code;
}
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
(*ppRes) = NULL;
return code;
}
@ -3295,7 +3412,10 @@ int32_t doStreamSessionEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOp
// 4.dataVersion
tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);
// 5.checksum
// 5.basicInfo
tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
// 6.checksum
if (isParent) {
if (buf) {
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
@ -3313,13 +3433,14 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe
int32_t lino = 0;
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
void* pDataEnd = POINTER_SHIFT(buf, len);
if (!pInfo) {
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
// 5.checksum
// 6.checksum
if (isParent) {
int32_t dataLen = len - sizeof(uint32_t);
void* pCksum = POINTER_SHIFT(buf, dataLen);
@ -3328,6 +3449,7 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
pDataEnd = pCksum;
}
// 1.streamAggSup.pResultRows
@ -3366,6 +3488,12 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe
(*ppBuf) = buf;
}
// 5.basicInfo
if (buf < pDataEnd) {
code = decodeStreamBasicInfo(&buf, &pInfo->basic);
QUERY_CHECK_CODE(code, lino, _end);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
@ -3926,6 +4054,9 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);
code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
if (downstream) {
pInfo->basic.primaryPkIndex = -1;
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
@ -4162,6 +4293,10 @@ int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhys
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionSemiReloadState);
}
code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), pPhyNode->type, false, OP_NOT_OPENED, pInfo,
pTaskInfo);
@ -4230,6 +4365,8 @@ void destroyStreamStateOperatorInfo(void* param) {
&pInfo->groupResInfo);
pInfo->pOperator = NULL;
}
destroyStreamBasicInfo(&pInfo->basic);
destroyStreamAggSupporter(&pInfo->streamAggSup);
clearGroupResInfo(&pInfo->groupResInfo);
taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);
@ -4284,12 +4421,39 @@ bool compareWinStateKey(SStateKeys* left, SStateKeys* right) {
return compareVal(left->pData, right);
}
static void getNextStateWin(const SStreamAggSupporter* pAggSup, SStateWindowInfo* pNextWin, bool asc) {
SStreamStateCur* pCur = NULL;
if (pAggSup == NULL || pNextWin == NULL) {
return;
}
if (asc)
pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);
else
pCur = pAggSup->stateStore.streamStateSessionSeekKeyPrev(pAggSup->pState, &pNextWin->winInfo.sessionWin);
int32_t nextSize = pAggSup->resultRowSize;
int32_t winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,
(void**)&pNextWin->winInfo.pStatePos, &nextSize);
if (winCode != TSDB_CODE_SUCCESS) {
SET_SESSION_WIN_INVALID(pNextWin->winInfo);
} else {
pNextWin->pStateKey =
(SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));
pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);
pNextWin->pStateKey->type = pAggSup->stateKeyType;
pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);
pNextWin->pStateKey->isNull = false;
pNextWin->winInfo.isOutput = true;
}
pAggSup->stateStore.streamStateFreeCur(pCur);
}
int32_t getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SStateWindowInfo* pCurWin,
SStateWindowInfo* pNextWin) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamStateCur* pCur = NULL;
int32_t size = pAggSup->resultRowSize;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t size = pAggSup->resultRowSize;
pCurWin->winInfo.sessionWin.groupId = pKey->groupId;
pCurWin->winInfo.sessionWin.win.skey = pKey->win.skey;
pCurWin->winInfo.sessionWin.win.ekey = pKey->win.ekey;
@ -4313,24 +4477,9 @@ int32_t getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey,
pCurWin->winInfo.sessionWin.win.ekey);
pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;
pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);
int32_t nextSize = pAggSup->resultRowSize;
int32_t winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,
(void**)&pNextWin->winInfo.pStatePos, &nextSize);
if (winCode != TSDB_CODE_SUCCESS) {
SET_SESSION_WIN_INVALID(pNextWin->winInfo);
} else {
pNextWin->pStateKey =
(SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));
pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);
pNextWin->pStateKey->type = pAggSup->stateKeyType;
pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);
pNextWin->pStateKey->isNull = false;
pNextWin->winInfo.isOutput = true;
}
getNextStateWin(pAggSup, pNextWin, true);
_end:
pAggSup->stateStore.streamStateFreeCur(pCur);
qDebug("===stream===get state next win buff. skey:%" PRId64 ", endkey:%" PRId64,
pNextWin->winInfo.sessionWin.win.skey, pNextWin->winInfo.sessionWin.win.ekey);
if (code != TSDB_CODE_SUCCESS) {
@ -4340,9 +4489,8 @@ _end:
}
int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,
SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin) {
int32_t size = pAggSup->resultRowSize;
SStreamStateCur* pCur = NULL;
SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin, int32_t* pWinCode) {
int32_t size = pAggSup->resultRowSize;
pCurWin->winInfo.sessionWin.groupId = groupId;
pCurWin->winInfo.sessionWin.win.skey = ts;
pCurWin->winInfo.sessionWin.win.ekey = ts;
@ -4390,29 +4538,16 @@ int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
}
}
*pWinCode = winCode;
qDebug("===stream===set state cur win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,
pCurWin->winInfo.sessionWin.win.ekey);
pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;
pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);
int32_t nextSize = pAggSup->resultRowSize;
winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,
(void**)&pNextWin->winInfo.pStatePos, &nextSize);
if (winCode != TSDB_CODE_SUCCESS) {
SET_SESSION_WIN_INVALID(pNextWin->winInfo);
} else {
pNextWin->pStateKey =
(SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));
pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);
pNextWin->pStateKey->type = pAggSup->stateKeyType;
pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);
pNextWin->pStateKey->isNull = false;
pNextWin->winInfo.isOutput = true;
}
getNextStateWin(pAggSup, pNextWin, true);
qDebug("===stream===set state next win buff. skey:%" PRId64 ", endkey:%" PRId64,
pNextWin->winInfo.sessionWin.win.skey, pNextWin->winInfo.sessionWin.win.ekey);
_end:
pAggSup->stateStore.streamStateFreeCur(pCur);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
@ -4472,7 +4607,7 @@ static bool isWinResult(SSessionKey* pKey, SSHashObj* pSeUpdate, SSHashObj* pRes
if (tSimpleHashGet(pSeUpdate, &checkKey, sizeof(SSessionKey)) != NULL) {
return true;
}
if (tSimpleHashGet(pResults, &checkKey, sizeof(SSessionKey)) != NULL) {
return true;
}
@ -4493,6 +4628,8 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
SResultRow* pResult = NULL;
int32_t winRows = 0;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
@ -4528,9 +4665,31 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
bool allEqual = true;
SStateWindowInfo curWin = {0};
SStateWindowInfo nextWin = {0};
code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin);
int32_t winCode = TSDB_CODE_SUCCESS;
code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS && pTaskInfo->streamInfo.eventTypes) {
SStateWindowInfo prevWin = {.winInfo.sessionWin = curWin.winInfo.sessionWin};
getNextStateWin(pAggSup, &prevWin, false);
qDebug("===stream===get state prev win buff. skey:%" PRId64 ", endkey:%" PRId64,
prevWin.winInfo.sessionWin.win.skey, prevWin.winInfo.sessionWin.win.ekey);
releaseOutputBuf(pAggSup->pState, prevWin.winInfo.pStatePos, &pAPI->stateStore);
// For ordered data, the previous window's closure did not record the corresponding state values, so they need to
// be added here.
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE) &&
IS_VALID_SESSION_WIN(prevWin.winInfo)) {
code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &prevWin.winInfo.sessionWin, prevWin.pStateKey,
curWin.pStateKey, true, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, curWin.pStateKey,
prevWin.pStateKey, false, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
}
if (isWinResult(&nextWin.winInfo.sessionWin, pSeUpdated, pAggSup->pResultRows) == false) {
releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore);
}
@ -4578,6 +4737,14 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
QUERY_CHECK_CODE(code, lino, _end);
}
// If this is a windown recalculation, add the corresponding state values here since the next window may not require
// recalculation.
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {
code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &curWin.winInfo.sessionWin, curWin.pStateKey,
nextWin.pStateKey, false, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
}
_end:
@ -4621,7 +4788,10 @@ int32_t doStreamStateEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
// 4.dataVersion
tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);
// 5.checksum
// 5.basicInfo
tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
// 6.checksum
if (isParent) {
if (buf) {
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
@ -4640,12 +4810,13 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
int32_t lino = 0;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
void* pDataEnd = POINTER_SHIFT(buf, len);
if (!pInfo) {
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
// 5.checksum
// 6.checksum
if (isParent) {
int32_t dataLen = len - sizeof(uint32_t);
void* pCksum = POINTER_SHIFT(buf, dataLen);
@ -4654,6 +4825,7 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
code = TSDB_CODE_FAILED;
QUERY_CHECK_CODE(code, lino, _end);
}
pDataEnd = pCksum;
}
// 1.streamAggSup.pResultRows
@ -4693,6 +4865,12 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
(*ppBuf) = buf;
}
// 5.basicInfo
if (buf < pDataEnd) {
code = decodeStreamBasicInfo(&buf, &pInfo->basic);
QUERY_CHECK_CODE(code, lino, _end);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
@ -4720,23 +4898,53 @@ void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) {
static int32_t buildStateResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
bool addNotifyEvent = false;
addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
if (addNotifyEvent) {
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
(*ppRes) = pInfo->pDelRes;
return code;
}
doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes);
doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes,
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
if (pBInfo->pRes->info.rows > 0) {
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
if (addNotifyEvent) {
code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
}
(*ppRes) = pBInfo->pRes;
return code;
}
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pNotifySup->pEventBlock;
return code;
}
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
}
(*ppRes) = NULL;
return code;
}
@ -5122,6 +5330,10 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAggNext, NULL, destroyStreamStateOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);
code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
QUERY_CHECK_CODE(code, lino, _error);
@ -5331,8 +5543,9 @@ _end:
return code;
}
static int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,
SReadHandle* pHandle, SOperatorInfo** pOptrInfo) {
static int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
SOperatorInfo** pOptrInfo) {
QRY_PARAM_CHECK(pOptrInfo);
int32_t code = TSDB_CODE_SUCCESS;
@ -5455,6 +5668,9 @@ static int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);
code = initStreamBasicInfo(&pInfo->basic, pOperator);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->recvGetAll = false;
code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

2
source/libs/parser/src/parTranslater.c
View File

@ -11420,7 +11420,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
!hasTbnameFunction(pSelect->pPartitionByList) && pSelect->pWindow != NULL &&
pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"Event window for stream on super table must patitioned by table name");
"Event window for stream on super table must partitioned by table name");
}
if (pSelect->pWindow != NULL && pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW &&

36
source/libs/stream/src/streamSessionState.c
View File

@ -562,6 +562,33 @@ SStreamStateCur* sessionWinStateSeekKeyCurrentPrev(SStreamFileState* pFileState,
pCur->pStreamFileState = pFileState;
return pCur;
}
SStreamStateCur* sessionWinStateSeekKeyPrev(SStreamFileState *pFileState, const SSessionKey *pWinKey) {
SArray* pWinStates = NULL;
int32_t index = -1;
SStreamStateCur *pCur = seekKeyCurrentPrev_buff(pFileState, pWinKey, &pWinStates, &index);
if (pCur) {
int32_t cmpRes= sessionStateRangeKeyCompare(pWinKey, pWinStates, index);
if (cmpRes > 0) {
return pCur;
} else if (cmpRes == 0 && index > 0) {
sessionWinStateMoveToPrev(pCur);
return pCur;
}
streamStateFreeCur(pCur);
pCur = NULL;
}
void* pFileStore = getStateFileStore(pFileState);
pCur = streamStateSessionSeekKeyPrev_rocksdb(pFileStore, pWinKey);
if (!pCur) {
return NULL;
}
pCur->buffIndex = -1;
pCur->pStreamFileState = pFileState;
return pCur;
}
static void transformCursor(SStreamFileState* pFileState, SStreamStateCur* pCur) {
if (!pCur) {
return;
@ -747,6 +774,15 @@ void sessionWinStateMoveToNext(SStreamStateCur* pCur) {
}
}
void sessionWinStateMoveToPrev(SStreamStateCur* pCur) {
qTrace("move cursor to prev");
if (pCur && pCur->buffIndex >= 1) {
pCur->buffIndex--;
} else {
streamStateCurPrev_rocksdb(pCur);
}
}
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey,
range_cmpr_fn cmpFn) {
SStreamStateCur* pCur = sessionWinStateSeekKeyCurrentPrev(pFileState, key);

4
source/libs/stream/src/streamState.c
View File

@ -440,6 +440,10 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, cons
return sessionWinStateSeekKeyCurrentNext(pState->pFileState, key);
}
SStreamStateCur *streamStateSessionSeekKeyPrev(SStreamState *pState, const SSessionKey *key) {
return sessionWinStateSeekKeyPrev(pState->pFileState, key);
}
SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key) {
return sessionWinStateSeekKeyNext(pState->pFileState, key);
}

4
source/libs/stream/src/streamTask.c
View File

@ -331,6 +331,8 @@ void tFreeStreamTask(void* pParam) {
taosMemoryFreeClear(pTask->notifyInfo.stbFullName);
tDeleteSchemaWrapper(pTask->notifyInfo.pSchemaWrapper);
pTask->notifyEventStat = (STaskNotifyEventStat){0};
taosMemoryFree(pTask);
stDebug("s-task:0x%x free task completed", taskId);
}
@ -988,6 +990,7 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc)
pDst->startTime = pSrc->startTime;
pDst->hTaskId = pSrc->hTaskId;
pDst->notifyEventStat = pSrc->notifyEventStat;
}
STaskStatusEntry streamTaskGetStatusEntry(SStreamTask* pTask) {
@ -1016,6 +1019,7 @@ STaskStatusEntry streamTaskGetStatusEntry(SStreamTask* pTask) {
.outputThroughput = SIZE_IN_KiB(pExecInfo->outputThroughput),
.startCheckpointId = pExecInfo->startCheckpointId,
.startCheckpointVer = pExecInfo->startCheckpointVer,
.notifyEventStat = pTask->notifyEventStat,
};
return entry;
}