feat(stream)[TS-5469]: add more stream event notifications
- Add event notifications for all window types - Implement window invalidation notifications - Add statistics for event notification delivery
This commit is contained in:
parent
c0aa91ff1e
commit
fc2d2672b9
|
@ -195,6 +195,8 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const
|
|||
// For the VAR_DATA_TYPE type, if a row already has data before inserting it (judged by offset != -1),
|
||||
// it will be inserted at the original position and the old data will be overwritten.
|
||||
int32_t colDataSetValOrCover(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull);
|
||||
int32_t varColSetVarData(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pVarData, int32_t varDataLen,
|
||||
bool isNull);
|
||||
int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData);
|
||||
int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows,
|
||||
bool trimValue);
|
||||
|
|
|
@ -2966,6 +2966,22 @@ typedef struct {
|
|||
int8_t notifyHistory;
|
||||
} SCMCreateStreamReq;
|
||||
|
||||
typedef struct STaskNotifyEventStat {
|
||||
int64_t notifyEventAddTimes; // call times of add function
|
||||
int64_t notifyEventAddElems; // elements added by add function
|
||||
double notifyEventAddCostSec; // time cost of add function
|
||||
int64_t notifyEventPushTimes; // call times of push function
|
||||
int64_t notifyEventPushElems; // elements pushed by push function
|
||||
double notifyEventPushCostSec; // time cost of push function
|
||||
int64_t notifyEventPackTimes; // call times of pack function
|
||||
int64_t notifyEventPackElems; // elements packed by pack function
|
||||
double notifyEventPackCostSec; // time cost of pack function
|
||||
int64_t notifyEventSendTimes; // call times of send function
|
||||
int64_t notifyEventSendElems; // elements sent by send function
|
||||
double notifyEventSendCostSec; // time cost of send function
|
||||
int64_t notifyEventHoldElems; // elements hold due to watermark
|
||||
} STaskNotifyEventStat;
|
||||
|
||||
typedef struct {
|
||||
int64_t streamId;
|
||||
} SCMCreateStreamRsp;
|
||||
|
|
|
@ -99,7 +99,7 @@ int32_t qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId);
|
|||
int32_t qSetStreamOpOpen(qTaskInfo_t tinfo);
|
||||
|
||||
int32_t qSetStreamNotifyInfo(qTaskInfo_t tinfo, int32_t eventTypes, const SSchemaWrapper* pSchemaWrapper,
|
||||
const char* stbFullName, bool newSubTableRule);
|
||||
const char* stbFullName, bool newSubTableRule, STaskNotifyEventStat* pNotifyEventStat);
|
||||
|
||||
/**
|
||||
* Set multiple input data blocks for the stream scan.
|
||||
|
|
|
@ -431,6 +431,7 @@ typedef struct SStateStore {
|
|||
int32_t (*updateInfoSerialize)(SEncoder* pEncoder, const SUpdateInfo* pInfo);
|
||||
int32_t (*updateInfoDeserialize)(SDecoder* pDeCoder, SUpdateInfo* pInfo);
|
||||
|
||||
SStreamStateCur* (*streamStateSessionSeekKeyPrev)(SStreamState* pState, const SSessionKey* key);
|
||||
SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key);
|
||||
SStreamStateCur* (*streamStateCountSeekKeyPrev)(SStreamState* pState, const SSessionKey* pKey, COUNT_TYPE count);
|
||||
SStreamStateCur* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key);
|
||||
|
|
|
@ -573,6 +573,7 @@ typedef enum EStreamNotifyOptionSetFlag {
|
|||
} EStreamNotifyOptionSetFlag;
|
||||
|
||||
typedef enum EStreamNotifyEventType {
|
||||
SNOTIFY_EVENT_WINDOW_INVALIDATION = 0,
|
||||
SNOTIFY_EVENT_WINDOW_OPEN = BIT_FLAG_MASK(0),
|
||||
SNOTIFY_EVENT_WINDOW_CLOSE = BIT_FLAG_MASK(1),
|
||||
} EStreamNotifyEventType;
|
||||
|
|
|
@ -65,6 +65,7 @@ int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* r
|
|||
int32_t streamStateSessionAllocWinBuffByNextPosition(SStreamState* pState, SStreamStateCur* pCur,
|
||||
const SSessionKey* pKey, void** pVal, int32_t* pVLen);
|
||||
|
||||
SStreamStateCur *streamStateSessionSeekKeyPrev(SStreamState *pState, const SSessionKey *key);
|
||||
SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key);
|
||||
SStreamStateCur* streamStateCountSeekKeyPrev(SStreamState* pState, const SSessionKey* pKey, COUNT_TYPE count);
|
||||
SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key);
|
||||
|
@ -162,4 +163,4 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2);
|
|||
}
|
||||
#endif
|
||||
|
||||
#endif /* ifndef _STREAM_STATE_H_ */
|
||||
#endif /* ifndef _STREAM_STATE_H_ */
|
||||
|
|
|
@ -463,6 +463,7 @@ struct SStreamTask {
|
|||
SUpstreamInfo upstreamInfo;
|
||||
STaskCheckInfo taskCheckInfo;
|
||||
SNotifyInfo notifyInfo;
|
||||
STaskNotifyEventStat notifyEventStat;
|
||||
|
||||
// the followings attributes don't be serialized
|
||||
SScanhistorySchedInfo schedHistoryInfo;
|
||||
|
@ -632,6 +633,7 @@ typedef struct STaskStatusEntry {
|
|||
int64_t startCheckpointVer;
|
||||
int64_t hTaskId;
|
||||
STaskCkptInfo checkpointInfo;
|
||||
STaskNotifyEventStat notifyEventStat;
|
||||
} STaskStatusEntry;
|
||||
|
||||
//typedef struct SNodeUpdateInfo {
|
||||
|
|
|
@ -100,10 +100,12 @@ void sessionWinStateCleanup(void* pBuff);
|
|||
SStreamStateCur* createStateCursor(SStreamFileState* pFileState);
|
||||
SStreamStateCur* sessionWinStateSeekKeyCurrentPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey);
|
||||
SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
|
||||
SStreamStateCur* sessionWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey);
|
||||
SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
|
||||
SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey, COUNT_TYPE count);
|
||||
int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
|
||||
void sessionWinStateMoveToNext(SStreamStateCur* pCur);
|
||||
void sessionWinStateMoveToPrev(SStreamStateCur* pCur);
|
||||
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey,
|
||||
range_cmpr_fn cmpFn);
|
||||
|
||||
|
|
|
@ -246,6 +246,7 @@ typedef enum ELogicConditionType {
|
|||
#define TSDB_USER_CGROUP_LEN (TSDB_USER_LEN + TSDB_CGROUP_LEN) // it is a null-terminated string
|
||||
#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string
|
||||
#define TSDB_STREAM_NOTIFY_URL_LEN 128 // it includes the terminating '\0'
|
||||
#define TSDB_STREAM_NOTIFY_STAT_LEN 350 // it includes the terminating '\0'
|
||||
#define TSDB_DB_NAME_LEN 65
|
||||
#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
|
||||
#define TSDB_PRIVILEDGE_CONDITION_LEN 48 * 1024
|
||||
|
|
|
@ -56,6 +56,7 @@ typedef struct STaskStatusEntry {
|
|||
int64_t startCheckpointVer;
|
||||
int64_t hTaskId;
|
||||
STaskCkptInfo checkpointInfo;
|
||||
STaskNotifyEventStat notifyEventStat;
|
||||
} STaskStatusEntry;
|
||||
|
||||
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo) {
|
||||
|
@ -523,6 +524,19 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) {
|
|||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->startCheckpointId));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->startCheckpointVer));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->hTaskId));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventAddTimes));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventAddElems));
|
||||
TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventAddCostSec));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPushTimes));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPushElems));
|
||||
TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventPushCostSec));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPackTimes));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventPackElems));
|
||||
TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventPackCostSec));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventSendTimes));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventSendElems));
|
||||
TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->notifyEventStat.notifyEventSendCostSec));
|
||||
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->notifyEventStat.notifyEventHoldElems));
|
||||
}
|
||||
|
||||
int32_t numOfVgs = taosArrayGetSize(pReq->pUpdateNodes);
|
||||
|
@ -596,6 +610,20 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) {
|
|||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.startCheckpointVer));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.hTaskId));
|
||||
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventAddTimes));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventAddElems));
|
||||
TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventAddCostSec));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPushTimes));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPushElems));
|
||||
TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventPushCostSec));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPackTimes));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventPackElems));
|
||||
TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventPackCostSec));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventSendTimes));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventSendElems));
|
||||
TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.notifyEventStat.notifyEventSendCostSec));
|
||||
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.notifyEventStat.notifyEventHoldElems));
|
||||
|
||||
entry.id.taskId = taskId;
|
||||
if (taosArrayPush(pReq->pTaskStatus, &entry) == NULL) {
|
||||
TAOS_CHECK_EXIT(terrno);
|
||||
|
@ -837,4 +865,4 @@ int32_t tDecodeStreamTaskRunReq(SDecoder* pDecoder, SStreamTaskRunReq* pReq) {
|
|||
|
||||
_exit:
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -212,6 +212,7 @@ static const SSysDbTableSchema streamTaskSchema[] = {
|
|||
{.name = "extra_info", .bytes = 25 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
|
||||
{.name = "history_task_id", .bytes = 16 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
|
||||
{.name = "history_task_status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
|
||||
{.name = "notify_event_stat", .bytes = TSDB_STREAM_NOTIFY_STAT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
|
||||
};
|
||||
|
||||
static const SSysDbTableSchema userTblsSchema[] = {
|
||||
|
|
|
@ -165,6 +165,55 @@ int32_t colDataSetValOrCover(SColumnInfoData* pColumnInfoData, uint32_t rowIndex
|
|||
return colDataSetValHelp(pColumnInfoData, rowIndex, pData, isNull);
|
||||
}
|
||||
|
||||
int32_t varColSetVarData(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pVarData, int32_t varDataLen,
|
||||
bool isNull) {
|
||||
if (!IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
}
|
||||
|
||||
if (isNull || pVarData == NULL) {
|
||||
pColumnInfoData->varmeta.offset[rowIndex] = -1; // it is a null value of VAR type.
|
||||
pColumnInfoData->hasNull = true;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t dataLen = VARSTR_HEADER_SIZE + varDataLen;
|
||||
if (pColumnInfoData->varmeta.offset[rowIndex] > 0) {
|
||||
pColumnInfoData->varmeta.length = pColumnInfoData->varmeta.offset[rowIndex];
|
||||
}
|
||||
|
||||
SVarColAttr* pAttr = &pColumnInfoData->varmeta;
|
||||
if (pAttr->allocLen < pAttr->length + dataLen) {
|
||||
uint32_t newSize = pAttr->allocLen;
|
||||
if (newSize <= 1) {
|
||||
newSize = 8;
|
||||
}
|
||||
|
||||
while (newSize < pAttr->length + dataLen) {
|
||||
newSize = newSize * 1.5;
|
||||
if (newSize > UINT32_MAX) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
}
|
||||
|
||||
char* buf = taosMemoryRealloc(pColumnInfoData->pData, newSize);
|
||||
if (buf == NULL) {
|
||||
return terrno;
|
||||
}
|
||||
|
||||
pColumnInfoData->pData = buf;
|
||||
pAttr->allocLen = newSize;
|
||||
}
|
||||
|
||||
uint32_t len = pColumnInfoData->varmeta.length;
|
||||
pColumnInfoData->varmeta.offset[rowIndex] = len;
|
||||
|
||||
(void)memmove(varDataVal(pColumnInfoData->pData + len), pVarData, varDataLen);
|
||||
varDataSetLen(pColumnInfoData->pData + len, varDataLen);
|
||||
pColumnInfoData->varmeta.length += dataLen;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx,
|
||||
const char* pData) {
|
||||
int32_t type = pColumnInfoData->info.type;
|
||||
|
|
|
@ -765,7 +765,7 @@ static int32_t addStreamTaskNotifyInfo(const SCMCreateStreamReq *createReq, cons
|
|||
TSDB_CHECK_NULL(pTask->notifyInfo.pNotifyAddrUrls, code, lino, _end, terrno);
|
||||
pTask->notifyInfo.notifyEventTypes = createReq->notifyEventTypes;
|
||||
pTask->notifyInfo.notifyErrorHandle = createReq->notifyErrorHandle;
|
||||
pTask->notifyInfo.streamName = taosStrdup(createReq->name);
|
||||
pTask->notifyInfo.streamName = taosStrdup(mndGetDbStr(createReq->name));
|
||||
TSDB_CHECK_NULL(pTask->notifyInfo.streamName, code, lino, _end, terrno);
|
||||
pTask->notifyInfo.stbFullName = taosStrdup(createReq->targetStbFullName);
|
||||
TSDB_CHECK_NULL(pTask->notifyInfo.stbFullName, code, lino, _end, terrno);
|
||||
|
|
|
@ -1309,8 +1309,8 @@ int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDataBlo
|
|||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
|
||||
// input queue
|
||||
char vbuf[40] = {0};
|
||||
char buf[38] = {0};
|
||||
char vbuf[TSDB_STREAM_NOTIFY_STAT_LEN + 2] = {0};
|
||||
char buf[TSDB_STREAM_NOTIFY_STAT_LEN] = {0};
|
||||
const char *queueInfoStr = "%4.2f MiB (%6.2f%)";
|
||||
snprintf(buf, tListLen(buf), queueInfoStr, pe->inputQUsed, pe->inputRate);
|
||||
STR_TO_VARSTR(vbuf, buf);
|
||||
|
@ -1503,6 +1503,47 @@ int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDataBlo
|
|||
code = colDataSetVal(pColInfo, numOfRows, 0, true);
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
|
||||
// notify_event_stat
|
||||
int32_t offset =0;
|
||||
if (pe->notifyEventStat.notifyEventAddTimes > 0) {
|
||||
offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Add %" PRId64 "x, %" PRId64 " elems in %lfs; ",
|
||||
pe->notifyEventStat.notifyEventAddTimes, pe->notifyEventStat.notifyEventAddElems,
|
||||
pe->notifyEventStat.notifyEventAddCostSec);
|
||||
}
|
||||
if (pe->notifyEventStat.notifyEventPushTimes > 0) {
|
||||
offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Push %" PRId64 "x, %" PRId64 " elems in %lfs; ",
|
||||
pe->notifyEventStat.notifyEventPushTimes, pe->notifyEventStat.notifyEventPushElems,
|
||||
pe->notifyEventStat.notifyEventPushCostSec);
|
||||
}
|
||||
if (pe->notifyEventStat.notifyEventPackTimes > 0) {
|
||||
offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Pack %" PRId64 "x, %" PRId64 " elems in %lfs; ",
|
||||
pe->notifyEventStat.notifyEventPackTimes, pe->notifyEventStat.notifyEventPackElems,
|
||||
pe->notifyEventStat.notifyEventPackCostSec);
|
||||
}
|
||||
if (pe->notifyEventStat.notifyEventSendTimes > 0) {
|
||||
offset += tsnprintf(buf + offset, sizeof(buf) - offset, "Send %" PRId64 "x, %" PRId64 " elems in %lfs; ",
|
||||
pe->notifyEventStat.notifyEventSendTimes, pe->notifyEventStat.notifyEventSendElems,
|
||||
pe->notifyEventStat.notifyEventSendCostSec);
|
||||
}
|
||||
if (pe->notifyEventStat.notifyEventHoldElems > 0) {
|
||||
offset += tsnprintf(buf + offset, sizeof(buf) - offset, "[Hold %" PRId64 " elems] ",
|
||||
pe->notifyEventStat.notifyEventHoldElems);
|
||||
}
|
||||
TSDB_CHECK_CONDITION(offset < sizeof(buf), code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
|
||||
buf[offset] = '\0';
|
||||
|
||||
STR_TO_VARSTR(vbuf, buf);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
TSDB_CHECK_NULL(pColInfo, code, lino, _end, terrno);
|
||||
|
||||
if (offset == 0) {
|
||||
colDataSetNULL(pColInfo, numOfRows);
|
||||
} else {
|
||||
code = colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code) {
|
||||
mError("error happens during build task attr result blocks, lino:%d, code:%s", lino, tstrerror(code));
|
||||
|
@ -1689,4 +1730,4 @@ int32_t mndCheckForSnode(SMnode *pMnode, SDbObj *pSrcDb) {
|
|||
mError("snode not existed when trying to create stream in db with multiple replica");
|
||||
return TSDB_CODE_SNODE_NOT_DEPLOYED;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -97,6 +97,7 @@ void initStateStoreAPI(SStateStore* pStore) {
|
|||
pStore->updateInfoSerialize = updateInfoSerialize;
|
||||
pStore->updateInfoDeserialize = updateInfoDeserialize;
|
||||
|
||||
pStore->streamStateSessionSeekKeyPrev = streamStateSessionSeekKeyPrev;
|
||||
pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext;
|
||||
pStore->streamStateCountSeekKeyPrev = streamStateCountSeekKeyPrev;
|
||||
pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev;
|
||||
|
@ -123,4 +124,4 @@ void initStateStoreAPI(SStateStore* pStore) {
|
|||
void initFunctionStateStore(SFunctionStateStore* pStore) {
|
||||
pStore->streamStateFuncPut = streamStateFuncPut;
|
||||
pStore->streamStateFuncGet = streamStateFuncGet;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -20,14 +20,16 @@
|
|||
#include "curl/curl.h"
|
||||
#endif
|
||||
|
||||
#define STREAM_EVENT_NOTIFY_RETRY_MS 50 // 50ms
|
||||
#define STREAM_EVENT_NOTIFY_RETRY_MS 50 // 50 ms
|
||||
#define STREAM_EVENT_NOTIFY_MESSAAGE_SIZE_KB 8 * 1024 // 8 MB
|
||||
#define STREAM_EVENT_NOTIFY_FRAME_SIZE 256 * 1024 // 256 KB
|
||||
|
||||
typedef struct SStreamNotifyHandle {
|
||||
TdThreadMutex mutex;
|
||||
#ifndef WINDOWS
|
||||
CURL* curl;
|
||||
CURL* curl;
|
||||
#endif
|
||||
char* url;
|
||||
char* url;
|
||||
} SStreamNotifyHandle;
|
||||
|
||||
struct SStreamNotifyHandleMap {
|
||||
|
@ -49,6 +51,7 @@ static void stopStreamNotifyConn(SStreamNotifyHandle* pHandle) {
|
|||
}
|
||||
// TODO: add wait mechanism for peer connection close response
|
||||
curl_easy_cleanup(pHandle->curl);
|
||||
pHandle->curl = NULL;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
@ -258,7 +261,8 @@ _end:
|
|||
}
|
||||
|
||||
static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBlocks, char** pMsg,
|
||||
int32_t* nNotifyEvents) {
|
||||
int32_t* nNotifyEvents, STaskNotifyEventStat* pNotifyEventStat,
|
||||
int32_t* pBlockIdx) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
int32_t numOfBlocks = 0;
|
||||
|
@ -268,15 +272,20 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
|
|||
char* msgHeader = NULL;
|
||||
const char* msgTail = "]}]}";
|
||||
char* msg = NULL;
|
||||
int64_t startTime = 0;
|
||||
int64_t endTime = 0;
|
||||
int32_t nBlocks = 0;
|
||||
|
||||
TSDB_CHECK_NULL(pMsg, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
TSDB_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
*pMsg = NULL;
|
||||
numOfBlocks = taosArrayGetSize(pBlocks);
|
||||
*nNotifyEvents = 0;
|
||||
|
||||
for (int32_t i = 0; i < numOfBlocks; ++i) {
|
||||
for (int32_t i = *pBlockIdx; i < numOfBlocks; ++i) {
|
||||
SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
|
||||
nBlocks++;
|
||||
if (pDataBlock == NULL || pDataBlock->info.type != STREAM_NOTIFY_EVENT) {
|
||||
continue;
|
||||
}
|
||||
|
@ -287,13 +296,19 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
|
|||
msgLen += varDataLen(val) + 1;
|
||||
}
|
||||
*nNotifyEvents += pDataBlock->info.rows;
|
||||
if (msgLen >= STREAM_EVENT_NOTIFY_MESSAAGE_SIZE_KB * 1024) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
*pBlockIdx += nBlocks;
|
||||
|
||||
if (msgLen == 0) {
|
||||
// skip since no notification events found
|
||||
goto _end;
|
||||
}
|
||||
|
||||
startTime = taosGetMonoTimestampMs();
|
||||
code = getStreamNotifyEventHeader(streamName, &msgHeader);
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
msgHeaderLen = strlen(msgHeader);
|
||||
|
@ -306,7 +321,7 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
|
|||
TAOS_STRNCPY(p, msgHeader, msgHeaderLen);
|
||||
p += msgHeaderLen - msgTailLen;
|
||||
|
||||
for (int32_t i = 0; i < numOfBlocks; ++i) {
|
||||
for (int32_t i = *pBlockIdx - nBlocks; i < *pBlockIdx; ++i) {
|
||||
SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
|
||||
if (pDataBlock == NULL || pDataBlock->info.type != STREAM_NOTIFY_EVENT) {
|
||||
continue;
|
||||
|
@ -328,6 +343,11 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl
|
|||
*pMsg = msg;
|
||||
msg = NULL;
|
||||
|
||||
endTime = taosGetMonoTimestampMs();
|
||||
pNotifyEventStat->notifyEventPackTimes++;
|
||||
pNotifyEventStat->notifyEventPackElems += *nNotifyEvents;
|
||||
pNotifyEventStat->notifyEventPackCostSec += (endTime - startTime) / 1000.0;
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tqError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
|
@ -354,9 +374,21 @@ static int32_t sendSingleStreamNotify(SStreamNotifyHandle* pHandle, char* msg) {
|
|||
TSDB_CHECK_NULL(pHandle->curl, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
totalLen = strlen(msg);
|
||||
while (sentLen < totalLen) {
|
||||
res = curl_ws_send(pHandle->curl, msg + sentLen, totalLen - sentLen, &nbytes, 0, CURLWS_TEXT);
|
||||
if (totalLen > 0) {
|
||||
// send PING frame to check if the connection is still alive
|
||||
res = curl_ws_send(pHandle->curl, "", 0, (size_t*)&sentLen, 0, CURLWS_PING);
|
||||
TSDB_CHECK_CONDITION(res == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
|
||||
}
|
||||
sentLen = 0;
|
||||
while (sentLen < totalLen) {
|
||||
size_t chunkSize = TMIN(totalLen - sentLen, STREAM_EVENT_NOTIFY_FRAME_SIZE);
|
||||
if (sentLen == 0) {
|
||||
res = curl_ws_send(pHandle->curl, msg, chunkSize, &nbytes, totalLen, CURLWS_TEXT | CURLWS_OFFSET);
|
||||
TSDB_CHECK_CONDITION(res == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
|
||||
} else {
|
||||
res = curl_ws_send(pHandle->curl, msg + sentLen, chunkSize, &nbytes, 0, CURLWS_TEXT | CURLWS_OFFSET);
|
||||
TSDB_CHECK_CONDITION(res == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED);
|
||||
}
|
||||
sentLen += nbytes;
|
||||
}
|
||||
|
||||
|
@ -379,6 +411,9 @@ int32_t tqSendAllNotifyEvents(const SArray* pBlocks, SStreamTask* pTask, SVnode*
|
|||
int32_t nNotifyAddr = 0;
|
||||
int32_t nNotifyEvents = 0;
|
||||
SStreamNotifyHandle* pHandle = NULL;
|
||||
int64_t startTime = 0;
|
||||
int64_t endTime = 0;
|
||||
int32_t blockIdx = 0;
|
||||
|
||||
TSDB_CHECK_NULL(pTask, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
TSDB_CHECK_NULL(pVnode, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
@ -388,50 +423,61 @@ int32_t tqSendAllNotifyEvents(const SArray* pBlocks, SStreamTask* pTask, SVnode*
|
|||
goto _end;
|
||||
}
|
||||
|
||||
code = packupStreamNotifyEvent(pTask->notifyInfo.streamName, pBlocks, &msg, &nNotifyEvents);
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
if (msg == NULL) {
|
||||
goto _end;
|
||||
}
|
||||
while (blockIdx < taosArrayGetSize(pBlocks)) {
|
||||
code = packupStreamNotifyEvent(pTask->notifyInfo.streamName, pBlocks, &msg, &nNotifyEvents, &pTask->notifyEventStat,
|
||||
&blockIdx);
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
if (msg == NULL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
tqDebug("stream task %s prepare to send %d notify events, total msg length: %" PRIu64, pTask->notifyInfo.streamName,
|
||||
nNotifyEvents, (uint64_t)strlen(msg));
|
||||
tqDebug("stream task %s prepare to send %d notify events, total msg length: %" PRIu64, pTask->notifyInfo.streamName,
|
||||
nNotifyEvents, (uint64_t)strlen(msg));
|
||||
|
||||
for (int32_t i = 0; i < nNotifyAddr; ++i) {
|
||||
if (streamTaskShouldStop(pTask)) {
|
||||
break;
|
||||
}
|
||||
const char* url = taosArrayGetP(pTask->notifyInfo.pNotifyAddrUrls, i);
|
||||
code = acquireStreamNotifyHandle(pVnode->pNotifyHandleMap, url, &pHandle);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tqError("failed to get stream notify handle of %s", url);
|
||||
if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
|
||||
// retry for event message sending in PAUSE error handling mode
|
||||
taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
|
||||
--i;
|
||||
continue;
|
||||
} else {
|
||||
// simply ignore the failure in DROP error handling mode
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
continue;
|
||||
startTime = taosGetMonoTimestampMs();
|
||||
for (int32_t i = 0; i < nNotifyAddr; ++i) {
|
||||
if (streamTaskShouldStop(pTask)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
code = sendSingleStreamNotify(pHandle, msg);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tqError("failed to send stream notify handle to %s since %s", url, tstrerror(code));
|
||||
if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
|
||||
// retry for event message sending in PAUSE error handling mode
|
||||
taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
|
||||
--i;
|
||||
} else {
|
||||
// simply ignore the failure in DROP error handling mode
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
const char* url = taosArrayGetP(pTask->notifyInfo.pNotifyAddrUrls, i);
|
||||
code = acquireStreamNotifyHandle(pVnode->pNotifyHandleMap, url, &pHandle);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tqError("failed to get stream notify handle of %s", url);
|
||||
if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
|
||||
// retry for event message sending in PAUSE error handling mode
|
||||
taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
|
||||
--i;
|
||||
continue;
|
||||
} else {
|
||||
// simply ignore the failure in DROP error handling mode
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tqDebug("stream task %s send %d notify events to %s successfully", pTask->notifyInfo.streamName, nNotifyEvents,
|
||||
url);
|
||||
code = sendSingleStreamNotify(pHandle, msg);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tqError("failed to send stream notify handle to %s since %s", url, tstrerror(code));
|
||||
if (pTask->notifyInfo.notifyErrorHandle == SNOTIFY_ERROR_HANDLE_PAUSE) {
|
||||
// retry for event message sending in PAUSE error handling mode
|
||||
taosMsleep(STREAM_EVENT_NOTIFY_RETRY_MS);
|
||||
--i;
|
||||
} else {
|
||||
// simply ignore the failure in DROP error handling mode
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
}
|
||||
} else {
|
||||
tqDebug("stream task %s send %d notify events to %s successfully", pTask->notifyInfo.streamName, nNotifyEvents,
|
||||
url);
|
||||
}
|
||||
releaseStreamNotifyHandle(&pHandle);
|
||||
}
|
||||
releaseStreamNotifyHandle(&pHandle);
|
||||
|
||||
endTime = taosGetMonoTimestampMs();
|
||||
pTask->notifyEventStat.notifyEventSendTimes++;
|
||||
pTask->notifyEventStat.notifyEventSendElems += nNotifyEvents;
|
||||
pTask->notifyEventStat.notifyEventSendCostSec += (endTime - startTime) / 1000.0;
|
||||
|
||||
taosMemoryFreeClear(msg);
|
||||
}
|
||||
|
||||
_end:
|
||||
|
|
|
@ -87,9 +87,9 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) {
|
|||
return code;
|
||||
}
|
||||
|
||||
code =
|
||||
qSetStreamNotifyInfo(pTask->exec.pExecutor, pTask->notifyInfo.notifyEventTypes,
|
||||
pTask->notifyInfo.pSchemaWrapper, pTask->notifyInfo.stbFullName, IS_NEW_SUBTB_RULE(pTask));
|
||||
code = qSetStreamNotifyInfo(pTask->exec.pExecutor, pTask->notifyInfo.notifyEventTypes,
|
||||
pTask->notifyInfo.pSchemaWrapper, pTask->notifyInfo.stbFullName,
|
||||
IS_NEW_SUBTB_RULE(pTask), &pTask->notifyEventStat);
|
||||
if (code) {
|
||||
tqError("s-task:%s failed to set stream notify info, code:%s", pTask->id.idStr, tstrerror(code));
|
||||
return code;
|
||||
|
|
|
@ -223,6 +223,7 @@ void initStateStoreAPI(SStateStore* pStore) {
|
|||
pStore->updateInfoSerialize = updateInfoSerialize;
|
||||
pStore->updateInfoDeserialize = updateInfoDeserialize;
|
||||
|
||||
pStore->streamStateSessionSeekKeyPrev = streamStateSessionSeekKeyPrev;
|
||||
pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext;
|
||||
pStore->streamStateCountSeekKeyPrev = streamStateCountSeekKeyPrev;
|
||||
pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev;
|
||||
|
|
|
@ -450,16 +450,17 @@ typedef struct STimeWindowAggSupp {
|
|||
} STimeWindowAggSupp;
|
||||
|
||||
typedef struct SStreamNotifyEventSupp {
|
||||
SArray* pWindowEvents; // Array of SStreamNotifyEvent, storing window events and trigger values.
|
||||
SHashObj* pTableNameHashMap; // Hash map from groupid to the dest child table name.
|
||||
SHashObj* pResultHashMap; // Hash map from groupid+skey to the window agg result.
|
||||
SSDataBlock* pEventBlock; // The datablock contains all window events and results.
|
||||
SHashObj* pWindowEventHashMap; // Hash map from gorupid+skey+eventType to the list node of window event.
|
||||
SHashObj* pTableNameHashMap; // Hash map from groupid to the dest child table name.
|
||||
SSDataBlock* pEventBlock; // The datablock contains all window events and results.
|
||||
SArray* pSessionKeys;
|
||||
const char* windowType;
|
||||
} SStreamNotifyEventSupp;
|
||||
|
||||
typedef struct SSteamOpBasicInfo {
|
||||
int32_t primaryPkIndex;
|
||||
bool updateOperatorInfo;
|
||||
SStreamNotifyEventSupp windowEventSup;
|
||||
SStreamNotifyEventSupp notifyEventSup;
|
||||
} SSteamOpBasicInfo;
|
||||
|
||||
typedef struct SStreamFillSupporter {
|
||||
|
@ -1053,7 +1054,7 @@ int32_t saveDeleteRes(SSHashObj* pStDelete, SSessionKey key);
|
|||
void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey);
|
||||
void doBuildDeleteDataBlock(struct SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite);
|
||||
void doBuildSessionResult(struct SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo,
|
||||
SSDataBlock* pBlock);
|
||||
SSDataBlock* pBlock, SArray* pSessionKeys);
|
||||
int32_t getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo);
|
||||
void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin,
|
||||
SResultWindowInfo* pNextWin);
|
||||
|
@ -1090,7 +1091,7 @@ void freeResetOperatorParams(struct SOperatorInfo* pOperator, SOperatorParamT
|
|||
int32_t getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int32_t idx, bool clearParam,
|
||||
SSDataBlock** pResBlock);
|
||||
void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, EStreamType mode, SSessionKey* pDelRange);
|
||||
void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey);
|
||||
void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey);
|
||||
|
||||
int32_t saveDeleteInfo(SArray* pWins, SSessionKey key);
|
||||
void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins);
|
||||
|
|
|
@ -18,9 +18,9 @@
|
|||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "executorInt.h"
|
||||
#endif
|
||||
|
||||
#include "executorInt.h"
|
||||
|
||||
#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str)
|
||||
|
||||
|
@ -59,22 +59,23 @@ typedef struct STaskStopInfo {
|
|||
} STaskStopInfo;
|
||||
|
||||
typedef struct {
|
||||
STqOffsetVal currentOffset; // for tmq
|
||||
SMqBatchMetaRsp btMetaRsp; // for tmq fetching meta
|
||||
int8_t sourceExcluded;
|
||||
int64_t snapshotVer;
|
||||
SSchemaWrapper* schema;
|
||||
char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor
|
||||
int8_t recoverStep;
|
||||
int8_t recoverScanFinished;
|
||||
SQueryTableDataCond tableCond;
|
||||
SVersionRange fillHistoryVer;
|
||||
STimeWindow fillHistoryWindow;
|
||||
SStreamState* pState;
|
||||
int32_t eventTypes; // event types to notify
|
||||
SSchemaWrapper* notifyResultSchema; // agg result to notify
|
||||
char* stbFullName; // used to generate dest child table name
|
||||
bool newSubTableRule; // used to generate dest child table name
|
||||
STqOffsetVal currentOffset; // for tmq
|
||||
SMqBatchMetaRsp btMetaRsp; // for tmq fetching meta
|
||||
int8_t sourceExcluded;
|
||||
int64_t snapshotVer;
|
||||
SSchemaWrapper* schema;
|
||||
char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor
|
||||
int8_t recoverStep;
|
||||
int8_t recoverScanFinished;
|
||||
SQueryTableDataCond tableCond;
|
||||
SVersionRange fillHistoryVer;
|
||||
STimeWindow fillHistoryWindow;
|
||||
SStreamState* pState;
|
||||
int32_t eventTypes; // event types to notify
|
||||
SSchemaWrapper* notifyResultSchema; // agg result to notify
|
||||
char* stbFullName; // used to generate dest child table name
|
||||
bool newSubTableRule; // used to generate dest child table name
|
||||
STaskNotifyEventStat* pNotifyEventStat; // used to store notify event statistics
|
||||
} SStreamTaskInfo;
|
||||
|
||||
struct SExecTaskInfo {
|
||||
|
|
|
@ -30,20 +30,31 @@ extern "C" {
|
|||
#define FILL_POS_MID 2
|
||||
#define FILL_POS_END 3
|
||||
|
||||
#define HAS_NON_ROW_DATA(pRowData) (pRowData->key == INT64_MIN)
|
||||
#define HAS_ROW_DATA(pRowData) (pRowData && pRowData->key != INT64_MIN)
|
||||
#define HAS_NON_ROW_DATA(pRowData) (pRowData->key == INT64_MIN)
|
||||
#define HAS_ROW_DATA(pRowData) (pRowData && pRowData->key != INT64_MIN)
|
||||
|
||||
#define IS_INVALID_WIN_KEY(ts) ((ts) == INT64_MIN)
|
||||
#define IS_VALID_WIN_KEY(ts) ((ts) != INT64_MIN)
|
||||
#define SET_WIN_KEY_INVALID(ts) ((ts) = INT64_MIN)
|
||||
#define IS_INVALID_WIN_KEY(ts) ((ts) == INT64_MIN)
|
||||
#define IS_VALID_WIN_KEY(ts) ((ts) != INT64_MIN)
|
||||
#define SET_WIN_KEY_INVALID(ts) ((ts) = INT64_MIN)
|
||||
|
||||
#define IS_NORMAL_INTERVAL_OP(op) \
|
||||
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || \
|
||||
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
|
||||
|
||||
#define IS_NORMAL_SESSION_OP(op) \
|
||||
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \
|
||||
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
|
||||
|
||||
#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE)
|
||||
|
||||
#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT)
|
||||
|
||||
#define IS_NORMAL_COUNT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT)
|
||||
|
||||
#define IS_CONTINUE_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL)
|
||||
|
||||
#define IS_FILL_CONST_VALUE(type) ((type == TSDB_FILL_NULL || type == TSDB_FILL_NULL_F || type == TSDB_FILL_SET_VALUE || type == TSDB_FILL_SET_VALUE_F))
|
||||
#define IS_FILL_CONST_VALUE(type) \
|
||||
((type == TSDB_FILL_NULL || type == TSDB_FILL_NULL_F || type == TSDB_FILL_SET_VALUE || type == TSDB_FILL_SET_VALUE_F))
|
||||
|
||||
typedef struct SSliceRowData {
|
||||
TSKEY key;
|
||||
|
@ -57,11 +68,13 @@ typedef struct SSlicePoint {
|
|||
SRowBuffPos* pResPos;
|
||||
} SSlicePoint;
|
||||
|
||||
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
|
||||
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);
|
||||
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo);
|
||||
int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo);
|
||||
void destroyStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo);
|
||||
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
|
||||
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);
|
||||
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo);
|
||||
int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo, const struct SOperatorInfo* pOperator);
|
||||
void destroyStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo);
|
||||
int32_t encodeStreamBasicInfo(void** buf, const SSteamOpBasicInfo* pBasicInfo);
|
||||
int32_t decodeStreamBasicInfo(void** buf, SSteamOpBasicInfo* pBasicInfo);
|
||||
|
||||
int64_t getDeleteMarkFromOption(SStreamNodeOption* pOption);
|
||||
void removeDeleteResults(SSHashObj* pUpdatedMap, SArray* pDelWins);
|
||||
|
@ -98,7 +111,7 @@ SResultCellData* getSliceResultCell(SResultCellData* pRowVal, int32_t index, int
|
|||
int32_t getDownstreamRes(struct SOperatorInfo* downstream, SSDataBlock** ppRes, SColumnInfo** ppPkCol);
|
||||
void destroyFlusedppPos(void* ppRes);
|
||||
void doBuildStreamIntervalResult(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
|
||||
SGroupResInfo* pGroupResInfo);
|
||||
SGroupResInfo* pGroupResInfo, SArray* pSessionKeys);
|
||||
void transBlockToSliceResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SSliceRowData* pRowVal,
|
||||
int32_t rowSize, void* pPkData, SColumnInfoData* pPkCol, int32_t* pCellOffsetInfo);
|
||||
int32_t getQualifiedRowNumDesc(SExprSupp* pExprSup, SSDataBlock* pBlock, TSKEY* tsCols, int32_t rowId, bool ignoreNull);
|
||||
|
@ -112,10 +125,25 @@ TSKEY compareTs(void* pKey);
|
|||
|
||||
int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
const SSDataBlock* pInputBlock, const SNodeList* pCondCols, int32_t ri,
|
||||
SStreamNotifyEventSupp* sup);
|
||||
int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SSchemaWrapper* pSchemaWrapper,
|
||||
SStreamNotifyEventSupp* sup);
|
||||
int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup);
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t addStateAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
const SStateKeys* pCurState, const SStateKeys* pAnotherState, bool onlyUpdate,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t addIntervalAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t addSessionAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t addCountAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SArray* pSessionKeys,
|
||||
const SSchemaWrapper* pSchemaWrapper, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t addAggDeleteNotifyEvent(const SSDataBlock* pDeleteBlock, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat);
|
||||
int32_t removeOutdatedNotifyEvents(STimeWindowAggSupp* pTwSup, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
@ -251,7 +251,7 @@ int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) {
|
|||
}
|
||||
|
||||
int32_t qSetStreamNotifyInfo(qTaskInfo_t tinfo, int32_t eventTypes, const SSchemaWrapper* pSchemaWrapper,
|
||||
const char* stbFullName, bool newSubTableRule) {
|
||||
const char* stbFullName, bool newSubTableRule, STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
SStreamTaskInfo *pStreamInfo = NULL;
|
||||
|
||||
|
@ -267,6 +267,7 @@ int32_t qSetStreamNotifyInfo(qTaskInfo_t tinfo, int32_t eventTypes, const SSchem
|
|||
}
|
||||
pStreamInfo->stbFullName = taosStrdup(stbFullName);
|
||||
pStreamInfo->newSubTableRule = newSubTableRule;
|
||||
pStreamInfo->pNotifyEventStat = pNotifyEventStat;
|
||||
|
||||
_end:
|
||||
return code;
|
||||
|
|
|
@ -25,7 +25,6 @@
|
|||
#include "tlog.h"
|
||||
#include "ttime.h"
|
||||
|
||||
#define IS_NORMAL_COUNT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT)
|
||||
#define STREAM_COUNT_OP_STATE_NAME "StreamCountHistoryState"
|
||||
#define STREAM_COUNT_OP_CHECKPOINT_NAME "StreamCountOperator_Checkpoint"
|
||||
|
||||
|
@ -56,6 +55,8 @@ void destroyStreamCountAggOperatorInfo(void* param) {
|
|||
&pInfo->groupResInfo);
|
||||
pInfo->pOperator = NULL;
|
||||
}
|
||||
|
||||
destroyStreamBasicInfo(&pInfo->basic);
|
||||
destroyStreamAggSupporter(&pInfo->streamAggSup);
|
||||
cleanupExprSupp(&pInfo->scalarSupp);
|
||||
clearGroupResInfo(&pInfo->groupResInfo);
|
||||
|
@ -79,10 +80,9 @@ void destroyStreamCountAggOperatorInfo(void* param) {
|
|||
bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; }
|
||||
|
||||
int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin,
|
||||
SBuffInfo* pBuffInfo) {
|
||||
SBuffInfo* pBuffInfo, int32_t* pWinCode) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
int32_t size = pAggSup->resultRowSize;
|
||||
pCurWin->winInfo.sessionWin.groupId = groupId;
|
||||
pCurWin->winInfo.sessionWin.win.skey = ts;
|
||||
|
@ -90,19 +90,21 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
|
|||
|
||||
if (isSlidingCountWindow(pAggSup)) {
|
||||
if (pBuffInfo->winBuffOp == CREATE_NEW_WINDOW) {
|
||||
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
code =
|
||||
pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
|
||||
pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
winCode = TSDB_CODE_FAILED;
|
||||
*pWinCode = TSDB_CODE_FAILED;
|
||||
} else if (pBuffInfo->winBuffOp == MOVE_NEXT_WINDOW) {
|
||||
QUERY_CHECK_NULL(pBuffInfo->pCur, code, lino, _end, terrno);
|
||||
pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur);
|
||||
winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
if (winCode == TSDB_CODE_FAILED) {
|
||||
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
*pWinCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
if (*pWinCode == TSDB_CODE_FAILED) {
|
||||
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
|
||||
pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos,
|
||||
&size);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
} else {
|
||||
reuseOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->stateStore);
|
||||
|
@ -110,11 +112,12 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
|
|||
} else {
|
||||
pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin,
|
||||
pAggSup->windowCount);
|
||||
winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
if (winCode == TSDB_CODE_FAILED) {
|
||||
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
*pWinCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size);
|
||||
if (*pWinCode == TSDB_CODE_FAILED) {
|
||||
code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
|
||||
pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos,
|
||||
&size);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
} else {
|
||||
reuseOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->stateStore);
|
||||
|
@ -126,11 +129,11 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
|
|||
} else {
|
||||
code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin,
|
||||
pAggSup->windowCount,
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size, &winCode);
|
||||
(void**)&pCurWin->winInfo.pStatePos, &size, pWinCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
if (winCode == TSDB_CODE_SUCCESS) {
|
||||
if (*pWinCode == TSDB_CODE_SUCCESS) {
|
||||
pCurWin->winInfo.isOutput = true;
|
||||
}
|
||||
pCurWin->pWindowCount =
|
||||
|
@ -297,10 +300,18 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
continue;
|
||||
}
|
||||
SCountWindowInfo curWin = {0};
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
buffInfo.rebuildWindow = false;
|
||||
code = setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo);
|
||||
code = setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo, &winCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (winCode != TSDB_CODE_SUCCESS &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
|
||||
code = addCountAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, &pInfo->basic.notifyEventSup,
|
||||
pTaskInfo->streamInfo.pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
if (!inCountSlidingWindow(pAggSup, &curWin.winInfo.sessionWin.win, &pSDataBlock->info)) {
|
||||
buffInfo.winBuffOp = MOVE_NEXT_WINDOW;
|
||||
continue;
|
||||
|
@ -375,23 +386,54 @@ _end:
|
|||
|
||||
static int32_t buildCountResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
SOptrBasicInfo* pBInfo = &pInfo->binfo;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
bool addNotifyEvent = false;
|
||||
addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
|
||||
doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->pDelRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
|
||||
doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes,
|
||||
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
|
||||
if (pBInfo->pRes->info.rows > 0) {
|
||||
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
|
||||
pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pBInfo->pRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
|
||||
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
(*ppRes) = pNotifySup->pEventBlock;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
}
|
||||
(*ppRes) = NULL;
|
||||
return code;
|
||||
}
|
||||
|
@ -423,7 +465,10 @@ int32_t doStreamCountEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
|
|||
// 3.dataVersion
|
||||
tlen += taosEncodeFixedI32(buf, pInfo->dataVersion);
|
||||
|
||||
// 4.checksum
|
||||
// 4.basicInfo
|
||||
tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
|
||||
|
||||
// 5.checksum
|
||||
if (isParent) {
|
||||
if (buf) {
|
||||
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
|
||||
|
@ -441,12 +486,13 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
int32_t lino = 0;
|
||||
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
void* pDataEnd = POINTER_SHIFT(buf, len);
|
||||
if (!pInfo) {
|
||||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
// 4.checksum
|
||||
// 5.checksum
|
||||
if (isParent) {
|
||||
int32_t dataLen = len - sizeof(uint32_t);
|
||||
void* pCksum = POINTER_SHIFT(buf, dataLen);
|
||||
|
@ -454,6 +500,7 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
pDataEnd = pCksum;
|
||||
}
|
||||
|
||||
// 1.streamAggSup.pResultRows
|
||||
|
@ -462,9 +509,10 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
for (int32_t i = 0; i < mapSize; i++) {
|
||||
SSessionKey key = {0};
|
||||
SCountWindowInfo curWin = {0};
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
buf = decodeSSessionKey(buf, &key);
|
||||
SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL};
|
||||
code = setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo);
|
||||
code = setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo, &winCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize);
|
||||
|
@ -479,6 +527,12 @@ int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
// 3.dataVersion
|
||||
buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);
|
||||
|
||||
// 4.basicInfo
|
||||
if (buf < pDataEnd) {
|
||||
code = decodeStreamBasicInfo(&buf, &pInfo->basic);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
|
@ -851,7 +905,7 @@ int32_t createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
|
|||
QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);
|
||||
pInfo->binfo.pRes = pResBlock;
|
||||
|
||||
SExprInfo* pExprInfo = NULL;
|
||||
SExprInfo* pExprInfo = NULL;
|
||||
code = createExprInfo(pCountNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
|
@ -925,6 +979,9 @@ int32_t createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
|
|||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
if (downstream) {
|
||||
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
|
||||
&pInfo->twAggSup, &pInfo->basic);
|
||||
|
|
|
@ -30,7 +30,6 @@
|
|||
#include "tlog.h"
|
||||
#include "ttime.h"
|
||||
|
||||
#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT)
|
||||
#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState"
|
||||
#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint"
|
||||
|
||||
|
@ -135,7 +134,8 @@ void reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) {
|
|||
}
|
||||
|
||||
int32_t setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupId, bool* pStart, bool* pEnd,
|
||||
int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey, int32_t* pWinCode) {
|
||||
int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey,
|
||||
int32_t* pWinCode) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
|
@ -179,7 +179,7 @@ int32_t setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t gro
|
|||
SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId};
|
||||
code = pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
(*pWinCode) = TSDB_CODE_FAILED;
|
||||
(*pWinCode) = TSDB_CODE_FAILED;
|
||||
|
||||
setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin);
|
||||
pCurWin->pWinFlag->startFlag = start;
|
||||
|
@ -335,6 +335,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
SColumnInfoData* pColStart = NULL;
|
||||
SColumnInfoData* pColEnd = NULL;
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
|
||||
pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
|
||||
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
|
||||
|
@ -395,17 +397,19 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
&nextWinKey, &winCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN) &&
|
||||
*(bool*)colDataGetNumData(pColStart, i) && winCode != TSDB_CODE_SUCCESS) {
|
||||
if (winCode != TSDB_CODE_SUCCESS &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN) &&
|
||||
*(bool*)colDataGetNumData(pColStart, i)) {
|
||||
code = addEventAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, pSDataBlock,
|
||||
pInfo->pStartCondCols, i, &pInfo->basic.windowEventSup);
|
||||
pInfo->pStartCondCols, i, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);
|
||||
bool rebuild = false;
|
||||
code = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData,
|
||||
rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild, &winRows);
|
||||
code = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData,
|
||||
(bool*)pColEnd->pData, rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild,
|
||||
&winRows);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (rebuild) {
|
||||
|
@ -471,7 +475,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
|
||||
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {
|
||||
code = addEventAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &curWin.winInfo.sessionWin, pSDataBlock,
|
||||
pInfo->pEndCondCols, i + winRows - 1, &pInfo->basic.windowEventSup);
|
||||
pInfo->pEndCondCols, i + winRows - 1, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
}
|
||||
|
@ -513,7 +517,10 @@ int32_t doStreamEventEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
|
|||
// 3.dataVersion
|
||||
tlen += taosEncodeFixedI32(buf, pInfo->dataVersion);
|
||||
|
||||
// 4.checksum
|
||||
// 4.basicInfo
|
||||
tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
|
||||
|
||||
// 5.checksum
|
||||
if (buf) {
|
||||
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
|
||||
tlen += taosEncodeFixedU32(buf, cksum);
|
||||
|
@ -529,13 +536,14 @@ int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
int32_t lino = 0;
|
||||
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
void* pDataEnd = POINTER_SHIFT(buf, len);
|
||||
if (!pInfo) {
|
||||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
|
||||
// 4.checksum
|
||||
// 5.checksum
|
||||
int32_t dataLen = len - sizeof(uint32_t);
|
||||
void* pCksum = POINTER_SHIFT(buf, dataLen);
|
||||
if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {
|
||||
|
@ -543,6 +551,7 @@ int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
pDataEnd = pCksum;
|
||||
|
||||
// 1.streamAggSup.pResultRows
|
||||
int32_t mapSize = 0;
|
||||
|
@ -567,6 +576,12 @@ int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
// 3.dataVersion
|
||||
buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);
|
||||
|
||||
// 4.basicInfo
|
||||
if (buf < pDataEnd) {
|
||||
code = decodeStreamBasicInfo(&buf, &pInfo->basic);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
|
@ -598,33 +613,45 @@ static int32_t buildEventResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
|
|||
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
|
||||
SOptrBasicInfo* pBInfo = &pInfo->binfo;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
bool addNotifyEvent = false;
|
||||
addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
|
||||
doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->pDelRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes);
|
||||
doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes,
|
||||
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
|
||||
if (pBInfo->pRes->info.rows > 0) {
|
||||
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {
|
||||
code = addAggResultNotifyEvent(pBInfo->pRes, pTaskInfo->streamInfo.notifyResultSchema, &pInfo->basic.windowEventSup);
|
||||
if (addNotifyEvent) {
|
||||
code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
|
||||
pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pBInfo->pRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = buildNotifyEventBlock(pTaskInfo, &pInfo->basic.windowEventSup);
|
||||
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (pInfo->basic.windowEventSup.pEventBlock->info.rows > 0) {
|
||||
printDataBlock(pInfo->basic.windowEventSup.pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
(*ppRes) = pInfo->basic.windowEventSup.pEventBlock;
|
||||
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
|
||||
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
(*ppRes) = pNotifySup->pEventBlock;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
_end:
|
||||
(*ppRes) = NULL;
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
@ -800,7 +827,7 @@ void streamEventReleaseState(SOperatorInfo* pOperator) {
|
|||
int32_t resSize = winSize + sizeof(TSKEY);
|
||||
char* pBuff = taosMemoryCalloc(1, resSize);
|
||||
if (!pBuff) {
|
||||
return ;
|
||||
return;
|
||||
}
|
||||
memcpy(pBuff, pInfo->historyWins->pData, winSize);
|
||||
memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));
|
||||
|
@ -954,9 +981,9 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
|
|||
QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);
|
||||
pInfo->binfo.pRes = pResBlock;
|
||||
|
||||
SExprSupp* pExpSup = &pOperator->exprSupp;
|
||||
int32_t numOfCols = 0;
|
||||
SExprInfo* pExprInfo = NULL;
|
||||
SExprSupp* pExpSup = &pOperator->exprSupp;
|
||||
int32_t numOfCols = 0;
|
||||
SExprInfo* pExprInfo = NULL;
|
||||
code = createExprInfo(pEventNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
|
@ -1006,7 +1033,6 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
|
|||
pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);
|
||||
QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno);
|
||||
pInfo->destHasPrimaryKey = pEventNode->window.destHasPrimaryKey;
|
||||
initStreamBasicInfo(&pInfo->basic);
|
||||
|
||||
pInfo->pOperator = pOperator;
|
||||
setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED,
|
||||
|
@ -1026,6 +1052,10 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
|
|||
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAggNext, NULL, destroyStreamEventOperatorInfo,
|
||||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
|
||||
&pInfo->twAggSup, &pInfo->basic);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
|
|
@ -16,53 +16,58 @@
|
|||
#include "streamexecutorInt.h"
|
||||
|
||||
#include "executorInt.h"
|
||||
#include "operator.h"
|
||||
#include "tdatablock.h"
|
||||
|
||||
#define NOTIFY_EVENT_NAME_CACHE_LIMIT_MB 16
|
||||
|
||||
typedef struct SStreamNotifyEvent {
|
||||
uint64_t gid;
|
||||
TSKEY skey;
|
||||
char* content;
|
||||
bool isEnd;
|
||||
int64_t eventType;
|
||||
STimeWindow win;
|
||||
cJSON* pJson;
|
||||
} SStreamNotifyEvent;
|
||||
|
||||
#define NOTIFY_EVENT_KEY_SIZE \
|
||||
((sizeof(((struct SStreamNotifyEvent*)0)->gid) + sizeof(((struct SStreamNotifyEvent*)0)->eventType)) + \
|
||||
sizeof(((struct SStreamNotifyEvent*)0)->win.skey))
|
||||
|
||||
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type) {
|
||||
if (type != STREAM_GET_ALL && type != STREAM_CHECKPOINT) {
|
||||
pBasicInfo->updateOperatorInfo = true;
|
||||
}
|
||||
}
|
||||
|
||||
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo) {
|
||||
return pBasicInfo->updateOperatorInfo;
|
||||
}
|
||||
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo) { return pBasicInfo->updateOperatorInfo; }
|
||||
|
||||
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo) {
|
||||
pBasicInfo->updateOperatorInfo = false;
|
||||
}
|
||||
void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo) { pBasicInfo->updateOperatorInfo = false; }
|
||||
|
||||
static void destroyStreamWindowEvent(void* ptr) {
|
||||
SStreamNotifyEvent* pEvent = ptr;
|
||||
if (pEvent == NULL || pEvent->content == NULL) return;
|
||||
cJSON_free(pEvent->content);
|
||||
SStreamNotifyEvent* pEvent = (SStreamNotifyEvent*)ptr;
|
||||
if (pEvent) {
|
||||
if (pEvent->pJson) {
|
||||
cJSON_Delete(pEvent->pJson);
|
||||
}
|
||||
*pEvent = (SStreamNotifyEvent){0};
|
||||
}
|
||||
}
|
||||
|
||||
static void destroyStreamNotifyEventSupp(SStreamNotifyEventSupp* sup) {
|
||||
if (sup == NULL) return;
|
||||
taosArrayDestroyEx(sup->pWindowEvents, destroyStreamWindowEvent);
|
||||
taosHashCleanup(sup->pWindowEventHashMap);
|
||||
taosHashCleanup(sup->pTableNameHashMap);
|
||||
taosHashCleanup(sup->pResultHashMap);
|
||||
blockDataDestroy(sup->pEventBlock);
|
||||
taosArrayDestroy(sup->pSessionKeys);
|
||||
*sup = (SStreamNotifyEventSupp){0};
|
||||
}
|
||||
|
||||
static int32_t initStreamNotifyEventSupp(SStreamNotifyEventSupp *sup) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SSDataBlock* pBlock = NULL;
|
||||
static int32_t initStreamNotifyEventSupp(SStreamNotifyEventSupp* sup, const char* windowType, int32_t resCapacity) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SSDataBlock* pBlock = NULL;
|
||||
SColumnInfoData infoData = {0};
|
||||
|
||||
if (sup == NULL) {
|
||||
if (sup == NULL || sup->pWindowEventHashMap != NULL) {
|
||||
goto _end;
|
||||
}
|
||||
|
||||
|
@ -77,15 +82,18 @@ static int32_t initStreamNotifyEventSupp(SStreamNotifyEventSupp *sup) {
|
|||
code = blockDataAppendColInfo(pBlock, &infoData);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
sup->pWindowEvents = taosArrayInit(0, sizeof(SStreamNotifyEvent));
|
||||
QUERY_CHECK_NULL(sup->pWindowEvents, code, lino, _end, terrno);
|
||||
sup->pWindowEventHashMap = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
|
||||
QUERY_CHECK_NULL(sup->pWindowEventHashMap, code, lino, _end, terrno);
|
||||
taosHashSetFreeFp(sup->pWindowEventHashMap, destroyStreamWindowEvent);
|
||||
sup->pTableNameHashMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
|
||||
QUERY_CHECK_NULL(sup->pTableNameHashMap, code, lino, _end, terrno);
|
||||
sup->pResultHashMap = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
|
||||
QUERY_CHECK_NULL(sup->pResultHashMap, code, lino, _end, terrno);
|
||||
taosHashSetFreeFp(sup->pResultHashMap, destroyStreamWindowEvent);
|
||||
sup->pEventBlock = pBlock;
|
||||
pBlock = NULL;
|
||||
code = blockDataEnsureCapacity(sup->pEventBlock, resCapacity);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
sup->windowType = windowType;
|
||||
sup->pSessionKeys = taosArrayInit(resCapacity, sizeof(SSessionKey));
|
||||
QUERY_CHECK_NULL(sup->pSessionKeys, code, lino, _end, terrno);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
@ -100,17 +108,99 @@ _end:
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo) {
|
||||
int32_t initStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo, const struct SOperatorInfo* pOperator) {
|
||||
pBasicInfo->primaryPkIndex = -1;
|
||||
pBasicInfo->updateOperatorInfo = false;
|
||||
return initStreamNotifyEventSupp(&pBasicInfo->windowEventSup);
|
||||
const char* windowType = NULL;
|
||||
if (IS_NORMAL_INTERVAL_OP(pOperator)) {
|
||||
windowType = "Time";
|
||||
} else if (IS_NORMAL_SESSION_OP(pOperator)) {
|
||||
windowType = "Session";
|
||||
} else if (IS_NORMAL_STATE_OP(pOperator)) {
|
||||
windowType = "State";
|
||||
} else if (IS_NORMAL_EVENT_OP(pOperator)) {
|
||||
windowType = "Event";
|
||||
} else if (IS_NORMAL_COUNT_OP(pOperator)) {
|
||||
windowType = "Count";
|
||||
} else {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
return initStreamNotifyEventSupp(&pBasicInfo->notifyEventSup, windowType, pOperator->resultInfo.capacity);
|
||||
}
|
||||
|
||||
void destroyStreamBasicInfo(SSteamOpBasicInfo* pBasicInfo) {
|
||||
destroyStreamNotifyEventSupp(&pBasicInfo->windowEventSup);
|
||||
destroyStreamNotifyEventSupp(&pBasicInfo->notifyEventSup);
|
||||
}
|
||||
|
||||
static void streamNotifyGetEventWindowId(const SSessionKey* pSessionKey, char *buf) {
|
||||
static int32_t encodeStreamNotifyEventSupp(void** buf, const SStreamNotifyEventSupp* sup) {
|
||||
int32_t tlen = 0;
|
||||
void* pIter = NULL;
|
||||
char* str = NULL;
|
||||
|
||||
if (sup == NULL) {
|
||||
return tlen;
|
||||
}
|
||||
|
||||
tlen += taosEncodeFixedI32(buf, taosHashGetSize(sup->pWindowEventHashMap));
|
||||
pIter = taosHashIterate(sup->pWindowEventHashMap, NULL);
|
||||
while (pIter) {
|
||||
const SStreamNotifyEvent* pEvent = (const SStreamNotifyEvent*)pIter;
|
||||
str = cJSON_PrintUnformatted(pEvent->pJson);
|
||||
|
||||
tlen += taosEncodeFixedU64(buf, pEvent->gid);
|
||||
tlen += taosEncodeFixedI64(buf, pEvent->eventType);
|
||||
tlen += taosEncodeFixedI64(buf, pEvent->win.skey);
|
||||
tlen += taosEncodeFixedI64(buf, pEvent->win.ekey);
|
||||
tlen += taosEncodeString(buf, str);
|
||||
cJSON_free(str);
|
||||
pIter = taosHashIterate(sup->pWindowEventHashMap, pIter);
|
||||
}
|
||||
return tlen;
|
||||
}
|
||||
|
||||
static int32_t decodeStreamNotifyEventSupp(void** buf, SStreamNotifyEventSupp* sup) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
void* p = *buf;
|
||||
int32_t size = 0;
|
||||
uint64_t len = 0;
|
||||
SStreamNotifyEvent item = {0};
|
||||
|
||||
p = taosDecodeFixedI32(p, &size);
|
||||
for (int32_t i = 0; i < size; i++) {
|
||||
p = taosDecodeFixedU64(p, &item.gid);
|
||||
p = taosDecodeFixedI64(p, &item.eventType);
|
||||
p = taosDecodeFixedI64(p, &item.win.skey);
|
||||
p = taosDecodeFixedI64(p, &item.win.ekey);
|
||||
p = taosDecodeVariantU64(p, &len);
|
||||
item.pJson = cJSON_Parse(p);
|
||||
if (item.pJson == NULL) {
|
||||
qWarn("failed to parse the json content since %s", cJSON_GetErrorPtr());
|
||||
}
|
||||
QUERY_CHECK_NULL(item.pJson, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
|
||||
p = POINTER_SHIFT(p, len);
|
||||
code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
item.pJson = NULL;
|
||||
}
|
||||
*buf = p;
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
destroyStreamWindowEvent(&item);
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t encodeStreamBasicInfo(void** buf, const SSteamOpBasicInfo* pBasicInfo) {
|
||||
return encodeStreamNotifyEventSupp(buf, &pBasicInfo->notifyEventSup);
|
||||
}
|
||||
|
||||
int32_t decodeStreamBasicInfo(void** buf, SSteamOpBasicInfo* pBasicInfo) {
|
||||
return decodeStreamNotifyEventSupp(buf, &pBasicInfo->notifyEventSup);
|
||||
}
|
||||
|
||||
static void streamNotifyGetEventWindowId(const SSessionKey* pSessionKey, char* buf) {
|
||||
uint64_t hash = 0;
|
||||
uint64_t ar[2];
|
||||
|
||||
|
@ -123,60 +213,60 @@ static void streamNotifyGetEventWindowId(const SSessionKey* pSessionKey, char *b
|
|||
#define JSON_CHECK_ADD_ITEM(obj, str, item) \
|
||||
QUERY_CHECK_CONDITION(cJSON_AddItemToObjectCS(obj, str, item), code, lino, _end, TSDB_CODE_OUT_OF_MEMORY)
|
||||
|
||||
static int32_t jsonAddColumnField(const char* colName, const SColumnInfoData* pColData, int32_t ri, cJSON* obj) {
|
||||
static int32_t jsonAddColumnField(const char* colName, int16_t type, bool isNull, const char* pData, cJSON* obj) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
char* temp = NULL;
|
||||
|
||||
QUERY_CHECK_NULL(colName, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_CONDITION(isNull || (pData != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(obj, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
if (colDataIsNull_s(pColData, ri)) {
|
||||
if (isNull) {
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNull());
|
||||
goto _end;
|
||||
}
|
||||
|
||||
switch (pColData->info.type) {
|
||||
switch (type) {
|
||||
case TSDB_DATA_TYPE_BOOL: {
|
||||
bool val = *(bool*)colDataGetNumData(pColData, ri);
|
||||
bool val = *(const bool*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateBool(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_TINYINT: {
|
||||
int8_t val = *(int8_t*)colDataGetNumData(pColData, ri);
|
||||
int8_t val = *(const int8_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_SMALLINT: {
|
||||
int16_t val = *(int16_t*)colDataGetNumData(pColData, ri);
|
||||
int16_t val = *(const int16_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_INT: {
|
||||
int32_t val = *(int32_t*)colDataGetNumData(pColData, ri);
|
||||
int32_t val = *(const int32_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_BIGINT:
|
||||
case TSDB_DATA_TYPE_TIMESTAMP: {
|
||||
int64_t val = *(int64_t*)colDataGetNumData(pColData, ri);
|
||||
int64_t val = *(const int64_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_FLOAT: {
|
||||
float val = *(float*)colDataGetNumData(pColData, ri);
|
||||
float val = *(const float*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_DOUBLE: {
|
||||
double val = *(double*)colDataGetNumData(pColData, ri);
|
||||
double val = *(const double*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
@ -185,8 +275,8 @@ static int32_t jsonAddColumnField(const char* colName, const SColumnInfoData* pC
|
|||
case TSDB_DATA_TYPE_NCHAR: {
|
||||
// cJSON requires null-terminated strings, but this data is not null-terminated,
|
||||
// so we need to manually copy the string and add null termination.
|
||||
const char* src = varDataVal(colDataGetVarData(pColData, ri));
|
||||
int32_t len = varDataLen(colDataGetVarData(pColData, ri));
|
||||
const char* src = varDataVal(pData);
|
||||
int32_t len = varDataLen(pData);
|
||||
temp = cJSON_malloc(len + 1);
|
||||
QUERY_CHECK_NULL(temp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
|
||||
memcpy(temp, src, len);
|
||||
|
@ -202,25 +292,25 @@ static int32_t jsonAddColumnField(const char* colName, const SColumnInfoData* pC
|
|||
}
|
||||
|
||||
case TSDB_DATA_TYPE_UTINYINT: {
|
||||
uint8_t val = *(uint8_t*)colDataGetNumData(pColData, ri);
|
||||
uint8_t val = *(const uint8_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_USMALLINT: {
|
||||
uint16_t val = *(uint16_t*)colDataGetNumData(pColData, ri);
|
||||
uint16_t val = *(const uint16_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_UINT: {
|
||||
uint32_t val = *(uint32_t*)colDataGetNumData(pColData, ri);
|
||||
uint32_t val = *(const uint32_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
||||
case TSDB_DATA_TYPE_UBIGINT: {
|
||||
uint64_t val = *(uint64_t*)colDataGetNumData(pColData, ri);
|
||||
uint64_t val = *(const uint64_t*)pData;
|
||||
JSON_CHECK_ADD_ITEM(obj, colName, cJSON_CreateNumber(val));
|
||||
break;
|
||||
}
|
||||
|
@ -241,53 +331,91 @@ _end:
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
const SSDataBlock* pInputBlock, const SNodeList* pCondCols, int32_t ri,
|
||||
SStreamNotifyEventSupp* sup) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SNode* node = NULL;
|
||||
cJSON* event = NULL;
|
||||
cJSON* fields = NULL;
|
||||
cJSON* cond = NULL;
|
||||
SStreamNotifyEvent item = {0};
|
||||
char windowId[32];
|
||||
static cJSON* createBasicAggNotifyEvent(const char* windowType, EStreamNotifyEventType eventType,
|
||||
const SSessionKey* pSessionKey) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
const char* eventTypeStr = NULL;
|
||||
cJSON* event = NULL;
|
||||
char windowId[32];
|
||||
|
||||
QUERY_CHECK_NULL(windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pSessionKey, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pInputBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pInputBlock->pDataBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pCondCols, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
qDebug("add stream notify event from event window, type: %s, start: %" PRId64 ", end: %" PRId64,
|
||||
(eventType == SNOTIFY_EVENT_WINDOW_OPEN) ? "WINDOW_OPEN" : "WINDOW_CLOSE", pSessionKey->win.skey,
|
||||
pSessionKey->win.ekey);
|
||||
if (eventType == SNOTIFY_EVENT_WINDOW_OPEN) {
|
||||
eventTypeStr = "WINDOW_OPEN";
|
||||
} else if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
|
||||
eventTypeStr = "WINDOW_CLOSE";
|
||||
} else if (eventType == SNOTIFY_EVENT_WINDOW_INVALIDATION) {
|
||||
eventTypeStr = "WINDOW_INVALIDATION";
|
||||
} else {
|
||||
QUERY_CHECK_CONDITION(false, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
}
|
||||
|
||||
qDebug("add stream notify event from %s Window, type: %s, start: %" PRId64 ", end: %" PRId64, windowType,
|
||||
eventTypeStr, pSessionKey->win.skey, pSessionKey->win.ekey);
|
||||
|
||||
event = cJSON_CreateObject();
|
||||
QUERY_CHECK_NULL(event, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
|
||||
|
||||
// add basic info
|
||||
streamNotifyGetEventWindowId(pSessionKey, windowId);
|
||||
if (eventType == SNOTIFY_EVENT_WINDOW_OPEN) {
|
||||
JSON_CHECK_ADD_ITEM(event, "eventType", cJSON_CreateStringReference("WINDOW_OPEN"));
|
||||
} else if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
|
||||
JSON_CHECK_ADD_ITEM(event, "eventType", cJSON_CreateStringReference("WINDOW_CLOSE"));
|
||||
}
|
||||
JSON_CHECK_ADD_ITEM(event, "eventType", cJSON_CreateStringReference(eventTypeStr));
|
||||
JSON_CHECK_ADD_ITEM(event, "eventTime", cJSON_CreateNumber(taosGetTimestampMs()));
|
||||
JSON_CHECK_ADD_ITEM(event, "windowId", cJSON_CreateStringReference(windowId));
|
||||
JSON_CHECK_ADD_ITEM(event, "windowType", cJSON_CreateStringReference("Event"));
|
||||
JSON_CHECK_ADD_ITEM(event, "windowId", cJSON_CreateString(windowId));
|
||||
JSON_CHECK_ADD_ITEM(event, "windowType", cJSON_CreateStringReference(windowType));
|
||||
JSON_CHECK_ADD_ITEM(event, "windowStart", cJSON_CreateNumber(pSessionKey->win.skey));
|
||||
if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
|
||||
JSON_CHECK_ADD_ITEM(event, "windowEnd", cJSON_CreateNumber(pSessionKey->win.ekey));
|
||||
if (eventType != SNOTIFY_EVENT_WINDOW_OPEN) {
|
||||
if (strcmp(windowType, "Time") == 0) {
|
||||
JSON_CHECK_ADD_ITEM(event, "windowEnd", cJSON_CreateNumber(pSessionKey->win.ekey + 1));
|
||||
} else {
|
||||
JSON_CHECK_ADD_ITEM(event, "windowEnd", cJSON_CreateNumber(pSessionKey->win.ekey));
|
||||
}
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
terrno = code;
|
||||
cJSON_Delete(event);
|
||||
event = NULL;
|
||||
}
|
||||
return event;
|
||||
}
|
||||
|
||||
int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
const SSDataBlock* pInputBlock, const SNodeList* pCondCols, int32_t ri,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
cJSON* event = NULL;
|
||||
cJSON* fields = NULL;
|
||||
cJSON* cond = NULL;
|
||||
const SNode* pNode = NULL;
|
||||
int32_t origSize = 0;
|
||||
int64_t startTime = 0;
|
||||
int64_t endTime = 0;
|
||||
SStreamNotifyEvent item = {0};
|
||||
|
||||
QUERY_CHECK_NULL(pInputBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pInputBlock->pDataBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pCondCols, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup->windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
startTime = taosGetMonoTimestampMs();
|
||||
event = createBasicAggNotifyEvent(sup->windowType, eventType, pSessionKey);
|
||||
QUERY_CHECK_NULL(event, code, lino, _end, terrno);
|
||||
|
||||
// create fields object to store matched column values
|
||||
fields = cJSON_CreateObject();
|
||||
QUERY_CHECK_NULL(fields, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
|
||||
FOREACH(node, pCondCols) {
|
||||
SColumnNode* pColDef = (SColumnNode*)node;
|
||||
SColumnInfoData* pColData = taosArrayGet(pInputBlock->pDataBlock, pColDef->slotId);
|
||||
code = jsonAddColumnField(pColDef->colName, pColData, ri, fields);
|
||||
FOREACH(pNode, pCondCols) {
|
||||
const SColumnNode* pColDef = (const SColumnNode*)pNode;
|
||||
const SColumnInfoData* pColData = taosArrayGet(pInputBlock->pDataBlock, pColDef->slotId);
|
||||
code = jsonAddColumnField(pColDef->colName, pColData->info.type, colDataIsNull_s(pColData, ri),
|
||||
colDataGetData(pColData, ri), fields);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
|
@ -297,16 +425,24 @@ int32_t addEventAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionK
|
|||
JSON_CHECK_ADD_ITEM(cond, "conditionIndex", cJSON_CreateNumber(0));
|
||||
JSON_CHECK_ADD_ITEM(cond, "fieldValues", fields);
|
||||
fields = NULL;
|
||||
JSON_CHECK_ADD_ITEM(event, "triggerConditions", cond);
|
||||
JSON_CHECK_ADD_ITEM(event, "triggerCondition", cond);
|
||||
cond = NULL;
|
||||
|
||||
// convert json object to string value
|
||||
item.gid = pSessionKey->groupId;
|
||||
item.skey = pSessionKey->win.skey;
|
||||
item.isEnd = (eventType == SNOTIFY_EVENT_WINDOW_CLOSE);
|
||||
item.content = cJSON_PrintUnformatted(event);
|
||||
QUERY_CHECK_NULL(taosArrayPush(sup->pWindowEvents, &item), code, lino, _end, terrno);
|
||||
item.content = NULL;
|
||||
item.win = pSessionKey->win;
|
||||
item.eventType = eventType;
|
||||
item.pJson = event;
|
||||
event = NULL;
|
||||
origSize = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
item.pJson = NULL;
|
||||
|
||||
endTime = taosGetMonoTimestampMs();
|
||||
pNotifyEventStat->notifyEventAddTimes++;
|
||||
pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
|
||||
pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
|
||||
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
@ -325,50 +461,204 @@ _end:
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SSchemaWrapper* pSchemaWrapper,
|
||||
SStreamNotifyEventSupp* sup) {
|
||||
int32_t addStateAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
const SStateKeys* pCurState, const SStateKeys* pAnotherState, bool onlyUpdate,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SNode * node = NULL;
|
||||
cJSON* event = NULL;
|
||||
cJSON* result = NULL;
|
||||
int32_t origSize = 0;
|
||||
int64_t startTime = 0;
|
||||
int64_t endTime = 0;
|
||||
SStreamNotifyEvent item = {0};
|
||||
|
||||
QUERY_CHECK_NULL(pCurState, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup->windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
item.gid = pSessionKey->groupId;
|
||||
item.win = pSessionKey->win;
|
||||
item.eventType = eventType;
|
||||
// Check if the notify event exists for update
|
||||
if (onlyUpdate && taosHashGet(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE) == NULL) {
|
||||
goto _end;
|
||||
}
|
||||
|
||||
startTime = taosGetMonoTimestampMs();
|
||||
event = createBasicAggNotifyEvent(sup->windowType, eventType, pSessionKey);
|
||||
QUERY_CHECK_NULL(event, code, lino, _end, terrno);
|
||||
|
||||
// add state value
|
||||
if (eventType == SNOTIFY_EVENT_WINDOW_OPEN) {
|
||||
if (pAnotherState) {
|
||||
code = jsonAddColumnField("prevState", pAnotherState->type, pAnotherState->isNull, pAnotherState->pData, event);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
} else {
|
||||
code = jsonAddColumnField("prevState", pCurState->type, true, NULL, event);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
}
|
||||
code = jsonAddColumnField("curState", pCurState->type, pCurState->isNull, pCurState->pData, event);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (eventType == SNOTIFY_EVENT_WINDOW_CLOSE) {
|
||||
if (pAnotherState) {
|
||||
code = jsonAddColumnField("nextState", pAnotherState->type, pAnotherState->isNull, pAnotherState->pData, event);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
} else {
|
||||
code = jsonAddColumnField("nextState", pCurState->type, true, NULL, event);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
}
|
||||
|
||||
item.pJson = event;
|
||||
event = NULL;
|
||||
origSize = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
item.pJson = NULL;
|
||||
|
||||
endTime = taosGetMonoTimestampMs();
|
||||
pNotifyEventStat->notifyEventAddTimes++;
|
||||
pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
|
||||
pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
|
||||
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
destroyStreamWindowEvent(&item);
|
||||
if (event != NULL) {
|
||||
cJSON_Delete(event);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t addNormalAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
cJSON* event = NULL;
|
||||
int32_t origSize = 0;
|
||||
int64_t startTime = 0;
|
||||
int64_t endTime = 0;
|
||||
SStreamNotifyEvent item = {0};
|
||||
|
||||
QUERY_CHECK_NULL(pSessionKey, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup->windowType, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
startTime = taosGetMonoTimestampMs();
|
||||
event = createBasicAggNotifyEvent(sup->windowType, eventType, pSessionKey);
|
||||
QUERY_CHECK_NULL(event, code, lino, _end, terrno);
|
||||
|
||||
item.gid = pSessionKey->groupId;
|
||||
item.win = pSessionKey->win;
|
||||
item.eventType = eventType;
|
||||
item.pJson = event;
|
||||
event = NULL;
|
||||
origSize = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
item.pJson = NULL;
|
||||
|
||||
endTime = taosGetMonoTimestampMs();
|
||||
pNotifyEventStat->notifyEventAddTimes++;
|
||||
pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
|
||||
pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
|
||||
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
destroyStreamWindowEvent(&item);
|
||||
if (event != NULL) {
|
||||
cJSON_Delete(event);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t addIntervalAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
|
||||
return addNormalAggNotifyEvent(eventType, pSessionKey, sup, pNotifyEventStat);
|
||||
}
|
||||
|
||||
int32_t addSessionAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
|
||||
return addNormalAggNotifyEvent(eventType, pSessionKey, sup, pNotifyEventStat);
|
||||
}
|
||||
|
||||
int32_t addCountAggNotifyEvent(EStreamNotifyEventType eventType, const SSessionKey* pSessionKey,
|
||||
SStreamNotifyEventSupp* sup, STaskNotifyEventStat* pNotifyEventStat) {
|
||||
return addNormalAggNotifyEvent(eventType, pSessionKey, sup, pNotifyEventStat);
|
||||
}
|
||||
|
||||
int32_t addAggResultNotifyEvent(const SSDataBlock* pResultBlock, const SArray* pSessionKeys,
|
||||
const SSchemaWrapper* pSchemaWrapper, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
cJSON* result = NULL;
|
||||
int32_t origSize = 0;
|
||||
int64_t startTime = 0;
|
||||
int64_t endTime = 0;
|
||||
SStreamNotifyEvent item = {0};
|
||||
SColumnInfoData* pWstartCol = NULL;
|
||||
|
||||
QUERY_CHECK_NULL(pResultBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pSessionKeys, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pSchemaWrapper, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
qDebug("add %" PRId64 " stream notify results from window agg", pResultBlock->info.rows);
|
||||
startTime = taosGetMonoTimestampMs();
|
||||
origSize = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
|
||||
pWstartCol = taosArrayGet(pResultBlock->pDataBlock, 0);
|
||||
for (int32_t i = 0; i< pResultBlock->info.rows; ++i) {
|
||||
event = cJSON_CreateObject();
|
||||
QUERY_CHECK_NULL(event, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
|
||||
for (int32_t i = 0; i < pResultBlock->info.rows; ++i) {
|
||||
const SSessionKey* pSessionKey = taosArrayGet(pSessionKeys, i);
|
||||
item.gid = pSessionKey->groupId;
|
||||
item.win = pSessionKey->win;
|
||||
item.eventType = SNOTIFY_EVENT_WINDOW_CLOSE;
|
||||
SStreamNotifyEvent* pItem = taosHashGet(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE);
|
||||
if (pItem == NULL) {
|
||||
item.pJson = createBasicAggNotifyEvent(sup->windowType, SNOTIFY_EVENT_WINDOW_CLOSE, pSessionKey);
|
||||
QUERY_CHECK_NULL(item.pJson, code, lino, _end, terrno);
|
||||
if (strcmp(sup->windowType, "Event") == 0) {
|
||||
JSON_CHECK_ADD_ITEM(item.pJson, "triggerCondition", cJSON_CreateNull());
|
||||
} else if (strcmp(sup->windowType, "State") == 0) {
|
||||
JSON_CHECK_ADD_ITEM(item.pJson, "curState", cJSON_CreateNull());
|
||||
JSON_CHECK_ADD_ITEM(item.pJson, "nextState", cJSON_CreateNull());
|
||||
}
|
||||
code = taosHashPut(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE, &item, sizeof(SStreamNotifyEvent));
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
item.pJson = NULL;
|
||||
pItem = taosHashGet(sup->pWindowEventHashMap, &item, NOTIFY_EVENT_KEY_SIZE);
|
||||
QUERY_CHECK_NULL(pItem, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
|
||||
}
|
||||
|
||||
// convert the result row into json
|
||||
result = cJSON_CreateObject();
|
||||
QUERY_CHECK_NULL(result, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
|
||||
for (int32_t j = 0; j < pSchemaWrapper->nCols; ++j) {
|
||||
SSchema *pCol = pSchemaWrapper->pSchema + j;
|
||||
SColumnInfoData *pColData = taosArrayGet(pResultBlock->pDataBlock, pCol->colId - 1);
|
||||
code = jsonAddColumnField(pCol->name, pColData, i, result);
|
||||
const SSchema* pCol = pSchemaWrapper->pSchema + j;
|
||||
const SColumnInfoData* pColData = taosArrayGet(pResultBlock->pDataBlock, pCol->colId - 1);
|
||||
code = jsonAddColumnField(pCol->name, pColData->info.type, colDataIsNull_s(pColData, i),
|
||||
colDataGetData(pColData, i), result);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
JSON_CHECK_ADD_ITEM(event, "result", result);
|
||||
JSON_CHECK_ADD_ITEM(pItem->pJson, "result", result);
|
||||
result = NULL;
|
||||
|
||||
item.gid = pResultBlock->info.id.groupId;
|
||||
item.skey = *(uint64_t*)colDataGetNumData(pWstartCol, i);
|
||||
item.content = cJSON_PrintUnformatted(event);
|
||||
code = taosHashPut(sup->pResultHashMap, &item.gid, sizeof(item.gid) + sizeof(item.skey), &item, sizeof(item));
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
item.content = NULL;
|
||||
|
||||
cJSON_Delete(event);
|
||||
event = NULL;
|
||||
}
|
||||
|
||||
endTime = taosGetMonoTimestampMs();
|
||||
pNotifyEventStat->notifyEventAddTimes++;
|
||||
pNotifyEventStat->notifyEventAddElems += taosHashGetSize(sup->pWindowEventHashMap) - origSize;
|
||||
pNotifyEventStat->notifyEventAddCostSec += (endTime - startTime) / 1000.0;
|
||||
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
|
@ -377,8 +667,38 @@ _end:
|
|||
if (result != NULL) {
|
||||
cJSON_Delete(result);
|
||||
}
|
||||
if (event != NULL) {
|
||||
cJSON_Delete(event);
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t addAggDeleteNotifyEvent(const SSDataBlock* pDeleteBlock, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SSessionKey sessionKey = {0};
|
||||
SColumnInfoData* pWstartCol = NULL;
|
||||
SColumnInfoData* pWendCol = NULL;
|
||||
SColumnInfoData* pGroupIdCol = NULL;
|
||||
|
||||
QUERY_CHECK_NULL(pDeleteBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(sup, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pNotifyEventStat, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
qDebug("add %" PRId64 " stream notify delete events from window agg", pDeleteBlock->info.rows);
|
||||
|
||||
pWstartCol = taosArrayGet(pDeleteBlock->pDataBlock, START_TS_COLUMN_INDEX);
|
||||
pWendCol = taosArrayGet(pDeleteBlock->pDataBlock, END_TS_COLUMN_INDEX);
|
||||
pGroupIdCol = taosArrayGet(pDeleteBlock->pDataBlock, GROUPID_COLUMN_INDEX);
|
||||
for (int32_t i = 0; i < pDeleteBlock->info.rows; ++i) {
|
||||
sessionKey.win.skey = *(int64_t*)colDataGetNumData(pWstartCol, i);
|
||||
sessionKey.win.ekey = *(int64_t*)colDataGetNumData(pWendCol, i);
|
||||
sessionKey.groupId = *(uint64_t*)colDataGetNumData(pGroupIdCol, i);
|
||||
code = addNormalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_INVALIDATION, &sessionKey, sup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
@ -418,97 +738,42 @@ _end:
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t streamNotifyFillTableName(const char* tableName, const SStreamNotifyEvent* pEvent,
|
||||
const SStreamNotifyEvent* pResult, char** pVal) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
static const char* prefix = "{\"tableName\":\"";
|
||||
uint64_t prefixLen = 0;
|
||||
uint64_t nameLen = 0;
|
||||
uint64_t eventLen = 0;
|
||||
uint64_t resultLen = 0;
|
||||
uint64_t valLen = 0;
|
||||
char* val = NULL;
|
||||
char* p = NULL;
|
||||
|
||||
QUERY_CHECK_NULL(tableName, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pEvent, code, lino , _end, TSDB_CODE_INVALID_PARA);
|
||||
QUERY_CHECK_NULL(pVal, code, lino , _end, TSDB_CODE_INVALID_PARA);
|
||||
|
||||
*pVal = NULL;
|
||||
prefixLen = strlen(prefix);
|
||||
nameLen = strlen(tableName);
|
||||
eventLen = strlen(pEvent->content);
|
||||
|
||||
if (pResult != NULL) {
|
||||
resultLen = strlen(pResult->content);
|
||||
valLen = VARSTR_HEADER_SIZE + prefixLen + nameLen + eventLen + resultLen;
|
||||
} else {
|
||||
valLen = VARSTR_HEADER_SIZE + prefixLen + nameLen + eventLen + 1;
|
||||
}
|
||||
val = taosMemoryMalloc(valLen);
|
||||
QUERY_CHECK_NULL(val, code, lino, _end, terrno);
|
||||
varDataSetLen(val, valLen - VARSTR_HEADER_SIZE);
|
||||
|
||||
p = varDataVal(val);
|
||||
TAOS_STRNCPY(p, prefix, prefixLen);
|
||||
p += prefixLen;
|
||||
TAOS_STRNCPY(p, tableName, nameLen);
|
||||
p += nameLen;
|
||||
*(p++) = '\"';
|
||||
TAOS_STRNCPY(p, pEvent->content, eventLen);
|
||||
*p = ',';
|
||||
|
||||
if (pResult != NULL) {
|
||||
p += eventLen - 1;
|
||||
TAOS_STRNCPY(p, pResult->content, resultLen);
|
||||
*p = ',';
|
||||
}
|
||||
*pVal = val;
|
||||
val = NULL;
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
if (val != NULL) {
|
||||
taosMemoryFreeClear(val);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup) {
|
||||
int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SColumnInfoData* pEventStrCol = NULL;
|
||||
int32_t nWindowEvents = 0;
|
||||
int32_t nWindowResults = 0;
|
||||
char* val = NULL;
|
||||
SColumnInfoData* pEventStrCol = NULL;
|
||||
int64_t startTime = 0;
|
||||
int64_t endTime = 0;
|
||||
void* pIter = NULL;
|
||||
|
||||
if (pTaskInfo == NULL || sup == NULL) {
|
||||
if (pTaskInfo == NULL || sup == NULL || sup->pEventBlock == NULL || pNotifyEventStat == NULL) {
|
||||
goto _end;
|
||||
}
|
||||
|
||||
QUERY_CHECK_NULL(sup->pEventBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
blockDataCleanup(sup->pEventBlock);
|
||||
nWindowEvents = taosArrayGetSize(sup->pWindowEvents);
|
||||
nWindowResults = taosHashGetSize(sup->pResultHashMap);
|
||||
qDebug("start to build stream notify event block, nWindowEvents: %d, nWindowResults: %d", nWindowEvents,
|
||||
nWindowResults);
|
||||
if (nWindowEvents == 0) {
|
||||
goto _end;
|
||||
}
|
||||
|
||||
code = blockDataEnsureCapacity(sup->pEventBlock, nWindowEvents);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
startTime = taosGetMonoTimestampMs();
|
||||
blockDataCleanup(sup->pEventBlock);
|
||||
nWindowEvents = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
qDebug("start to build stream notify event block, nWindowEvents: %d", nWindowEvents);
|
||||
|
||||
pEventStrCol = taosArrayGet(sup->pEventBlock->pDataBlock, NOTIFY_EVENT_STR_COLUMN_INDEX);
|
||||
QUERY_CHECK_NULL(pEventStrCol, code, lino, _end, terrno);
|
||||
|
||||
for (int32_t i = 0; i < nWindowEvents; ++i) {
|
||||
SStreamNotifyEvent* pResult = NULL;
|
||||
SStreamNotifyEvent* pEvent = taosArrayGet(sup->pWindowEvents, i);
|
||||
char* tableName = taosHashGet(sup->pTableNameHashMap, &pEvent->gid, sizeof(pEvent->gid));
|
||||
// Append all events content into data block.
|
||||
pIter = taosHashIterate(sup->pWindowEventHashMap, NULL);
|
||||
while (pIter) {
|
||||
const SStreamNotifyEvent* pEvent = (const SStreamNotifyEvent*)pIter;
|
||||
pIter = taosHashIterate(sup->pWindowEventHashMap, pIter);
|
||||
if (pEvent->eventType == SNOTIFY_EVENT_WINDOW_CLOSE && !cJSON_HasObjectItem(pEvent->pJson, "result")) {
|
||||
// current WINDOW_CLOSE event cannot be pushed yet due to watermark
|
||||
continue;
|
||||
}
|
||||
|
||||
// get name of the dest child table
|
||||
char* tableName = taosHashGet(sup->pTableNameHashMap, &pEvent->gid, sizeof(&pEvent->gid));
|
||||
if (tableName == NULL) {
|
||||
code = streamNotifyGetDestTableName(pTaskInfo, pEvent->gid, &tableName);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
@ -518,32 +783,73 @@ int32_t buildNotifyEventBlock(const SExecTaskInfo* pTaskInfo, SStreamNotifyEvent
|
|||
tableName = taosHashGet(sup->pTableNameHashMap, &pEvent->gid, sizeof(pEvent->gid));
|
||||
QUERY_CHECK_NULL(tableName, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
|
||||
}
|
||||
if (pEvent->isEnd) {
|
||||
pResult = taosHashGet(sup->pResultHashMap, &pEvent->gid, sizeof(pEvent->gid) + sizeof(pEvent->skey));
|
||||
QUERY_CHECK_NULL(pResult, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
|
||||
}
|
||||
code = streamNotifyFillTableName(tableName, pEvent, pResult, &val);
|
||||
JSON_CHECK_ADD_ITEM(pEvent->pJson, "tableName", cJSON_CreateStringReference(tableName));
|
||||
|
||||
// convert the json object into string and append it into the block
|
||||
char* str = cJSON_PrintUnformatted(pEvent->pJson);
|
||||
QUERY_CHECK_NULL(str, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
|
||||
int32_t len = strlen(str);
|
||||
code = varColSetVarData(pEventStrCol, sup->pEventBlock->info.rows, str, len, false);
|
||||
cJSON_free(str);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
code = colDataSetVal(pEventStrCol, i, val, false);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
taosMemoryFreeClear(val);
|
||||
sup->pEventBlock->info.rows++;
|
||||
code = taosHashRemove(sup->pWindowEventHashMap, pEvent, NOTIFY_EVENT_KEY_SIZE);
|
||||
if (code == TSDB_CODE_NOT_FOUND) {
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
}
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (sup->pEventBlock->info.rows >= sup->pEventBlock->info.capacity) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (taosHashGetMemSize(sup->pTableNameHashMap) >= NOTIFY_EVENT_NAME_CACHE_LIMIT_MB * 1024 * 1024) {
|
||||
taosHashClear(sup->pTableNameHashMap);
|
||||
}
|
||||
|
||||
endTime = taosGetMonoTimestampMs();
|
||||
if (sup->pEventBlock->info.rows > 0) {
|
||||
pNotifyEventStat->notifyEventPushTimes++;
|
||||
pNotifyEventStat->notifyEventPushElems += sup->pEventBlock->info.rows;
|
||||
pNotifyEventStat->notifyEventPushCostSec += (endTime - startTime) / 1000.0;
|
||||
}
|
||||
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
if (val != NULL) {
|
||||
taosMemoryFreeClear(val);
|
||||
}
|
||||
if (sup != NULL) {
|
||||
taosArrayClearEx(sup->pWindowEvents, destroyStreamWindowEvent);
|
||||
taosHashClear(sup->pResultHashMap);
|
||||
if (pIter) {
|
||||
taosHashCancelIterate(sup->pWindowEventHashMap, pIter);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t removeOutdatedNotifyEvents(STimeWindowAggSupp* pTwSup, SStreamNotifyEventSupp* sup,
|
||||
STaskNotifyEventStat* pNotifyEventStat) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
void* pIter = NULL;
|
||||
|
||||
if (pTwSup || sup == NULL || pNotifyEventStat == NULL) {
|
||||
goto _end;
|
||||
}
|
||||
|
||||
pIter = taosHashIterate(sup->pWindowEventHashMap, NULL);
|
||||
while (pIter) {
|
||||
const SStreamNotifyEvent* pEvent = (const SStreamNotifyEvent*)pIter;
|
||||
pIter = taosHashIterate(sup->pWindowEventHashMap, pIter);
|
||||
if (isOverdue(pEvent->win.ekey, pTwSup)) {
|
||||
code = taosHashRemove(sup->pWindowEventHashMap, pEvent, NOTIFY_EVENT_KEY_SIZE);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
}
|
||||
|
||||
pNotifyEventStat->notifyEventHoldElems = taosHashGetSize(sup->pWindowEventHashMap);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
|
|
@ -87,23 +87,48 @@ static int32_t buildIntervalSliceResult(SOperatorInfo* pOperator, SSDataBlock**
|
|||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
uint16_t opType = pOperator->operatorType;
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
bool addNotifyEvent = false;
|
||||
addNotifyEvent = IS_NORMAL_INTERVAL_OP(pOperator) &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
|
||||
doBuildDeleteResultImpl(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pState, pInfo->pDelWins, &pInfo->delIndex,
|
||||
pInfo->pDelRes);
|
||||
if (pInfo->pDelRes->info.rows != 0) {
|
||||
// process the rest of the data
|
||||
printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->pDelRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
doBuildStreamIntervalResult(pOperator, pInfo->streamAggSup.pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
|
||||
doBuildStreamIntervalResult(pOperator, pInfo->streamAggSup.pState, pInfo->binfo.pRes, &pInfo->groupResInfo,
|
||||
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
|
||||
if (pInfo->binfo.pRes->info.rows != 0) {
|
||||
printDataBlock(pInfo->binfo.pRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggResultNotifyEvent(pInfo->binfo.pRes, pNotifySup->pSessionKeys,
|
||||
pTaskInfo->streamInfo.notifyResultSchema, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->binfo.pRes;
|
||||
goto _end;
|
||||
}
|
||||
|
||||
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
|
||||
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(opType), GET_TASKID(pTaskInfo));
|
||||
(*ppRes) = pNotifySup->pEventBlock;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
|
@ -316,6 +341,14 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc
|
|||
code = setIntervalSliceOutputBuf(&curPoint, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator) &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
|
||||
SSessionKey key = {.win = curWin, .groupId = groupId};
|
||||
code = addIntervalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &key, &pInfo->basic.notifyEventSup,
|
||||
pTaskInfo->streamInfo.pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
resetIntervalSliceFunctionKey(pSup->pCtx, numOfOutput);
|
||||
if (pInfo->hasInterpoFunc && IS_VALID_WIN_KEY(prevPoint.winKey.win.skey) && curPoint.winKey.win.skey != curTs) {
|
||||
doStreamSliceInterpolation(prevPoint.pLastRow, curPoint.winKey.win.skey, curTs, pBlock, startPos, &pOperator->exprSupp, INTERVAL_SLICE_START, pInfo->pOffsetInfo);
|
||||
|
@ -652,8 +685,9 @@ int32_t createStreamIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiN
|
|||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamIntervalSliceReleaseState, streamIntervalSliceReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic);
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
if (downstream) {
|
||||
code = initIntervalSliceDownStream(downstream, &pInfo->streamAggSup, pPhyNode->type, pInfo->primaryTsIndex,
|
||||
&pInfo->twAggSup, &pInfo->basic, &pInfo->interval, pInfo->hasInterpoFunc);
|
||||
|
|
|
@ -150,7 +150,6 @@ void destroyStreamTimeSliceOperatorInfo(void* param) {
|
|||
&pInfo->groupResInfo);
|
||||
pInfo->pOperator = NULL;
|
||||
}
|
||||
destroyStreamBasicInfo(&pInfo->basic);
|
||||
colDataDestroy(&pInfo->twAggSup.timeWindowData);
|
||||
destroyStreamAggSupporter(&pInfo->streamAggSup);
|
||||
resetPrevAndNextWindow(pInfo->pFillSup);
|
||||
|
@ -2202,7 +2201,7 @@ int32_t createStreamTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
|
|||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamTimeSliceReleaseState, streamTimeSliceReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic);
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
if (downstream) {
|
||||
code = initTimeSliceDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
|
||||
|
|
|
@ -32,11 +32,6 @@
|
|||
#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL)
|
||||
|
||||
#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
|
||||
#define IS_NORMAL_SESSION_OP(op) \
|
||||
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \
|
||||
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
|
||||
|
||||
#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE)
|
||||
|
||||
#define DEAULT_DELETE_MARK INT64_MAX
|
||||
#define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState"
|
||||
|
@ -480,6 +475,8 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) {
|
|||
false);
|
||||
pInfo->pOperator = NULL;
|
||||
}
|
||||
|
||||
destroyStreamBasicInfo(&pInfo->basic);
|
||||
cleanupAggSup(&pInfo->aggSup);
|
||||
clearGroupResInfo(&pInfo->groupResInfo);
|
||||
taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);
|
||||
|
@ -917,7 +914,7 @@ int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SSta
|
|||
}
|
||||
|
||||
void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,
|
||||
SGroupResInfo* pGroupResInfo) {
|
||||
SGroupResInfo* pGroupResInfo, SArray* pSessionKeys) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
@ -991,6 +988,14 @@ void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBl
|
|||
}
|
||||
}
|
||||
|
||||
if (pSessionKeys) {
|
||||
SSessionKey key = {.groupId = groupId, .win = pRow->win};
|
||||
for (int32_t j = 0; j < pRow->numOfRows; ++j) {
|
||||
const void* px = taosArrayPush(pSessionKeys, &key);
|
||||
QUERY_CHECK_NULL(px, code, lino, _end, terrno);
|
||||
}
|
||||
}
|
||||
|
||||
pBlock->info.rows += pRow->numOfRows;
|
||||
}
|
||||
|
||||
|
@ -1005,19 +1010,20 @@ _end:
|
|||
}
|
||||
|
||||
void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
|
||||
SGroupResInfo* pGroupResInfo) {
|
||||
SGroupResInfo* pGroupResInfo, SArray* pSessionKeys) {
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
// set output datablock version
|
||||
pBlock->info.version = pTaskInfo->version;
|
||||
|
||||
blockDataCleanup(pBlock);
|
||||
taosArrayClear(pSessionKeys);
|
||||
if (!hasRemainResults(pGroupResInfo)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// clear the existed group id
|
||||
pBlock->info.id.groupId = 0;
|
||||
buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo);
|
||||
buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);
|
||||
}
|
||||
|
||||
static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
|
||||
|
@ -1150,6 +1156,14 @@ static int32_t doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS
|
|||
pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->stateStore, &winCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator) &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
|
||||
SSessionKey key = {.win = nextWin, .groupId = groupId};
|
||||
code = addIntervalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &key, &pInfo->basic.notifyEventSup,
|
||||
pTaskInfo->streamInfo.pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
pResult = (SResultRow*)pResPos->pRowBuff;
|
||||
|
||||
if (IS_FINAL_INTERVAL_OP(pOperator)) {
|
||||
|
@ -1371,7 +1385,10 @@ int32_t doStreamIntervalEncodeOpState(void** buf, int32_t len, SOperatorInfo* pO
|
|||
// 5.dataVersion
|
||||
tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);
|
||||
|
||||
// 6.checksum
|
||||
// 6.basicInfo
|
||||
tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
|
||||
|
||||
// 7.checksum
|
||||
if (buf) {
|
||||
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
|
||||
tlen += taosEncodeFixedU32(buf, cksum);
|
||||
|
@ -1387,18 +1404,20 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
int32_t lino = 0;
|
||||
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
void* pDataEnd = POINTER_SHIFT(buf, len);
|
||||
if (!pInfo) {
|
||||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
// 6.checksum
|
||||
// 7.checksum
|
||||
int32_t dataLen = len - sizeof(uint32_t);
|
||||
void* pCksum = POINTER_SHIFT(buf, dataLen);
|
||||
if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {
|
||||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
pDataEnd = pCksum;
|
||||
|
||||
// 1.pResultRowHashTable
|
||||
int32_t mapSize = 0;
|
||||
|
@ -1454,6 +1473,12 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
// 5.dataVersion
|
||||
buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);
|
||||
|
||||
// 6.basicInfo
|
||||
if (buf < pDataEnd) {
|
||||
code = decodeStreamBasicInfo(&buf, &pInfo->basic);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
|
@ -1503,8 +1528,12 @@ _end:
|
|||
static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
|
||||
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
uint16_t opType = pOperator->operatorType;
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
bool addNotifyEvent = false;
|
||||
|
||||
// check if query task is closed or not
|
||||
if (isTaskKilled(pTaskInfo)) {
|
||||
|
@ -1512,6 +1541,8 @@ static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes
|
|||
return code;
|
||||
}
|
||||
|
||||
addNotifyEvent = IS_NORMAL_INTERVAL_OP(pOperator) &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
|
||||
if (IS_FINAL_INTERVAL_OP(pOperator)) {
|
||||
doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
|
||||
if (pInfo->pPullDataRes->info.rows != 0) {
|
||||
|
@ -1526,17 +1557,42 @@ static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes
|
|||
if (pInfo->pDelRes->info.rows != 0) {
|
||||
// process the rest of the data
|
||||
printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->pDelRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo);
|
||||
doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo,
|
||||
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
|
||||
if (pInfo->binfo.pRes->info.rows != 0) {
|
||||
printDataBlock(pInfo->binfo.pRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggResultNotifyEvent(pInfo->binfo.pRes, pNotifySup->pSessionKeys,
|
||||
pTaskInfo->streamInfo.notifyResultSchema, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->binfo.pRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
|
||||
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
(*ppRes) = pNotifySup->pEventBlock;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
}
|
||||
(*ppRes) = NULL;
|
||||
return code;
|
||||
}
|
||||
|
@ -2023,7 +2079,7 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN
|
|||
pInfo->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;
|
||||
pInfo->pState->pResultRowStore.resultRowPut = putResultRowToBuf;
|
||||
pInfo->pState->pExprSupp = &pOperator->exprSupp;
|
||||
|
||||
|
||||
code =
|
||||
pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,
|
||||
compareTs, pInfo->pState, pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo),
|
||||
|
@ -2069,6 +2125,10 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN
|
|||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
}
|
||||
setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
|
||||
pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
|
||||
pInfo->basic.primaryPkIndex = -1;
|
||||
|
@ -2120,6 +2180,8 @@ void destroyStreamSessionAggOperatorInfo(void* param) {
|
|||
&pInfo->groupResInfo);
|
||||
pInfo->pOperator = NULL;
|
||||
}
|
||||
|
||||
destroyStreamBasicInfo(&pInfo->basic);
|
||||
destroyStreamAggSupporter(&pInfo->streamAggSup);
|
||||
cleanupExprSupp(&pInfo->scalarSupp);
|
||||
clearGroupResInfo(&pInfo->groupResInfo);
|
||||
|
@ -2248,8 +2310,8 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, in
|
|||
}
|
||||
|
||||
if (stateType == STREAM_STATE_BUFF_SORT) {
|
||||
pSup->pState->pFileState = NULL;
|
||||
code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize,
|
||||
pSup->pState->pFileState = NULL;
|
||||
code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize,
|
||||
funResSize, sesionTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr,
|
||||
pHandle->checkpointId, stateType, &pSup->pState->pFileState);
|
||||
} else if (stateType == STREAM_STATE_BUFF_HASH_SORT || stateType == STREAM_STATE_BUFF_HASH_SEARCH) {
|
||||
|
@ -2309,24 +2371,23 @@ bool inWinRange(STimeWindow* range, STimeWindow* cur) {
|
|||
void clearOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { pAPI->streamStateClearBuff(pState, pPos); }
|
||||
|
||||
int32_t setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
|
||||
SResultWindowInfo* pCurWin) {
|
||||
SResultWindowInfo* pCurWin, int32_t* pWinCode) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
pCurWin->sessionWin.groupId = groupId;
|
||||
pCurWin->sessionWin.win.skey = startTs;
|
||||
pCurWin->sessionWin.win.ekey = endTs;
|
||||
int32_t size = pAggSup->resultRowSize;
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap,
|
||||
(void**)&pCurWin->pStatePos, &size, &winCode);
|
||||
(void**)&pCurWin->pStatePos, &size, pWinCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) {
|
||||
winCode = TSDB_CODE_FAILED;
|
||||
if (*pWinCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) {
|
||||
*pWinCode = TSDB_CODE_FAILED;
|
||||
clearOutputBuf(pAggSup->pState, pCurWin->pStatePos, &pAggSup->pSessionAPI->stateStore);
|
||||
}
|
||||
|
||||
if (winCode == TSDB_CODE_SUCCESS) {
|
||||
if (*pWinCode == TSDB_CODE_SUCCESS) {
|
||||
pCurWin->isOutput = true;
|
||||
if (pCurWin->pStatePos->needFree) {
|
||||
pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->sessionWin);
|
||||
|
@ -2692,9 +2753,17 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
|
|||
continue;
|
||||
}
|
||||
SResultWindowInfo winInfo = {0};
|
||||
code = setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo);
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
code = setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo, &winCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_SESSION_OP(pOperator) &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
|
||||
code = addSessionAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &winInfo.sessionWin, &pInfo->basic.notifyEventSup,
|
||||
pTaskInfo->streamInfo.pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
// coverity scan error
|
||||
if (!winInfo.pStatePos) {
|
||||
continue;
|
||||
|
@ -2908,7 +2977,9 @@ static int32_t rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray,
|
|||
|
||||
if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pWinKey->win, &childWin.sessionWin.win)) {
|
||||
if (num == 0) {
|
||||
code = setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin);
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
code = setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin,
|
||||
&winCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
parentWin.sessionWin = childWin.sessionWin;
|
||||
|
@ -3051,7 +3122,7 @@ void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayL
|
|||
}
|
||||
|
||||
int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,
|
||||
SGroupResInfo* pGroupResInfo) {
|
||||
SGroupResInfo* pGroupResInfo, SArray* pSessionKeys) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
@ -3131,6 +3202,13 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa
|
|||
}
|
||||
}
|
||||
|
||||
if (pSessionKeys) {
|
||||
for (int32_t j = 0; j < pRow->numOfRows; ++j) {
|
||||
const void* px = taosArrayPush(pSessionKeys, pKey);
|
||||
QUERY_CHECK_NULL(px, code, lino, _end, terrno);
|
||||
}
|
||||
}
|
||||
|
||||
pBlock->info.dataLoad = 1;
|
||||
pBlock->info.rows += pRow->numOfRows;
|
||||
}
|
||||
|
@ -3144,7 +3222,8 @@ _end:
|
|||
return code;
|
||||
}
|
||||
|
||||
void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock) {
|
||||
void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock,
|
||||
SArray* pSessionKeys) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
@ -3152,6 +3231,7 @@ void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo*
|
|||
pBlock->info.version = pTaskInfo->version;
|
||||
|
||||
blockDataCleanup(pBlock);
|
||||
taosArrayClear(pSessionKeys);
|
||||
if (!hasRemainResults(pGroupResInfo)) {
|
||||
cleanupGroupResInfo(pGroupResInfo);
|
||||
goto _end;
|
||||
|
@ -3159,7 +3239,7 @@ void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo*
|
|||
|
||||
// clear the existed group id
|
||||
pBlock->info.id.groupId = 0;
|
||||
code = buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo);
|
||||
code = buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (pBlock->info.rows == 0) {
|
||||
|
@ -3174,23 +3254,60 @@ _end:
|
|||
|
||||
static int32_t buildSessionResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
SOptrBasicInfo* pBInfo = &pInfo->binfo;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
bool addNotifyEvent = false;
|
||||
addNotifyEvent = IS_NORMAL_SESSION_OP(pOperator) &&
|
||||
BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
|
||||
doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->pDelRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes);
|
||||
doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes,
|
||||
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
|
||||
if (pBInfo->pRes->info.rows > 0) {
|
||||
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
// Adjust the window end time based on the Session Window gap
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pNotifySup->pSessionKeys); ++i) {
|
||||
SSessionKey* pKey = taosArrayGet(pNotifySup->pSessionKeys, i);
|
||||
pKey->win.ekey += pAggSup->gap;
|
||||
}
|
||||
code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
|
||||
pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pBInfo->pRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
|
||||
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
(*ppRes) = pNotifySup->pEventBlock;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
}
|
||||
(*ppRes) = NULL;
|
||||
return code;
|
||||
}
|
||||
|
@ -3295,7 +3412,10 @@ int32_t doStreamSessionEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOp
|
|||
// 4.dataVersion
|
||||
tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);
|
||||
|
||||
// 5.checksum
|
||||
// 5.basicInfo
|
||||
tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
|
||||
|
||||
// 6.checksum
|
||||
if (isParent) {
|
||||
if (buf) {
|
||||
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
|
||||
|
@ -3313,13 +3433,14 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe
|
|||
int32_t lino = 0;
|
||||
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
void* pDataEnd = POINTER_SHIFT(buf, len);
|
||||
if (!pInfo) {
|
||||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
|
||||
// 5.checksum
|
||||
// 6.checksum
|
||||
if (isParent) {
|
||||
int32_t dataLen = len - sizeof(uint32_t);
|
||||
void* pCksum = POINTER_SHIFT(buf, dataLen);
|
||||
|
@ -3328,6 +3449,7 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe
|
|||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
pDataEnd = pCksum;
|
||||
}
|
||||
|
||||
// 1.streamAggSup.pResultRows
|
||||
|
@ -3366,6 +3488,12 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe
|
|||
(*ppBuf) = buf;
|
||||
}
|
||||
|
||||
// 5.basicInfo
|
||||
if (buf < pDataEnd) {
|
||||
code = decodeStreamBasicInfo(&buf, &pInfo->basic);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
|
@ -3926,6 +4054,9 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode
|
|||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
if (downstream) {
|
||||
pInfo->basic.primaryPkIndex = -1;
|
||||
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
|
||||
|
@ -4162,6 +4293,10 @@ int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhys
|
|||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionSemiReloadState);
|
||||
}
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), pPhyNode->type, false, OP_NOT_OPENED, pInfo,
|
||||
pTaskInfo);
|
||||
|
||||
|
@ -4230,6 +4365,8 @@ void destroyStreamStateOperatorInfo(void* param) {
|
|||
&pInfo->groupResInfo);
|
||||
pInfo->pOperator = NULL;
|
||||
}
|
||||
|
||||
destroyStreamBasicInfo(&pInfo->basic);
|
||||
destroyStreamAggSupporter(&pInfo->streamAggSup);
|
||||
clearGroupResInfo(&pInfo->groupResInfo);
|
||||
taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);
|
||||
|
@ -4284,12 +4421,39 @@ bool compareWinStateKey(SStateKeys* left, SStateKeys* right) {
|
|||
return compareVal(left->pData, right);
|
||||
}
|
||||
|
||||
static void getNextStateWin(const SStreamAggSupporter* pAggSup, SStateWindowInfo* pNextWin, bool asc) {
|
||||
SStreamStateCur* pCur = NULL;
|
||||
|
||||
if (pAggSup == NULL || pNextWin == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (asc)
|
||||
pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);
|
||||
else
|
||||
pCur = pAggSup->stateStore.streamStateSessionSeekKeyPrev(pAggSup->pState, &pNextWin->winInfo.sessionWin);
|
||||
int32_t nextSize = pAggSup->resultRowSize;
|
||||
int32_t winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,
|
||||
(void**)&pNextWin->winInfo.pStatePos, &nextSize);
|
||||
if (winCode != TSDB_CODE_SUCCESS) {
|
||||
SET_SESSION_WIN_INVALID(pNextWin->winInfo);
|
||||
} else {
|
||||
pNextWin->pStateKey =
|
||||
(SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));
|
||||
pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);
|
||||
pNextWin->pStateKey->type = pAggSup->stateKeyType;
|
||||
pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);
|
||||
pNextWin->pStateKey->isNull = false;
|
||||
pNextWin->winInfo.isOutput = true;
|
||||
}
|
||||
pAggSup->stateStore.streamStateFreeCur(pCur);
|
||||
}
|
||||
|
||||
int32_t getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SStateWindowInfo* pCurWin,
|
||||
SStateWindowInfo* pNextWin) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SStreamStateCur* pCur = NULL;
|
||||
int32_t size = pAggSup->resultRowSize;
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
int32_t size = pAggSup->resultRowSize;
|
||||
pCurWin->winInfo.sessionWin.groupId = pKey->groupId;
|
||||
pCurWin->winInfo.sessionWin.win.skey = pKey->win.skey;
|
||||
pCurWin->winInfo.sessionWin.win.ekey = pKey->win.ekey;
|
||||
|
@ -4313,24 +4477,9 @@ int32_t getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey,
|
|||
pCurWin->winInfo.sessionWin.win.ekey);
|
||||
|
||||
pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;
|
||||
pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);
|
||||
int32_t nextSize = pAggSup->resultRowSize;
|
||||
int32_t winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,
|
||||
(void**)&pNextWin->winInfo.pStatePos, &nextSize);
|
||||
if (winCode != TSDB_CODE_SUCCESS) {
|
||||
SET_SESSION_WIN_INVALID(pNextWin->winInfo);
|
||||
} else {
|
||||
pNextWin->pStateKey =
|
||||
(SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));
|
||||
pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);
|
||||
pNextWin->pStateKey->type = pAggSup->stateKeyType;
|
||||
pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);
|
||||
pNextWin->pStateKey->isNull = false;
|
||||
pNextWin->winInfo.isOutput = true;
|
||||
}
|
||||
getNextStateWin(pAggSup, pNextWin, true);
|
||||
|
||||
_end:
|
||||
pAggSup->stateStore.streamStateFreeCur(pCur);
|
||||
qDebug("===stream===get state next win buff. skey:%" PRId64 ", endkey:%" PRId64,
|
||||
pNextWin->winInfo.sessionWin.win.skey, pNextWin->winInfo.sessionWin.win.ekey);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
@ -4340,9 +4489,8 @@ _end:
|
|||
}
|
||||
|
||||
int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,
|
||||
SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin) {
|
||||
int32_t size = pAggSup->resultRowSize;
|
||||
SStreamStateCur* pCur = NULL;
|
||||
SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin, int32_t* pWinCode) {
|
||||
int32_t size = pAggSup->resultRowSize;
|
||||
pCurWin->winInfo.sessionWin.groupId = groupId;
|
||||
pCurWin->winInfo.sessionWin.win.skey = ts;
|
||||
pCurWin->winInfo.sessionWin.win.ekey = ts;
|
||||
|
@ -4390,29 +4538,16 @@ int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group
|
|||
}
|
||||
}
|
||||
|
||||
*pWinCode = winCode;
|
||||
|
||||
qDebug("===stream===set state cur win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,
|
||||
pCurWin->winInfo.sessionWin.win.ekey);
|
||||
|
||||
pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;
|
||||
pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);
|
||||
int32_t nextSize = pAggSup->resultRowSize;
|
||||
winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,
|
||||
(void**)&pNextWin->winInfo.pStatePos, &nextSize);
|
||||
if (winCode != TSDB_CODE_SUCCESS) {
|
||||
SET_SESSION_WIN_INVALID(pNextWin->winInfo);
|
||||
} else {
|
||||
pNextWin->pStateKey =
|
||||
(SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));
|
||||
pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);
|
||||
pNextWin->pStateKey->type = pAggSup->stateKeyType;
|
||||
pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);
|
||||
pNextWin->pStateKey->isNull = false;
|
||||
pNextWin->winInfo.isOutput = true;
|
||||
}
|
||||
getNextStateWin(pAggSup, pNextWin, true);
|
||||
qDebug("===stream===set state next win buff. skey:%" PRId64 ", endkey:%" PRId64,
|
||||
pNextWin->winInfo.sessionWin.win.skey, pNextWin->winInfo.sessionWin.win.ekey);
|
||||
_end:
|
||||
pAggSup->stateStore.streamStateFreeCur(pCur);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
|
@ -4472,7 +4607,7 @@ static bool isWinResult(SSessionKey* pKey, SSHashObj* pSeUpdate, SSHashObj* pRes
|
|||
if (tSimpleHashGet(pSeUpdate, &checkKey, sizeof(SSessionKey)) != NULL) {
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
if (tSimpleHashGet(pResults, &checkKey, sizeof(SSessionKey)) != NULL) {
|
||||
return true;
|
||||
}
|
||||
|
@ -4493,6 +4628,8 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
SResultRow* pResult = NULL;
|
||||
int32_t winRows = 0;
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
|
||||
pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
|
||||
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
|
||||
|
@ -4528,9 +4665,31 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
bool allEqual = true;
|
||||
SStateWindowInfo curWin = {0};
|
||||
SStateWindowInfo nextWin = {0};
|
||||
code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin);
|
||||
int32_t winCode = TSDB_CODE_SUCCESS;
|
||||
code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin, &winCode);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
if (winCode != TSDB_CODE_SUCCESS && pTaskInfo->streamInfo.eventTypes) {
|
||||
SStateWindowInfo prevWin = {.winInfo.sessionWin = curWin.winInfo.sessionWin};
|
||||
getNextStateWin(pAggSup, &prevWin, false);
|
||||
qDebug("===stream===get state prev win buff. skey:%" PRId64 ", endkey:%" PRId64,
|
||||
prevWin.winInfo.sessionWin.win.skey, prevWin.winInfo.sessionWin.win.ekey);
|
||||
releaseOutputBuf(pAggSup->pState, prevWin.winInfo.pStatePos, &pAPI->stateStore);
|
||||
// For ordered data, the previous window's closure did not record the corresponding state values, so they need to
|
||||
// be added here.
|
||||
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE) &&
|
||||
IS_VALID_SESSION_WIN(prevWin.winInfo)) {
|
||||
code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &prevWin.winInfo.sessionWin, prevWin.pStateKey,
|
||||
curWin.pStateKey, true, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {
|
||||
code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, curWin.pStateKey,
|
||||
prevWin.pStateKey, false, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
}
|
||||
|
||||
if (isWinResult(&nextWin.winInfo.sessionWin, pSeUpdated, pAggSup->pResultRows) == false) {
|
||||
releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore);
|
||||
}
|
||||
|
@ -4578,6 +4737,14 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
// If this is a windown recalculation, add the corresponding state values here since the next window may not require
|
||||
// recalculation.
|
||||
if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {
|
||||
code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &curWin.winInfo.sessionWin, curWin.pStateKey,
|
||||
nextWin.pStateKey, false, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
}
|
||||
|
||||
_end:
|
||||
|
@ -4621,7 +4788,10 @@ int32_t doStreamStateEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
|
|||
// 4.dataVersion
|
||||
tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);
|
||||
|
||||
// 5.checksum
|
||||
// 5.basicInfo
|
||||
tlen += encodeStreamBasicInfo(buf, &pInfo->basic);
|
||||
|
||||
// 6.checksum
|
||||
if (isParent) {
|
||||
if (buf) {
|
||||
uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));
|
||||
|
@ -4640,12 +4810,13 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
int32_t lino = 0;
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
void* pDataEnd = POINTER_SHIFT(buf, len);
|
||||
if (!pInfo) {
|
||||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
// 5.checksum
|
||||
// 6.checksum
|
||||
if (isParent) {
|
||||
int32_t dataLen = len - sizeof(uint32_t);
|
||||
void* pCksum = POINTER_SHIFT(buf, dataLen);
|
||||
|
@ -4654,6 +4825,7 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
code = TSDB_CODE_FAILED;
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
pDataEnd = pCksum;
|
||||
}
|
||||
|
||||
// 1.streamAggSup.pResultRows
|
||||
|
@ -4693,6 +4865,12 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
|
|||
(*ppBuf) = buf;
|
||||
}
|
||||
|
||||
// 5.basicInfo
|
||||
if (buf < pDataEnd) {
|
||||
code = decodeStreamBasicInfo(&buf, &pInfo->basic);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
|
@ -4720,23 +4898,53 @@ void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) {
|
|||
|
||||
static int32_t buildStateResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
|
||||
SOptrBasicInfo* pBInfo = &pInfo->binfo;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
||||
SStreamNotifyEventSupp* pNotifySup = &pInfo->basic.notifyEventSup;
|
||||
STaskNotifyEventStat* pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;
|
||||
bool addNotifyEvent = false;
|
||||
addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);
|
||||
doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pInfo->pDelRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes);
|
||||
doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes,
|
||||
addNotifyEvent ? pNotifySup->pSessionKeys : NULL);
|
||||
if (pBInfo->pRes->info.rows > 0) {
|
||||
printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
if (addNotifyEvent) {
|
||||
code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,
|
||||
pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
(*ppRes) = pBInfo->pRes;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {
|
||||
printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
|
||||
(*ppRes) = pNotifySup->pEventBlock;
|
||||
return code;
|
||||
}
|
||||
|
||||
code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);
|
||||
QUERY_CHECK_CODE(code, lino, _end);
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));
|
||||
}
|
||||
(*ppRes) = NULL;
|
||||
return code;
|
||||
}
|
||||
|
@ -5122,6 +5330,10 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
|
|||
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAggNext, NULL, destroyStreamStateOperatorInfo,
|
||||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
|
||||
&pInfo->twAggSup, &pInfo->basic);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
@ -5331,8 +5543,9 @@ _end:
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,
|
||||
SReadHandle* pHandle, SOperatorInfo** pOptrInfo) {
|
||||
static int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
|
||||
SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
|
||||
SOperatorInfo** pOptrInfo) {
|
||||
QRY_PARAM_CHECK(pOptrInfo);
|
||||
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
@ -5455,6 +5668,9 @@ static int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream,
|
|||
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
|
||||
setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);
|
||||
|
||||
code = initStreamBasicInfo(&pInfo->basic, pOperator);
|
||||
QUERY_CHECK_CODE(code, lino, _error);
|
||||
|
||||
pInfo->recvGetAll = false;
|
||||
|
||||
code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);
|
||||
|
|
|
@ -11420,7 +11420,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
|
|||
!hasTbnameFunction(pSelect->pPartitionByList) && pSelect->pWindow != NULL &&
|
||||
pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW) {
|
||||
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
|
||||
"Event window for stream on super table must patitioned by table name");
|
||||
"Event window for stream on super table must partitioned by table name");
|
||||
}
|
||||
|
||||
if (pSelect->pWindow != NULL && pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW &&
|
||||
|
|
|
@ -562,6 +562,33 @@ SStreamStateCur* sessionWinStateSeekKeyCurrentPrev(SStreamFileState* pFileState,
|
|||
pCur->pStreamFileState = pFileState;
|
||||
return pCur;
|
||||
}
|
||||
|
||||
SStreamStateCur* sessionWinStateSeekKeyPrev(SStreamFileState *pFileState, const SSessionKey *pWinKey) {
|
||||
SArray* pWinStates = NULL;
|
||||
int32_t index = -1;
|
||||
SStreamStateCur *pCur = seekKeyCurrentPrev_buff(pFileState, pWinKey, &pWinStates, &index);
|
||||
if (pCur) {
|
||||
int32_t cmpRes= sessionStateRangeKeyCompare(pWinKey, pWinStates, index);
|
||||
if (cmpRes > 0) {
|
||||
return pCur;
|
||||
} else if (cmpRes == 0 && index > 0) {
|
||||
sessionWinStateMoveToPrev(pCur);
|
||||
return pCur;
|
||||
}
|
||||
streamStateFreeCur(pCur);
|
||||
pCur = NULL;
|
||||
}
|
||||
|
||||
void* pFileStore = getStateFileStore(pFileState);
|
||||
pCur = streamStateSessionSeekKeyPrev_rocksdb(pFileStore, pWinKey);
|
||||
if (!pCur) {
|
||||
return NULL;
|
||||
}
|
||||
pCur->buffIndex = -1;
|
||||
pCur->pStreamFileState = pFileState;
|
||||
return pCur;
|
||||
}
|
||||
|
||||
static void transformCursor(SStreamFileState* pFileState, SStreamStateCur* pCur) {
|
||||
if (!pCur) {
|
||||
return;
|
||||
|
@ -747,6 +774,15 @@ void sessionWinStateMoveToNext(SStreamStateCur* pCur) {
|
|||
}
|
||||
}
|
||||
|
||||
void sessionWinStateMoveToPrev(SStreamStateCur* pCur) {
|
||||
qTrace("move cursor to prev");
|
||||
if (pCur && pCur->buffIndex >= 1) {
|
||||
pCur->buffIndex--;
|
||||
} else {
|
||||
streamStateCurPrev_rocksdb(pCur);
|
||||
}
|
||||
}
|
||||
|
||||
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey,
|
||||
range_cmpr_fn cmpFn) {
|
||||
SStreamStateCur* pCur = sessionWinStateSeekKeyCurrentPrev(pFileState, key);
|
||||
|
|
|
@ -440,6 +440,10 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, cons
|
|||
return sessionWinStateSeekKeyCurrentNext(pState->pFileState, key);
|
||||
}
|
||||
|
||||
SStreamStateCur *streamStateSessionSeekKeyPrev(SStreamState *pState, const SSessionKey *key) {
|
||||
return sessionWinStateSeekKeyPrev(pState->pFileState, key);
|
||||
}
|
||||
|
||||
SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key) {
|
||||
return sessionWinStateSeekKeyNext(pState->pFileState, key);
|
||||
}
|
||||
|
|
|
@ -331,6 +331,8 @@ void tFreeStreamTask(void* pParam) {
|
|||
taosMemoryFreeClear(pTask->notifyInfo.stbFullName);
|
||||
tDeleteSchemaWrapper(pTask->notifyInfo.pSchemaWrapper);
|
||||
|
||||
pTask->notifyEventStat = (STaskNotifyEventStat){0};
|
||||
|
||||
taosMemoryFree(pTask);
|
||||
stDebug("s-task:0x%x free task completed", taskId);
|
||||
}
|
||||
|
@ -988,6 +990,7 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc)
|
|||
|
||||
pDst->startTime = pSrc->startTime;
|
||||
pDst->hTaskId = pSrc->hTaskId;
|
||||
pDst->notifyEventStat = pSrc->notifyEventStat;
|
||||
}
|
||||
|
||||
STaskStatusEntry streamTaskGetStatusEntry(SStreamTask* pTask) {
|
||||
|
@ -1016,6 +1019,7 @@ STaskStatusEntry streamTaskGetStatusEntry(SStreamTask* pTask) {
|
|||
.outputThroughput = SIZE_IN_KiB(pExecInfo->outputThroughput),
|
||||
.startCheckpointId = pExecInfo->startCheckpointId,
|
||||
.startCheckpointVer = pExecInfo->startCheckpointVer,
|
||||
.notifyEventStat = pTask->notifyEventStat,
|
||||
};
|
||||
return entry;
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue