diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 0a33798f0c..e91b7c71ea 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -78,7 +78,7 @@ typedef struct SMetaEntry { } smaEntry; }; - uint8_t* pBuf; + uint8_t* pBuf; SColCmprWrapper colCmpr; // col compress alg } SMetaEntry; @@ -105,15 +105,15 @@ typedef struct SMTbCursor { } SMTbCursor; typedef struct SMCtbCursor { - struct SMeta* pMeta; - void* pCur; - tb_uid_t suid; - void* pKey; - void* pVal; - int kLen; - int vLen; - int8_t paused; - int lock; + struct SMeta* pMeta; + void* pCur; + tb_uid_t suid; + void* pKey; + void* pVal; + int kLen; + int vLen; + int8_t paused; + int lock; } SMCtbCursor; typedef struct SRowBuffPos { @@ -135,17 +135,17 @@ typedef struct SMetaTableInfo { typedef struct SSnapContext { struct SMeta* pMeta; - int64_t snapVersion; - void* pCur; - int64_t suid; - int8_t subType; - SHashObj* idVersion; - SHashObj* suidInfo; - SArray* idList; - int32_t index; - int8_t withMeta; - int8_t queryMeta; // true-get meta, false-get data - bool hasPrimaryKey; + int64_t snapVersion; + void* pCur; + int64_t suid; + int8_t subType; + SHashObj* idVersion; + SHashObj* suidInfo; + SArray* idList; + int32_t index; + int8_t withMeta; + int8_t queryMeta; // true-get meta, false-get data + bool hasPrimaryKey; } SSnapContext; typedef struct { @@ -229,15 +229,15 @@ typedef struct SStoreTqReader { bool (*tqReaderCurrentBlockConsumed)(); struct SWalReader* (*tqReaderGetWalReader)(); // todo remove it -// int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it + // int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it int32_t (*tqReaderSetSubmitMsg)(); // todo remove it -// bool (*tqReaderNextBlockFilterOut)(); + // bool (*tqReaderNextBlockFilterOut)(); } SStoreTqReader; typedef struct SStoreSnapshotFn { - bool (*taosXGetTablePrimaryKey)(SSnapContext *ctx); - void (*taosXSetTablePrimaryKey)(SSnapContext *ctx, int64_t uid); + bool (*taosXGetTablePrimaryKey)(SSnapContext* ctx); + void (*taosXSetTablePrimaryKey)(SSnapContext* ctx, int64_t uid); int32_t (*setForSnapShot)(SSnapContext* ctx, int64_t uid); int32_t (*destroySnapshot)(SSnapContext* ctx); SMetaTableInfo (*getMetaTableInfoFromSnapshot)(SSnapContext* ctx); @@ -270,7 +270,7 @@ typedef struct SStoreMeta { int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen, double selectivityRatio); - void* (*storeGetIndexInfo)(void *pVnode); + void* (*storeGetIndexInfo)(void* pVnode); void* (*getInvertIndex)(void* pVnode); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] int32_t (*getChildTableList)(void* pVnode, int64_t suid, SArray* list); @@ -329,29 +329,31 @@ typedef struct { typedef struct SStateStore { int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname); - int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache); + int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, + int32_t* pWinCode); - int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); - int32_t (*streamStateReleaseBuf)(SStreamState* pState, void* pVal, bool used); - int32_t (*streamStateClearBuff)(SStreamState* pState, void* pVal); + int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, + int32_t* pWinCode); + void (*streamStateReleaseBuf)(SStreamState* pState, void* pVal, bool used); + void (*streamStateClearBuff)(SStreamState* pState, void* pVal); void (*streamStateFreeVal)(void* val); int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); - int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); + int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode); bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key); int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal); - int32_t (*streamStateDel)(SStreamState* pState, const SWinKey* key); - int32_t (*streamStateClear)(SStreamState* pState); + void (*streamStateDel)(SStreamState* pState, const SWinKey* key); + void (*streamStateClear)(SStreamState* pState); void (*streamStateSetNumber)(SStreamState* pState, int32_t number, int32_t tsIdex); - int32_t (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); + void (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen); int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); - int32_t (*streamStateFillDel)(SStreamState* pState, const SWinKey* key); + void (*streamStateFillDel)(SStreamState* pState, const SWinKey* key); - int32_t (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur); - int32_t (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur); + void (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur); + void (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur); SStreamStateCur* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key); SStreamStateCur* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key); @@ -363,26 +365,28 @@ typedef struct SStateStore { int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, - int32_t* pVLen); + int32_t* pVLen, int32_t* pWinCode); int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen); int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); - int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key); - int32_t (*streamStateSessionReset)(SStreamState* pState, void* pVal); - int32_t (*streamStateSessionClear)(SStreamState* pState); + void (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key); + void (*streamStateSessionReset)(SStreamState* pState, void* pVal); + void (*streamStateSessionClear)(SStreamState* pState); int32_t (*streamStateSessionGetKVByCur)(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, - state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode); int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); int32_t (*streamStateCountGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); int32_t (*streamStateSessionAllocWinBuffByNextPosition)(SStreamState* pState, SStreamStateCur* pCur, const SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t (*streamStateCountWinAddIfNotExist)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, - void** ppVal, int32_t* pVLen); + void** ppVal, int32_t* pVLen, int32_t* pWinCode); int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); - int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); - TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); + int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, + int32_t pkLen, SUpdateInfo** ppInfo); + TSKEY(*updateInfoFillBlockData) + (SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); bool (*isIncrementalTimeStamp)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); @@ -391,7 +395,8 @@ typedef struct SStateStore { void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count); int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); - int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); + int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, + SUpdateInfo** ppInfo); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo); int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen); @@ -413,7 +418,7 @@ typedef struct SStateStore { SStreamState* (*streamStateOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId); void (*streamStateClose)(SStreamState* pState, bool remove); int32_t (*streamStateBegin)(SStreamState* pState); - int32_t (*streamStateCommit)(SStreamState* pState); + void (*streamStateCommit)(SStreamState* pState); void (*streamStateDestroy)(SStreamState* pState, bool remove); int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts); diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 924682f223..700cc5ba7f 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -294,7 +294,7 @@ typedef struct SPoint { void *val; } SPoint; -int32_t taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2, +void taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2, int32_t inputType); #define LEASTSQUARES_DOUBLE_ITEM_LENGTH 25 diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 5768160fdb..06da578749 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -32,7 +32,7 @@ extern "C" { SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId); void streamStateClose(SStreamState* pState, bool remove); int32_t streamStateBegin(SStreamState* pState); -int32_t streamStateCommit(SStreamState* pState); +void streamStateCommit(SStreamState* pState); void streamStateDestroy(SStreamState* pState, bool remove); int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark); int32_t streamStateDelTaskDb(SStreamState* pState); @@ -41,22 +41,23 @@ int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen); int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); -int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); +int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode); bool streamStateCheck(SStreamState* pState, const SWinKey* key); int32_t streamStateGetByPos(SStreamState* pState, void* pos, void** pVal); -int32_t streamStateDel(SStreamState* pState, const SWinKey* key); -int32_t streamStateClear(SStreamState* pState); +void streamStateDel(SStreamState* pState, const SWinKey* key); +void streamStateClear(SStreamState* pState); void streamStateSetNumber(SStreamState* pState, int32_t number, int32_t tsIdex); -int32_t streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); +void streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen); // session window -int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); +int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen, + int32_t* pWinCode); int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen); int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); -int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key); -int32_t streamStateSessionReset(SStreamState* pState, void* pVal); -int32_t streamStateSessionClear(SStreamState* pState); +void streamStateSessionDel(SStreamState* pState, const SSessionKey* key); +void streamStateSessionReset(SStreamState* pState, void* pVal); +void streamStateSessionClear(SStreamState* pState); int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); @@ -70,21 +71,22 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, cons // state window int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, - state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode); // fill int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); -int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key); +void streamStateFillDel(SStreamState* pState, const SWinKey* key); -int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); -int32_t streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used); -int32_t streamStateClearBuff(SStreamState* pState, void* pVal); +int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, + int32_t* pWinCode); +void streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used); +void streamStateClearBuff(SStreamState* pState, void* pVal); void streamStateFreeVal(void* val); // count window int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal, - int32_t* pVLen); + int32_t* pVLen, int32_t* pWinCode); int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key); @@ -97,11 +99,11 @@ void streamStateResetCur(SStreamStateCur* pCur); int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); -int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur); -int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); +void streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur); +void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname); -int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache); +int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode); void streamStateReloadInfo(SStreamState* pState, TSKEY ts); diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index 68b9c4baa2..7d7001fdf9 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -18,10 +18,10 @@ #include "os.h" +#include "storageapi.h" #include "tarray.h" #include "tdef.h" #include "tlist.h" -#include "storageapi.h" #ifdef __cplusplus extern "C" { @@ -30,53 +30,56 @@ extern "C" { typedef struct SStreamFileState SStreamFileState; typedef SList SStreamSnapshot; -typedef void* (*_state_buff_get_fn)(void* pRowBuff, const void* pKey, size_t keyLen); +typedef void* (*_state_buff_get_fn)(void* pRowBuff, const void* pKey, size_t keyLen); typedef int32_t (*_state_buff_remove_fn)(void* pRowBuff, const void* pKey, size_t keyLen); typedef int32_t (*_state_buff_remove_by_pos_fn)(SStreamFileState* pState, SRowBuffPos* pPos); -typedef void (*_state_buff_cleanup_fn)(void* pRowBuff); -typedef void* (*_state_buff_create_statekey_fn)(SRowBuffPos* pPos, int64_t num); +typedef void (*_state_buff_cleanup_fn)(void* pRowBuff); +typedef void* (*_state_buff_create_statekey_fn)(SRowBuffPos* pPos, int64_t num); typedef int32_t (*_state_file_remove_fn)(SStreamFileState* pFileState, const void* pKey); typedef int32_t (*_state_file_get_fn)(SStreamFileState* pFileState, void* pKey, void* data, int32_t* pDataLen); typedef int32_t (*_state_file_clear_fn)(SStreamState* pState); -typedef int32_t (*_state_fun_get_fn) (SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); +typedef int32_t (*_state_fun_get_fn)(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, + int32_t* pVLen, int32_t* pWinCode); typedef int32_t (*range_cmpr_fn)(const SSessionKey* pWin1, const SSessionKey* pWin2); SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, - GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, - int64_t checkpointId, int8_t type); + GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId, + int8_t type); void streamFileStateDestroy(SStreamFileState* pFileState); void streamFileStateClear(SStreamFileState* pFileState); bool needClearDiskBuff(SStreamFileState* pFileState); void streamFileStateReleaseBuff(SStreamFileState* pFileState, SRowBuffPos* pPos, bool used); -int32_t streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); +void streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); -int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); -int32_t deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen); +int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen, + int32_t* pWinCode); +void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen); int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** pVal); bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen); void putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); SStreamSnapshot* getSnapshot(SStreamFileState* pFileState); -int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState); -int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId); +void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState); +void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId); int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list); int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark); int32_t streamFileStateGetSelectRowSize(SStreamFileState* pFileState); void streamFileStateReloadInfo(SStreamFileState* pFileState, TSKEY ts); -void* getRowStateBuff(SStreamFileState* pFileState); -void* getStateFileStore(SStreamFileState* pFileState); -bool isDeteled(SStreamFileState* pFileState, TSKEY ts); -bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap); +void* getRowStateBuff(SStreamFileState* pFileState); +void* getStateFileStore(SStreamFileState* pFileState); +bool isDeteled(SStreamFileState* pFileState, TSKEY ts); +bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap); SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState); -int32_t getRowStateRowSize(SStreamFileState* pFileState); +int32_t getRowStateRowSize(SStreamFileState* pFileState); // session window -int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen); +int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen, + int32_t* pWinCode); int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen); @@ -85,7 +88,7 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen); SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen); -int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId); +void recoverSesssion(SStreamFileState* pFileState, int64_t ckId); void sessionWinStateClear(SStreamFileState* pFileState); void sessionWinStateCleanup(void* pBuff); @@ -94,20 +97,23 @@ SStreamStateCur* sessionWinStateSeekKeyCurrentPrev(SStreamFileState* pFileState, SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState, const SSessionKey* pWinKey); SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey); SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey, COUNT_TYPE count); -int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); -int32_t sessionWinStateMoveToNext(SStreamStateCur* pCur); -int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey, range_cmpr_fn cmpFn); +int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); +void sessionWinStateMoveToNext(SStreamStateCur* pCur); +int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey, + range_cmpr_fn cmpFn); // state window int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, - state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode); // count window -int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen); +int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, + int32_t* pVLen, int32_t* pWinCode); int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen); -//function -int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); +// function +int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen, + int32_t* pWinCode); int32_t getFunctionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); #ifdef __cplusplus diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 6db3c780e2..c9acead357 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -221,10 +221,10 @@ enum { }; typedef enum ETableCountState { - TABLE_COUNT_STATE_NONE = 0, // before start scan - TABLE_COUNT_STATE_SCAN = 1, // cur group scanning - TABLE_COUNT_STATE_PROCESSED = 2, // cur group processed - TABLE_COUNT_STATE_END = 3, // finish or noneed to process + TABLE_COUNT_STATE_NONE = 0, // before start scan + TABLE_COUNT_STATE_SCAN = 1, // cur group scanning + TABLE_COUNT_STATE_PROCESSED = 2, // cur group processed + TABLE_COUNT_STATE_END = 3, // finish or noneed to process } ETableCountState; typedef struct SAggSupporter { @@ -272,14 +272,14 @@ typedef struct STableScanInfo { int32_t scanTimes; SSDataBlock* pResBlock; SHashObj* pIgnoreTables; - SSampleExecInfo sample; // sample execution info - int32_t tableStartIndex; // current group scan start - int32_t tableEndIndex; // current group scan end + SSampleExecInfo sample; // sample execution info + int32_t tableStartIndex; // current group scan start + int32_t tableEndIndex; // current group scan end int32_t currentGroupId; - int32_t currentTable; + int32_t currentTable; int8_t scanMode; int8_t assignBlockUid; - uint8_t countState; // empty table count state + uint8_t countState; // empty table count state bool hasGroupByTag; bool filesetDelimited; bool needCountEmptyTable; @@ -291,19 +291,19 @@ typedef enum ESubTableInputType { } ESubTableInputType; typedef struct STmsSubTableInput { - STsdbReader* pReader; + STsdbReader* pReader; SQueryTableDataCond tblCond; - STableKeyInfo* pKeyInfo; - bool bInMemReader; - ESubTableInputType type; - SSDataBlock* pReaderBlock; + STableKeyInfo* pKeyInfo; + bool bInMemReader; + ESubTableInputType type; + SSDataBlock* pReaderBlock; - SArray* aBlockPages; + SArray* aBlockPages; SSDataBlock* pPageBlock; - int32_t pageIdx; + int32_t pageIdx; - int32_t rowIdx; - int64_t* aTs; + int32_t rowIdx; + int64_t* aTs; SSDataBlock* pInputBlock; } STmsSubTableInput; @@ -312,10 +312,10 @@ typedef struct STmsSubTablesMergeInfo { SBlockOrderInfo* pTsOrderInfo; SBlockOrderInfo* pPkOrderInfo; - int32_t numSubTables; - STmsSubTableInput* aInputs; + int32_t numSubTables; + STmsSubTableInput* aInputs; SMultiwayMergeTreeInfo* pTree; - int32_t numSubTablesCompleted; + int32_t numSubTablesCompleted; int32_t numTableBlocksInMem; SDiskbasedBuf* pBlocksBuf; @@ -382,7 +382,7 @@ typedef struct STagScanInfo { SArray* aUidTags; // SArray SArray* aFilterIdxs; // SArray SStorageAPI* pStorageAPI; - SLimitInfo limitInfo; + SLimitInfo limitInfo; } STagScanInfo; typedef enum EStreamScanMode { @@ -400,18 +400,18 @@ enum { }; typedef struct SStreamAggSupporter { - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row - SSDataBlock* pScanBlock; - SStreamState* pState; - int64_t gap; // stream session window gap - SqlFunctionCtx* pDummyCtx; // for combine - SSHashObj* pResultRows; - int32_t stateKeySize; - int16_t stateKeyType; - SDiskbasedBuf* pResultBuf; - SStateStore stateStore; - STimeWindow winRange; - SStorageAPI* pSessionAPI; + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + SSDataBlock* pScanBlock; + SStreamState* pState; + int64_t gap; // stream session window gap + SqlFunctionCtx* pDummyCtx; // for combine + SSHashObj* pResultRows; + int32_t stateKeySize; + int16_t stateKeyType; + SDiskbasedBuf* pResultBuf; + SStateStore stateStore; + STimeWindow winRange; + SStorageAPI* pSessionAPI; struct SUpdateInfo* pUpdateInfo; int32_t windowCount; int32_t windowSliding; @@ -577,7 +577,7 @@ typedef struct SOpCheckPointInfo { } SOpCheckPointInfo; typedef struct SStreamIntervalOperatorInfo { - SOptrBasicInfo binfo; // basic info + SOptrBasicInfo binfo; // basic info SSteamOpBasicInfo basic; SAggSupporter aggSup; // aggregate supporter SExprSupp scalarSupp; // supporter for perform scalar function @@ -835,7 +835,8 @@ void doBuildResultDatablock(struct SOperatorInfo* pOperator, SOptrBasicInfo* pbI * @brief copydata from hash table, instead of copying from SGroupResInfo's pRow */ int32_t doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, - SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold, bool ignoreGroup); + SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold, + bool ignoreGroup); bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); bool hasSlimitOffsetInfo(SLimitInfo* pLimitInfo); @@ -855,7 +856,8 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* de extern void doDestroyExchangeOperatorInfo(void* param); -int32_t doFilterImpl(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo, SColumnInfoData** pResCol); +int32_t doFilterImpl(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo, + SColumnInfoData** pResCol); int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock, int32_t rows, SExecTaskInfo* pTask, STableMetaCacheInfo* pCache); @@ -865,8 +867,9 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, void setVgIdColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int32_t vgId); void setVgVerColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int64_t vgVer); -int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); -void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput); +int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, + int32_t* rowEntryInfoOffset); +void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput); SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, int32_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, @@ -894,8 +897,8 @@ bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, SStateStore* pStore); -void appendDataToSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, - uint64_t* pGp, void* pTbName); +void appendDataToSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp, + void* pTbName); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); @@ -906,7 +909,7 @@ bool groupbyTbname(SNodeList* pGroupList); void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, int64_t* pData); -void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, +int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI); SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag); @@ -929,36 +932,39 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, i SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore, SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr, SStorageAPI* pApi, int32_t tsIndex); -int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, - STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic); -void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins); +int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, + int32_t tsColIndex, STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic); +int32_t getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins); void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey); -void deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd); -int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated); +int32_t deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, + SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd); +void getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated); int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed); -int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar); +int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar); int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2); void removeSessionDeleteResults(SSHashObj* pHashMap, SArray* pWins); int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult, - int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput, - struct SOperatorInfo* pOperator, int64_t winDelta); -int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo); + int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput, + struct SOperatorInfo* pOperator, int64_t winDelta); +void setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo); int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo); int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated); -void saveDeleteRes(SSHashObj* pStDelete, SSessionKey key); +int32_t saveDeleteRes(SSHashObj* pStDelete, SSessionKey key); void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey); void doBuildDeleteDataBlock(struct SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite); -void doBuildSessionResult(struct SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock); +void doBuildSessionResult(struct SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, + SSDataBlock* pBlock); void getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo); void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin); -void compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup, +int32_t compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup, SExecTaskInfo* pTaskInfo, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap); -int32_t releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI); +void releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI); void resetWinRange(STimeWindow* winRange); -bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); +bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, + TSKEY ts, void* pPkVal, int32_t len); int64_t getDeleteMark(SWindowPhysiNode* pWinPhyNode, int64_t interval); void resetUnCloseSessionWinInfo(SSHashObj* winMap); void setStreamOperatorCompleted(struct SOperatorInfo* pOperator); @@ -979,26 +985,24 @@ void freeExchangeGetBasicOperatorParam(void* pParam); void freeOperatorParam(SOperatorParam* pParam, SOperatorParamType type); void freeResetOperatorParams(struct SOperatorInfo* pOperator, SOperatorParamType type, bool allFree); SSDataBlock* getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int32_t idx, bool clearParam); -void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, EStreamType mode, SSessionKey* pDelRange); -bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey); +void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, EStreamType mode, SSessionKey* pDelRange); +void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey); -void saveDeleteInfo(SArray* pWins, SSessionKey key); -void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins); -void copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted); -void copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest); +int32_t saveDeleteInfo(SArray* pWins, SSessionKey key); +void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins); +int32_t copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted); +void copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest); bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo); bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd, EStreamType blockType); bool compareVal(const char* v, const SStateKeys* pKey); bool inWinRange(STimeWindow* range, STimeWindow* cur); -void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result); +int32_t doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result); int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, TSKEY* primaryKeys, int32_t prevPosition, int32_t order); void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, int32_t status); - - #ifdef __cplusplus } #endif diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index c7236a88e8..2e65ca2a7a 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -179,7 +179,7 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData size_t numOfGroupCols = taosArrayGetSize(pGroupCols); for (int32_t i = 0; i < numOfGroupCols; ++i) { - SColumn* pCol = (SColumn*) taosArrayGet(pGroupCols, i); + SColumn* pCol = (SColumn*)taosArrayGet(pGroupCols, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); // valid range check. todo: return error code. @@ -372,7 +372,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { bool hasRemainResultByHash(SOperatorInfo* pOperator) { SGroupbyOperatorInfo* pInfo = pOperator->info; - SSHashObj* pHashmap = pInfo->aggSup.pResultRowHashTable; + SSHashObj* pHashmap = pInfo->aggSup.pResultRowHashTable; return pInfo->groupResInfo.index < tSimpleHashGetSize(pHashmap); } @@ -413,7 +413,7 @@ void doBuildResultDatablockByHash(SOperatorInfo* pOperator, SOptrBasicInfo* pbIn static SSDataBlock* buildGroupResultDataBlockByHash(SOperatorInfo* pOperator) { SGroupbyOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pRes = pInfo->binfo.pRes; + SSDataBlock* pRes = pInfo->binfo.pRes; // after filter, if result block turn to null, get next from whole set while (1) { @@ -448,8 +448,8 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { return buildGroupResultDataBlockByHash(pOperator); } SGroupResInfo* pGroupResInfo = &pInfo->groupResInfo; - - int32_t order = pInfo->binfo.inputTsOrder; + + int32_t order = pInfo->binfo.inputTsOrder; int64_t st = taosGetTimestampUs(); SOperatorInfo* downstream = pOperator->pDownstream[0]; @@ -585,7 +585,7 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc blockDataDestroy(pDstBlock); return NULL; } - for(int i = 0; i < numOfCols; ++i) { + for (int i = 0; i < numOfCols; ++i) { pDstBlock->pBlockAgg[i].colId = -1; } } @@ -601,12 +601,12 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc if (pDataBlock->pBlockAgg && pDataBlock->pBlockAgg[slotId].colId != -1) { pDstBlock->pBlockAgg[i] = pDataBlock->pBlockAgg[slotId]; } else { - int32_t code = doEnsureCapacity(pDst, &pDstBlock->info, pDataBlock->info.rows, false); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - blockDataDestroy(pDstBlock); - return NULL; - } + int32_t code = doEnsureCapacity(pDst, &pDstBlock->info, pDataBlock->info.rows, false); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + blockDataDestroy(pDstBlock); + return NULL; + } colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); } @@ -840,9 +840,9 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { SDataGroupInfo* pGroupInfo = (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) { - if(pGroupInfo != NULL) { + if (pGroupInfo != NULL) { SSDataBlock* ret = buildPartitionResultForNotLoadBlock(pGroupInfo); - if(ret != NULL) return ret; + if (ret != NULL) return ret; } // try next group data if (pInfo->groupIndex + 1 >= taosArrayGetSize(pInfo->sortedGroupArray)) { @@ -885,7 +885,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { pInfo->binfo.pRes->info.id.groupId = pGroupInfo->groupId; pInfo->binfo.pRes->info.dataLoad = 1; pInfo->orderedRows = 0; - } else if (pInfo->pOrderInfoArr == NULL) { + } else if (pInfo->pOrderInfoArr == NULL) { qError("Exception, remainRows not zero, but pOrderInfoArr is NULL"); } @@ -1069,8 +1069,9 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - pInfo->rowCapacity = blockDataGetCapacityInRow(pInfo->binfo.pRes, getBufPageSize(pInfo->pBuf), - blockDataGetSerialMetaSize(taosArrayGetSize(pInfo->binfo.pRes->pDataBlock))); + pInfo->rowCapacity = + blockDataGetCapacityInRow(pInfo->binfo.pRes, getBufPageSize(pInfo->pBuf), + blockDataGetSerialMetaSize(taosArrayGetSize(pInfo->binfo.pRes->pDataBlock))); pInfo->columnOffset = setupColumnOffset(pInfo->binfo.pRes, pInfo->rowCapacity); code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pInfo->pGroupCols); if (code != TSDB_CODE_SUCCESS) { @@ -1084,8 +1085,8 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->fpSet = - createOperatorFpSet(optrDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -1110,8 +1111,8 @@ int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; - SResultRow* pResultRow = - doSetResultOutBufByKey(pBuf, pResultRowInfo, (char*)pData, bytes, true, groupId, pTaskInfo, false, pAggSup, false); + SResultRow* pResultRow = doSetResultOutBufByKey(pBuf, pResultRowInfo, (char*)pData, bytes, true, groupId, pTaskInfo, + false, pAggSup, false); return setResultRowInitCtx(pResultRow, pCtx, numOfCols, pOperator->exprSupp.rowEntryInfoOffset); } @@ -1134,6 +1135,8 @@ static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo static bool hasRemainTbName(SStreamPartitionOperatorInfo* pInfo) { return pInfo->pTbNameIte != NULL; } static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -1159,8 +1162,13 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { } pDest->info.parTbName[0] = 0; if (pInfo->tbnameCalSup.numOfExprs > 0) { - void* tbname = NULL; - if (pAPI->stateStore.streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname, false) == 0) { + void* tbname = NULL; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAPI->stateStore.streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname, + false, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode == TSDB_CODE_SUCCESS) { memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); pAPI->stateStore.streamStateFreeVal(tbname); } @@ -1174,20 +1182,35 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { pOperator->resultInfo.totalRows += pDest->info.rows; pInfo->parIte = taosHashIterate(pInfo->pPartitions, pInfo->parIte); ASSERT(pDest->info.rows > 0); + +_end: + if (code != TSDB_CODE_SUCCESS) { + blockDataCleanup(pDest); + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } printDataBlock(pDest, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); return pDest; } -void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, - SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI) { - void* pValue = NULL; - if (pAPI->streamStateGetParName(pState, groupId, &pValue, true) != 0) { +int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, + SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + void* pValue = NULL; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAPI->streamStateGetParName(pState, groupId, &pValue, true, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode != TSDB_CODE_SUCCESS) { SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId); memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); pTmpBlock->info.id.groupId = groupId; char* tbName = pSrcBlock->info.parTbName; if (pTableSup->numOfExprs > 0) { - projectApplyFunctions(pTableSup->pExprInfo, pDestBlock, pTmpBlock, pTableSup->pCtx, pTableSup->numOfExprs, NULL); + code = projectApplyFunctions(pTableSup->pExprInfo, pDestBlock, pTmpBlock, pTableSup->pCtx, pTableSup->numOfExprs, + NULL); + TSDB_CHECK_CODE(code, lino, _end); + SColumnInfoData* pTbCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); memset(tbName, 0, TSDB_TABLE_NAME_LEN); int32_t len = 0; @@ -1198,7 +1221,8 @@ void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1); len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); memcpy(tbName, varDataVal(pData), len); - pAPI->streamStatePutParName(pState, groupId, tbName); + code = pAPI->streamStatePutParName(pState, groupId, tbName); + TSDB_CHECK_CODE(code, lino, _end); } memcpy(pTmpBlock->info.parTbName, tbName, len); pDestBlock->info.rows--; @@ -1209,42 +1233,58 @@ void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup } if (pTagSup->numOfExprs > 0) { - projectApplyFunctions(pTagSup->pExprInfo, pDestBlock, pTmpBlock, pTagSup->pCtx, pTagSup->numOfExprs, NULL); + code = projectApplyFunctions(pTagSup->pExprInfo, pDestBlock, pTmpBlock, pTagSup->pCtx, pTagSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); pDestBlock->info.rows--; } else { memcpy(pDestBlock->info.parTbName, pTmpBlock->info.parTbName, TSDB_TABLE_NAME_LEN); } void* pGpIdCol = taosArrayGet(pDestBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); - colDataSetVal(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false); + code = colDataSetVal(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false); + TSDB_CHECK_CODE(code, lino, _end); pDestBlock->info.rows++; blockDataDestroy(pTmpBlock); } else { memcpy(pSrcBlock->info.parTbName, pValue, TSDB_TABLE_NAME_LEN); } pAPI->streamStateFreeVal(pValue); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) { - SExecTaskInfo* pTask = pOperator->pTaskInfo; - +static int32_t buildStreamCreateTableResult(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SExecTaskInfo* pTask = pOperator->pTaskInfo; SStreamPartitionOperatorInfo* pInfo = pOperator->info; + SSDataBlock* pSrc = pInfo->pInputDataBlock; if ((pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) || taosHashGetSize(pInfo->pPartitions) == 0) { - return NULL; + goto _end; } blockDataCleanup(pInfo->pCreateTbRes); - blockDataEnsureCapacity(pInfo->pCreateTbRes, taosHashGetSize(pInfo->pPartitions)); - SSDataBlock* pSrc = pInfo->pInputDataBlock; + code = blockDataEnsureCapacity(pInfo->pCreateTbRes, taosHashGetSize(pInfo->pPartitions)); + TSDB_CHECK_CODE(code, lino, _end); if (pInfo->pTbNameIte != NULL) { SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte; int32_t rowId = *(int32_t*)taosArrayGet(pParInfo->rowIds, 0); - appendCreateTableRow(pTask->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, - pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes, &pTask->storageAPI.stateStore); + code = appendCreateTableRow(pTask->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, pParInfo->groupId, + pSrc, rowId, pInfo->pCreateTbRes, &pTask->storageAPI.stateStore); + TSDB_CHECK_CODE(code, lino, _end); pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, pInfo->pTbNameIte); } - return pInfo->pCreateTbRes->info.rows > 0 ? pInfo->pCreateTbRes : NULL; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDataBlock* pBlock) { @@ -1267,18 +1307,20 @@ static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDat } static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStreamPartitionOperatorInfo* pInfo = pOperator->info; + if (pOperator->status == OP_EXEC_DONE) { return NULL; } - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStreamPartitionOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pCtRes = NULL; - if (hasRemainTbName(pInfo)) { - pCtRes = buildStreamCreateTableResult(pOperator); - if (pCtRes != NULL) { - return pCtRes; + code = buildStreamCreateTableResult(pOperator); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->pCreateTbRes->info.rows > 0) { + return pInfo->pCreateTbRes; } } @@ -1310,7 +1352,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { } break; case STREAM_CREATE_CHILD_TABLE: case STREAM_RETRIEVE: - case STREAM_CHECKPOINT: + case STREAM_CHECKPOINT: case STREAM_GET_ALL: { return pBlock; } @@ -1330,11 +1372,18 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { pInfo->parIte = taosHashIterate(pInfo->pPartitions, NULL); pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, NULL); - pCtRes = buildStreamCreateTableResult(pOperator); - if (pCtRes != NULL) { - return pCtRes; + code = buildStreamCreateTableResult(pOperator); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->pCreateTbRes->info.rows > 0) { + return pInfo->pCreateTbRes; } return buildStreamPartitionResult(pOperator); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return NULL; } static void destroyStreamPartitionOperatorInfo(void* param) { @@ -1358,9 +1407,10 @@ static void destroyStreamPartitionOperatorInfo(void* param) { taosMemoryFreeClear(param); } -int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; +int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, + SExprSupp* pTbnameExpr) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI; if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -1372,7 +1422,8 @@ int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pPar pScanInfo->pPartScalarSup = pExpr; pScanInfo->pPartTbnameSup = pTbnameExpr; if (!pScanInfo->pUpdateInfo) { - code = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); + code = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, + pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); } return code; } @@ -1512,8 +1563,9 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamHashPartition, NULL, - destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); code = initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 678d8d569e..cf464ad918 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1866,17 +1866,29 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS return TSDB_CODE_SUCCESS; } -static void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowId) { +static int32_t calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowId) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; blockDataCleanup(pInfo->pCreateTbRes); if (pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) { pBlock->info.parTbName[0] = 0; } else { - appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, - pBlock->info.id.groupId, pBlock, rowId, pInfo->pCreateTbRes, &pInfo->stateStore); + code = appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, + &pInfo->tagCalSup, pBlock->info.id.groupId, pBlock, rowId, pInfo->pCreateTbRes, + &pInfo->stateStore); + TSDB_CHECK_CODE(code, lino, _end); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t generatePartitionDelResBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); @@ -1897,9 +1909,13 @@ static int32_t generatePartitionDelResBlock(SStreamScanInfo* pInfo, SSDataBlock* for (int32_t preJ = 0; preJ < pPreRes->info.rows; preJ++) { groupId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, preJ); if (pInfo->pPartTbnameSup) { - void* parTbname = NULL; - int32_t code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname, false); - if (code != TSDB_CODE_SUCCESS) { + void* parTbname = NULL; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, + &parTbname, false, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode != TSDB_CODE_SUCCESS) { calBlockTbName(pInfo, pPreRes, preJ); memcpy(varDataVal(tbname), pPreRes->info.parTbName, strlen(pPreRes->info.parTbName)); } else { @@ -1912,10 +1928,17 @@ static int32_t generatePartitionDelResBlock(SStreamScanInfo* pInfo, SSDataBlock* tbname[0] == 0 ? NULL : tbname); } } - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t generateDeleteResultBlockImpl(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { + int32_t code = TSDB_CODE_SUCCESS; +int32_t lino = 0; SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); @@ -1944,8 +1967,11 @@ static int32_t generateDeleteResultBlockImpl(SStreamScanInfo* pInfo, SSDataBlock } if (pInfo->tbnameCalSup.pExprInfo) { void* parTbname = NULL; - int32_t code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname, false); - if (code != TSDB_CODE_SUCCESS) { + int32_t winCode = TSDB_CODE_SUCCESS; + code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname, false, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode != TSDB_CODE_SUCCESS) { SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, srcStartTsCol[i], srcStartTsCol[i], ver); printDataBlock(pPreRes, "pre res", GET_TASKID(pInfo->pStreamScanOp->pTaskInfo)); calBlockTbName(pInfo, pPreRes, 0); @@ -1959,7 +1985,12 @@ static int32_t generateDeleteResultBlockImpl(SStreamScanInfo* pInfo, SSDataBlock appendDataToSpecialBlock(pDestBlock, srcStartTsCol + i, srcEndTsCol + i, srcUidData + i, &groupId, tbname[0] == 0 ? NULL : tbname); } - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { @@ -2565,6 +2596,8 @@ static bool isStreamWindow(SStreamScanInfo* pInfo) { static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; const char* id = GET_TASKID(pTaskInfo); @@ -2667,7 +2700,11 @@ FETCH_NEXT_BLOCK: SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); SSDataBlock* pBlock = pPacked->pDataBlock; if (pBlock->info.parTbName[0]) { - pAPI->stateStore.streamStatePutParName(pStreamInfo->pState, pBlock->info.id.groupId, pBlock->info.parTbName); + code = + pAPI->stateStore.streamStatePutParName(pStreamInfo->pState, pBlock->info.id.groupId, pBlock->info.parTbName); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } // TODO move into scan @@ -3161,17 +3198,27 @@ _end: } void streamScanReloadState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamScanInfo* pInfo = pOperator->info; if (!pInfo->pState) { return; } void* pBuff = NULL; int32_t len = 0; - pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), &pBuff, &len); + code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, + strlen(STREAM_SCAN_OP_STATE_NAME), &pBuff, &len); + TSDB_CHECK_CODE(code, lino, _end); + SUpdateInfo* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); - int32_t code = pInfo->stateStore.updateInfoDeserialize(pBuff, len, pUpInfo); + if (!pUpInfo) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + int32_t winCode = pInfo->stateStore.updateInfoDeserialize(pBuff, len, pUpInfo); taosMemoryFree(pBuff); - if (code == TSDB_CODE_SUCCESS && pInfo->pUpdateInfo) { + if (winCode == TSDB_CODE_SUCCESS && pInfo->pUpdateInfo) { if (pInfo->pUpdateInfo->minTS < 0) { pInfo->stateStore.updateInfoDestroy(pInfo->pUpdateInfo); pInfo->pUpdateInfo = pUpInfo; @@ -3181,9 +3228,9 @@ void streamScanReloadState(SOperatorInfo* pOperator) { ASSERT(pInfo->pUpdateInfo->minTS > pUpInfo->minTS); pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pUpInfo->maxDataVersion); SHashObj* curMap = pInfo->pUpdateInfo->pMap; - void *pIte = taosHashIterate(curMap, NULL); + void* pIte = taosHashIterate(curMap, NULL); while (pIte != NULL) { - size_t keySize = 0; + size_t keySize = 0; int64_t* pUid = taosHashGetKey(pIte, &keySize); taosHashPut(pUpInfo->pMap, pUid, sizeof(int64_t), pIte, sizeof(TSKEY)); pIte = taosHashIterate(curMap, pIte); @@ -3196,6 +3243,11 @@ void streamScanReloadState(SOperatorInfo* pOperator) { } else { pInfo->stateStore.updateInfoDestroy(pUpInfo); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } void addPrimaryKeyCol(SSDataBlock* pBlock, uint8_t type, int32_t bytes) { diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 14f2dfbada..a1702885c8 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -55,13 +55,13 @@ void destroyStreamCountAggOperatorInfo(void* param) { pInfo->pUpdated = NULL; colDataDestroy(&pInfo->twAggSup.timeWindowData); - blockDataDestroy(pInfo->pDelRes); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); tSimpleHashCleanup(pInfo->pStUpdated); tSimpleHashCleanup(pInfo->pStDeleted); cleanupGroupResInfo(&pInfo->groupResInfo); taosArrayDestroy(pInfo->historyWins); - blockDataDestroy(pInfo->pCheckpointRes); + pInfo->pCheckpointRes = blockDataDestroy(pInfo->pCheckpointRes); tSimpleHashCleanup(pInfo->pPkDeleted); @@ -70,9 +70,11 @@ void destroyStreamCountAggOperatorInfo(void* param) { bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; } -void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin, - SBuffInfo* pBuffInfo) { +int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin, + SBuffInfo* pBuffInfo) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t winCode = TSDB_CODE_SUCCESS; int32_t size = pAggSup->resultRowSize; pCurWin->winInfo.sessionWin.groupId = groupId; pCurWin->winInfo.sessionWin.win.skey = ts; @@ -80,26 +82,30 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, if (isSlidingCountWindow(pAggSup)) { if (pBuffInfo->winBuffOp == CREATE_NEW_WINDOW) { - pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, - (void**)&pCurWin->winInfo.pStatePos, &size); - code = TSDB_CODE_FAILED; + code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, + (void**)&pCurWin->winInfo.pStatePos, &size); + TSDB_CHECK_CODE(code, lino, _end); + + winCode = TSDB_CODE_FAILED; } else if (pBuffInfo->winBuffOp == MOVE_NEXT_WINDOW) { ASSERT(pBuffInfo->pCur); pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur); - code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, - (void**)&pCurWin->winInfo.pStatePos, &size); - if (code == TSDB_CODE_FAILED) { - pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, - (void**)&pCurWin->winInfo.pStatePos, &size); + winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, + (void**)&pCurWin->winInfo.pStatePos, &size); + if (winCode == TSDB_CODE_FAILED) { + code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, + (void**)&pCurWin->winInfo.pStatePos, &size); + TSDB_CHECK_CODE(code, lino, _end); } } else { pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount); - code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, - (void**)&pCurWin->winInfo.pStatePos, &size); - if (code == TSDB_CODE_FAILED) { - pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, - (void**)&pCurWin->winInfo.pStatePos, &size); + winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, + (void**)&pCurWin->winInfo.pStatePos, &size); + if (winCode == TSDB_CODE_FAILED) { + code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, + (void**)&pCurWin->winInfo.pStatePos, &size); + TSDB_CHECK_CODE(code, lino, _end); } } if (ts < pCurWin->winInfo.sessionWin.win.ekey) { @@ -108,10 +114,11 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, } else { code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, - (void**)&pCurWin->winInfo.pStatePos, &size); + (void**)&pCurWin->winInfo.pStatePos, &size, &winCode); + TSDB_CHECK_CODE(code, lino, _end); } - if (code == TSDB_CODE_SUCCESS) { + if (winCode == TSDB_CODE_SUCCESS) { pCurWin->winInfo.isOutput = true; } pCurWin->pWindowCount = @@ -120,19 +127,34 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, if (*pCurWin->pWindowCount == pAggSup->windowCount) { pBuffInfo->rebuildWindow = true; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void removeCountResult(SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey) { SSessionKey key = {0}; getSessionHashKey(pKey, &key); - tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); - tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); + int32_t code = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + if (code != TSDB_CODE_SUCCESS) { + qWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } + + code = tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); + if (code != TSDB_CODE_SUCCESS) { + qWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowInfo* pWinInfo, TSKEY* pTs, int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { SSessionKey sWinKey = pWinInfo->winInfo.sessionWin; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t num = 0; for (int32_t i = start; i < rows; i++) { if (pTs[i] < pWinInfo->winInfo.sessionWin.win.ekey) { @@ -150,7 +172,10 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI if (pWinInfo->winInfo.sessionWin.win.skey > pTs[start]) { needDelState = true; if (pStDeleted && pWinInfo->winInfo.isOutput) { - saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin); + code = saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } pWinInfo->winInfo.sessionWin.win.skey = pTs[start]; @@ -218,11 +243,12 @@ bool inCountSlidingWindow(SStreamAggSupporter* pAggSup, STimeWindow* pWin, SData static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pStUpdated, SSHashObj* pStDeleted) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamCountAggOperatorInfo* pInfo = pOperator->info; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; uint64_t groupId = pSDataBlock->info.id.groupId; - int64_t code = TSDB_CODE_SUCCESS; SResultRow* pResult = NULL; int32_t rows = pSDataBlock->info.rows; int32_t winRows = 0; @@ -236,8 +262,14 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; - blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2); + if (!pStartTsCol) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; + code = blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2); + TSDB_CHECK_CODE(code, lino, _end); + SStreamStateCur* pCur = NULL; COUNT_TYPE slidingRows = 0; @@ -250,7 +282,9 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } SCountWindowInfo curWin = {0}; buffInfo.rebuildWindow = false; - setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo); + code = setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo); + TSDB_CHECK_CODE(code, lino, _end); + if (!inCountSlidingWindow(pAggSup, &curWin.winInfo.sessionWin.win, &pSDataBlock->info)) { buffInfo.winBuffOp = MOVE_NEXT_WINDOW; continue; @@ -276,28 +310,27 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator, 0); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - qError("%s do stream count aggregate impl error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - break; - } - saveSessionOutputBuf(pAggSup, &curWin.winInfo); + TSDB_CHECK_CODE(code, lino, _end); + + code = saveSessionOutputBuf(pAggSup, &curWin.winInfo); + TSDB_CHECK_CODE(code, lino, _end); if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_COUNT_OP(pOperator)) { - saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { code = saveResult(curWin.winInfo, pStUpdated); - if (code != TSDB_CODE_SUCCESS) { - qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - break; - } + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { curWin.winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &key); - tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + code = + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } if (isSlidingCountWindow(pAggSup)) { @@ -314,6 +347,11 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } i += winRows; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } destroySBuffInfo(pAggSup, &buffInfo); } @@ -376,10 +414,13 @@ int32_t doStreamCountEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper return tlen; } -void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent) { +int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamCountAggOperatorInfo* pInfo = pOperator->info; if (!pInfo) { - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } // 4.checksum @@ -387,8 +428,8 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato int32_t dataLen = len - sizeof(uint32_t); void* pCksum = POINTER_SHIFT(buf, dataLen); if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { - qError("stream count state is invalid"); - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } } @@ -400,10 +441,13 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SCountWindowInfo curWin = {0}; buf = decodeSSessionKey(buf, &key); SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL}; - setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo); + code = setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo); + TSDB_CHECK_CODE(code, lino, _end); + buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize); - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, - sizeof(SResultWindowInfo)); + code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, + sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } // 2.twAggSup @@ -411,21 +455,37 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato // 3.dataVersion buf = taosDecodeFixedI64(buf, &pInfo->dataVersion); - return buf; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void doStreamCountSaveCheckpoint(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + void* pBuf = NULL; SStreamCountAggOperatorInfo* pInfo = pOperator->info; if (needSaveStreamOperatorInfo(&pInfo->basic)) { int32_t len = doStreamCountEncodeOpState(NULL, 0, pOperator, true); - void* buf = taosMemoryCalloc(1, len); - void* pBuf = buf; + pBuf = taosMemoryCalloc(1, len); + if (!pBuf) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } len = doStreamCountEncodeOpState(&pBuf, len, pOperator, true); pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME, - strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), buf, len); - taosMemoryFree(buf); + strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), pBuf, len); saveStreamOperatorStateComplete(&pInfo->basic); } + +_end: + taosMemoryFreeClear(pBuf); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { @@ -467,7 +527,9 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { } } -void doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) { +int32_t doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); @@ -482,35 +544,61 @@ void doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SAr SSessionKey key = {.win.skey = startDatas[i], .win.ekey = endDatas[i], .groupId = gpDatas[i]}; while (1) { SSessionKey curWin = {0}; - int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin); - if (code == TSDB_CODE_FAILED) { + int32_t winCode = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin); + if (winCode != TSDB_CODE_SUCCESS) { break; } doDeleteSessionWindow(pAggSup, &curWin); if (result) { - saveDeleteInfo(result, curWin); + code = saveDeleteInfo(result, curWin); + TSDB_CHECK_CODE(code, lino, _end); } } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -void deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, - SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd) { +int32_t deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, + SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); + if (!pWins) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + if (isSlidingCountWindow(pAggSup)) { - doDeleteCountWindows(pAggSup, pBlock, pWins); + code = doDeleteCountWindows(pAggSup, pBlock, pWins); + TSDB_CHECK_CODE(code, lino, _end); } else { - doDeleteTimeWindows(pAggSup, pBlock, pWins); + code = doDeleteTimeWindows(pAggSup, pBlock, pWins); + TSDB_CHECK_CODE(code, lino, _end); } removeSessionResults(pAggSup, pMapUpdate, pWins); - copyDeleteWindowInfo(pWins, pMapDelete); + code = copyDeleteWindowInfo(pWins, pMapDelete); + TSDB_CHECK_CODE(code, lino, _end); if (needAdd) { - copyDeleteWindowInfo(pWins, pPkDelete); + code = copyDeleteWindowInfo(pWins, pPkDelete); + TSDB_CHECK_CODE(code, lino, _end); } taosArrayDestroy(pWins); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExprSupp* pSup = &pOperator->exprSupp; SStreamCountAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; @@ -558,7 +646,9 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { bool add = pInfo->destHasPrimaryKey && IS_NORMAL_COUNT_OP(pOperator); - deleteCountWinState(&pInfo->streamAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, pInfo->pPkDeleted, add); + code = deleteCountWinState(&pInfo->streamAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, pInfo->pPkDeleted, + add); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_CLEAR) { doResetCountWindows(&pInfo->streamAggSup, pBlock); @@ -572,7 +662,8 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_CHECKPOINT) { pAggSup->stateStore.streamStateCommit(pAggSup->pState); doStreamCountSaveCheckpoint(pOperator); - copyDataBlock(pInfo->pCheckpointRes, pBlock); + code = copyDataBlock(pInfo->pCheckpointRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); continue; } else { ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); @@ -580,7 +671,8 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); @@ -591,12 +683,17 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated); - copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + code = closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated); + TSDB_CHECK_CODE(code, lino, _end); + + code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + TSDB_CHECK_CODE(code, lino, _end); + removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated); initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); if (pInfo->destHasPrimaryKey && IS_NORMAL_COUNT_OP(pOperator)) { copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pStDeleted); @@ -607,14 +704,24 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { return opRes; } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } setStreamOperatorCompleted(pOperator); return NULL; } void streamCountReleaseState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamEventAggOperatorInfo* pInfo = pOperator->info; int32_t resSize = sizeof(TSKEY); char* pBuff = taosMemoryCalloc(1, resSize); + if (pBuff) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _end); + } memcpy(pBuff, &pInfo->twAggSup.maxTs, sizeof(TSKEY)); qDebug("===stream=== count window operator relase state. "); pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_STATE_NAME, @@ -625,17 +732,25 @@ void streamCountReleaseState(SOperatorInfo* pOperator) { if (downstream->fpSet.releaseStreamStateFn) { downstream->fpSet.releaseStreamStateFn(downstream); } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } void streamCountReloadState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamCountAggOperatorInfo* pInfo = pOperator->info; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; int32_t size = 0; void* pBuf = NULL; - int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME, - strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size); - TSKEY ts = *(TSKEY*)pBuf; + code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME, + strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size); + TSDB_CHECK_CODE(code, lino, _end); + + TSKEY ts = *(TSKEY*)pBuf; pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); taosMemoryFree(pBuf); @@ -644,6 +759,11 @@ void streamCountReloadState(SOperatorInfo* pOperator) { downstream->fpSet.reloadStreamStateFn(downstream); } reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -725,7 +845,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME, strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), &buff, &len); if (res == TSDB_CODE_SUCCESS) { - doStreamCountDecodeOpState(buff, len, pOperator, true); + code = doStreamCountDecodeOpState(buff, len, pOperator, true); + TSDB_CHECK_CODE(code, lino, _error); taosMemoryFree(buff); } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamCountAgg, NULL, destroyStreamCountAggOperatorInfo, diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index e6ff4fe242..f79f29dc55 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -60,14 +60,14 @@ void destroyStreamEventOperatorInfo(void* param) { taosArrayDestroy(pInfo->pChildren); } colDataDestroy(&pInfo->twAggSup.timeWindowData); - blockDataDestroy(pInfo->pDelRes); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); tSimpleHashCleanup(pInfo->pSeUpdated); tSimpleHashCleanup(pInfo->pAllUpdated); tSimpleHashCleanup(pInfo->pSeDeleted); cleanupGroupResInfo(&pInfo->groupResInfo); taosArrayDestroy(pInfo->historyWins); - blockDataDestroy(pInfo->pCheckpointRes); + pInfo->pDelRes = blockDataDestroy(pInfo->pCheckpointRes); tSimpleHashCleanup(pInfo->pPkDeleted); @@ -108,14 +108,15 @@ int32_t getEndCondIndex(bool* pEnd, int32_t start, int32_t rows) { static bool isWindowIncomplete(SEventWindowInfo* pWinInfo) { return !(pWinInfo->pWinFlag->startFlag && pWinInfo->pWinFlag->endFlag); } -int32_t reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { +void reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { pAPI->streamStateReleaseBuf(pState, pPos, true); - return TSDB_CODE_SUCCESS; } -void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupId, bool* pStart, bool* pEnd, - int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey) { +int32_t setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupId, bool* pStart, bool* pEnd, + int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t winCode = TSDB_CODE_SUCCESS; int32_t size = pAggSup->resultRowSize; TSKEY ts = pTs[index]; bool start = pStart[index]; @@ -128,8 +129,8 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI SSessionKey leftWinKey = {.groupId = groupId}; void* pVal = NULL; int32_t len = 0; - code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len); - if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win)) { + winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len); + if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win)) { bool inWin = isInTimeWindow(&leftWinKey.win, ts, 0); setEventWindowInfo(pAggSup, &leftWinKey, pVal, pCurWin); if (inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag)) { @@ -140,9 +141,9 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI pAggSup->stateStore.streamStateFreeCur(pCur); pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); SSessionKey rightWinKey = {.groupId = groupId}; - code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len); + winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len); bool inWin = isInTimeWindow(&rightWinKey.win, ts, 0); - if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) { + if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) { int32_t endi = getEndCondIndex(pEnd, index, rows); if (endi < 0 || pTs[endi] >= rightWinKey.win.skey) { setEventWindowInfo(pAggSup, &rightWinKey, pVal, pCurWin); @@ -152,7 +153,9 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI } SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId}; - pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len); + code = pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len); + TSDB_CHECK_CODE(code, lino, _error); + setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin); pCurWin->pWinFlag->startFlag = start; pCurWin->pWinFlag->endFlag = end; @@ -162,8 +165,8 @@ _end: reuseOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->stateStore); pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur); pNextWinKey->groupId = groupId; - code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, pNextWinKey, NULL, 0); - if (code != TSDB_CODE_SUCCESS) { + winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, pNextWinKey, NULL, 0); + if (winCode != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pNextWinKey); } if (pCurWin->winInfo.pStatePos->needFree) { @@ -172,11 +175,19 @@ _end: pAggSup->stateStore.streamStateFreeCur(pCur); qDebug("===stream===set event next win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey, pCurWin->winInfo.sessionWin.win.ekey); + +_error: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey, TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; *pRebuild = false; if (!pWinInfo->pWinFlag->startFlag && !(starts[start])) { return 1; @@ -198,7 +209,10 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW if (pWin->skey > pTsData[i]) { if (pStDeleted && pWinInfo->winInfo.isOutput) { - saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin); + code = saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->winInfo.sessionWin); pWin->skey = pTsData[i]; @@ -232,10 +246,11 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - int32_t winNum = 0; SStreamEventAggOperatorInfo* pInfo = pOperator->info; SResultRow* pCurResult = NULL; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; @@ -252,18 +267,23 @@ static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pC break; } setEventWindowFlag(pAggSup, &nextWinInfo); - compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, - pStDeleted, false); + code = compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, + pStUpdated, pStDeleted, false); + TSDB_CHECK_CODE(code, lino, _end); + pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag; - winNum++; } - return winNum; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -bool doDeleteEventWindow(SStreamAggSupporter* pAggSup, SSHashObj* pSeUpdated, SSessionKey* pKey) { +void doDeleteEventWindow(SStreamAggSupporter* pAggSup, SSHashObj* pSeUpdated, SSessionKey* pKey) { pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey); removeSessionResult(pAggSup, pSeUpdated, pAggSup->pResultRows, pKey); - return true; } static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated, @@ -273,7 +293,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl SStreamEventAggOperatorInfo* pInfo = pOperator->info; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; uint64_t groupId = pSDataBlock->info.id.groupId; - int64_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; TSKEY* tsCols = NULL; SResultRow* pResult = NULL; int32_t winRows = 0; @@ -289,6 +310,10 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + if (!pColDataInfo) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } tsCols = (int64_t*)pColDataInfo->pData; } else { return; @@ -302,7 +327,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl goto _end; } int32_t statusStart = 0; - filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart); + code = filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart); + TSDB_CHECK_CODE(code, lino, _end); SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; @@ -313,10 +339,13 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } int32_t statusEnd = 0; - filterExecute(pInfo->pEndCondInfo, pSDataBlock, &pColEnd, NULL, paramEnd.numOfCols, &statusEnd); + code = filterExecute(pInfo->pEndCondInfo, pSDataBlock, &pColEnd, NULL, paramEnd.numOfCols, &statusEnd); + TSDB_CHECK_CODE(code, lino, _end); int32_t rows = pSDataBlock->info.rows; - blockDataEnsureCapacity(pAggSup->pScanBlock, rows); + code = blockDataEnsureCapacity(pAggSup->pScanBlock, rows); + TSDB_CHECK_CODE(code, lino, _end); + for (int32_t i = 0; i < rows; i += winRows) { if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, &pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i], NULL, 0)) { @@ -327,8 +356,10 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl bool allEqual = true; SEventWindowInfo curWin = {0}; SSessionKey nextWinKey = {0}; - setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, - &nextWinKey); + code = setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, + &nextWinKey); + TSDB_CHECK_CODE(code, lino, _end); + setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); bool rebuild = false; winRows = @@ -339,28 +370,35 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); - tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); + code = tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); + TSDB_CHECK_CODE(code, lino, _end); + doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin); if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) && !isWindowIncomplete(&curWin)) { - saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + TSDB_CHECK_CODE(code, lino, _end); } releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); SSessionKey tmpSeInfo = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &tmpSeInfo); - tSimpleHashPut(pStDeleted, &tmpSeInfo, sizeof(SSessionKey), NULL, 0); + code = tSimpleHashPut(pStDeleted, &tmpSeInfo, sizeof(SSessionKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); continue; } code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator, 0); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - break; - } - compactEventWindow(pOperator, &curWin, pInfo->pSeUpdated, pInfo->pSeDeleted, false); - saveSessionOutputBuf(pAggSup, &curWin.winInfo); + TSDB_CHECK_CODE(code, lino, _end); + + code = compactEventWindow(pOperator, &curWin, pInfo->pSeUpdated, pInfo->pSeDeleted, false); + TSDB_CHECK_CODE(code, lino, _end); + + code = saveSessionOutputBuf(pAggSup, &curWin.winInfo); + TSDB_CHECK_CODE(code, lino, _end); if (pInfo->isHistoryOp) { - saveResult(curWin.winInfo, pInfo->pAllUpdated); + code = saveResult(curWin.winInfo, pInfo->pAllUpdated); + TSDB_CHECK_CODE(code, lino, _end); } if (isWindowIncomplete(&curWin)) { @@ -369,18 +407,22 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator)) { - saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveResult(curWin.winInfo, pSeUpdated); + code = saveResult(curWin.winInfo, pSeUpdated); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { curWin.winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &key); - tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + code = + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } } @@ -389,6 +431,9 @@ _end: taosMemoryFree(pColStart); colDataDestroy(pColEnd); taosMemoryFree(pColEnd); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } int32_t doStreamEventEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator) { @@ -429,10 +474,13 @@ int32_t doStreamEventEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper return tlen; } -void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator) { +int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamEventAggOperatorInfo* pInfo = pOperator->info; if (!pInfo) { - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; @@ -440,9 +488,9 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato int32_t dataLen = len - sizeof(uint32_t); void* pCksum = POINTER_SHIFT(buf, dataLen); if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { - ASSERT(0); // debug qError("stream event state is invalid"); - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } // 1.streamAggSup.pResultRows @@ -452,10 +500,16 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); - pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &winfo.sessionWin, pAggSup->gap, - (void**)&winfo.pStatePos, &pAggSup->resultRowSize); + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAggSup->stateStore.streamStateSessionAddIfNotExist( + pAggSup->pState, &winfo.sessionWin, pAggSup->gap, (void**)&winfo.pStatePos, &pAggSup->resultRowSize, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + ASSERT(winCode == TSDB_CODE_SUCCESS); + buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + code = + tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } // 2.twAggSup @@ -463,7 +517,12 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato // 3.dataVersion buf = taosDecodeFixedI64(buf, &pInfo->dataVersion); - return buf; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void doStreamEventSaveCheckpoint(SOperatorInfo* pOperator) { @@ -504,6 +563,8 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { return NULL; } + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExprSupp* pSup = &pOperator->exprSupp; SStreamEventAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; @@ -549,7 +610,9 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { bool add = pInfo->destHasPrimaryKey && IS_NORMAL_EVENT_OP(pOperator); - deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted, pInfo->pPkDeleted, add); + code = deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted, + pInfo->pPkDeleted, add); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { pInfo->recvGetAll = true; @@ -561,7 +624,8 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState); doStreamEventSaveCheckpoint(pOperator); pInfo->reCkBlock = true; - copyDataBlock(pInfo->pCheckpointRes, pBlock); + code = copyDataBlock(pInfo->pCheckpointRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); continue; } else { ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); @@ -569,7 +633,8 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); @@ -579,14 +644,27 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated); - copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + code = closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated); + TSDB_CHECK_CODE(code, lino, _end); + + code = copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + TSDB_CHECK_CODE(code, lino, _end); + removeSessionDeleteResults(pInfo->pSeDeleted, pInfo->pUpdated); if (pInfo->isHistoryOp) { SArray* pHisWins = taosArrayInit(16, sizeof(SEventWindowInfo)); - copyUpdateResult(&pInfo->pAllUpdated, pHisWins, sessionKeyCompareAsc); - getMaxTsWins(pHisWins, pInfo->historyWins); + if (!pHisWins) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + code = copyUpdateResult(&pInfo->pAllUpdated, pHisWins, sessionKeyCompareAsc); + TSDB_CHECK_CODE(code, lino, _end); + + code = getMaxTsWins(pHisWins, pInfo->historyWins); + TSDB_CHECK_CODE(code, lino, _end); + taosArrayDestroy(pHisWins); } if (pInfo->destHasPrimaryKey && IS_NORMAL_EVENT_OP(pOperator)) { @@ -595,12 +673,18 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); SSDataBlock* resBlock = buildEventResult(pOperator); if (resBlock != NULL) { return resBlock; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } setStreamOperatorCompleted(pOperator); return NULL; } @@ -626,6 +710,8 @@ void streamEventReleaseState(SOperatorInfo* pOperator) { } void streamEventReloadState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamEventAggOperatorInfo* pInfo = pOperator->info; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; resetWinRange(&pAggSup->winRange); @@ -633,9 +719,11 @@ void streamEventReloadState(SOperatorInfo* pOperator) { SSessionKey seKey = {.win.skey = INT64_MIN, .win.ekey = INT64_MIN, .groupId = 0}; int32_t size = 0; void* pBuf = NULL; - int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_EVENT_OP_STATE_NAME, - strlen(STREAM_EVENT_OP_STATE_NAME), &pBuf, &size); - int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); + code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_EVENT_OP_STATE_NAME, + strlen(STREAM_EVENT_OP_STATE_NAME), &pBuf, &size); + TSDB_CHECK_CODE(code, lino, _end); + + int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); qDebug("===stream=== event window operator reload state. get result count:%d", num); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); @@ -663,15 +751,19 @@ void streamEventReloadState(SOperatorInfo* pOperator) { } setEventWindowFlag(pAggSup, &curInfo); if (!curInfo.pWinFlag->startFlag || curInfo.pWinFlag->endFlag) { - saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + TSDB_CHECK_CODE(code, lino, _end); continue; } - compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false); + code = compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false); qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey, curInfo.winInfo.sessionWin.groupId); + TSDB_CHECK_CODE(code, lino, _end); + if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { - saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + TSDB_CHECK_CODE(code, lino, _end); } if (!curInfo.pWinFlag->endFlag) { @@ -679,14 +771,18 @@ void streamEventReloadState(SOperatorInfo* pOperator) { } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveResult(curInfo.winInfo, pInfo->pSeUpdated); + code = saveResult(curInfo.winInfo, pInfo->pSeUpdated); + TSDB_CHECK_CODE(code, lino, _end); } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { if (!isCloseWindow(&curInfo.winInfo.sessionWin.win, &pInfo->twAggSup)) { - saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin); + code = saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin); + TSDB_CHECK_CODE(code, lino, _end); } SSessionKey key = {0}; getSessionHashKey(&curInfo.winInfo.sessionWin, &key); - tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, sizeof(SResultWindowInfo)); + code = + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } } taosMemoryFree(pBuf); @@ -696,6 +792,11 @@ void streamEventReloadState(SOperatorInfo* pOperator) { downstream->fpSet.reloadStreamStateFn(downstream); } reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -787,8 +888,9 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME, strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), &buff, &len); if (res == TSDB_CODE_SUCCESS) { - doStreamEventDecodeOpState(buff, len, pOperator); + code = doStreamEventDecodeOpState(buff, len, pOperator); taosMemoryFree(buff); + TSDB_CHECK_CODE(code, lino, _error); } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo, diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index 5f188c2c8a..518f7a33fb 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -26,12 +26,11 @@ #include "ttime.h" #include "function.h" +#include "operator.h" #include "querynodes.h" +#include "querytask.h" #include "tdatablock.h" #include "tfill.h" -#include "operator.h" -#include "querytask.h" - #define FILL_POS_INVALID 0 #define FILL_POS_START 1 @@ -56,8 +55,8 @@ TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { return win.skey; } -void setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) { - colDataSetVal(pCol, rowId, pCell->pData, pCell->isNull); +int32_t setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) { + return colDataSetVal(pCol, rowId, pCell->pData, pCell->isNull); } SResultCellData* getResultCell(SResultRowData* pRaw, int32_t index) { @@ -83,7 +82,7 @@ void* destroyFillColumnInfo(SFillColInfo* pFillCol, int32_t start, int32_t end) return NULL; } -void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { +void destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); tSimpleHashCleanup(pFillSup->pResMap); pFillSup->pResMap = NULL; @@ -96,36 +95,36 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { taosMemoryFree(pFillSup->nextNext.pRowVal); taosMemoryFree(pFillSup); - return NULL; } void destroySPoint(void* ptr) { - SPoint* point = (SPoint*) ptr; + SPoint* point = (SPoint*)ptr; taosMemoryFreeClear(point->val); } -void* destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) { +void destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) { taosArrayDestroyEx(pFillLinear->pEndPoints, destroySPoint); taosArrayDestroyEx(pFillLinear->pNextEndPoints, destroySPoint); taosMemoryFree(pFillLinear); - return NULL; } -void* destroyStreamFillInfo(SStreamFillInfo* pFillInfo) { + +void destroyStreamFillInfo(SStreamFillInfo* pFillInfo) { if (pFillInfo->type == TSDB_FILL_SET_VALUE || pFillInfo->type == TSDB_FILL_SET_VALUE_F || pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) { taosMemoryFreeClear(pFillInfo->pResRow->pRowVal); taosMemoryFreeClear(pFillInfo->pResRow); } - pFillInfo->pLinearInfo = destroyStreamFillLinearInfo(pFillInfo->pLinearInfo); + destroyStreamFillLinearInfo(pFillInfo->pLinearInfo); + pFillInfo->pLinearInfo = NULL; + taosArrayDestroy(pFillInfo->delRanges); taosMemoryFree(pFillInfo); - return NULL; } static void destroyStreamFillOperatorInfo(void* param) { SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param; - pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo); - pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup); + destroyStreamFillInfo(pInfo->pFillInfo); + destroyStreamFillSupporter(pInfo->pFillSup); pInfo->pRes = blockDataDestroy(pInfo->pRes); pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); @@ -161,19 +160,21 @@ void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupI int32_t curVLen = 0; int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); - ASSERT(code == TSDB_CODE_SUCCESS); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } pFillSup->cur.key = key.ts; } void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - void* pState = pOperator->pTaskInfo->streamInfo.pState; + void* pState = pOperator->pTaskInfo->streamInfo.pState; resetPrevAndNextWindow(pFillSup, pState, pAPI); SWinKey key = {.ts = ts, .groupId = groupId}; void* curVal = NULL; int32_t curVLen = 0; - bool hasCurKey = true; + bool hasCurKey = true; int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); if (code == TSDB_CODE_SUCCESS) { pFillSup->cur.key = key.ts; @@ -196,15 +197,10 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, pFillSup->prev.pRowVal = preVal; if (hasCurKey) { - code = pAPI->stateStore.streamStateCurNext(pState, pCur); - ASSERT(code == TSDB_CODE_SUCCESS); + pAPI->stateStore.streamStateCurNext(pState, pCur); } - code = pAPI->stateStore.streamStateCurNext(pState, pCur); - if (code != TSDB_CODE_SUCCESS) { - pAPI->stateStore.streamStateFreeCur(pCur); - pCur = NULL; - } + pAPI->stateStore.streamStateCurNext(pState, pCur); } else { pAPI->stateStore.streamStateFreeCur(pCur); pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key); @@ -218,16 +214,14 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, pFillSup->next.key = nextKey.ts; pFillSup->next.pRowVal = nextVal; if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { - code = pAPI->stateStore.streamStateCurNext(pState, pCur); + pAPI->stateStore.streamStateCurNext(pState, pCur); + SWinKey nextNextKey = {.groupId = groupId}; + void* nextNextVal = NULL; + int32_t nextNextVLen = 0; + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); if (code == TSDB_CODE_SUCCESS) { - SWinKey nextNextKey = {.groupId = groupId}; - void* nextNextVal = NULL; - int32_t nextNextVLen = 0; - code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); - if (code == TSDB_CODE_SUCCESS) { - pFillSup->nextNext.key = nextNextKey.ts; - pFillSup->nextNext.pRowVal = nextNextVal; - } + pFillSup->nextNext.key = nextNextKey.ts; + pFillSup->nextNext.pRowVal = nextNextVal; } } } @@ -286,13 +280,12 @@ static void calcDeltaData(SSDataBlock* pBlock, int32_t rowId, SResultRowData* pR } } -static void calcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, - int32_t numOfCol) { +static void calcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, int32_t numOfCol) { for (int32_t i = 0; i < numOfCol; i++) { if (!pFillCol[i].notFillCol) { int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); SResultCellData* pECell = getResultCell(pEndRow, slotId); - SPoint* pPoint = taosArrayGet(pEndPoins, slotId); + SPoint* pPoint = taosArrayGet(pEndPoins, slotId); pPoint->key = pEndRow->key; memcpy(pPoint->val, pECell->pData, pECell->bytes); } @@ -489,7 +482,10 @@ static bool checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t group if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) { return false; } - tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); + int32_t code = tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } return true; } @@ -513,7 +509,10 @@ static bool buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFill bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pColData, pBlock->info.rows); if (!filled) { SResultCellData* pCell = getResultCell(pResRow, slotId); - setRowCell(pColData, pBlock->info.rows, pCell); + int32_t code = setRowCell(pColData, pBlock->info.rows, pCell); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } } pBlock->info.rows++; @@ -531,7 +530,11 @@ static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; if (inWinRange(&pFillSup->winRange, &st)) { - buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); + bool res = buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); + if (!res) { + int32_t code = TSDB_CODE_FAILED; + qError("%s failed at line %d since block is full", __func__, __LINE__); + } } pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, pFillSup->interval.precision); @@ -540,10 +543,11 @@ static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { - uint64_t groupId = pBlock->info.id.groupId; - SWinKey key = {.groupId = groupId, .ts = pFillInfo->current}; + uint64_t groupId = pBlock->info.id.groupId; + SWinKey key = {.groupId = groupId, .ts = pFillInfo->current}; STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; - if ( ( pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId) ) || !inWinRange(&pFillSup->winRange, &st) ) { + if ((pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId)) || + !inWinRange(&pFillSup->winRange, &st)) { pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, pFillSup->interval.precision); pFillInfo->pLinearInfo->winIndex++; @@ -566,7 +570,10 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* if (pFillCol->notFillCol) { bool filled = fillIfWindowPseudoColumn(&tmp, pFillCol, pColData, index); if (!filled) { - setRowCell(pColData, index, pCell); + int32_t code = setRowCell(pColData, index, pCell); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } } else { if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) { @@ -575,7 +582,7 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* } SPoint* pEnd = taosArrayGet(pFillInfo->pLinearInfo->pEndPoints, slotId); double vCell = 0; - SPoint start = {0}; + SPoint start = {0}; start.key = pFillInfo->pResRow->key; start.val = pCell->pData; @@ -583,7 +590,10 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* cur.key = pFillInfo->current; cur.val = taosMemoryCalloc(1, pCell->bytes); taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type); - colDataSetVal(pColData, index, (const char*)cur.val, false); + int32_t code = colDataSetVal(pColData, index, (const char*)cur.val, false); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } destroySPoint(&cur); } } @@ -604,7 +614,11 @@ static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SRes static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { if (pFillInfo->needFill == false) { - buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); + bool res = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); + if (!res) { + int32_t code = TSDB_CODE_FAILED; + qError("%s failed at line %d since block is full", __func__, __LINE__); + } return; } @@ -663,6 +677,8 @@ static void doFillResults(SOperatorInfo* pOperator, SStreamFillSupporter* pFillS } static void doStreamFillImpl(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamFillOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamFillSupporter* pFillSup = pInfo->pFillSup; @@ -685,20 +701,28 @@ static void doStreamFillImpl(SOperatorInfo* pOperator) { keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); if (pInfo->pRes->info.rows == pInfo->pRes->info.capacity) { - blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + code = blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + TSDB_CHECK_CODE(code, lino, _end); return; } pInfo->srcRowIndex++; } doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); - blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + code = blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + TSDB_CHECK_CODE(code, lino, _end); blockDataCleanup(pInfo->pSrcBlock); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } -static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { - SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; - void* pState = pOp->pTaskInfo->streamInfo.pState; - +static int32_t buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; + void* pState = pOp->pTaskInfo->streamInfo.pState; SSDataBlock* pBlock = delRes; SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); @@ -707,61 +731,101 @@ static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_ SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); SColumnInfoData* pTbNameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - colDataSetVal(pStartCol, pBlock->info.rows, (const char*)&start, false); - colDataSetVal(pEndCol, pBlock->info.rows, (const char*)&end, false); + code = colDataSetVal(pStartCol, pBlock->info.rows, (const char*)&start, false); + TSDB_CHECK_CODE(code, lino, _end); + + code = colDataSetVal(pEndCol, pBlock->info.rows, (const char*)&end, false); + TSDB_CHECK_CODE(code, lino, _end); + colDataSetNULL(pUidCol, pBlock->info.rows); - colDataSetVal(pGroupCol, pBlock->info.rows, (const char*)&groupId, false); + code = colDataSetVal(pGroupCol, pBlock->info.rows, (const char*)&groupId, false); + TSDB_CHECK_CODE(code, lino, _end); + colDataSetNULL(pCalStartCol, pBlock->info.rows); colDataSetNULL(pCalEndCol, pBlock->info.rows); SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - void* tbname = NULL; - pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname, false); - if (tbname == NULL) { + void* tbname = NULL; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname, false, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + if (winCode != TSDB_CODE_SUCCESS) { colDataSetNULL(pTableCol, pBlock->info.rows); } else { + ASSERT(tbname); char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); - colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + code = colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + TSDB_CHECK_CODE(code, lino, _end); pAPI->stateStore.streamStateFreeVal(tbname); } pBlock->info.rows++; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId, - SSDataBlock* delRes) { +static int32_t buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId, + SSDataBlock* delRes) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillSupporter* pFillSup = pInfo->pFillSup; if (hasPrevWindow(pFillSup)) { TSKEY start = getNextWindowTs(pFillSup->prev.key, &pFillSup->interval); - buildDeleteRange(pOperator, start, endTs, groupId, delRes); + code = buildDeleteRange(pOperator, start, endTs, groupId, delRes); + TSDB_CHECK_CODE(code, lino, _end); } else if (hasNextWindow(pFillSup)) { TSKEY end = getPrevWindowTs(pFillSup->next.key, &pFillSup->interval); - buildDeleteRange(pOperator, startTs, end, groupId, delRes); + code = buildDeleteRange(pOperator, startTs, end, groupId, delRes); + TSDB_CHECK_CODE(code, lino, _end); } else { - buildDeleteRange(pOperator, startTs, endTs, groupId, delRes); + code = buildDeleteRange(pOperator, startTs, endTs, groupId, delRes); + TSDB_CHECK_CODE(code, lino, _end); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; +static int32_t doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStreamFillOperatorInfo* pInfo = pOperator->info; getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup); setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo); SWinKey key = {.ts = startTs, .groupId = groupId}; pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); if (!pInfo->pFillInfo->needFill) { - buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes); + code = buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes); + TSDB_CHECK_CODE(code, lino, _end); } else { STimeRange tw = { .skey = startTs, .ekey = endTs, .groupId = groupId, }; - taosArrayPush(pInfo->pFillInfo->delRanges, &tw); + void* tmp = taosArrayPush(pInfo->pFillInfo->delRanges, &tw); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void getWindowInfoByKey(SStorageAPI* pAPI, void* pState, TSKEY ts, int64_t groupId, SResultRowData* pWinData) { @@ -799,7 +863,8 @@ static void doDeleteFillFinalize(SOperatorInfo* pOperator) { getWindowFromDiscBuf(pOperator, range->skey, range->groupId, pInfo->pFillSup); TSKEY realEnd = range->ekey + 1; if (pInfo->pFillInfo->type == TSDB_FILL_NEXT && pInfo->pFillSup->next.key != realEnd) { - getWindowInfoByKey(pAPI, pOperator->pTaskInfo->streamInfo.pState, realEnd, range->groupId, &pInfo->pFillSup->next); + getWindowInfoByKey(pAPI, pOperator->pTaskInfo->streamInfo.pState, realEnd, range->groupId, + &pInfo->pFillSup->next); } setDeleteFillValueInfo(range->skey, range->ekey, pInfo->pFillSup, pInfo->pFillInfo); pFillInfo->delIndex++; @@ -810,9 +875,10 @@ static void doDeleteFillFinalize(SOperatorInfo* pOperator) { } } -static void doDeleteFillResult(SOperatorInfo* pOperator) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - +static int32_t doDeleteFillResult(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillInfo* pFillInfo = pInfo->pFillInfo; SSDataBlock* pBlock = pInfo->pSrcDelBlock; @@ -837,7 +903,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { while (pInfo->srcDelRowIndex < pBlock->info.rows) { TSKEY delTs = tsStarts[pInfo->srcDelRowIndex]; uint64_t delGroupId = groupIds[pInfo->srcDelRowIndex]; - int32_t code = TSDB_CODE_SUCCESS; + int32_t winCode = TSDB_CODE_SUCCESS; if (groupId != delGroupId) { break; } @@ -847,10 +913,8 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { SWinKey delKey = {.groupId = delGroupId, .ts = delTs}; if (delTs == nextKey.ts) { - code = pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); - if (code == TSDB_CODE_SUCCESS) { - code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, NULL, NULL); - } + pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); + winCode = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, NULL, NULL); // ts will be deleted later if (delTs != ts) { pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); @@ -858,7 +922,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); } endTs = TMAX(delTs, nextKey.ts - 1); - if (code != TSDB_CODE_SUCCESS) { + if (winCode != TSDB_CODE_SUCCESS) { break; } } @@ -866,10 +930,17 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { } pAPI->stateStore.streamStateFreeCur(pCur); - doDeleteFillResultImpl(pOperator, ts, endTs, groupId); + code = doDeleteFillResultImpl(pOperator, ts, endTs, groupId); + TSDB_CHECK_CODE(code, lino, _end); } pFillInfo->current = pFillInfo->end + 1; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) { @@ -879,25 +950,41 @@ static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) { pInfo->pFillInfo->delIndex = 0; } -static void doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock, SSDataBlock* pDstBlock) { +static int32_t doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock, + SSDataBlock* pDstBlock) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamFillOperatorInfo* pInfo = pOperator->info; SExprSupp* pSup = &pOperator->exprSupp; blockDataCleanup(pDstBlock); - blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows); + code = blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows); + TSDB_CHECK_CODE(code, lino, _end); + setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); - projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + code = projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); pDstBlock->info.rows = 0; pSup = &pInfo->pFillSup->notFillExprSup; setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); - projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + code = projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); + pDstBlock->info.id.groupId = pSrcBlock->info.id.groupId; - blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol); + code = blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamFillOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -959,7 +1046,9 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { pInfo->srcDelRowIndex = 0; blockDataCleanup(pInfo->pDelRes); pInfo->pFillSup->hasDelete = true; - doDeleteFillResult(pOperator); + code = doDeleteFillResult(pOperator); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); return pInfo->pDelRes; @@ -967,9 +1056,11 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { continue; } break; case STREAM_NORMAL: - case STREAM_INVALID: + case STREAM_INVALID: case STREAM_PULL_DATA: { - doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock); + code = doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock); + TSDB_CHECK_CODE(code, lino, _end); + memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); pInfo->srcRowIndex = -1; } break; @@ -983,7 +1074,9 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { } doStreamFillImpl(pOperator); - doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo); + code = doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo); + TSDB_CHECK_CODE(code, lino, _end); + memcpy(pInfo->pRes->info.parTbName, pInfo->pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN); pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; if (pInfo->pRes->info.rows > 0) { @@ -1003,6 +1096,14 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); return pInfo->pRes; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + setOperatorCompleted(pOperator); + resetStreamFillInfo(pInfo); + return NULL; } static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { @@ -1026,9 +1127,12 @@ static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); if (!pFillSup) { - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } pFillSup->numOfFillCols = numOfFillCols; int32_t numOfNotFillCols = 0; @@ -1040,48 +1144,89 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod pFillSup->interval = *pInterval; pFillSup->pAPI = pAPI; - int32_t code = initResultBuf(pFillSup); - if (code != TSDB_CODE_SUCCESS) { - destroyStreamFillSupporter(pFillSup); - return NULL; - } + code = initResultBuf(pFillSup); + TSDB_CHECK_CODE(code, lino, _end); SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore); - if (code != TSDB_CODE_SUCCESS) { - destroyStreamFillSupporter(pFillSup); - return NULL; - } + TSDB_CHECK_CODE(code, lino, _end); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pFillSup->pResMap = tSimpleHashInit(16, hashFn); pFillSup->hasDelete = false; + +_end: + if (code != TSDB_CODE_SUCCESS) { + destroyStreamFillSupporter(pFillSup); + pFillSup = NULL; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return pFillSup; } SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SStreamFillInfo)); + if (!pFillInfo) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + pFillInfo->start = INT64_MIN; pFillInfo->current = INT64_MIN; pFillInfo->end = INT64_MIN; pFillInfo->preRowKey = INT64_MIN; pFillInfo->needFill = false; pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo)); + if (!pFillInfo) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + pFillInfo->pLinearInfo->hasNext = false; pFillInfo->pLinearInfo->nextEnd = INT64_MIN; pFillInfo->pLinearInfo->pEndPoints = NULL; pFillInfo->pLinearInfo->pNextEndPoints = NULL; if (pFillSup->type == TSDB_FILL_LINEAR) { pFillInfo->pLinearInfo->pEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); + if (!pFillInfo->pLinearInfo->pEndPoints) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + pFillInfo->pLinearInfo->pNextEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); + if (!pFillInfo->pLinearInfo->pNextEndPoints) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) { SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); - SPoint value = {0}; + SPoint value = {0}; value.val = taosMemoryCalloc(1, pColData->info.bytes); - taosArrayPush(pFillInfo->pLinearInfo->pEndPoints, &value); + if (!value.val) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + void* tmpRes = taosArrayPush(pFillInfo->pLinearInfo->pEndPoints, &value); + if (!tmpRes) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } value.val = taosMemoryCalloc(1, pColData->info.bytes); - taosArrayPush(pFillInfo->pLinearInfo->pNextEndPoints, &value); + if (!value.val) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + tmpRes = taosArrayPush(pFillInfo->pLinearInfo->pNextEndPoints, &value); + if (!tmpRes) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } } pFillInfo->pLinearInfo->winIndex = 0; @@ -1090,8 +1235,18 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* if (pFillSup->type == TSDB_FILL_SET_VALUE || pFillSup->type == TSDB_FILL_SET_VALUE_F || pFillSup->type == TSDB_FILL_NULL || pFillSup->type == TSDB_FILL_NULL_F) { pFillInfo->pResRow = taosMemoryCalloc(1, sizeof(SResultRowData)); + if (!pFillInfo->pResRow) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + pFillInfo->pResRow->key = INT64_MIN; pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize); + if (!pFillInfo->pResRow->pRowVal) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); SResultCellData* pCell = getResultCell(pFillInfo->pResRow, i); @@ -1102,17 +1257,32 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pFillInfo->type = pFillSup->type; pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeRange)); + if (!pFillInfo->delRanges) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + pFillInfo->delIndex = 0; pFillInfo->curGroupId = 0; return pFillInfo; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + destroyStreamFillInfo(pFillInfo); + return NULL; } SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFillOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval; @@ -1120,14 +1290,18 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI); if (!pInfo->pFillSup) { - goto _error; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); - blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _error); + + code = blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _error); pInfo->pFillInfo = initStreamFillInfo(pInfo->pFillSup, pInfo->pRes); if (!pInfo->pFillInfo) { @@ -1166,42 +1340,43 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi } pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); - blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity); + if (!pInfo->pDelRes) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } + + code = blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _error); pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId; pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId; int32_t numOfOutputCols = 0; - int32_t code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, - &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, &numOfOutputCols, + COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); + TSDB_CHECK_CODE(code, lino, _error); code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); pInfo->srcRowIndex = -1; setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); return pOperator; _error: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } destroyStreamFillOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); pTaskInfo->code = code; diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 96c6712866..1ee6270677 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -171,8 +171,7 @@ static int32_t saveWinResult(SWinKey* pKey, SRowBuffPos* pPos, SSHashObj* pUpdat static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SRowBuffPos* pPos, SSHashObj* pUpdatedMap) { SWinKey key = {.ts = ts, .groupId = groupId}; - saveWinResult(&key, pPos, pUpdatedMap); - return TSDB_CODE_SUCCESS; + return saveWinResult(&key, pPos, pUpdatedMap); } static void removeResults(SArray* pWins, SSHashObj* pUpdatedMap) { @@ -183,7 +182,8 @@ static void removeResults(SArray* pWins, SSHashObj* pUpdatedMap) { if (tmp) { void* value = *(void**)tmp; taosMemoryFree(value); - tSimpleHashRemove(pUpdatedMap, pW, sizeof(SWinKey)); + int32_t tmpRes = tSimpleHashRemove(pUpdatedMap, pW, sizeof(SWinKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); } } } @@ -219,20 +219,22 @@ bool isOverdue(TSKEY ekey, STimeWindowAggSupp* pTwSup) { bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) { return isOverdue(pWin->ekey, pTwSup); } -static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) { +static void doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) { SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStreamIntervalOperatorInfo* pInfo = pOperator->info; SWinKey key = {.ts = ts, .groupId = groupId}; - tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey)); + int32_t tmpRes = tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); pAPI->stateStore.streamStateDel(pInfo->pState, &key); - return true; } static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } -static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDataBlock* pBlock, SArray* pUpWins, - SSHashObj* pUpdatedMap, SHashObj* pInvalidWins) { +static int32_t doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDataBlock* pBlock, SArray* pUpWins, + SSHashObj* pUpdatedMap, SHashObj* pInvalidWins) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startTsCols = (TSKEY*)pStartTsCol->pData; @@ -269,7 +271,8 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa if (pInvalidWins) { qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts, winRes.groupId, childId); - taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0); + code = taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); } SArray* chArray = *(void**)chIds; @@ -281,16 +284,26 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa continue; } } - bool res = doDeleteWindow(pOperator, win.skey, winGpId); - if (pUpWins && res) { - taosArrayPush(pUpWins, &winRes); + doDeleteWindow(pOperator, win.skey, winGpId); + if (pUpWins) { + void* tmp = taosArrayPush(pUpWins, &winRes); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } if (pUpdatedMap) { - tSimpleHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); + int32_t tmpRes = tSimpleHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); } getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); } while (win.ekey <= endTsCols[i]); } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SSHashObj* resWins) { @@ -317,6 +330,8 @@ static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp SHashObj* pPullDataMap, SSHashObj* closeWins, SArray* pDelWins, SOperatorInfo* pOperator) { qDebug("===stream===close interval window"); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; void* pIte = NULL; int32_t iter = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; @@ -351,15 +366,19 @@ static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp } if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { - int32_t code = saveWinResult(pWinKey, *(SRowBuffPos**)pIte, closeWins); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = saveWinResult(pWinKey, *(SRowBuffPos**)pIte, closeWins); + TSDB_CHECK_CODE(code, lino, _end); } - tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter); + int32_t tmpRes = tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); } } - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) { @@ -370,22 +389,30 @@ STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) { static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWins, int32_t* index, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; blockDataCleanup(pBlock); int32_t size = taosArrayGetSize(pWins); if (*index == size) { *index = 0; taosArrayClear(pWins); - return; + goto _end; } - blockDataEnsureCapacity(pBlock, size - *index); + code = blockDataEnsureCapacity(pBlock, size - *index); + TSDB_CHECK_CODE(code, lino, _end); + uint64_t uid = 0; for (int32_t i = *index; i < size; i++) { SWinKey* pWin = taosArrayGet(pWins, i); void* tbname = NULL; - pInfo->stateStore.streamStateGetParName(pInfo->pState, pWin->groupId, &tbname, false); - if (tbname == NULL) { + int32_t winCode = TSDB_CODE_SUCCESS; + code = pInfo->stateStore.streamStateGetParName(pInfo->pState, pWin->groupId, &tbname, false, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode != TSDB_CODE_SUCCESS) { appendDataToSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL); } else { + ASSERT(tbname); char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); appendDataToSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName); @@ -393,6 +420,11 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin pInfo->stateStore.streamStateFreeVal(tbname); (*index)++; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } void destroyFlusedPos(void* pRes) { @@ -433,11 +465,11 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { taosHashCleanup(pInfo->pPullDataMap); taosHashCleanup(pInfo->pFinalPullDataMap); taosArrayDestroy(pInfo->pPullWins); - blockDataDestroy(pInfo->pPullDataRes); + pInfo->pPullDataRes = blockDataDestroy(pInfo->pPullDataRes); taosArrayDestroy(pInfo->pDelWins); - blockDataDestroy(pInfo->pDelRes); - blockDataDestroy(pInfo->pMidRetriveRes); - blockDataDestroy(pInfo->pMidPulloverRes); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + pInfo->pMidRetriveRes = blockDataDestroy(pInfo->pMidRetriveRes); + pInfo->pMidPulloverRes = blockDataDestroy(pInfo->pMidPulloverRes); pInfo->stateStore.streamFileStateDestroy(pInfo->pState->pFileState); taosArrayDestroy(pInfo->pMidPullDatas); @@ -454,7 +486,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { pInfo->pUpdatedMap = NULL; tSimpleHashCleanup(pInfo->pDeletedMap); - blockDataDestroy(pInfo->pCheckpointRes); + pInfo->pCheckpointRes = blockDataDestroy(pInfo->pCheckpointRes); taosMemoryFreeClear(param); } @@ -508,8 +540,10 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt pInfo->basic.primaryPkIndex = pScanInfo->primaryKeyIndex; } -void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput, - SExecTaskInfo* pTaskInfo, SColumnInfoData* pTimeWindowData) { +int32_t compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput, + SExecTaskInfo* pTaskInfo, SColumnInfoData* pTimeWindowData) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; for (int32_t k = 0; k < numOfOutput; ++k) { if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) { if (!pTimeWindowData) { @@ -525,19 +559,27 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 SScalarParam out = {.columnData = &idata}; SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData}; - pDestCtx[k].sfp.process(&tw, 1, &out); + code = pDestCtx[k].sfp.process(&tw, 1, &out); + TSDB_CHECK_CODE(code, lino, _end); + pEntryInfo->numOfRes = 1; } else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) { - int32_t code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]); - if (code != TSDB_CODE_SUCCESS) { - qError("%s apply combine functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code)); - } + code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]); + TSDB_CHECK_CODE(code, lino, _end); } else if (pDestCtx[k].fpSet.combine == NULL) { char* funName = fmGetFuncName(pDestCtx[k].functionId); qError("%s error, combine funcion for %s is not implemented", GET_TASKID(pTaskInfo), funName); taosMemoryFreeClear(funName); + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } bool hasIntervalWindow(void* pState, SWinKey* pKey, SStateStore* pStore) { @@ -546,19 +588,28 @@ bool hasIntervalWindow(void* pState, SWinKey* pKey, SStateStore* pStore) { int32_t setIntervalOutputBuf(void* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, - SAggSupporter* pAggSup, SStateStore* pStore) { + SAggSupporter* pAggSup, SStateStore* pStore, int32_t* pWinCode) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SWinKey key = {.ts = win->skey, .groupId = groupId}; char* value = NULL; int32_t size = pAggSup->resultRowSize; - int32_t code = pStore->streamStateAddIfNotExist(pState, &key, (void**)&value, &size); + code = pStore->streamStateAddIfNotExist(pState, &key, (void**)&value, &size, pWinCode); + TSDB_CHECK_CODE(code, lino, _end); *pResult = (SRowBuffPos*)value; SResultRow* res = (SResultRow*)((*pResult)->pRowBuff); // set time window for current result res->win = (*win); - setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset); + code = setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } @@ -582,12 +633,24 @@ int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC); } -void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) { +int32_t addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SArray* childIds = taosArrayInit(8, sizeof(int32_t)); for (int32_t i = 0; i < size; i++) { - taosArrayPush(childIds, &i); + void* tmp = taosArrayPush(childIds, &i); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } - taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*)); + code = taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*)); + TSDB_CHECK_CODE(code, lino, _end); +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) { @@ -606,12 +669,16 @@ static void clearSpecialDataBlock(SSDataBlock* pBlock) { } static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; clearSpecialDataBlock(pBlock); int32_t size = taosArrayGetSize(array); if (size - (*pIndex) == 0) { - return; + goto _end; } - blockDataEnsureCapacity(pBlock, size - (*pIndex)); + code = blockDataEnsureCapacity(pBlock, size - (*pIndex)); + TSDB_CHECK_CODE(code, lino, _end); + SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); @@ -619,22 +686,40 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); for (; (*pIndex) < size; (*pIndex)++) { SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex)); - colDataSetVal(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); - colDataSetVal(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); - colDataSetVal(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false); - colDataSetVal(pCalStartTs, pBlock->info.rows, (const char*)&pWin->calWin.skey, false); - colDataSetVal(pCalEndTs, pBlock->info.rows, (const char*)&pWin->calWin.ekey, false); + code = colDataSetVal(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); + TSDB_CHECK_CODE(code, lino, _end); + + code = colDataSetVal(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); + TSDB_CHECK_CODE(code, lino, _end); + + code = colDataSetVal(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false); + TSDB_CHECK_CODE(code, lino, _end); + + code = colDataSetVal(pCalStartTs, pBlock->info.rows, (const char*)&pWin->calWin.skey, false); + TSDB_CHECK_CODE(code, lino, _end); + + code = colDataSetVal(pCalEndTs, pBlock->info.rows, (const char*)&pWin->calWin.ekey, false); + TSDB_CHECK_CODE(code, lino, _end); + pBlock->info.rows++; } if ((*pIndex) == size) { *pIndex = 0; taosArrayClear(array); } - blockDataUpdateTsWindow(pBlock, 0); + code = blockDataUpdateTsWindow(pBlock, 0); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } -static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, - SArray* pPullWins, int32_t numOfCh, SOperatorInfo* pOperator) { +static int32_t processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, + SArray* pPullWins, int32_t numOfCh, SOperatorInfo* pOperator, bool* pBeOver) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); TSKEY* tsData = (TSKEY*)pStartCol->pData; @@ -658,13 +743,15 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina if (taosArrayGetSize(chArray) == 0) { // pull data is over taosArrayDestroy(chArray); - taosHashRemove(pMap, &winRes, sizeof(SWinKey)); + int32_t tmpRes = taosHashRemove(pMap, &winRes, sizeof(SWinKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); res = true; qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts); void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey)); if (pFinalCh) { - taosHashRemove(pFinalMap, &winRes, sizeof(SWinKey)); + int32_t tmpRes = taosHashRemove(pFinalMap, &winRes, sizeof(SWinKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); doDeleteWindow(pOperator, winRes.ts, winRes.groupId); STimeWindow nextWin = getFinalTimeWindow(winRes.ts, pInterval); SPullWindowInfo pull = {.window = nextWin, @@ -676,12 +763,25 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina winRes.ts, winRes.groupId, numOfCh); if (IS_MID_INTERVAL_OP(pOperator)) { SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info; - taosArrayPush(pInfo->pMidPullDatas, &winRes); + + void* tmp = taosArrayPush(pInfo->pMidPullDatas, &winRes); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } else if (savePullWindow(&pull, pPullWins) == TSDB_CODE_SUCCESS) { - taosArrayPush(pInfo->pDelWins, &winRes); - addPullWindow(pMap, &winRes, numOfCh); + void* tmp = taosArrayPush(pInfo->pDelWins, &winRes); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + code = addPullWindow(pMap, &winRes, numOfCh); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); + code = tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); } qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh); } @@ -692,10 +792,20 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina winTs = taosTimeAdd(winTs, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); } } - return res; + if (pBeOver) { + *pBeOver = res; + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, int32_t childId) { +static int32_t addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, int32_t childId) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t size = taosArrayGetSize(wins); for (int32_t i = 0; i < size; i++) { SWinKey* winKey = taosArrayGet(wins, i); @@ -706,9 +816,12 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i .window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; // add pull data request if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { - addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild); + code = addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0); + code = tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); } qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild); } @@ -718,10 +831,17 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i qDebug("===stream===check final retrive %" PRId64 ",chid:%d", winKey->ts, index); if (index == -1) { qDebug("===stream===add final retrive %" PRId64, winKey->ts); - taosHashPut(pInfo->pFinalPullDataMap, winKey, sizeof(SWinKey), NULL, 0); + code = taosHashPut(pInfo->pFinalPullDataMap, winKey, sizeof(SWinKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); } } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void clearFunctionContext(SExprSupp* pSup) { @@ -734,11 +854,12 @@ int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SSta return pStore->streamStateGetByPos(pState, pPos, (void**)pResult); } -int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, - SGroupResInfo* pGroupResInfo) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - +void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, + SGroupResInfo* pGroupResInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SExprInfo* pExprInfo = pSup->pExprInfo; int32_t numOfExprs = pSup->numOfExprs; int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; @@ -749,9 +870,9 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { SRowBuffPos* pPos = *(SRowBuffPos**)taosArrayGet(pGroupResInfo->pRows, i); SResultRow* pRow = NULL; - int32_t code = getOutputBuf(pState, pPos, &pRow, &pAPI->stateStore); - uint64_t groupId = ((SWinKey*)pPos->pKey)->groupId; - ASSERT(code == 0); + code = getOutputBuf(pState, pPos, &pRow, &pAPI->stateStore); + TSDB_CHECK_CODE(code, lino, _end); + uint64_t groupId = ((SWinKey*)pPos->pKey)->groupId; doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); // no results, continue to check the next one if (pRow->numOfRows == 0) { @@ -760,9 +881,12 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat } if (pBlock->info.id.groupId == 0) { pBlock->info.id.groupId = groupId; - void* tbname = NULL; - if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, - false) < 0) { + void* tbname = NULL; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, + false, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + if (winCode != TSDB_CODE_SUCCESS) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); @@ -800,7 +924,8 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { - colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + code = colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + TSDB_CHECK_CODE(code, lino, _end); } } } @@ -809,8 +934,13 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat } pBlock->info.dataLoad = 1; - blockDataUpdateTsWindow(pBlock, 0); - return TSDB_CODE_SUCCESS; + code = blockDataUpdateTsWindow(pBlock, 0); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, @@ -844,6 +974,8 @@ bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkInde static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId, SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info; pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); @@ -871,7 +1003,8 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) { qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64, pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); - blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex); + code = blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex); + TSDB_CHECK_CODE(code, lino, _end); // timestamp of the data is incorrect if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { @@ -922,9 +1055,12 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat .window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; // add pull data request if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { - addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild); + code = addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); + code = tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); } } } else { @@ -950,13 +1086,13 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat } } - int32_t code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput, - pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->stateStore); + int32_t winCode = TSDB_CODE_SUCCESS; + code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput, + pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->stateStore, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + pResult = (SResultRow*)pResPos->pRowBuff; - if (pResult == NULL) { - qError("%s set interval output buff error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - break; - } + if (IS_FINAL_INTERVAL_OP(pOperator)) { forwardRows = 1; } else { @@ -969,17 +1105,20 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat .groupId = groupId, }; - if (pInfo->destHasPrimaryKey && code == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) { - tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0); + if (pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) { + code = tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { - saveWinResult(&key, pResPos, pUpdatedMap); + code = saveWinResult(&key, pResPos, pUpdatedMap); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { pResPos->beUpdated = true; - tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES); + code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, 1); @@ -1001,6 +1140,10 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat break; } } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } static inline int winPosCmprImpl(const void* pKey1, const void* pKey2) { @@ -1098,15 +1241,27 @@ int32_t encodeSPullWindowInfoArray(void** buf, SArray* pPullInfos) { return tlen; } -void* decodeSPullWindowInfoArray(void* buf, SArray* pPullInfos) { +int32_t decodeSPullWindowInfoArray(void* buf, SArray* pPullInfos, void** ppBuf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t size = 0; buf = taosDecodeFixedI32(buf, &size); for (int32_t i = 0; i < size; i++) { SPullWindowInfo item = {0}; buf = decodeSPullWindowInfo(buf, &item); - taosArrayPush(pPullInfos, &item); + void* tmp = taosArrayPush(pPullInfos, &item); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } - return buf; + (*ppBuf) = buf; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t doStreamIntervalEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator) { @@ -1167,17 +1322,20 @@ int32_t doStreamIntervalEncodeOpState(void** buf, int32_t len, SOperatorInfo* pO } void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; if (!pInfo) { - return; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } // 6.checksum int32_t dataLen = len - sizeof(uint32_t); void* pCksum = POINTER_SHIFT(buf, dataLen); if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { - qError("stream interval state is invalid"); - return; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } // 1.pResultRowHashTable @@ -1188,8 +1346,13 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera buf = decodeSWinKey(buf, &key); SRowBuffPos* pPos = NULL; int32_t resSize = pInfo->aggSup.resultRowSize; - pInfo->stateStore.streamStateAddIfNotExist(pInfo->pState, &key, (void**)&pPos, &resSize); - tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pPos, POINTER_BYTES); + int32_t winCode = TSDB_CODE_SUCCESS; + code = pInfo->stateStore.streamStateAddIfNotExist(pInfo->pState, &key, (void**)&pPos, &resSize, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + ASSERT(winCode == TSDB_CODE_SUCCESS); + + code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pPos, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } // 2.twAggSup @@ -1201,22 +1364,38 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera for (int32_t i = 0; i < size; i++) { SWinKey key = {0}; SArray* pArray = taosArrayInit(0, sizeof(int32_t)); + if (!pArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + buf = decodeSWinKey(buf, &key); int32_t chSize = 0; buf = taosDecodeFixedI32(buf, &chSize); for (int32_t i = 0; i < chSize; i++) { int32_t chId = 0; buf = taosDecodeFixedI32(buf, &chId); - taosArrayPush(pArray, &chId); + void* tmp = taosArrayPush(pArray, &chId); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } - taosHashPut(pInfo->pPullDataMap, &key, sizeof(SWinKey), &pArray, POINTER_BYTES); + code = taosHashPut(pInfo->pPullDataMap, &key, sizeof(SWinKey), &pArray, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } // 4.pPullWins - buf = decodeSPullWindowInfoArray(buf, pInfo->pPullWins); + code = decodeSPullWindowInfoArray(buf, pInfo->pPullWins, &buf); + TSDB_CHECK_CODE(code, lino, _end); // 5.dataVersion buf = taosDecodeFixedI64(buf, &pInfo->dataVersion); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) { @@ -1233,14 +1412,26 @@ void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) { } } -static void copyIntervalDeleteKey(SSHashObj* pMap, SArray* pWins) { +static int32_t copyIntervalDeleteKey(SSHashObj* pMap, SArray* pWins) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pMap, pIte, &iter)) != NULL) { void* pKey = tSimpleHashGetKey(pIte, NULL); - taosArrayPush(pWins, pKey); + void* tmp = taosArrayPush(pWins, pKey); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } tSimpleHashClear(pMap); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static SSDataBlock* buildIntervalResult(SOperatorInfo* pOperator) { @@ -1280,18 +1471,31 @@ static SSDataBlock* buildIntervalResult(SOperatorInfo* pOperator) { } int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(*ppWinUpdated, pIte, &iter)) != NULL) { - taosArrayPush(pUpdated, pIte); + void* tmp = taosArrayPush(pUpdated, pIte); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } taosArraySort(pUpdated, compar); tSimpleHashCleanup(*ppWinUpdated); *ppWinUpdated = NULL; - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; @@ -1375,18 +1579,30 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pBlock->info.type == STREAM_CLEAR) { SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL; - doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap); + code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap); + TSDB_CHECK_CODE(code, lino, _end); + if (IS_FINAL_INTERVAL_OP(pOperator)) { int32_t chId = getChildIndex(pBlock); - addRetriveWindow(delWins, pInfo, chId); + code = addRetriveWindow(delWins, pInfo, chId); + TSDB_CHECK_CODE(code, lino, _end); + if (pBlock->info.type != STREAM_CLEAR) { - taosArrayAddAll(pInfo->pDelWins, delWins); + void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } taosArrayDestroy(delWins); continue; } removeResults(delWins, pInfo->pUpdatedMap); - taosArrayAddAll(pInfo->pDelWins, delWins); + void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } taosArrayDestroy(delWins); doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); @@ -1404,27 +1620,34 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { break; } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_INTERVAL_OP(pOperator)) { pInfo->recvGetAll = true; - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); + code = getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_RETRIEVE) { if (!IS_FINAL_INTERVAL_OP(pOperator)) { pInfo->recvRetrive = true; - copyDataBlock(pInfo->pMidRetriveRes, pBlock); + code = copyDataBlock(pInfo->pMidRetriveRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE; - doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pInfo->pUpdatedMap, NULL); + code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pInfo->pUpdatedMap, NULL); + TSDB_CHECK_CODE(code, lino, _end); break; } continue; } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_INTERVAL_OP(pOperator)) { - processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins, - pInfo->numOfChild, pOperator); + code = processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins, + pInfo->numOfChild, pOperator, NULL); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { return pBlock; } else if (pBlock->info.type == STREAM_CHECKPOINT) { pAPI->stateStore.streamStateCommit(pInfo->pState); doStreamIntervalSaveCheckpoint(pOperator); - copyDataBlock(pInfo->pCheckpointRes, pBlock); + code = copyDataBlock(pInfo->pCheckpointRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + continue; } else if (IS_FINAL_INTERVAL_OP(pOperator) && pBlock->info.type == STREAM_MID_RETRIEVE) { continue; @@ -1434,7 +1657,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap, pInfo->pDeletedMap); @@ -1447,19 +1671,24 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins); } if (IS_FINAL_INTERVAL_OP(pOperator)) { - closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, - pInfo->pPullDataMap, pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); + code = closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, + pInfo->pPullDataMap, pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->destHasPrimaryKey) { - copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins); + code = copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins); + TSDB_CHECK_CODE(code, lino, _end); } } pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs; - copyUpdateResult(&pInfo->pUpdatedMap, pInfo->pUpdated, winPosCmprImpl); + code = copyUpdateResult(&pInfo->pUpdatedMap, pInfo->pUpdated, winPosCmprImpl); + TSDB_CHECK_CODE(code, lino, _end); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); SSDataBlock* resBlock = buildIntervalResult(pOperator); if (resBlock != NULL) { @@ -1472,6 +1701,11 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pMidRetriveRes; } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + setStreamOperatorCompleted(pOperator); return NULL; } @@ -1530,25 +1764,32 @@ static void streamIntervalReleaseState(SOperatorInfo* pOperator) { } void streamIntervalReloadState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL && pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) { int32_t size = 0; void* pBuf = NULL; - int32_t code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, - strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size); - if (code == 0) { - TSKEY ts = *(TSKEY*)pBuf; - taosMemoryFree(pBuf); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); - pInfo->stateStore.streamStateReloadInfo(pInfo->pState, ts); - } + code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, + strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size); + TSDB_CHECK_CODE(code, lino, _end); + + TSKEY ts = *(TSKEY*)pBuf; + taosMemoryFree(pBuf); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); + pInfo->stateStore.streamStateReloadInfo(pInfo->pState, ts); } SOperatorInfo* downstream = pOperator->pDownstream[0]; if (downstream->fpSet.reloadStreamStateFn) { downstream->fpSet.reloadStreamStateFn(downstream); } reloadFromDownStream(downstream, pInfo); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -1622,8 +1863,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo)); pInfo->pullIndex = 0; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); - pInfo->pFinalPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); + pInfo->pPullDataMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK); + pInfo->pFinalPullDataMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK); pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE); pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pInfo->ignoreExpiredDataSaved = false; @@ -1700,7 +1941,7 @@ _error: void destroyStreamAggSupporter(SStreamAggSupporter* pSup) { tSimpleHashCleanup(pSup->pResultRows); destroyDiskbasedBuf(pSup->pResultBuf); - blockDataDestroy(pSup->pScanBlock); + pSup->pScanBlock = blockDataDestroy(pSup->pScanBlock); pSup->stateStore.streamFileStateDestroy(pSup->pState->pFileState); taosMemoryFreeClear(pSup->pState); taosMemoryFreeClear(pSup->pDummyCtx); @@ -1725,14 +1966,14 @@ void destroyStreamSessionAggOperatorInfo(void* param) { } colDataDestroy(&pInfo->twAggSup.timeWindowData); - blockDataDestroy(pInfo->pDelRes); - blockDataDestroy(pInfo->pWinBlock); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + pInfo->pWinBlock = blockDataDestroy(pInfo->pWinBlock); tSimpleHashCleanup(pInfo->pStUpdated); tSimpleHashCleanup(pInfo->pStDeleted); cleanupGroupResInfo(&pInfo->groupResInfo); taosArrayDestroy(pInfo->historyWins); - blockDataDestroy(pInfo->pCheckpointRes); + pInfo->pCheckpointRes = blockDataDestroy(pInfo->pCheckpointRes); tSimpleHashCleanup(pInfo->pPkDeleted); taosMemoryFreeClear(param); @@ -1869,24 +2110,27 @@ bool inWinRange(STimeWindow* range, STimeWindow* cur) { return false; } -int32_t clearOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { - return pAPI->streamStateClearBuff(pState, pPos); -} +void clearOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { pAPI->streamStateClearBuff(pState, pPos); } -void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, - SResultWindowInfo* pCurWin) { +int32_t setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, + SResultWindowInfo* pCurWin) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; pCurWin->sessionWin.groupId = groupId; pCurWin->sessionWin.win.skey = startTs; pCurWin->sessionWin.win.ekey = endTs; int32_t size = pAggSup->resultRowSize; - int32_t code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, - pAggSup->gap, (void**)&pCurWin->pStatePos, &size); - if (code == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) { - code = TSDB_CODE_FAILED; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap, + (void**)&pCurWin->pStatePos, &size, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) { + winCode = TSDB_CODE_FAILED; clearOutputBuf(pAggSup->pState, pCurWin->pStatePos, &pAggSup->pSessionAPI->stateStore); } - if (code == TSDB_CODE_SUCCESS) { + if (winCode == TSDB_CODE_SUCCESS) { pCurWin->isOutput = true; if (pCurWin->pStatePos->needFree) { pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->sessionWin); @@ -1897,43 +2141,62 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT } qDebug("===stream===set session window buff .start:%" PRId64 ",end:%" PRId64 ",groupid:%" PRIu64, pCurWin->sessionWin.win.skey, pCurWin->sessionWin.win.ekey, pCurWin->sessionWin.groupId); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -int32_t getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo) { +void getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo, + int32_t* pWinCode) { int32_t size = 0; - int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, - (void**)&pWinInfo->pStatePos, &size); - if (code != TSDB_CODE_SUCCESS) { - return code; + (*pWinCode) = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, + (void**)&pWinInfo->pStatePos, &size); + if ((*pWinCode) != TSDB_CODE_SUCCESS) { + return; } pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur); - return TSDB_CODE_SUCCESS; } -void saveDeleteInfo(SArray* pWins, SSessionKey key) { + +int32_t saveDeleteInfo(SArray* pWins, SSessionKey key) { // key.win.ekey = key.win.skey; - taosArrayPush(pWins, &key); -} - -void saveDeleteRes(SSHashObj* pStDelete, SSessionKey key) { - key.win.ekey = key.win.skey; - tSimpleHashPut(pStDelete, &key, sizeof(SSessionKey), NULL, 0); -} - -int32_t releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { - pAPI->streamStateReleaseBuf(pState, pPos, false); + void* res = taosArrayPush(pWins, &key); + if (!res) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return TSDB_CODE_OUT_OF_MEMORY; + } return TSDB_CODE_SUCCESS; } +int32_t saveDeleteRes(SSHashObj* pStDelete, SSessionKey key) { + key.win.ekey = key.win.skey; + return tSimpleHashPut(pStDelete, &key, sizeof(SSessionKey), NULL, 0); +} + +void releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { + pAPI->streamStateReleaseBuf(pState, pPos, false); +} + void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SSessionKey key = {0}; getSessionHashKey(pKey, &key); void* pVal = tSimpleHashGet(pHashMap, &key, sizeof(SSessionKey)); if (pVal) { releaseOutputBuf(pAggSup->pState, *(void**)pVal, &pAggSup->pSessionAPI->stateStore); - tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + code = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + if (code != TSDB_CODE_SUCCESS) { + qWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } + } + code = tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); + if (code != TSDB_CODE_SUCCESS) { + qWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); } - tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); } void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey) { @@ -1951,7 +2214,8 @@ void removeSessionDeleteResults(SSHashObj* pHashMap, SArray* pWins) { if (!pWin) continue; SSessionKey key = {0}; getSessionHashKey(&pWin->sessionWin, &key); - tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); } } @@ -1968,7 +2232,8 @@ void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SAr void* pVal = tSimpleHashGet(pHashMap, &key, sizeof(SSessionKey)); if (pVal) { releaseOutputBuf(pAggSup->pState, *(void**)pVal, &pAggSup->pSessionAPI->stateStore); - tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); } } } @@ -1982,7 +2247,10 @@ int32_t updateSessionWindowInfo(SStreamAggSupporter* pAggSup, SResultWindowInfo* } if (pWinInfo->sessionWin.win.skey > pStartTs[i]) { if (pStDeleted && pWinInfo->isOutput) { - saveDeleteRes(pStDeleted, pWinInfo->sessionWin); + int32_t code = saveDeleteRes(pStDeleted, pWinInfo->sessionWin); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->sessionWin); pWinInfo->sessionWin.win.skey = pStartTs[i]; @@ -2008,32 +2276,39 @@ static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pR int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput, SOperatorInfo* pOperator, int64_t winDelta) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - int32_t code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset); - if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } + code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset); + TSDB_CHECK_CODE(code, lino, _end); + updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, winDelta); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput); - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) { +void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) { pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey); SSessionKey hashKey = {0}; getSessionHashKey(pKey, &hashKey); - tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey)); - return true; + int32_t code = tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey)); + if (code != TSDB_CODE_SUCCESS) { + qWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } -int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo) { +void setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo) { void* pVal = tSimpleHashGet(pStUpdated, &pWinInfo->sessionWin, sizeof(SSessionKey)); if (pVal) { SResultWindowInfo* pWin = pVal; pWinInfo->isOutput = pWin->isOutput; } - return TSDB_CODE_SUCCESS; } void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin, @@ -2051,14 +2326,20 @@ void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, pAggSup->stateStore.streamStateFreeCur(pCur); } -void compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup, - SExecTaskInfo* pTaskInfo, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, - SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap) { +int32_t compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup, + SExecTaskInfo* pTaskInfo, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, + SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SResultRow* pCurResult = NULL; int32_t numOfOutput = pSup->numOfExprs; - initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); + code = initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); + TSDB_CHECK_CODE(code, lino, _end); + SResultRow* pWinResult = NULL; - initSessionOutputBuf(pNextWin, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset); + code = initSessionOutputBuf(pNextWin, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset); + TSDB_CHECK_CODE(code, lino, _end); + pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, pNextWin->sessionWin.win.ekey); memcpy(pCurWin->pStatePos->pKey, &pCurWin->sessionWin, sizeof(SSessionKey)); @@ -2067,30 +2348,44 @@ void compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindo winDelta = pAggSup->gap; } updateTimeWindowInfo(&pTwAggSup->timeWindowData, &pCurWin->sessionWin.win, winDelta); - compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pTwAggSup->timeWindowData); - tSimpleHashRemove(pStUpdated, &pNextWin->sessionWin, sizeof(SSessionKey)); + code = compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pTwAggSup->timeWindowData); + TSDB_CHECK_CODE(code, lino, _end); + + int32_t tmpRes = tSimpleHashRemove(pStUpdated, &pNextWin->sessionWin, sizeof(SSessionKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); + if (pNextWin->isOutput && pStDeleted) { qDebug("===stream=== save delete window info %" PRId64 ", %" PRIu64, pNextWin->sessionWin.win.skey, pNextWin->sessionWin.groupId); - saveDeleteRes(pStDeleted, pNextWin->sessionWin); + int32_t code = saveDeleteRes(pStDeleted, pNextWin->sessionWin); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } removeSessionResult(pAggSup, pStUpdated, pAggSup->pResultRows, &pNextWin->sessionWin); doDeleteSessionWindow(pAggSup, &pNextWin->sessionWin); releaseOutputBuf(pAggSup->pState, pNextWin->pStatePos, &pAggSup->pSessionAPI->stateStore); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated, - SSHashObj* pStDeleted, bool addGap) { - SExprSupp* pSup = &pOperator->exprSupp; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - int32_t winNum = 0; - + SSHashObj* pStDeleted, bool addGap, int32_t* pWinNum) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + int32_t winNum = 0; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SResultRow* pCurResult = NULL; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; - // initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); + // Just look for the window behind StartIndex while (1) { SResultWindowInfo winInfo = {0}; @@ -2100,10 +2395,20 @@ static int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); break; } - compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, &winInfo, pStUpdated, pStDeleted, true); + code = + compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, &winInfo, pStUpdated, pStDeleted, true); + TSDB_CHECK_CODE(code, lino, _end); winNum++; } - return winNum; + if (pWinNum) { + (*pWinNum) = winNum; + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void compactSessionSemiWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin) { @@ -2143,7 +2448,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData SStreamSessionAggOperatorInfo* pInfo = pOperator->info; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; uint64_t groupId = pSDataBlock->info.id.groupId; - int64_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SResultRow* pResult = NULL; int32_t rows = pSDataBlock->info.rows; int32_t winRows = 0; @@ -2185,7 +2491,9 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData continue; } SResultWindowInfo winInfo = {0}; - setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo); + code = setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo); + TSDB_CHECK_CODE(code, lino, _end); + // coverity scan error if (!winInfo.pStatePos) { continue; @@ -2200,36 +2508,42 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData } code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator, winDelta); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - qError("%s do stream session aggregate impl error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - break; - } - compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted, addGap); - saveSessionOutputBuf(pAggSup, &winInfo); + TSDB_CHECK_CODE(code, lino, _end); + + code = compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted, addGap, NULL); + TSDB_CHECK_CODE(code, lino, _end); + + code = saveSessionOutputBuf(pAggSup, &winInfo); + TSDB_CHECK_CODE(code, lino, _end); if (pInfo->destHasPrimaryKey && winInfo.isOutput && IS_NORMAL_SESSION_OP(pOperator)) { - saveDeleteRes(pInfo->pPkDeleted, winInfo.sessionWin); + code = saveDeleteRes(pInfo->pPkDeleted, winInfo.sessionWin); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { code = saveResult(winInfo, pStUpdated); - if (code != TSDB_CODE_SUCCESS) { - qError("%s do stream session aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), - tstrerror(code)); - break; - } + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&winInfo.sessionWin, &key); - tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); + code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } i += winRows; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } -void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) { +int32_t doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); @@ -2245,10 +2559,17 @@ void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArr } doDeleteSessionWindow(pAggSup, &curWin); if (result) { - saveDeleteInfo(result, curWin); + code = saveDeleteInfo(result, curWin); + TSDB_CHECK_CODE(code, lino, _end); } } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) { @@ -2273,6 +2594,8 @@ inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) { } void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; blockDataCleanup(pBlock); @@ -2280,7 +2603,9 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo if (size == 0) { return; } - blockDataEnsureCapacity(pBlock, size); + code = blockDataEnsureCapacity(pBlock, size); + TSDB_CHECK_CODE(code, lino, _end); + int32_t iter = 0; while (((*Ite) = tSimpleHashIterate(pStDeleted, *Ite, &iter)) != NULL) { if (pBlock->info.rows + 1 > pBlock->info.capacity) { @@ -2288,38 +2613,63 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo } SSessionKey* res = tSimpleHashGetKey(*Ite, NULL); SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - colDataSetVal(pStartTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); + code = colDataSetVal(pStartTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); + TSDB_CHECK_CODE(code, lino, _end); + SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); - colDataSetVal(pEndTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); + code = colDataSetVal(pEndTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); + TSDB_CHECK_CODE(code, lino, _end); + SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); colDataSetNULL(pUidCol, pBlock->info.rows); + SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - colDataSetVal(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false); + code = colDataSetVal(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false); + TSDB_CHECK_CODE(code, lino, _end); + SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); colDataSetNULL(pCalStCol, pBlock->info.rows); + SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); colDataSetNULL(pCalEdCol, pBlock->info.rows); SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); + if (!pTableCol) { + TSDB_CHECK_CODE(code, lino, _end); + } - void* tbname = NULL; - pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname, false); - if (tbname == NULL) { + void* tbname = NULL; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname, false, + &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode != TSDB_CODE_SUCCESS) { colDataSetNULL(pTableCol, pBlock->info.rows); } else { char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); - colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + code = colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + TSDB_CHECK_CODE(code, lino, _end); pAPI->stateStore.streamStateFreeVal(tbname); } pBlock->info.rows += 1; } + +_end: if ((*Ite) == NULL) { tSimpleHashClear(pStDeleted); } + + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } -static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) { +static int32_t rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) { + int32_t winCode = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; @@ -2346,23 +2696,22 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS while (1) { SResultWindowInfo childWin = {0}; childWin.sessionWin = *pWinKey; - int32_t code = getSessionWinBuf(pChAggSup, pCur, &childWin); + getSessionWinBuf(pChAggSup, pCur, &childWin, &winCode); - if (code == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &childWin.sessionWin.win)) { + if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &childWin.sessionWin.win)) { releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore); continue; } - if (code == TSDB_CODE_SUCCESS && inWinRange(&pWinKey->win, &childWin.sessionWin.win)) { + if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pWinKey->win, &childWin.sessionWin.win)) { if (num == 0) { - setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin); + code = setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin); + TSDB_CHECK_CODE(code, lino, _end); + parentWin.sessionWin = childWin.sessionWin; memcpy(parentWin.pStatePos->pKey, &parentWin.sessionWin, sizeof(SSessionKey)); code = initSessionOutputBuf(&parentWin, &pResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore); - break; - } + TSDB_CHECK_CODE(code, lino, _end); } num++; parentWin.sessionWin.win.skey = TMIN(parentWin.sessionWin.win.skey, childWin.sessionWin.win.skey); @@ -2370,10 +2719,17 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS memcpy(parentWin.pStatePos->pKey, &parentWin.sessionWin, sizeof(SSessionKey)); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin.sessionWin.win, pAggSup->gap); - initSessionOutputBuf(&childWin, &pChResult, pChild->exprSupp.pCtx, numOfOutput, - pChild->exprSupp.rowEntryInfoOffset); - compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); - compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL, true); + code = initSessionOutputBuf(&childWin, &pChResult, pChild->exprSupp.pCtx, numOfOutput, + pChild->exprSupp.rowEntryInfoOffset); + TSDB_CHECK_CODE(code, lino, _end); + + code = compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, + &pInfo->twAggSup.timeWindowData); + TSDB_CHECK_CODE(code, lino, _end); + + code = compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL, true, NULL); + TSDB_CHECK_CODE(code, lino, _end); + releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore); } else { releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore); @@ -2383,42 +2739,65 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS pAPI->stateStore.streamStateFreeCur(pCur); } if (num > 0) { - saveResult(parentWin, pStUpdated); - saveSessionOutputBuf(pAggSup, &parentWin); + code = saveResult(parentWin, pStUpdated); + TSDB_CHECK_CODE(code, lino, _end); + + code = saveSessionOutputBuf(pAggSup, &parentWin); + TSDB_CHECK_CODE(code, lino, _end); } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } + return code; } int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { SResultWindowInfo* pWinInfo = pIte; if (isCloseWindow(&pWinInfo->sessionWin.win, pTwSup)) { if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) { - int32_t code = saveResult(*pWinInfo, pClosed); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = saveResult(*pWinInfo, pClosed); + TSDB_CHECK_CODE(code, lino, _end); } SSessionKey* pKey = tSimpleHashGetKey(pIte, NULL); - tSimpleHashIterateRemove(pHashMap, pKey, sizeof(SSessionKey), &pIte, &iter); + code = tSimpleHashIterateRemove(pHashMap, pKey, sizeof(SSessionKey), &pIte, &iter); + TSDB_CHECK_CODE(code, lino, _end); } } - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs) { +static int32_t closeChildSessionWindow(SArray* pChildren, TSKEY maxTs) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t size = taosArrayGetSize(pChildren); for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info; pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs); - closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL); + code = closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL); + TSDB_CHECK_CODE(code, lino, _end); } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) { +void getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) { void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { @@ -2427,20 +2806,32 @@ int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) { continue; } pWinInfo->pStatePos->beUpdated = false; - saveResult(*pWinInfo, pStUpdated); + int32_t code = saveResult(*pWinInfo, pStUpdated); + if (code != TSDB_CODE_SUCCESS) { + pWinInfo->pStatePos->beUpdated = true; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } - return TSDB_CODE_SUCCESS; } -void copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) { +int32_t copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t size = taosArrayGetSize(pResWins); for (int32_t i = 0; i < size; i++) { SSessionKey* pWinKey = taosArrayGet(pResWins, i); if (!pWinKey) continue; SSessionKey winInfo = {0}; getSessionHashKey(pWinKey, &winInfo); - tSimpleHashPut(pStDeleted, &winInfo, sizeof(SSessionKey), NULL, 0); + code = tSimpleHashPut(pStDeleted, &winInfo, sizeof(SSessionKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } // the allocated memory comes from outer function. @@ -2453,6 +2844,8 @@ void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayL int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; SExprInfo* pExprInfo = pSup->pExprInfo; @@ -2471,9 +2864,13 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa if (pBlock->info.id.groupId == 0) { pBlock->info.id.groupId = pKey->groupId; - void* tbname = NULL; - if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, - false) < 0) { + void* tbname = NULL; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, + &tbname, false, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode != TSDB_CODE_SUCCESS) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); @@ -2487,8 +2884,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa } int32_t code = pAPI->stateStore.streamStateGetByPos(pState, pPos, (void**)&pRow); - - if (code == -1) { + if (code == TSDB_CODE_FAILED) { // for history qWarn("===stream===not found session result key:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64 "", pKey->win.skey, pKey->win.ekey, pKey->groupId); @@ -2515,11 +2911,8 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); if (pCtx[j].fpSet.finalize) { - int32_t code1 = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); - if (TAOS_FAILED(code1)) { - qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code1)); - T_LONG_JMP(pTaskInfo->env, code1); - } + code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + TSDB_CHECK_CODE(code, lino, _end); } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { // do nothing, todo refactor } else { @@ -2528,7 +2921,8 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { - colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + code = colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + TSDB_CHECK_CODE(code, lino, _end); } } } @@ -2536,11 +2930,19 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa pBlock->info.dataLoad = 1; pBlock->info.rows += pRow->numOfRows; } - blockDataUpdateTsWindow(pBlock, 0); - return TSDB_CODE_SUCCESS; + code = blockDataUpdateTsWindow(pBlock, 0); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; // set output datablock version pBlock->info.version = pTaskInfo->version; @@ -2548,15 +2950,22 @@ void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* blockDataCleanup(pBlock); if (!hasRemainResults(pGroupResInfo)) { cleanupGroupResInfo(pGroupResInfo); - return; + goto _end; } // clear the existed group id pBlock->info.id.groupId = 0; - buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo); + code = buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo); + TSDB_CHECK_CODE(code, lino, _end); + if (pBlock->info.rows == 0) { cleanupGroupResInfo(pGroupResInfo); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } static SSDataBlock* buildSessionResult(SOperatorInfo* pOperator) { @@ -2578,26 +2987,43 @@ static SSDataBlock* buildSessionResult(SOperatorInfo* pOperator) { return NULL; } -void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins) { +int32_t getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t size = taosArrayGetSize(pAllWins); if (size == 0) { - return; + goto _end; } SResultWindowInfo* pWinInfo = taosArrayGet(pAllWins, size - 1); SSessionKey* pSeKey = &pWinInfo->sessionWin; - taosArrayPush(pMaxWins, pSeKey); + void* tmp = taosArrayPush(pMaxWins, pSeKey); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + if (pSeKey->groupId == 0) { - return; + goto _end; } uint64_t preGpId = pSeKey->groupId; for (int32_t i = size - 2; i >= 0; i--) { pWinInfo = taosArrayGet(pAllWins, i); pSeKey = &pWinInfo->sessionWin; if (preGpId != pSeKey->groupId) { - taosArrayPush(pMaxWins, pSeKey); + void* tmp = taosArrayPush(pMaxWins, pSeKey); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } preGpId = pSeKey->groupId; } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t encodeSSessionKey(void** buf, SSessionKey* key) { @@ -2674,10 +3100,13 @@ int32_t doStreamSessionEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOp return tlen; } -void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent) { +int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent, void** ppBuf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; if (!pInfo) { - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; @@ -2687,7 +3116,8 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera void* pCksum = POINTER_SHIFT(buf, dataLen); if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { qError("stream session state is invalid"); - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } } @@ -2698,10 +3128,16 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); - pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &winfo.sessionWin, pAggSup->gap, - (void**)&winfo.pStatePos, &pAggSup->resultRowSize); + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAggSup->stateStore.streamStateSessionAddIfNotExist( + pAggSup->pState, &winfo.sessionWin, pAggSup->gap, (void**)&winfo.pStatePos, &pAggSup->resultRowSize, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + ASSERT(winCode == TSDB_CODE_SUCCESS); + buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + code = + tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } // 2.twAggSup @@ -2713,12 +3149,21 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera ASSERT(size <= taosArrayGetSize(pInfo->pChildren)); for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i); - buf = doStreamSessionDecodeOpState(buf, 0, pChOp, false); + code = doStreamSessionDecodeOpState(buf, 0, pChOp, false, &buf); + TSDB_CHECK_CODE(code, lino, _end); } // 4.dataVersion buf = taosDecodeFixedI64(buf, &pInfo->dataVersion); - return buf; + if (ppBuf) { + (*ppBuf) = buf; + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) { @@ -2753,12 +3198,17 @@ void copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest) { size_t keyLen = 0; while ((pIte = tSimpleHashIterate(source, pIte, &iter)) != NULL) { SSessionKey* pKey = tSimpleHashGetKey(pIte, &keyLen); - saveDeleteRes(dest, *pKey); + int32_t code = saveDeleteRes(dest, *pKey); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } tSimpleHashClear(source); } static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExprSupp* pSup = &pOperator->exprSupp; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; @@ -2808,19 +3258,27 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); // gap must be 0 - doDeleteTimeWindows(pAggSup, pBlock, pWins); + code = doDeleteTimeWindows(pAggSup, pBlock, pWins); + TSDB_CHECK_CODE(code, lino, _end); + removeSessionResults(pAggSup, pInfo->pStUpdated, pWins); if (IS_FINAL_SESSION_OP(pOperator)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; // gap must be 0 - doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, NULL); - rebuildSessionWindow(pOperator, pWins, pInfo->pStUpdated); + code = doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, NULL); + TSDB_CHECK_CODE(code, lino, _end); + + code = rebuildSessionWindow(pOperator, pWins, pInfo->pStUpdated); + TSDB_CHECK_CODE(code, lino, _end); } - copyDeleteWindowInfo(pWins, pInfo->pStDeleted); + code = copyDeleteWindowInfo(pWins, pInfo->pStDeleted); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->destHasPrimaryKey && IS_NORMAL_SESSION_OP(pOperator)) { - copyDeleteWindowInfo(pWins, pInfo->pPkDeleted); + code = copyDeleteWindowInfo(pWins, pInfo->pPkDeleted); + TSDB_CHECK_CODE(code, lino, _end); } taosArrayDestroy(pWins); continue; @@ -2833,7 +3291,9 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_CHECKPOINT) { pAggSup->stateStore.streamStateCommit(pAggSup->pState); doStreamSessionSaveCheckpoint(pOperator); - copyDataBlock(pInfo->pCheckpointRes, pBlock); + code = copyDataBlock(pInfo->pCheckpointRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + continue; } else { ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); @@ -2841,7 +3301,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); @@ -2856,9 +3317,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0, NULL); if (!pChildOp) { qError("%s create stream child of final session error", GET_TASKID(pTaskInfo)); - T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + void* tmp = taosArrayPush(pInfo->pChildren, &pChildOp); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } - taosArrayPush(pInfo->pChildren, &pChildOp); } SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); @@ -2870,27 +3336,39 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated); - closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); - copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + code = closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated); + TSDB_CHECK_CODE(code, lino, _end); + + code = closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); + TSDB_CHECK_CODE(code, lino, _end); + + code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + TSDB_CHECK_CODE(code, lino, _end); + if (!pInfo->destHasPrimaryKey) { removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated); } if (pInfo->isHistoryOp) { - getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->destHasPrimaryKey && IS_NORMAL_SESSION_OP(pOperator)) { copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pStDeleted); } initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); SSDataBlock* opRes = buildSessionResult(pOperator); if (opRes) { return opRes; } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } setStreamOperatorCompleted(pOperator); return NULL; } @@ -2945,6 +3423,8 @@ void reloadAggSupFromDownStream(SOperatorInfo* downstream, SStreamAggSupporter* } void streamSessionSemiReloadState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; resetWinRange(&pAggSup->winRange); @@ -2952,10 +3432,12 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { SResultWindowInfo winInfo = {0}; int32_t size = 0; void* pBuf = NULL; - int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, - strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); - int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); - SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; + code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + TSDB_CHECK_CODE(code, lino, _end); + + int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); + SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); for (int32_t i = 0; i < num; i++) { SResultWindowInfo winInfo = {0}; @@ -2964,7 +3446,8 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { continue; } compactSessionSemiWindow(pOperator, &winInfo); - saveSessionOutputBuf(pAggSup, &winInfo); + code = saveSessionOutputBuf(pAggSup, &winInfo); + TSDB_CHECK_CODE(code, lino, _end); } TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY)); taosMemoryFree(pBuf); @@ -2976,17 +3459,27 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { downstream->fpSet.reloadStreamStateFn(downstream); } reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } void streamSessionReloadState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; resetWinRange(&pAggSup->winRange); - int32_t size = 0; - void* pBuf = NULL; - int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, - strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + int32_t size = 0; + void* pBuf = NULL; + code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + + TSDB_CHECK_CODE(code, lino, _end); + int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); @@ -3005,22 +3498,36 @@ void streamSessionReloadState(SOperatorInfo* pOperator) { if (!IS_VALID_SESSION_WIN(winInfo)) { continue; } - int32_t winNum = compactSessionWindow(pOperator, &winInfo, pInfo->pStUpdated, pInfo->pStDeleted, true); + + int32_t winNum = 0; + code = compactSessionWindow(pOperator, &winInfo, pInfo->pStUpdated, pInfo->pStDeleted, true, &winNum); + TSDB_CHECK_CODE(code, lino, _end); + if (winNum > 0) { qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, winInfo.sessionWin.win.skey, winInfo.sessionWin.groupId); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveResult(winInfo, pInfo->pStUpdated); + code = saveResult(winInfo, pInfo->pStUpdated); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { if (!isCloseWindow(&winInfo.sessionWin.win, &pInfo->twAggSup)) { - saveDeleteRes(pInfo->pStDeleted, winInfo.sessionWin); + code = saveDeleteRes(pInfo->pStDeleted, winInfo.sessionWin); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } SSessionKey key = {0}; getSessionHashKey(&winInfo.sessionWin, &key); - tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); + code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } } - saveSessionOutputBuf(pAggSup, &winInfo); + code = saveSessionOutputBuf(pAggSup, &winInfo); + TSDB_CHECK_CODE(code, lino, _end); } taosMemoryFree(pBuf); @@ -3029,6 +3536,11 @@ void streamSessionReloadState(SOperatorInfo* pOperator) { downstream->fpSet.reloadStreamStateFn(downstream); } reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -3122,8 +3634,9 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME, strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), &buff, &len); if (res == TSDB_CODE_SUCCESS) { - doStreamSessionDecodeOpState(buff, len, pOperator, true); + code = doStreamSessionDecodeOpState(buff, len, pOperator, true, NULL); taosMemoryFree(buff); + TSDB_CHECK_CODE(code, lino, _error); } } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, @@ -3157,19 +3670,38 @@ static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) { pInfo->clearState = false; } -void deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, - SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd) { +int32_t deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, + SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); - doDeleteTimeWindows(pAggSup, pBlock, pWins); + if (!pWins) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + code = doDeleteTimeWindows(pAggSup, pBlock, pWins); + TSDB_CHECK_CODE(code, lino, _end); + removeSessionResults(pAggSup, pMapUpdate, pWins); - copyDeleteWindowInfo(pWins, pMapDelete); + code = copyDeleteWindowInfo(pWins, pMapDelete); + TSDB_CHECK_CODE(code, lino, _end); + if (needAdd) { - copyDeleteWindowInfo(pWins, pPkDelete); + code = copyDeleteWindowInfo(pWins, pPkDelete); + TSDB_CHECK_CODE(code, lino, _end); } taosArrayDestroy(pWins); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; TSKEY maxTs = INT64_MIN; @@ -3228,7 +3760,8 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { // gap must be 0 - deleteSessionWinState(pAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, NULL, false); + code = deleteSessionWinState(pAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, NULL, false); + TSDB_CHECK_CODE(code, lino, _end); pInfo->clearState = true; break; } else if (pBlock->info.type == STREAM_GET_ALL) { @@ -3246,7 +3779,8 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); @@ -3257,22 +3791,31 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs; - copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + TSDB_CHECK_CODE(code, lino, _end); + removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated); if (pInfo->isHistoryOp) { - getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + TSDB_CHECK_CODE(code, lino, _end); } initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); SSDataBlock* opRes = buildSessionResult(pOperator); if (opRes) { return opRes; } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + clearFunctionContext(&pOperator->exprSupp); // semi session operator clear disk buffer clearStreamSessionOperator(pInfo); @@ -3283,10 +3826,12 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild, SReadHandle* pHandle) { - int32_t code = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle); if (pOperator == NULL) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } SStorageAPI* pAPI = &pTaskInfo->storageAPI; @@ -3307,12 +3852,17 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream for (int32_t i = 0; i < numOfChild; i++) { SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0, pHandle); if (pChildOp == NULL) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info; pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; pAPI->stateStore.streamStateSetNumber(pChInfo->streamAggSup.pState, i, pInfo->primaryTsIndex); - taosArrayPush(pInfo->pChildren, &pChildOp); + void* tmp = taosArrayPush(pInfo->pChildren, &pChildOp); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } } void* buff = NULL; @@ -3321,8 +3871,9 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME, strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), &buff, &len); if (res == TSDB_CODE_SUCCESS) { - doStreamSessionDecodeOpState(buff, len, pOperator, true); + code = doStreamSessionDecodeOpState(buff, len, pOperator, true, NULL); taosMemoryFree(buff); + TSDB_CHECK_CODE(code, lino, _error); } } @@ -3338,6 +3889,9 @@ _error: } taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return NULL; } @@ -3359,13 +3913,13 @@ void destroyStreamStateOperatorInfo(void* param) { taosArrayDestroy(pInfo->pChildren); } colDataDestroy(&pInfo->twAggSup.timeWindowData); - blockDataDestroy(pInfo->pDelRes); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); tSimpleHashCleanup(pInfo->pSeUpdated); tSimpleHashCleanup(pInfo->pSeDeleted); cleanupGroupResInfo(&pInfo->groupResInfo); taosArrayDestroy(pInfo->historyWins); - blockDataDestroy(pInfo->pCheckpointRes); + pInfo->pCheckpointRes = blockDataDestroy(pInfo->pCheckpointRes); tSimpleHashCleanup(pInfo->pPkDeleted); taosMemoryFreeClear(param); @@ -3442,15 +3996,21 @@ void getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SS pNextWin->winInfo.sessionWin.win.skey, pNextWin->winInfo.sessionWin.win.ekey); } -void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData, - SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin) { - int32_t size = pAggSup->resultRowSize; +int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData, + SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin) { + int32_t size = pAggSup->resultRowSize; + SStreamStateCur* pCur = NULL; pCurWin->winInfo.sessionWin.groupId = groupId; pCurWin->winInfo.sessionWin.win.skey = ts; pCurWin->winInfo.sessionWin.win.ekey = ts; - int32_t code = pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, - pKeyData, pAggSup->stateKeySize, compareStateKey, - (void**)&pCurWin->winInfo.pStatePos, &size); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, + pAggSup->stateKeySize, compareStateKey, + (void**)&pCurWin->winInfo.pStatePos, &size, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + pCurWin->pStateKey = (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize)); pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys); @@ -3458,8 +4018,8 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys); pCurWin->pStateKey->isNull = false; - if (code == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->winInfo.sessionWin.win)) { - code = TSDB_CODE_FAILED; + if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->winInfo.sessionWin.win)) { + winCode = TSDB_CODE_FAILED; clearOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); pCurWin->pStateKey = (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize)); @@ -3474,7 +4034,7 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pCurWin->winInfo.sessionWin.win.ekey); } - if (code == TSDB_CODE_SUCCESS) { + if (winCode == TSDB_CODE_SUCCESS) { pCurWin->winInfo.isOutput = true; if (pCurWin->winInfo.pStatePos->needFree) { pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin); @@ -3491,12 +4051,11 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pCurWin->winInfo.sessionWin.win.ekey); pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin; - SStreamStateCur* pCur = - pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin); + pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin); int32_t nextSize = pAggSup->resultRowSize; - code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, - (void**)&pNextWin->winInfo.pStatePos, &nextSize); - if (code != TSDB_CODE_SUCCESS) { + winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, + (void**)&pNextWin->winInfo.pStatePos, &nextSize); + if (winCode != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_INVALID(pNextWin->winInfo); } else { pNextWin->pStateKey = @@ -3507,9 +4066,14 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pNextWin->pStateKey->isNull = false; pNextWin->winInfo.isOutput = true; } - pAggSup->stateStore.streamStateFreeCur(pCur); qDebug("===stream===set state next win buff. skey:%" PRId64 ", endkey:%" PRId64, pNextWin->winInfo.sessionWin.win.skey, pNextWin->winInfo.sessionWin.win.ekey); +_end: + pAggSup->stateStore.streamStateFreeCur(pCur); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t updateStateWindowInfo(SStreamAggSupporter* pAggSup, SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, @@ -3533,7 +4097,10 @@ int32_t updateStateWindowInfo(SStreamAggSupporter* pAggSup, SStateWindowInfo* pW if (pWinInfo->winInfo.sessionWin.win.skey > pTs[i]) { if (pSeDeleted && pWinInfo->winInfo.isOutput) { - saveDeleteRes(pSeDeleted, pWinInfo->winInfo.sessionWin); + int32_t code = saveDeleteRes(pSeDeleted, pWinInfo->winInfo.sessionWin); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } removeSessionResult(pAggSup, pSeUpdated, pResultRows, &pWinInfo->winInfo.sessionWin); pWinInfo->winInfo.sessionWin.win.skey = pTs[i]; @@ -3555,7 +4122,8 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl SStreamStateAggOperatorInfo* pInfo = pOperator->info; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; uint64_t groupId = pSDataBlock->info.id.groupId; - int64_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; TSKEY* tsCols = NULL; SResultRow* pResult = NULL; int32_t winRows = 0; @@ -3569,13 +4137,19 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + if (!pColDataInfo) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } tsCols = (int64_t*)pColDataInfo->pData; } else { return; } int32_t rows = pSDataBlock->info.rows; - blockDataEnsureCapacity(pAggSup->pScanBlock, rows); + code = blockDataEnsureCapacity(pAggSup->pScanBlock, rows); + TSDB_CHECK_CODE(code, lino, _end); + SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); for (int32_t i = 0; i < rows; i += winRows) { if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, @@ -3589,7 +4163,9 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl bool allEqual = true; SStateWindowInfo curWin = {0}; SStateWindowInfo nextWin = {0}; - setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin); + code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin); + TSDB_CHECK_CODE(code, lino, _end); + releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore); setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); @@ -3599,38 +4175,45 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); - tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); + code = tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); + TSDB_CHECK_CODE(code, lino, _end); + doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin); releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); continue; } + code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator, 0); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - qError("%s do one window aggregate impl error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - break; - } - saveSessionOutputBuf(pAggSup, &curWin.winInfo); + TSDB_CHECK_CODE(code, lino, _end); + + code = saveSessionOutputBuf(pAggSup, &curWin.winInfo); + TSDB_CHECK_CODE(code, lino, _end); if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_STATE_OP(pOperator)) { - saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { code = saveResult(curWin.winInfo, pSeUpdated); - if (code != TSDB_CODE_SUCCESS) { - qError("%s do stream state aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - break; - } + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { curWin.winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &key); - tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + code = + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } int32_t doStreamStateEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator, bool isParent) { @@ -3681,12 +4264,15 @@ int32_t doStreamStateEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper return tlen; } -void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent) { +int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent, void** ppBuf) { SStreamStateAggOperatorInfo* pInfo = pOperator->info; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; if (!pInfo) { - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; // 5.checksum if (isParent) { @@ -3694,7 +4280,8 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato void* pCksum = POINTER_SHIFT(buf, dataLen); if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { qError("stream state_window state is invalid"); - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } } @@ -3705,11 +4292,16 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); - pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, pAggSup->stateKeySize, - compareStateKey, (void**)&winfo.pStatePos, - &pAggSup->resultRowSize); + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAggSup->stateStore.streamStateStateAddIfNotExist( + pAggSup->pState, &winfo.sessionWin, NULL, pAggSup->stateKeySize, compareStateKey, (void**)&winfo.pStatePos, + &pAggSup->resultRowSize, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + code = + tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } // 2.twAggSup @@ -3721,12 +4313,22 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato ASSERT(size <= taosArrayGetSize(pInfo->pChildren)); for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i); - buf = doStreamStateDecodeOpState(buf, 0, pChOp, false); + code = doStreamStateDecodeOpState(buf, 0, pChOp, false, &buf); + TSDB_CHECK_CODE(code, lino, _end); } // 4.dataVersion buf = taosDecodeFixedI64(buf, &pInfo->dataVersion); - return buf; + + if (ppBuf) { + (*ppBuf) = buf; + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) { @@ -3767,6 +4369,8 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return NULL; } + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExprSupp* pSup = &pOperator->exprSupp; SStreamStateAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; @@ -3812,7 +4416,9 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { bool add = pInfo->destHasPrimaryKey && IS_NORMAL_STATE_OP(pOperator); - deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted, pInfo->pPkDeleted, add); + code = deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted, + pInfo->pPkDeleted, add); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { pInfo->recvGetAll = true; @@ -3823,7 +4429,9 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_CHECKPOINT) { pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState); doStreamStateSaveCheckpoint(pOperator); - copyDataBlock(pInfo->pCheckpointRes, pBlock); + code = copyDataBlock(pInfo->pCheckpointRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + continue; } else { ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); @@ -3831,7 +4439,8 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); @@ -3841,12 +4450,17 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated); - copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + code = closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated); + TSDB_CHECK_CODE(code, lino, _end); + + code = copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + TSDB_CHECK_CODE(code, lino, _end); + removeSessionDeleteResults(pInfo->pSeDeleted, pInfo->pUpdated); if (pInfo->isHistoryOp) { - getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->destHasPrimaryKey && IS_NORMAL_STATE_OP(pOperator)) { copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pSeDeleted); @@ -3854,12 +4468,18 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); SSDataBlock* resBlock = buildStateResult(pOperator); if (resBlock != NULL) { return resBlock; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } setStreamOperatorCompleted(pOperator); return NULL; } @@ -3883,16 +4503,18 @@ void streamStateReleaseState(SOperatorInfo* pOperator) { } } -static void compactStateWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, - SSHashObj* pStUpdated, SSHashObj* pStDeleted) { +static int32_t compactStateWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, + SSHashObj* pStUpdated, SSHashObj* pStDeleted) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamStateAggOperatorInfo* pInfo = pOperator->info; - compactTimeWindow(pSup, &pInfo->streamAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, pNextWin, pStUpdated, pStDeleted, - false); + return compactTimeWindow(pSup, &pInfo->streamAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, pNextWin, pStUpdated, + pStDeleted, false); } void streamStateReloadState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamStateAggOperatorInfo* pInfo = pOperator->info; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; resetWinRange(&pAggSup->winRange); @@ -3900,9 +4522,11 @@ void streamStateReloadState(SOperatorInfo* pOperator) { SSessionKey seKey = {.win.skey = INT64_MIN, .win.ekey = INT64_MIN, .groupId = 0}; int32_t size = 0; void* pBuf = NULL; - int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME, - strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); - int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); + code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME, + strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); + TSDB_CHECK_CODE(code, lino, _end); + + int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); qDebug("===stream=== reload state. get result count:%d", num); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); @@ -3929,25 +4553,38 @@ void streamStateReloadState(SOperatorInfo* pOperator) { qDebug("===stream=== reload state. next window info %" PRId64 ", %" PRIu64 ", compare:%d", nextInfo.winInfo.sessionWin.win.skey, nextInfo.winInfo.sessionWin.groupId, cpRes); if (cpRes) { - compactStateWindow(pOperator, &curInfo.winInfo, &nextInfo.winInfo, pInfo->pSeUpdated, pInfo->pSeDeleted); + code = compactStateWindow(pOperator, &curInfo.winInfo, &nextInfo.winInfo, pInfo->pSeUpdated, pInfo->pSeDeleted); qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey, curInfo.winInfo.sessionWin.groupId); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveResult(curInfo.winInfo, pInfo->pSeUpdated); + code = saveResult(curInfo.winInfo, pInfo->pSeUpdated); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { if (!isCloseWindow(&curInfo.winInfo.sessionWin.win, &pInfo->twAggSup)) { - saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin); + code = saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } SSessionKey key = {0}; getSessionHashKey(&curInfo.winInfo.sessionWin, &key); - tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, sizeof(SResultWindowInfo)); + code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, + sizeof(SResultWindowInfo)); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } } else if (IS_VALID_SESSION_WIN(nextInfo.winInfo)) { releaseOutputBuf(pAggSup->pState, nextInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); } if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { - saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + TSDB_CHECK_CODE(code, lino, _end); } } taosMemoryFree(pBuf); @@ -3957,6 +4594,11 @@ void streamStateReloadState(SOperatorInfo* pOperator) { downstream->fpSet.reloadStreamStateFn(downstream); } reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -4042,8 +4684,9 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME, strlen(STREAM_STATE_OP_CHECKPOINT_NAME), &buff, &len); if (res == TSDB_CODE_SUCCESS) { - doStreamStateDecodeOpState(buff, len, pOperator, true); + code = doStreamStateDecodeOpState(buff, len, pOperator, true, NULL); taosMemoryFree(buff); + TSDB_CHECK_CODE(code, lino, _error); } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, @@ -4079,6 +4722,8 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type #endif static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; @@ -4137,11 +4782,13 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { - doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pInfo->pUpdatedMap, NULL); + code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pInfo->pUpdatedMap, NULL); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { pInfo->recvGetAll = true; - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); + code = getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); @@ -4150,7 +4797,9 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { pAPI->stateStore.streamStateCommit(pInfo->pState); doStreamIntervalSaveCheckpoint(pOperator); pInfo->reCkBlock = true; - copyDataBlock(pInfo->pCheckpointRes, pBlock); + code = copyDataBlock(pInfo->pCheckpointRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + continue; } else { ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); @@ -4163,7 +4812,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the @@ -4184,26 +4834,42 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (!pInfo->destHasPrimaryKey) { removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins); } - closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, - pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); + code = closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, + pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->destHasPrimaryKey && IS_NORMAL_INTERVAL_OP(pOperator)) { - copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins); + code = copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins); + TSDB_CHECK_CODE(code, lino, _end); } void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) { - taosArrayPush(pInfo->pUpdated, pIte); + void* tmp = taosArrayPush(pInfo->pUpdated, pIte); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } taosArraySort(pInfo->pUpdated, winPosCmprImpl); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); + tSimpleHashCleanup(pInfo->pUpdatedMap); pInfo->pUpdatedMap = NULL; return buildIntervalResult(pOperator); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + setStreamOperatorCompleted(pOperator); + return NULL; } SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -4338,6 +5004,8 @@ _error: } static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pUpdatedMap) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info; pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); @@ -4387,20 +5055,21 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS continue; } - int32_t code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput, - pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->stateStore); + int32_t winCode = TSDB_CODE_SUCCESS; + code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput, + pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->stateStore, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + pResult = (SResultRow*)pResPos->pRowBuff; - if (pResult == NULL) { - qError("%s set interval output buff error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - break; - } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveWinResult(&key, pResPos, pUpdatedMap); + code = saveWinResult(&key, pResPos, pUpdatedMap); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { - tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES); + code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, 1); @@ -4416,7 +5085,8 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64, pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); - blockDataUpdateTsWindow(pSDataBlock, 0); + code = blockDataUpdateTsWindow(pSDataBlock, 0); + TSDB_CHECK_CODE(code, lino, _end); // timestamp of the data is incorrect if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { @@ -4429,18 +5099,31 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS break; } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } -static void addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) { +static int32_t addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t size = taosArrayGetSize(wins); for (int32_t i = 0; i < size; i++) { SWinKey* winKey = taosArrayGet(wins, i); void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey)); if (!chIds) { - addPullWindow(pMidPullMap, winKey, numOfChild); + code = addPullWindow(pMidPullMap, winKey, numOfChild); qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild); + TSDB_CHECK_CODE(code, lino, _end); } } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static SSDataBlock* buildMidIntervalResult(SOperatorInfo* pOperator) { @@ -4472,6 +5155,8 @@ static SSDataBlock* buildMidIntervalResult(SOperatorInfo* pOperator) { } static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; @@ -4542,9 +5227,20 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); - doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, pInfo->pFinalPullDataMap); + if (!delWins) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + code = + doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, pInfo->pFinalPullDataMap); + TSDB_CHECK_CODE(code, lino, _end); + removeResults(delWins, pInfo->pUpdatedMap); - taosArrayAddAll(pInfo->pDelWins, delWins); + void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } taosArrayDestroy(delWins); doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); @@ -4563,10 +5259,14 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { return pBlock; } else if (pBlock->info.type == STREAM_PULL_OVER) { - pInfo->recvPullover = processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, - pInfo->pPullWins, pInfo->numOfChild, pOperator); + code = processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins, + pInfo->numOfChild, pOperator, &pInfo->recvPullover); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->recvPullover) { - copyDataBlock(pInfo->pMidPulloverRes, pBlock); + code = copyDataBlock(pInfo->pMidPulloverRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + pInfo->clearState = true; break; } @@ -4574,15 +5274,27 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_CHECKPOINT) { pAPI->stateStore.streamStateCommit(pInfo->pState); doStreamIntervalSaveCheckpoint(pOperator); - copyDataBlock(pInfo->pCheckpointRes, pBlock); + code = copyDataBlock(pInfo->pCheckpointRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + continue; } else if (pBlock->info.type == STREAM_MID_RETRIEVE) { SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); - doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, NULL); - addMidRetriveWindow(delWins, pInfo->pPullDataMap, pInfo->numOfChild); + if (!delWins) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, NULL); + TSDB_CHECK_CODE(code, lino, _end); + + code = addMidRetriveWindow(delWins, pInfo->pPullDataMap, pInfo->numOfChild); + TSDB_CHECK_CODE(code, lino, _end); + taosArrayDestroy(delWins); pInfo->recvRetrive = true; - copyDataBlock(pInfo->pMidRetriveRes, pBlock); + code = copyDataBlock(pInfo->pMidRetriveRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE; pInfo->clearState = true; break; @@ -4592,7 +5304,8 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamMidIntervalAggImpl(pOperator, pBlock, pInfo->pUpdatedMap); @@ -4607,7 +5320,11 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) { - taosArrayPush(pInfo->pUpdated, pIte); + void* tmp = taosArrayPush(pInfo->pUpdated, pIte); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } tSimpleHashCleanup(pInfo->pUpdatedMap); @@ -4616,7 +5333,8 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); SSDataBlock* resBlock = buildIntervalResult(pOperator); if (resBlock != NULL) { @@ -4633,6 +5351,11 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); clearStreamIntervalOperator(pInfo); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return NULL; } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index f16e187789..2be8024aa8 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -602,7 +602,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes; } -int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, +void taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType) { double v1 = -1, v2 = -1; GET_TYPED_DATA(v1, double, inputType, point1->val); @@ -615,8 +615,6 @@ int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* r = (v1 < 1 || v2 < 1) ? 0 : 1; } SET_TYPED_DATA(point->val, outputType, r); - - return TSDB_CODE_SUCCESS; } int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity) { diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index f0647f44a3..c3eeb12209 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -161,11 +161,11 @@ int32_t streamStatePut_rocksdb(SStreamState* pState, const SWinKey* key, const v int32_t streamStateGet_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateDel_rocksdb(SStreamState* pState, const SWinKey* key); int32_t streamStateClear_rocksdb(SStreamState* pState); -int32_t streamStateCurNext_rocksdb(SStreamState* pState, SStreamStateCur* pCur); +void streamStateCurNext_rocksdb(SStreamStateCur* pCur); int32_t streamStateGetFirst_rocksdb(SStreamState* pState, SWinKey* key); int32_t streamStateGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateAddIfNotExist_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); -int32_t streamStateCurPrev_rocksdb(SStreamStateCur* pCur); +void streamStateCurPrev_rocksdb(SStreamStateCur* pCur); int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); SStreamStateCur* streamStateGetAndCheckCur_rocksdb(SStreamState* pState, SWinKey* key); SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key); @@ -193,7 +193,7 @@ int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSes int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); -int32_t streamStateSessionClear_rocksdb(SStreamState* pState); +void streamStateSessionClear_rocksdb(SStreamState* pState); int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 8b87019ee0..abb5200009 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -3289,12 +3289,10 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) { return 0; } -int32_t streamStateCurNext_rocksdb(SStreamState* pState, SStreamStateCur* pCur) { - if (!pCur) { - return -1; +void streamStateCurNext_rocksdb(SStreamStateCur* pCur) { + if (pCur) { + rocksdb_iter_next(pCur->iter); } - rocksdb_iter_next(pCur->iter); - return 0; } int32_t streamStateGetFirst_rocksdb(SStreamState* pState, SWinKey* key) { stDebug("streamStateGetFirst_rocksdb"); @@ -3337,11 +3335,10 @@ int32_t streamStateAddIfNotExist_rocksdb(SStreamState* pState, const SWinKey* ke memset(*pVal, 0, size); return 0; } -int32_t streamStateCurPrev_rocksdb(SStreamStateCur* pCur) { - if (!pCur) return -1; - - rocksdb_iter_prev(pCur->iter); - return 0; +void streamStateCurPrev_rocksdb(SStreamStateCur* pCur) { + if (pCur) { + rocksdb_iter_prev(pCur->iter); + } } int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { if (!pCur) return -1; @@ -3990,7 +3987,7 @@ int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSes } if (c > 0) { - streamStateCurNext_rocksdb(pState, pCur); + streamStateCurNext_rocksdb(pCur); code = streamStateSessionGetKVByCur_rocksdb(pCur, &resKey, NULL, 0); if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { *curKey = resKey; @@ -4037,7 +4034,7 @@ int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKe goto _end; } taosMemoryFreeClear(*pVal); - streamStateCurNext_rocksdb(pState, pCur); + streamStateCurNext_rocksdb(pCur); } else { *key = originKey; streamStateFreeCur(pCur); @@ -4063,7 +4060,7 @@ _end: streamStateFreeCur(pCur); return res; } -int32_t streamStateSessionClear_rocksdb(SStreamState* pState) { +void streamStateSessionClear_rocksdb(SStreamState* pState) { stDebug("streamStateSessionClear_rocksdb"); SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0}; SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext_rocksdb(pState, &key); @@ -4083,10 +4080,9 @@ int32_t streamStateSessionClear_rocksdb(SStreamState* pState) { } taosMemoryFreeClear(buf); - streamStateCurNext_rocksdb(pState, pCur); + streamStateCurNext_rocksdb(pCur); } streamStateFreeCur(pCur); - return -1; } int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { @@ -4114,7 +4110,7 @@ int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* goto _end; } - streamStateCurNext_rocksdb(pState, pCur); + streamStateCurNext_rocksdb(pCur); } else { *key = tmpKey; streamStateFreeCur(pCur); diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 0887f9e965..bca9f5cdfb 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -81,21 +81,47 @@ SStreamStateCur* createSessionStateCursor(SStreamFileState* pFileState) { return pCur; } -static SRowBuffPos* addNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey) { +static int32_t addNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey, + SRowBuffPos** ppPos) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); ASSERT(pNewPos->pRowBuff); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); - taosArrayPush(pWinInfos, &pNewPos); - return pNewPos; + void* tmp = taosArrayPush(pWinInfos, &pNewPos); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + (*ppPos) = pNewPos; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static SRowBuffPos* insertNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey, - int32_t index) { +static int32_t insertNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey, + int32_t index, SRowBuffPos** ppPos) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); ASSERT(pNewPos->pRowBuff); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); - taosArrayInsert(pWinInfos, index, &pNewPos); - return pNewPos; + void* tmp = taosArrayInsert(pWinInfos, index, &pNewPos); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + *ppPos = pNewPos; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen) { @@ -103,7 +129,7 @@ SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKe memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); pNewPos->needFree = true; pNewPos->beFlushed = true; - if(p) { + if (p) { memcpy(pNewPos->pRowBuff, p, *pVLen); } else { int32_t len = getRowStateRowSize(pFileState); @@ -113,9 +139,11 @@ SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKe return pNewPos; } -int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, - int32_t* pVLen) { - int32_t code = TSDB_CODE_SUCCESS; +int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen, + int32_t* pWinCode) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + (*pWinCode) = TSDB_CODE_SUCCESS; SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SArray* pWinStates = NULL; void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); @@ -123,7 +151,12 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, pWinStates = (SArray*)(*ppBuff); } else { pWinStates = taosArrayInit(16, POINTER_BYTES); - tSimpleHashPut(pSessionBuff, &pKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + if (!pWinStates) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + code = tSimpleHashPut(pSessionBuff, &pKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } TSKEY startTs = pKey->win.skey; @@ -136,12 +169,13 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pKey, p, pVLen); - code = code_file; + (*pWinCode) = code_file; qDebug("===stream===0 get session win:%" PRId64 ",%" PRId64 " from disc, res %d", startTs, endTs, code_file); } else { - (*pVal) = addNewSessionWindow(pFileState, pWinStates, pKey); - code = TSDB_CODE_FAILED; + code = addNewSessionWindow(pFileState, pWinStates, pKey, (SRowBuffPos**)pVal); + (*pWinCode) = TSDB_CODE_FAILED; taosMemoryFree(p); + TSDB_CHECK_CODE(code, lino, _end); } goto _end; } @@ -181,7 +215,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pKey, p, pVLen); - code = code_file; + (*pWinCode) = code_file; qDebug("===stream===1 get session win:%" PRId64 ",%" PRId64 " from disc, res %d", startTs, endTs, code_file); goto _end; } else { @@ -191,25 +225,36 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, } if (index == size - 1) { - (*pVal) = addNewSessionWindow(pFileState, pWinStates, pKey); - code = TSDB_CODE_FAILED; + code = addNewSessionWindow(pFileState, pWinStates, pKey, (SRowBuffPos**)pVal); + TSDB_CHECK_CODE(code, lino, _end); + + (*pWinCode) = TSDB_CODE_FAILED; goto _end; } - (*pVal) = insertNewSessionWindow(pFileState, pWinStates, pKey, index + 1); - code = TSDB_CODE_FAILED; + + code = insertNewSessionWindow(pFileState, pWinStates, pKey, index + 1, (SRowBuffPos**)pVal); + TSDB_CHECK_CODE(code, lino, _end); + + (*pWinCode) = TSDB_CODE_FAILED; _end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } -int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen) { +int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen, + int32_t* pWinCode) { SWinKey* pTmpkey = pKey; ASSERT(keyLen == sizeof(SWinKey)); SSessionKey pWinKey = {.groupId = pTmpkey->groupId, .win.skey = pTmpkey->ts, .win.ekey = pTmpkey->ts}; - return getSessionWinResultBuff(pFileState, &pWinKey, 0, pVal, pVLen); + return getSessionWinResultBuff(pFileState, &pWinKey, 0, pVal, pVLen, pWinCode); } int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SSessionKey* pKey = pPos->pKey; SArray* pWinStates = NULL; @@ -218,26 +263,47 @@ int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) pWinStates = (SArray*)(*ppBuff); } else { pWinStates = taosArrayInit(16, POINTER_BYTES); - tSimpleHashPut(pSessionBuff, &pKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + if (!pWinStates) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + code = tSimpleHashPut(pSessionBuff, &pKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } int32_t size = taosArrayGetSize(pWinStates); if (size == 0) { - taosArrayPush(pWinStates, &pPos); + void* tmp = taosArrayPush(pWinStates, &pPos); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } goto _end; } // find the first position which is smaller than the pKey int32_t index = binarySearch(pWinStates, size, pKey, sessionStateKeyCompare); if (index >= 0) { - taosArrayInsert(pWinStates, index, &pPos); + void* tmp = taosArrayInsert(pWinStates, index, &pPos); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } else { - taosArrayInsert(pWinStates, 0, &pPos); + void* tmp = taosArrayInsert(pWinStates, 0, &pPos); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } _end: pPos->needFree = false; - return TSDB_CODE_SUCCESS; + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen) { @@ -299,28 +365,42 @@ int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffP int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SRowBuffPos* pNewPos = NULL; SSHashObj* pSessionBuff = getRowStateBuff(pFileState); void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); SArray* pWinStates = NULL; if (!ppBuff) { pWinStates = taosArrayInit(16, POINTER_BYTES); - tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + if (!pWinStates) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + code = tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } else { pWinStates = (SArray*)(*ppBuff); } if (!pCur) { - pNewPos = addNewSessionWindow(pFileState, pWinStates, pWinKey); + code = addNewSessionWindow(pFileState, pWinStates, pWinKey, &pNewPos); + TSDB_CHECK_CODE(code, lino, _end); + goto _end; } int32_t size = taosArrayGetSize(pWinStates); if (pCur->buffIndex >= 0) { if (pCur->buffIndex >= size) { - pNewPos = addNewSessionWindow(pFileState, pWinStates, pWinKey); + code = addNewSessionWindow(pFileState, pWinStates, pWinKey, &pNewPos); + TSDB_CHECK_CODE(code, lino, _end); + goto _end; } - pNewPos = insertNewSessionWindow(pFileState, pWinStates, pWinKey, pCur->buffIndex); + code = insertNewSessionWindow(pFileState, pWinStates, pWinKey, pCur->buffIndex, &pNewPos); + TSDB_CHECK_CODE(code, lino, _end); + goto _end; } else { if (size > 0) { @@ -328,8 +408,10 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream if (sessionWinKeyCmpr(pWinKey, pPos->pKey) >= 0) { // pCur is invalid SSessionKey pTmpKey = *pWinKey; - int32_t code = getSessionWinResultBuff(pFileState, &pTmpKey, 0, (void**)&pNewPos, pVLen); - ASSERT(code == TSDB_CODE_FAILED); + int32_t winCode = TSDB_CODE_SUCCESS; + code = getSessionWinResultBuff(pFileState, &pTmpKey, 0, (void**)&pNewPos, pVLen, &winCode); + ASSERT(winCode == TSDB_CODE_FAILED); + TSDB_CHECK_CODE(code, lino, _end); goto _end; } } @@ -341,7 +423,10 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream _end: (*ppVal) = pNewPos; - return TSDB_CODE_SUCCESS; + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void sessionWinStateClear(SStreamFileState* pFileState) { @@ -484,12 +569,12 @@ SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SS SArray* pWinStates = NULL; int32_t index = -1; SStreamStateCur* pBuffCur = seekKeyCurrentPrev_buff(pFileState, pWinKey, &pWinStates, &index); - int32_t resSize = getRowStateRowSize(pFileState); - COUNT_TYPE winCount = 0; + int32_t resSize = getRowStateRowSize(pFileState); + COUNT_TYPE winCount = 0; if (pBuffCur) { while (index >= 0) { SRowBuffPos* pPos = taosArrayGetP(pWinStates, index); - winCount = *((COUNT_TYPE*) ((char*)pPos->pRowBuff + (resSize - sizeof(COUNT_TYPE)))); + winCount = *((COUNT_TYPE*)((char*)pPos->pRowBuff + (resSize - sizeof(COUNT_TYPE)))); if (sessionStateRangeKeyCompare(pWinKey, pWinStates, index) == 0 || winCount < count) { index--; } else if (index >= 0) { @@ -503,19 +588,19 @@ SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SS pBuffCur->buffIndex = 0; } - void* pFileStore = getStateFileStore(pFileState); + void* pFileStore = getStateFileStore(pFileState); SStreamStateCur* pCur = streamStateSessionSeekKeyPrev_rocksdb(pFileStore, pWinKey); if (pCur) { pCur->pStreamFileState = pFileState; SSessionKey key = {0}; - void* pVal = NULL; - int len = 0; - int32_t code = streamStateSessionGetKVByCur_rocksdb(pCur, &key, &pVal, &len); + void* pVal = NULL; + int len = 0; + int32_t code = streamStateSessionGetKVByCur_rocksdb(pCur, &key, &pVal, &len); if (code == TSDB_CODE_FAILED) { streamStateFreeCur(pCur); return pBuffCur; } - winCount = *((COUNT_TYPE*) ((char*)pVal + (resSize - sizeof(COUNT_TYPE)))); + winCount = *((COUNT_TYPE*)((char*)pVal + (resSize - sizeof(COUNT_TYPE)))); taosMemoryFreeClear(pVal); streamStateFreeCur(pBuffCur); if (sessionRangeKeyCmpr(pWinKey, &key) != 0 && winCount == count) { @@ -529,7 +614,7 @@ SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SS streamStateCurNext(pFileStore, pCur); return pCur; } - winCount = *((COUNT_TYPE*) ((char*)pVal + (resSize - sizeof(COUNT_TYPE)))); + winCount = *((COUNT_TYPE*)((char*)pVal + (resSize - sizeof(COUNT_TYPE)))); taosMemoryFreeClear(pVal); if (sessionRangeKeyCmpr(pWinKey, &key) == 0 || winCount < count) { streamStateCurPrev(pFileStore, pCur); @@ -590,17 +675,17 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void return code; } -int32_t sessionWinStateMoveToNext(SStreamStateCur* pCur) { +void sessionWinStateMoveToNext(SStreamStateCur* pCur) { qTrace("move cursor to next"); if (pCur && pCur->buffIndex >= 0) { pCur->buffIndex++; } else { - streamStateCurNext_rocksdb(NULL, pCur); + streamStateCurNext_rocksdb(pCur); } - return TSDB_CODE_SUCCESS; } -int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey, range_cmpr_fn cmpFn) { +int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey, + range_cmpr_fn cmpFn) { SStreamStateCur* pCur = sessionWinStateSeekKeyCurrentPrev(pFileState, key); SSessionKey tmpKey = *key; int32_t code = sessionWinStateGetKVByCur(pCur, &tmpKey, NULL, NULL); @@ -639,18 +724,28 @@ _end: } int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, - state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + (*pWinCode) = TSDB_CODE_SUCCESS; + SSessionKey* pWinKey = key; TSKEY gap = 0; - int32_t code = TSDB_CODE_SUCCESS; SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SArray* pWinStates = NULL; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } else { pWinStates = taosArrayInit(16, POINTER_BYTES); - tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + if (!pWinStates) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + code = tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } TSKEY startTs = pWinKey->win.skey; @@ -663,13 +758,14 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch int32_t code_file = streamStateStateAddIfNotExist_rocksdb(pFileStore, pWinKey, pKeyData, keyDataLen, fn, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); - code = code_file; + (*pWinCode) = code_file; qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); } else { - (*pVal) = addNewSessionWindow(pFileState, pWinStates, key); - code = TSDB_CODE_FAILED; + code = addNewSessionWindow(pFileState, pWinStates, key, (SRowBuffPos**)pVal); + (*pWinCode) = TSDB_CODE_FAILED; taosMemoryFree(p); + TSDB_CHECK_CODE(code, lino, _end); } goto _end; } @@ -714,7 +810,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch streamStateStateAddIfNotExist_rocksdb(pFileStore, pWinKey, pKeyData, keyDataLen, fn, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); - code = code_file; + (*pWinCode) = code_file; qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); goto _end; @@ -725,14 +821,21 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch } if (index == size - 1) { - (*pVal) = addNewSessionWindow(pFileState, pWinStates, key); - code = TSDB_CODE_FAILED; + code = addNewSessionWindow(pFileState, pWinStates, key, (SRowBuffPos**)pVal); + TSDB_CHECK_CODE(code, lino, _end); + + (*pWinCode) = TSDB_CODE_FAILED; goto _end; } - (*pVal) = insertNewSessionWindow(pFileState, pWinStates, key, index + 1); - code = TSDB_CODE_FAILED; + code = insertNewSessionWindow(pFileState, pWinStates, key, index + 1, (SRowBuffPos**)pVal); + TSDB_CHECK_CODE(code, lino, _end); + + (*pWinCode) = TSDB_CODE_FAILED; _end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } @@ -751,18 +854,30 @@ int32_t getCountWinStateFromDisc(SStreamState* pState, SSessionKey* pKey, void** return code; } -int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen) { - SSessionKey* pWinKey = pKey; - const TSKEY gap = 0; +int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, + int32_t* pVLen, int32_t* pWinCount) { int32_t code = TSDB_CODE_SUCCESS; - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - SArray* pWinStates = NULL; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + int32_t lino = 0; + (*pWinCount) = TSDB_CODE_SUCCESS; + + SSessionKey* pWinKey = pKey; + const TSKEY gap = 0; + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + SArray* pWinStates = NULL; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } else { pWinStates = taosArrayInit(16, POINTER_BYTES); - tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + if (!pWinStates) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + code = tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } TSKEY startTs = pWinKey->win.skey; @@ -772,11 +887,11 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C if (size == 0) { void* pFileStore = getStateFileStore(pFileState); void* pRockVal = NULL; - code = getCountWinStateFromDisc(pFileStore, pWinKey, &pRockVal, pVLen); - if (code == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { + (*pWinCount) = getCountWinStateFromDisc(pFileStore, pWinKey, &pRockVal, pVLen); + if ((*pWinCount) == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, - pWinKey->win.ekey, code); - if (code == TSDB_CODE_SUCCESS) { + pWinKey->win.ekey, (*pWinCount)); + if ((*pWinCount) == TSDB_CODE_SUCCESS) { int32_t valSize = *pVLen; COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)((char*)(pRockVal) + (valSize - sizeof(COUNT_TYPE))); if (inSessionWindow(pWinKey, startTs, gap) || (*pWinStateCout) < winCount) { @@ -789,8 +904,10 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C (*pVal) = createSessionWinBuff(pFileState, pWinKey, NULL, NULL); taosMemoryFree(pRockVal); } else { - (*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey); - code = TSDB_CODE_FAILED; + code = addNewSessionWindow(pFileState, pWinStates, pWinKey, (SRowBuffPos**)pVal); + TSDB_CHECK_CODE(code, lino, _end); + + (*pWinCount) = TSDB_CODE_FAILED; } goto _end; } @@ -802,8 +919,8 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C if (index >= 0) { pPos = taosArrayGetP(pWinStates, index); - COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)( (char*)(pPos->pRowBuff) + (valSize - sizeof(COUNT_TYPE)) ); - if (inSessionWindow(pPos->pKey, startTs, gap) || (index == size - 1 && (*pWinStateCout) < winCount) ) { + COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)((char*)(pPos->pRowBuff) + (valSize - sizeof(COUNT_TYPE))); + if (inSessionWindow(pPos->pKey, startTs, gap) || (index == size - 1 && (*pWinStateCout) < winCount)) { (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; @@ -825,7 +942,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C if (tmpKey.win.ekey < pFirstWinKey->win.skey) { *pWinKey = tmpKey; (*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen); - code = code_file; + (*pWinCount) = code_file; qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); goto _end; @@ -845,25 +962,32 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C goto _end; } - (*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey); - code = TSDB_CODE_FAILED; + code = addNewSessionWindow(pFileState, pWinStates, pWinKey, (SRowBuffPos**)pVal); + TSDB_CHECK_CODE(code, lino, _end); + + (*pWinCount) = TSDB_CODE_FAILED; _end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen) { SSessionKey* pWinKey = pKey; - const TSKEY gap = 0; - int32_t code = TSDB_CODE_SUCCESS; - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - SArray* pWinStates = NULL; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + const TSKEY gap = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + SArray* pWinStates = NULL; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } else { pWinStates = taosArrayInit(16, POINTER_BYTES); - tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + code = tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } TSKEY startTs = pWinKey->win.skey; @@ -871,24 +995,27 @@ int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey int32_t size = taosArrayGetSize(pWinStates); if (size == 0) { - void* pFileStore = getStateFileStore(pFileState); - void* p = NULL; + void* pFileStore = getStateFileStore(pFileState); + void* p = NULL; int32_t code_file = getCountWinStateFromDisc(pFileStore, pWinKey, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS && isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); - code = code_file; - qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); + qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, + pWinKey->win.ekey, code_file); } else { - (*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey); - code = TSDB_CODE_FAILED; + code = addNewSessionWindow(pFileState, pWinStates, pWinKey, (SRowBuffPos**)pVal); taosMemoryFree(p); + TSDB_CHECK_CODE(code, lino, _end); } - goto _end; } else { - (*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey); + code = addNewSessionWindow(pFileState, pWinStates, pWinKey, (SRowBuffPos**)pVal); + TSDB_CHECK_CODE(code, lino, _end); } _end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index a817070d5b..7c1d3a8baa 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -99,18 +99,20 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { } SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState)); stDebug("open stream state %p, %s", pState, path); if (pState == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } pState->pTdbState = taosMemoryCalloc(1, sizeof(STdbState)); if (pState->pTdbState == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; streamStateDestroy(pState, true); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } SStreamTask* pStreamTask = pTask; @@ -118,101 +120,27 @@ SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, i pState->taskId = taskId; sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-0x%x", pState->streamId, pState->taskId); - streamTaskSetDb(pStreamTask->pMeta, pTask, pState->pTdbState->idstr); + code = streamTaskSetDb(pStreamTask->pMeta, pTask, pState->pTdbState->idstr); + TSDB_CHECK_CODE(code, lino, _end); -#ifdef USE_ROCKSDB SStreamMeta* pMeta = pStreamTask->pMeta; pState->pTdbState->pOwner = pTask; pState->pFileState = NULL; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT); - pState->parNameMap = tSimpleHashInit(1024, hashFn); + if (!pState->parNameMap) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } stInfo("open state %p on backend %p 0x%" PRIx64 "-%d succ", pState, pMeta->streamBackend, pState->streamId, pState->taskId); return pState; -#else - - char cfgPath[1030]; - sprintf(cfgPath, "%s/cfg", statePath); - - szPage = szPage < 0 ? 4096 : szPage; - pages = pages < 0 ? 256 : pages; - char cfg[1024]; - memset(cfg, 0, 1024); - TdFilePtr pCfgFile = taosOpenFile(cfgPath, TD_FILE_READ); - if (pCfgFile != NULL) { - int64_t size = 0; - taosFStatFile(pCfgFile, &size, NULL); - if (size > 0) { - taosReadFile(pCfgFile, cfg, size); - sscanf(cfg, "%d\n%d\n", &szPage, &pages); - } - } else { - int32_t code = taosMulModeMkDir(statePath, 0755, false); - if (code == 0) { - pCfgFile = taosOpenFile(cfgPath, TD_FILE_WRITE | TD_FILE_CREATE); - sprintf(cfg, "%d\n%d\n", szPage, pages); - taosWriteFile(pCfgFile, cfg, strlen(cfg)); - } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } - taosCloseFile(&pCfgFile); - - if (tdbOpen(statePath, szPage, pages, &pState->pTdbState->db, 1, 0, NULL) < 0) { - goto _err; - } - - // open state storage backend - if (tdbTbOpen("state.db", sizeof(SStateKey), -1, stateKeyCmpr, pState->pTdbState->db, &pState->pTdbState->pStateDb, - 0) < 0) { - goto _err; - } - - // todo refactor - if (tdbTbOpen("fill.state.db", sizeof(SWinKey), -1, winKeyCmpr, pState->pTdbState->db, - &pState->pTdbState->pFillStateDb, 0) < 0) { - goto _err; - } - - if (tdbTbOpen("session.state.db", sizeof(SStateSessionKey), -1, stateSessionKeyCmpr, pState->pTdbState->db, - &pState->pTdbState->pSessionStateDb, 0) < 0) { - goto _err; - } - - if (tdbTbOpen("func.state.db", sizeof(STupleKey), -1, STupleKeyCmpr, pState->pTdbState->db, - &pState->pTdbState->pFuncStateDb, 0) < 0) { - goto _err; - } - - if (tdbTbOpen("parname.state.db", sizeof(int64_t), TSDB_TABLE_NAME_LEN, NULL, pState->pTdbState->db, - &pState->pTdbState->pParNameDb, 0) < 0) { - goto _err; - } - - if (tdbTbOpen("partag.state.db", sizeof(int64_t), -1, NULL, pState->pTdbState->db, &pState->pTdbState->pParTagDb, 0) < - 0) { - goto _err; - } - - if (streamStateBegin(pState) < 0) { - goto _err; - } - - pState->pTdbState->pOwner = pTask; - - return pState; - -_err: - tdbTbClose(pState->pTdbState->pStateDb); - tdbTbClose(pState->pTdbState->pFuncStateDb); - tdbTbClose(pState->pTdbState->pFillStateDb); - tdbTbClose(pState->pTdbState->pSessionStateDb); - tdbTbClose(pState->pTdbState->pParNameDb); - tdbTbClose(pState->pTdbState->pParTagDb); - tdbClose(pState->pTdbState->db); - streamStateDestroy(pState, false); return NULL; -#endif } int32_t streamStateDelTaskDb(SStreamState* pState) { @@ -223,112 +151,68 @@ int32_t streamStateDelTaskDb(SStreamState* pState) { } void streamStateClose(SStreamState* pState, bool remove) { SStreamTask* pTask = pState->pTdbState->pOwner; -#ifdef USE_ROCKSDB streamStateDestroy(pState, remove); -#else - tdbCommit(pState->pTdbState->db, pState->pTdbState->txn); - tdbPostCommit(pState->pTdbState->db, pState->pTdbState->txn); - tdbTbClose(pState->pTdbState->pStateDb); - tdbTbClose(pState->pTdbState->pFuncStateDb); - tdbTbClose(pState->pTdbState->pFillStateDb); - tdbTbClose(pState->pTdbState->pSessionStateDb); - tdbTbClose(pState->pTdbState->pParNameDb); - tdbTbClose(pState->pTdbState->pParTagDb); - tdbClose(pState->pTdbState->db); -#endif } -int32_t streamStateBegin(SStreamState* pState) { -#ifdef USE_ROCKSDB - return 0; -#else - if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn, NULL, NULL, NULL, - TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - tdbAbort(pState->pTdbState->db, pState->pTdbState->txn); - return -1; - } - return 0; -#endif -} +int32_t streamStateBegin(SStreamState* pState) { return 0; } -int32_t streamStateCommit(SStreamState* pState) { -#ifdef USE_ROCKSDB +void streamStateCommit(SStreamState* pState) { if (pState->pFileState) { SStreamSnapshot* pShot = getSnapshot(pState->pFileState); flushSnapshot(pState->pFileState, pShot, true); } - return 0; -#else - if (tdbCommit(pState->pTdbState->db, pState->pTdbState->txn) < 0) { - return -1; - } - if (tdbPostCommit(pState->pTdbState->db, pState->pTdbState->txn) < 0) { - return -1; - } - - if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn, NULL, NULL, NULL, - TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - return -1; - } - return 0; -#endif } int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { -#ifdef USE_ROCKSDB - void* pVal = NULL; - int32_t len = getRowStateRowSize(pState->pFileState); - int32_t code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), &pVal, &len); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + void* pVal = NULL; + int32_t len = getRowStateRowSize(pState->pFileState); + code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), &pVal, &len); + TSDB_CHECK_CODE(code, lino, _end); + char* buf = ((SRowBuffPos*)pVal)->pRowBuff; uint32_t rowSize = streamFileStateGetSelectRowSize(pState->pFileState); memcpy(buf + len - rowSize, value, vLen); - return TSDB_CODE_SUCCESS; -#else - return tdbTbUpsert(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), value, vLen, pState->pTdbState->txn); -#endif + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB - void* pVal = NULL; - int32_t len = getRowStateRowSize(pState->pFileState); - int32_t code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), (void**)(&pVal), &len); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + void* pVal = NULL; + int32_t len = getRowStateRowSize(pState->pFileState); + code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), (void**)(&pVal), &len); + TSDB_CHECK_CODE(code, lino, _end); + char* buf = ((SRowBuffPos*)pVal)->pRowBuff; uint32_t rowSize = streamFileStateGetSelectRowSize(pState->pFileState); *ppVal = buf + len - rowSize; streamStateReleaseBuf(pState, pVal, false); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; -#else - return tdbTbGet(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), ppVal, pVLen); -#endif } // todo refactor int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { -#ifdef USE_ROCKSDB return 0; // return streamStatePut_rocksdb(pState, key, value, vLen); -#else - SStateKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbUpsert(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), value, vLen, pState->pTdbState->txn); -#endif } -int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB - return getRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), pVal, pVLen); -#else - SStateKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbGet(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), pVal, pVLen); -#endif +int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode) { + return getRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), pVal, pVLen, pWinCode); } bool streamStateCheck(SStreamState* pState, const SWinKey* key) { -#ifdef USE_ROCKSDB return hasRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey)); -#else - SStateKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbGet(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), NULL, NULL); -#endif } int32_t streamStateGetByPos(SStreamState* pState, void* pos, void** pVal) { @@ -338,330 +222,110 @@ int32_t streamStateGetByPos(SStreamState* pState, void* pos, void** pVal) { } // todo refactor -int32_t streamStateDel(SStreamState* pState, const SWinKey* key) { -#ifdef USE_ROCKSDB - return deleteRowBuff(pState->pFileState, key, sizeof(SWinKey)); -#else - SStateKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbDelete(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), pState->pTdbState->txn); -#endif +void streamStateDel(SStreamState* pState, const SWinKey* key) { + deleteRowBuff(pState->pFileState, key, sizeof(SWinKey)); } // todo refactor int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { -#ifdef USE_ROCKSDB return streamStateFillPut_rocksdb(pState, key, value, vLen); -#else - return tdbTbUpsert(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), value, vLen, pState->pTdbState->txn); -#endif } // todo refactor int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB return streamStateFillGet_rocksdb(pState, key, pVal, pVLen); -#else - return tdbTbGet(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), pVal, pVLen); -#endif } // todo refactor -int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key) { -#ifdef USE_ROCKSDB - return streamStateFillDel_rocksdb(pState, key); -#else - return tdbTbDelete(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), pState->pTdbState->txn); -#endif +void streamStateFillDel(SStreamState* pState, const SWinKey* key) { + int32_t code = streamStateFillDel_rocksdb(pState, key); + qError("%s at line %d res %d", __func__, __LINE__, code); } -int32_t streamStateClear(SStreamState* pState) { -#ifdef USE_ROCKSDB - streamFileStateClear(pState->pFileState); - return 0; -#else - SWinKey key = {.ts = 0, .groupId = 0}; - streamStatePut(pState, &key, NULL, 0); - while (1) { - SStreamStateCur* pCur = streamStateSeekKeyNext(pState, &key); - SWinKey delKey = {0}; - int32_t code = streamStateGetKVByCur(pCur, &delKey, NULL, 0); - streamStateFreeCur(pCur); - if (code == 0) { - streamStateDel(pState, &delKey); - } else { - break; - } - } - return 0; -#endif -} +void streamStateClear(SStreamState* pState) { streamFileStateClear(pState->pFileState); } void streamStateSetNumber(SStreamState* pState, int32_t number, int32_t tsIdex) { pState->number = number; pState->tsIndex = tsIdex; } -int32_t streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen) { -#ifdef USE_ROCKSDB - int32_t code = 0; +void streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; char* cfName = "default"; void* batch = streamStateCreateBatch(); code = streamStatePutBatch(pState, cfName, batch, pKey, pVal, vLen, 0); - if (code != 0) { - streamStateDestroyBatch(batch); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); + code = streamStatePutBatch_rocksdb(pState, batch); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } streamStateDestroyBatch(batch); - return code; -#else - return 0; -#endif } int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen) { -#ifdef USE_ROCKSDB - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; code = streamDefaultGet_rocksdb(pState, pKey, pVal, pLen); return code; -#else - return 0; -#endif } -int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB - return streamStateGet(pState, key, pVal, pVLen); -#else - // todo refactor - int32_t size = *pVLen; - if (streamStateGet(pState, key, pVal, pVLen) == 0) { - return 0; - } - *pVal = tdbRealloc(NULL, size); - memset(*pVal, 0, size); - return 0; -#endif +int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, + int32_t* pWinCode) { + return streamStateGet(pState, key, pVal, pVLen, pWinCode); } -int32_t streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used) { +void streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used) { // todo refactor if (!pVal) { - return 0; + return; } -#ifdef USE_ROCKSDB streamFileStateReleaseBuff(pState->pFileState, pVal, used); -#else - streamStateFreeVal(pVal); -#endif - return 0; } -int32_t streamStateClearBuff(SStreamState* pState, void* pVal) { - return streamFileStateClearBuff(pState->pFileState, pVal); -} +void streamStateClearBuff(SStreamState* pState, void* pVal) { streamFileStateClearBuff(pState->pFileState, pVal); } SStreamStateCur* streamStateFillGetCur(SStreamState* pState, const SWinKey* key) { -#ifdef USE_ROCKSDB return streamStateFillGetCur_rocksdb(pState, key); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) return NULL; - tdbTbcOpen(pState->pTdbState->pFillStateDb, &pCur->pCur, NULL); - - int32_t c = 0; - tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c); - if (c != 0) { - streamStateFreeCur(pCur); - return NULL; - } - return pCur; -#endif } SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key) { -#ifdef USE_ROCKSDB return streamStateGetAndCheckCur_rocksdb(pState, key); -#else - SStreamStateCur* pCur = streamStateFillGetCur(pState, key); - if (pCur) { - int32_t code = streamStateGetGroupKVByCur(pCur, key, NULL, 0); - if (code == 0) { - return pCur; - } - streamStateFreeCur(pCur); - } - return NULL; -#endif } int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB return streamStateGetKVByCur_rocksdb(pCur, pKey, pVal, pVLen); -#else - if (!pCur) { - return -1; - } - const SStateKey* pKTmp = NULL; - int32_t kLen; - if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { - return -1; - } - if (pKTmp->opNum != pCur->number) { - return -1; - } - *pKey = pKTmp->key; - return 0; -#endif } int32_t streamStateFillGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB return streamStateFillGetKVByCur_rocksdb(pCur, pKey, pVal, pVLen); -#else - if (!pCur) { - return -1; - } - const SWinKey* pKTmp = NULL; - int32_t kLen; - if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { - return -1; - } - *pKey = *pKTmp; - return 0; -#endif } int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB return streamStateGetGroupKVByCur_rocksdb(pCur, pKey, pVal, pVLen); -#else - if (!pCur) { - return -1; - } - uint64_t groupId = pKey->groupId; - int32_t code = streamStateFillGetKVByCur(pCur, pKey, pVal, pVLen); - if (code == 0) { - if (pKey->groupId == groupId) { - return 0; - } - } - return -1; -#endif } SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key) { -#ifdef USE_ROCKSDB return streamStateSeekKeyNext_rocksdb(pState, key); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) { - return NULL; - } - pCur->number = pState->number; - if (tdbTbcOpen(pState->pTdbState->pStateDb, &pCur->pCur, NULL) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - SStateKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; - if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateKey), &c) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - if (c > 0) return pCur; - - if (tdbTbcMoveToNext(pCur->pCur) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - return pCur; -#endif } SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key) { -#ifdef USE_ROCKSDB return streamStateFillSeekKeyNext_rocksdb(pState, key); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (!pCur) { - return NULL; - } - if (tdbTbcOpen(pState->pTdbState->pFillStateDb, &pCur->pCur, NULL) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - int32_t c = 0; - if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - if (c > 0) return pCur; - - if (tdbTbcMoveToNext(pCur->pCur) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - return pCur; -#endif } SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* key) { -#ifdef USE_ROCKSDB return streamStateFillSeekKeyPrev_rocksdb(pState, key); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) { - return NULL; - } - if (tdbTbcOpen(pState->pTdbState->pFillStateDb, &pCur->pCur, NULL) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - int32_t c = 0; - if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - if (c < 0) return pCur; - - if (tdbTbcMoveToPrev(pCur->pCur) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - return pCur; -#endif } -int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur) { -#ifdef USE_ROCKSDB - return sessionWinStateMoveToNext(pCur); -#else - if (!pCur) { - return -1; - } - // - return tdbTbcMoveToNext(pCur->pCur); -#endif -} +void streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur) { sessionWinStateMoveToNext(pCur); } -int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur) { -#ifdef USE_ROCKSDB +void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur) { qTrace("move cursor to next"); - return streamStateCurPrev_rocksdb(pCur); -#else - if (!pCur) { - return -1; - } - return tdbTbcMoveToPrev(pCur->pCur); -#endif + streamStateCurPrev_rocksdb(pCur); } void streamStateResetCur(SStreamStateCur* pCur) { @@ -672,8 +336,6 @@ void streamStateResetCur(SStreamStateCur* pCur) { if (pCur->snapshot) rocksdb_release_snapshot(pCur->db, pCur->snapshot); if (pCur->readOpt) rocksdb_readoptions_destroy(pCur->readOpt); - tdbTbcClose(pCur->pCur); - memset(pCur, 0, sizeof(SStreamStateCur)); pCur->buffIndex = -1; @@ -687,16 +349,9 @@ void streamStateFreeCur(SStreamStateCur* pCur) { taosMemoryFree(pCur); } -void streamStateFreeVal(void* val) { -#ifdef USE_ROCKSDB - taosMemoryFree(val); -#else - tdbFree(val); -#endif -} +void streamStateFreeVal(void* val) { taosMemoryFree(val); } int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen) { -#ifdef USE_ROCKSDB int32_t code = TSDB_CODE_SUCCESS; SRowBuffPos* pos = (SRowBuffPos*)value; if (pos->needFree) { @@ -715,438 +370,133 @@ int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void } } return code; -#else - SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbUpsert(pState->pTdbState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), value, vLen, - pState->pTdbState->txn); -#endif } int32_t streamStateSessionAllocWinBuffByNextPosition(SStreamState* pState, SStreamStateCur* pCur, const SSessionKey* pKey, void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB return allocSessioncWinBuffByNextPosition(pState->pFileState, pCur, pKey, pVal, pVLen); -#else - return TSDB_CODE_FAILED; -#endif } int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB return getSessionFlushedBuff(pState->pFileState, key, pVal, pVLen); -#else - - SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pState, key); - SSessionKey resKey = *key; - void* tmp = NULL; - int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, &tmp, pVLen); - if (code == 0) { - if (key->win.skey != resKey.win.skey) { - code = -1; - } else { - *key = resKey; - *pVal = tdbRealloc(NULL, *pVLen); - memcpy(*pVal, tmp, *pVLen); - } - } - streamStateFreeCur(pCur); - return code; -#endif } -int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { -#ifdef USE_ROCKSDB +void streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { qDebug("===stream===delete skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64, key->win.skey, key->win.ekey, key->groupId); - return deleteRowBuff(pState->pFileState, key, sizeof(SSessionKey)); -#else - SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbDelete(pState->pTdbState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), pState->pTdbState->txn); -#endif + deleteRowBuff(pState->pFileState, key, sizeof(SSessionKey)); } -int32_t streamStateSessionReset(SStreamState* pState, void* pVal) { +void streamStateSessionReset(SStreamState* pState, void* pVal) { int32_t len = getRowStateRowSize(pState->pFileState); memset(pVal, 0, len); - return TSDB_CODE_SUCCESS; } SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key) { -#ifdef USE_ROCKSDB return sessionWinStateSeekKeyCurrentPrev(pState->pFileState, key); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) { - return NULL; - } - pCur->number = pState->number; - if (tdbTbcOpen(pState->pTdbState->pSessionStateDb, &pCur->pCur, NULL) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; - if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - if (c >= 0) return pCur; - - if (tdbTbcMoveToPrev(pCur->pCur) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - return pCur; -#endif } SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, const SSessionKey* key) { -#ifdef USE_ROCKSDB return sessionWinStateSeekKeyCurrentNext(pState->pFileState, key); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) { - return NULL; - } - pCur->number = pState->number; - if (tdbTbcOpen(pState->pTdbState->pSessionStateDb, &pCur->pCur, NULL) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; - if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - if (c <= 0) return pCur; - - if (tdbTbcMoveToNext(pCur->pCur) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - return pCur; -#endif } SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key) { -#ifdef USE_ROCKSDB return sessionWinStateSeekKeyNext(pState->pFileState, key); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) { - return NULL; - } - pCur->number = pState->number; - if (tdbTbcOpen(pState->pTdbState->pSessionStateDb, &pCur->pCur, NULL) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; - if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - if (c < 0) return pCur; - - if (tdbTbcMoveToNext(pCur->pCur) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - return pCur; -#endif } SStreamStateCur* streamStateCountSeekKeyPrev(SStreamState* pState, const SSessionKey* key, COUNT_TYPE count) { -#ifdef USE_ROCKSDB return countWinStateSeekKeyPrev(pState->pFileState, key, count); -#else -#endif } int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB return sessionWinStateGetKVByCur(pCur, pKey, pVal, pVLen); -#else - if (!pCur) { - return -1; - } - SStateSessionKey* pKTmp = NULL; - int32_t kLen; - if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, (const void**)pVal, pVLen) < 0) { - return -1; - } - if (pKTmp->opNum != pCur->number) { - return -1; - } - if (pKey->groupId != 0 && pKey->groupId != pKTmp->key.groupId) { - return -1; - } - *pKey = pKTmp->key; - return 0; -#endif } -int32_t streamStateSessionClear(SStreamState* pState) { -#ifdef USE_ROCKSDB +void streamStateSessionClear(SStreamState* pState) { sessionWinStateClear(pState->pFileState); - return streamStateSessionClear_rocksdb(pState); -#else - SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0}; - SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pState, &key); - while (1) { - SSessionKey delKey = {0}; - void* buf = NULL; - int32_t size = 0; - int32_t code = streamStateSessionGetKVByCur(pCur, &delKey, &buf, &size); - if (code == 0 && size > 0) { - memset(buf, 0, size); - streamStateSessionPut(pState, &delKey, buf, size); - } else { - break; - } - streamStateCurNext(pState, pCur); - } - streamStateFreeCur(pCur); - return 0; -#endif + streamStateSessionClear_rocksdb(pState); } int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { -#ifdef USE_ROCKSDB return sessionWinStateGetKeyByRange(pState->pFileState, key, curKey, sessionRangeKeyCmpr); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) { - return -1; - } - pCur->number = pState->number; - if (tdbTbcOpen(pState->pTdbState->pSessionStateDb, &pCur->pCur, NULL) < 0) { - streamStateFreeCur(pCur); - return -1; - } - - SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; - if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { - streamStateFreeCur(pCur); - return -1; - } - - SSessionKey resKey = *key; - int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); - if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { - *curKey = resKey; - streamStateFreeCur(pCur); - return code; - } - - if (c > 0) { - streamStateCurNext(pState, pCur); - code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); - if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { - *curKey = resKey; - streamStateFreeCur(pCur); - return code; - } - } else if (c < 0) { - streamStateCurPrev(pState, pCur); - code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); - if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { - *curKey = resKey; - streamStateFreeCur(pCur); - return code; - } - } - - streamStateFreeCur(pCur); - return -1; -#endif } int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { -#ifdef USE_ROCKSDB return sessionWinStateGetKeyByRange(pState->pFileState, key, curKey, countRangeKeyEqual); -#else -#endif } -int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, - int32_t* pVLen) { -#ifdef USE_ROCKSDB - return getSessionWinResultBuff(pState->pFileState, key, gap, pVal, pVLen); -#else - // todo refactor - int32_t res = 0; - SSessionKey originKey = *key; - SSessionKey searchKey = *key; - searchKey.win.skey = key->win.skey - gap; - searchKey.win.ekey = key->win.ekey + gap; - int32_t valSize = *pVLen; - void* tmp = tdbRealloc(NULL, valSize); - if (!tmp) { - return -1; - } - - SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pState, key); - int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); - if (code == 0) { - if (sessionRangeKeyCmpr(&searchKey, key) == 0) { - memcpy(tmp, *pVal, valSize); - streamStateSessionDel(pState, key); - goto _end; - } - streamStateCurNext(pState, pCur); - } else { - *key = originKey; - streamStateFreeCur(pCur); - pCur = streamStateSessionSeekKeyNext(pState, key); - } - - code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); - if (code == 0) { - if (sessionRangeKeyCmpr(&searchKey, key) == 0) { - memcpy(tmp, *pVal, valSize); - streamStateSessionDel(pState, key); - goto _end; - } - } - - *key = originKey; - res = 1; - memset(tmp, 0, valSize); - -_end: - - *pVal = tmp; - streamStateFreeCur(pCur); - return res; - -#endif +int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen, + int32_t* pWinCode) { + return getSessionWinResultBuff(pState->pFileState, key, gap, pVal, pVLen, pWinCode); } int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, - state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode) { // todo refactor - -#ifdef USE_ROCKSDB - return getStateWinResultBuff(pState->pFileState, key, pKeyData, keyDataLen, fn, pVal, pVLen); -#else - int32_t res = 0; - SSessionKey tmpKey = *key; - int32_t valSize = *pVLen; - void* tmp = tdbRealloc(NULL, valSize); - if (!tmp) { - return -1; - } - - SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pState, key); - int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); - if (code == 0) { - if (key->win.skey <= tmpKey.win.skey && tmpKey.win.ekey <= key->win.ekey) { - memcpy(tmp, *pVal, valSize); - streamStateSessionDel(pState, key); - goto _end; - } - - void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); - if (fn(pKeyData, stateKey) == true) { - memcpy(tmp, *pVal, valSize); - streamStateSessionDel(pState, key); - goto _end; - } - - streamStateCurNext(pState, pCur); - } else { - *key = tmpKey; - streamStateFreeCur(pCur); - pCur = streamStateSessionSeekKeyNext(pState, key); - } - - code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); - if (code == 0) { - void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); - if (fn(pKeyData, stateKey) == true) { - memcpy(tmp, *pVal, valSize); - streamStateSessionDel(pState, key); - goto _end; - } - } - - *key = tmpKey; - res = 1; - memset(tmp, 0, valSize); - -_end: - - *pVal = tmp; - streamStateFreeCur(pCur); - return res; -#endif + return getStateWinResultBuff(pState->pFileState, key, pKeyData, keyDataLen, fn, pVal, pVLen, pWinCode); } int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) { -#ifdef USE_ROCKSDB + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t)) == NULL) { if (tSimpleHashGetSize(pState->parNameMap) < MAX_TABLE_NAME_NUM) { - tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN); + code = tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN); + TSDB_CHECK_CODE(code, lino, _end); } - streamStatePutParName_rocksdb(pState, groupId, tbname); + code = streamStatePutParName_rocksdb(pState, groupId, tbname); + TSDB_CHECK_CODE(code, lino, _end); } - return TSDB_CODE_SUCCESS; -#else - return tdbTbUpsert(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN, - pState->pTdbState->txn); -#endif + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache) { -#ifdef USE_ROCKSDB - void* pStr = tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t)); +int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + void* pStr = tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t)); if (!pStr) { if (onlyCache && tSimpleHashGetSize(pState->parNameMap) < MAX_TABLE_NAME_NUM) { - return TSDB_CODE_FAILED; + (*pWinCode) = TSDB_CODE_FAILED; } - int32_t code = streamStateGetParName_rocksdb(pState, groupId, pVal); - if (code == TSDB_CODE_SUCCESS && tSimpleHashGetSize(pState->parNameMap) < MAX_TABLE_NAME_NUM) { - tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), *pVal, TSDB_TABLE_NAME_LEN); + (*pWinCode) = streamStateGetParName_rocksdb(pState, groupId, pVal); + if ((*pWinCode) == TSDB_CODE_SUCCESS && tSimpleHashGetSize(pState->parNameMap) < MAX_TABLE_NAME_NUM) { + code = tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), *pVal, TSDB_TABLE_NAME_LEN); + TSDB_CHECK_CODE(code, lino, _end); } - return code; + goto _end; } *pVal = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); + if (!(*pVal)) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + memcpy(*pVal, pStr, TSDB_TABLE_NAME_LEN); - return TSDB_CODE_SUCCESS; -#else - int32_t len; - return tdbTbGet(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), pVal, &len); -#endif + (*pWinCode) = TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void streamStateDestroy(SStreamState* pState, bool remove) { -#ifdef USE_ROCKSDB streamFileStateDestroy(pState->pFileState); // streamStateDestroy_rocksdb(pState, remove); tSimpleHashCleanup(pState->parNameMap); // do nothong -#endif taosMemoryFreeClear(pState->pTdbState); taosMemoryFreeClear(pState); } int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark) { -#ifdef USE_ROCKSDB return deleteExpiredCheckPoint(pState->pFileState, mark); -#else - return 0; -#endif } void streamStateReloadInfo(SStreamState* pState, TSKEY ts) { streamFileStateReloadInfo(pState->pFileState, ts); } @@ -1172,8 +522,9 @@ SStreamStateCur* createStreamStateCursor() { } // count window -int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal, int32_t* pVLen) { - return getCountWinResultBuff(pState->pFileState, pKey, winCount, ppVal, pVLen); +int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal, + int32_t* pVLen, int32_t* pWinCode) { + return getCountWinResultBuff(pState->pFileState, pKey, winCount, ppVal, pVLen, pWinCode); } int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen) { diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 61b7e02a4c..e3e07136c6 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -183,7 +183,11 @@ int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, b TSKEY dumy = 0; for (uint64_t i = 0; i < DEFAULT_BUCKET_SIZE; ++i) { - taosArrayPush(pInfo->pTsBuckets, &dumy); + void* tmp = taosArrayPush(pInfo->pTsBuckets, &dumy); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } pInfo->numBuckets = DEFAULT_BUCKET_SIZE; pInfo->pCloseWinSBF = NULL; @@ -251,7 +255,11 @@ static int32_t getSBf(SUpdateInfo* pInfo, TSKEY ts, SScalableBf** ppSBf) { code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &res); TSDB_CHECK_CODE(code, lino, _end); - taosArrayPush(pInfo->pTsSBFs, &res); + void* tmp = taosArrayPush(pInfo->pTsSBFs, &res); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } (*ppSBf) = res; @@ -543,7 +551,11 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { TSKEY ts = INT64_MIN; for (int32_t i = 0; i < size; i++) { if (tDecodeI64(&decoder, &ts) < 0) return -1; - taosArrayPush(pInfo->pTsBuckets, &ts); + void* tmp = taosArrayPush(pInfo->pTsBuckets, &ts); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } } if (tDecodeU64(&decoder, &pInfo->numBuckets) < 0) return -1; @@ -556,7 +568,11 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { code = tScalableBfDecode(&decoder, &pSBf); TSDB_CHECK_CODE(code, lino, _error); - taosArrayPush(pInfo->pTsSBFs, &pSBf); + void* tmp = taosArrayPush(pInfo->pTsSBFs, &pSBf); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } } if (tDecodeU64(&decoder, &pInfo->numSBFs) < 0) return -1; diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index fb745f86cb..7f8892e961 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -57,7 +57,6 @@ struct SStreamFileState { _state_file_remove_fn stateFileRemoveFn; _state_file_get_fn stateFileGetFn; - _state_file_clear_fn stateFileClearFn; _state_fun_get_fn stateFunctionGetFn; }; @@ -157,7 +156,6 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ pFileState->stateFileRemoveFn = intervalFileRemoveFn; pFileState->stateFileGetFn = intervalFileGetFn; - pFileState->stateFileClearFn = streamStateClear_rocksdb; pFileState->cfName = taosStrdup("state"); pFileState->stateFunctionGetFn = getRowBuff; } else { @@ -169,7 +167,6 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ pFileState->stateFileRemoveFn = sessionFileRemoveFn; pFileState->stateFileGetFn = sessionFileGetFn; - pFileState->stateFileClearFn = streamStateSessionClear_rocksdb; pFileState->cfName = taosStrdup("sess"); pFileState->stateFunctionGetFn = getSessionRowBuff; } @@ -396,12 +393,10 @@ void* getFreeBuff(SStreamFileState* pFileState) { return ptr; } -int32_t streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { +void streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { if (pPos->pRowBuff) { memset(pPos->pRowBuff, 0, pFileState->rowSize); - return TSDB_CODE_SUCCESS; } - return TSDB_CODE_FAILED; } SRowBuffPos* getNewRowPos(SStreamFileState* pFileState) { @@ -441,8 +436,11 @@ SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState) { return newPos; } -int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen) { +int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen, + int32_t* pWinCode) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + (*pWinCode) = TSDB_CODE_SUCCESS; pFileState->maxTs = TMAX(pFileState->maxTs, pFileState->getTs(pKey)); SRowBuffPos** pos = tSimpleHashGet(pFileState->rowStateBuff, pKey, keyLen); if (pos) { @@ -450,40 +448,45 @@ int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, voi *pVal = *pos; (*pos)->beUsed = true; (*pos)->beFlushed = false; - return code; + goto _end; } SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); ASSERT(pNewPos->pRowBuff); memcpy(pNewPos->pKey, pKey, keyLen); - code = TSDB_CODE_FAILED; + (*pWinCode) = TSDB_CODE_FAILED; TSKEY ts = pFileState->getTs(pKey); if (!isDeteled(pFileState, ts) && isFlushedState(pFileState, ts, 0)) { int32_t len = 0; void* p = NULL; - code = streamStateGet_rocksdb(pFileState->pFileStore, pKey, &p, &len); - qDebug("===stream===get %" PRId64 " from disc, res %d", ts, code); - if (code == TSDB_CODE_SUCCESS) { + (*pWinCode) = streamStateGet_rocksdb(pFileState->pFileStore, pKey, &p, &len); + qDebug("===stream===get %" PRId64 " from disc, res %d", ts, (*pWinCode)); + if ((*pWinCode) == TSDB_CODE_SUCCESS) { memcpy(pNewPos->pRowBuff, p, len); } taosMemoryFree(p); } - tSimpleHashPut(pFileState->rowStateBuff, pKey, keyLen, &pNewPos, POINTER_BYTES); + code = tSimpleHashPut(pFileState->rowStateBuff, pKey, keyLen, &pNewPos, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); + if (pVal) { *pVLen = pFileState->rowSize; *pVal = pNewPos; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } -int32_t deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) { +void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) { int32_t code_buff = pFileState->stateBuffRemoveFn(pFileState->rowStateBuff, pKey, keyLen); + qTrace("%s at line %d res:%s", __func__, __LINE__, code_buff); int32_t code_file = pFileState->stateFileRemoveFn(pFileState, pKey); - if (code_buff == TSDB_CODE_SUCCESS || code_file == TSDB_CODE_SUCCESS) { - return TSDB_CODE_SUCCESS; - } - return TSDB_CODE_FAILED; + qTrace("%s at line %d res:%s", __func__, __LINE__, code_file); } int32_t resetRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) { @@ -549,8 +552,9 @@ SStreamSnapshot* getSnapshot(SStreamFileState* pFileState) { return pFileState->usedBuffs; } -int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState) { +void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SListIter iter = {0}; tdListInitIter(pSnapshot, &iter, TD_LIST_FORWARD); @@ -563,8 +567,17 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, int32_t len = pFileState->rowSize + sizeof(uint64_t) + sizeof(int32_t) + 64; char* buf = taosMemoryCalloc(1, len); + if (!buf) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } void* batch = streamStateCreateBatch(); + if (!batch) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + while ((pNode = tdListNext(&iter)) != NULL && code == TSDB_CODE_SUCCESS) { SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data; if (pPos->beFlushed || !pPos->pRowBuff) { @@ -575,14 +588,16 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, qDebug("===stream===flushed start:%" PRId64, pFileState->getTs(pPos->pKey)); if (streamStateGetBatchSize(batch) >= BATCH_LIMIT) { - streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); + code = streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); streamStateClearBatch(batch); + TSDB_CHECK_CODE(code, lino, _end); } void* pSKey = pFileState->stateBuffCreateStateKeyFn(pPos, ((SStreamState*)pFileState->pFileStore)->number); code = streamStatePutBatchOptimize(pFileState->pFileStore, idx, batch, pSKey, pPos->pRowBuff, pFileState->rowSize, 0, buf); taosMemoryFreeClear(pSKey); + TSDB_CHECK_CODE(code, lino, _end); // todo handle failure memset(buf, 0, len); } @@ -590,7 +605,8 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, int32_t numOfElems = streamStateGetBatchSize(batch); if (numOfElems > 0) { - streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); + code = streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); + TSDB_CHECK_CODE(code, lino, _end); } else { goto _end; } @@ -606,14 +622,19 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, int32_t len = 0; streamFileStateEncode(&pFileState->flushMark, &valBuf, &len); qDebug("===stream===flushMark write:%" PRId64, pFileState->flushMark); - streamStatePutBatch(pFileState->pFileStore, "default", batch, STREAM_STATE_INFO_NAME, valBuf, len, 0); + code = streamStatePutBatch(pFileState->pFileStore, "default", batch, STREAM_STATE_INFO_NAME, valBuf, len, 0); taosMemoryFree(valBuf); - streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); + TSDB_CHECK_CODE(code, lino, _end); + + code = streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); + TSDB_CHECK_CODE(code, lino, _end); } _end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } streamStateDestroyBatch(batch); - return code; } int32_t forceRemoveCheckpoint(SStreamFileState* pFileState, int64_t checkpointId) { @@ -660,27 +681,26 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { ts = atol((char*)buf); if (ts < mark) { // statekey winkey.ts < mark - forceRemoveCheckpoint(pFileState, i); + int32_t tmpRes = forceRemoveCheckpoint(pFileState, i); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); break; } } return code; } -int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { - int code = TSDB_CODE_SUCCESS; +void recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { + int32_t code = TSDB_CODE_SUCCESS; if (pFileState->maxTs != INT64_MIN) { int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs) ? INT64_MIN : pFileState->maxTs - pFileState->deleteMark; - deleteExpiredCheckPoint(pFileState, mark); + int32_t tmpRes = deleteExpiredCheckPoint(pFileState, mark); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); } SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileState->pFileStore, INT64_MAX); - if (pCur == NULL) { - return -1; - } - int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); + int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); while (code == TSDB_CODE_SUCCESS) { if (pFileState->curRowCount >= recoverNum) { break; @@ -690,31 +710,32 @@ int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { int32_t vlen = 0; SSessionKey key = {0}; code = streamStateSessionGetKVByCur_rocksdb(pCur, &key, &pVal, &vlen); - if (code != 0) { + if (code != TSDB_CODE_SUCCESS) { break; } SRowBuffPos* pPos = createSessionWinBuff(pFileState, &key, pVal, &vlen); - putSessionWinResultBuff(pFileState, pPos); + code = putSessionWinResultBuff(pFileState, pPos); + if (code != TSDB_CODE_SUCCESS) { + break; + } + code = streamStateSessionCurPrev_rocksdb(pCur); } streamStateFreeCur(pCur); - return code; } -int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { +void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { int32_t code = TSDB_CODE_SUCCESS; if (pFileState->maxTs != INT64_MIN) { int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs) ? INT64_MIN : pFileState->maxTs - pFileState->deleteMark; - deleteExpiredCheckPoint(pFileState, mark); + int32_t tmpRes = deleteExpiredCheckPoint(pFileState, mark); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); } SStreamStateCur* pCur = streamStateSeekToLast_rocksdb(pFileState->pFileStore); - if (pCur == NULL) { - return -1; - } - int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); + int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); while (code == TSDB_CODE_SUCCESS) { if (pFileState->curRowCount >= recoverNum) { break; @@ -740,11 +761,9 @@ int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { destroyRowBuffPos(pNewPos); break; } - code = streamStateCurPrev_rocksdb(pCur); + streamStateCurPrev_rocksdb(pCur); } streamStateFreeCur(pCur); - - return TSDB_CODE_SUCCESS; } int32_t streamFileStateGetSelectRowSize(SStreamFileState* pFileState) { return pFileState->selectivityRowSize; } @@ -759,7 +778,8 @@ void* getRowStateBuff(SStreamFileState* pFileState) { return pFileState->rowStat void* getStateFileStore(SStreamFileState* pFileState) { return pFileState->pFileStore; } bool isDeteled(SStreamFileState* pFileState, TSKEY ts) { - return pFileState->deleteMark != INT64_MAX && pFileState->maxTs > 0 && ts < (pFileState->maxTs - pFileState->deleteMark); + return pFileState->deleteMark != INT64_MAX && pFileState->maxTs > 0 && + ts < (pFileState->maxTs - pFileState->deleteMark); } bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap) { return ts <= (pFileState->flushMark + gap); } @@ -767,5 +787,6 @@ bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap) { return int32_t getRowStateRowSize(SStreamFileState* pFileState) { return pFileState->rowSize; } int32_t getFunctionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen) { - return pFileState->stateFunctionGetFn(pFileState, pKey, keyLen, pVal, pVLen); + int32_t winCode = TSDB_CODE_SUCCESS; + return pFileState->stateFunctionGetFn(pFileState, pKey, keyLen, pVal, pVLen, &winCode); } diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 4a4dfd2653..2e6dfbc96d 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -233,7 +233,11 @@ int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) { SBloomFilter* pBF = NULL; code = tBloomFilterDecode(pDecoder, &pBF); TSDB_CHECK_CODE(code, lino, _error); - taosArrayPush(pSBf->bfArray, &pBF); + void* tmpRes = taosArrayPush(pSBf->bfArray, &pBF); + if (!tmpRes) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } } if (tDecodeU32(pDecoder, &pSBf->growth) < 0) { code = TSDB_CODE_FAILED;