xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into fix/TD-30990

This commit is contained in:
Minglei Jin 2024-07-22 08:53:15 +08:00
parent b0b9dd7d5f 2f0768e12d
commit ecc4b50a28
150 changed files with 4982 additions and 3407 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Jenkinsfile2
View File

@ -401,7 +401,7 @@ pipeline {
} }
} }
stage('linux test') { stage('linux test') {
agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || worker03 || slave215 || slave217 || slave219 "} agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "}
options { skipDefaultCheckout() } options { skipDefaultCheckout() }
when { when {
changeRequest() changeRequest()
@ -416,7 +416,7 @@ pipeline {
echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}" echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}"
} }
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
timeout(time: 150, unit: 'MINUTES'){ timeout(time: 200, unit: 'MINUTES'){
pre_test() pre_test()
script { script {
sh ''' sh '''
@ -454,7 +454,7 @@ pipeline {
cd ${WKC}/tests/parallel_test cd ${WKC}/tests/parallel_test
export DEFAULT_RETRY_TIME=2 export DEFAULT_RETRY_TIME=2
date date
''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 900 ''' + extra_param + ''' ''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 1200 ''' + extra_param + '''
''' '''
} }
} }

2
include/common/tdatablock.h
View File

@ -253,7 +253,7 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc); int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc);
SSDataBlock* createDataBlock(); SSDataBlock* createDataBlock();
void* blockDataDestroy(SSDataBlock* pBlock); void blockDataDestroy(SSDataBlock* pBlock);
void blockDataFreeRes(SSDataBlock* pBlock); void blockDataFreeRes(SSDataBlock* pBlock);
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
SSDataBlock* createSpecialDataBlock(EStreamType type); SSDataBlock* createSpecialDataBlock(EStreamType type);

60
include/libs/executor/storageapi.h
View File

@ -229,15 +229,15 @@ typedef struct SStoreTqReader {
bool (*tqReaderCurrentBlockConsumed)(); bool (*tqReaderCurrentBlockConsumed)();
struct SWalReader* (*tqReaderGetWalReader)(); // todo remove it struct SWalReader* (*tqReaderGetWalReader)(); // todo remove it
// int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it // int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it
int32_t (*tqReaderSetSubmitMsg)(); // todo remove it int32_t (*tqReaderSetSubmitMsg)(); // todo remove it
// bool (*tqReaderNextBlockFilterOut)(); // bool (*tqReaderNextBlockFilterOut)();
} SStoreTqReader; } SStoreTqReader;
typedef struct SStoreSnapshotFn { typedef struct SStoreSnapshotFn {
bool (*taosXGetTablePrimaryKey)(SSnapContext *ctx); bool (*taosXGetTablePrimaryKey)(SSnapContext* ctx);
void (*taosXSetTablePrimaryKey)(SSnapContext *ctx, int64_t uid); void (*taosXSetTablePrimaryKey)(SSnapContext* ctx, int64_t uid);
int32_t (*setForSnapShot)(SSnapContext* ctx, int64_t uid); int32_t (*setForSnapShot)(SSnapContext* ctx, int64_t uid);
int32_t (*destroySnapshot)(SSnapContext* ctx); int32_t (*destroySnapshot)(SSnapContext* ctx);
SMetaTableInfo (*getMetaTableInfoFromSnapshot)(SSnapContext* ctx); SMetaTableInfo (*getMetaTableInfoFromSnapshot)(SSnapContext* ctx);
@ -270,7 +270,7 @@ typedef struct SStoreMeta {
int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
int32_t payloadLen, double selectivityRatio); int32_t payloadLen, double selectivityRatio);
void* (*storeGetIndexInfo)(void *pVnode); void* (*storeGetIndexInfo)(void* pVnode);
void* (*getInvertIndex)(void* pVnode); void* (*getInvertIndex)(void* pVnode);
// support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter]
int32_t (*getChildTableList)(void* pVnode, int64_t suid, SArray* list); int32_t (*getChildTableList)(void* pVnode, int64_t suid, SArray* list);
@ -329,29 +329,31 @@ typedef struct {
typedef struct SStateStore { typedef struct SStateStore {
int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname); int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache); int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache,
int32_t* pWinCode);
int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
int32_t (*streamStateReleaseBuf)(SStreamState* pState, void* pVal, bool used); int32_t* pWinCode);
int32_t (*streamStateClearBuff)(SStreamState* pState, void* pVal); void (*streamStateReleaseBuf)(SStreamState* pState, void* pVal, bool used);
void (*streamStateClearBuff)(SStreamState* pState, void* pVal);
void (*streamStateFreeVal)(void* val); void (*streamStateFreeVal)(void* val);
int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key); bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key);
int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal); int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal);
int32_t (*streamStateDel)(SStreamState* pState, const SWinKey* key); void (*streamStateDel)(SStreamState* pState, const SWinKey* key);
int32_t (*streamStateClear)(SStreamState* pState); void (*streamStateClear)(SStreamState* pState);
void (*streamStateSetNumber)(SStreamState* pState, int32_t number, int32_t tsIdex); void (*streamStateSetNumber)(SStreamState* pState, int32_t number, int32_t tsIdex);
int32_t (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); void (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen); int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t (*streamStateFillDel)(SStreamState* pState, const SWinKey* key); void (*streamStateFillDel)(SStreamState* pState, const SWinKey* key);
int32_t (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur); void (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur);
int32_t (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur); void (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur);
SStreamStateCur* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key); SStreamStateCur* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key);
SStreamStateCur* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key); SStreamStateCur* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key);
@ -363,26 +365,29 @@ typedef struct SStateStore {
int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal,
int32_t* pVLen); int32_t* pVLen, int32_t* pWinCode);
int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen); int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen);
int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen,
int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key); int32_t* pWinCode);
int32_t (*streamStateSessionReset)(SStreamState* pState, void* pVal); void (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key);
int32_t (*streamStateSessionClear)(SStreamState* pState); void (*streamStateSessionReset)(SStreamState* pState, void* pVal);
void (*streamStateSessionClear)(SStreamState* pState);
int32_t (*streamStateSessionGetKVByCur)(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t (*streamStateSessionGetKVByCur)(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode);
int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t (*streamStateCountGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); int32_t (*streamStateCountGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t (*streamStateSessionAllocWinBuffByNextPosition)(SStreamState* pState, SStreamStateCur* pCur, int32_t (*streamStateSessionAllocWinBuffByNextPosition)(SStreamState* pState, SStreamStateCur* pCur,
const SSessionKey* pKey, void** pVal, int32_t* pVLen); const SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*streamStateCountWinAddIfNotExist)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, int32_t (*streamStateCountWinAddIfNotExist)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount,
void** ppVal, int32_t* pVLen); void** ppVal, int32_t* pVLen, int32_t* pWinCode);
int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType,
TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); int32_t pkLen, SUpdateInfo** ppInfo);
int32_t (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol,
int32_t primaryKeyCol, TSKEY* pMaxResTs);
bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
bool (*isIncrementalTimeStamp)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool (*isIncrementalTimeStamp)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
@ -391,7 +396,8 @@ typedef struct SStateStore {
void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count); void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count);
int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count);
int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen,
SUpdateInfo** ppInfo);
void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo);
void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo); void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo);
int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen); int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen);
@ -413,7 +419,7 @@ typedef struct SStateStore {
SStreamState* (*streamStateOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId); SStreamState* (*streamStateOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void (*streamStateClose)(SStreamState* pState, bool remove); void (*streamStateClose)(SStreamState* pState, bool remove);
int32_t (*streamStateBegin)(SStreamState* pState); int32_t (*streamStateBegin)(SStreamState* pState);
int32_t (*streamStateCommit)(SStreamState* pState); void (*streamStateCommit)(SStreamState* pState);
void (*streamStateDestroy)(SStreamState* pState, bool remove); void (*streamStateDestroy)(SStreamState* pState, bool remove);
int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark);
void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts); void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts);

2
include/libs/function/function.h
View File

@ -294,7 +294,7 @@ typedef struct SPoint {
void *val; void *val;
} SPoint; } SPoint;
int32_t taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2, void taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2,
int32_t inputType); int32_t inputType);
#define LEASTSQUARES_DOUBLE_ITEM_LENGTH 25 #define LEASTSQUARES_DOUBLE_ITEM_LENGTH 25

1
include/libs/qcom/query.h
View File

@ -192,6 +192,7 @@ typedef struct SBoundColInfo {
int16_t* pColIndex; // bound index => schema index int16_t* pColIndex; // bound index => schema index
int32_t numOfCols; int32_t numOfCols;
int32_t numOfBound; int32_t numOfBound;
bool hasBoundCols;
} SBoundColInfo; } SBoundColInfo;
typedef struct STableColsData { typedef struct STableColsData {

40
include/libs/stream/streamState.h
View File

@ -32,7 +32,7 @@ extern "C" {
SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId); SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void streamStateClose(SStreamState* pState, bool remove); void streamStateClose(SStreamState* pState, bool remove);
int32_t streamStateBegin(SStreamState* pState); int32_t streamStateBegin(SStreamState* pState);
int32_t streamStateCommit(SStreamState* pState); void streamStateCommit(SStreamState* pState);
void streamStateDestroy(SStreamState* pState, bool remove); void streamStateDestroy(SStreamState* pState, bool remove);
int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark); int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark);
int32_t streamStateDelTaskDb(SStreamState* pState); int32_t streamStateDelTaskDb(SStreamState* pState);
@ -41,22 +41,23 @@ int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void*
int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen); int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen);
int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
bool streamStateCheck(SStreamState* pState, const SWinKey* key); bool streamStateCheck(SStreamState* pState, const SWinKey* key);
int32_t streamStateGetByPos(SStreamState* pState, void* pos, void** pVal); int32_t streamStateGetByPos(SStreamState* pState, void* pos, void** pVal);
int32_t streamStateDel(SStreamState* pState, const SWinKey* key); void streamStateDel(SStreamState* pState, const SWinKey* key);
int32_t streamStateClear(SStreamState* pState); void streamStateClear(SStreamState* pState);
void streamStateSetNumber(SStreamState* pState, int32_t number, int32_t tsIdex); void streamStateSetNumber(SStreamState* pState, int32_t number, int32_t tsIdex);
int32_t streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); void streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen); int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
// session window // session window
int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen); int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen);
int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key); void streamStateSessionDel(SStreamState* pState, const SSessionKey* key);
int32_t streamStateSessionReset(SStreamState* pState, void* pVal); void streamStateSessionReset(SStreamState* pState, void* pVal);
int32_t streamStateSessionClear(SStreamState* pState); void streamStateSessionClear(SStreamState* pState);
int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
@ -70,21 +71,22 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, cons
// state window // state window
int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode);
// fill // fill
int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key); void streamStateFillDel(SStreamState* pState, const SWinKey* key);
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
int32_t streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used); int32_t* pWinCode);
int32_t streamStateClearBuff(SStreamState* pState, void* pVal); void streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used);
void streamStateClearBuff(SStreamState* pState, void* pVal);
void streamStateFreeVal(void* val); void streamStateFreeVal(void* val);
// count window // count window
int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal, int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal,
int32_t* pVLen); int32_t* pVLen, int32_t* pWinCode);
int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key); SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key);
@ -97,11 +99,11 @@ void streamStateResetCur(SStreamStateCur* pCur);
int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur); void streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname); int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache); int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode);
void streamStateReloadInfo(SStreamState* pState, TSKEY ts); void streamStateReloadInfo(SStreamState* pState, TSKEY ts);

10
include/libs/stream/tstream.h
View File

@ -629,7 +629,7 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask);
int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg); int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg);
int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code); int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code);
int32_t streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo); void streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo);
#if 0 #if 0
SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId); SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId);
#endif #endif
@ -659,10 +659,10 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask);
int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt); int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt);
int32_t streamExecTask(SStreamTask* pTask); int32_t streamExecTask(SStreamTask* pTask);
int32_t streamResumeTask(SStreamTask* pTask); void streamResumeTask(SStreamTask* pTask);
int32_t streamTrySchedExec(SStreamTask* pTask); int32_t streamTrySchedExec(SStreamTask* pTask);
int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType); int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType);
int32_t streamTaskResumeInFuture(SStreamTask* pTask); void streamTaskResumeInFuture(SStreamTask* pTask);
void streamTaskClearSchedIdleInfo(SStreamTask* pTask); void streamTaskClearSchedIdleInfo(SStreamTask* pTask);
void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime); void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime);
@ -675,8 +675,8 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa
int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp); int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp);
// check downstream status // check downstream status
int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask); void streamTaskStartMonitorCheckRsp(SStreamTask* pTask);
int32_t streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id); void streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id);
void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo); void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo);
// fill-history task // fill-history task

49
include/libs/stream/tstreamFileState.h
View File

@ -18,10 +18,10 @@
#include "os.h" #include "os.h"
#include "storageapi.h"
#include "tarray.h" #include "tarray.h"
#include "tdef.h" #include "tdef.h"
#include "tlist.h" #include "tlist.h"
#include "storageapi.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -32,7 +32,7 @@ typedef SList SStreamSnapshot;
typedef void* (*_state_buff_get_fn)(void* pRowBuff, const void* pKey, size_t keyLen); typedef void* (*_state_buff_get_fn)(void* pRowBuff, const void* pKey, size_t keyLen);
typedef int32_t (*_state_buff_remove_fn)(void* pRowBuff, const void* pKey, size_t keyLen); typedef int32_t (*_state_buff_remove_fn)(void* pRowBuff, const void* pKey, size_t keyLen);
typedef int32_t (*_state_buff_remove_by_pos_fn)(SStreamFileState* pState, SRowBuffPos* pPos); typedef void (*_state_buff_remove_by_pos_fn)(SStreamFileState* pState, SRowBuffPos* pPos);
typedef void (*_state_buff_cleanup_fn)(void* pRowBuff); typedef void (*_state_buff_cleanup_fn)(void* pRowBuff);
typedef void* (*_state_buff_create_statekey_fn)(SRowBuffPos* pPos, int64_t num); typedef void* (*_state_buff_create_statekey_fn)(SRowBuffPos* pPos, int64_t num);
@ -40,28 +40,30 @@ typedef int32_t (*_state_file_remove_fn)(SStreamFileState* pFileState, const voi
typedef int32_t (*_state_file_get_fn)(SStreamFileState* pFileState, void* pKey, void* data, int32_t* pDataLen); typedef int32_t (*_state_file_get_fn)(SStreamFileState* pFileState, void* pKey, void* data, int32_t* pDataLen);
typedef int32_t (*_state_file_clear_fn)(SStreamState* pState); typedef int32_t (*_state_file_clear_fn)(SStreamState* pState);
typedef int32_t (*_state_fun_get_fn) (SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); typedef int32_t (*_state_fun_get_fn)(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal,
int32_t* pVLen, int32_t* pWinCode);
typedef int32_t (*range_cmpr_fn)(const SSessionKey* pWin1, const SSessionKey* pWin2); typedef int32_t (*range_cmpr_fn)(const SSessionKey* pWin1, const SSessionKey* pWin2);
SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize,
GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId,
int64_t checkpointId, int8_t type); int8_t type);
void streamFileStateDestroy(SStreamFileState* pFileState); void streamFileStateDestroy(SStreamFileState* pFileState);
void streamFileStateClear(SStreamFileState* pFileState); void streamFileStateClear(SStreamFileState* pFileState);
bool needClearDiskBuff(SStreamFileState* pFileState); bool needClearDiskBuff(SStreamFileState* pFileState);
void streamFileStateReleaseBuff(SStreamFileState* pFileState, SRowBuffPos* pPos, bool used); void streamFileStateReleaseBuff(SStreamFileState* pFileState, SRowBuffPos* pPos, bool used);
int32_t streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); void streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
int32_t deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen); int32_t* pWinCode);
void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen);
int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** pVal); int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** pVal);
bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen); bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen);
void putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); int32_t putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
SStreamSnapshot* getSnapshot(SStreamFileState* pFileState); SStreamSnapshot* getSnapshot(SStreamFileState* pFileState);
int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState); void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState);
int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId); void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId);
int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list); int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list);
int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark); int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark);
@ -76,16 +78,18 @@ SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState);
int32_t getRowStateRowSize(SStreamFileState* pFileState); int32_t getRowStateRowSize(SStreamFileState* pFileState);
// session window // session window
int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen); int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen); int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen);
int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos); void deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos);
int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur,
const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen); const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen);
SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen); SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen);
int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId); void recoverSesssion(SStreamFileState* pFileState, int64_t ckId);
void sessionWinStateClear(SStreamFileState* pFileState); void sessionWinStateClear(SStreamFileState* pFileState);
void sessionWinStateCleanup(void* pBuff); void sessionWinStateCleanup(void* pBuff);
@ -95,19 +99,22 @@ SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState,
SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey); SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey, COUNT_TYPE count); SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey, COUNT_TYPE count);
int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t sessionWinStateMoveToNext(SStreamStateCur* pCur); void sessionWinStateMoveToNext(SStreamStateCur* pCur);
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey, range_cmpr_fn cmpFn); int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey,
range_cmpr_fn cmpFn);
// state window // state window
int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode);
// count window // count window
int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen); int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal,
int32_t* pVLen, int32_t* pWinCode);
int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
//function // function
int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t getFunctionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); int32_t getFunctionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen);
#ifdef __cplusplus #ifdef __cplusplus

9
include/libs/stream/tstreamUpdate.h
View File

@ -25,9 +25,12 @@
extern "C" { extern "C" {
#endif #endif
int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen,
int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); SUpdateInfo** ppInfo);
TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen,
SUpdateInfo** ppInfo);
int32_t updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol,
TSKEY* pMaxResTs);
bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid); bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid);
void updateInfoDestroy(SUpdateInfo* pInfo); void updateInfoDestroy(SUpdateInfo* pInfo);

1
include/util/taoserror.h
View File

@ -556,6 +556,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x061B) #define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x061B)
#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061C) #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061C)
#define TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE TAOS_DEF_ERROR_CODE(0, 0x061D) #define TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE TAOS_DEF_ERROR_CODE(0, 0x061D)
#define TSDB_CODE_TDB_INCONSISTENT_DB_ID TAOS_DEF_ERROR_CODE(0, 0x061E)
// query // query
#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700)

2
include/util/tlist.h
View File

@ -224,7 +224,7 @@ void tdListInit(SList *list, int32_t eleSize);
void tdListEmpty(SList *list); void tdListEmpty(SList *list);
SList *tdListNew(int32_t eleSize); SList *tdListNew(int32_t eleSize);
void *tdListFree(SList *list); void *tdListFree(SList *list);
void *tdListFreeP(SList *list, FDelete fp); void tdListFreeP(SList *list, FDelete fp);
void tdListPrependNode(SList *list, SListNode *node); void tdListPrependNode(SList *list, SListNode *node);
void tdListAppendNode(SList *list, SListNode *node); void tdListAppendNode(SList *list, SListNode *node);
int32_t tdListPrepend(SList *list, void *data); int32_t tdListPrepend(SList *list, void *data);

2
include/util/tscalablebf.h
View File

@ -34,7 +34,7 @@ typedef struct SScalableBf {
int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf); int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf);
int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len); int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len);
int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len); int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len, int32_t* winRes);
int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len); int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len);
void tScalableBfDestroy(SScalableBf* pSBf); void tScalableBfDestroy(SScalableBf* pSBf);
int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder); int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder);

12
source/common/src/tdatablock.c
View File

@ -1523,9 +1523,9 @@ void blockDataFreeRes(SSDataBlock* pBlock) {
memset(&pBlock->info, 0, sizeof(SDataBlockInfo)); memset(&pBlock->info, 0, sizeof(SDataBlockInfo));
} }
void* blockDataDestroy(SSDataBlock* pBlock) { void blockDataDestroy(SSDataBlock* pBlock) {
if (pBlock == NULL) { if (pBlock == NULL) {
return NULL; return;
} }
if (IS_VAR_DATA_TYPE(pBlock->info.pks[0].type)) { if (IS_VAR_DATA_TYPE(pBlock->info.pks[0].type)) {
@ -1535,7 +1535,6 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
blockDataFreeRes(pBlock); blockDataFreeRes(pBlock);
taosMemoryFreeClear(pBlock); taosMemoryFreeClear(pBlock);
return NULL;
} }
// todo remove it // todo remove it
@ -2468,19 +2467,18 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) {
int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, char* cname) { int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, char* cname) {
if (stbFullName[0] == 0) { if (stbFullName[0] == 0) {
terrno = TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
return TSDB_CODE_FAILED;
} }
SArray* tags = taosArrayInit(0, sizeof(SSmlKv)); SArray* tags = taosArrayInit(0, sizeof(SSmlKv));
if (tags == NULL) { if (tags == NULL) {
return TSDB_CODE_FAILED; return TSDB_CODE_OUT_OF_MEMORY;
} }
if (cname == NULL) { if (cname == NULL) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
taosArrayDestroy(tags); taosArrayDestroy(tags);
return TSDB_CODE_FAILED; return terrno;
} }
int8_t type = TSDB_DATA_TYPE_UBIGINT; int8_t type = TSDB_DATA_TYPE_UBIGINT;

1
source/dnode/vnode/src/tq/tqStreamTask.c
View File

@ -365,6 +365,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
code = streamTrySchedExec(pTask); code = streamTrySchedExec(pTask);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
streamMetaReleaseTask(pStreamMeta, pTask); streamMetaReleaseTask(pStreamMeta, pTask);
taosArrayDestroy(pTaskList);
return -1; return -1;
} }
} }

39
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -92,10 +92,12 @@ _exit:
static int32_t tsdbSaveFSToFile(STsdbFS *pFS, const char *fname) { static int32_t tsdbSaveFSToFile(STsdbFS *pFS, const char *fname) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
TdFilePtr pFD = NULL;
uint8_t *pData = NULL;
// encode to binary // encode to binary
int32_t size = tsdbFSToBinary(NULL, pFS) + sizeof(TSCKSUM); int32_t size = tsdbFSToBinary(NULL, pFS) + sizeof(TSCKSUM);
uint8_t *pData = taosMemoryMalloc(size); pData = taosMemoryMalloc(size);
if (pData == NULL) { if (pData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
@ -104,32 +106,26 @@ static int32_t tsdbSaveFSToFile(STsdbFS *pFS, const char *fname) {
taosCalcChecksumAppend(0, pData, size); taosCalcChecksumAppend(0, pData, size);
// save to file // save to file
TdFilePtr pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFD == NULL) { if (pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
int64_t n = taosWriteFile(pFD, pData, size); int64_t n = taosWriteFile(pFD, pData, size);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
taosCloseFile(&pFD);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (taosFsyncFile(pFD) < 0) { if (taosFsyncFile(pFD) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
taosCloseFile(&pFD);
TSDB_CHECK_CODE(code, lino, _exit);
} }
taosCloseFile(&pFD);
_exit: _exit:
if (pData) taosMemoryFree(pData);
if (code) { if (code) {
tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname);
} }
taosMemoryFree(pData);
taosCloseFile(&pFD);
return code; return code;
} }
@ -180,11 +176,6 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
// if (size != tsdbLogicToFileSize(pTsdb->fs.pDelFile->size, pTsdb->pVnode->config.tsdbPageSize)) {
// code = TSDB_CODE_FILE_CORRUPTED;
// TSDB_CHECK_CODE(code, lino, _exit);
// }
} }
// SArray<SDFileSet> // SArray<SDFileSet>
@ -327,17 +318,15 @@ static int32_t load_fs(const char *fname, STsdbFS *pFS) {
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
taosCloseFile(&pFD);
// decode binary
code = tsdbBinaryToFS(pData, size, pFS); code = tsdbBinaryToFS(pData, size, pFS);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (pData) taosMemoryFree(pData);
if (code) { if (code) {
tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname);
} }
taosMemoryFree(pData);
taosCloseFile(&pFD);
return code; return code;
} }
@ -383,7 +372,11 @@ static int32_t tsdbNewFileSet(STsdb *pTsdb, SDFileSet *pSetTo, SDFileSet *pSetFr
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
*pSetTo = (SDFileSet){.diskId = pSetFrom->diskId, .fid = pSetFrom->fid, .nSttF = 0}; *pSetTo = (SDFileSet){
.diskId = pSetFrom->diskId,
.fid = pSetFrom->fid,
.nSttF = 0,
};
// head // head
pSetTo->pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); pSetTo->pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile));

124
source/dnode/vnode/src/tsdb/tsdbFS2.c
View File

@ -41,7 +41,9 @@ static const char *gCurrentFname[] = {
static int32_t create_fs(STsdb *pTsdb, STFileSystem **fs) { static int32_t create_fs(STsdb *pTsdb, STFileSystem **fs) {
fs[0] = taosMemoryCalloc(1, sizeof(*fs[0])); fs[0] = taosMemoryCalloc(1, sizeof(*fs[0]));
if (fs[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (fs[0] == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
fs[0]->tsdb = pTsdb; fs[0]->tsdb = pTsdb;
tsem_init(&fs[0]->canEdit, 0, 1); tsem_init(&fs[0]->canEdit, 0, 1);
@ -76,68 +78,74 @@ int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype) {
static int32_t save_json(const cJSON *json, const char *fname) { static int32_t save_json(const cJSON *json, const char *fname) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
char *data = NULL;
TdFilePtr fp = NULL;
char *data = cJSON_PrintUnformatted(json); data = cJSON_PrintUnformatted(json);
if (data == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (data == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
TdFilePtr fp = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); fp = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (fp == NULL) { if (fp == NULL) {
code = TAOS_SYSTEM_ERROR(code); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit);
goto _exit;
} }
if (taosWriteFile(fp, data, strlen(data)) < 0) { if (taosWriteFile(fp, data, strlen(data)) < 0) {
code = TAOS_SYSTEM_ERROR(code); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit);
goto _exit;
} }
if (taosFsyncFile(fp) < 0) { if (taosFsyncFile(fp) < 0) {
code = TAOS_SYSTEM_ERROR(code); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit);
goto _exit;
} }
taosCloseFile(&fp);
_exit: _exit:
if (code) {
tsdbError("%s failed at %s:%d since %s", __func__, fname, __LINE__, tstrerror(code));
}
taosMemoryFree(data); taosMemoryFree(data);
taosCloseFile(&fp);
return code; return code;
} }
static int32_t load_json(const char *fname, cJSON **json) { static int32_t load_json(const char *fname, cJSON **json) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
char *data = NULL; char *data = NULL;
TdFilePtr fp = taosOpenFile(fname, TD_FILE_READ); TdFilePtr fp = taosOpenFile(fname, TD_FILE_READ);
if (fp == NULL) return TAOS_SYSTEM_ERROR(code); if (fp == NULL) {
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
int64_t size; int64_t size;
if (taosFStatFile(fp, &size, NULL) < 0) { if (taosFStatFile(fp, &size, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(code); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit);
goto _exit;
} }
data = taosMemoryMalloc(size + 1); data = taosMemoryMalloc(size + 1);
if (data == NULL) { if (data == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _exit;
} }
if (taosReadFile(fp, data, size) < 0) { if (taosReadFile(fp, data, size) < 0) {
code = TAOS_SYSTEM_ERROR(code); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit);
goto _exit;
} }
data[size] = '\0'; data[size] = '\0';
json[0] = cJSON_Parse(data); json[0] = cJSON_Parse(data);
if (json[0] == NULL) { if (json[0] == NULL) {
code = TSDB_CODE_FILE_CORRUPTED; TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
goto _exit;
} }
_exit: _exit:
if (code) {
tsdbError("%s failed at %s:%d since %s", __func__, fname, __LINE__, tstrerror(code));
json[0] = NULL;
}
taosCloseFile(&fp); taosCloseFile(&fp);
if (data) taosMemoryFree(data); taosMemoryFree(data);
if (code) json[0] = NULL;
return code; return code;
} }
@ -146,26 +154,25 @@ int32_t save_fs(const TFileSetArray *arr, const char *fname) {
int32_t lino = 0; int32_t lino = 0;
cJSON *json = cJSON_CreateObject(); cJSON *json = cJSON_CreateObject();
if (!json) return TSDB_CODE_OUT_OF_MEMORY; if (json == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
// fmtv // fmtv
if (cJSON_AddNumberToObject(json, "fmtv", 1) == NULL) { if (cJSON_AddNumberToObject(json, "fmtv", 1) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// fset // fset
cJSON *ajson = cJSON_AddArrayToObject(json, "fset"); cJSON *ajson = cJSON_AddArrayToObject(json, "fset");
if (!ajson) { if (!ajson) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
const STFileSet *fset; const STFileSet *fset;
TARRAY2_FOREACH(arr, fset) { TARRAY2_FOREACH(arr, fset) {
cJSON *item = cJSON_CreateObject(); cJSON *item = cJSON_CreateObject();
if (!item) { if (!item) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
cJSON_AddItemToArray(ajson, item); cJSON_AddItemToArray(ajson, item);
@ -226,14 +233,17 @@ static int32_t load_fs(STsdb *pTsdb, const char *fname, TFileSetArray *arr) {
_exit: _exit:
if (code) { if (code) {
tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); tsdbError("%s failed at %sP%d since %s, fname:%s", __func__, __FILE__, lino, tstrerror(code), fname);
}
if (json) {
cJSON_Delete(json);
} }
if (json) cJSON_Delete(json);
return code; return code;
} }
static int32_t apply_commit(STFileSystem *fs) { static int32_t apply_commit(STFileSystem *fs) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
TFileSetArray *fsetArray1 = fs->fSetArr; TFileSetArray *fsetArray1 = fs->fSetArr;
TFileSetArray *fsetArray2 = fs->fSetArrTmp; TFileSetArray *fsetArray2 = fs->fSetArrTmp;
int32_t i1 = 0, i2 = 0; int32_t i1 = 0, i2 = 0;
@ -250,15 +260,15 @@ static int32_t apply_commit(STFileSystem *fs) {
} else if (fset1->fid > fset2->fid) { } else if (fset1->fid > fset2->fid) {
// create new file set with fid of fset2->fid // create new file set with fid of fset2->fid
code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1); code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1);
if (code) return code; TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn); code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn);
if (code) return code; TSDB_CHECK_CODE(code, lino, _exit);
i1++; i1++;
i2++; i2++;
} else { } else {
// edit // edit
code = tsdbTFileSetApplyEdit(fs->tsdb, fset2, fset1); code = tsdbTFileSetApplyEdit(fs->tsdb, fset2, fset1);
if (code) return code; TSDB_CHECK_CODE(code, lino, _exit);
i1++; i1++;
i2++; i2++;
} }
@ -269,15 +279,19 @@ static int32_t apply_commit(STFileSystem *fs) {
} else { } else {
// create new file set with fid of fset2->fid // create new file set with fid of fset2->fid
code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1); code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1);
if (code) return code; TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn); code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn);
if (code) return code; TSDB_CHECK_CODE(code, lino, _exit);
i1++; i1++;
i2++; i2++;
} }
} }
return 0; _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(fs->tsdb->pVnode), lino, code);
}
return code;
} }
static int32_t commit_edit(STFileSystem *fs) { static int32_t commit_edit(STFileSystem *fs) {
@ -303,7 +317,8 @@ static int32_t commit_edit(STFileSystem *fs) {
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(fs->tsdb->pVnode), __func__, lino, tstrerror(code)); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(fs->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbInfo("vgId:%d %s success, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype); tsdbInfo("vgId:%d %s success, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype);
} }
@ -335,7 +350,8 @@ static int32_t abort_edit(STFileSystem *fs) {
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed since %s", TD_VID(fs->tsdb->pVnode), __func__, tstrerror(code)); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(fs->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbInfo("vgId:%d %s success, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype); tsdbInfo("vgId:%d %s success, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype);
} }
@ -360,13 +376,6 @@ static int32_t tsdbFSDoScanAndFixFile(STFileSystem *fs, const STFileObj *fobj) {
} }
found = true; found = true;
/*
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
long s3_size = s3Size(object_name);
if (s3_size > 0) {
return 0;
}
*/
} }
if (!found) { if (!found) {
@ -376,16 +385,6 @@ static int32_t tsdbFSDoScanAndFixFile(STFileSystem *fs, const STFileObj *fobj) {
} }
} }
{ // TODO: check file size
// int64_t fsize;
// if (taosStatFile(fobj->fname, &fsize, NULL, NULL) < 0) {
// code = TAOS_SYSTEM_ERROR(terrno);
// tsdbError("vgId:%d %s failed since file:%s stat failed, reason:%s", TD_VID(fs->tsdb->pVnode), __func__,
// fobj->fname, tstrerror(code));
// return code;
// }
}
return 0; return 0;
} }
@ -408,6 +407,7 @@ static int32_t tsdbFSAddEntryToFileObjHash(STFileHash *hash, const char *fname)
static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) { static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
// init hash table // init hash table
@ -415,14 +415,13 @@ static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) {
hash->numBucket = 4096; hash->numBucket = 4096;
hash->buckets = taosMemoryCalloc(hash->numBucket, sizeof(STFileHashEntry *)); hash->buckets = taosMemoryCalloc(hash->numBucket, sizeof(STFileHashEntry *));
if (hash->buckets == NULL) { if (hash->buckets == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
return code;
} }
// vnode.json // vnode.json
current_fname(fs->tsdb, fname, TSDB_FCURRENT); current_fname(fs->tsdb, fname, TSDB_FCURRENT);
code = tsdbFSAddEntryToFileObjHash(hash, fname); code = tsdbFSAddEntryToFileObjHash(hash, fname);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
// other // other
STFileSet *fset = NULL; STFileSet *fset = NULL;
@ -431,7 +430,7 @@ static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) {
for (int32_t i = 0; i < TSDB_FTYPE_MAX; i++) { for (int32_t i = 0; i < TSDB_FTYPE_MAX; i++) {
if (fset->farr[i] != NULL) { if (fset->farr[i] != NULL) {
code = tsdbFSAddEntryToFileObjHash(hash, fset->farr[i]->fname); code = tsdbFSAddEntryToFileObjHash(hash, fset->farr[i]->fname);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
@ -441,13 +440,14 @@ static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) {
STFileObj *fobj; STFileObj *fobj;
TARRAY2_FOREACH(lvl->fobjArr, fobj) { TARRAY2_FOREACH(lvl->fobjArr, fobj) {
code = tsdbFSAddEntryToFileObjHash(hash, fobj->fname); code = tsdbFSAddEntryToFileObjHash(hash, fobj->fname);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(fs->tsdb->pVnode), lino, code);
tsdbFSDestroyFileObjHash(hash); tsdbFSDestroyFileObjHash(hash);
} }
return code; return code;

4
source/dnode/vnode/src/tsdb/tsdbFSetRAW.c
View File

@ -35,7 +35,9 @@ int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **wri
int32_t lino = 0; int32_t lino = 0;
writer[0] = taosMemoryCalloc(1, sizeof(SFSetRAWWriter)); writer[0] = taosMemoryCalloc(1, sizeof(SFSetRAWWriter));
if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (writer[0] == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
writer[0]->config[0] = config[0]; writer[0]->config[0] = config[0];

7
source/dnode/vnode/src/tsdb/tsdbFSetRW.c
View File

@ -44,9 +44,10 @@ static int32_t tsdbFSetWriteTableDataBegin(SFSetWriter *writer, const TABLEID *t
writer->ctx->tbid->uid = tbid->uid; writer->ctx->tbid->uid = tbid->uid;
code = tsdbUpdateSkmTb(writer->config->tsdb, writer->ctx->tbid, writer->skmTb); code = tsdbUpdateSkmTb(writer->config->tsdb, writer->ctx->tbid, writer->skmTb);
TSDB_CHECK_CODE(code , lino, _exit);
code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, tbid->suid ? tbid->suid : tbid->uid, &writer->pColCmprObj); code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, tbid->suid ? tbid->suid : tbid->uid, &writer->pColCmprObj);
// TSDB_CHECK_CODE(code, lino, _exit); // TODO: TSDB_CHECK_CODE(code, lino, _exit);
writer->blockDataIdx = 0; writer->blockDataIdx = 0;
for (int32_t i = 0; i < ARRAY_SIZE(writer->blockData); i++) { for (int32_t i = 0; i < ARRAY_SIZE(writer->blockData); i++) {
@ -136,7 +137,9 @@ int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer) {
int32_t lino = 0; int32_t lino = 0;
writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); writer[0] = taosMemoryCalloc(1, sizeof(*writer[0]));
if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (writer[0] == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
writer[0]->config[0] = config[0]; writer[0]->config[0] = config[0];

3
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -4648,7 +4648,8 @@ void tsdbReaderClose2(STsdbReader* pReader) {
} }
if (pReader->resBlockInfo.freeBlock) { if (pReader->resBlockInfo.freeBlock) {
pReader->resBlockInfo.pResBlock = blockDataDestroy(pReader->resBlockInfo.pResBlock); blockDataDestroy(pReader->resBlockInfo.pResBlock);
pReader->resBlockInfo.pResBlock = NULL;
} }
taosMemoryFree(pSupInfo->colId); taosMemoryFree(pSupInfo->colId);

342
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -16,10 +16,12 @@
#include "cos.h" #include "cos.h"
#include "crypt.h" #include "crypt.h"
#include "tsdb.h" #include "tsdb.h"
#include "tsdbDef.h"
#include "vnd.h" #include "vnd.h"
static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
const char *path = pFD->path; const char *path = pFD->path;
int32_t szPage = pFD->szPage; int32_t szPage = pFD->szPage;
int32_t flag = pFD->flag; int32_t flag = pFD->flag;
@ -38,70 +40,27 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
char *dot = strrchr(lc_path, '.'); char *dot = strrchr(lc_path, '.');
if (!dot) { if (!dot) {
tsdbError("unexpected path: %s", lc_path); tsdbError("unexpected path: %s", lc_path);
code = TAOS_SYSTEM_ERROR(ENOENT); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(ENOENT), lino, _exit);
goto _exit;
} }
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", pFD->lcn); snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", pFD->lcn);
pFD->pFD = taosOpenFile(lc_path, flag); pFD->pFD = taosOpenFile(lc_path, flag);
if (pFD->pFD == NULL) { if (pFD->pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
// taosMemoryFree(pFD);
goto _exit;
} }
if (taosStatFile(lc_path, &lc_size, NULL, NULL) < 0) { if (taosStatFile(lc_path, &lc_size, NULL, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
// taosCloseFile(&pFD->pFD);
// taosMemoryFree(pFD);
goto _exit;
} }
} else { } else {
tsdbInfo("no file: %s", path); tsdbInfo("no file: %s", path);
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
// taosMemoryFree(pFD);
goto _exit;
} }
pFD->s3File = 1; pFD->s3File = 1;
/*
const char *object_name = taosDirEntryBaseName((char *)path);
long s3_size = 0;
if (tsS3Enabled) {
long size = s3Size(object_name);
if (size < 0) {
code = terrno = TSDB_CODE_FAILED_TO_CONNECT_S3;
goto _exit;
}
s3_size = size;
}
if (tsS3Enabled && !strncmp(path + strlen(path) - 5, ".data", 5) && s3_size > 0) {
#ifndef S3_BLOCK_CACHE
s3EvictCache(path, s3_size);
s3Get(object_name, path);
pFD->pFD = taosOpenFile(path, flag);
if (pFD->pFD == NULL) {
code = TAOS_SYSTEM_ERROR(ENOENT);
// taosMemoryFree(pFD);
goto _exit;
}
#else
pFD->pFD = (TdFilePtr)&pFD->s3File;
int32_t vid = 0;
sscanf(object_name, "v%df%dver%" PRId64 ".data", &vid, &pFD->fid, &pFD->cid);
pFD->objName = object_name;
// pFD->szFile = s3_size;
#endif
*/
} }
pFD->pBuf = taosMemoryCalloc(1, szPage); pFD->pBuf = taosMemoryCalloc(1, szPage);
if (pFD->pBuf == NULL) { if (pFD->pBuf == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
// taosCloseFile(&pFD->pFD);
// taosMemoryFree(pFD);
goto _exit;
} }
if (lc_size > 0) { if (lc_size > 0) {
@ -114,11 +73,7 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
// not check file size when reading data files. // not check file size when reading data files.
if (flag != TD_FILE_READ /* && !pFD->s3File*/) { if (flag != TD_FILE_READ /* && !pFD->s3File*/) {
if (!lc_size && taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) { if (!lc_size && taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
// taosMemoryFree(pFD->pBuf);
// taosCloseFile(&pFD->pFD);
// taosMemoryFree(pFD);
goto _exit;
} }
} }
@ -126,12 +81,16 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
pFD->szFile = pFD->szFile / szPage; pFD->szFile = pFD->szFile / szPage;
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
// =============== PAGE-WISE FILE =============== // =============== PAGE-WISE FILE ===============
int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn) { int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
STsdbFD *pFD = NULL; STsdbFD *pFD = NULL;
int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; int32_t szPage = pTsdb->pVnode->config.tsdbPageSize;
@ -139,8 +98,7 @@ int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppF
pFD = (STsdbFD *)taosMemoryCalloc(1, sizeof(*pFD) + strlen(path) + 1); pFD = (STsdbFD *)taosMemoryCalloc(1, sizeof(*pFD) + strlen(path) + 1);
if (pFD == NULL) { if (pFD == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _exit;
} }
pFD->path = (char *)&pFD[1]; pFD->path = (char *)&pFD[1];
@ -155,6 +113,9 @@ int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppF
*ppFD = pFD; *ppFD = pFD;
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pTsdb->pVnode), lino, code);
}
return code; return code;
} }
@ -172,12 +133,11 @@ void tsdbCloseFile(STsdbFD **ppFD) {
static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey) { static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
if (!pFD->pFD) { if (!pFD->pFD) {
code = tsdbOpenFileImpl(pFD); code = tsdbOpenFileImpl(pFD);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _exit;
}
} }
if (pFD->pgno > 0) { if (pFD->pgno > 0) {
@ -193,8 +153,7 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e
int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
goto _exit;
} }
taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage); taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage);
@ -223,8 +182,7 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e
n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage); n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
goto _exit;
} }
if (pFD->szFile < pFD->pgno) { if (pFD->szFile < pFD->pgno) {
@ -234,18 +192,20 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e
pFD->pgno = 0; pFD->pgno = 0;
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgorithm, char *encryptKey) { static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
// ASSERT(pgno <= pFD->szFile); // ASSERT(pgno <= pFD->szFile);
if (!pFD->pFD) { if (!pFD->pFD) {
code = tsdbOpenFileImpl(pFD); code = tsdbOpenFileImpl(pFD);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _exit;
}
} }
int64_t offset = PAGE_OFFSET(pgno, pFD->szPage); int64_t offset = PAGE_OFFSET(pgno, pFD->szPage);
@ -257,43 +217,19 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor
offset -= chunkoffset; offset -= chunkoffset;
} }
ASSERT(offset >= 0); ASSERT(offset >= 0);
/*
if (pFD->s3File) {
LRUHandle *handle = NULL;
pFD->blkno = (pgno + tsS3BlockSize - 1) / tsS3BlockSize;
code = tsdbCacheGetBlockS3(pFD->pTsdb->bCache, pFD, &handle);
if (code != TSDB_CODE_SUCCESS || handle == NULL) {
tsdbCacheRelease(pFD->pTsdb->bCache, handle);
if (code == TSDB_CODE_SUCCESS && !handle) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
goto _exit;
}
uint8_t *pBlock = (uint8_t *)taosLRUCacheValue(pFD->pTsdb->bCache, handle);
int64_t blk_offset = (pFD->blkno - 1) * tsS3BlockSize * pFD->szPage;
memcpy(pFD->pBuf, pBlock + (offset - blk_offset), pFD->szPage);
tsdbCacheRelease(pFD->pTsdb->bCache, handle);
} else {
*/
// seek // seek
int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
goto _exit;
} }
// read // read
n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage); n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
goto _exit;
} else if (n < pFD->szPage) { } else if (n < pFD->szPage) {
code = TSDB_CODE_FILE_CORRUPTED; TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
goto _exit;
} }
//} //}
@ -322,19 +258,22 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor
// check // check
if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
code = TSDB_CODE_FILE_CORRUPTED; TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
goto _exit;
} }
pFD->pgno = pgno; pFD->pgno = pgno;
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm, int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
char *encryptKey) { char *encryptKey) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
int64_t bOffset = fOffset % pFD->szPage; int64_t bOffset = fOffset % pFD->szPage;
@ -343,11 +282,11 @@ int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t
do { do {
if (pFD->pgno != pgno) { if (pFD->pgno != pgno) {
code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey); code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
if (pgno <= pFD->szFile) { if (pgno <= pFD->szFile) {
code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey); code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
pFD->pgno = pgno; pFD->pgno = pgno;
} }
@ -362,12 +301,16 @@ int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t
} while (n < size); } while (n < size);
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm, static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
char *encryptKey) { char *encryptKey) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
int64_t n = 0; int64_t n = 0;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
@ -380,7 +323,7 @@ static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int6
while (n < size) { while (n < size) {
if (pFD->pgno != pgno) { if (pFD->pgno != pgno) {
code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey); code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} }
int64_t nRead = TMIN(szPgCont - bOffset, size - n); int64_t nRead = TMIN(szPgCont - bOffset, size - n);
@ -392,15 +335,20 @@ static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int6
} }
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config; SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config;
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize; int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int64_t cOffset = offset % chunksize; int64_t cOffset = offset % chunksize;
int64_t n = 0; int64_t n = 0;
char *buf = NULL;
char *object_name = taosDirEntryBaseName(pFD->path); char *object_name = taosDirEntryBaseName(pFD->path);
char object_name_prefix[TSDB_FILENAME_LEN]; char object_name_prefix[TSDB_FILENAME_LEN];
@ -410,11 +358,13 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo
char *dot = strrchr(object_name_prefix, '.'); char *dot = strrchr(object_name_prefix, '.');
if (!dot) { if (!dot) {
tsdbError("unexpected path: %s", object_name_prefix); tsdbError("unexpected path: %s", object_name_prefix);
code = TAOS_SYSTEM_ERROR(ENOENT); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(ENOENT), lino, _exit);
goto _exit;
} }
char *buf = taosMemoryCalloc(1, size); buf = taosMemoryCalloc(1, size);
if (buf == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
for (int32_t chunkno = offset / chunksize + 1; n < size; ++chunkno) { for (int32_t chunkno = offset / chunksize + 1; n < size; ++chunkno) {
int64_t nRead = TMIN(chunksize - cOffset, size - n); int64_t nRead = TMIN(chunksize - cOffset, size - n);
@ -423,20 +373,14 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo
// read last chunk // read last chunk
int64_t ret = taosLSeekFile(pFD->pFD, chunksize * (chunkno - pFD->lcn) + cOffset, SEEK_SET); int64_t ret = taosLSeekFile(pFD->pFD, chunksize * (chunkno - pFD->lcn) + cOffset, SEEK_SET);
if (ret < 0) { if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
taosMemoryFree(buf);
goto _exit;
} }
ret = taosReadFile(pFD->pFD, buf + n, nRead); ret = taosReadFile(pFD->pFD, buf + n, nRead);
if (ret < 0) { if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
taosMemoryFree(buf);
goto _exit;
} else if (ret < nRead) { } else if (ret < nRead) {
code = TSDB_CODE_FILE_CORRUPTED; TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
taosMemoryFree(buf);
goto _exit;
} }
} else { } else {
uint8_t *pBlock = NULL; uint8_t *pBlock = NULL;
@ -444,10 +388,7 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", chunkno); snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", chunkno);
code = s3GetObjectBlock(object_name_prefix, cOffset, nRead, check, &pBlock); code = s3GetObjectBlock(object_name_prefix, cOffset, nRead, check, &pBlock);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _exit);
taosMemoryFree(buf);
goto _exit;
}
memcpy(buf + n, pBlock, nRead); memcpy(buf + n, pBlock, nRead);
taosMemoryFree(pBlock); taosMemoryFree(pBlock);
@ -457,14 +398,19 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo
cOffset = 0; cOffset = 0;
} }
*ppBlock = buf;
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
taosMemoryFree(buf);
} else {
*ppBlock = (uint8_t *)buf;
}
return code; return code;
} }
static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) { static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
int64_t n = 0; int64_t n = 0;
int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage); int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage);
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
@ -486,7 +432,7 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
if (handle) { if (handle) {
tsdbCacheRelease(pFD->pTsdb->pgCache, handle); tsdbCacheRelease(pFD->pTsdb->pgCache, handle);
} }
goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} }
if (!handle) { if (!handle) {
@ -499,8 +445,7 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
// check // check
if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
code = TSDB_CODE_FILE_CORRUPTED; TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
goto _exit;
} }
pFD->pgno = pgno; pFD->pgno = pgno;
@ -525,16 +470,9 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
} }
int64_t retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage; int64_t retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage;
/*
code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, 1, &pBlock);
if (code != TSDB_CODE_SUCCESS) {
goto _exit;
}
*/
code = tsdbReadFileBlock(pFD, retrieve_offset, retrieve_size, 1, &pBlock); code = tsdbReadFileBlock(pFD, retrieve_offset, retrieve_size, 1, &pBlock);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _exit);
goto _exit;
}
// 3, Store Pages in Cache // 3, Store Pages in Cache
int nPage = pgnoEnd - pgno + 1; int nPage = pgnoEnd - pgno + 1;
for (int i = 0; i < nPage; ++i) { for (int i = 0; i < nPage; ++i) {
@ -550,8 +488,7 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
// check // check
if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
code = TSDB_CODE_FILE_CORRUPTED; TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
goto _exit;
} }
pFD->pgno = pgno; pFD->pgno = pgno;
@ -568,59 +505,72 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
} }
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint, int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint,
int32_t encryptAlgorithm, char *encryptKey) { int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
if (!pFD->pFD) { if (!pFD->pFD) {
code = tsdbOpenFileImpl(pFD); code = tsdbOpenFileImpl(pFD);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _exit;
}
} }
if (pFD->s3File && pFD->lcn > 1 /* && tsS3BlockSize < 0*/) { if (pFD->s3File && pFD->lcn > 1 /* && tsS3BlockSize < 0*/) {
return tsdbReadFileS3(pFD, offset, pBuf, size, szHint); code = tsdbReadFileS3(pFD, offset, pBuf, size, szHint);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
return tsdbReadFileImp(pFD, offset, pBuf, size, encryptAlgorithm, encryptKey); code = tsdbReadFileImp(pFD, offset, pBuf, size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint, int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint,
int32_t encryptAlgorithm, char *encryptKey) { int32_t encryptAlgorithm, char *encryptKey) {
int32_t code; int32_t code;
int32_t lino;
code = tBufferEnsureCapacity(buffer, buffer->size + size); code = tBufferEnsureCapacity(buffer, buffer->size + size);
if (code) return code; TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadFile(pFD, offset, (uint8_t *)tBufferGetDataEnd(buffer), size, szHint, encryptAlgorithm, encryptKey); code = tsdbReadFile(pFD, offset, (uint8_t *)tBufferGetDataEnd(buffer), size, szHint, encryptAlgorithm, encryptKey);
if (code) return code; TSDB_CHECK_CODE(code, lino, _exit);
buffer->size += size; buffer->size += size;
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey) { int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0; int32_t code = 0;
/* int32_t lino;
if (pFD->s3File) {
tsdbWarn("%s file: %s", __func__, pFD->path);
return code;
}
*/
code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey); code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
if (taosFsyncFile(pFD->pFD) < 0) { if (taosFsyncFile(pFD->pFD) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
goto _exit;
} }
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
@ -635,8 +585,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
// alloc // alloc
pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader)); pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) { if (pReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
pReader->pTsdb = pTsdb; pReader->pTsdb = pTsdb;
pReader->pSet = pSet; pReader->pSet = pSet;
@ -711,22 +660,25 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) {
int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SHeadFile *pHeadFile = pReader->pSet->pHeadF; SHeadFile *pHeadFile = pReader->pSet->pHeadF;
int64_t offset = pHeadFile->offset; int64_t offset = pHeadFile->offset;
int64_t size = pHeadFile->size - offset; int64_t size = pHeadFile->size - offset;
taosArrayClear(aBlockIdx); taosArrayClear(aBlockIdx);
if (size == 0) return code; if (size == 0) {
return code;
}
// alloc // alloc
code = tRealloc(&pReader->aBuf[0], size); code = tRealloc(&pReader->aBuf[0], size);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// read // read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// decode // decode
int64_t n = 0; int64_t n = 0;
@ -735,37 +687,39 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
n += tGetBlockIdx(pReader->aBuf[0] + n, &blockIdx); n += tGetBlockIdx(pReader->aBuf[0] + n, &blockIdx);
if (taosArrayPush(aBlockIdx, &blockIdx) == NULL) { if (taosArrayPush(aBlockIdx, &blockIdx) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
} }
ASSERT(n == size); ASSERT(n == size);
return code; _exit:
if (code) {
_err: TSDB_ERROR_LOG(TD_VID(pReader->pTsdb->pVnode), lino, code);
tsdbError("vgId:%d, read block idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) { int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SSttFile *pSttFile = pReader->pSet->aSttF[iStt]; SSttFile *pSttFile = pReader->pSet->aSttF[iStt];
int64_t offset = pSttFile->offset; int64_t offset = pSttFile->offset;
int64_t size = pSttFile->size - offset; int64_t size = pSttFile->size - offset;
taosArrayClear(aSttBlk); taosArrayClear(aSttBlk);
if (size == 0) return code; if (size == 0) {
return 0;
}
// alloc // alloc
code = tRealloc(&pReader->aBuf[0], size); code = tRealloc(&pReader->aBuf[0], size);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// read // read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// decode // decode
int64_t n = 0; int64_t n = 0;
@ -774,46 +728,45 @@ int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) {
n += tGetSttBlk(pReader->aBuf[0] + n, &sttBlk); n += tGetSttBlk(pReader->aBuf[0] + n, &sttBlk);
if (taosArrayPush(aSttBlk, &sttBlk) == NULL) { if (taosArrayPush(aSttBlk, &sttBlk) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
} }
ASSERT(n == size); ASSERT(n == size);
return code; _exit:
if (code) {
_err: TSDB_ERROR_LOG(TD_VID(pReader->pTsdb->pVnode), lino, code);
tsdbError("vgId:%d, read stt blk failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mDataBlk) { int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mDataBlk) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
int64_t offset = pBlockIdx->offset; int64_t offset = pBlockIdx->offset;
int64_t size = pBlockIdx->size; int64_t size = pBlockIdx->size;
// alloc // alloc
code = tRealloc(&pReader->aBuf[0], size); code = tRealloc(&pReader->aBuf[0], size);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// read // read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// decode // decode
int64_t n = tGetMapData(pReader->aBuf[0], mDataBlk); int64_t n = tGetMapData(pReader->aBuf[0], mDataBlk);
if (n < 0) { if (n < 0) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
ASSERT(n == size); ASSERT(n == size);
return code; _exit:
if (code) {
_err: TSDB_ERROR_LOG(TD_VID(pReader->pTsdb->pVnode), lino, code);
tsdbError("vgId:%d, read block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
@ -834,8 +787,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb
// alloc // alloc
pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader)); pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader));
if (pDelFReader == NULL) { if (pDelFReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _exit;
} }
// open impl // open impl
@ -844,15 +796,13 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb
tsdbDelFileName(pTsdb, pFile, fname); tsdbDelFileName(pTsdb, pFile, fname);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pDelFReader->pReadH, 0); code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pDelFReader->pReadH, 0);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
taosMemoryFree(pDelFReader);
goto _exit;
}
_exit: _exit:
if (code) { if (code) {
*ppReader = NULL; *ppReader = NULL;
tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); TSDB_ERROR_LOG(TD_VID(pTsdb->pVnode), lino, code);
taosMemoryFree(pDelFReader);
} else { } else {
*ppReader = pDelFReader; *ppReader = pDelFReader;
} }
@ -881,6 +831,7 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData
int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer) { int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
int64_t offset = pDelIdx->offset; int64_t offset = pDelIdx->offset;
int64_t size = pDelIdx->size; int64_t size = pDelIdx->size;
int64_t n; int64_t n;
@ -889,13 +840,13 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa
// alloc // alloc
code = tRealloc(&pReader->aBuf[0], size); code = tRealloc(&pReader->aBuf[0], size);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// read // read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// // decode // // decode
n = 0; n = 0;
@ -907,22 +858,22 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa
continue; continue;
} }
if (taosArrayPush(aDelData, &delData) == NULL) { if (taosArrayPush(aDelData, &delData) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
} }
ASSERT(n == size); ASSERT(n == size);
return code; _exit:
if (code) {
_err: TSDB_ERROR_LOG(TD_VID(pReader->pTsdb->pVnode), lino, code);
tsdbError("vgId:%d, read del data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
int32_t n; int32_t n;
int64_t offset = pReader->fDel.offset; int64_t offset = pReader->fDel.offset;
int64_t size = pReader->fDel.size - offset; int64_t size = pReader->fDel.size - offset;
@ -931,13 +882,13 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
// alloc // alloc
code = tRealloc(&pReader->aBuf[0], size); code = tRealloc(&pReader->aBuf[0], size);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// read // read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// decode // decode
n = 0; n = 0;
@ -947,16 +898,15 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx); n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx);
if (taosArrayPush(aDelIdx, &delIdx) == NULL) { if (taosArrayPush(aDelIdx, &delIdx) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
} }
ASSERT(n == size); ASSERT(n == size);
return code; _exit:
if (code) {
_err: TSDB_ERROR_LOG(TD_VID(pReader->pTsdb->pVnode), lino, code);
tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }

17
source/dnode/vnode/src/vnd/vnodeAsync.c
View File

@ -190,10 +190,10 @@ static int32_t vnodeAsyncCancelAllTasks(SVAsync *async, SArray *cancelArray) {
task->prev->next = task->next; task->prev->next = task->next;
task->next->prev = task->prev; task->next->prev = task->prev;
if (task->cancel) { if (task->cancel) {
taosArrayPush(cancelArray, &(SVATaskCancelInfo){ TAOS_UNUSED(taosArrayPush(cancelArray, &(SVATaskCancelInfo){
.cancel = task->cancel, .cancel = task->cancel,
.arg = task->arg, .arg = task->arg,
}); }));
} }
vnodeAsyncTaskDone(async, task); vnodeAsyncTaskDone(async, task);
} }
@ -206,6 +206,9 @@ static void *vnodeAsyncLoop(void *arg) {
SVWorker *worker = (SVWorker *)arg; SVWorker *worker = (SVWorker *)arg;
SVAsync *async = worker->async; SVAsync *async = worker->async;
SArray *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo)); SArray *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo));
if (cancelArray == NULL) {
return NULL;
}
setThreadName(async->label); setThreadName(async->label);
@ -466,7 +469,7 @@ int32_t vnodeAsyncOpen(int32_t numOfThreads) {
vnodeAsyncSetWorkers(2, numOfThreads); vnodeAsyncSetWorkers(2, numOfThreads);
_exit: _exit:
return 0; return code;
} }
int32_t vnodeAsyncClose() { int32_t vnodeAsyncClose() {
@ -748,10 +751,10 @@ int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) {
task->prev->next = task->next; task->prev->next = task->next;
task->next->prev = task->prev; task->next->prev = task->prev;
if (task->cancel) { if (task->cancel) {
taosArrayPush(cancelArray, &(SVATaskCancelInfo){ TAOS_UNUSED(taosArrayPush(cancelArray, &(SVATaskCancelInfo){
.cancel = task->cancel, .cancel = task->cancel,
.arg = task->arg, .arg = task->arg,
}); }));
} }
vnodeAsyncTaskDone(async, task); vnodeAsyncTaskDone(async, task);
} }
@ -763,10 +766,10 @@ int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) {
channel->scheduled->prev->next = channel->scheduled->next; channel->scheduled->prev->next = channel->scheduled->next;
channel->scheduled->next->prev = channel->scheduled->prev; channel->scheduled->next->prev = channel->scheduled->prev;
if (channel->scheduled->cancel) { if (channel->scheduled->cancel) {
taosArrayPush(cancelArray, &(SVATaskCancelInfo){ TAOS_UNUSED(taosArrayPush(cancelArray, &(SVATaskCancelInfo){
.cancel = channel->scheduled->cancel, .cancel = channel->scheduled->cancel,
.arg = channel->scheduled->arg, .arg = channel->scheduled->arg,
}); }));
} }
vnodeAsyncTaskDone(async, channel->scheduled); vnodeAsyncTaskDone(async, channel->scheduled);
} }

22
source/dnode/vnode/src/vnd/vnodeBufPool.c
View File

@ -16,13 +16,12 @@
#include "vnd.h" #include "vnd.h"
/* ------------------------ STRUCTURES ------------------------ */ /* ------------------------ STRUCTURES ------------------------ */
static int vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBufPool **ppPool) { static int32_t vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBufPool **ppPool) {
SVBufPool *pPool; SVBufPool *pPool;
pPool = taosMemoryMalloc(sizeof(SVBufPool) + size); pPool = taosMemoryMalloc(sizeof(SVBufPool) + size);
if (pPool == NULL) { if (pPool == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
memset(pPool, 0, sizeof(SVBufPool)); memset(pPool, 0, sizeof(SVBufPool));
@ -44,14 +43,12 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBufPoo
pPool->lock = taosMemoryMalloc(sizeof(TdThreadSpinlock)); pPool->lock = taosMemoryMalloc(sizeof(TdThreadSpinlock));
if (!pPool->lock) { if (!pPool->lock) {
taosMemoryFree(pPool); taosMemoryFree(pPool);
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
if (taosThreadSpinInit(pPool->lock, 0) != 0) { if (taosThreadSpinInit(pPool->lock, 0) != 0) {
taosMemoryFree((void *)pPool->lock); taosMemoryFree((void *)pPool->lock);
taosMemoryFree(pPool); taosMemoryFree(pPool);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
} else { } else {
pPool->lock = NULL; pPool->lock = NULL;
@ -77,10 +74,11 @@ int vnodeOpenBufPool(SVnode *pVnode) {
for (int i = 0; i < VNODE_BUFPOOL_SEGMENTS; i++) { for (int i = 0; i < VNODE_BUFPOOL_SEGMENTS; i++) {
// create pool // create pool
if (vnodeBufPoolCreate(pVnode, i, size, &pVnode->aBufPool[i])) { int32_t code;
if ((code = vnodeBufPoolCreate(pVnode, i, size, &pVnode->aBufPool[i]))) {
vError("vgId:%d, failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); vError("vgId:%d, failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno));
vnodeCloseBufPool(pVnode); vnodeCloseBufPool(pVnode);
return -1; return code;
} }
// add to free list // add to free list
@ -274,8 +272,6 @@ _exit:
} }
int32_t vnodeBufPoolRegisterQuery(SVBufPool *pPool, SQueryNode *pQNode) { int32_t vnodeBufPoolRegisterQuery(SVBufPool *pPool, SQueryNode *pQNode) {
int32_t code = 0;
taosThreadMutexLock(&pPool->mutex); taosThreadMutexLock(&pPool->mutex);
pQNode->pNext = pPool->qList.pNext; pQNode->pNext = pPool->qList.pNext;
@ -285,9 +281,7 @@ int32_t vnodeBufPoolRegisterQuery(SVBufPool *pPool, SQueryNode *pQNode) {
pPool->nQuery++; pPool->nQuery++;
taosThreadMutexUnlock(&pPool->mutex); taosThreadMutexUnlock(&pPool->mutex);
return 0;
_exit:
return code;
} }
void vnodeBufPoolDeregisterQuery(SVBufPool *pPool, SQueryNode *pQNode, bool proactive) { void vnodeBufPoolDeregisterQuery(SVBufPool *pPool, SQueryNode *pQNode, bool proactive) {

3
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -394,8 +394,7 @@ int vnodeValidateTableHash(SVnode *pVnode, char *tableFName) {
} }
if (hashValue < pVnode->config.hashBegin || hashValue > pVnode->config.hashEnd) { if (hashValue < pVnode->config.hashBegin || hashValue > pVnode->config.hashEnd) {
terrno = TSDB_CODE_VND_HASH_MISMATCH; return terrno = TSDB_CODE_VND_HASH_MISMATCH;
return -1;
} }
return 0; return 0;

6
source/dnode/vnode/src/vnd/vnodeInitApi.c
View File

@ -130,16 +130,16 @@ void initTqAPI(SStoreTqReader* pTq) {
pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed; pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed;
pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it
// pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it // pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it
pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it
pTq->tqGetResultBlock = tqGetResultBlock; pTq->tqGetResultBlock = tqGetResultBlock;
// pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; // pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut;
pTq->tqGetResultBlockTime = tqGetResultBlockTime; pTq->tqGetResultBlockTime = tqGetResultBlockTime;
pTq->tqGetStreamExecProgress = tqGetStreamExecInfo; pTq->tqGetStreamExecProgress = tqGetStreamExecInfo;
} }
void initStateStoreAPI(SStateStore* pStore) { void initStateStoreAPI(SStateStore* pStore) {
pStore->streamFileStateInit = streamFileStateInit; pStore->streamFileStateInit = streamFileStateInit;

20
source/dnode/vnode/src/vnd/vnodeModule.c
View File

@ -19,31 +19,19 @@
static volatile int32_t VINIT = 0; static volatile int32_t VINIT = 0;
int vnodeInit(int nthreads) { int vnodeInit(int nthreads) {
int32_t init; if (atomic_val_compare_exchange_32(&VINIT, 0, 1)) {
init = atomic_val_compare_exchange_32(&VINIT, 0, 1);
if (init) {
return 0; return 0;
} }
if (vnodeAsyncOpen(nthreads) != 0) { TAOS_CHECK_RETURN(vnodeAsyncOpen(nthreads));
return -1; TAOS_CHECK_RETURN(walInit());
}
if (walInit() < 0) {
return -1;
}
return 0; return 0;
} }
void vnodeCleanup() { void vnodeCleanup() {
int32_t init = atomic_val_compare_exchange_32(&VINIT, 1, 0); if (atomic_val_compare_exchange_32(&VINIT, 1, 0) == 0) return;
if (init == 0) return;
// set stop
vnodeAsyncClose(); vnodeAsyncClose();
walCleanUp(); walCleanUp();
smaCleanUp(); smaCleanUp();
} }

566
source/libs/catalog/test/catalogTests.cpp
View File

File diff suppressed because it is too large Load Diff

52
source/libs/executor/inc/executorInt.h
View File

@ -835,7 +835,8 @@ void doBuildResultDatablock(struct SOperatorInfo* pOperator, SOptrBasicInfo* pbI
* @brief copydata from hash table, instead of copying from SGroupResInfo's pRow * @brief copydata from hash table, instead of copying from SGroupResInfo's pRow
*/ */
int32_t doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, int32_t doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold, bool ignoreGroup); SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold,
bool ignoreGroup);
bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo);
bool hasSlimitOffsetInfo(SLimitInfo* pLimitInfo); bool hasSlimitOffsetInfo(SLimitInfo* pLimitInfo);
@ -855,7 +856,8 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* de
extern void doDestroyExchangeOperatorInfo(void* param); extern void doDestroyExchangeOperatorInfo(void* param);
int32_t doFilterImpl(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo, SColumnInfoData** pResCol); int32_t doFilterImpl(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo,
SColumnInfoData** pResCol);
int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo); int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo);
int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock, int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock,
int32_t rows, SExecTaskInfo* pTask, STableMetaCacheInfo* pCache); int32_t rows, SExecTaskInfo* pTask, STableMetaCacheInfo* pCache);
@ -865,7 +867,8 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData,
void setVgIdColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int32_t vgId); void setVgIdColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int32_t vgId);
void setVgVerColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int64_t vgVer); void setVgVerColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int64_t vgVer);
int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput,
int32_t* rowEntryInfoOffset);
void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput); void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput);
SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData,
@ -894,8 +897,8 @@ bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup);
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup);
bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup,
SStateStore* pStore); SStateStore* pStore);
void appendDataToSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, void appendDataToSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp,
uint64_t* pGp, void* pTbName); void* pTbName);
uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId);
@ -906,7 +909,7 @@ bool groupbyTbname(SNodeList* pGroupList);
void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order);
int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order,
int64_t* pData); int64_t* pData);
void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId,
SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI); SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI);
SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag); SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag);
@ -929,12 +932,13 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, i
SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore, SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore,
SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr, SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr,
SStorageAPI* pApi, int32_t tsIndex); SStorageAPI* pApi, int32_t tsIndex);
int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type,
STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic); int32_t tsColIndex, STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic);
void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins); int32_t getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins);
void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList);
void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey); void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey);
void deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd); int32_t deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate,
SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd);
int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated); int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated);
int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed); int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed);
int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar); int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar);
@ -943,22 +947,24 @@ void removeSessionDeleteResults(SSHashObj* pHashMap, SArray* pWins);
int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult,
int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput, int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput,
struct SOperatorInfo* pOperator, int64_t winDelta); struct SOperatorInfo* pOperator, int64_t winDelta);
int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo); void setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo);
int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo); int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo);
int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated); int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated);
void saveDeleteRes(SSHashObj* pStDelete, SSessionKey key); int32_t saveDeleteRes(SSHashObj* pStDelete, SSessionKey key);
void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey); void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey);
void doBuildDeleteDataBlock(struct SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite); void doBuildDeleteDataBlock(struct SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite);
void doBuildSessionResult(struct SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock); void doBuildSessionResult(struct SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo,
void getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo); SSDataBlock* pBlock);
int32_t getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo);
void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin, void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin,
SResultWindowInfo* pNextWin); SResultWindowInfo* pNextWin);
void compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup, int32_t compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup,
SExecTaskInfo* pTaskInfo, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, SExecTaskInfo* pTaskInfo, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin,
SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap); SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap);
int32_t releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI); void releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI);
void resetWinRange(STimeWindow* winRange); void resetWinRange(STimeWindow* winRange);
bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId,
TSKEY ts, void* pPkVal, int32_t len);
int64_t getDeleteMark(SWindowPhysiNode* pWinPhyNode, int64_t interval); int64_t getDeleteMark(SWindowPhysiNode* pWinPhyNode, int64_t interval);
void resetUnCloseSessionWinInfo(SSHashObj* winMap); void resetUnCloseSessionWinInfo(SSHashObj* winMap);
void setStreamOperatorCompleted(struct SOperatorInfo* pOperator); void setStreamOperatorCompleted(struct SOperatorInfo* pOperator);
@ -980,25 +986,23 @@ void freeOperatorParam(SOperatorParam* pParam, SOperatorParamType type);
void freeResetOperatorParams(struct SOperatorInfo* pOperator, SOperatorParamType type, bool allFree); void freeResetOperatorParams(struct SOperatorInfo* pOperator, SOperatorParamType type, bool allFree);
SSDataBlock* getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int32_t idx, bool clearParam); SSDataBlock* getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int32_t idx, bool clearParam);
void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, EStreamType mode, SSessionKey* pDelRange); void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, EStreamType mode, SSessionKey* pDelRange);
bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey); void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey);
void saveDeleteInfo(SArray* pWins, SSessionKey key); int32_t saveDeleteInfo(SArray* pWins, SSessionKey key);
void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins); void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins);
void copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted); int32_t copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted);
void copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest); int32_t copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest);
bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo); bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo);
bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd, EStreamType blockType); bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd, EStreamType blockType);
bool compareVal(const char* v, const SStateKeys* pKey); bool compareVal(const char* v, const SStateKeys* pKey);
bool inWinRange(STimeWindow* range, STimeWindow* cur); bool inWinRange(STimeWindow* range, STimeWindow* cur);
void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result); int32_t doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result);
int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
TSKEY* primaryKeys, int32_t prevPosition, int32_t order); TSKEY* primaryKeys, int32_t prevPosition, int32_t order);
void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, int32_t status); void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, int32_t status);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

2
source/libs/executor/src/exchangeoperator.c
View File

@ -607,7 +607,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pCo
blockDataAppendColInfo(pBlock, &idata); blockDataAppendColInfo(pBlock, &idata);
} }
blockDecode(pBlock, pStart); (void) blockDecode(pBlock, pStart);
blockDataEnsureCapacity(pRes, pBlock->info.rows); blockDataEnsureCapacity(pRes, pBlock->info.rows);
// data from mnode // data from mnode

2
source/libs/executor/src/executorInt.c
View File

@ -970,7 +970,7 @@ void cleanupExprSupp(SExprSupp* pSupp) {
taosMemoryFree(pSupp->rowEntryInfoOffset); taosMemoryFree(pSupp->rowEntryInfoOffset);
} }
void cleanupBasicInfo(SOptrBasicInfo* pInfo) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } void cleanupBasicInfo(SOptrBasicInfo* pInfo) { blockDataDestroy(pInfo->pRes); pInfo->pRes = NULL;}
bool groupbyTbname(SNodeList* pGroupList) { bool groupbyTbname(SNodeList* pGroupList) {
bool bytbname = false; bool bytbname = false;

6
source/libs/executor/src/filloperator.c
View File

@ -310,8 +310,10 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
void destroyFillOperatorInfo(void* param) { void destroyFillOperatorInfo(void* param) {
SFillOperatorInfo* pInfo = (SFillOperatorInfo*)param; SFillOperatorInfo* pInfo = (SFillOperatorInfo*)param;
pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo);
pInfo->pRes = blockDataDestroy(pInfo->pRes); blockDataDestroy(pInfo->pRes);
pInfo->pFinalRes = blockDataDestroy(pInfo->pFinalRes); pInfo->pRes = NULL;
blockDataDestroy(pInfo->pFinalRes);
pInfo->pFinalRes = NULL;
cleanupExprSupp(&pInfo->noFillExprSupp); cleanupExprSupp(&pInfo->noFillExprSupp);

128
source/libs/executor/src/groupoperator.c
View File

@ -179,7 +179,7 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData
size_t numOfGroupCols = taosArrayGetSize(pGroupCols); size_t numOfGroupCols = taosArrayGetSize(pGroupCols);
for (int32_t i = 0; i < numOfGroupCols; ++i) { for (int32_t i = 0; i < numOfGroupCols; ++i) {
SColumn* pCol = (SColumn*) taosArrayGet(pGroupCols, i); SColumn* pCol = (SColumn*)taosArrayGet(pGroupCols, i);
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId);
// valid range check. todo: return error code. // valid range check. todo: return error code.
@ -585,7 +585,7 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc
blockDataDestroy(pDstBlock); blockDataDestroy(pDstBlock);
return NULL; return NULL;
} }
for(int i = 0; i < numOfCols; ++i) { for (int i = 0; i < numOfCols; ++i) {
pDstBlock->pBlockAgg[i].colId = -1; pDstBlock->pBlockAgg[i].colId = -1;
} }
} }
@ -840,9 +840,9 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) {
SDataGroupInfo* pGroupInfo = SDataGroupInfo* pGroupInfo =
(pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL;
if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) { if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) {
if(pGroupInfo != NULL) { if (pGroupInfo != NULL) {
SSDataBlock* ret = buildPartitionResultForNotLoadBlock(pGroupInfo); SSDataBlock* ret = buildPartitionResultForNotLoadBlock(pGroupInfo);
if(ret != NULL) return ret; if (ret != NULL) return ret;
} }
// try next group data // try next group data
if (pInfo->groupIndex + 1 >= taosArrayGetSize(pInfo->sortedGroupArray)) { if (pInfo->groupIndex + 1 >= taosArrayGetSize(pInfo->sortedGroupArray)) {
@ -1069,7 +1069,8 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
goto _error; goto _error;
} }
pInfo->rowCapacity = blockDataGetCapacityInRow(pInfo->binfo.pRes, getBufPageSize(pInfo->pBuf), pInfo->rowCapacity =
blockDataGetCapacityInRow(pInfo->binfo.pRes, getBufPageSize(pInfo->pBuf),
blockDataGetSerialMetaSize(taosArrayGetSize(pInfo->binfo.pRes->pDataBlock))); blockDataGetSerialMetaSize(taosArrayGetSize(pInfo->binfo.pRes->pDataBlock)));
pInfo->columnOffset = setupColumnOffset(pInfo->binfo.pRes, pInfo->rowCapacity); pInfo->columnOffset = setupColumnOffset(pInfo->binfo.pRes, pInfo->rowCapacity);
code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pInfo->pGroupCols); code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pInfo->pGroupCols);
@ -1084,8 +1085,8 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.numOfExprs = numOfCols;
pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.pExprInfo = pExprInfo;
pOperator->fpSet = pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo,
createOperatorFpSet(optrDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -1110,8 +1111,8 @@ int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo,
SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo; SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo;
SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
SResultRow* pResultRow = SResultRow* pResultRow = doSetResultOutBufByKey(pBuf, pResultRowInfo, (char*)pData, bytes, true, groupId, pTaskInfo,
doSetResultOutBufByKey(pBuf, pResultRowInfo, (char*)pData, bytes, true, groupId, pTaskInfo, false, pAggSup, false); false, pAggSup, false);
return setResultRowInitCtx(pResultRow, pCtx, numOfCols, pOperator->exprSupp.rowEntryInfoOffset); return setResultRowInitCtx(pResultRow, pCtx, numOfCols, pOperator->exprSupp.rowEntryInfoOffset);
} }
@ -1134,6 +1135,8 @@ static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo
static bool hasRemainTbName(SStreamPartitionOperatorInfo* pInfo) { return pInfo->pTbNameIte != NULL; } static bool hasRemainTbName(SStreamPartitionOperatorInfo* pInfo) { return pInfo->pTbNameIte != NULL; }
static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -1160,7 +1163,12 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
pDest->info.parTbName[0] = 0; pDest->info.parTbName[0] = 0;
if (pInfo->tbnameCalSup.numOfExprs > 0) { if (pInfo->tbnameCalSup.numOfExprs > 0) {
void* tbname = NULL; void* tbname = NULL;
if (pAPI->stateStore.streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname, false) == 0) { int32_t winCode = TSDB_CODE_SUCCESS;
code = pAPI->stateStore.streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname,
false, &winCode);
TSDB_CHECK_CODE(code, lino, _end);
if (winCode == TSDB_CODE_SUCCESS) {
memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
pAPI->stateStore.streamStateFreeVal(tbname); pAPI->stateStore.streamStateFreeVal(tbname);
} }
@ -1174,20 +1182,35 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
pOperator->resultInfo.totalRows += pDest->info.rows; pOperator->resultInfo.totalRows += pDest->info.rows;
pInfo->parIte = taosHashIterate(pInfo->pPartitions, pInfo->parIte); pInfo->parIte = taosHashIterate(pInfo->pPartitions, pInfo->parIte);
ASSERT(pDest->info.rows > 0); ASSERT(pDest->info.rows > 0);
_end:
if (code != TSDB_CODE_SUCCESS) {
blockDataCleanup(pDest);
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
printDataBlock(pDest, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); printDataBlock(pDest, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
return pDest; return pDest;
} }
void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId,
SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI) { SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
void* pValue = NULL; void* pValue = NULL;
if (pAPI->streamStateGetParName(pState, groupId, &pValue, true) != 0) { int32_t winCode = TSDB_CODE_SUCCESS;
code = pAPI->streamStateGetParName(pState, groupId, &pValue, true, &winCode);
TSDB_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS) {
SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId); SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId);
memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN);
pTmpBlock->info.id.groupId = groupId; pTmpBlock->info.id.groupId = groupId;
char* tbName = pSrcBlock->info.parTbName; char* tbName = pSrcBlock->info.parTbName;
if (pTableSup->numOfExprs > 0) { if (pTableSup->numOfExprs > 0) {
projectApplyFunctions(pTableSup->pExprInfo, pDestBlock, pTmpBlock, pTableSup->pCtx, pTableSup->numOfExprs, NULL); code = projectApplyFunctions(pTableSup->pExprInfo, pDestBlock, pTmpBlock, pTableSup->pCtx, pTableSup->numOfExprs,
NULL);
TSDB_CHECK_CODE(code, lino, _end);
SColumnInfoData* pTbCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); SColumnInfoData* pTbCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX);
memset(tbName, 0, TSDB_TABLE_NAME_LEN); memset(tbName, 0, TSDB_TABLE_NAME_LEN);
int32_t len = 0; int32_t len = 0;
@ -1198,7 +1221,8 @@ void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup
void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1); void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1);
len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1);
memcpy(tbName, varDataVal(pData), len); memcpy(tbName, varDataVal(pData), len);
pAPI->streamStatePutParName(pState, groupId, tbName); code = pAPI->streamStatePutParName(pState, groupId, tbName);
TSDB_CHECK_CODE(code, lino, _end);
} }
memcpy(pTmpBlock->info.parTbName, tbName, len); memcpy(pTmpBlock->info.parTbName, tbName, len);
pDestBlock->info.rows--; pDestBlock->info.rows--;
@ -1209,42 +1233,58 @@ void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup
} }
if (pTagSup->numOfExprs > 0) { if (pTagSup->numOfExprs > 0) {
projectApplyFunctions(pTagSup->pExprInfo, pDestBlock, pTmpBlock, pTagSup->pCtx, pTagSup->numOfExprs, NULL); code = projectApplyFunctions(pTagSup->pExprInfo, pDestBlock, pTmpBlock, pTagSup->pCtx, pTagSup->numOfExprs, NULL);
TSDB_CHECK_CODE(code, lino, _end);
pDestBlock->info.rows--; pDestBlock->info.rows--;
} else { } else {
memcpy(pDestBlock->info.parTbName, pTmpBlock->info.parTbName, TSDB_TABLE_NAME_LEN); memcpy(pDestBlock->info.parTbName, pTmpBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
} }
void* pGpIdCol = taosArrayGet(pDestBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); void* pGpIdCol = taosArrayGet(pDestBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX);
colDataSetVal(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false); code = colDataSetVal(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false);
TSDB_CHECK_CODE(code, lino, _end);
pDestBlock->info.rows++; pDestBlock->info.rows++;
blockDataDestroy(pTmpBlock); blockDataDestroy(pTmpBlock);
} else { } else {
memcpy(pSrcBlock->info.parTbName, pValue, TSDB_TABLE_NAME_LEN); memcpy(pSrcBlock->info.parTbName, pValue, TSDB_TABLE_NAME_LEN);
} }
pAPI->streamStateFreeVal(pValue); pAPI->streamStateFreeVal(pValue);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) { static int32_t buildStreamCreateTableResult(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTask = pOperator->pTaskInfo; SExecTaskInfo* pTask = pOperator->pTaskInfo;
SStreamPartitionOperatorInfo* pInfo = pOperator->info; SStreamPartitionOperatorInfo* pInfo = pOperator->info;
SSDataBlock* pSrc = pInfo->pInputDataBlock;
if ((pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) || if ((pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) ||
taosHashGetSize(pInfo->pPartitions) == 0) { taosHashGetSize(pInfo->pPartitions) == 0) {
return NULL; goto _end;
} }
blockDataCleanup(pInfo->pCreateTbRes); blockDataCleanup(pInfo->pCreateTbRes);
blockDataEnsureCapacity(pInfo->pCreateTbRes, taosHashGetSize(pInfo->pPartitions)); code = blockDataEnsureCapacity(pInfo->pCreateTbRes, taosHashGetSize(pInfo->pPartitions));
SSDataBlock* pSrc = pInfo->pInputDataBlock; TSDB_CHECK_CODE(code, lino, _end);
if (pInfo->pTbNameIte != NULL) { if (pInfo->pTbNameIte != NULL) {
SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte; SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte;
int32_t rowId = *(int32_t*)taosArrayGet(pParInfo->rowIds, 0); int32_t rowId = *(int32_t*)taosArrayGet(pParInfo->rowIds, 0);
appendCreateTableRow(pTask->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, code = appendCreateTableRow(pTask->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, pParInfo->groupId,
pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes, &pTask->storageAPI.stateStore); pSrc, rowId, pInfo->pCreateTbRes, &pTask->storageAPI.stateStore);
TSDB_CHECK_CODE(code, lino, _end);
pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, pInfo->pTbNameIte); pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, pInfo->pTbNameIte);
} }
return pInfo->pCreateTbRes->info.rows > 0 ? pInfo->pCreateTbRes : NULL;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDataBlock* pBlock) { static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDataBlock* pBlock) {
@ -1267,18 +1307,20 @@ static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDat
} }
static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamPartitionOperatorInfo* pInfo = pOperator->info;
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
return NULL; return NULL;
} }
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamPartitionOperatorInfo* pInfo = pOperator->info;
SSDataBlock* pCtRes = NULL;
if (hasRemainTbName(pInfo)) { if (hasRemainTbName(pInfo)) {
pCtRes = buildStreamCreateTableResult(pOperator); code = buildStreamCreateTableResult(pOperator);
if (pCtRes != NULL) { TSDB_CHECK_CODE(code, lino, _end);
return pCtRes; if (pInfo->pCreateTbRes && pInfo->pCreateTbRes->info.rows > 0) {
return pInfo->pCreateTbRes;
} }
} }
@ -1330,11 +1372,18 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) {
pInfo->parIte = taosHashIterate(pInfo->pPartitions, NULL); pInfo->parIte = taosHashIterate(pInfo->pPartitions, NULL);
pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, NULL); pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, NULL);
pCtRes = buildStreamCreateTableResult(pOperator); code = buildStreamCreateTableResult(pOperator);
if (pCtRes != NULL) { TSDB_CHECK_CODE(code, lino, _end);
return pCtRes; if (pInfo->pCreateTbRes && pInfo->pCreateTbRes->info.rows > 0) {
return pInfo->pCreateTbRes;
} }
return buildStreamPartitionResult(pOperator); return buildStreamPartitionResult(pOperator);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return NULL;
} }
static void destroyStreamPartitionOperatorInfo(void* param) { static void destroyStreamPartitionOperatorInfo(void* param) {
@ -1358,7 +1407,8 @@ static void destroyStreamPartitionOperatorInfo(void* param) {
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) { int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr,
SExprSupp* pTbnameExpr) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0; int32_t lino = 0;
SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI; SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI;
@ -1372,7 +1422,8 @@ int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pPar
pScanInfo->pPartScalarSup = pExpr; pScanInfo->pPartScalarSup = pExpr;
pScanInfo->pPartTbnameSup = pTbnameExpr; pScanInfo->pPartTbnameSup = pTbnameExpr;
if (!pScanInfo->pUpdateInfo) { if (!pScanInfo->pUpdateInfo) {
code = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); code = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate,
pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);
} }
return code; return code;
} }
@ -1512,8 +1563,9 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
pInfo, pTaskInfo); pInfo, pTaskInfo);
pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.numOfExprs = numOfCols;
pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.pExprInfo = pExprInfo;
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamHashPartition, NULL, pOperator->fpSet =
destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); createOperatorFpSet(optrDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState);
code = initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup); code = initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup);

3
source/libs/executor/src/hashjoinoperator.c
View File

@ -1065,7 +1065,8 @@ static void destroyHashJoinOperator(void* param) {
hJoinFreeTableInfo(&pJoinOperator->tbs[0]); hJoinFreeTableInfo(&pJoinOperator->tbs[0]);
hJoinFreeTableInfo(&pJoinOperator->tbs[1]); hJoinFreeTableInfo(&pJoinOperator->tbs[1]);
pJoinOperator->finBlk = blockDataDestroy(pJoinOperator->finBlk); blockDataDestroy(pJoinOperator->finBlk);
pJoinOperator->finBlk = NULL;
taosMemoryFreeClear(pJoinOperator->pResColMap); taosMemoryFreeClear(pJoinOperator->pResColMap);
taosArrayDestroyEx(pJoinOperator->pRowBufs, hJoinFreeBufPage); taosArrayDestroyEx(pJoinOperator->pRowBufs, hJoinFreeBufPage);

12
source/libs/executor/src/mergejoin.c
View File

@ -3305,8 +3305,10 @@ void mJoinDestroyWindowCtx(SMJoinOperatorInfo* pJoin) {
mWinJoinResetWindowCache(pCtx, &pCtx->cache); mWinJoinResetWindowCache(pCtx, &pCtx->cache);
pCtx->finBlk = blockDataDestroy(pCtx->finBlk); blockDataDestroy(pCtx->finBlk);
pCtx->cache.outBlk = blockDataDestroy(pCtx->cache.outBlk); pCtx->finBlk = NULL;
blockDataDestroy(pCtx->cache.outBlk);
pCtx->cache.outBlk = NULL;
taosArrayDestroy(pCtx->cache.grps); taosArrayDestroy(pCtx->cache.grps);
} }
@ -3378,9 +3380,11 @@ int32_t mJoinInitWindowCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* p
void mJoinDestroyMergeCtx(SMJoinOperatorInfo* pJoin) { void mJoinDestroyMergeCtx(SMJoinOperatorInfo* pJoin) {
SMJoinMergeCtx* pCtx = &pJoin->ctx.mergeCtx; SMJoinMergeCtx* pCtx = &pJoin->ctx.mergeCtx;
blockDataDestroy(pCtx->finBlk);
blockDataDestroy(pCtx->midBlk);
pCtx->finBlk = blockDataDestroy(pCtx->finBlk); pCtx->finBlk = NULL;
pCtx->midBlk = blockDataDestroy(pCtx->midBlk); pCtx->midBlk = NULL;
} }

10
source/libs/executor/src/mergeoperator.c
View File

@ -232,8 +232,11 @@ int32_t getSortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, u
void destroySortMergeOperatorInfo(void* param) { void destroySortMergeOperatorInfo(void* param) {
SSortMergeInfo* pSortMergeInfo = param; SSortMergeInfo* pSortMergeInfo = param;
pSortMergeInfo->pInputBlock = blockDataDestroy(pSortMergeInfo->pInputBlock); blockDataDestroy(pSortMergeInfo->pInputBlock);
pSortMergeInfo->pIntermediateBlock = blockDataDestroy(pSortMergeInfo->pIntermediateBlock); pSortMergeInfo->pInputBlock = NULL;
blockDataDestroy(pSortMergeInfo->pIntermediateBlock);
pSortMergeInfo->pIntermediateBlock = NULL;
taosArrayDestroy(pSortMergeInfo->matchInfo.pList); taosArrayDestroy(pSortMergeInfo->matchInfo.pList);
@ -429,7 +432,8 @@ SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) {
void destroyMultiwayMergeOperatorInfo(void* param) { void destroyMultiwayMergeOperatorInfo(void* param) {
SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)param; SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)param;
pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); blockDataDestroy(pInfo->binfo.pRes);
pInfo->binfo.pRes = NULL;
if (NULL != gMultiwayMergeFps[pInfo->type].closeFn) { if (NULL != gMultiwayMergeFps[pInfo->type].closeFn) {
(*gMultiwayMergeFps[pInfo->type].closeFn)(&pInfo->sortMergeInfo); (*gMultiwayMergeFps[pInfo->type].closeFn)(&pInfo->sortMergeInfo);

94
source/libs/executor/src/scanoperator.c
View File

@ -1870,17 +1870,29 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowId) { static int32_t calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowId) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
blockDataCleanup(pInfo->pCreateTbRes); blockDataCleanup(pInfo->pCreateTbRes);
if (pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) { if (pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) {
pBlock->info.parTbName[0] = 0; pBlock->info.parTbName[0] = 0;
} else { } else {
appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, code = appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup,
pBlock->info.id.groupId, pBlock, rowId, pInfo->pCreateTbRes, &pInfo->stateStore); &pInfo->tagCalSup, pBlock->info.id.groupId, pBlock, rowId, pInfo->pCreateTbRes,
&pInfo->stateStore);
TSDB_CHECK_CODE(code, lino, _end);
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static int32_t generatePartitionDelResBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { static int32_t generatePartitionDelResBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX);
SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
@ -1902,8 +1914,12 @@ static int32_t generatePartitionDelResBlock(SStreamScanInfo* pInfo, SSDataBlock*
groupId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, preJ); groupId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, preJ);
if (pInfo->pPartTbnameSup) { if (pInfo->pPartTbnameSup) {
void* parTbname = NULL; void* parTbname = NULL;
int32_t code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname, false); int32_t winCode = TSDB_CODE_SUCCESS;
if (code != TSDB_CODE_SUCCESS) { code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId,
&parTbname, false, &winCode);
TSDB_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS) {
calBlockTbName(pInfo, pPreRes, preJ); calBlockTbName(pInfo, pPreRes, preJ);
memcpy(varDataVal(tbname), pPreRes->info.parTbName, strlen(pPreRes->info.parTbName)); memcpy(varDataVal(tbname), pPreRes->info.parTbName, strlen(pPreRes->info.parTbName));
} else { } else {
@ -1916,10 +1932,17 @@ static int32_t generatePartitionDelResBlock(SStreamScanInfo* pInfo, SSDataBlock*
tbname[0] == 0 ? NULL : tbname); tbname[0] == 0 ? NULL : tbname);
} }
} }
return TSDB_CODE_SUCCESS;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static int32_t generateDeleteResultBlockImpl(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { static int32_t generateDeleteResultBlockImpl(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX);
SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
@ -1948,8 +1971,11 @@ static int32_t generateDeleteResultBlockImpl(SStreamScanInfo* pInfo, SSDataBlock
} }
if (pInfo->tbnameCalSup.pExprInfo) { if (pInfo->tbnameCalSup.pExprInfo) {
void* parTbname = NULL; void* parTbname = NULL;
int32_t code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname, false); int32_t winCode = TSDB_CODE_SUCCESS;
if (code != TSDB_CODE_SUCCESS) { code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname, false, &winCode);
TSDB_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS) {
SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, srcStartTsCol[i], srcStartTsCol[i], ver); SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, srcStartTsCol[i], srcStartTsCol[i], ver);
printDataBlock(pPreRes, "pre res", GET_TASKID(pInfo->pStreamScanOp->pTaskInfo)); printDataBlock(pPreRes, "pre res", GET_TASKID(pInfo->pStreamScanOp->pTaskInfo));
calBlockTbName(pInfo, pPreRes, 0); calBlockTbName(pInfo, pPreRes, 0);
@ -1963,7 +1989,12 @@ static int32_t generateDeleteResultBlockImpl(SStreamScanInfo* pInfo, SSDataBlock
appendDataToSpecialBlock(pDestBlock, srcStartTsCol + i, srcEndTsCol + i, srcUidData + i, &groupId, appendDataToSpecialBlock(pDestBlock, srcStartTsCol + i, srcEndTsCol + i, srcUidData + i, &groupId,
tbname[0] == 0 ? NULL : tbname); tbname[0] == 0 ? NULL : tbname);
} }
return TSDB_CODE_SUCCESS;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
@ -2569,6 +2600,8 @@ static bool isStreamWindow(SStreamScanInfo* pInfo) {
static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
// NOTE: this operator does never check if current status is done or not // NOTE: this operator does never check if current status is done or not
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
const char* id = GET_TASKID(pTaskInfo); const char* id = GET_TASKID(pTaskInfo);
@ -2630,7 +2663,8 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
if (pInfo->pRecoverRes != NULL) { if (pInfo->pRecoverRes != NULL) {
calBlockTbName(pInfo, pInfo->pRecoverRes, 0); calBlockTbName(pInfo, pInfo->pRecoverRes, 0);
if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) { if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) {
TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex, pInfo->primaryKeyIndex); TSKEY maxTs = INT64_MIN;
pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex, pInfo->primaryKeyIndex, &maxTs);
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
} }
if (pInfo->pCreateTbRes->info.rows > 0) { if (pInfo->pCreateTbRes->info.rows > 0) {
@ -2671,7 +2705,11 @@ FETCH_NEXT_BLOCK:
SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current);
SSDataBlock* pBlock = pPacked->pDataBlock; SSDataBlock* pBlock = pPacked->pDataBlock;
if (pBlock->info.parTbName[0]) { if (pBlock->info.parTbName[0]) {
code =
pAPI->stateStore.streamStatePutParName(pStreamInfo->pState, pBlock->info.id.groupId, pBlock->info.parTbName); pAPI->stateStore.streamStatePutParName(pStreamInfo->pState, pBlock->info.id.groupId, pBlock->info.parTbName);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
} }
// TODO move into scan // TODO move into scan
@ -3165,17 +3203,27 @@ _end:
} }
void streamScanReloadState(SOperatorInfo* pOperator) { void streamScanReloadState(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamScanInfo* pInfo = pOperator->info; SStreamScanInfo* pInfo = pOperator->info;
if (!pInfo->pState) { if (!pInfo->pState) {
return; return;
} }
void* pBuff = NULL; void* pBuff = NULL;
int32_t len = 0; int32_t len = 0;
pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), &pBuff, &len); code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME,
strlen(STREAM_SCAN_OP_STATE_NAME), &pBuff, &len);
TSDB_CHECK_CODE(code, lino, _end);
SUpdateInfo* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); SUpdateInfo* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo));
int32_t code = pInfo->stateStore.updateInfoDeserialize(pBuff, len, pUpInfo); if (!pUpInfo) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
int32_t winCode = pInfo->stateStore.updateInfoDeserialize(pBuff, len, pUpInfo);
taosMemoryFree(pBuff); taosMemoryFree(pBuff);
if (code == TSDB_CODE_SUCCESS && pInfo->pUpdateInfo) { if (winCode == TSDB_CODE_SUCCESS && pInfo->pUpdateInfo) {
if (pInfo->pUpdateInfo->minTS < 0) { if (pInfo->pUpdateInfo->minTS < 0) {
pInfo->stateStore.updateInfoDestroy(pInfo->pUpdateInfo); pInfo->stateStore.updateInfoDestroy(pInfo->pUpdateInfo);
pInfo->pUpdateInfo = pUpInfo; pInfo->pUpdateInfo = pUpInfo;
@ -3185,7 +3233,7 @@ void streamScanReloadState(SOperatorInfo* pOperator) {
ASSERT(pInfo->pUpdateInfo->minTS > pUpInfo->minTS); ASSERT(pInfo->pUpdateInfo->minTS > pUpInfo->minTS);
pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pUpInfo->maxDataVersion); pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pUpInfo->maxDataVersion);
SHashObj* curMap = pInfo->pUpdateInfo->pMap; SHashObj* curMap = pInfo->pUpdateInfo->pMap;
void *pIte = taosHashIterate(curMap, NULL); void* pIte = taosHashIterate(curMap, NULL);
while (pIte != NULL) { while (pIte != NULL) {
size_t keySize = 0; size_t keySize = 0;
int64_t* pUid = taosHashGetKey(pIte, &keySize); int64_t* pUid = taosHashGetKey(pIte, &keySize);
@ -3200,6 +3248,11 @@ void streamScanReloadState(SOperatorInfo* pOperator) {
} else { } else {
pInfo->stateStore.updateInfoDestroy(pUpInfo); pInfo->stateStore.updateInfoDestroy(pUpInfo);
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
} }
void addPrimaryKeyCol(SSDataBlock* pBlock, uint8_t type, int32_t bytes) { void addPrimaryKeyCol(SSDataBlock* pBlock, uint8_t type, int32_t bytes) {
@ -3789,7 +3842,8 @@ static void destroyTagScanOperatorInfo(void* param) {
taosArrayDestroy(pInfo->aFilterIdxs); taosArrayDestroy(pInfo->aFilterIdxs);
taosArrayDestroyEx(pInfo->aUidTags, tagScanFreeUidTag); taosArrayDestroyEx(pInfo->aUidTags, tagScanFreeUidTag);
pInfo->pRes = blockDataDestroy(pInfo->pRes); blockDataDestroy(pInfo->pRes);
pInfo->pRes = NULL;
taosArrayDestroy(pInfo->matchInfo.pList); taosArrayDestroy(pInfo->matchInfo.pList);
pInfo->pTableListInfo = tableListDestroy(pInfo->pTableListInfo); pInfo->pTableListInfo = tableListDestroy(pInfo->pTableListInfo);
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
@ -4742,15 +4796,19 @@ void destroyTableMergeScanOperatorInfo(void* param) {
pTableScanInfo->pSortHandle = NULL; pTableScanInfo->pSortHandle = NULL;
taosHashCleanup(pTableScanInfo->mSkipTables); taosHashCleanup(pTableScanInfo->mSkipTables);
pTableScanInfo->mSkipTables = NULL; pTableScanInfo->mSkipTables = NULL;
pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); blockDataDestroy(pTableScanInfo->pSortInputBlock);
pTableScanInfo->pSortInputBlock = NULL;
// end one reader variable // end one reader variable
cleanupQueryTableDataCond(&pTableScanInfo->base.cond); cleanupQueryTableDataCond(&pTableScanInfo->base.cond);
destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI);
pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); blockDataDestroy(pTableScanInfo->pResBlock);
pTableScanInfo->pResBlock = NULL;
// remove it from the task->result list // remove it from the task->result list
pTableScanInfo->pReaderBlock = blockDataDestroy(pTableScanInfo->pReaderBlock); blockDataDestroy(pTableScanInfo->pReaderBlock);
pTableScanInfo->pReaderBlock = NULL;
taosArrayDestroy(pTableScanInfo->pSortInfo); taosArrayDestroy(pTableScanInfo->pSortInfo);
stopSubTablesTableMergeScan(pTableScanInfo); stopSubTablesTableMergeScan(pTableScanInfo);

6
source/libs/executor/src/sortoperator.c
View File

@ -368,7 +368,8 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
void destroySortOperatorInfo(void* param) { void destroySortOperatorInfo(void* param) {
SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param; SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param;
pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); blockDataDestroy(pInfo->binfo.pRes);
pInfo->binfo.pRes = NULL;
tsortDestroySortHandle(pInfo->pSortHandle); tsortDestroySortHandle(pInfo->pSortHandle);
taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->pSortInfo);
@ -611,7 +612,8 @@ int32_t getGroupSortExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, u
void destroyGroupSortOperatorInfo(void* param) { void destroyGroupSortOperatorInfo(void* param) {
SGroupSortOperatorInfo* pInfo = (SGroupSortOperatorInfo*)param; SGroupSortOperatorInfo* pInfo = (SGroupSortOperatorInfo*)param;
pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); blockDataDestroy(pInfo->binfo.pRes);
pInfo->binfo.pRes = NULL;
taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->pSortInfo);
taosArrayDestroy(pInfo->matchInfo.pList); taosArrayDestroy(pInfo->matchInfo.pList);

250
source/libs/executor/src/streamcountwindowoperator.c
View File

@ -70,9 +70,11 @@ void destroyStreamCountAggOperatorInfo(void* param) {
bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; } bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; }
void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin, int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin,
SBuffInfo* pBuffInfo) { SBuffInfo* pBuffInfo) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t winCode = TSDB_CODE_SUCCESS;
int32_t size = pAggSup->resultRowSize; int32_t size = pAggSup->resultRowSize;
pCurWin->winInfo.sessionWin.groupId = groupId; pCurWin->winInfo.sessionWin.groupId = groupId;
pCurWin->winInfo.sessionWin.win.skey = ts; pCurWin->winInfo.sessionWin.win.skey = ts;
@ -80,26 +82,30 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
if (isSlidingCountWindow(pAggSup)) { if (isSlidingCountWindow(pAggSup)) {
if (pBuffInfo->winBuffOp == CREATE_NEW_WINDOW) { if (pBuffInfo->winBuffOp == CREATE_NEW_WINDOW) {
pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size); (void**)&pCurWin->winInfo.pStatePos, &size);
code = TSDB_CODE_FAILED; TSDB_CHECK_CODE(code, lino, _end);
winCode = TSDB_CODE_FAILED;
} else if (pBuffInfo->winBuffOp == MOVE_NEXT_WINDOW) { } else if (pBuffInfo->winBuffOp == MOVE_NEXT_WINDOW) {
ASSERT(pBuffInfo->pCur); ASSERT(pBuffInfo->pCur);
pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur); pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur);
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size); (void**)&pCurWin->winInfo.pStatePos, &size);
if (code == TSDB_CODE_FAILED) { if (winCode == TSDB_CODE_FAILED) {
pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size); (void**)&pCurWin->winInfo.pStatePos, &size);
TSDB_CHECK_CODE(code, lino, _end);
} }
} else { } else {
pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount); pAggSup->windowCount);
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size); (void**)&pCurWin->winInfo.pStatePos, &size);
if (code == TSDB_CODE_FAILED) { if (winCode == TSDB_CODE_FAILED) {
pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size); (void**)&pCurWin->winInfo.pStatePos, &size);
TSDB_CHECK_CODE(code, lino, _end);
} }
} }
if (ts < pCurWin->winInfo.sessionWin.win.ekey) { if (ts < pCurWin->winInfo.sessionWin.win.ekey) {
@ -108,10 +114,11 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
} else { } else {
code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount, pAggSup->windowCount,
(void**)&pCurWin->winInfo.pStatePos, &size); (void**)&pCurWin->winInfo.pStatePos, &size, &winCode);
TSDB_CHECK_CODE(code, lino, _end);
} }
if (code == TSDB_CODE_SUCCESS) { if (winCode == TSDB_CODE_SUCCESS) {
pCurWin->winInfo.isOutput = true; pCurWin->winInfo.isOutput = true;
} }
pCurWin->pWindowCount = pCurWin->pWindowCount =
@ -120,18 +127,33 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
if (*pCurWin->pWindowCount == pAggSup->windowCount) { if (*pCurWin->pWindowCount == pAggSup->windowCount) {
pBuffInfo->rebuildWindow = true; pBuffInfo->rebuildWindow = true;
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static void removeCountResult(SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey) { static void removeCountResult(SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey) {
SSessionKey key = {0}; SSessionKey key = {0};
getSessionHashKey(pKey, &key); getSessionHashKey(pKey, &key);
tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); int32_t code = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));
tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); if (code != TSDB_CODE_SUCCESS) {
qInfo("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
}
code = tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey));
if (code != TSDB_CODE_SUCCESS) {
qInfo("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
}
} }
static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowInfo* pWinInfo, TSKEY* pTs, static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowInfo* pWinInfo, TSKEY* pTs,
int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated, int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool* pRebuild) { SSHashObj* pStDeleted, bool* pRebuild, int32_t* pWinRows) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SSessionKey sWinKey = pWinInfo->winInfo.sessionWin; SSessionKey sWinKey = pWinInfo->winInfo.sessionWin;
int32_t num = 0; int32_t num = 0;
for (int32_t i = start; i < rows; i++) { for (int32_t i = start; i < rows; i++) {
@ -150,7 +172,8 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI
if (pWinInfo->winInfo.sessionWin.win.skey > pTs[start]) { if (pWinInfo->winInfo.sessionWin.win.skey > pTs[start]) {
needDelState = true; needDelState = true;
if (pStDeleted && pWinInfo->winInfo.isOutput) { if (pStDeleted && pWinInfo->winInfo.isOutput) {
saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin); code = saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin);
TSDB_CHECK_CODE(code, lino, _end);
} }
pWinInfo->winInfo.sessionWin.win.skey = pTs[start]; pWinInfo->winInfo.sessionWin.win.skey = pTs[start];
@ -169,7 +192,13 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI
} }
} }
return maxNum; (*pWinRows) = maxNum;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, EStreamType mode, SSessionKey* pDelRange) { void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, EStreamType mode, SSessionKey* pDelRange) {
@ -218,11 +247,12 @@ bool inCountSlidingWindow(SStreamAggSupporter* pAggSup, STimeWindow* pWin, SData
static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pStUpdated, static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pStUpdated,
SSHashObj* pStDeleted) { SSHashObj* pStDeleted) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamCountAggOperatorInfo* pInfo = pOperator->info; SStreamCountAggOperatorInfo* pInfo = pOperator->info;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs; int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
uint64_t groupId = pSDataBlock->info.id.groupId; uint64_t groupId = pSDataBlock->info.id.groupId;
int64_t code = TSDB_CODE_SUCCESS;
SResultRow* pResult = NULL; SResultRow* pResult = NULL;
int32_t rows = pSDataBlock->info.rows; int32_t rows = pSDataBlock->info.rows;
int32_t winRows = 0; int32_t winRows = 0;
@ -236,8 +266,14 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
} }
SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
if (!pStartTsCol) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; TSKEY* startTsCols = (int64_t*)pStartTsCol->pData;
blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2); code = blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2);
TSDB_CHECK_CODE(code, lino, _end);
SStreamStateCur* pCur = NULL; SStreamStateCur* pCur = NULL;
COUNT_TYPE slidingRows = 0; COUNT_TYPE slidingRows = 0;
@ -250,7 +286,9 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
} }
SCountWindowInfo curWin = {0}; SCountWindowInfo curWin = {0};
buffInfo.rebuildWindow = false; buffInfo.rebuildWindow = false;
setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo); code = setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo);
TSDB_CHECK_CODE(code, lino, _end);
if (!inCountSlidingWindow(pAggSup, &curWin.winInfo.sessionWin.win, &pSDataBlock->info)) { if (!inCountSlidingWindow(pAggSup, &curWin.winInfo.sessionWin.win, &pSDataBlock->info)) {
buffInfo.winBuffOp = MOVE_NEXT_WINDOW; buffInfo.winBuffOp = MOVE_NEXT_WINDOW;
continue; continue;
@ -258,8 +296,9 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
setSessionWinOutputInfo(pStUpdated, &curWin.winInfo); setSessionWinOutputInfo(pStUpdated, &curWin.winInfo);
slidingRows = *curWin.pWindowCount; slidingRows = *curWin.pWindowCount;
if (!buffInfo.rebuildWindow) { if (!buffInfo.rebuildWindow) {
winRows = updateCountWindowInfo(pAggSup, &curWin, startTsCols, i, rows, pAggSup->windowCount, pStUpdated, code = updateCountWindowInfo(pAggSup, &curWin, startTsCols, i, rows, pAggSup->windowCount, pStUpdated,
pStDeleted, &buffInfo.rebuildWindow); pStDeleted, &buffInfo.rebuildWindow, &winRows);
TSDB_CHECK_CODE(code, lino, _end);
} }
if (buffInfo.rebuildWindow) { if (buffInfo.rebuildWindow) {
SSessionKey range = {0}; SSessionKey range = {0};
@ -276,28 +315,27 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
} }
code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,
pOperator, 0); pOperator, 0);
if (code != TSDB_CODE_SUCCESS || pResult == NULL) { TSDB_CHECK_CODE(code, lino, _end);
qError("%s do stream count aggregate impl error, code %s", GET_TASKID(pTaskInfo), tstrerror(code));
break; code = saveSessionOutputBuf(pAggSup, &curWin.winInfo);
} TSDB_CHECK_CODE(code, lino, _end);
saveSessionOutputBuf(pAggSup, &curWin.winInfo);
if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_COUNT_OP(pOperator)) { if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_COUNT_OP(pOperator)) {
saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin);
TSDB_CHECK_CODE(code, lino, _end);
} }
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
code = saveResult(curWin.winInfo, pStUpdated); code = saveResult(curWin.winInfo, pStUpdated);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _end);
qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code));
break;
}
} }
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
curWin.winInfo.pStatePos->beUpdated = true; curWin.winInfo.pStatePos->beUpdated = true;
SSessionKey key = {0}; SSessionKey key = {0};
getSessionHashKey(&curWin.winInfo.sessionWin, &key); getSessionHashKey(&curWin.winInfo.sessionWin, &key);
code =
tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
TSDB_CHECK_CODE(code, lino, _end);
} }
if (isSlidingCountWindow(pAggSup)) { if (isSlidingCountWindow(pAggSup)) {
@ -314,6 +352,11 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
} }
i += winRows; i += winRows;
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
destroySBuffInfo(pAggSup, &buffInfo); destroySBuffInfo(pAggSup, &buffInfo);
} }
@ -376,10 +419,13 @@ int32_t doStreamCountEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
return tlen; return tlen;
} }
void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent) { int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamCountAggOperatorInfo* pInfo = pOperator->info; SStreamCountAggOperatorInfo* pInfo = pOperator->info;
if (!pInfo) { if (!pInfo) {
return buf; code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
} }
// 4.checksum // 4.checksum
@ -387,8 +433,8 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
int32_t dataLen = len - sizeof(uint32_t); int32_t dataLen = len - sizeof(uint32_t);
void* pCksum = POINTER_SHIFT(buf, dataLen); void* pCksum = POINTER_SHIFT(buf, dataLen);
if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {
qError("stream count state is invalid"); code = TSDB_CODE_FAILED;
return buf; TSDB_CHECK_CODE(code, lino, _end);
} }
} }
@ -400,10 +446,13 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
SCountWindowInfo curWin = {0}; SCountWindowInfo curWin = {0};
buf = decodeSSessionKey(buf, &key); buf = decodeSSessionKey(buf, &key);
SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL}; SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL};
setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo); code = setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo);
TSDB_CHECK_CODE(code, lino, _end);
buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize); buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize);
tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo,
sizeof(SResultWindowInfo)); sizeof(SResultWindowInfo));
TSDB_CHECK_CODE(code, lino, _end);
} }
// 2.twAggSup // 2.twAggSup
@ -411,21 +460,37 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
// 3.dataVersion // 3.dataVersion
buf = taosDecodeFixedI64(buf, &pInfo->dataVersion); buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);
return buf;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
void doStreamCountSaveCheckpoint(SOperatorInfo* pOperator) { void doStreamCountSaveCheckpoint(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
void* pBuf = NULL;
SStreamCountAggOperatorInfo* pInfo = pOperator->info; SStreamCountAggOperatorInfo* pInfo = pOperator->info;
if (needSaveStreamOperatorInfo(&pInfo->basic)) { if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamCountEncodeOpState(NULL, 0, pOperator, true); int32_t len = doStreamCountEncodeOpState(NULL, 0, pOperator, true);
void* buf = taosMemoryCalloc(1, len); pBuf = taosMemoryCalloc(1, len);
void* pBuf = buf; if (!pBuf) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
len = doStreamCountEncodeOpState(&pBuf, len, pOperator, true); len = doStreamCountEncodeOpState(&pBuf, len, pOperator, true);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME, pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME,
strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), buf, len); strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), pBuf, len);
taosMemoryFree(buf);
saveStreamOperatorStateComplete(&pInfo->basic); saveStreamOperatorStateComplete(&pInfo->basic);
} }
_end:
taosMemoryFreeClear(pBuf);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
} }
void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
@ -467,7 +532,9 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
} }
} }
void doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) { int32_t doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; TSKEY* startDatas = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
@ -482,35 +549,61 @@ void doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SAr
SSessionKey key = {.win.skey = startDatas[i], .win.ekey = endDatas[i], .groupId = gpDatas[i]}; SSessionKey key = {.win.skey = startDatas[i], .win.ekey = endDatas[i], .groupId = gpDatas[i]};
while (1) { while (1) {
SSessionKey curWin = {0}; SSessionKey curWin = {0};
int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin); int32_t winCode = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin);
if (code == TSDB_CODE_FAILED) { if (winCode != TSDB_CODE_SUCCESS) {
break; break;
} }
doDeleteSessionWindow(pAggSup, &curWin); doDeleteSessionWindow(pAggSup, &curWin);
if (result) { if (result) {
saveDeleteInfo(result, curWin); code = saveDeleteInfo(result, curWin);
TSDB_CHECK_CODE(code, lino, _end);
} }
} }
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
void deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, int32_t deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate,
SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd) { SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));
if (!pWins) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
if (isSlidingCountWindow(pAggSup)) { if (isSlidingCountWindow(pAggSup)) {
doDeleteCountWindows(pAggSup, pBlock, pWins); code = doDeleteCountWindows(pAggSup, pBlock, pWins);
TSDB_CHECK_CODE(code, lino, _end);
} else { } else {
doDeleteTimeWindows(pAggSup, pBlock, pWins); code = doDeleteTimeWindows(pAggSup, pBlock, pWins);
TSDB_CHECK_CODE(code, lino, _end);
} }
removeSessionResults(pAggSup, pMapUpdate, pWins); removeSessionResults(pAggSup, pMapUpdate, pWins);
copyDeleteWindowInfo(pWins, pMapDelete); code = copyDeleteWindowInfo(pWins, pMapDelete);
TSDB_CHECK_CODE(code, lino, _end);
if (needAdd) { if (needAdd) {
copyDeleteWindowInfo(pWins, pPkDelete); code = copyDeleteWindowInfo(pWins, pPkDelete);
TSDB_CHECK_CODE(code, lino, _end);
} }
taosArrayDestroy(pWins); taosArrayDestroy(pWins);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
SStreamCountAggOperatorInfo* pInfo = pOperator->info; SStreamCountAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo; SOptrBasicInfo* pBInfo = &pInfo->binfo;
@ -558,21 +651,25 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
bool add = pInfo->destHasPrimaryKey && IS_NORMAL_COUNT_OP(pOperator); bool add = pInfo->destHasPrimaryKey && IS_NORMAL_COUNT_OP(pOperator);
deleteCountWinState(&pInfo->streamAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, pInfo->pPkDeleted, add); code = deleteCountWinState(&pInfo->streamAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, pInfo->pPkDeleted,
add);
TSDB_CHECK_CODE(code, lino, _end);
continue; continue;
} else if (pBlock->info.type == STREAM_CLEAR) { } else if (pBlock->info.type == STREAM_CLEAR) {
doResetCountWindows(&pInfo->streamAggSup, pBlock); doResetCountWindows(&pInfo->streamAggSup, pBlock);
continue; continue;
} else if (pBlock->info.type == STREAM_GET_ALL) { } else if (pBlock->info.type == STREAM_GET_ALL) {
pInfo->recvGetAll = true; pInfo->recvGetAll = true;
getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated); code = getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated);
TSDB_CHECK_CODE(code, lino, _end);
continue; continue;
} else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {
return pBlock; return pBlock;
} else if (pBlock->info.type == STREAM_CHECKPOINT) { } else if (pBlock->info.type == STREAM_CHECKPOINT) {
pAggSup->stateStore.streamStateCommit(pAggSup->pState); pAggSup->stateStore.streamStateCommit(pAggSup->pState);
doStreamCountSaveCheckpoint(pOperator); doStreamCountSaveCheckpoint(pOperator);
copyDataBlock(pInfo->pCheckpointRes, pBlock); code = copyDataBlock(pInfo->pCheckpointRes, pBlock);
TSDB_CHECK_CODE(code, lino, _end);
continue; continue;
} else { } else {
ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type");
@ -580,7 +677,8 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
if (pInfo->scalarSupp.pExprInfo != NULL) { if (pInfo->scalarSupp.pExprInfo != NULL) {
SExprSupp* pExprSup = &pInfo->scalarSupp; SExprSupp* pExprSup = &pInfo->scalarSupp;
projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
TSDB_CHECK_CODE(code, lino, _end);
} }
// the pDataBlock are always the same one, no need to call this again // the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
@ -591,15 +689,21 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
// restore the value // restore the value
pOperator->status = OP_RES_TO_RETURN; pOperator->status = OP_RES_TO_RETURN;
closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated); code = closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated);
copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc); TSDB_CHECK_CODE(code, lino, _end);
code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc);
TSDB_CHECK_CODE(code, lino, _end);
removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated); removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated);
initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);
pInfo->pUpdated = NULL; pInfo->pUpdated = NULL;
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
TSDB_CHECK_CODE(code, lino, _end);
if (pInfo->destHasPrimaryKey && IS_NORMAL_COUNT_OP(pOperator)) { if (pInfo->destHasPrimaryKey && IS_NORMAL_COUNT_OP(pOperator)) {
copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pStDeleted); code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pStDeleted);
TSDB_CHECK_CODE(code, lino, _end);
} }
SSDataBlock* opRes = buildCountResult(pOperator); SSDataBlock* opRes = buildCountResult(pOperator);
@ -607,14 +711,24 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
return opRes; return opRes;
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
setStreamOperatorCompleted(pOperator); setStreamOperatorCompleted(pOperator);
return NULL; return NULL;
} }
void streamCountReleaseState(SOperatorInfo* pOperator) { void streamCountReleaseState(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamEventAggOperatorInfo* pInfo = pOperator->info; SStreamEventAggOperatorInfo* pInfo = pOperator->info;
int32_t resSize = sizeof(TSKEY); int32_t resSize = sizeof(TSKEY);
char* pBuff = taosMemoryCalloc(1, resSize); char* pBuff = taosMemoryCalloc(1, resSize);
if (pBuff) {
code = terrno;
TSDB_CHECK_CODE(code, lino, _end);
}
memcpy(pBuff, &pInfo->twAggSup.maxTs, sizeof(TSKEY)); memcpy(pBuff, &pInfo->twAggSup.maxTs, sizeof(TSKEY));
qDebug("===stream=== count window operator relase state. "); qDebug("===stream=== count window operator relase state. ");
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_STATE_NAME, pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_STATE_NAME,
@ -625,16 +739,24 @@ void streamCountReleaseState(SOperatorInfo* pOperator) {
if (downstream->fpSet.releaseStreamStateFn) { if (downstream->fpSet.releaseStreamStateFn) {
downstream->fpSet.releaseStreamStateFn(downstream); downstream->fpSet.releaseStreamStateFn(downstream);
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
} }
void streamCountReloadState(SOperatorInfo* pOperator) { void streamCountReloadState(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamCountAggOperatorInfo* pInfo = pOperator->info; SStreamCountAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
int32_t size = 0; int32_t size = 0;
void* pBuf = NULL; void* pBuf = NULL;
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME, code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME,
strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size); strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size);
TSDB_CHECK_CODE(code, lino, _end);
TSKEY ts = *(TSKEY*)pBuf; TSKEY ts = *(TSKEY*)pBuf;
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);
taosMemoryFree(pBuf); taosMemoryFree(pBuf);
@ -644,6 +766,11 @@ void streamCountReloadState(SOperatorInfo* pOperator) {
downstream->fpSet.reloadStreamStateFn(downstream); downstream->fpSet.reloadStreamStateFn(downstream);
} }
reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
} }
SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
@ -725,7 +852,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME, pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME,
strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), &buff, &len); strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), &buff, &len);
if (res == TSDB_CODE_SUCCESS) { if (res == TSDB_CODE_SUCCESS) {
doStreamCountDecodeOpState(buff, len, pOperator, true); code = doStreamCountDecodeOpState(buff, len, pOperator, true);
TSDB_CHECK_CODE(code, lino, _error);
taosMemoryFree(buff); taosMemoryFree(buff);
} }
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamCountAgg, NULL, destroyStreamCountAggOperatorInfo, pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamCountAgg, NULL, destroyStreamCountAggOperatorInfo,

252
source/libs/executor/src/streameventwindowoperator.c
View File

@ -108,14 +108,15 @@ int32_t getEndCondIndex(bool* pEnd, int32_t start, int32_t rows) {
static bool isWindowIncomplete(SEventWindowInfo* pWinInfo) { static bool isWindowIncomplete(SEventWindowInfo* pWinInfo) {
return !(pWinInfo->pWinFlag->startFlag && pWinInfo->pWinFlag->endFlag); return !(pWinInfo->pWinFlag->startFlag && pWinInfo->pWinFlag->endFlag);
} }
int32_t reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { void reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) {
pAPI->streamStateReleaseBuf(pState, pPos, true); pAPI->streamStateReleaseBuf(pState, pPos, true);
return TSDB_CODE_SUCCESS;
} }
void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupId, bool* pStart, bool* pEnd, int32_t setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupId, bool* pStart, bool* pEnd,
int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey) { int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t winCode = TSDB_CODE_SUCCESS;
int32_t size = pAggSup->resultRowSize; int32_t size = pAggSup->resultRowSize;
TSKEY ts = pTs[index]; TSKEY ts = pTs[index];
bool start = pStart[index]; bool start = pStart[index];
@ -128,8 +129,8 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI
SSessionKey leftWinKey = {.groupId = groupId}; SSessionKey leftWinKey = {.groupId = groupId};
void* pVal = NULL; void* pVal = NULL;
int32_t len = 0; int32_t len = 0;
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len); winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len);
if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win)) { if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win)) {
bool inWin = isInTimeWindow(&leftWinKey.win, ts, 0); bool inWin = isInTimeWindow(&leftWinKey.win, ts, 0);
setEventWindowInfo(pAggSup, &leftWinKey, pVal, pCurWin); setEventWindowInfo(pAggSup, &leftWinKey, pVal, pCurWin);
if (inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag)) { if (inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag)) {
@ -140,9 +141,9 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI
pAggSup->stateStore.streamStateFreeCur(pCur); pAggSup->stateStore.streamStateFreeCur(pCur);
pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin);
SSessionKey rightWinKey = {.groupId = groupId}; SSessionKey rightWinKey = {.groupId = groupId};
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len); winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len);
bool inWin = isInTimeWindow(&rightWinKey.win, ts, 0); bool inWin = isInTimeWindow(&rightWinKey.win, ts, 0);
if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) { if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) {
int32_t endi = getEndCondIndex(pEnd, index, rows); int32_t endi = getEndCondIndex(pEnd, index, rows);
if (endi < 0 || pTs[endi] >= rightWinKey.win.skey) { if (endi < 0 || pTs[endi] >= rightWinKey.win.skey) {
setEventWindowInfo(pAggSup, &rightWinKey, pVal, pCurWin); setEventWindowInfo(pAggSup, &rightWinKey, pVal, pCurWin);
@ -152,7 +153,9 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI
} }
SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId}; SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId};
pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len); code = pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len);
TSDB_CHECK_CODE(code, lino, _error);
setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin); setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin);
pCurWin->pWinFlag->startFlag = start; pCurWin->pWinFlag->startFlag = start;
pCurWin->pWinFlag->endFlag = end; pCurWin->pWinFlag->endFlag = end;
@ -162,8 +165,8 @@ _end:
reuseOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->stateStore); reuseOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->stateStore);
pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur); pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur);
pNextWinKey->groupId = groupId; pNextWinKey->groupId = groupId;
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, pNextWinKey, NULL, 0); winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, pNextWinKey, NULL, 0);
if (code != TSDB_CODE_SUCCESS) { if (winCode != TSDB_CODE_SUCCESS) {
SET_SESSION_WIN_KEY_INVALID(pNextWinKey); SET_SESSION_WIN_KEY_INVALID(pNextWinKey);
} }
if (pCurWin->winInfo.pStatePos->needFree) { if (pCurWin->winInfo.pStatePos->needFree) {
@ -172,14 +175,24 @@ _end:
pAggSup->stateStore.streamStateFreeCur(pCur); pAggSup->stateStore.streamStateFreeCur(pCur);
qDebug("===stream===set event next win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey, qDebug("===stream===set event next win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,
pCurWin->winInfo.sessionWin.win.ekey); pCurWin->winInfo.sessionWin.win.ekey);
_error:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey, int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey,
TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start,
SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild,
int32_t* pWinRow) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
*pRebuild = false; *pRebuild = false;
if (!pWinInfo->pWinFlag->startFlag && !(starts[start])) { if (!pWinInfo->pWinFlag->startFlag && !(starts[start])) {
return 1; (*pWinRow) = 1;
goto _end;
} }
TSKEY maxTs = INT64_MAX; TSKEY maxTs = INT64_MAX;
@ -193,12 +206,14 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW
for (int32_t i = start; i < rows; ++i) { for (int32_t i = start; i < rows; ++i) {
if (pTsData[i] >= maxTs) { if (pTsData[i] >= maxTs) {
return i - start; (*pWinRow) = i - start;
goto _end;
} }
if (pWin->skey > pTsData[i]) { if (pWin->skey > pTsData[i]) {
if (pStDeleted && pWinInfo->winInfo.isOutput) { if (pStDeleted && pWinInfo->winInfo.isOutput) {
saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin); code = saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin);
TSDB_CHECK_CODE(code, lino, _end);
} }
removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->winInfo.sessionWin); removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->winInfo.sessionWin);
pWin->skey = pTsData[i]; pWin->skey = pTsData[i];
@ -215,7 +230,8 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW
} else { } else {
*pRebuild = true; *pRebuild = true;
pWinInfo->pWinFlag->endFlag |= ends[i]; pWinInfo->pWinFlag->endFlag |= ends[i];
return i + 1 - start; (*pWinRow) = i + 1 - start;
goto _end;
} }
memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey)); memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey));
@ -224,18 +240,26 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW
if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey) { if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey) {
*pRebuild = true; *pRebuild = true;
} }
return i + 1 - start; (*pWinRow) = i + 1 - start;
goto _end;
} }
} }
return rows - start; (*pWinRow) = rows - start;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated, static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool addGap) { SSHashObj* pStDeleted, bool addGap) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
int32_t winNum = 0;
SStreamEventAggOperatorInfo* pInfo = pOperator->info; SStreamEventAggOperatorInfo* pInfo = pOperator->info;
SResultRow* pCurResult = NULL; SResultRow* pCurResult = NULL;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs; int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
@ -252,18 +276,23 @@ static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pC
break; break;
} }
setEventWindowFlag(pAggSup, &nextWinInfo); setEventWindowFlag(pAggSup, &nextWinInfo);
compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, code = compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo,
pStDeleted, false); pStUpdated, pStDeleted, false);
TSDB_CHECK_CODE(code, lino, _end);
pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag; pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag;
winNum++;
} }
return winNum;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
bool doDeleteEventWindow(SStreamAggSupporter* pAggSup, SSHashObj* pSeUpdated, SSessionKey* pKey) { void doDeleteEventWindow(SStreamAggSupporter* pAggSup, SSHashObj* pSeUpdated, SSessionKey* pKey) {
pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey); pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey);
removeSessionResult(pAggSup, pSeUpdated, pAggSup->pResultRows, pKey); removeSessionResult(pAggSup, pSeUpdated, pAggSup->pResultRows, pKey);
return true;
} }
static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated, static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated,
@ -273,7 +302,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
SStreamEventAggOperatorInfo* pInfo = pOperator->info; SStreamEventAggOperatorInfo* pInfo = pOperator->info;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs; int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
uint64_t groupId = pSDataBlock->info.id.groupId; uint64_t groupId = pSDataBlock->info.id.groupId;
int64_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
TSKEY* tsCols = NULL; TSKEY* tsCols = NULL;
SResultRow* pResult = NULL; SResultRow* pResult = NULL;
int32_t winRows = 0; int32_t winRows = 0;
@ -289,6 +319,10 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
if (pSDataBlock->pDataBlock != NULL) { if (pSDataBlock->pDataBlock != NULL) {
SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
if (!pColDataInfo) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
tsCols = (int64_t*)pColDataInfo->pData; tsCols = (int64_t*)pColDataInfo->pData;
} else { } else {
return; return;
@ -302,7 +336,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
goto _end; goto _end;
} }
int32_t statusStart = 0; int32_t statusStart = 0;
filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart); code = filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart);
TSDB_CHECK_CODE(code, lino, _end);
SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock),
.pDataBlock = pSDataBlock->pDataBlock}; .pDataBlock = pSDataBlock->pDataBlock};
@ -313,10 +348,13 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
} }
int32_t statusEnd = 0; int32_t statusEnd = 0;
filterExecute(pInfo->pEndCondInfo, pSDataBlock, &pColEnd, NULL, paramEnd.numOfCols, &statusEnd); code = filterExecute(pInfo->pEndCondInfo, pSDataBlock, &pColEnd, NULL, paramEnd.numOfCols, &statusEnd);
TSDB_CHECK_CODE(code, lino, _end);
int32_t rows = pSDataBlock->info.rows; int32_t rows = pSDataBlock->info.rows;
blockDataEnsureCapacity(pAggSup->pScanBlock, rows); code = blockDataEnsureCapacity(pAggSup->pScanBlock, rows);
TSDB_CHECK_CODE(code, lino, _end);
for (int32_t i = 0; i < rows; i += winRows) { for (int32_t i = 0; i < rows; i += winRows) {
if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo,
&pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i], NULL, 0)) { &pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i], NULL, 0)) {
@ -327,40 +365,50 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
bool allEqual = true; bool allEqual = true;
SEventWindowInfo curWin = {0}; SEventWindowInfo curWin = {0};
SSessionKey nextWinKey = {0}; SSessionKey nextWinKey = {0};
setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, code = setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin,
&nextWinKey); &nextWinKey);
TSDB_CHECK_CODE(code, lino, _end);
setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);
bool rebuild = false; bool rebuild = false;
winRows = code = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData,
updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild, &winRows);
rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild); TSDB_CHECK_CODE(code, lino, _end);
ASSERT(winRows >= 1); ASSERT(winRows >= 1);
if (rebuild) { if (rebuild) {
uint64_t uid = 0; uint64_t uid = 0;
appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); code = tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
TSDB_CHECK_CODE(code, lino, _end);
doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin); doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin);
if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) && if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) &&
!isWindowIncomplete(&curWin)) { !isWindowIncomplete(&curWin)) {
saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin);
TSDB_CHECK_CODE(code, lino, _end);
} }
releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);
SSessionKey tmpSeInfo = {0}; SSessionKey tmpSeInfo = {0};
getSessionHashKey(&curWin.winInfo.sessionWin, &tmpSeInfo); getSessionHashKey(&curWin.winInfo.sessionWin, &tmpSeInfo);
tSimpleHashPut(pStDeleted, &tmpSeInfo, sizeof(SSessionKey), NULL, 0); code = tSimpleHashPut(pStDeleted, &tmpSeInfo, sizeof(SSessionKey), NULL, 0);
TSDB_CHECK_CODE(code, lino, _end);
continue; continue;
} }
code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,
pOperator, 0); pOperator, 0);
if (code != TSDB_CODE_SUCCESS || pResult == NULL) { TSDB_CHECK_CODE(code, lino, _end);
break;
} code = compactEventWindow(pOperator, &curWin, pInfo->pSeUpdated, pInfo->pSeDeleted, false);
compactEventWindow(pOperator, &curWin, pInfo->pSeUpdated, pInfo->pSeDeleted, false); TSDB_CHECK_CODE(code, lino, _end);
saveSessionOutputBuf(pAggSup, &curWin.winInfo);
code = saveSessionOutputBuf(pAggSup, &curWin.winInfo);
TSDB_CHECK_CODE(code, lino, _end);
if (pInfo->isHistoryOp) { if (pInfo->isHistoryOp) {
saveResult(curWin.winInfo, pInfo->pAllUpdated); code = saveResult(curWin.winInfo, pInfo->pAllUpdated);
TSDB_CHECK_CODE(code, lino, _end);
} }
if (isWindowIncomplete(&curWin)) { if (isWindowIncomplete(&curWin)) {
@ -369,18 +417,22 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
} }
if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator)) { if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator)) {
saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin);
TSDB_CHECK_CODE(code, lino, _end);
} }
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
saveResult(curWin.winInfo, pSeUpdated); code = saveResult(curWin.winInfo, pSeUpdated);
TSDB_CHECK_CODE(code, lino, _end);
} }
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
curWin.winInfo.pStatePos->beUpdated = true; curWin.winInfo.pStatePos->beUpdated = true;
SSessionKey key = {0}; SSessionKey key = {0};
getSessionHashKey(&curWin.winInfo.sessionWin, &key); getSessionHashKey(&curWin.winInfo.sessionWin, &key);
code =
tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
TSDB_CHECK_CODE(code, lino, _end);
} }
} }
@ -389,6 +441,9 @@ _end:
taosMemoryFree(pColStart); taosMemoryFree(pColStart);
colDataDestroy(pColEnd); colDataDestroy(pColEnd);
taosMemoryFree(pColEnd); taosMemoryFree(pColEnd);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
} }
int32_t doStreamEventEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator) { int32_t doStreamEventEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator) {
@ -429,10 +484,13 @@ int32_t doStreamEventEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
return tlen; return tlen;
} }
void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator) { int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamEventAggOperatorInfo* pInfo = pOperator->info; SStreamEventAggOperatorInfo* pInfo = pOperator->info;
if (!pInfo) { if (!pInfo) {
return buf; code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
} }
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
@ -440,9 +498,9 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
int32_t dataLen = len - sizeof(uint32_t); int32_t dataLen = len - sizeof(uint32_t);
void* pCksum = POINTER_SHIFT(buf, dataLen); void* pCksum = POINTER_SHIFT(buf, dataLen);
if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {
ASSERT(0); // debug
qError("stream event state is invalid"); qError("stream event state is invalid");
return buf; code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
} }
// 1.streamAggSup.pResultRows // 1.streamAggSup.pResultRows
@ -452,10 +510,16 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
SSessionKey key = {0}; SSessionKey key = {0};
SResultWindowInfo winfo = {0}; SResultWindowInfo winfo = {0};
buf = decodeSSessionKey(buf, &key); buf = decodeSSessionKey(buf, &key);
pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &winfo.sessionWin, pAggSup->gap, int32_t winCode = TSDB_CODE_SUCCESS;
(void**)&winfo.pStatePos, &pAggSup->resultRowSize); code = pAggSup->stateStore.streamStateSessionAddIfNotExist(
pAggSup->pState, &winfo.sessionWin, pAggSup->gap, (void**)&winfo.pStatePos, &pAggSup->resultRowSize, &winCode);
TSDB_CHECK_CODE(code, lino, _end);
ASSERT(winCode == TSDB_CODE_SUCCESS);
buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);
code =
tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo));
TSDB_CHECK_CODE(code, lino, _end);
} }
// 2.twAggSup // 2.twAggSup
@ -463,7 +527,12 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
// 3.dataVersion // 3.dataVersion
buf = taosDecodeFixedI64(buf, &pInfo->dataVersion); buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);
return buf;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
void doStreamEventSaveCheckpoint(SOperatorInfo* pOperator) { void doStreamEventSaveCheckpoint(SOperatorInfo* pOperator) {
@ -504,6 +573,8 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) {
return NULL; return NULL;
} }
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
SStreamEventAggOperatorInfo* pInfo = pOperator->info; SStreamEventAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo; SOptrBasicInfo* pBInfo = &pInfo->binfo;
@ -549,11 +620,14 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) {
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) { pBlock->info.type == STREAM_CLEAR) {
bool add = pInfo->destHasPrimaryKey && IS_NORMAL_EVENT_OP(pOperator); bool add = pInfo->destHasPrimaryKey && IS_NORMAL_EVENT_OP(pOperator);
deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted, pInfo->pPkDeleted, add); code = deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted,
pInfo->pPkDeleted, add);
TSDB_CHECK_CODE(code, lino, _end);
continue; continue;
} else if (pBlock->info.type == STREAM_GET_ALL) { } else if (pBlock->info.type == STREAM_GET_ALL) {
pInfo->recvGetAll = true; pInfo->recvGetAll = true;
getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated); code = getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated);
TSDB_CHECK_CODE(code, lino, _end);
continue; continue;
} else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {
return pBlock; return pBlock;
@ -561,7 +635,8 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) {
pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState); pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);
doStreamEventSaveCheckpoint(pOperator); doStreamEventSaveCheckpoint(pOperator);
pInfo->reCkBlock = true; pInfo->reCkBlock = true;
copyDataBlock(pInfo->pCheckpointRes, pBlock); code = copyDataBlock(pInfo->pCheckpointRes, pBlock);
TSDB_CHECK_CODE(code, lino, _end);
continue; continue;
} else { } else {
ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type");
@ -569,7 +644,8 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) {
if (pInfo->scalarSupp.pExprInfo != NULL) { if (pInfo->scalarSupp.pExprInfo != NULL) {
SExprSupp* pExprSup = &pInfo->scalarSupp; SExprSupp* pExprSup = &pInfo->scalarSupp;
projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
TSDB_CHECK_CODE(code, lino, _end);
} }
// the pDataBlock are always the same one, no need to call this again // the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
@ -579,28 +655,48 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) {
// restore the value // restore the value
pOperator->status = OP_RES_TO_RETURN; pOperator->status = OP_RES_TO_RETURN;
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated); code = closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated);
copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc); TSDB_CHECK_CODE(code, lino, _end);
code = copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc);
TSDB_CHECK_CODE(code, lino, _end);
removeSessionDeleteResults(pInfo->pSeDeleted, pInfo->pUpdated); removeSessionDeleteResults(pInfo->pSeDeleted, pInfo->pUpdated);
if (pInfo->isHistoryOp) { if (pInfo->isHistoryOp) {
SArray* pHisWins = taosArrayInit(16, sizeof(SEventWindowInfo)); SArray* pHisWins = taosArrayInit(16, sizeof(SEventWindowInfo));
copyUpdateResult(&pInfo->pAllUpdated, pHisWins, sessionKeyCompareAsc); if (!pHisWins) {
getMaxTsWins(pHisWins, pInfo->historyWins); code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
code = copyUpdateResult(&pInfo->pAllUpdated, pHisWins, sessionKeyCompareAsc);
TSDB_CHECK_CODE(code, lino, _end);
code = getMaxTsWins(pHisWins, pInfo->historyWins);
TSDB_CHECK_CODE(code, lino, _end);
taosArrayDestroy(pHisWins); taosArrayDestroy(pHisWins);
} }
if (pInfo->destHasPrimaryKey && IS_NORMAL_EVENT_OP(pOperator)) { if (pInfo->destHasPrimaryKey && IS_NORMAL_EVENT_OP(pOperator)) {
copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pSeDeleted); code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pSeDeleted);
TSDB_CHECK_CODE(code, lino, _end);
} }
initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);
pInfo->pUpdated = NULL; pInfo->pUpdated = NULL;
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
TSDB_CHECK_CODE(code, lino, _end);
SSDataBlock* resBlock = buildEventResult(pOperator); SSDataBlock* resBlock = buildEventResult(pOperator);
if (resBlock != NULL) { if (resBlock != NULL) {
return resBlock; return resBlock;
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
setStreamOperatorCompleted(pOperator); setStreamOperatorCompleted(pOperator);
return NULL; return NULL;
} }
@ -626,6 +722,8 @@ void streamEventReleaseState(SOperatorInfo* pOperator) {
} }
void streamEventReloadState(SOperatorInfo* pOperator) { void streamEventReloadState(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamEventAggOperatorInfo* pInfo = pOperator->info; SStreamEventAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
resetWinRange(&pAggSup->winRange); resetWinRange(&pAggSup->winRange);
@ -633,8 +731,10 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
SSessionKey seKey = {.win.skey = INT64_MIN, .win.ekey = INT64_MIN, .groupId = 0}; SSessionKey seKey = {.win.skey = INT64_MIN, .win.ekey = INT64_MIN, .groupId = 0};
int32_t size = 0; int32_t size = 0;
void* pBuf = NULL; void* pBuf = NULL;
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_EVENT_OP_STATE_NAME, code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_EVENT_OP_STATE_NAME,
strlen(STREAM_EVENT_OP_STATE_NAME), &pBuf, &size); strlen(STREAM_EVENT_OP_STATE_NAME), &pBuf, &size);
TSDB_CHECK_CODE(code, lino, _end);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
qDebug("===stream=== event window operator reload state. get result count:%d", num); qDebug("===stream=== event window operator reload state. get result count:%d", num);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
@ -656,22 +756,28 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
SEventWindowInfo curInfo = {0}; SEventWindowInfo curInfo = {0};
qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey, qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey,
pSeKeyBuf[i].groupId, i); pSeKeyBuf[i].groupId, i);
getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo); code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo);
TSDB_CHECK_CODE(code, lino, _end);
// event window has been deleted // event window has been deleted
if (!IS_VALID_SESSION_WIN(curInfo.winInfo)) { if (!IS_VALID_SESSION_WIN(curInfo.winInfo)) {
continue; continue;
} }
setEventWindowFlag(pAggSup, &curInfo); setEventWindowFlag(pAggSup, &curInfo);
if (!curInfo.pWinFlag->startFlag || curInfo.pWinFlag->endFlag) { if (!curInfo.pWinFlag->startFlag || curInfo.pWinFlag->endFlag) {
saveSessionOutputBuf(pAggSup, &curInfo.winInfo); code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo);
TSDB_CHECK_CODE(code, lino, _end);
continue; continue;
} }
compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false); code = compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false);
qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey, qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,
curInfo.winInfo.sessionWin.groupId); curInfo.winInfo.sessionWin.groupId);
TSDB_CHECK_CODE(code, lino, _end);
if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { if (IS_VALID_SESSION_WIN(curInfo.winInfo)) {
saveSessionOutputBuf(pAggSup, &curInfo.winInfo); code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo);
TSDB_CHECK_CODE(code, lino, _end);
} }
if (!curInfo.pWinFlag->endFlag) { if (!curInfo.pWinFlag->endFlag) {
@ -679,14 +785,18 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
} }
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
saveResult(curInfo.winInfo, pInfo->pSeUpdated); code = saveResult(curInfo.winInfo, pInfo->pSeUpdated);
TSDB_CHECK_CODE(code, lino, _end);
} else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
if (!isCloseWindow(&curInfo.winInfo.sessionWin.win, &pInfo->twAggSup)) { if (!isCloseWindow(&curInfo.winInfo.sessionWin.win, &pInfo->twAggSup)) {
saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin); code = saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin);
TSDB_CHECK_CODE(code, lino, _end);
} }
SSessionKey key = {0}; SSessionKey key = {0};
getSessionHashKey(&curInfo.winInfo.sessionWin, &key); getSessionHashKey(&curInfo.winInfo.sessionWin, &key);
code =
tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, sizeof(SResultWindowInfo)); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, sizeof(SResultWindowInfo));
TSDB_CHECK_CODE(code, lino, _end);
} }
} }
taosMemoryFree(pBuf); taosMemoryFree(pBuf);
@ -696,6 +806,11 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
downstream->fpSet.reloadStreamStateFn(downstream); downstream->fpSet.reloadStreamStateFn(downstream);
} }
reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
} }
SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
@ -787,8 +902,9 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME, pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME,
strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), &buff, &len); strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), &buff, &len);
if (res == TSDB_CODE_SUCCESS) { if (res == TSDB_CODE_SUCCESS) {
doStreamEventDecodeOpState(buff, len, pOperator); code = doStreamEventDecodeOpState(buff, len, pOperator);
taosMemoryFree(buff); taosMemoryFree(buff);
TSDB_CHECK_CODE(code, lino, _error);
} }
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo, pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo,

464
source/libs/executor/src/streamfilloperator.c
View File

@ -26,12 +26,11 @@
#include "ttime.h" #include "ttime.h"
#include "function.h" #include "function.h"
#include "operator.h"
#include "querynodes.h" #include "querynodes.h"
#include "querytask.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "tfill.h" #include "tfill.h"
#include "operator.h"
#include "querytask.h"
#define FILL_POS_INVALID 0 #define FILL_POS_INVALID 0
#define FILL_POS_START 1 #define FILL_POS_START 1
@ -56,8 +55,8 @@ TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) {
return win.skey; return win.skey;
} }
void setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) { int32_t setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) {
colDataSetVal(pCol, rowId, pCell->pData, pCell->isNull); return colDataSetVal(pCol, rowId, pCell->pData, pCell->isNull);
} }
SResultCellData* getResultCell(SResultRowData* pRaw, int32_t index) { SResultCellData* getResultCell(SResultRowData* pRaw, int32_t index) {
@ -83,7 +82,7 @@ void* destroyFillColumnInfo(SFillColInfo* pFillCol, int32_t start, int32_t end)
return NULL; return NULL;
} }
void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { void destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) {
pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols);
tSimpleHashCleanup(pFillSup->pResMap); tSimpleHashCleanup(pFillSup->pResMap);
pFillSup->pResMap = NULL; pFillSup->pResMap = NULL;
@ -96,39 +95,42 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) {
taosMemoryFree(pFillSup->nextNext.pRowVal); taosMemoryFree(pFillSup->nextNext.pRowVal);
taosMemoryFree(pFillSup); taosMemoryFree(pFillSup);
return NULL;
} }
void destroySPoint(void* ptr) { void destroySPoint(void* ptr) {
SPoint* point = (SPoint*) ptr; SPoint* point = (SPoint*)ptr;
taosMemoryFreeClear(point->val); taosMemoryFreeClear(point->val);
} }
void* destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) { void destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) {
taosArrayDestroyEx(pFillLinear->pEndPoints, destroySPoint); taosArrayDestroyEx(pFillLinear->pEndPoints, destroySPoint);
taosArrayDestroyEx(pFillLinear->pNextEndPoints, destroySPoint); taosArrayDestroyEx(pFillLinear->pNextEndPoints, destroySPoint);
taosMemoryFree(pFillLinear); taosMemoryFree(pFillLinear);
return NULL;
} }
void* destroyStreamFillInfo(SStreamFillInfo* pFillInfo) {
void destroyStreamFillInfo(SStreamFillInfo* pFillInfo) {
if (pFillInfo->type == TSDB_FILL_SET_VALUE || pFillInfo->type == TSDB_FILL_SET_VALUE_F || if (pFillInfo->type == TSDB_FILL_SET_VALUE || pFillInfo->type == TSDB_FILL_SET_VALUE_F ||
pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) { pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) {
taosMemoryFreeClear(pFillInfo->pResRow->pRowVal); taosMemoryFreeClear(pFillInfo->pResRow->pRowVal);
taosMemoryFreeClear(pFillInfo->pResRow); taosMemoryFreeClear(pFillInfo->pResRow);
} }
pFillInfo->pLinearInfo = destroyStreamFillLinearInfo(pFillInfo->pLinearInfo); destroyStreamFillLinearInfo(pFillInfo->pLinearInfo);
pFillInfo->pLinearInfo = NULL;
taosArrayDestroy(pFillInfo->delRanges); taosArrayDestroy(pFillInfo->delRanges);
taosMemoryFree(pFillInfo); taosMemoryFree(pFillInfo);
return NULL;
} }
static void destroyStreamFillOperatorInfo(void* param) { static void destroyStreamFillOperatorInfo(void* param) {
SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param; SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param;
pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo); destroyStreamFillInfo(pInfo->pFillInfo);
pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup); destroyStreamFillSupporter(pInfo->pFillSup);
pInfo->pRes = blockDataDestroy(pInfo->pRes); blockDataDestroy(pInfo->pRes);
pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); pInfo->pRes = NULL;
pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); blockDataDestroy(pInfo->pSrcBlock);
pInfo->pSrcBlock = NULL;
blockDataDestroy(pInfo->pDelRes);
pInfo->pDelRes = NULL;
taosArrayDestroy(pInfo->matchInfo.pList); taosArrayDestroy(pInfo->matchInfo.pList);
pInfo->matchInfo.pList = NULL; pInfo->matchInfo.pList = NULL;
taosMemoryFree(pInfo); taosMemoryFree(pInfo);
@ -151,20 +153,6 @@ void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStora
resetFillWindow(&pFillSup->nextNext); resetFillWindow(&pFillSup->nextNext);
} }
void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) {
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
void* pState = pOperator->pTaskInfo->streamInfo.pState;
resetPrevAndNextWindow(pFillSup, pState, pAPI);
SWinKey key = {.ts = ts, .groupId = groupId};
int32_t curVLen = 0;
int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen);
ASSERT(code == TSDB_CODE_SUCCESS);
pFillSup->cur.key = key.ts;
}
void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) {
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
void* pState = pOperator->pTaskInfo->streamInfo.pState; void* pState = pOperator->pTaskInfo->streamInfo.pState;
@ -196,15 +184,10 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId,
pFillSup->prev.pRowVal = preVal; pFillSup->prev.pRowVal = preVal;
if (hasCurKey) { if (hasCurKey) {
code = pAPI->stateStore.streamStateCurNext(pState, pCur); pAPI->stateStore.streamStateCurNext(pState, pCur);
ASSERT(code == TSDB_CODE_SUCCESS);
} }
code = pAPI->stateStore.streamStateCurNext(pState, pCur); pAPI->stateStore.streamStateCurNext(pState, pCur);
if (code != TSDB_CODE_SUCCESS) {
pAPI->stateStore.streamStateFreeCur(pCur);
pCur = NULL;
}
} else { } else {
pAPI->stateStore.streamStateFreeCur(pCur); pAPI->stateStore.streamStateFreeCur(pCur);
pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key); pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key);
@ -218,8 +201,7 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId,
pFillSup->next.key = nextKey.ts; pFillSup->next.key = nextKey.ts;
pFillSup->next.pRowVal = nextVal; pFillSup->next.pRowVal = nextVal;
if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) {
code = pAPI->stateStore.streamStateCurNext(pState, pCur); pAPI->stateStore.streamStateCurNext(pState, pCur);
if (code == TSDB_CODE_SUCCESS) {
SWinKey nextNextKey = {.groupId = groupId}; SWinKey nextNextKey = {.groupId = groupId};
void* nextNextVal = NULL; void* nextNextVal = NULL;
int32_t nextNextVLen = 0; int32_t nextNextVLen = 0;
@ -230,7 +212,6 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId,
} }
} }
} }
}
pAPI->stateStore.streamStateFreeCur(pCur); pAPI->stateStore.streamStateFreeCur(pCur);
} }
@ -286,8 +267,7 @@ static void calcDeltaData(SSDataBlock* pBlock, int32_t rowId, SResultRowData* pR
} }
} }
static void calcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, static void calcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, int32_t numOfCol) {
int32_t numOfCol) {
for (int32_t i = 0; i < numOfCol; i++) { for (int32_t i = 0; i < numOfCol; i++) {
if (!pFillCol[i].notFillCol) { if (!pFillCol[i].notFillCol) {
int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i);
@ -484,22 +464,40 @@ void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillS
ASSERT(pFillInfo->pos != FILL_POS_INVALID); ASSERT(pFillInfo->pos != FILL_POS_INVALID);
} }
static bool checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t groupId) { static int32_t checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t groupId, bool* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SWinKey key = {.groupId = groupId, .ts = ts}; SWinKey key = {.groupId = groupId, .ts = ts};
if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) { if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) {
return false; (*pRes) = false;
} }
tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); code = tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0);
return true; TSDB_CHECK_CODE(code, lino, _end);
(*pRes) = true;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static bool buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFillSup, TSKEY ts, SSDataBlock* pBlock) { static int32_t buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFillSup, TSKEY ts, SSDataBlock* pBlock,
bool* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pBlock->info.rows >= pBlock->info.capacity) { if (pBlock->info.rows >= pBlock->info.capacity) {
return false; (*pRes) = false;
goto _end;
} }
uint64_t groupId = pBlock->info.id.groupId; uint64_t groupId = pBlock->info.id.groupId;
if (pFillSup->hasDelete && !checkResult(pFillSup, ts, groupId)) { bool ckRes = true;
return true; code = checkResult(pFillSup, ts, groupId, &ckRes);
TSDB_CHECK_CODE(code, lino, _end);
if (pFillSup->hasDelete && !ckRes) {
(*pRes) = true;
goto _end;
} }
for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) {
SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; SFillColInfo* pFillCol = pFillSup->pAllColInfo + i;
@ -513,11 +511,18 @@ static bool buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFill
bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pColData, pBlock->info.rows); bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pColData, pBlock->info.rows);
if (!filled) { if (!filled) {
SResultCellData* pCell = getResultCell(pResRow, slotId); SResultCellData* pCell = getResultCell(pResRow, slotId);
setRowCell(pColData, pBlock->info.rows, pCell); code = setRowCell(pColData, pBlock->info.rows, pCell);
TSDB_CHECK_CODE(code, lino, _end);
} }
} }
pBlock->info.rows++; pBlock->info.rows++;
return true; (*pRes) = true;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static bool hasRemainCalc(SStreamFillInfo* pFillInfo) { static bool hasRemainCalc(SStreamFillInfo* pFillInfo) {
@ -528,22 +533,38 @@ static bool hasRemainCalc(SStreamFillInfo* pFillInfo) {
} }
static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) {
STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current};
if (inWinRange(&pFillSup->winRange, &st)) { if (inWinRange(&pFillSup->winRange, &st)) {
buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); bool res = true;
code = buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock, &res);
TSDB_CHECK_CODE(code, lino, _end);
} }
pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
pFillSup->interval.precision); pFillSup->interval.precision);
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
} }
static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) {
uint64_t groupId = pBlock->info.id.groupId; uint64_t groupId = pBlock->info.id.groupId;
SWinKey key = {.groupId = groupId, .ts = pFillInfo->current}; SWinKey key = {.groupId = groupId, .ts = pFillInfo->current};
STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current};
if ( ( pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId) ) || !inWinRange(&pFillSup->winRange, &st) ) { bool ckRes = true;
code = checkResult(pFillSup, pFillInfo->current, groupId, &ckRes);
TSDB_CHECK_CODE(code, lino, _end);
if ((pFillSup->hasDelete && !ckRes) ||
!inWinRange(&pFillSup->winRange, &st)) {
pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
pFillSup->interval.precision); pFillSup->interval.precision);
pFillInfo->pLinearInfo->winIndex++; pFillInfo->pLinearInfo->winIndex++;
@ -566,7 +587,8 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo*
if (pFillCol->notFillCol) { if (pFillCol->notFillCol) {
bool filled = fillIfWindowPseudoColumn(&tmp, pFillCol, pColData, index); bool filled = fillIfWindowPseudoColumn(&tmp, pFillCol, pColData, index);
if (!filled) { if (!filled) {
setRowCell(pColData, index, pCell); code = setRowCell(pColData, index, pCell);
TSDB_CHECK_CODE(code, lino, _end);
} }
} else { } else {
if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) { if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) {
@ -583,7 +605,8 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo*
cur.key = pFillInfo->current; cur.key = pFillInfo->current;
cur.val = taosMemoryCalloc(1, pCell->bytes); cur.val = taosMemoryCalloc(1, pCell->bytes);
taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type); taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type);
colDataSetVal(pColData, index, (const char*)cur.val, false); code = colDataSetVal(pColData, index, (const char*)cur.val, false);
TSDB_CHECK_CODE(code, lino, _end);
destroySPoint(&cur); destroySPoint(&cur);
} }
} }
@ -591,6 +614,11 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo*
pFillSup->interval.precision); pFillSup->interval.precision);
pBlock->info.rows++; pBlock->info.rows++;
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
} }
static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) { static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) {
@ -603,13 +631,19 @@ static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SRes
} }
static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
bool res = false;
if (pFillInfo->needFill == false) { if (pFillInfo->needFill == false) {
buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res);
TSDB_CHECK_CODE(code, lino, _end);
return; return;
} }
if (pFillInfo->pos == FILL_POS_START) { if (pFillInfo->pos == FILL_POS_START) {
if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res);
TSDB_CHECK_CODE(code, lino, _end);
if (res) {
pFillInfo->pos = FILL_POS_INVALID; pFillInfo->pos = FILL_POS_INVALID;
} }
} }
@ -619,7 +653,9 @@ static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter*
doStreamFillLinear(pFillSup, pFillInfo, pRes); doStreamFillLinear(pFillSup, pFillInfo, pRes);
if (pFillInfo->pos == FILL_POS_MID) { if (pFillInfo->pos == FILL_POS_MID) {
if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res);
TSDB_CHECK_CODE(code, lino, _end);
if (res) {
pFillInfo->pos = FILL_POS_INVALID; pFillInfo->pos = FILL_POS_INVALID;
} }
} }
@ -634,10 +670,17 @@ static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter*
} }
} }
if (pFillInfo->pos == FILL_POS_END) { if (pFillInfo->pos == FILL_POS_END) {
if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res);
TSDB_CHECK_CODE(code, lino, _end);
if (res) {
pFillInfo->pos = FILL_POS_INVALID; pFillInfo->pos = FILL_POS_INVALID;
} }
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
} }
void keepBlockRowInDiscBuf(SOperatorInfo* pOperator, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, TSKEY* tsCol, void keepBlockRowInDiscBuf(SOperatorInfo* pOperator, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, TSKEY* tsCol,
@ -663,6 +706,8 @@ static void doFillResults(SOperatorInfo* pOperator, SStreamFillSupporter* pFillS
} }
static void doStreamFillImpl(SOperatorInfo* pOperator) { static void doStreamFillImpl(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamFillSupporter* pFillSup = pInfo->pFillSup; SStreamFillSupporter* pFillSup = pInfo->pFillSup;
@ -685,20 +730,28 @@ static void doStreamFillImpl(SOperatorInfo* pOperator) {
keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize);
doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes);
if (pInfo->pRes->info.rows == pInfo->pRes->info.capacity) { if (pInfo->pRes->info.rows == pInfo->pRes->info.capacity) {
blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); code = blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol);
TSDB_CHECK_CODE(code, lino, _end);
return; return;
} }
pInfo->srcRowIndex++; pInfo->srcRowIndex++;
} }
doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes);
blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); code = blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol);
TSDB_CHECK_CODE(code, lino, _end);
blockDataCleanup(pInfo->pSrcBlock); blockDataCleanup(pInfo->pSrcBlock);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
} }
static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { static int32_t buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI;
void* pState = pOp->pTaskInfo->streamInfo.pState; void* pState = pOp->pTaskInfo->streamInfo.pState;
SSDataBlock* pBlock = delRes; SSDataBlock* pBlock = delRes;
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
@ -707,45 +760,74 @@ static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_
SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
SColumnInfoData* pTbNameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); SColumnInfoData* pTbNameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
colDataSetVal(pStartCol, pBlock->info.rows, (const char*)&start, false); code = colDataSetVal(pStartCol, pBlock->info.rows, (const char*)&start, false);
colDataSetVal(pEndCol, pBlock->info.rows, (const char*)&end, false); TSDB_CHECK_CODE(code, lino, _end);
code = colDataSetVal(pEndCol, pBlock->info.rows, (const char*)&end, false);
TSDB_CHECK_CODE(code, lino, _end);
colDataSetNULL(pUidCol, pBlock->info.rows); colDataSetNULL(pUidCol, pBlock->info.rows);
colDataSetVal(pGroupCol, pBlock->info.rows, (const char*)&groupId, false); code = colDataSetVal(pGroupCol, pBlock->info.rows, (const char*)&groupId, false);
TSDB_CHECK_CODE(code, lino, _end);
colDataSetNULL(pCalStartCol, pBlock->info.rows); colDataSetNULL(pCalStartCol, pBlock->info.rows);
colDataSetNULL(pCalEndCol, pBlock->info.rows); colDataSetNULL(pCalEndCol, pBlock->info.rows);
SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
void* tbname = NULL; void* tbname = NULL;
pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname, false); int32_t winCode = TSDB_CODE_SUCCESS;
if (tbname == NULL) { code = pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname, false, &winCode);
TSDB_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS) {
colDataSetNULL(pTableCol, pBlock->info.rows); colDataSetNULL(pTableCol, pBlock->info.rows);
} else { } else {
ASSERT(tbname);
char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); code = colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false);
TSDB_CHECK_CODE(code, lino, _end);
pAPI->stateStore.streamStateFreeVal(tbname); pAPI->stateStore.streamStateFreeVal(tbname);
} }
pBlock->info.rows++; pBlock->info.rows++;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId, static int32_t buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId,
SSDataBlock* delRes) { SSDataBlock* delRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillOperatorInfo* pInfo = pOperator->info;
SStreamFillSupporter* pFillSup = pInfo->pFillSup; SStreamFillSupporter* pFillSup = pInfo->pFillSup;
if (hasPrevWindow(pFillSup)) { if (hasPrevWindow(pFillSup)) {
TSKEY start = getNextWindowTs(pFillSup->prev.key, &pFillSup->interval); TSKEY start = getNextWindowTs(pFillSup->prev.key, &pFillSup->interval);
buildDeleteRange(pOperator, start, endTs, groupId, delRes); code = buildDeleteRange(pOperator, start, endTs, groupId, delRes);
TSDB_CHECK_CODE(code, lino, _end);
} else if (hasNextWindow(pFillSup)) { } else if (hasNextWindow(pFillSup)) {
TSKEY end = getPrevWindowTs(pFillSup->next.key, &pFillSup->interval); TSKEY end = getPrevWindowTs(pFillSup->next.key, &pFillSup->interval);
buildDeleteRange(pOperator, startTs, end, groupId, delRes); code = buildDeleteRange(pOperator, startTs, end, groupId, delRes);
TSDB_CHECK_CODE(code, lino, _end);
} else { } else {
buildDeleteRange(pOperator, startTs, endTs, groupId, delRes); code = buildDeleteRange(pOperator, startTs, endTs, groupId, delRes);
TSDB_CHECK_CODE(code, lino, _end);
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { static int32_t doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillOperatorInfo* pInfo = pOperator->info;
getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup); getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup);
@ -753,15 +835,26 @@ static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKE
SWinKey key = {.ts = startTs, .groupId = groupId}; SWinKey key = {.ts = startTs, .groupId = groupId};
pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
if (!pInfo->pFillInfo->needFill) { if (!pInfo->pFillInfo->needFill) {
buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes); code = buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes);
TSDB_CHECK_CODE(code, lino, _end);
} else { } else {
STimeRange tw = { STimeRange tw = {
.skey = startTs, .skey = startTs,
.ekey = endTs, .ekey = endTs,
.groupId = groupId, .groupId = groupId,
}; };
taosArrayPush(pInfo->pFillInfo->delRanges, &tw); void* tmp = taosArrayPush(pInfo->pFillInfo->delRanges, &tw);
if (!tmp) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
} }
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static void getWindowInfoByKey(SStorageAPI* pAPI, void* pState, TSKEY ts, int64_t groupId, SResultRowData* pWinData) { static void getWindowInfoByKey(SStorageAPI* pAPI, void* pState, TSKEY ts, int64_t groupId, SResultRowData* pWinData) {
@ -799,7 +892,8 @@ static void doDeleteFillFinalize(SOperatorInfo* pOperator) {
getWindowFromDiscBuf(pOperator, range->skey, range->groupId, pInfo->pFillSup); getWindowFromDiscBuf(pOperator, range->skey, range->groupId, pInfo->pFillSup);
TSKEY realEnd = range->ekey + 1; TSKEY realEnd = range->ekey + 1;
if (pInfo->pFillInfo->type == TSDB_FILL_NEXT && pInfo->pFillSup->next.key != realEnd) { if (pInfo->pFillInfo->type == TSDB_FILL_NEXT && pInfo->pFillSup->next.key != realEnd) {
getWindowInfoByKey(pAPI, pOperator->pTaskInfo->streamInfo.pState, realEnd, range->groupId, &pInfo->pFillSup->next); getWindowInfoByKey(pAPI, pOperator->pTaskInfo->streamInfo.pState, realEnd, range->groupId,
&pInfo->pFillSup->next);
} }
setDeleteFillValueInfo(range->skey, range->ekey, pInfo->pFillSup, pInfo->pFillInfo); setDeleteFillValueInfo(range->skey, range->ekey, pInfo->pFillSup, pInfo->pFillInfo);
pFillInfo->delIndex++; pFillInfo->delIndex++;
@ -810,9 +904,10 @@ static void doDeleteFillFinalize(SOperatorInfo* pOperator) {
} }
} }
static void doDeleteFillResult(SOperatorInfo* pOperator) { static int32_t doDeleteFillResult(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillOperatorInfo* pInfo = pOperator->info;
SStreamFillInfo* pFillInfo = pInfo->pFillInfo; SStreamFillInfo* pFillInfo = pInfo->pFillInfo;
SSDataBlock* pBlock = pInfo->pSrcDelBlock; SSDataBlock* pBlock = pInfo->pSrcDelBlock;
@ -837,7 +932,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
while (pInfo->srcDelRowIndex < pBlock->info.rows) { while (pInfo->srcDelRowIndex < pBlock->info.rows) {
TSKEY delTs = tsStarts[pInfo->srcDelRowIndex]; TSKEY delTs = tsStarts[pInfo->srcDelRowIndex];
uint64_t delGroupId = groupIds[pInfo->srcDelRowIndex]; uint64_t delGroupId = groupIds[pInfo->srcDelRowIndex];
int32_t code = TSDB_CODE_SUCCESS; int32_t winCode = TSDB_CODE_SUCCESS;
if (groupId != delGroupId) { if (groupId != delGroupId) {
break; break;
} }
@ -847,10 +942,8 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
SWinKey delKey = {.groupId = delGroupId, .ts = delTs}; SWinKey delKey = {.groupId = delGroupId, .ts = delTs};
if (delTs == nextKey.ts) { if (delTs == nextKey.ts) {
code = pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur);
if (code == TSDB_CODE_SUCCESS) { winCode = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, NULL, NULL);
code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, NULL, NULL);
}
// ts will be deleted later // ts will be deleted later
if (delTs != ts) { if (delTs != ts) {
pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey);
@ -858,7 +951,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey);
} }
endTs = TMAX(delTs, nextKey.ts - 1); endTs = TMAX(delTs, nextKey.ts - 1);
if (code != TSDB_CODE_SUCCESS) { if (winCode != TSDB_CODE_SUCCESS) {
break; break;
} }
} }
@ -866,10 +959,17 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
} }
pAPI->stateStore.streamStateFreeCur(pCur); pAPI->stateStore.streamStateFreeCur(pCur);
doDeleteFillResultImpl(pOperator, ts, endTs, groupId); code = doDeleteFillResultImpl(pOperator, ts, endTs, groupId);
TSDB_CHECK_CODE(code, lino, _end);
} }
pFillInfo->current = pFillInfo->end + 1; pFillInfo->current = pFillInfo->end + 1;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) { static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) {
@ -879,25 +979,41 @@ static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) {
pInfo->pFillInfo->delIndex = 0; pInfo->pFillInfo->delIndex = 0;
} }
static void doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock, SSDataBlock* pDstBlock) { static int32_t doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock,
SSDataBlock* pDstBlock) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillOperatorInfo* pInfo = pOperator->info;
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
blockDataCleanup(pDstBlock); blockDataCleanup(pDstBlock);
blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows); code = blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows);
TSDB_CHECK_CODE(code, lino, _end);
setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false);
projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); code = projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL);
TSDB_CHECK_CODE(code, lino, _end);
pDstBlock->info.rows = 0; pDstBlock->info.rows = 0;
pSup = &pInfo->pFillSup->notFillExprSup; pSup = &pInfo->pFillSup->notFillExprSup;
setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false);
projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); code = projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL);
TSDB_CHECK_CODE(code, lino, _end);
pDstBlock->info.id.groupId = pSrcBlock->info.id.groupId; pDstBlock->info.id.groupId = pSrcBlock->info.id.groupId;
blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol); code = blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -959,7 +1075,9 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) {
pInfo->srcDelRowIndex = 0; pInfo->srcDelRowIndex = 0;
blockDataCleanup(pInfo->pDelRes); blockDataCleanup(pInfo->pDelRes);
pInfo->pFillSup->hasDelete = true; pInfo->pFillSup->hasDelete = true;
doDeleteFillResult(pOperator); code = doDeleteFillResult(pOperator);
TSDB_CHECK_CODE(code, lino, _end);
if (pInfo->pDelRes->info.rows > 0) { if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
return pInfo->pDelRes; return pInfo->pDelRes;
@ -969,7 +1087,9 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) {
case STREAM_NORMAL: case STREAM_NORMAL:
case STREAM_INVALID: case STREAM_INVALID:
case STREAM_PULL_DATA: { case STREAM_PULL_DATA: {
doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock); code = doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock);
TSDB_CHECK_CODE(code, lino, _end);
memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
pInfo->srcRowIndex = -1; pInfo->srcRowIndex = -1;
} break; } break;
@ -983,7 +1103,9 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) {
} }
doStreamFillImpl(pOperator); doStreamFillImpl(pOperator);
doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo); code = doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo);
TSDB_CHECK_CODE(code, lino, _end);
memcpy(pInfo->pRes->info.parTbName, pInfo->pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN); memcpy(pInfo->pRes->info.parTbName, pInfo->pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; pOperator->resultInfo.totalRows += pInfo->pRes->info.rows;
if (pInfo->pRes->info.rows > 0) { if (pInfo->pRes->info.rows > 0) {
@ -1003,6 +1125,14 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) {
pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; pOperator->resultInfo.totalRows += pInfo->pRes->info.rows;
printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
return pInfo->pRes; return pInfo->pRes;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
setOperatorCompleted(pOperator);
resetStreamFillInfo(pInfo);
return NULL;
} }
static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { static int32_t initResultBuf(SStreamFillSupporter* pFillSup) {
@ -1026,9 +1156,12 @@ static int32_t initResultBuf(SStreamFillSupporter* pFillSup) {
static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval,
SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) { SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter));
if (!pFillSup) { if (!pFillSup) {
return NULL; code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
} }
pFillSup->numOfFillCols = numOfFillCols; pFillSup->numOfFillCols = numOfFillCols;
int32_t numOfNotFillCols = 0; int32_t numOfNotFillCols = 0;
@ -1040,48 +1173,89 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod
pFillSup->interval = *pInterval; pFillSup->interval = *pInterval;
pFillSup->pAPI = pAPI; pFillSup->pAPI = pAPI;
int32_t code = initResultBuf(pFillSup); code = initResultBuf(pFillSup);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _end);
destroyStreamFillSupporter(pFillSup);
return NULL;
}
SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols);
code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore); code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _end);
destroyStreamFillSupporter(pFillSup);
return NULL;
}
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pFillSup->pResMap = tSimpleHashInit(16, hashFn); pFillSup->pResMap = tSimpleHashInit(16, hashFn);
pFillSup->hasDelete = false; pFillSup->hasDelete = false;
_end:
if (code != TSDB_CODE_SUCCESS) {
destroyStreamFillSupporter(pFillSup);
pFillSup = NULL;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return pFillSup; return pFillSup;
} }
SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SStreamFillInfo)); SStreamFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SStreamFillInfo));
if (!pFillInfo) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
pFillInfo->start = INT64_MIN; pFillInfo->start = INT64_MIN;
pFillInfo->current = INT64_MIN; pFillInfo->current = INT64_MIN;
pFillInfo->end = INT64_MIN; pFillInfo->end = INT64_MIN;
pFillInfo->preRowKey = INT64_MIN; pFillInfo->preRowKey = INT64_MIN;
pFillInfo->needFill = false; pFillInfo->needFill = false;
pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo)); pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo));
if (!pFillInfo) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
pFillInfo->pLinearInfo->hasNext = false; pFillInfo->pLinearInfo->hasNext = false;
pFillInfo->pLinearInfo->nextEnd = INT64_MIN; pFillInfo->pLinearInfo->nextEnd = INT64_MIN;
pFillInfo->pLinearInfo->pEndPoints = NULL; pFillInfo->pLinearInfo->pEndPoints = NULL;
pFillInfo->pLinearInfo->pNextEndPoints = NULL; pFillInfo->pLinearInfo->pNextEndPoints = NULL;
if (pFillSup->type == TSDB_FILL_LINEAR) { if (pFillSup->type == TSDB_FILL_LINEAR) {
pFillInfo->pLinearInfo->pEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); pFillInfo->pLinearInfo->pEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint));
if (!pFillInfo->pLinearInfo->pEndPoints) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
pFillInfo->pLinearInfo->pNextEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); pFillInfo->pLinearInfo->pNextEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint));
if (!pFillInfo->pLinearInfo->pNextEndPoints) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) { for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) {
SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i);
SPoint value = {0}; SPoint value = {0};
value.val = taosMemoryCalloc(1, pColData->info.bytes); value.val = taosMemoryCalloc(1, pColData->info.bytes);
taosArrayPush(pFillInfo->pLinearInfo->pEndPoints, &value); if (!value.val) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
void* tmpRes = taosArrayPush(pFillInfo->pLinearInfo->pEndPoints, &value);
if (!tmpRes) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
value.val = taosMemoryCalloc(1, pColData->info.bytes); value.val = taosMemoryCalloc(1, pColData->info.bytes);
taosArrayPush(pFillInfo->pLinearInfo->pNextEndPoints, &value); if (!value.val) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
tmpRes = taosArrayPush(pFillInfo->pLinearInfo->pNextEndPoints, &value);
if (!tmpRes) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
} }
} }
pFillInfo->pLinearInfo->winIndex = 0; pFillInfo->pLinearInfo->winIndex = 0;
@ -1090,8 +1264,18 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
if (pFillSup->type == TSDB_FILL_SET_VALUE || pFillSup->type == TSDB_FILL_SET_VALUE_F || if (pFillSup->type == TSDB_FILL_SET_VALUE || pFillSup->type == TSDB_FILL_SET_VALUE_F ||
pFillSup->type == TSDB_FILL_NULL || pFillSup->type == TSDB_FILL_NULL_F) { pFillSup->type == TSDB_FILL_NULL || pFillSup->type == TSDB_FILL_NULL_F) {
pFillInfo->pResRow = taosMemoryCalloc(1, sizeof(SResultRowData)); pFillInfo->pResRow = taosMemoryCalloc(1, sizeof(SResultRowData));
if (!pFillInfo->pResRow) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
pFillInfo->pResRow->key = INT64_MIN; pFillInfo->pResRow->key = INT64_MIN;
pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize); pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize);
if (!pFillInfo->pResRow->pRowVal) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i);
SResultCellData* pCell = getResultCell(pFillInfo->pResRow, i); SResultCellData* pCell = getResultCell(pFillInfo->pResRow, i);
@ -1102,17 +1286,32 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
pFillInfo->type = pFillSup->type; pFillInfo->type = pFillSup->type;
pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeRange)); pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeRange));
if (!pFillInfo->delRanges) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
pFillInfo->delIndex = 0; pFillInfo->delIndex = 0;
pFillInfo->curGroupId = 0; pFillInfo->curGroupId = 0;
return pFillInfo; return pFillInfo;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
destroyStreamFillInfo(pFillInfo);
return NULL;
} }
SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode,
SExecTaskInfo* pTaskInfo) { SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFillOperatorInfo)); SStreamFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFillOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
goto _error; code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
} }
SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval; SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval;
@ -1120,14 +1319,18 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols);
pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI); pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI);
if (!pInfo->pFillSup) { if (!pInfo->pFillSup) {
goto _error; code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
} }
initResultSizeInfo(&pOperator->resultInfo, 4096); initResultSizeInfo(&pOperator->resultInfo, 4096);
pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); code = blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity); TSDB_CHECK_CODE(code, lino, _error);
code = blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity);
TSDB_CHECK_CODE(code, lino, _error);
pInfo->pFillInfo = initStreamFillInfo(pInfo->pFillSup, pInfo->pRes); pInfo->pFillInfo = initStreamFillInfo(pInfo->pFillSup, pInfo->pRes);
if (!pInfo->pFillInfo) { if (!pInfo->pFillInfo) {
@ -1166,42 +1369,43 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
} }
pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity); if (!pInfo->pDelRes) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
code = blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity);
TSDB_CHECK_CODE(code, lino, _error);
pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId; pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId;
pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId; pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId;
int32_t numOfOutputCols = 0; int32_t numOfOutputCols = 0;
int32_t code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, &numOfOutputCols,
&numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
pInfo->srcRowIndex = -1; pInfo->srcRowIndex = -1;
setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo,
pTaskInfo); pTaskInfo);
pOperator->fpSet = pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo,
createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState);
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
return pOperator; return pOperator;
_error: _error:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
destroyStreamFillOperatorInfo(pInfo); destroyStreamFillOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
pTaskInfo->code = code; pTaskInfo->code = code;

1634
source/libs/executor/src/streamtimewindowoperator.c
View File

File diff suppressed because it is too large Load Diff

4
source/libs/executor/src/tfill.c
View File

@ -602,7 +602,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma
return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes; return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes;
} }
int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, void taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2,
int32_t inputType) { int32_t inputType) {
double v1 = -1, v2 = -1; double v1 = -1, v2 = -1;
GET_TYPED_DATA(v1, double, inputType, point1->val); GET_TYPED_DATA(v1, double, inputType, point1->val);
@ -615,8 +615,6 @@ int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint*
r = (v1 < 1 || v2 < 1) ? 0 : 1; r = (v1 < 1 || v2 < 1) ? 0 : 1;
} }
SET_TYPED_DATA(point->val, outputType, r); SET_TYPED_DATA(point->val, outputType, r);
return TSDB_CODE_SUCCESS;
} }
int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity) { int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity) {

3
source/libs/executor/src/timesliceoperator.c
View File

@ -1087,7 +1087,8 @@ _error:
void destroyTimeSliceOperatorInfo(void* param) { void destroyTimeSliceOperatorInfo(void* param) {
STimeSliceOperatorInfo* pInfo = (STimeSliceOperatorInfo*)param; STimeSliceOperatorInfo* pInfo = (STimeSliceOperatorInfo*)param;
pInfo->pRes = blockDataDestroy(pInfo->pRes); blockDataDestroy(pInfo->pRes);
pInfo->pRes = NULL;
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pPrevRow); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pPrevRow); ++i) {
SGroupKeys* pKey = taosArrayGet(pInfo->pPrevRow, i); SGroupKeys* pKey = taosArrayGet(pInfo->pPrevRow, i);

3
source/libs/executor/src/tsort.c
View File

@ -555,7 +555,8 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT
(*numOfCompleted) += 1; (*numOfCompleted) += 1;
pSource->src.rowIndex = -1; pSource->src.rowIndex = -1;
pSource->pageIndex = -1; pSource->pageIndex = -1;
pSource->src.pBlock = blockDataDestroy(pSource->src.pBlock); blockDataDestroy(pSource->src.pBlock);
pSource->src.pBlock = NULL;
} else { } else {
if (pSource->pageIndex % 512 == 0) { if (pSource->pageIndex % 512 == 0) {
qDebug("begin source %p page %d", pSource, pSource->pageIndex); qDebug("begin source %p page %d", pSource, pSource->pageIndex);

1
source/libs/parser/inc/parInsertUtil.h
View File

@ -45,6 +45,7 @@ int16_t insFindCol(struct SToken *pColname, int16_t start, int16_t end, SSchema
void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag, int64_t suid, const char *sname, void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag, int64_t suid, const char *sname,
SArray *tagName, uint8_t tagNum, int32_t ttl); SArray *tagName, uint8_t tagNum, int32_t ttl);
int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo);
void insResetBoundColsInfo(SBoundColInfo *pInfo);
void insInitColValues(STableMeta *pTableMeta, SArray *aColValues); void insInitColValues(STableMeta *pTableMeta, SArray *aColValues);
void insCheckTableDataOrder(STableDataCxt *pTableCxt, SRowKey *rowKey); void insCheckTableDataOrder(STableDataCxt *pTableCxt, SRowKey *rowKey);
int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta,

3
source/libs/parser/src/parInsertSql.c
View File

@ -183,6 +183,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, E
} }
pBoundInfo->numOfBound = 0; pBoundInfo->numOfBound = 0;
pBoundInfo->hasBoundCols = true;
bool hasPK = pTableMeta->tableInfo.numOfPKs; bool hasPK = pTableMeta->tableInfo.numOfPKs;
int16_t numOfBoundPKs = 0; int16_t numOfBoundPKs = 0;
@ -1379,6 +1380,8 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOp
if (NULL != pStmt->pBoundCols) { if (NULL != pStmt->pBoundCols) {
return parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_COLUMNS, pStmt->pTableMeta, &pTableCxt->boundColsInfo); return parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_COLUMNS, pStmt->pTableMeta, &pTableCxt->boundColsInfo);
} else if (pTableCxt->boundColsInfo.hasBoundCols) {
insResetBoundColsInfo(&pTableCxt->boundColsInfo);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;

9
source/libs/parser/src/parInsertUtil.c
View File

@ -180,6 +180,7 @@ void insInitColValues(STableMeta* pTableMeta, SArray* aColValues) { initColValue
int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) {
pInfo->numOfCols = numOfBound; pInfo->numOfCols = numOfBound;
pInfo->numOfBound = numOfBound; pInfo->numOfBound = numOfBound;
pInfo->hasBoundCols = false;
pInfo->pColIndex = taosMemoryCalloc(numOfBound, sizeof(int16_t)); pInfo->pColIndex = taosMemoryCalloc(numOfBound, sizeof(int16_t));
if (NULL == pInfo->pColIndex) { if (NULL == pInfo->pColIndex) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -190,6 +191,14 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void insResetBoundColsInfo(SBoundColInfo* pInfo) {
pInfo->numOfBound = pInfo->numOfCols;
pInfo->hasBoundCols = false;
for (int32_t i = 0; i < pInfo->numOfCols; ++i) {
pInfo->pColIndex[i] = i;
}
}
void insCheckTableDataOrder(STableDataCxt* pTableCxt, SRowKey* rowKey) { void insCheckTableDataOrder(STableDataCxt* pTableCxt, SRowKey* rowKey) {
// once the data block is disordered, we do NOT keep last timestamp any more // once the data block is disordered, we do NOT keep last timestamp any more
if (!pTableCxt->ordered) { if (!pTableCxt->ordered) {

6
source/libs/stream/inc/streamBackendRocksdb.h
View File

@ -161,11 +161,11 @@ int32_t streamStatePut_rocksdb(SStreamState* pState, const SWinKey* key, const v
int32_t streamStateGet_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateGet_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateDel_rocksdb(SStreamState* pState, const SWinKey* key); int32_t streamStateDel_rocksdb(SStreamState* pState, const SWinKey* key);
int32_t streamStateClear_rocksdb(SStreamState* pState); int32_t streamStateClear_rocksdb(SStreamState* pState);
int32_t streamStateCurNext_rocksdb(SStreamState* pState, SStreamStateCur* pCur); void streamStateCurNext_rocksdb(SStreamStateCur* pCur);
int32_t streamStateGetFirst_rocksdb(SStreamState* pState, SWinKey* key); int32_t streamStateGetFirst_rocksdb(SStreamState* pState, SWinKey* key);
int32_t streamStateGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t streamStateAddIfNotExist_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateAddIfNotExist_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateCurPrev_rocksdb(SStreamStateCur* pCur); void streamStateCurPrev_rocksdb(SStreamStateCur* pCur);
int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
SStreamStateCur* streamStateGetAndCheckCur_rocksdb(SStreamState* pState, SWinKey* key); SStreamStateCur* streamStateGetAndCheckCur_rocksdb(SStreamState* pState, SWinKey* key);
SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key);
@ -193,7 +193,7 @@ int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSes
int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal,
int32_t* pVLen); int32_t* pVLen);
int32_t streamStateSessionClear_rocksdb(SStreamState* pState); void streamStateSessionClear_rocksdb(SStreamState* pState);
int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, char* pKeyData,
int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen);

8
source/libs/stream/inc/streamInt.h
View File

@ -208,7 +208,7 @@ int32_t streamQueueOpen(int64_t cap, SStreamQueue** pQ);
void streamQueueClose(SStreamQueue* pQueue, int32_t taskId); void streamQueueClose(SStreamQueue* pQueue, int32_t taskId);
void streamQueueProcessSuccess(SStreamQueue* queue); void streamQueueProcessSuccess(SStreamQueue* queue);
void streamQueueProcessFail(SStreamQueue* queue); void streamQueueProcessFail(SStreamQueue* queue);
void* streamQueueNextItem(SStreamQueue* pQueue); void streamQueueNextItem(SStreamQueue* pQueue, SStreamQueueItem** pItem);
void streamFreeQitem(SStreamQueueItem* data); void streamFreeQitem(SStreamQueueItem* data);
int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); int32_t streamQueueGetItemSize(const SStreamQueue* pQueue);
@ -226,7 +226,7 @@ int32_t streamTaskDownloadCheckpointData(const char* id, char* path);
int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask); int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask);
int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask); int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask);
int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId, void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId,
int32_t childId, SEpSet* pEpset, int64_t checkpointId); int32_t childId, SEpSet* pEpset, int64_t checkpointId);
int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId, int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId,
int64_t checkpointId, SRpcMsg* pMsg); int64_t checkpointId, SRpcMsg* pMsg);
@ -236,6 +236,10 @@ typedef int32_t (*__stream_async_exec_fn_t)(void* param);
int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code); int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code);
void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock); void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock);
void streamMutexLock(TdThreadMutex *pMutex);
void streamMutexUnlock(TdThreadMutex *pMutex);
void streamMutexDestroy(TdThreadMutex *pMutex);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

70
source/libs/stream/src/streamBackendRocksdb.c
View File

@ -887,8 +887,8 @@ _EXIT:
rocksdb_options_destroy(opts); rocksdb_options_destroy(opts);
rocksdb_cache_destroy(cache); rocksdb_cache_destroy(cache);
rocksdb_env_destroy(env); rocksdb_env_destroy(env);
taosThreadMutexDestroy(&pHandle->mutex); streamMutexDestroy(&pHandle->mutex);
taosThreadMutexDestroy(&pHandle->cfMutex); streamMutexDestroy(&pHandle->cfMutex);
taosHashCleanup(pHandle->cfInst); taosHashCleanup(pHandle->cfInst);
tdListFree(pHandle->list); tdListFree(pHandle->list);
taosMemoryFree(pHandle); taosMemoryFree(pHandle);
@ -923,9 +923,9 @@ void streamBackendCleanup(void* arg) {
} }
tdListFree(pHandle->list); tdListFree(pHandle->list);
taosThreadMutexDestroy(&pHandle->mutex); streamMutexDestroy(&pHandle->mutex);
taosThreadMutexDestroy(&pHandle->cfMutex); streamMutexDestroy(&pHandle->cfMutex);
stDebug("destroy stream backend :%p", pHandle); stDebug("destroy stream backend :%p", pHandle);
taosMemoryFree(pHandle); taosMemoryFree(pHandle);
return; return;
@ -1393,7 +1393,7 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) {
// vnode task->db // vnode task->db
SStreamMeta* pMeta = arg; SStreamMeta* pMeta = arg;
taosThreadMutexLock(&pMeta->backendMutex); streamMutexLock(&pMeta->backendMutex);
void* pIter = taosHashIterate(pMeta->pTaskDbUnique, NULL); void* pIter = taosHashIterate(pMeta->pTaskDbUnique, NULL);
int32_t code = 0; int32_t code = 0;
@ -1434,14 +1434,14 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) {
pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter); pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter);
} }
taosThreadMutexUnlock(&pMeta->backendMutex); streamMutexUnlock(&pMeta->backendMutex);
return code; return code;
} }
int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) { int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) {
if (pSnapInfo == NULL) return 0; if (pSnapInfo == NULL) return 0;
SStreamMeta* pMeta = arg; SStreamMeta* pMeta = arg;
int32_t code = 0; int32_t code = 0;
taosThreadMutexLock(&pMeta->backendMutex); streamMutexLock(&pMeta->backendMutex);
char buf[128] = {0}; char buf[128] = {0};
for (int i = 0; i < taosArrayGetSize(pSnapInfo); i++) { for (int i = 0; i < taosArrayGetSize(pSnapInfo); i++) {
@ -1457,7 +1457,7 @@ int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) {
taskDbUnRefChkp(*pTaskDb, pSnap->chkpId); taskDbUnRefChkp(*pTaskDb, pSnap->chkpId);
} }
taosThreadMutexUnlock(&pMeta->backendMutex); streamMutexUnlock(&pMeta->backendMutex);
return 0; return 0;
} }
#ifdef BUILD_NO_CALL #ifdef BUILD_NO_CALL
@ -1697,17 +1697,17 @@ int32_t streamBackendDoCheckpoint(void* arg, int64_t chkpId, int64_t processVer)
SListNode* streamBackendAddCompare(void* backend, void* arg) { SListNode* streamBackendAddCompare(void* backend, void* arg) {
SBackendWrapper* pHandle = (SBackendWrapper*)backend; SBackendWrapper* pHandle = (SBackendWrapper*)backend;
SListNode* node = NULL; SListNode* node = NULL;
taosThreadMutexLock(&pHandle->mutex); streamMutexLock(&pHandle->mutex);
node = tdListAdd(pHandle->list, arg); node = tdListAdd(pHandle->list, arg);
taosThreadMutexUnlock(&pHandle->mutex); streamMutexUnlock(&pHandle->mutex);
return node; return node;
} }
void streamBackendDelCompare(void* backend, void* arg) { void streamBackendDelCompare(void* backend, void* arg) {
SBackendWrapper* pHandle = (SBackendWrapper*)backend; SBackendWrapper* pHandle = (SBackendWrapper*)backend;
SListNode* node = NULL; SListNode* node = NULL;
taosThreadMutexLock(&pHandle->mutex); streamMutexLock(&pHandle->mutex);
node = tdListPopNode(pHandle->list, arg); node = tdListPopNode(pHandle->list, arg);
taosThreadMutexUnlock(&pHandle->mutex); streamMutexUnlock(&pHandle->mutex);
if (node) { if (node) {
streamStateDestroyCompar(node->data); streamStateDestroyCompar(node->data);
taosMemoryFree(node); taosMemoryFree(node);
@ -2461,9 +2461,9 @@ int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** sta
void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) { void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) {
STaskDbWrapper* p = pTaskDb; STaskDbWrapper* p = pTaskDb;
taosThreadMutexLock(&p->mutex); streamMutexLock(&p->mutex);
p->chkpId = chkpId; p->chkpId = chkpId;
taosThreadMutexUnlock(&p->mutex); streamMutexUnlock(&p->mutex);
} }
STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) { STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) {
@ -2622,7 +2622,7 @@ void taskDbDestroy(void* pDb, bool flush) {
taosMemoryFree(wrapper->pCfOpts); taosMemoryFree(wrapper->pCfOpts);
taosMemoryFree(wrapper->pCfParams); taosMemoryFree(wrapper->pCfParams);
taosThreadMutexDestroy(&wrapper->mutex); streamMutexDestroy(&wrapper->mutex);
taskDbDestroyChkpOpt(wrapper); taskDbDestroyChkpOpt(wrapper);
@ -2957,7 +2957,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
SBackendWrapper* handle = backend; SBackendWrapper* handle = backend;
SBackendCfWrapper* pBackendCfWrapper = taosMemoryCalloc(1, sizeof(SBackendCfWrapper)); SBackendCfWrapper* pBackendCfWrapper = taosMemoryCalloc(1, sizeof(SBackendCfWrapper));
taosThreadMutexLock(&handle->cfMutex); streamMutexLock(&handle->cfMutex);
RocksdbCfInst** ppInst = taosHashGet(handle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); RocksdbCfInst** ppInst = taosHashGet(handle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1);
if (ppInst != NULL && *ppInst != NULL) { if (ppInst != NULL && *ppInst != NULL) {
RocksdbCfInst* inst = *ppInst; RocksdbCfInst* inst = *ppInst;
@ -2970,7 +2970,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
pBackendCfWrapper->param = inst->param; pBackendCfWrapper->param = inst->param;
pBackendCfWrapper->pBackend = handle; pBackendCfWrapper->pBackend = handle;
pBackendCfWrapper->pComparNode = inst->pCompareNode; pBackendCfWrapper->pComparNode = inst->pCompareNode;
taosThreadMutexUnlock(&handle->cfMutex); streamMutexUnlock(&handle->cfMutex);
pBackendCfWrapper->backendId = pState->streamBackendRid; pBackendCfWrapper->backendId = pState->streamBackendRid;
memcpy(pBackendCfWrapper->idstr, pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr)); memcpy(pBackendCfWrapper->idstr, pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr));
@ -2987,7 +2987,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
inst->rOpt = NULL; inst->rOpt = NULL;
return 0; return 0;
} }
taosThreadMutexUnlock(&handle->cfMutex); streamMutexUnlock(&handle->cfMutex);
char* err = NULL; char* err = NULL;
int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]);
@ -3046,14 +3046,14 @@ void streamStateCloseBackend(SStreamState* pState, bool remove) {
stInfo("start to close state on backend: %p", pHandle); stInfo("start to close state on backend: %p", pHandle);
taosThreadMutexLock(&pHandle->cfMutex); streamMutexLock(&pHandle->cfMutex);
RocksdbCfInst** ppInst = taosHashGet(pHandle->cfInst, wrapper->idstr, strlen(pState->pTdbState->idstr) + 1); RocksdbCfInst** ppInst = taosHashGet(pHandle->cfInst, wrapper->idstr, strlen(pState->pTdbState->idstr) + 1);
if (ppInst != NULL && *ppInst != NULL) { if (ppInst != NULL && *ppInst != NULL) {
RocksdbCfInst* inst = *ppInst; RocksdbCfInst* inst = *ppInst;
taosMemoryFree(inst); taosMemoryFree(inst);
taosHashRemove(pHandle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); taosHashRemove(pHandle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1);
} }
taosThreadMutexUnlock(&pHandle->cfMutex); streamMutexUnlock(&pHandle->cfMutex);
char* status[] = {"close", "drop"}; char* status[] = {"close", "drop"};
stInfo("start to %s state %p on backendWrapper %p %s", status[remove == false ? 0 : 1], pState, wrapper, stInfo("start to %s state %p on backendWrapper %p %s", status[remove == false ? 0 : 1], pState, wrapper,
@ -3085,7 +3085,7 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) {
return -1; return -1;
} }
taosThreadMutexLock(&wrapper->mutex); streamMutexLock(&wrapper->mutex);
rocksdb_column_family_handle_t* cf = wrapper->pCf[idx]; rocksdb_column_family_handle_t* cf = wrapper->pCf[idx];
if (cf == NULL) { if (cf == NULL) {
@ -3100,7 +3100,7 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) {
wrapper->pCf[idx] = cf; wrapper->pCf[idx] = cf;
} }
} }
taosThreadMutexUnlock(&wrapper->mutex); streamMutexUnlock(&wrapper->mutex);
} }
return idx; return idx;
@ -3289,12 +3289,10 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) {
return 0; return 0;
} }
int32_t streamStateCurNext_rocksdb(SStreamState* pState, SStreamStateCur* pCur) { void streamStateCurNext_rocksdb(SStreamStateCur* pCur) {
if (!pCur) { if (pCur) {
return -1;
}
rocksdb_iter_next(pCur->iter); rocksdb_iter_next(pCur->iter);
return 0; }
} }
int32_t streamStateGetFirst_rocksdb(SStreamState* pState, SWinKey* key) { int32_t streamStateGetFirst_rocksdb(SStreamState* pState, SWinKey* key) {
stDebug("streamStateGetFirst_rocksdb"); stDebug("streamStateGetFirst_rocksdb");
@ -3337,11 +3335,10 @@ int32_t streamStateAddIfNotExist_rocksdb(SStreamState* pState, const SWinKey* ke
memset(*pVal, 0, size); memset(*pVal, 0, size);
return 0; return 0;
} }
int32_t streamStateCurPrev_rocksdb(SStreamStateCur* pCur) { void streamStateCurPrev_rocksdb(SStreamStateCur* pCur) {
if (!pCur) return -1; if (pCur) {
rocksdb_iter_prev(pCur->iter); rocksdb_iter_prev(pCur->iter);
return 0; }
} }
int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) {
if (!pCur) return -1; if (!pCur) return -1;
@ -3990,7 +3987,7 @@ int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSes
} }
if (c > 0) { if (c > 0) {
streamStateCurNext_rocksdb(pState, pCur); streamStateCurNext_rocksdb(pCur);
code = streamStateSessionGetKVByCur_rocksdb(pCur, &resKey, NULL, 0); code = streamStateSessionGetKVByCur_rocksdb(pCur, &resKey, NULL, 0);
if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) {
*curKey = resKey; *curKey = resKey;
@ -4037,7 +4034,7 @@ int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKe
goto _end; goto _end;
} }
taosMemoryFreeClear(*pVal); taosMemoryFreeClear(*pVal);
streamStateCurNext_rocksdb(pState, pCur); streamStateCurNext_rocksdb(pCur);
} else { } else {
*key = originKey; *key = originKey;
streamStateFreeCur(pCur); streamStateFreeCur(pCur);
@ -4063,7 +4060,7 @@ _end:
streamStateFreeCur(pCur); streamStateFreeCur(pCur);
return res; return res;
} }
int32_t streamStateSessionClear_rocksdb(SStreamState* pState) { void streamStateSessionClear_rocksdb(SStreamState* pState) {
stDebug("streamStateSessionClear_rocksdb"); stDebug("streamStateSessionClear_rocksdb");
SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0}; SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0};
SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext_rocksdb(pState, &key); SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext_rocksdb(pState, &key);
@ -4083,10 +4080,9 @@ int32_t streamStateSessionClear_rocksdb(SStreamState* pState) {
} }
taosMemoryFreeClear(buf); taosMemoryFreeClear(buf);
streamStateCurNext_rocksdb(pState, pCur); streamStateCurNext_rocksdb(pCur);
} }
streamStateFreeCur(pCur); streamStateFreeCur(pCur);
return -1;
} }
int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, char* pKeyData,
int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) {
@ -4114,7 +4110,7 @@ int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey*
goto _end; goto _end;
} }
streamStateCurNext_rocksdb(pState, pCur); streamStateCurNext_rocksdb(pCur);
} else { } else {
*key = tmpKey; *key = tmpKey;
streamStateFreeCur(pCur); streamStateFreeCur(pCur);

141
source/libs/stream/src/streamCheckStatus.c
View File

@ -23,10 +23,10 @@
static void processDownstreamReadyRsp(SStreamTask* pTask); static void processDownstreamReadyRsp(SStreamTask* pTask);
static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId); static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId);
static void rspMonitorFn(void* param, void* tmrId); static void rspMonitorFn(void* param, void* tmrId);
static int32_t streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs); static void streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs);
static int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id); static int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id);
static int32_t streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* id); static void streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* id);
static int32_t streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, int32_t vgId, const char* id); static void streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, int32_t vgId, const char* id);
static void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p); static void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p);
static void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList); static void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList);
static void handleNotReadyDownstreamTask(SStreamTask* pTask, SArray* pNotReadyList); static void handleNotReadyDownstreamTask(SStreamTask* pTask, SArray* pNotReadyList);
@ -41,9 +41,9 @@ static SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t
int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage, int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage,
int64_t* oldStage) { int64_t* oldStage) {
SStreamUpstreamEpInfo* pInfo = NULL; SStreamUpstreamEpInfo* pInfo = NULL;
int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo); streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) { if (pInfo == NULL) {
return code; return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
*oldStage = pInfo->stage; *oldStage = pInfo->stage;
@ -65,21 +65,21 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_
", prev:%" PRId64, ", prev:%" PRId64,
id, upstreamTaskId, vgId, stage, pInfo->stage); id, upstreamTaskId, vgId, stage, pInfo->stage);
// record the checkpoint failure id and sent to mnode // record the checkpoint failure id and sent to mnode
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
ETaskStatus status = streamTaskGetStatus(pTask).state; ETaskStatus status = streamTaskGetStatus(pTask).state;
if (status == TASK_STATUS__CK) { if (status == TASK_STATUS__CK) {
streamTaskSetFailedCheckpointId(pTask); streamTaskSetFailedCheckpointId(pTask);
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
} }
if (pInfo->stage != stage) { if (pInfo->stage != stage) {
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
ETaskStatus status = streamTaskGetStatus(pTask).state; ETaskStatus status = streamTaskGetStatus(pTask).state;
if (status == TASK_STATUS__CK) { if (status == TASK_STATUS__CK) {
streamTaskSetFailedCheckpointId(pTask); streamTaskSetFailedCheckpointId(pTask);
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return TASK_UPSTREAM_NEW_STAGE; return TASK_UPSTREAM_NEW_STAGE;
} else if (pTask->status.downstreamReady != 1) { } else if (pTask->status.downstreamReady != 1) {
@ -120,7 +120,7 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) {
idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, pRange->range.minVer, idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, pRange->range.minVer,
pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId); pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId);
streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, &pTask->outputInfo.fixedDispatcher.epSet); (void) streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, &pTask->outputInfo.fixedDispatcher.epSet);
} else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
streamTaskStartMonitorCheckRsp(pTask); streamTaskStartMonitorCheckRsp(pTask);
@ -140,7 +140,7 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) {
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 stDebug("s-task:%s (vgId:%d) stage:%" PRId64
" check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, reqId:0x%" PRIx64, " check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, reqId:0x%" PRIx64,
idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, req.reqId); idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, req.reqId);
streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); (void) streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
} }
} else { // for sink task, set it ready directly. } else { // for sink task, set it ready directly.
stDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", idstr, pTask->info.nodeId); stDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", idstr, pTask->info.nodeId);
@ -265,7 +265,7 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tEncoderInit(&encoder, (uint8_t*)abuf, len); tEncoderInit(&encoder, (uint8_t*)abuf, len);
tEncodeStreamTaskCheckRsp(&encoder, pRsp); (void) tEncodeStreamTaskCheckRsp(&encoder, pRsp);
tEncoderClear(&encoder); tEncoderClear(&encoder);
SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = *pRpcInfo}; SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = *pRpcInfo};
@ -274,14 +274,23 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa
return 0; return 0;
} }
int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask) { void streamTaskStartMonitorCheckRsp(SStreamTask* pTask) {
int32_t vgId = pTask->pMeta->vgId;
STaskCheckInfo* pInfo = &pTask->taskCheckInfo; STaskCheckInfo* pInfo = &pTask->taskCheckInfo;
taosThreadMutexLock(&pInfo->checkInfoLock);
streamMutexLock(&pInfo->checkInfoLock);
// drop procedure already started, not start check downstream now
ETaskStatus s = streamTaskGetStatus(pTask).state;
if (s == TASK_STATUS__DROPPING) {
streamMutexUnlock(&pInfo->checkInfoLock);
return;
}
int32_t code = streamTaskStartCheckDownstream(pInfo, pTask->id.idStr); int32_t code = streamTaskStartCheckDownstream(pInfo, pTask->id.idStr);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
return TSDB_CODE_FAILED; return;
} }
/*SStreamTask* p = */ streamMetaAcquireOneTask(pTask); // add task ref here /*SStreamTask* p = */ streamMetaAcquireOneTask(pTask); // add task ref here
@ -293,20 +302,18 @@ int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask) {
if (pInfo->checkRspTmr == NULL) { if (pInfo->checkRspTmr == NULL) {
pInfo->checkRspTmr = taosTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer); pInfo->checkRspTmr = taosTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer);
} else { } else {
taosTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr); streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, "check-status-monitor");
} }
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
return 0;
} }
int32_t streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id) { void streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id) {
taosThreadMutexLock(&pInfo->checkInfoLock); streamMutexLock(&pInfo->checkInfoLock);
pInfo->stopCheckProcess = 1; pInfo->stopCheckProcess = 1;
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
stDebug("s-task:%s set stop check-rsp monitor flag", id); stDebug("s-task:%s set stop check-rsp monitor flag", id);
return TSDB_CODE_SUCCESS;
} }
void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) {
@ -316,21 +323,21 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) {
pInfo->pList = NULL; pInfo->pList = NULL;
if (pInfo->checkRspTmr != NULL) { if (pInfo->checkRspTmr != NULL) {
/*bool ret = */ taosTmrStop(pInfo->checkRspTmr); (void) taosTmrStop(pInfo->checkRspTmr);
pInfo->checkRspTmr = NULL; pInfo->checkRspTmr = NULL;
} }
taosThreadMutexDestroy(&pInfo->checkInfoLock); streamMutexDestroy(&pInfo->checkInfoLock);
} }
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void processDownstreamReadyRsp(SStreamTask* pTask) { void processDownstreamReadyRsp(SStreamTask* pTask) {
EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST; EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST;
streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); (void) streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL);
int64_t checkTs = pTask->execInfo.checkTs; int64_t checkTs = pTask->execInfo.checkTs;
int64_t readyTs = pTask->execInfo.readyTs; int64_t readyTs = pTask->execInfo.readyTs;
streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true); (void) streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true);
if (pTask->status.taskStatus == TASK_STATUS__HALT) { if (pTask->status.taskStatus == TASK_STATUS__HALT) {
ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask) && (pTask->info.fillHistory == 0)); ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask) && (pTask->info.fillHistory == 0));
@ -338,21 +345,25 @@ void processDownstreamReadyRsp(SStreamTask* pTask) {
// halt it self for count window stream task until the related fill history task completed. // halt it self for count window stream task until the related fill history task completed.
stDebug("s-task:%s level:%d initial status is %s from mnode, set it to be halt", pTask->id.idStr, stDebug("s-task:%s level:%d initial status is %s from mnode, set it to be halt", pTask->id.idStr,
pTask->info.taskLevel, streamTaskGetStatusStr(pTask->status.taskStatus)); pTask->info.taskLevel, streamTaskGetStatusStr(pTask->status.taskStatus));
streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT); int32_t code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT);
if (code != 0) {
// todo: handle error
}
} }
// start the related fill-history task, when current task is ready // start the related fill-history task, when current task is ready
// not invoke in success callback due to the deadlock. // not invoke in success callback due to the deadlock.
// todo: let's retry
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
stDebug("s-task:%s try to launch related fill-history task", pTask->id.idStr); stDebug("s-task:%s try to launch related fill-history task", pTask->id.idStr);
streamLaunchFillHistoryTask(pTask); (void) streamLaunchFillHistoryTask(pTask);
} }
} }
void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) {
int32_t vgId = pTask->pMeta->vgId; int32_t vgId = pTask->pMeta->vgId;
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList); int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList);
bool existed = false; bool existed = false;
for (int i = 0; i < num; ++i) { for (int i = 0; i < num; ++i) {
@ -365,16 +376,18 @@ void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) {
if (!existed) { if (!existed) {
SDownstreamTaskEpset t = {.nodeId = nodeId}; SDownstreamTaskEpset t = {.nodeId = nodeId};
taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t); void* p = taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t);
if (p == NULL) {
// todo let's retry
}
stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", pTask->id.idStr, vgId, stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", pTask->id.idStr, vgId,
t.nodeId, (num + 1)); t.nodeId, (num + 1));
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
} }
int32_t streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs) { void streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs) {
taosArrayClear(pInfo->pList); taosArrayClear(pInfo->pList);
if (pOutputInfo->type == TASK_OUTPUT__FIXED_DISPATCH) { if (pOutputInfo->type == TASK_OUTPUT__FIXED_DISPATCH) {
@ -387,7 +400,6 @@ int32_t streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOut
pInfo->startTs = startTs; pInfo->startTs = startTs;
pInfo->timeoutStartTs = startTs; pInfo->timeoutStartTs = startTs;
pInfo->stopCheckProcess = 0; pInfo->stopCheckProcess = 0;
return TSDB_CODE_SUCCESS;
} }
SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId) { SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId) {
@ -403,7 +415,7 @@ SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId)
int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t status, int64_t rspTs, int64_t reqId, int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t status, int64_t rspTs, int64_t reqId,
int32_t* pNotReady, const char* id) { int32_t* pNotReady, const char* id) {
taosThreadMutexLock(&pInfo->checkInfoLock); streamMutexLock(&pInfo->checkInfoLock);
SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId); SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId);
if (p != NULL) { if (p != NULL) {
@ -411,7 +423,7 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t
stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64 stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64
" expired check-rsp recv from downstream task:0x%x, discarded", " expired check-rsp recv from downstream task:0x%x, discarded",
id, reqId, p->reqId, taskId); id, reqId, p->reqId, taskId);
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -425,11 +437,11 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t
p->status = status; p->status = status;
p->rspTs = rspTs; p->rspTs = rspTs;
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
stError("s-task:%s unexpected check rsp msg, invalid downstream task:0x%x, reqId:%" PRIx64 " discarded", id, taskId, stError("s-task:%s unexpected check rsp msg, invalid downstream task:0x%x, reqId:%" PRIx64 " discarded", id, taskId,
reqId); reqId);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
@ -450,9 +462,9 @@ int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* id) { void streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* id) {
if (lock) { if (lock) {
taosThreadMutexLock(&pInfo->checkInfoLock); streamMutexLock(&pInfo->checkInfoLock);
} }
if (pInfo->inCheckProcess) { if (pInfo->inCheckProcess) {
@ -474,27 +486,28 @@ int32_t streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char*
} }
if (lock) { if (lock) {
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
} }
return 0;
} }
int32_t streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, int32_t vgId, const char* id) { // todo: retry until success
void streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, int32_t vgId, const char* id) {
SDownstreamStatusInfo info = {.taskId = taskId, .status = -1, .vgId = vgId, .reqId = reqId, .rspTs = 0}; SDownstreamStatusInfo info = {.taskId = taskId, .status = -1, .vgId = vgId, .reqId = reqId, .rspTs = 0};
taosThreadMutexLock(&pInfo->checkInfoLock); streamMutexLock(&pInfo->checkInfoLock);
SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId); SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId);
if (p != NULL) { if (p != NULL) {
stDebug("s-task:%s check info to task:0x%x already sent", id, taskId); stDebug("s-task:%s check info to task:0x%x already sent", id, taskId);
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
return TSDB_CODE_SUCCESS; return;
} }
taosArrayPush(pInfo->pList, &info); void* px = taosArrayPush(pInfo->pList, &info);
if (px == NULL) {
// todo: retry
}
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
return TSDB_CODE_SUCCESS;
} }
void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) {
@ -519,7 +532,7 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) {
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64, id, stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64, id,
pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId); pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId);
streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet); (void) streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet);
} else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) { } else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
SArray* vgInfo = pOutputInfo->shuffleDispatcher.dbInfo.pVgroupInfos; SArray* vgInfo = pOutputInfo->shuffleDispatcher.dbInfo.pVgroupInfos;
int32_t numOfVgs = taosArrayGetSize(vgInfo); int32_t numOfVgs = taosArrayGetSize(vgInfo);
@ -533,7 +546,7 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) {
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 stDebug("s-task:%s (vgId:%d) stage:%" PRId64
" re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d reqId:0x%" PRIx64, " re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d reqId:0x%" PRIx64,
id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, p->reqId); id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, p->reqId);
streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); (void) streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
break; break;
} }
} }
@ -556,12 +569,12 @@ void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, i
if (p->rspTs == 0) { // not response yet if (p->rspTs == 0) { // not response yet
ASSERT(p->status == -1); ASSERT(p->status == -1);
if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec. if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec.
taosArrayPush(pTimeoutList, &p->taskId); (void) taosArrayPush(pTimeoutList, &p->taskId);
} else { // el < CHECK_NOT_RSP_DURATION } else { // el < CHECK_NOT_RSP_DURATION
(*numOfNotRsp) += 1; // do nothing and continue waiting for their rsp (*numOfNotRsp) += 1; // do nothing and continue waiting for their rsp
} }
} else { } else {
taosArrayPush(pNotReadyList, &p->taskId); (void) taosArrayPush(pNotReadyList, &p->taskId);
} }
} }
} }
@ -676,7 +689,7 @@ void rspMonitorFn(void* param, void* tmrId) {
// not record the failed of the current task if try to close current vnode // not record the failed of the current task if try to close current vnode
// otherwise, the put of message operation may incur invalid read of message queue. // otherwise, the put of message operation may incur invalid read of message queue.
if (!pMeta->closeFlag) { if (!pMeta->closeFlag) {
addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); (void) addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId);
} }
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
@ -692,14 +705,14 @@ void rspMonitorFn(void* param, void* tmrId) {
return; return;
} }
taosThreadMutexLock(&pInfo->checkInfoLock); streamMutexLock(&pInfo->checkInfoLock);
if (pInfo->notReadyTasks == 0) { if (pInfo->notReadyTasks == 0) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, pStat.name, stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, pStat.name,
vgId, ref); vgId, ref);
streamTaskCompleteCheckRsp(pInfo, false, id); streamTaskCompleteCheckRsp(pInfo, false, id);
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return; return;
} }
@ -723,7 +736,7 @@ void rspMonitorFn(void* param, void* tmrId) {
id, pStat.name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); id, pStat.name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref);
streamTaskCompleteCheckRsp(pInfo, false, id); streamTaskCompleteCheckRsp(pInfo, false, id);
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
taosArrayDestroy(pNotReadyList); taosArrayDestroy(pNotReadyList);
@ -743,9 +756,9 @@ void rspMonitorFn(void* param, void* tmrId) {
id, pStat.name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); id, pStat.name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref);
streamTaskCompleteCheckRsp(pInfo, false, id); streamTaskCompleteCheckRsp(pInfo, false, id);
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); (void) addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
taosArrayDestroy(pNotReadyList); taosArrayDestroy(pNotReadyList);
@ -761,8 +774,8 @@ void rspMonitorFn(void* param, void* tmrId) {
handleTimeoutDownstreamTasks(pTask, pTimeoutList); handleTimeoutDownstreamTasks(pTask, pTimeoutList);
} }
taosTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr); streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, "check-status-monitor");
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
stDebug( stDebug(
"s-task:%s vgId:%d continue checking rsp in 300ms, total:%d, notRsp:%d, notReady:%d, fault:%d, timeout:%d, " "s-task:%s vgId:%d continue checking rsp in 300ms, total:%d, notRsp:%d, notReady:%d, fault:%d, timeout:%d, "

132
source/libs/stream/src/streamCheckpoint.c
View File

@ -77,6 +77,7 @@ int32_t createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int6
} }
// this message must be put into inputq successfully, continue retrying until it succeeds // this message must be put into inputq successfully, continue retrying until it succeeds
// todo must be success
int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId, int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId,
int32_t srcTaskId) { int32_t srcTaskId) {
SStreamDataBlock* pCheckpoint = NULL; SStreamDataBlock* pCheckpoint = NULL;
@ -119,7 +120,7 @@ int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTri
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER, pRsp->checkpointId, pRsp->transId, (void)appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER, pRsp->checkpointId, pRsp->transId,
pRsp->upstreamTaskId); pRsp->upstreamTaskId);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -164,7 +165,7 @@ int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStream
int32_t code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock); int32_t code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock);
if (code == 0) { if (code == 0) {
ASSERT(pTask->chkInfo.pActiveInfo->dispatchTrigger == false); ASSERT(pTask->chkInfo.pActiveInfo->dispatchTrigger == false);
streamDispatchStreamBlock(pTask); code = streamDispatchStreamBlock(pTask);
} else { } else {
stError("s-task:%s failed to put checkpoint into outputQ, code:%s", pTask->id.idStr, tstrerror(code)); stError("s-task:%s failed to put checkpoint into outputQ, code:%s", pTask->id.idStr, tstrerror(code));
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
@ -184,12 +185,12 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pTask->chkInfo.checkpointId > checkpointId) { if (pTask->chkInfo.checkpointId > checkpointId) {
stError("s-task:%s vgId:%d current checkpointId:%" PRId64 stError("s-task:%s vgId:%d current checkpointId:%" PRId64
" recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard", " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard",
id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId); id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId);
code = taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
return code; return code;
@ -199,13 +200,16 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
{ // send checkpoint-ready msg to upstream { // send checkpoint-ready msg to upstream
SRpcMsg msg = {0}; SRpcMsg msg = {0};
SStreamUpstreamEpInfo* pInfo = NULL; SStreamUpstreamEpInfo* pInfo = NULL;
code = streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId, &pInfo); streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) { if (pInfo == NULL) {
return code; streamMutexUnlock(&pTask->lock);
return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg); code = initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg);
tmsgSendReq(&pInfo->epSet, &msg); if (code == TSDB_CODE_SUCCESS) {
(void)tmsgSendReq(&pInfo->epSet, &msg);
}
} }
stWarn( stWarn(
@ -214,7 +218,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
id, vgId, pBlock->srcTaskId); id, vgId, pBlock->srcTaskId);
streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId); streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId);
code = taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
return code; return code;
@ -225,7 +229,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64 stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64
" discard", " discard",
id, vgId, pActiveInfo->activeId, checkpointId); id, vgId, pActiveInfo->activeId, checkpointId);
code = taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
return code; return code;
@ -235,7 +239,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
"s-task:%s vgId:%d all upstream checkpoint-trigger recv, discard this checkpoint-trigger, " "s-task:%s vgId:%d all upstream checkpoint-trigger recv, discard this checkpoint-trigger, "
"checkpointId:%" PRId64 " transId:%d", "checkpointId:%" PRId64 " transId:%d",
id, vgId, checkpointId, transId); id, vgId, checkpointId, transId);
code = taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
return code; return code;
} }
@ -250,7 +254,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
", prev recvTs:%" PRId64 " discard", ", prev recvTs:%" PRId64 " discard",
pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs); pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs);
code = taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
return code; return code;
} }
@ -259,7 +263,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
} }
} }
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
stDebug("s-task:%s vgId:%d start to handle the checkpoint-trigger block, checkpointId:%" PRId64 " ver:%" PRId64 stDebug("s-task:%s vgId:%d start to handle the checkpoint-trigger block, checkpointId:%" PRId64 " ver:%" PRId64
", transId:%d current active checkpointId:%" PRId64, ", transId:%d current active checkpointId:%" PRId64,
@ -284,7 +288,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
if (pActiveInfo->pChkptTriggerTmr == NULL) { if (pActiveInfo->pChkptTriggerTmr == NULL) {
pActiveInfo->pChkptTriggerTmr = taosTmrStart(checkpointTriggerMonitorFn, 100, pTask, streamTimer); pActiveInfo->pChkptTriggerTmr = taosTmrStart(checkpointTriggerMonitorFn, 100, pTask, streamTimer);
} else { } else {
taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr); streamTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr, vgId, "trigger-recv-monitor");
} }
} }
@ -300,9 +304,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) { if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId); stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId);
continueDispatchCheckpointTriggerBlock(pBlock, pTask); (void)continueDispatchCheckpointTriggerBlock(pBlock, pTask); // todo handle this failure
} else { // only one task exists, no need to dispatch downstream info } else { // only one task exists, no need to dispatch downstream info
appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId, -1); (void)appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId,
-1);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
} }
} else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) { } else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) {
@ -312,8 +317,9 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
pTask->execInfo.checkpoint += 1; pTask->execInfo.checkpoint += 1;
} }
// todo: handle this
// update the child Id for downstream tasks // update the child Id for downstream tasks
streamAddCheckpointReadyMsg(pTask, pBlock->srcTaskId, pTask->info.selfChildId, checkpointId); (void) streamAddCheckpointReadyMsg(pTask, pBlock->srcTaskId, pTask->info.selfChildId, checkpointId);
// there are still some upstream tasks not send checkpoint request, do nothing and wait for then // there are still some upstream tasks not send checkpoint request, do nothing and wait for then
if (pActiveInfo->allUpstreamTriggerRecv != 1) { if (pActiveInfo->allUpstreamTriggerRecv != 1) {
@ -325,7 +331,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
if (taskLevel == TASK_LEVEL__SINK) { if (taskLevel == TASK_LEVEL__SINK) {
stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, send ready msg to upstream", id, num); stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, send ready msg to upstream", id, num);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
streamTaskBuildCheckpoint(pTask); (void)streamTaskBuildCheckpoint(pTask); // todo: not handle error yet
} else { // source & agg tasks need to forward the checkpoint msg downwards } else { // source & agg tasks need to forward the checkpoint msg downwards
stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, forwards to downstream", id, num); stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, forwards to downstream", id, num);
@ -369,7 +375,7 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId
return -1; return -1;
} }
(void) taosThreadMutexLock(&pInfo->lock); streamMutexLock(&pInfo->lock);
// only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task // only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task
int32_t size = taosArrayGetSize(pInfo->pCheckpointReadyRecvList); int32_t size = taosArrayGetSize(pInfo->pCheckpointReadyRecvList);
@ -391,16 +397,16 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId
.transId = pInfo->transId, .transId = pInfo->transId,
.streamId = pTask->id.streamId, .streamId = pTask->id.streamId,
.downstreamNodeId = downstreamNodeId}; .downstreamNodeId = downstreamNodeId};
taosArrayPush(pInfo->pCheckpointReadyRecvList, &info); (void)taosArrayPush(pInfo->pCheckpointReadyRecvList, &info);
} }
int32_t notReady = total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList); int32_t notReady = total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList);
int32_t transId = pInfo->transId; int32_t transId = pInfo->transId;
(void) taosThreadMutexUnlock(&pInfo->lock); streamMutexUnlock(&pInfo->lock);
if (notReady == 0) { if (notReady == 0) {
stDebug("s-task:%s all downstream tasks have completed build checkpoint, do checkpoint for current task", id); stDebug("s-task:%s all downstream tasks have completed build checkpoint, do checkpoint for current task", id);
appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, checkpointId, transId, -1); (void)appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, checkpointId, transId, -1);
} }
return 0; return 0;
@ -411,7 +417,7 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream
int64_t now = taosGetTimestampMs(); int64_t now = taosGetTimestampMs();
int32_t numOfConfirmed = 0; int32_t numOfConfirmed = 0;
(void) taosThreadMutexLock(&pInfo->lock); streamMutexLock(&pInfo->lock);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) {
STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i); STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i);
if (pReadyInfo->upstreamTaskId == upstreamTaskId && pReadyInfo->checkpointId == checkpointId) { if (pReadyInfo->upstreamTaskId == upstreamTaskId && pReadyInfo->checkpointId == checkpointId) {
@ -432,7 +438,7 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream
stDebug("s-task:%s send checkpoint-ready msg to %d upstream confirmed, checkpointId:%" PRId64, pTask->id.idStr, stDebug("s-task:%s send checkpoint-ready msg to %d upstream confirmed, checkpointId:%" PRId64, pTask->id.idStr,
numOfConfirmed, checkpointId); numOfConfirmed, checkpointId);
(void) taosThreadMutexUnlock(&pInfo->lock); streamMutexUnlock(&pInfo->lock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -440,12 +446,12 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) {
pTask->chkInfo.startTs = 0; // clear the recorded start time pTask->chkInfo.startTs = 0; // clear the recorded start time
streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks
(void) taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock); streamMutexLock(&pTask->chkInfo.pActiveInfo->lock);
streamTaskClearActiveInfo(pTask->chkInfo.pActiveInfo); streamTaskClearActiveInfo(pTask->chkInfo.pActiveInfo);
if (clearChkpReadyMsg) { if (clearChkpReadyMsg) {
streamClearChkptReadyMsg(pTask->chkInfo.pActiveInfo); streamClearChkptReadyMsg(pTask->chkInfo.pActiveInfo);
} }
(void) taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock); streamMutexUnlock(&pTask->chkInfo.pActiveInfo->lock);
} }
int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq) { int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq) {
@ -455,7 +461,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
SCheckpointInfo* pInfo = &pTask->chkInfo; SCheckpointInfo* pInfo = &pTask->chkInfo;
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pReq->checkpointId <= pInfo->checkpointId) { if (pReq->checkpointId <= pInfo->checkpointId) {
stDebug("s-task:%s vgId:%d latest checkpointId:%" PRId64 " checkpointVer:%" PRId64 stDebug("s-task:%s vgId:%d latest checkpointId:%" PRId64 " checkpointVer:%" PRId64
@ -463,7 +469,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
" transId:%d ignored", " transId:%d ignored",
id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer, id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer,
pReq->transId); pReq->transId);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
{ // destroy the related fill-history tasks { // destroy the related fill-history tasks
// drop task should not in the meta-lock, and drop the related fill-history task now // drop task should not in the meta-lock, and drop the related fill-history task now
@ -525,13 +531,14 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
pTask->status.taskStatus = TASK_STATUS__READY; pTask->status.taskStatus = TASK_STATUS__READY;
code = streamMetaSaveTask(pMeta, pTask); code = streamMetaSaveTask(pMeta, pTask);
streamMutexUnlock(&pTask->lock);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s vgId:%d failed to save task info after do checkpoint, checkpointId:%" PRId64 ", since %s", id, stError("s-task:%s vgId:%d failed to save task info after do checkpoint, checkpointId:%" PRId64 ", since %s", id,
vgId, pReq->checkpointId, terrstr()); vgId, pReq->checkpointId, terrstr());
return code; return code;
} }
(void) taosThreadMutexUnlock(&pTask->lock);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
// drop task should not in the meta-lock, and drop the related fill-history task now // drop task should not in the meta-lock, and drop the related fill-history task now
@ -730,9 +737,9 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
code = streamSendChkptReportMsg(pTask, &pTask->chkInfo, dropRelHTask); code = streamSendChkptReportMsg(pTask, &pTask->chkInfo, dropRelHTask);
} }
} else { // clear the checkpoint info if failed } else { // clear the checkpoint info if failed
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
streamTaskSetFailedCheckpointId(pTask); // set failed checkpoint id before clear the checkpoint info streamTaskSetFailedCheckpointId(pTask); // set failed checkpoint id before clear the checkpoint info
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE); code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE);
stDebug("s-task:%s clear checkpoint flag since gen checkpoint failed, checkpointId:%" PRId64, id, ckId); stDebug("s-task:%s clear checkpoint flag since gen checkpoint failed, checkpointId:%" PRId64, id, ckId);
@ -763,20 +770,20 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
} }
if (++pActiveInfo->checkCounter < 100) { if (++pActiveInfo->checkCounter < 100) {
taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr); streamTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr, vgId, "trigger-recv-monitor");
return; return;
} }
pActiveInfo->checkCounter = 0; pActiveInfo->checkCounter = 0;
stDebug("s-task:%s vgId:%d checkpoint-trigger monitor in tmr, ts:%" PRId64, id, vgId, now); stDebug("s-task:%s vgId:%d checkpoint-trigger monitor in tmr, ts:%" PRId64, id, vgId, now);
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state != TASK_STATUS__CK) { if (pState.state != TASK_STATUS__CK) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
return; return;
} }
@ -787,14 +794,14 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
stDebug("s-task:%s vgId:%d all checkpoint-trigger recv, quit from monitor checkpoint-trigger, ref:%d", id, vgId, stDebug("s-task:%s vgId:%d all checkpoint-trigger recv, quit from monitor checkpoint-trigger, ref:%d", id, vgId,
ref); ref);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
return; return;
} }
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
(void) taosThreadMutexLock(&pActiveInfo->lock); streamMutexLock(&pActiveInfo->lock);
// send msg to retrieve checkpoint trigger msg // send msg to retrieve checkpoint trigger msg
SArray* pList = pTask->upstreamInfo.pList; SArray* pList = pTask->upstreamInfo.pList;
@ -820,19 +827,19 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
if (!recved) { // make sure the inputQ is opened for not recv upstream checkpoint-trigger message if (!recved) { // make sure the inputQ is opened for not recv upstream checkpoint-trigger message
streamTaskOpenUpstreamInput(pTask, pInfo->taskId); streamTaskOpenUpstreamInput(pTask, pInfo->taskId);
taosArrayPush(pNotSendList, pInfo); (void)taosArrayPush(pNotSendList, pInfo);
} }
} }
// do send retrieve checkpoint trigger msg to upstream // do send retrieve checkpoint trigger msg to upstream
int32_t size = taosArrayGetSize(pNotSendList); int32_t size = taosArrayGetSize(pNotSendList);
doSendRetrieveTriggerMsg(pTask, pNotSendList); (void)doSendRetrieveTriggerMsg(pTask, pNotSendList);
(void) taosThreadMutexUnlock(&pActiveInfo->lock); streamMutexUnlock(&pActiveInfo->lock);
// check every 100ms // check every 100ms
if (size > 0) { if (size > 0) {
stDebug("s-task:%s start to monitor checkpoint-trigger in 10s", id); stDebug("s-task:%s start to monitor checkpoint-trigger in 10s", id);
taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr); streamTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr, vgId, "trigger-recv-monitor");
} else { } else {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s all checkpoint-trigger recved, quit from monitor checkpoint-trigger tmr, ref:%d", id, ref); stDebug("s-task:%s all checkpoint-trigger recved, quit from monitor checkpoint-trigger tmr, ref:%d", id, ref);
@ -863,7 +870,7 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) {
SRetrieveChkptTriggerReq* pReq = rpcMallocCont(sizeof(SRetrieveChkptTriggerReq)); SRetrieveChkptTriggerReq* pReq = rpcMallocCont(sizeof(SRetrieveChkptTriggerReq));
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
stError("vgId:%d failed to create msg to retrieve trigger msg for task:%s exec, code:out of memory", vgId, pId); stError("vgId:%d failed to create msg to retrieve trigger msg for task:%s exec, code:out of memory", vgId, pId);
continue; continue;
} }
@ -880,11 +887,16 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) {
initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE_TRIGGER, pReq, sizeof(SRetrieveChkptTriggerReq)); initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE_TRIGGER, pReq, sizeof(SRetrieveChkptTriggerReq));
code = tmsgSendReq(&pUpstreamTask->epSet, &rpcMsg); code = tmsgSendReq(&pUpstreamTask->epSet, &rpcMsg);
stDebug("s-task:%s vgId:%d send checkpoint-trigger retrieve msg to 0x%x(vgId:%d) checkpointId:%" PRId64, pId, vgId, if (code == TSDB_CODE_SUCCESS) {
pUpstreamTask->taskId, pUpstreamTask->nodeId, checkpointId); stDebug("s-task:%s vgId:%d send checkpoint-trigger retrieve msg to 0x%x(vgId:%d) checkpointId:%" PRId64, pId,
vgId, pUpstreamTask->taskId, pUpstreamTask->nodeId, checkpointId);
} else {
stError("s-task:%s vgId:%d failed to send checkpoint-trigger retrieve msg to 0x%x(vgId:%d) checkpointId:%" PRId64,
pId, vgId, pUpstreamTask->taskId, pUpstreamTask->nodeId, checkpointId);
}
} }
return TSDB_CODE_SUCCESS; return code;
} }
bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) { bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) {
@ -897,9 +909,9 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId)
return false; return false;
} }
(void) taosThreadMutexLock(&pInfo->lock); streamMutexLock(&pInfo->lock);
if (!pInfo->dispatchTrigger) { if (!pInfo->dispatchTrigger) {
(void) taosThreadMutexUnlock(&pInfo->lock); streamMutexUnlock(&pTask->lock);
return false; return false;
} }
@ -921,7 +933,7 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId)
id, pSendInfo->sendTs, before, pInfo->activeId, pInfo->transId); id, pSendInfo->sendTs, before, pInfo->activeId, pInfo->transId);
} }
(void) taosThreadMutexUnlock(&pInfo->lock); streamMutexUnlock(&pTask->lock);
return true; return true;
} }
@ -945,7 +957,7 @@ void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) {
SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
int64_t now = taosGetTimestampMs(); int64_t now = taosGetTimestampMs();
(void) taosThreadMutexLock(&pInfo->lock); streamMutexLock(&pInfo->lock);
// outputQ should be empty here // outputQ should be empty here
ASSERT(streamQueueGetNumOfUnAccessedItems(pTask->outputq.queue) == 0); ASSERT(streamQueueGetNumOfUnAccessedItems(pTask->outputq.queue) == 0);
@ -971,21 +983,21 @@ void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) {
} }
} }
(void) taosThreadMutexUnlock(&pInfo->lock); streamMutexUnlock(&pInfo->lock);
} }
int32_t streamTaskGetNumOfConfirmed(SStreamTask* pTask) { int32_t streamTaskGetNumOfConfirmed(SStreamTask* pTask) {
SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
int32_t num = 0; int32_t num = 0;
(void) taosThreadMutexLock(&pInfo->lock); streamMutexLock(&pInfo->lock);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) {
STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i);
if (p->recved) { if (p->recved) {
num++; num++;
} }
} }
(void) taosThreadMutexUnlock(&pInfo->lock); streamMutexUnlock(&pInfo->lock);
return num; return num;
} }
@ -993,7 +1005,7 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) {
SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
int32_t taskId = 0; int32_t taskId = 0;
(void) taosThreadMutexLock(&pInfo->lock); streamMutexLock(&pInfo->lock);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) {
STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i);
@ -1007,7 +1019,7 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) {
} }
} }
(void) taosThreadMutexUnlock(&pInfo->lock); streamMutexUnlock(&pInfo->lock);
int32_t numOfConfirmed = streamTaskGetNumOfConfirmed(pTask); int32_t numOfConfirmed = streamTaskGetNumOfConfirmed(pTask);
int32_t total = streamTaskGetNumOfDownstream(pTask); int32_t total = streamTaskGetNumOfDownstream(pTask);
@ -1184,16 +1196,16 @@ int32_t deleteCheckpointFile(const char* id, const char* name) {
int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask) { int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask) {
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pTask->status.sendConsensusChkptId == true) { if (pTask->status.sendConsensusChkptId == true) {
stDebug("s-task:%s already start to consensus-checkpointId, not start again before it completed", id); stDebug("s-task:%s already start to consensus-checkpointId, not start again before it completed", id);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else { } else {
pTask->status.sendConsensusChkptId = true; pTask->status.sendConsensusChkptId = true;
} }
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
ASSERT(pTask->pBackend == NULL); ASSERT(pTask->pBackend == NULL);
pTask->status.requireConsensusChkptId = true; pTask->status.requireConsensusChkptId = true;
@ -1207,12 +1219,12 @@ int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask) {
return code; return code;
} }
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState p = streamTaskGetStatus(pTask); SStreamTaskState p = streamTaskGetStatus(pTask);
if (p.state == TASK_STATUS__CK) { if (p.state == TASK_STATUS__CK) {
code = streamTaskSendCheckpointSourceRsp(pTask); code = streamTaskSendCheckpointSourceRsp(pTask);
} }
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return code; return code;
} }

61
source/libs/stream/src/streamData.c
View File

@ -49,7 +49,7 @@ int32_t createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t
pInput = p; pInput = p;
} }
blockDecode(pDataBlock, pInput); (void) blockDecode(pDataBlock, pInput);
if (pRetrieve->compressed && compLen < fullLen) { if (pRetrieve->compressed && compLen < fullLen) {
taosMemoryFree(pInput); taosMemoryFree(pInput);
@ -116,11 +116,11 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock
return terrno; return terrno;
} }
taosArrayPush(pArray, &(SSDataBlock){0}); (void) taosArrayPush(pArray, &(SSDataBlock){0});
SRetrieveTableRsp* pRetrieve = pReq->pRetrieve; SRetrieveTableRsp* pRetrieve = pReq->pRetrieve;
SSDataBlock* pDataBlock = taosArrayGet(pArray, 0); SSDataBlock* pDataBlock = taosArrayGet(pArray, 0);
blockDecode(pDataBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN); (void) blockDecode(pDataBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN);
// TODO: refactor // TODO: refactor
pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.skey = be64toh(pRetrieve->skey);
@ -156,28 +156,31 @@ void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit) {
taosFreeQitem(pDataSubmit); taosFreeQitem(pDataSubmit);
} }
SStreamMergedSubmit* streamMergedSubmitNew() { int32_t streamMergedSubmitNew(SStreamMergedSubmit** pSubmit) {
SStreamMergedSubmit* pMerged; *pSubmit = NULL;
int32_t code = taosAllocateQitem(sizeof(SStreamMergedSubmit), DEF_QITEM, 0, (void**)&pMerged); int32_t code = taosAllocateQitem(sizeof(SStreamMergedSubmit), DEF_QITEM, 0, (void**)pSubmit);
if (code) { if (code) {
terrno = code; return TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
pMerged->submits = taosArrayInit(0, sizeof(SPackedData)); (*pSubmit)->submits = taosArrayInit(0, sizeof(SPackedData));
if (pMerged->submits == NULL) { if ((*pSubmit)->submits == NULL) {
taosArrayDestroy(pMerged->submits); taosFreeQitem(*pSubmit);
taosFreeQitem(pMerged); *pSubmit = NULL;
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
pMerged->type = STREAM_INPUT__MERGED_SUBMIT; (*pSubmit)->type = STREAM_INPUT__MERGED_SUBMIT;
return pMerged; return TSDB_CODE_SUCCESS;
} }
int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubmit) { int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubmit) {
taosArrayPush(pMerged->submits, &pSubmit->submit); void* p = taosArrayPush(pMerged->submits, &pSubmit->submit);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (pSubmit->ver > pMerged->ver) { if (pSubmit->ver > pMerged->ver) {
pMerged->ver = pSubmit->ver; pMerged->ver = pSubmit->ver;
} }
@ -187,11 +190,12 @@ int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubm
// todo handle memory error // todo handle memory error
int32_t streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem, SStreamQueueItem** pRes) { int32_t streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem, SStreamQueueItem** pRes) {
*pRes = NULL; *pRes = NULL;
int32_t code = 0;
if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) { if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) {
SStreamDataBlock* pBlock = (SStreamDataBlock*)dst; SStreamDataBlock* pBlock = (SStreamDataBlock*)dst;
SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)pElem; SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)pElem;
taosArrayAddAll(pBlock->blocks, pBlockSrc->blocks); (void) taosArrayAddAll(pBlock->blocks, pBlockSrc->blocks);
taosArrayDestroy(pBlockSrc->blocks); taosArrayDestroy(pBlockSrc->blocks);
streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem)); streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem));
@ -201,33 +205,38 @@ int32_t streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem
} else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) {
SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)dst; SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)dst;
SStreamDataSubmit* pBlockSrc = (SStreamDataSubmit*)pElem; SStreamDataSubmit* pBlockSrc = (SStreamDataSubmit*)pElem;
streamMergeSubmit(pMerged, pBlockSrc);
code = streamMergeSubmit(pMerged, pBlockSrc);
streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem)); streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem));
taosFreeQitem(pElem); taosFreeQitem(pElem);
*pRes = dst; *pRes = dst;
*pRes = dst; *pRes = dst;
return TSDB_CODE_SUCCESS; return code;
} else if (dst->type == STREAM_INPUT__DATA_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) {
SStreamMergedSubmit* pMerged = streamMergedSubmitNew(); SStreamMergedSubmit* pMerged = NULL;
if (pMerged == NULL) { code = streamMergedSubmitNew(&pMerged);
return TSDB_CODE_OUT_OF_MEMORY; if (code != 0) {
return code;
} }
streamQueueItemIncSize((SStreamQueueItem*)pMerged, streamQueueItemGetSize(pElem)); streamQueueItemIncSize((SStreamQueueItem*)pMerged, streamQueueItemGetSize(pElem));
streamMergeSubmit(pMerged, (SStreamDataSubmit*)dst); code = streamMergeSubmit(pMerged, (SStreamDataSubmit*)dst);
streamMergeSubmit(pMerged, (SStreamDataSubmit*)pElem); if (code == 0) {
code = streamMergeSubmit(pMerged, (SStreamDataSubmit*)pElem);
}
taosFreeQitem(dst); taosFreeQitem(dst);
taosFreeQitem(pElem); taosFreeQitem(pElem);
*pRes = (SStreamQueueItem*)pMerged; *pRes = (SStreamQueueItem*)pMerged;
return TSDB_CODE_SUCCESS; return code;
} else { } else {
code = TSDB_CODE_FAILED;
stDebug("block type:%s not merged with existed blocks list, type:%d", streamQueueItemGetTypeStr(pElem->type), stDebug("block type:%s not merged with existed blocks list, type:%d", streamQueueItemGetTypeStr(pElem->type),
dst->type); dst->type);
return TSDB_CODE_FAILED; return code;
} }
} }

178
source/libs/stream/src/streamDispatch.c
View File

@ -121,7 +121,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, abuf, len); tEncoderInit(&encoder, abuf, len);
tEncodeStreamRetrieveReq(&encoder, req); (void) tEncodeStreamRetrieveReq(&encoder, req);
tEncoderClear(&encoder); tEncoderClear(&encoder);
SRpcMsg rpcMsg = {0}; SRpcMsg rpcMsg = {0};
@ -189,6 +189,7 @@ int32_t streamBroadcastToUpTasks(SStreamTask* pTask, const SSDataBlock* pBlock)
return code; return code;
} }
// no need to do anything if failed
int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet) { int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet) {
void* buf = NULL; void* buf = NULL;
int32_t code = -1; int32_t code = -1;
@ -202,7 +203,7 @@ int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq,
buf = rpcMallocCont(sizeof(SMsgHead) + tlen); buf = rpcMallocCont(sizeof(SMsgHead) + tlen);
if (buf == NULL) { if (buf == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
((SMsgHead*)buf)->vgId = htonl(nodeId); ((SMsgHead*)buf)->vgId = htonl(nodeId);
@ -220,8 +221,7 @@ int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq,
stDebug("s-task:%s (level:%d) send check msg to s-task:0x%" PRIx64 ":0x%x (vgId:%d)", pTask->id.idStr, stDebug("s-task:%s (level:%d) send check msg to s-task:0x%" PRIx64 ":0x%x (vgId:%d)", pTask->id.idStr,
pTask->info.taskLevel, pReq->streamId, pReq->downstreamTaskId, nodeId); pTask->info.taskLevel, pReq->streamId, pReq->downstreamTaskId, nodeId);
tmsgSendReq(pEpSet, &msg); return tmsgSendReq(pEpSet, &msg);
return 0;
} }
void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups) { void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups) {
@ -239,7 +239,7 @@ void clearBufferedDispatchMsg(SStreamTask* pTask) {
destroyDispatchMsg(pMsgInfo->pData, streamTaskGetNumOfDownstream(pTask)); destroyDispatchMsg(pMsgInfo->pData, streamTaskGetNumOfDownstream(pTask));
} }
taosThreadMutexLock(&pMsgInfo->lock); streamMutexLock(&pMsgInfo->lock);
pMsgInfo->checkpointId = -1; pMsgInfo->checkpointId = -1;
pMsgInfo->transId = -1; pMsgInfo->transId = -1;
@ -249,7 +249,7 @@ void clearBufferedDispatchMsg(SStreamTask* pTask) {
clearDispatchInfo(pMsgInfo); clearDispatchInfo(pMsgInfo);
taosArrayClear(pTask->msgInfo.pSendInfo); taosArrayClear(pTask->msgInfo.pSendInfo);
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
} }
static SStreamDispatchReq* createDispatchDataReq(SStreamTask* pTask, const SStreamDataBlock* pData) { static SStreamDispatchReq* createDispatchDataReq(SStreamTask* pTask, const SStreamDataBlock* pData) {
@ -424,9 +424,9 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch
} }
static void setNotInDispatchMonitor(SDispatchMsgInfo* pMsgInfo) { static void setNotInDispatchMonitor(SDispatchMsgInfo* pMsgInfo) {
taosThreadMutexLock(&pMsgInfo->lock); streamMutexLock(&pMsgInfo->lock);
pMsgInfo->inMonitor = 0; pMsgInfo->inMonitor = 0;
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
} }
static void setResendInfo(SDispatchEntry* pEntry, int64_t now) { static void setResendInfo(SDispatchEntry* pEntry, int64_t now) {
@ -440,13 +440,13 @@ static void addDispatchEntry(SDispatchMsgInfo* pMsgInfo, int32_t nodeId, int64_t
SDispatchEntry entry = {.nodeId = nodeId, .rspTs = -1, .status = 0, .sendTs = now}; SDispatchEntry entry = {.nodeId = nodeId, .rspTs = -1, .status = 0, .sendTs = now};
if (lock) { if (lock) {
taosThreadMutexLock(&pMsgInfo->lock); streamMutexLock(&pMsgInfo->lock);
} }
taosArrayPush(pMsgInfo->pSendInfo, &entry); (void) taosArrayPush(pMsgInfo->pSendInfo, &entry);
if (lock) { if (lock) {
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
} }
} }
@ -496,16 +496,16 @@ static void doMonitorDispatchData(void* param, void* tmrId) {
return; return;
} }
taosThreadMutexLock(&pMsgInfo->lock); streamMutexLock(&pMsgInfo->lock);
if (pTask->outputq.status == TASK_OUTPUT_STATUS__NORMAL) { if (pTask->outputq.status == TASK_OUTPUT_STATUS__NORMAL) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s not in dispatch procedure, abort from timer, ref:%d", pTask->id.idStr, ref); stDebug("s-task:%s not in dispatch procedure, abort from timer, ref:%d", pTask->id.idStr, ref);
pTask->msgInfo.inMonitor = 0; pTask->msgInfo.inMonitor = 0;
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
return; return;
} }
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
int32_t numOfFailed = getFailedDispatchInfo(pMsgInfo, now); int32_t numOfFailed = getFailedDispatchInfo(pMsgInfo, now);
if (numOfFailed == 0) { if (numOfFailed == 0) {
@ -577,8 +577,9 @@ static void doMonitorDispatchData(void* param, void* tmrId) {
} }
void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) { void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) {
int32_t vgId = pTask->pMeta->vgId;
if (pTask->msgInfo.pRetryTmr != NULL) { if (pTask->msgInfo.pRetryTmr != NULL) {
taosTmrReset(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr); streamTmrReset(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr, vgId, "dispatch-monitor-tmr");
} else { } else {
pTask->msgInfo.pRetryTmr = taosTmrStart(doMonitorDispatchData, waitDuration, pTask, streamTimer); pTask->msgInfo.pRetryTmr = taosTmrStart(doMonitorDispatchData, waitDuration, pTask, streamTimer);
} }
@ -612,7 +613,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
} }
} }
} else { } else {
buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName); (void) buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName);
} }
snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->outputInfo.shuffleDispatcher.dbInfo.db, snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->outputInfo.shuffleDispatcher.dbInfo.db,
@ -624,8 +625,10 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
SBlockName bln = {0}; SBlockName bln = {0};
bln.hashValue = hashValue; bln.hashValue = hashValue;
memcpy(bln.parTbName, pDataBlock->info.parTbName, strlen(pDataBlock->info.parTbName)); memcpy(bln.parTbName, pDataBlock->info.parTbName, strlen(pDataBlock->info.parTbName));
// failed to put into name buffer, no need to do anything
if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) { if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) {
tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName)); (void) tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName));
} }
} }
@ -633,14 +636,14 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
int32_t numOfVgroups = taosArrayGetSize(vgInfo); int32_t numOfVgroups = taosArrayGetSize(vgInfo);
// TODO: optimize search // TODO: optimize search
taosThreadMutexLock(&pTask->msgInfo.lock); streamMutexLock(&pTask->msgInfo.lock);
for (int32_t j = 0; j < numOfVgroups; j++) { for (int32_t j = 0; j < numOfVgroups; j++) {
SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j);
if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) {
if (streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]) < 0) { if (streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]) < 0) {
taosThreadMutexUnlock(&pTask->msgInfo.lock); streamMutexUnlock(&pTask->msgInfo.lock);
return -1; return -1;
} }
@ -655,7 +658,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
} }
} }
taosThreadMutexUnlock(&pTask->msgInfo.lock); streamMutexUnlock(&pTask->msgInfo.lock);
ASSERT(found); ASSERT(found);
return 0; return 0;
} }
@ -690,7 +693,8 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
ASSERT(pTask->msgInfo.pData == NULL); ASSERT(pTask->msgInfo.pData == NULL);
stDebug("s-task:%s start to dispatch msg, set output status:%d", id, pTask->outputq.status); stDebug("s-task:%s start to dispatch msg, set output status:%d", id, pTask->outputq.status);
SStreamDataBlock* pBlock = streamQueueNextItem(pTask->outputq.queue); SStreamDataBlock* pBlock = NULL;
streamQueueNextItem(pTask->outputq.queue, (SStreamQueueItem**)&pBlock);
if (pBlock == NULL) { if (pBlock == NULL) {
atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL); atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL);
stDebug("s-task:%s not dispatch since no elems in outputQ, output status:%d", id, pTask->outputq.status); stDebug("s-task:%s not dispatch since no elems in outputQ, output status:%d", id, pTask->outputq.status);
@ -703,9 +707,9 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
pTask->execInfo.dispatch += 1; pTask->execInfo.dispatch += 1;
taosThreadMutexLock(&pTask->msgInfo.lock); streamMutexLock(&pTask->msgInfo.lock);
initDispatchInfo(&pTask->msgInfo, pTask->execInfo.dispatch); initDispatchInfo(&pTask->msgInfo, pTask->execInfo.dispatch);
taosThreadMutexUnlock(&pTask->msgInfo.lock); streamMutexUnlock(&pTask->msgInfo.lock);
int32_t code = doBuildDispatchMsg(pTask, pBlock); int32_t code = doBuildDispatchMsg(pTask, pBlock);
if (code == 0) { if (code == 0) {
@ -719,7 +723,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
code = sendDispatchMsg(pTask, pTask->msgInfo.pData); code = sendDispatchMsg(pTask, pTask->msgInfo.pData);
taosThreadMutexLock(&pTask->msgInfo.lock); streamMutexLock(&pTask->msgInfo.lock);
if (pTask->msgInfo.inMonitor == 0) { if (pTask->msgInfo.inMonitor == 0) {
int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s start dispatch monitor tmr in %dms, ref:%d, dispatch code:%s", id, DISPATCH_RETRY_INTERVAL_MS, stDebug("s-task:%s start dispatch monitor tmr in %dms, ref:%d, dispatch code:%s", id, DISPATCH_RETRY_INTERVAL_MS,
@ -730,7 +734,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
stDebug("s-task:%s already in dispatch monitor tmr", id); stDebug("s-task:%s already in dispatch monitor tmr", id);
} }
taosThreadMutexUnlock(&pTask->msgInfo.lock); streamMutexUnlock(&pTask->msgInfo.lock);
// this block can not be deleted until it has been sent to downstream task successfully. // this block can not be deleted until it has been sent to downstream task successfully.
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -758,7 +762,7 @@ int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32
buf = rpcMallocCont(sizeof(SMsgHead) + tlen); buf = rpcMallocCont(sizeof(SMsgHead) + tlen);
if (buf == NULL) { if (buf == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId); ((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId);
@ -791,33 +795,33 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
if (++pActiveInfo->sendReadyCheckCounter < 100) { if (++pActiveInfo->sendReadyCheckCounter < 100) {
taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr); streamTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr, vgId, "chkpt-ready-monitor");
return; return;
} }
pActiveInfo->sendReadyCheckCounter = 0; pActiveInfo->sendReadyCheckCounter = 0;
stDebug("s-task:%s in sending checkpoint-ready msg monitor timer", id); stDebug("s-task:%s in sending checkpoint-ready msg monitor timer", id);
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state != TASK_STATUS__CK) { if (pState.state != TASK_STATUS__CK) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s vgId:%d status:%s not in checkpoint, quit from monitor checkpoint-ready send, ref:%d", id, vgId, stDebug("s-task:%s vgId:%d status:%s not in checkpoint, quit from monitor checkpoint-ready send, ref:%d", id, vgId,
pState.name, ref); pState.name, ref);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
return; return;
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
taosThreadMutexLock(&pActiveInfo->lock); streamMutexLock(&pActiveInfo->lock);
SArray* pList = pActiveInfo->pReadyMsgList; SArray* pList = pActiveInfo->pReadyMsgList;
int32_t num = taosArrayGetSize(pList); int32_t num = taosArrayGetSize(pList);
// active checkpoint info is cleared for now // active checkpoint info is cleared for now
if ((pActiveInfo->activeId == 0) && (pActiveInfo->transId == 0) && (num == 0) && (pTask->chkInfo.startTs == 0)) { if ((pActiveInfo->activeId == 0) && (pActiveInfo->transId == 0) && (num == 0) && (pTask->chkInfo.startTs == 0)) {
taosThreadMutexUnlock(&pActiveInfo->lock); streamMutexUnlock(&pActiveInfo->lock);
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stWarn("s-task:%s vgId:%d active checkpoint may be cleared, quit from readyMsg send tmr, ref:%d", id, vgId, ref); stWarn("s-task:%s vgId:%d active checkpoint may be cleared, quit from readyMsg send tmr, ref:%d", id, vgId, ref);
@ -835,7 +839,7 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
continue; continue;
} }
taosArrayPush(pNotRspList, &pInfo->upstreamTaskId); (void) taosArrayPush(pNotRspList, &pInfo->upstreamTaskId);
stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId,
pTask->info.taskLevel, pInfo->upstreamTaskId); pTask->info.taskLevel, pInfo->upstreamTaskId);
} }
@ -852,17 +856,25 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
if (taskId == pReadyInfo->upstreamTaskId) { // send msg again if (taskId == pReadyInfo->upstreamTaskId) { // send msg again
SRpcMsg msg = {0}; SRpcMsg msg = {0};
initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, pReadyInfo->childId, int32_t code = initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, pReadyInfo->childId,
checkpointId, &msg); checkpointId, &msg);
tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg); if (code == TSDB_CODE_SUCCESS) {
code = tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg);
if (code == TSDB_CODE_SUCCESS) {
stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x again", id, pTask->info.taskLevel, stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x again", id, pTask->info.taskLevel,
pReadyInfo->upstreamTaskId); pReadyInfo->upstreamTaskId);
} else {
stError("s-task:%s failed to send checkpoint-ready msg, try nex time in 10s", id);
}
} else {
stError("s-task:%s failed to prepare the checkpoint-ready msg, try nex time in 10s", id);
}
} }
} }
} }
taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr); streamTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr, vgId, "chkpt-ready-monitor");
taosThreadMutexUnlock(&pActiveInfo->lock); streamMutexUnlock(&pActiveInfo->lock);
} else { } else {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug( stDebug(
@ -871,7 +883,7 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
id, vgId, ref); id, vgId, ref);
streamClearChkptReadyMsg(pActiveInfo); streamClearChkptReadyMsg(pActiveInfo);
taosThreadMutexUnlock(&pActiveInfo->lock); streamMutexUnlock(&pActiveInfo->lock);
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
} }
@ -883,9 +895,10 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int32_t vgId = pTask->pMeta->vgId;
SArray* pList = pActiveInfo->pReadyMsgList; SArray* pList = pActiveInfo->pReadyMsgList;
taosThreadMutexLock(&pActiveInfo->lock); streamMutexLock(&pActiveInfo->lock);
int32_t num = taosArrayGetSize(pList); int32_t num = taosArrayGetSize(pList);
ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num); ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num);
@ -894,15 +907,22 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i); STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i);
SRpcMsg msg = {0}; SRpcMsg msg = {0};
initCheckpointReadyMsg(pTask, pInfo->upstreamNodeId, pInfo->upstreamTaskId, pInfo->childId, pInfo->checkpointId, int32_t code = initCheckpointReadyMsg(pTask, pInfo->upstreamNodeId, pInfo->upstreamTaskId, pInfo->childId, pInfo->checkpointId,
&msg); &msg);
tmsgSendReq(&pInfo->upstreamNodeEpset, &msg); if (code == TSDB_CODE_SUCCESS) {
code = tmsgSendReq(&pInfo->upstreamNodeEpset, &msg);
if (code == TSDB_CODE_SUCCESS) {
stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x", id, pTask->info.taskLevel, stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x", id, pTask->info.taskLevel,
pInfo->upstreamTaskId); pInfo->upstreamTaskId);
} else {
stError("s-task:%s failed to send checkpoint-ready msg, try nex time in 10s", id);
}
} else {
stError("s-task:%s failed to prepare the checkpoint-ready msg, try nex time in 10s", id);
}
} }
taosThreadMutexUnlock(&pActiveInfo->lock); streamMutexUnlock(&pActiveInfo->lock);
stDebug("s-task:%s level:%d checkpoint-ready msg sent to all %d upstreams", id, pTask->info.taskLevel, num); stDebug("s-task:%s level:%d checkpoint-ready msg sent to all %d upstreams", id, pTask->info.taskLevel, num);
// start to check if checkpoint ready msg has successfully received by upstream tasks. // start to check if checkpoint ready msg has successfully received by upstream tasks.
@ -914,7 +934,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
if (pActiveInfo->pSendReadyMsgTmr == NULL) { if (pActiveInfo->pSendReadyMsgTmr == NULL) {
pActiveInfo->pSendReadyMsgTmr = taosTmrStart(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer); pActiveInfo->pSendReadyMsgTmr = taosTmrStart(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer);
} else { } else {
taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr); streamTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr, vgId, "chkpt-ready-monitor");
} }
} }
@ -925,7 +945,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) {
SArray* pList = pTask->chkInfo.pActiveInfo->pReadyMsgList; SArray* pList = pTask->chkInfo.pActiveInfo->pReadyMsgList;
taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock); streamMutexLock(&pTask->chkInfo.pActiveInfo->lock);
ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE);
if (taosArrayGetSize(pList) == 1) { if (taosArrayGetSize(pList) == 1) {
@ -940,7 +960,7 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) {
pTask->info.taskLevel); pTask->info.taskLevel);
} }
taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock); streamMutexUnlock(&pTask->chkInfo.pActiveInfo->lock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -978,8 +998,8 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch
payloadLen += sizeof(SRetrieveTableRsp); payloadLen += sizeof(SRetrieveTableRsp);
taosArrayPush(pReq->dataLen, &payloadLen); (void) taosArrayPush(pReq->dataLen, &payloadLen);
taosArrayPush(pReq->data, &buf); (void) taosArrayPush(pReq->data, &buf);
pReq->totalLen += dataStrLen; pReq->totalLen += dataStrLen;
return 0; return 0;
@ -1056,7 +1076,7 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp
void* abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead));
tEncoderInit(&encoder, (uint8_t*)abuf, len); tEncoderInit(&encoder, (uint8_t*)abuf, len);
tEncodeStreamCheckpointSourceRsp(&encoder, &rsp); (void) tEncodeStreamCheckpointSourceRsp(&encoder, &rsp);
tEncoderClear(&encoder); tEncoderClear(&encoder);
initRpcMsg(pMsg, 0, pBuf, sizeof(SMsgHead) + len); initRpcMsg(pMsg, 0, pBuf, sizeof(SMsgHead) + len);
@ -1071,10 +1091,11 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
STaskCheckpointReadyInfo info = { STaskCheckpointReadyInfo info = {
.recvTs = taosGetTimestampMs(), .transId = pReq->transId, .checkpointId = pReq->checkpointId}; .recvTs = taosGetTimestampMs(), .transId = pReq->transId, .checkpointId = pReq->checkpointId};
streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, TSDB_CODE_SUCCESS); // todo retry until it success
(void) streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, TSDB_CODE_SUCCESS);
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
taosThreadMutexLock(&pActiveInfo->lock); streamMutexLock(&pActiveInfo->lock);
int32_t size = taosArrayGetSize(pActiveInfo->pReadyMsgList); int32_t size = taosArrayGetSize(pActiveInfo->pReadyMsgList);
if (size > 0) { if (size > 0) {
@ -1091,15 +1112,15 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
ASSERT(0); // failed to handle it ASSERT(0); // failed to handle it
} }
} else { } else {
taosArrayPush(pActiveInfo->pReadyMsgList, &info); (void) taosArrayPush(pActiveInfo->pReadyMsgList, &info);
stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size + 1); stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size + 1);
} }
taosThreadMutexUnlock(&pActiveInfo->lock); streamMutexUnlock(&pActiveInfo->lock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId, void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId,
int32_t childId, SEpSet* pEpset, int64_t checkpointId) { int32_t childId, SEpSet* pEpset, int64_t checkpointId) {
ASSERT(upstreamTaskId != 0); ASSERT(upstreamTaskId != 0);
@ -1109,8 +1130,6 @@ int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t up
pReadyInfo->recvTs = taosGetTimestampMs(); pReadyInfo->recvTs = taosGetTimestampMs();
pReadyInfo->checkpointId = checkpointId; pReadyInfo->checkpointId = checkpointId;
pReadyInfo->childId = childId; pReadyInfo->childId = childId;
return TSDB_CODE_SUCCESS;
} }
int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, int32_t index, int64_t checkpointId) { int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, int32_t index, int64_t checkpointId) {
@ -1119,9 +1138,9 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
} }
SStreamUpstreamEpInfo* pInfo = NULL; SStreamUpstreamEpInfo* pInfo = NULL;
int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo); streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) { if (pInfo == NULL) {
return code; return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
STaskCheckpointReadyInfo info = {0}; STaskCheckpointReadyInfo info = {0};
@ -1132,8 +1151,8 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
taosThreadMutexLock(&pActiveInfo->lock); streamMutexLock(&pActiveInfo->lock);
taosArrayPush(pActiveInfo->pReadyMsgList, &info); (void) taosArrayPush(pActiveInfo->pReadyMsgList, &info);
int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList); int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList);
int32_t total = streamTaskGetNumOfUpstream(pTask); int32_t total = streamTaskGetNumOfUpstream(pTask);
@ -1145,7 +1164,7 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
stDebug("s-task:%s %d/%d checkpoint-trigger recv", pTask->id.idStr, numOfRecv, total); stDebug("s-task:%s %d/%d checkpoint-trigger recv", pTask->id.idStr, numOfRecv, total);
} }
taosThreadMutexUnlock(&pActiveInfo->lock); streamMutexUnlock(&pActiveInfo->lock);
return 0; return 0;
} }
@ -1187,10 +1206,8 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId
if (delayDispatch) { if (delayDispatch) {
return 0; return 0;
} else { } else {
streamDispatchStreamBlock(pTask); return streamDispatchStreamBlock(pTask);
} }
return 0;
} }
static bool setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t code, int64_t now, int32_t* pNotRsp, const char* id) { static bool setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t code, int64_t now, int32_t* pNotRsp, const char* id) {
@ -1200,7 +1217,7 @@ static bool setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t
bool allRsp = false; bool allRsp = false;
*pNotRsp = 0; *pNotRsp = 0;
taosThreadMutexLock(&pMsgInfo->lock); streamMutexLock(&pMsgInfo->lock);
int32_t numOfDispatchBranch = taosArrayGetSize(pMsgInfo->pSendInfo); int32_t numOfDispatchBranch = taosArrayGetSize(pMsgInfo->pSendInfo);
for(int32_t i = 0; i < numOfDispatchBranch; ++i) { for(int32_t i = 0; i < numOfDispatchBranch; ++i) {
@ -1228,7 +1245,7 @@ static bool setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t
*pNotRsp = numOfDispatchBranch - numOfRsp; *pNotRsp = numOfDispatchBranch - numOfRsp;
allRsp = (numOfRsp == numOfDispatchBranch); allRsp = (numOfRsp == numOfDispatchBranch);
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
ASSERT(updated); ASSERT(updated);
return allRsp; return allRsp;
@ -1240,7 +1257,7 @@ bool isDispatchRspTimeout(SDispatchEntry* pEntry, int64_t now) {
int32_t getFailedDispatchInfo(SDispatchMsgInfo* pMsgInfo, int64_t now) { int32_t getFailedDispatchInfo(SDispatchMsgInfo* pMsgInfo, int64_t now) {
int32_t numOfFailed = 0; int32_t numOfFailed = 0;
taosThreadMutexLock(&pMsgInfo->lock); streamMutexLock(&pMsgInfo->lock);
for (int32_t j = 0; j < taosArrayGetSize(pMsgInfo->pSendInfo); ++j) { for (int32_t j = 0; j < taosArrayGetSize(pMsgInfo->pSendInfo); ++j) {
SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, j); SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, j);
@ -1248,7 +1265,7 @@ int32_t getFailedDispatchInfo(SDispatchMsgInfo* pMsgInfo, int64_t now) {
numOfFailed += 1; numOfFailed += 1;
} }
} }
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
return numOfFailed; return numOfFailed;
} }
@ -1260,9 +1277,9 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
bool allRsp = false; bool allRsp = false;
int32_t notRsp = 0; int32_t notRsp = 0;
taosThreadMutexLock(&pMsgInfo->lock); streamMutexLock(&pMsgInfo->lock);
int32_t msgId = pMsgInfo->msgId; int32_t msgId = pMsgInfo->msgId;
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
// follower not handle the dispatch rsp // follower not handle the dispatch rsp
if ((pTask->pMeta->role == NODE_ROLE_FOLLOWER) || (pTask->status.downstreamReady != 1)) { if ((pTask->pMeta->role == NODE_ROLE_FOLLOWER) || (pTask->status.downstreamReady != 1)) {
@ -1315,7 +1332,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
{ {
bool delayDispatch = (pMsgInfo->dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); bool delayDispatch = (pMsgInfo->dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER);
if (delayDispatch) { if (delayDispatch) {
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
// we only set the dispatch msg info for current checkpoint trans // we only set the dispatch msg info for current checkpoint trans
if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK && if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK &&
pTask->chkInfo.pActiveInfo->activeId == pMsgInfo->checkpointId) { pTask->chkInfo.pActiveInfo->activeId == pMsgInfo->checkpointId) {
@ -1329,7 +1346,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
" transId:%d discard, since expired", " transId:%d discard, since expired",
pTask->id.idStr, pMsgInfo->checkpointId, pMsgInfo->transId); pTask->id.idStr, pMsgInfo->checkpointId, pMsgInfo->transId);
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
} }
} }
} }
@ -1374,12 +1391,12 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
// now ready for next data output // now ready for next data output
atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL); atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL);
} else { } else {
handleDispatchSuccessRsp(pTask, pRsp->downstreamTaskId, pRsp->downstreamNodeId); code = handleDispatchSuccessRsp(pTask, pRsp->downstreamTaskId, pRsp->downstreamNodeId);
} }
} }
} }
return 0; return code;
} }
static int32_t buildDispatchRsp(const SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t status, void** pBuf) { static int32_t buildDispatchRsp(const SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t status, void** pBuf) {
@ -1438,9 +1455,9 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->totalLen, pReq->msgId); pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->totalLen, pReq->msgId);
SStreamUpstreamEpInfo* pInfo = NULL; SStreamUpstreamEpInfo* pInfo = NULL;
int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId, &pInfo); streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) { if (pInfo == NULL) {
return code; return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
if (pMeta->role == NODE_ROLE_FOLLOWER) { if (pMeta->role == NODE_ROLE_FOLLOWER) {
@ -1474,7 +1491,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
{ {
// do send response with the input status // do send response with the input status
code = buildDispatchRsp(pTask, pReq, status, &pRsp->pCont); int32_t code = buildDispatchRsp(pTask, pReq, status, &pRsp->pCont);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed to build dispatch rsp, msgId:%d, code:%s", id, pReq->msgId, tstrerror(code)); stError("s-task:%s failed to build dispatch rsp, msgId:%d, code:%s", id, pReq->msgId, tstrerror(code));
return code; return code;
@ -1484,6 +1501,5 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
tmsgSendRsp(pRsp); tmsgSendRsp(pRsp);
} }
streamTrySchedExec(pTask); return streamTrySchedExec(pTask);
return code;
} }

132
source/libs/stream/src/streamExec.c
View File

@ -24,7 +24,7 @@
#define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec #define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec
static int32_t streamTransferStateDoPrepare(SStreamTask* pTask); static int32_t streamTransferStateDoPrepare(SStreamTask* pTask);
static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks); static void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks);
bool streamTaskShouldStop(const SStreamTask* pTask) { bool streamTaskShouldStop(const SStreamTask* pTask) {
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
@ -52,8 +52,9 @@ static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBl
return code; return code;
} }
// not handle error, if dispatch failed, try next time.
// checkpoint trigger will be checked // checkpoint trigger will be checked
streamDispatchStreamBlock(pTask); (void) streamDispatchStreamBlock(pTask);
} }
return code; return code;
@ -90,7 +91,7 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray*
return code; return code;
} }
int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) { void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
void* pExecutor = pTask->exec.pExecutor; void* pExecutor = pTask->exec.pExecutor;
@ -108,7 +109,7 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t*
if (streamTaskShouldStop(pTask)) { if (streamTaskShouldStop(pTask)) {
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
return 0; return;
} }
SSDataBlock* output = NULL; SSDataBlock* output = NULL;
@ -128,11 +129,16 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t*
const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem; const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem;
ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1);
assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); (void) assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0));
block.info.type = STREAM_PULL_OVER; block.info.type = STREAM_PULL_OVER;
block.info.childId = pTask->info.selfChildId; block.info.childId = pTask->info.selfChildId;
taosArrayPush(pRes, &block);
void* p = taosArrayPush(pRes, &block);
if (p != NULL) {
numOfBlocks += 1; numOfBlocks += 1;
} else {
stError("s-task:%s failed to add retrieve block", pTask->id.idStr);
}
stDebug("s-task:%s(child %d) retrieve process completed, reqId:0x%" PRIx64 " dump results", pTask->id.idStr, stDebug("s-task:%s(child %d) retrieve process completed, reqId:0x%" PRIx64 " dump results", pTask->id.idStr,
pTask->info.selfChildId, pRetrieveBlock->reqId); pTask->info.selfChildId, pRetrieveBlock->reqId);
@ -151,13 +157,21 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t*
} }
SSDataBlock block = {0}; SSDataBlock block = {0};
assignOneDataBlock(&block, output); code = assignOneDataBlock(&block, output);
if (code) {
stError("s-task:%s failed to build result block due to out of memory", pTask->id.idStr);
continue;
}
block.info.childId = pTask->info.selfChildId; block.info.childId = pTask->info.selfChildId;
size += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block); size += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block);
numOfBlocks += 1; numOfBlocks += 1;
taosArrayPush(pRes, &block); void* p = taosArrayPush(pRes, &block);
if (p == NULL) {
stError("s-task:%s failed to add computing results, the final res may be incorrect", pTask->id.idStr);
}
stDebug("s-task:%s (child %d) executed and get %d result blocks, size:%.2fMiB", pTask->id.idStr, stDebug("s-task:%s (child %d) executed and get %d result blocks, size:%.2fMiB", pTask->id.idStr,
pTask->info.selfChildId, numOfBlocks, SIZE_IN_MiB(size)); pTask->info.selfChildId, numOfBlocks, SIZE_IN_MiB(size));
@ -166,8 +180,9 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t*
if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || size >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) { if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || size >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) {
ASSERT(numOfBlocks == taosArrayGetSize(pRes)); ASSERT(numOfBlocks == taosArrayGetSize(pRes));
code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks); code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks);
// todo: here we need continue retry to put it into output buffer
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return;
} }
pRes = NULL; pRes = NULL;
@ -182,16 +197,16 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t*
} else { } else {
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
} }
return code;
} }
static int32_t handleSanhistoryResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) { // todo contiuous try to create result blocks
static int32_t handleScanhistoryResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (taosArrayGetSize(pRes) > 0) { if (taosArrayGetSize(pRes) > 0) {
SStreamDataBlock* pStreamBlocks = NULL; SStreamDataBlock* pStreamBlocks = NULL;
code = createStreamBlockFromResults(NULL, pTask, size, pRes, &pStreamBlocks); code = createStreamBlockFromResults(NULL, pTask, size, pRes, &pStreamBlocks);
if (code) { if (code) {
stError("s-task:%s failed to build history result blocks", pTask->id.idStr);
return code; return code;
} }
@ -236,9 +251,13 @@ static void streamScanHistoryDataImpl(SStreamTask* pTask, SArray* pRes, int32_t*
} }
SSDataBlock block = {0}; SSDataBlock block = {0};
assignOneDataBlock(&block, output); (void) assignOneDataBlock(&block, output);
block.info.childId = pTask->info.selfChildId; block.info.childId = pTask->info.selfChildId;
taosArrayPush(pRes, &block);
void* p = taosArrayPush(pRes, &block);
if (p == NULL) {
stError("s-task:%s failed to add computing results, the final res may be incorrect", pTask->id.idStr);
}
(*pSize) += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block); (*pSize) += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block);
numOfBlocks += 1; numOfBlocks += 1;
@ -264,7 +283,7 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) {
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
if (!pTask->hTaskInfo.operatorOpen) { if (!pTask->hTaskInfo.operatorOpen) {
qSetStreamOpOpen(exec); (void) qSetStreamOpOpen(exec);
pTask->hTaskInfo.operatorOpen = true; pTask->hTaskInfo.operatorOpen = true;
} }
@ -301,8 +320,8 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) {
return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0); return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0);
} }
// dispatch the generated results // dispatch the generated results, todo fix error
/*int32_t code = */handleSanhistoryResultBlocks(pTask, pRes, size); (void) handleScanhistoryResultBlocks(pTask, pRes, size);
if (finished) { if (finished) {
return buildScanhistoryExecRet(TASK_SCANHISTORY_CONT, 0); return buildScanhistoryExecRet(TASK_SCANHISTORY_CONT, 0);
@ -323,14 +342,15 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
SStreamTask* pStreamTask = NULL; SStreamTask* pStreamTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId, &pStreamTask); int32_t code = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId, &pStreamTask);
if (pStreamTask == NULL) { if (pStreamTask == NULL || code != TSDB_CODE_SUCCESS) {
stError( stError(
"s-task:%s failed to find related stream task:0x%x, it may have been destroyed or closed, destroy the related " "s-task:%s failed to find related stream task:0x%x, it may have been destroyed or closed, destroy the related "
"fill-history task", "fill-history task",
id, (int32_t) pTask->streamTaskId.taskId); id, (int32_t) pTask->streamTaskId.taskId);
// 1. free it and remove fill-history task from disk meta-store // 1. free it and remove fill-history task from disk meta-store
streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0); // todo: this function should never be failed.
(void) streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0);
// 2. save to disk // 2. save to disk
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
@ -388,14 +408,14 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN, pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN,
pTimeWindow->ekey, p, pStreamTask->status.schedStatus); pTimeWindow->ekey, p, pStreamTask->status.schedStatus);
streamTaskResetTimewindowFilter(pStreamTask); (void) streamTaskResetTimewindowFilter(pStreamTask);
} else { } else {
stDebug("s-task:%s no need to update/reset filter time window for non-source tasks", pStreamTask->id.idStr); stDebug("s-task:%s no need to update/reset filter time window for non-source tasks", pStreamTask->id.idStr);
} }
// NOTE: transfer the ownership of executor state before handle the checkpoint block during stream exec // NOTE: transfer the ownership of executor state before handle the checkpoint block during stream exec
// 2. send msg to mnode to launch a checkpoint to keep the state for current stream // 2. send msg to mnode to launch a checkpoint to keep the state for current stream
streamTaskSendCheckpointReq(pStreamTask); (void) streamTaskSendCheckpointReq(pStreamTask);
// 3. assign the status to the value that will be kept in disk // 3. assign the status to the value that will be kept in disk
pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask).state; pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask).state;
@ -409,7 +429,7 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
static int32_t haltCallback(SStreamTask* pTask, void* param) { static int32_t haltCallback(SStreamTask* pTask, void* param) {
streamTaskOpenAllUpstreamInput(pTask); streamTaskOpenAllUpstreamInput(pTask);
streamTaskSendCheckpointReq(pTask); (void) streamTaskSendCheckpointReq(pTask);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -445,18 +465,19 @@ int32_t streamTransferStatePrepare(SStreamTask* pTask) {
} }
// set input // set input
static void doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_t* pVer, const char* id) { static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_t* pVer, const char* id) {
void* pExecutor = pTask->exec.pExecutor; void* pExecutor = pTask->exec.pExecutor;
int32_t code = 0;
const SStreamQueueItem* pItem = pInput; const SStreamQueueItem* pItem = pInput;
if (pItem->type == STREAM_INPUT__GET_RES) { if (pItem->type == STREAM_INPUT__GET_RES) {
const SStreamTrigger* pTrigger = (const SStreamTrigger*)pInput; const SStreamTrigger* pTrigger = (const SStreamTrigger*)pInput;
qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); code = qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
} else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE);
const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput; const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput;
qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); code = qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT);
stDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, id, pSubmit, stDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, id, pSubmit,
pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver);
ASSERT((*pVer) <= pSubmit->submit.ver); ASSERT((*pVer) <= pSubmit->submit.ver);
@ -468,7 +489,7 @@ static void doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_
SArray* pBlockList = pBlock->blocks; SArray* pBlockList = pBlock->blocks;
int32_t numOfBlocks = taosArrayGetSize(pBlockList); int32_t numOfBlocks = taosArrayGetSize(pBlockList);
stDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, id, numOfBlocks, pBlock->sourceVer); stDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, id, numOfBlocks, pBlock->sourceVer);
qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK); code = qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK);
} else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) {
const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)pInput; const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)pInput;
@ -477,24 +498,26 @@ static void doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_
int32_t numOfBlocks = taosArrayGetSize(pBlockList); int32_t numOfBlocks = taosArrayGetSize(pBlockList);
stDebug("s-task:%s %p set (merged) submit blocks as a batch, numOfBlocks:%d, ver:%" PRId64, id, pTask, numOfBlocks, stDebug("s-task:%s %p set (merged) submit blocks as a batch, numOfBlocks:%d, ver:%" PRId64, id, pTask, numOfBlocks,
pMerged->ver); pMerged->ver);
qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); code = qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT);
ASSERT((*pVer) <= pMerged->ver); ASSERT((*pVer) <= pMerged->ver);
(*pVer) = pMerged->ver; (*pVer) = pMerged->ver;
} else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) {
const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)pInput; const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)pInput;
qSetMultiStreamInput(pExecutor, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK); code = qSetMultiStreamInput(pExecutor, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
} else if (pItem->type == STREAM_INPUT__CHECKPOINT || pItem->type == STREAM_INPUT__CHECKPOINT_TRIGGER) { } else if (pItem->type == STREAM_INPUT__CHECKPOINT || pItem->type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
const SStreamDataBlock* pCheckpoint = (const SStreamDataBlock*)pInput; const SStreamDataBlock* pCheckpoint = (const SStreamDataBlock*)pInput;
qSetMultiStreamInput(pExecutor, pCheckpoint->blocks, 1, pItem->type); code = qSetMultiStreamInput(pExecutor, pCheckpoint->blocks, 1, pItem->type);
} else { } else {
ASSERT(0); ASSERT(0);
} }
return code;
} }
int32_t streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { void streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) {
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t level = pTask->info.taskLevel; int32_t level = pTask->info.taskLevel;
@ -505,7 +528,7 @@ int32_t streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBloc
if (remain > 0) { if (remain > 0) {
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
stDebug("s-task:%s receive upstream trans-state msg, not sent remain:%d", id, remain); stDebug("s-task:%s receive upstream trans-state msg, not sent remain:%d", id, remain);
return 0; return;
} }
} }
@ -526,7 +549,7 @@ int32_t streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBloc
pBlock->srcVgId = pTask->pMeta->vgId; pBlock->srcVgId = pTask->pMeta->vgId;
code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock); code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock);
if (code == 0) { if (code == 0) {
streamDispatchStreamBlock(pTask); (void) streamDispatchStreamBlock(pTask);
} else { // todo put into queue failed, retry } else { // todo put into queue failed, retry
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
} }
@ -540,11 +563,9 @@ int32_t streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBloc
code = streamTransferStatePrepare(pTask); code = streamTransferStatePrepare(pTask);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
/*int8_t status = */ streamTaskSetSchedStatusInactive(pTask); (void)streamTaskSetSchedStatusInactive(pTask);
} }
} }
return code;
} }
//static void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; } //static void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; }
@ -559,7 +580,11 @@ static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, i
stDebug("s-task:%s start to process batch blocks, num:%d, type:%s", id, num, streamQueueItemGetTypeStr(pBlock->type)); stDebug("s-task:%s start to process batch blocks, num:%d, type:%s", id, num, streamQueueItemGetTypeStr(pBlock->type));
doSetStreamInputBlock(pTask, pBlock, &ver, id); int32_t code = doSetStreamInputBlock(pTask, pBlock, &ver, id);
if(code) {
stError("s-task:%s failed to set input block, not exec for these blocks", id);
return;
}
int64_t totalSize = 0; int64_t totalSize = 0;
int32_t totalBlocks = 0; int32_t totalBlocks = 0;
@ -601,9 +626,9 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB
STaskId* pHTaskId = &pTask->hTaskInfo.id; STaskId* pHTaskId = &pTask->hTaskInfo.id;
SStreamTask* pHTask = NULL; SStreamTask* pHTask = NULL;
int32_t code = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask); int32_t code = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask);
if (pHTask != NULL) { if (code == TSDB_CODE_SUCCESS) { // ignore the error code.
streamTaskReleaseState(pHTask); (void) streamTaskReleaseState(pHTask);
streamTaskReloadState(pTask); (void) streamTaskReloadState(pTask);
stDebug("s-task:%s transfer state from fill-history task:%s, status:%s completed", id, pHTask->id.idStr, stDebug("s-task:%s transfer state from fill-history task:%s, status:%s completed", id, pHTask->id.idStr,
streamTaskGetStatus(pHTask).name); streamTaskGetStatus(pHTask).name);
@ -676,7 +701,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
// dispatch checkpoint msg to all downstream tasks // dispatch checkpoint msg to all downstream tasks
int32_t type = pInput->type; int32_t type = pInput->type;
if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) { if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput); (void) streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput);
continue; continue;
} }
@ -694,7 +719,10 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
if (type == STREAM_INPUT__DATA_BLOCK) { if (type == STREAM_INPUT__DATA_BLOCK) {
pTask->execInfo.sink.dataSize += blockSize; pTask->execInfo.sink.dataSize += blockSize;
stDebug("s-task:%s sink task start to sink %d blocks, size:%.2fKiB", id, numOfBlocks, SIZE_IN_KiB(blockSize)); stDebug("s-task:%s sink task start to sink %d blocks, size:%.2fKiB", id, numOfBlocks, SIZE_IN_KiB(blockSize));
doOutputResultBlockImpl(pTask, (SStreamDataBlock*)pInput); int32_t code = doOutputResultBlockImpl(pTask, (SStreamDataBlock*)pInput);
if (code != TSDB_CODE_SUCCESS) {
// todo handle error.
}
double el = (taosGetTimestampMs() - st) / 1000.0; double el = (taosGetTimestampMs() - st) / 1000.0;
if (fabs(el - 0.0) <= DBL_EPSILON) { if (fabs(el - 0.0) <= DBL_EPSILON) {
@ -712,11 +740,11 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
streamFreeQitem(pInput); streamFreeQitem(pInput);
} else { // todo other thread may change the status } else { // todo other thread may change the status
// do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed.
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state == TASK_STATUS__CK) { if (pState.state == TASK_STATUS__CK) {
stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name); stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name);
streamTaskBuildCheckpoint(pTask); (void) streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue
} else { // todo refactor } else { // todo refactor
int32_t code = 0; int32_t code = 0;
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
@ -732,7 +760,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
} }
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamFreeQitem(pInput); streamFreeQitem(pInput);
return 0; return 0;
} }
@ -765,21 +793,21 @@ bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) {
} }
} }
int32_t streamResumeTask(SStreamTask* pTask) { void streamResumeTask(SStreamTask* pTask) {
ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__ACTIVE); ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__ACTIVE);
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
while (1) { while (1) {
/*int32_t code = */ doStreamExecTask(pTask); (void) doStreamExecTask(pTask);
// check if continue // check if continue
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue); int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue);
if ((numOfItems == 0) || streamTaskShouldStop(pTask) || streamTaskShouldPause(pTask)) { if ((numOfItems == 0) || streamTaskShouldStop(pTask) || streamTaskShouldPause(pTask)) {
atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE);
streamTaskClearSchedIdleInfo(pTask); streamTaskClearSchedIdleInfo(pTask);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
setLastExecTs(pTask, taosGetTimestampMs()); setLastExecTs(pTask, taosGetTimestampMs());
@ -787,19 +815,19 @@ int32_t streamResumeTask(SStreamTask* pTask) {
stDebug("s-task:%s exec completed, status:%s, sched-status:%d, lastExecTs:%" PRId64, id, p, stDebug("s-task:%s exec completed, status:%s, sched-status:%d, lastExecTs:%" PRId64, id, p,
pTask->status.schedStatus, pTask->status.lastExecTs); pTask->status.schedStatus, pTask->status.lastExecTs);
return 0; return;
} else { } else {
// check if this task needs to be idle for a while // check if this task needs to be idle for a while
if (pTask->status.schedIdleTime > 0) { if (pTask->status.schedIdleTime > 0) {
streamTaskResumeInFuture(pTask); streamTaskResumeInFuture(pTask);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
setLastExecTs(pTask, taosGetTimestampMs()); setLastExecTs(pTask, taosGetTimestampMs());
return 0; return;
} }
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
} }
} }

12
source/libs/stream/src/streamHb.c
View File

@ -54,7 +54,7 @@ static bool existInHbMsg(SStreamHbMsg* pMsg, SDownstreamTaskEpset* pTaskEpset) {
static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) { static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) {
SStreamMeta* pMeta = pTask->pMeta; SStreamMeta* pMeta = pTask->pMeta;
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList); int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList);
for (int j = 0; j < num; ++j) { for (int j = 0; j < num; ++j) {
@ -73,7 +73,7 @@ static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) {
} }
taosArrayClear(pTask->outputInfo.pNodeEpsetUpdateList); taosArrayClear(pTask->outputInfo.pNodeEpsetUpdateList);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
} }
static int32_t doSendHbMsgInfo(SStreamHbMsg* pMsg, SStreamMeta* pMeta, SEpSet* pEpset) { static int32_t doSendHbMsgInfo(SStreamHbMsg* pMsg, SStreamMeta* pMeta, SEpSet* pEpset) {
@ -170,9 +170,9 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) {
continue; continue;
} }
(void) taosThreadMutexLock(&(*pTask)->lock); streamMutexLock(&(*pTask)->lock);
STaskStatusEntry entry = streamTaskGetStatusEntry(*pTask); STaskStatusEntry entry = streamTaskGetStatusEntry(*pTask);
(void) taosThreadMutexUnlock(&(*pTask)->lock); streamMutexUnlock(&(*pTask)->lock);
entry.inputRate = entry.inputQUsed * 100.0 / (2 * STREAM_TASK_QUEUE_CAPACITY_IN_SIZE); entry.inputRate = entry.inputQUsed * 100.0 / (2 * STREAM_TASK_QUEUE_CAPACITY_IN_SIZE);
if ((*pTask)->info.taskLevel == TASK_LEVEL__SINK) { if ((*pTask)->info.taskLevel == TASK_LEVEL__SINK) {
@ -190,9 +190,9 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) {
stInfo("s-task:%s set kill checkpoint trans in hbMsg, transId:%d, clear the active checkpointInfo", stInfo("s-task:%s set kill checkpoint trans in hbMsg, transId:%d, clear the active checkpointInfo",
(*pTask)->id.idStr, p->transId); (*pTask)->id.idStr, p->transId);
(void) taosThreadMutexLock(&(*pTask)->lock); streamMutexLock(&(*pTask)->lock);
streamTaskClearCheckInfo((*pTask), true); streamTaskClearCheckInfo((*pTask), true);
(void) taosThreadMutexUnlock(&(*pTask)->lock); streamMutexUnlock(&(*pTask)->lock);
} }
} }

57
source/libs/stream/src/streamMeta.c
View File

@ -57,10 +57,13 @@ static void streamMetaEnvInit() {
streamMetaId = taosOpenRef(64, streamMetaCloseImpl); streamMetaId = taosOpenRef(64, streamMetaCloseImpl);
metaRefMgtInit(); metaRefMgtInit();
streamTimerInit(); int32_t code = streamTimerInit();
if (code != 0) {
stError("failed to init stream meta env, start failed");
}
} }
void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } void streamMetaInit() { (void) taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); }
void streamMetaCleanup() { void streamMetaCleanup() {
taosCloseRef(streamBackendId); taosCloseRef(streamBackendId);
@ -89,11 +92,11 @@ void metaRefMgtCleanup() {
} }
taosHashCleanup(gMetaRefMgt.pTable); taosHashCleanup(gMetaRefMgt.pTable);
taosThreadMutexDestroy(&gMetaRefMgt.mutex); streamMutexDestroy(&gMetaRefMgt.mutex);
} }
int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) { int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) {
taosThreadMutexLock(&gMetaRefMgt.mutex); streamMutexLock(&gMetaRefMgt.mutex);
void* p = taosHashGet(gMetaRefMgt.pTable, &vgId, sizeof(vgId)); void* p = taosHashGet(gMetaRefMgt.pTable, &vgId, sizeof(vgId));
if (p == NULL) { if (p == NULL) {
@ -105,7 +108,7 @@ int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) {
taosArrayPush(list, &rid); taosArrayPush(list, &rid);
} }
taosThreadMutexUnlock(&gMetaRefMgt.mutex); streamMutexUnlock(&gMetaRefMgt.mutex);
return 0; return 0;
} }
@ -238,7 +241,7 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) {
int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key) { int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key) {
int64_t chkpId = pTask->chkInfo.checkpointId; int64_t chkpId = pTask->chkInfo.checkpointId;
taosThreadMutexLock(&pMeta->backendMutex); streamMutexLock(&pMeta->backendMutex);
void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key)); void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key));
if ((ppBackend != NULL) && (*ppBackend != NULL)) { if ((ppBackend != NULL) && (*ppBackend != NULL)) {
taskDbAddRef(*ppBackend); taskDbAddRef(*ppBackend);
@ -247,7 +250,7 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
pBackend->pMeta = pMeta; pBackend->pMeta = pMeta;
pTask->pBackend = pBackend; pTask->pBackend = pBackend;
taosThreadMutexUnlock(&pMeta->backendMutex); streamMutexUnlock(&pMeta->backendMutex);
stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend); stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend);
return 0; return 0;
} }
@ -260,11 +263,11 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
break; break;
} }
taosThreadMutexUnlock(&pMeta->backendMutex); streamMutexUnlock(&pMeta->backendMutex);
taosMsleep(1000); taosMsleep(1000);
stDebug("backend held by other task, restart later, path:%s, key:%s", pMeta->path, key); stDebug("backend held by other task, restart later, path:%s, key:%s", pMeta->path, key);
taosThreadMutexLock(&pMeta->backendMutex); streamMutexLock(&pMeta->backendMutex);
} }
int64_t tref = taosAddRef(taskDbWrapperId, pBackend); int64_t tref = taosAddRef(taskDbWrapperId, pBackend);
@ -276,7 +279,7 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
if (processVer != -1) pTask->chkInfo.processedVer = processVer; if (processVer != -1) pTask->chkInfo.processedVer = processVer;
taosHashPut(pMeta->pTaskDbUnique, key, strlen(key), &pBackend, sizeof(void*)); taosHashPut(pMeta->pTaskDbUnique, key, strlen(key), &pBackend, sizeof(void*));
taosThreadMutexUnlock(&pMeta->backendMutex); streamMutexUnlock(&pMeta->backendMutex);
stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend); stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend);
return 0; return 0;
@ -286,10 +289,10 @@ void streamMetaRemoveDB(void* arg, char* key) {
if (arg == NULL || key == NULL) return; if (arg == NULL || key == NULL) return;
SStreamMeta* pMeta = arg; SStreamMeta* pMeta = arg;
taosThreadMutexLock(&pMeta->backendMutex); streamMutexLock(&pMeta->backendMutex);
taosHashRemove(pMeta->pTaskDbUnique, key, strlen(key)); taosHashRemove(pMeta->pTaskDbUnique, key, strlen(key));
taosThreadMutexUnlock(&pMeta->backendMutex); streamMutexUnlock(&pMeta->backendMutex);
} }
int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, FTaskExpand expandTaskFn, int32_t vgId, int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, FTaskExpand expandTaskFn, int32_t vgId,
@ -541,7 +544,7 @@ void streamMetaCloseImpl(void* arg) {
pMeta->pHbInfo = NULL; pMeta->pHbInfo = NULL;
taosMemoryFree(pMeta->path); taosMemoryFree(pMeta->path);
taosThreadMutexDestroy(&pMeta->backendMutex); streamMutexDestroy(&pMeta->backendMutex);
taosCleanUpScheduler(pMeta->qHandle); taosCleanUpScheduler(pMeta->qHandle);
taosMemoryFree(pMeta->qHandle); taosMemoryFree(pMeta->qHandle);
@ -739,7 +742,10 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
streamMetaRLock(pMeta); streamMetaRLock(pMeta);
ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (ppTask) { if (ppTask) {
// to make sure check status will not start the check downstream status when we start to check timerActive count.
streamMutexLock(&pTask->taskCheckInfo.checkInfoLock);
timerActive = (*ppTask)->status.timerActive; timerActive = (*ppTask)->status.timerActive;
streamMutexUnlock(&pTask->taskCheckInfo.checkInfoLock);
} }
streamMetaRUnLock(pMeta); streamMetaRUnLock(pMeta);
@ -1132,7 +1138,7 @@ int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) {
continue; continue;
} }
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state == TASK_STATUS__CK) { if (pState.state == TASK_STATUS__CK) {
@ -1141,7 +1147,7 @@ int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) {
stDebug("s-task:%s status:%s not reset the checkpoint", pTask->id.idStr, pState.name); stDebug("s-task:%s status:%s not reset the checkpoint", pTask->id.idStr, pState.name);
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
} }
@ -1271,7 +1277,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) {
streamLaunchFillHistoryTask(pTask); streamLaunchFillHistoryTask(pTask);
} }
streamMetaAddTaskLaunchResult(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->checkTs, pInfo->readyTs, true); (void) streamMetaAddTaskLaunchResult(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->checkTs, pInfo->readyTs, true);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
continue; continue;
} }
@ -1388,16 +1394,16 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas
ASSERT(pTask->status.downstreamReady == 0); ASSERT(pTask->status.downstreamReady == 0);
// avoid initialization and destroy running concurrently. // avoid initialization and destroy running concurrently.
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pTask->pBackend == NULL) { if (pTask->pBackend == NULL) {
code = pMeta->expandTaskFn(pTask); code = pMeta->expandTaskFn(pTask);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs); streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs);
} }
} else { } else {
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
} }
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
@ -1440,7 +1446,6 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3
STaskId id = {.streamId = streamId, .taskId = taskId}; STaskId id = {.streamId = streamId, .taskId = taskId};
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
SStreamTask** p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); SStreamTask** p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (p == NULL) { // task does not exists in current vnode, not record the complete info if (p == NULL) { // task does not exists in current vnode, not record the complete info
stError("vgId:%d s-task:0x%x not exists discard the check downstream info", pMeta->vgId, taskId); stError("vgId:%d s-task:0x%x not exists discard the check downstream info", pMeta->vgId, taskId);
@ -1449,9 +1454,9 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3
} }
// clear the send consensus-checkpointId flag // clear the send consensus-checkpointId flag
taosThreadMutexLock(&(*p)->lock); streamMutexLock(&(*p)->lock);
(*p)->status.sendConsensusChkptId = false; (*p)->status.sendConsensusChkptId = false;
taosThreadMutexUnlock(&(*p)->lock); streamMutexUnlock(&(*p)->lock);
if (pStartInfo->startAllTasks != 1) { if (pStartInfo->startAllTasks != 1) {
int64_t el = endTs - startTs; int64_t el = endTs - startTs;
@ -1537,9 +1542,9 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t ta
streamMetaRUnLock(pMeta); streamMetaRUnLock(pMeta);
// add the failed task info, along with the related fill-history task info into tasks list. // add the failed task info, along with the related fill-history task info into tasks list.
streamMetaAddTaskLaunchResult(pMeta, streamId, taskId, startTs, now, false); (void) streamMetaAddTaskLaunchResult(pMeta, streamId, taskId, startTs, now, false);
if (hasFillhistoryTask) { if (hasFillhistoryTask) {
streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false); (void) streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false);
} }
} else { } else {
streamMetaRUnLock(pMeta); streamMetaRUnLock(pMeta);
@ -1554,12 +1559,12 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t ta
void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs) { void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs) {
int32_t startTs = pTask->execInfo.checkTs; int32_t startTs = pTask->execInfo.checkTs;
streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, startTs, failedTs, false); (void) streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, startTs, failedTs, false);
// automatically set the related fill-history task to be failed. // automatically set the related fill-history task to be failed.
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
STaskId* pId = &pTask->hTaskInfo.id; STaskId* pId = &pTask->hTaskInfo.id;
streamMetaAddTaskLaunchResult(pTask->pMeta, pId->streamId, pId->taskId, startTs, failedTs, false); (void) streamMetaAddTaskLaunchResult(pTask->pMeta, pId->streamId, pId->taskId, startTs, failedTs, false);
} }
} }

61
source/libs/stream/src/streamQueue.c
View File

@ -27,20 +27,24 @@ typedef struct SQueueReader {
int32_t waitDuration; // maximum wait time to format several block into a batch to process, unit: ms int32_t waitDuration; // maximum wait time to format several block into a batch to process, unit: ms
} SQueueReader; } SQueueReader;
#define streamQueueCurItem(_q) ((_q)->qItem)
static bool streamTaskExtractAvailableToken(STokenBucket* pBucket, const char* id); static bool streamTaskExtractAvailableToken(STokenBucket* pBucket, const char* id);
static void streamTaskPutbackToken(STokenBucket* pBucket); static void streamTaskPutbackToken(STokenBucket* pBucket);
static void streamTaskConsumeQuota(STokenBucket* pBucket, int32_t bytes); static void streamTaskConsumeQuota(STokenBucket* pBucket, int32_t bytes);
static void streamQueueCleanup(SStreamQueue* pQueue) { static void streamQueueCleanup(SStreamQueue* pQueue) {
void* qItem = NULL; SStreamQueueItem* qItem = NULL;
while ((qItem = streamQueueNextItem(pQueue)) != NULL) { while (1) {
streamQueueNextItem(pQueue, &qItem);
if (qItem == NULL) {
break;
}
streamFreeQitem(qItem); streamFreeQitem(qItem);
} }
pQueue->status = STREAM_QUEUE__SUCESS; pQueue->status = STREAM_QUEUE__SUCESS;
} }
static void* streamQueueCurItem(SStreamQueue* queue) { return queue->qItem; }
int32_t streamQueueOpen(int64_t cap, SStreamQueue** pQ) { int32_t streamQueueOpen(int64_t cap, SStreamQueue** pQ) {
*pQ = NULL; *pQ = NULL;
int32_t code = 0; int32_t code = 0;
@ -81,21 +85,22 @@ void streamQueueClose(SStreamQueue* pQueue, int32_t taskId) {
taosMemoryFree(pQueue); taosMemoryFree(pQueue);
} }
void* streamQueueNextItem(SStreamQueue* pQueue) { void streamQueueNextItem(SStreamQueue* pQueue, SStreamQueueItem** pItem) {
*pItem = NULL;
int8_t flag = atomic_exchange_8(&pQueue->status, STREAM_QUEUE__PROCESSING); int8_t flag = atomic_exchange_8(&pQueue->status, STREAM_QUEUE__PROCESSING);
if (flag == STREAM_QUEUE__FAILED) { if (flag == STREAM_QUEUE__FAILED) {
ASSERT(pQueue->qItem != NULL); ASSERT(pQueue->qItem != NULL);
return streamQueueCurItem(pQueue); *pItem = streamQueueCurItem(pQueue);
} else { } else {
pQueue->qItem = NULL; pQueue->qItem = NULL;
taosGetQitem(pQueue->qall, &pQueue->qItem); (void) taosGetQitem(pQueue->qall, &pQueue->qItem);
if (pQueue->qItem == NULL) { if (pQueue->qItem == NULL) {
taosReadAllQitems(pQueue->pQueue, pQueue->qall); (void) taosReadAllQitems(pQueue->pQueue, pQueue->qall);
taosGetQitem(pQueue->qall, &pQueue->qItem); (void) taosGetQitem(pQueue->qall, &pQueue->qItem);
} }
return streamQueueCurItem(pQueue); *pItem = streamQueueCurItem(pQueue);
} }
} }
@ -181,7 +186,8 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIte
return EXEC_CONTINUE; return EXEC_CONTINUE;
} }
SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputq.queue); SStreamQueueItem* qItem = NULL;
streamQueueNextItem(pTask->inputq.queue, (SStreamQueueItem**)&qItem);
if (qItem == NULL) { if (qItem == NULL) {
// restore the token to bucket // restore the token to bucket
if (*numOfBlocks > 0) { if (*numOfBlocks > 0) {
@ -338,7 +344,8 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem)
if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && type != STREAM_INPUT__CHECKPOINT_TRIGGER && if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && type != STREAM_INPUT__CHECKPOINT_TRIGGER &&
(pTask->info.delaySchedParam != 0)) { (pTask->info.delaySchedParam != 0)) {
atomic_val_compare_exchange_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE); (void)atomic_val_compare_exchange_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE,
TASK_TRIGGER_STATUS__ACTIVE);
stDebug("s-task:%s new data arrived, active the sched-trigger, triggerStatus:%d", pTask->id.idStr, stDebug("s-task:%s new data arrived, active the sched-trigger, triggerStatus:%d", pTask->id.idStr,
pTask->schedInfo.status); pTask->schedInfo.status);
} }
@ -347,18 +354,19 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem)
} }
int32_t streamTaskPutTranstateIntoInputQ(SStreamTask* pTask) { int32_t streamTaskPutTranstateIntoInputQ(SStreamTask* pTask) {
int32_t code; int32_t code = 0;
SStreamDataBlock* pTranstate; SStreamDataBlock* pTranstate = NULL;
SSDataBlock* pBlock = NULL;
code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock), (void**)&pTranstate); code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock), (void**)&pTranstate);
if (code) { if (code) {
return code; return code;
} }
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
if (pBlock == NULL) { if (pBlock == NULL) {
taosFreeQitem(pTranstate); code = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_OUT_OF_MEMORY; goto _err;
} }
pTranstate->type = STREAM_INPUT__TRANS_STATE; pTranstate->type = STREAM_INPUT__TRANS_STATE;
@ -368,15 +376,30 @@ int32_t streamTaskPutTranstateIntoInputQ(SStreamTask* pTask) {
pBlock->info.childId = pTask->info.selfChildId; pBlock->info.childId = pTask->info.selfChildId;
pTranstate->blocks = taosArrayInit(4, sizeof(SSDataBlock)); // pBlock; pTranstate->blocks = taosArrayInit(4, sizeof(SSDataBlock)); // pBlock;
taosArrayPush(pTranstate->blocks, pBlock); if (pTranstate->blocks == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
void* p = taosArrayPush(pTranstate->blocks, pBlock);
if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
taosMemoryFree(pBlock); taosMemoryFree(pBlock);
if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTranstate) < 0) { if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTranstate) < 0) {
return TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
} }
pTask->status.appendTranstateBlock = true; pTask->status.appendTranstateBlock = true;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_err:
taosMemoryFree(pBlock);
taosFreeQitem(pTranstate);
return code;
} }
// the result should be put into the outputQ in any cases, the result may be lost otherwise. // the result should be put into the outputQ in any cases, the result may be lost otherwise.

40
source/libs/stream/src/streamSched.c
View File

@ -36,7 +36,7 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask) {
int32_t streamTrySchedExec(SStreamTask* pTask) { int32_t streamTrySchedExec(SStreamTask* pTask) {
if (streamTaskSetSchedStatusWait(pTask)) { if (streamTaskSetSchedStatusWait(pTask)) {
streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pTask->id.streamId, pTask->id.taskId, 0); return streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pTask->id.streamId, pTask->id.taskId, 0);
} else { } else {
stTrace("s-task:%s not launch task since sched status:%d", pTask->id.idStr, pTask->status.schedStatus); stTrace("s-task:%s not launch task since sched status:%d", pTask->id.idStr, pTask->status.schedStatus);
} }
@ -47,10 +47,9 @@ int32_t streamTrySchedExec(SStreamTask* pTask) {
int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType) { int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType) {
SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq));
if (pRunReq == NULL) { if (pRunReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
stError("vgId:%d failed to create msg to start stream task:0x%x exec, type:%d, code:%s", vgId, taskId, execType, stError("vgId:%d failed to create msg to start stream task:0x%x exec, type:%d, code:%s", vgId, taskId, execType,
terrstr()); terrstr());
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
if (streamId != 0) { if (streamId != 0) {
@ -65,29 +64,27 @@ int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int3
pRunReq->reqType = execType; pRunReq->reqType = execType;
SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)};
tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg); return tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg);
return TSDB_CODE_SUCCESS;
} }
void streamTaskClearSchedIdleInfo(SStreamTask* pTask) { pTask->status.schedIdleTime = 0; } void streamTaskClearSchedIdleInfo(SStreamTask* pTask) { pTask->status.schedIdleTime = 0; }
void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; } void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; }
int32_t streamTaskResumeInFuture(SStreamTask* pTask) { void streamTaskResumeInFuture(SStreamTask* pTask) {
int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s task should idle, add into timer to retry in %dms, ref:%d", pTask->id.idStr, stDebug("s-task:%s task should idle, add into timer to retry in %dms, ref:%d", pTask->id.idStr,
pTask->status.schedIdleTime, ref); pTask->status.schedIdleTime, ref);
// add one ref count for task // add one ref count for task
/*SStreamTask* pAddRefTask = */ streamMetaAcquireOneTask(pTask); streamMetaAcquireOneTask(pTask);
if (pTask->schedInfo.pIdleTimer == NULL) { if (pTask->schedInfo.pIdleTimer == NULL) {
pTask->schedInfo.pIdleTimer = taosTmrStart(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer); pTask->schedInfo.pIdleTimer = taosTmrStart(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer);
} else { } else {
taosTmrReset(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer, &pTask->schedInfo.pIdleTimer); streamTmrReset(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer,
&pTask->schedInfo.pIdleTimer, pTask->pMeta->vgId, "resume-task-tmr");
} }
return TSDB_CODE_SUCCESS;
} }
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -97,7 +94,7 @@ void streamTaskResumeHelper(void* param, void* tmrId) {
SStreamTaskState p = streamTaskGetStatus(pTask); SStreamTaskState p = streamTaskGetStatus(pTask);
if (p.state == TASK_STATUS__DROPPING || p.state == TASK_STATUS__STOP) { if (p.state == TASK_STATUS__DROPPING || p.state == TASK_STATUS__STOP) {
streamTaskSetSchedStatusInactive(pTask); (void) streamTaskSetSchedStatusInactive(pTask);
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s status:%s not resume task, ref:%d", pId->idStr, p.name, ref); stDebug("s-task:%s status:%s not resume task, ref:%d", pId->idStr, p.name, ref);
@ -106,8 +103,10 @@ void streamTaskResumeHelper(void* param, void* tmrId) {
return; return;
} }
streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pId->streamId, pId->taskId, STREAM_EXEC_T_RESUME_TASK); int32_t code = streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pId->streamId, pId->taskId, STREAM_EXEC_T_RESUME_TASK);
if (code) {
stError("s-task:%s sched task failed, code:%s", pTask->id.idStr, strerror(code));
} else {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("trigger to resume s-task:%s after being idled for %dms, ref:%d", pId->idStr, pTask->status.schedIdleTime, stDebug("trigger to resume s-task:%s after being idled for %dms, ref:%d", pId->idStr, pTask->status.schedIdleTime,
ref); ref);
@ -115,12 +114,14 @@ void streamTaskResumeHelper(void* param, void* tmrId) {
// release the task ref count // release the task ref count
streamTaskClearSchedIdleInfo(pTask); streamTaskClearSchedIdleInfo(pTask);
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
}
} }
void streamTaskSchedHelper(void* param, void* tmrId) { void streamTaskSchedHelper(void* param, void* tmrId) {
SStreamTask* pTask = (void*)param; SStreamTask* pTask = (void*)param;
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int32_t nextTrigger = (int32_t)pTask->info.delaySchedParam; int32_t nextTrigger = (int32_t)pTask->info.delaySchedParam;
int32_t vgId = pTask->pMeta->vgId;
int8_t status = atomic_load_8(&pTask->schedInfo.status); int8_t status = atomic_load_8(&pTask->schedInfo.status);
stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger); stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger);
@ -140,7 +141,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) {
if (code) { if (code) {
stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory", stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory",
nextTrigger); nextTrigger);
taosTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr");
terrno = code; terrno = code;
return; return;
} }
@ -152,7 +153,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) {
stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory", stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory",
nextTrigger); nextTrigger);
taosTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr");
return; return;
} }
@ -161,13 +162,16 @@ void streamTaskSchedHelper(void* param, void* tmrId) {
code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger); code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr");
return; return;
} }
streamTrySchedExec(pTask); code = streamTrySchedExec(pTask);
if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed to sched to run, wait for next time", pTask->id.idStr);
}
} }
} }
taosTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr");
} }

374
source/libs/stream/src/streamSessionState.c
View File

@ -81,41 +81,90 @@ SStreamStateCur* createSessionStateCursor(SStreamFileState* pFileState) {
return pCur; return pCur;
} }
static SRowBuffPos* addNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey) { static int32_t addNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey,
SRowBuffPos** ppPos) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState);
ASSERT(pNewPos->pRowBuff); if (!pNewPos || !pNewPos->pRowBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey));
taosArrayPush(pWinInfos, &pNewPos); void* tmp = taosArrayPush(pWinInfos, &pNewPos);
return pNewPos; if (!tmp) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
(*ppPos) = pNewPos;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static SRowBuffPos* insertNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey, static int32_t insertNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey,
int32_t index) { int32_t index, SRowBuffPos** ppPos) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState);
ASSERT(pNewPos->pRowBuff); if (!pNewPos || !pNewPos->pRowBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey));
taosArrayInsert(pWinInfos, index, &pNewPos); void* tmp = taosArrayInsert(pWinInfos, index, &pNewPos);
return pNewPos; if (!tmp) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
*ppPos = pNewPos;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen) { SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState);
if (!pNewPos || !pNewPos->pRowBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey));
pNewPos->needFree = true; pNewPos->needFree = true;
pNewPos->beFlushed = true; pNewPos->beFlushed = true;
if(p) { if (p) {
memcpy(pNewPos->pRowBuff, p, *pVLen); memcpy(pNewPos->pRowBuff, p, *pVLen);
} else { } else {
int32_t len = getRowStateRowSize(pFileState); int32_t len = getRowStateRowSize(pFileState);
memset(pNewPos->pRowBuff, 0, len); memset(pNewPos->pRowBuff, 0, len);
} }
_end:
taosMemoryFree(p); taosMemoryFree(p);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL;
}
return pNewPos; return pNewPos;
} }
int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen,
int32_t* pVLen) { int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
(*pWinCode) = TSDB_CODE_SUCCESS;
SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SSHashObj* pSessionBuff = getRowStateBuff(pFileState);
SArray* pWinStates = NULL; SArray* pWinStates = NULL;
void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t));
@ -123,7 +172,12 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey,
pWinStates = (SArray*)(*ppBuff); pWinStates = (SArray*)(*ppBuff);
} else { } else {
pWinStates = taosArrayInit(16, POINTER_BYTES); pWinStates = taosArrayInit(16, POINTER_BYTES);
tSimpleHashPut(pSessionBuff, &pKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); if (!pWinStates) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
code = tSimpleHashPut(pSessionBuff, &pKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES);
TSDB_CHECK_CODE(code, lino, _end);
} }
TSKEY startTs = pKey->win.skey; TSKEY startTs = pKey->win.skey;
@ -136,12 +190,18 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey,
int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen);
if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) {
(*pVal) = createSessionWinBuff(pFileState, pKey, p, pVLen); (*pVal) = createSessionWinBuff(pFileState, pKey, p, pVLen);
code = code_file; if (!(*pVal)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
(*pWinCode) = code_file;
qDebug("===stream===0 get session win:%" PRId64 ",%" PRId64 " from disc, res %d", startTs, endTs, code_file); qDebug("===stream===0 get session win:%" PRId64 ",%" PRId64 " from disc, res %d", startTs, endTs, code_file);
} else { } else {
(*pVal) = addNewSessionWindow(pFileState, pWinStates, pKey); code = addNewSessionWindow(pFileState, pWinStates, pKey, (SRowBuffPos**)pVal);
code = TSDB_CODE_FAILED; (*pWinCode) = TSDB_CODE_FAILED;
taosMemoryFree(p); taosMemoryFree(p);
TSDB_CHECK_CODE(code, lino, _end);
} }
goto _end; goto _end;
} }
@ -181,7 +241,12 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey,
int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen);
if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) {
(*pVal) = createSessionWinBuff(pFileState, pKey, p, pVLen); (*pVal) = createSessionWinBuff(pFileState, pKey, p, pVLen);
code = code_file; if (!(*pVal)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
(*pWinCode) = code_file;
qDebug("===stream===1 get session win:%" PRId64 ",%" PRId64 " from disc, res %d", startTs, endTs, code_file); qDebug("===stream===1 get session win:%" PRId64 ",%" PRId64 " from disc, res %d", startTs, endTs, code_file);
goto _end; goto _end;
} else { } else {
@ -191,25 +256,36 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey,
} }
if (index == size - 1) { if (index == size - 1) {
(*pVal) = addNewSessionWindow(pFileState, pWinStates, pKey); code = addNewSessionWindow(pFileState, pWinStates, pKey, (SRowBuffPos**)pVal);
code = TSDB_CODE_FAILED; TSDB_CHECK_CODE(code, lino, _end);
(*pWinCode) = TSDB_CODE_FAILED;
goto _end; goto _end;
} }
(*pVal) = insertNewSessionWindow(pFileState, pWinStates, pKey, index + 1);
code = TSDB_CODE_FAILED; code = insertNewSessionWindow(pFileState, pWinStates, pKey, index + 1, (SRowBuffPos**)pVal);
TSDB_CHECK_CODE(code, lino, _end);
(*pWinCode) = TSDB_CODE_FAILED;
_end: _end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code; return code;
} }
int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen) { int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
int32_t* pWinCode) {
SWinKey* pTmpkey = pKey; SWinKey* pTmpkey = pKey;
ASSERT(keyLen == sizeof(SWinKey)); ASSERT(keyLen == sizeof(SWinKey));
SSessionKey pWinKey = {.groupId = pTmpkey->groupId, .win.skey = pTmpkey->ts, .win.ekey = pTmpkey->ts}; SSessionKey pWinKey = {.groupId = pTmpkey->groupId, .win.skey = pTmpkey->ts, .win.ekey = pTmpkey->ts};
return getSessionWinResultBuff(pFileState, &pWinKey, 0, pVal, pVLen); return getSessionWinResultBuff(pFileState, &pWinKey, 0, pVal, pVLen, pWinCode);
} }
int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SSHashObj* pSessionBuff = getRowStateBuff(pFileState);
SSessionKey* pKey = pPos->pKey; SSessionKey* pKey = pPos->pKey;
SArray* pWinStates = NULL; SArray* pWinStates = NULL;
@ -218,42 +294,74 @@ int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos)
pWinStates = (SArray*)(*ppBuff); pWinStates = (SArray*)(*ppBuff);
} else { } else {
pWinStates = taosArrayInit(16, POINTER_BYTES); pWinStates = taosArrayInit(16, POINTER_BYTES);
tSimpleHashPut(pSessionBuff, &pKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); if (!pWinStates) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
code = tSimpleHashPut(pSessionBuff, &pKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES);
TSDB_CHECK_CODE(code, lino, _end);
} }
int32_t size = taosArrayGetSize(pWinStates); int32_t size = taosArrayGetSize(pWinStates);
if (size == 0) { if (size == 0) {
taosArrayPush(pWinStates, &pPos); void* tmp = taosArrayPush(pWinStates, &pPos);
if (!tmp) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
goto _end; goto _end;
} }
// find the first position which is smaller than the pKey // find the first position which is smaller than the pKey
int32_t index = binarySearch(pWinStates, size, pKey, sessionStateKeyCompare); int32_t index = binarySearch(pWinStates, size, pKey, sessionStateKeyCompare);
if (index >= 0) { if (index >= 0) {
taosArrayInsert(pWinStates, index, &pPos); void* tmp = taosArrayInsert(pWinStates, index, &pPos);
if (!tmp) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
} else { } else {
taosArrayInsert(pWinStates, 0, &pPos); void* tmp = taosArrayInsert(pWinStates, 0, &pPos);
if (!tmp) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
} }
_end: _end:
pPos->needFree = false; pPos->needFree = false;
return TSDB_CODE_SUCCESS; if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen) { int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen, int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState);
if (!pNewPos || !pNewPos->pRowBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
pNewPos->needFree = true; pNewPos->needFree = true;
pNewPos->beFlushed = true; pNewPos->beFlushed = true;
void* pBuff = NULL; void* pBuff = NULL;
int32_t code = streamStateSessionGet_rocksdb(getStateFileStore(pFileState), pKey, &pBuff, pVLen); (*pWinCode) = streamStateSessionGet_rocksdb(getStateFileStore(pFileState), pKey, &pBuff, pVLen);
if (code != TSDB_CODE_SUCCESS) { if ((*pWinCode) != TSDB_CODE_SUCCESS) {
return code; goto _end;
} }
memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey));
memcpy(pNewPos->pRowBuff, pBuff, *pVLen); memcpy(pNewPos->pRowBuff, pBuff, *pVLen);
taosMemoryFreeClear(pBuff); taosMemoryFreeClear(pBuff);
(*pVal) = pNewPos; (*pVal) = pNewPos;
return TSDB_CODE_SUCCESS;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen) { int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen) {
@ -277,12 +385,12 @@ int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen)
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos) { void deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos) {
SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SSHashObj* pSessionBuff = getRowStateBuff(pFileState);
SSessionKey* pWinKey = (SSessionKey*)pPos->pKey; SSessionKey* pWinKey = (SSessionKey*)pPos->pKey;
void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t));
if (!ppBuff) { if (!ppBuff) {
return TSDB_CODE_SUCCESS; return;
} }
SArray* pWinStates = (SArray*)(*ppBuff); SArray* pWinStates = (SArray*)(*ppBuff);
int32_t size = taosArrayGetSize(pWinStates); int32_t size = taosArrayGetSize(pWinStates);
@ -294,33 +402,46 @@ int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffP
taosArrayRemove(pWinStates, index); taosArrayRemove(pWinStates, index);
} }
} }
return TSDB_CODE_SUCCESS;
} }
int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur,
const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen) { const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SRowBuffPos* pNewPos = NULL; SRowBuffPos* pNewPos = NULL;
SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SSHashObj* pSessionBuff = getRowStateBuff(pFileState);
void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t));
SArray* pWinStates = NULL; SArray* pWinStates = NULL;
if (!ppBuff) { if (!ppBuff) {
pWinStates = taosArrayInit(16, POINTER_BYTES); pWinStates = taosArrayInit(16, POINTER_BYTES);
tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); if (!pWinStates) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
code = tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES);
TSDB_CHECK_CODE(code, lino, _end);
} else { } else {
pWinStates = (SArray*)(*ppBuff); pWinStates = (SArray*)(*ppBuff);
} }
if (!pCur) { if (!pCur) {
pNewPos = addNewSessionWindow(pFileState, pWinStates, pWinKey); code = addNewSessionWindow(pFileState, pWinStates, pWinKey, &pNewPos);
TSDB_CHECK_CODE(code, lino, _end);
goto _end; goto _end;
} }
int32_t size = taosArrayGetSize(pWinStates); int32_t size = taosArrayGetSize(pWinStates);
if (pCur->buffIndex >= 0) { if (pCur->buffIndex >= 0) {
if (pCur->buffIndex >= size) { if (pCur->buffIndex >= size) {
pNewPos = addNewSessionWindow(pFileState, pWinStates, pWinKey); code = addNewSessionWindow(pFileState, pWinStates, pWinKey, &pNewPos);
TSDB_CHECK_CODE(code, lino, _end);
goto _end; goto _end;
} }
pNewPos = insertNewSessionWindow(pFileState, pWinStates, pWinKey, pCur->buffIndex); code = insertNewSessionWindow(pFileState, pWinStates, pWinKey, pCur->buffIndex, &pNewPos);
TSDB_CHECK_CODE(code, lino, _end);
goto _end; goto _end;
} else { } else {
if (size > 0) { if (size > 0) {
@ -328,12 +449,19 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream
if (sessionWinKeyCmpr(pWinKey, pPos->pKey) >= 0) { if (sessionWinKeyCmpr(pWinKey, pPos->pKey) >= 0) {
// pCur is invalid // pCur is invalid
SSessionKey pTmpKey = *pWinKey; SSessionKey pTmpKey = *pWinKey;
int32_t code = getSessionWinResultBuff(pFileState, &pTmpKey, 0, (void**)&pNewPos, pVLen); int32_t winCode = TSDB_CODE_SUCCESS;
ASSERT(code == TSDB_CODE_FAILED); code = getSessionWinResultBuff(pFileState, &pTmpKey, 0, (void**)&pNewPos, pVLen, &winCode);
ASSERT(winCode == TSDB_CODE_FAILED);
TSDB_CHECK_CODE(code, lino, _end);
goto _end; goto _end;
} }
} }
pNewPos = getNewRowPosForWrite(pFileState); pNewPos = getNewRowPosForWrite(pFileState);
if (!pNewPos || !pNewPos->pRowBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
memcpy(pNewPos->pKey, pWinKey, sizeof(SSessionKey)); memcpy(pNewPos->pKey, pWinKey, sizeof(SSessionKey));
pNewPos->needFree = true; pNewPos->needFree = true;
pNewPos->beFlushed = true; pNewPos->beFlushed = true;
@ -341,7 +469,10 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream
_end: _end:
(*ppVal) = pNewPos; (*ppVal) = pNewPos;
return TSDB_CODE_SUCCESS; if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
void sessionWinStateClear(SStreamFileState* pFileState) { void sessionWinStateClear(SStreamFileState* pFileState) {
@ -489,7 +620,7 @@ SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SS
if (pBuffCur) { if (pBuffCur) {
while (index >= 0) { while (index >= 0) {
SRowBuffPos* pPos = taosArrayGetP(pWinStates, index); SRowBuffPos* pPos = taosArrayGetP(pWinStates, index);
winCount = *((COUNT_TYPE*) ((char*)pPos->pRowBuff + (resSize - sizeof(COUNT_TYPE)))); winCount = *((COUNT_TYPE*)((char*)pPos->pRowBuff + (resSize - sizeof(COUNT_TYPE))));
if (sessionStateRangeKeyCompare(pWinKey, pWinStates, index) == 0 || winCount < count) { if (sessionStateRangeKeyCompare(pWinKey, pWinStates, index) == 0 || winCount < count) {
index--; index--;
} else if (index >= 0) { } else if (index >= 0) {
@ -515,7 +646,7 @@ SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SS
streamStateFreeCur(pCur); streamStateFreeCur(pCur);
return pBuffCur; return pBuffCur;
} }
winCount = *((COUNT_TYPE*) ((char*)pVal + (resSize - sizeof(COUNT_TYPE)))); winCount = *((COUNT_TYPE*)((char*)pVal + (resSize - sizeof(COUNT_TYPE))));
taosMemoryFreeClear(pVal); taosMemoryFreeClear(pVal);
streamStateFreeCur(pBuffCur); streamStateFreeCur(pBuffCur);
if (sessionRangeKeyCmpr(pWinKey, &key) != 0 && winCount == count) { if (sessionRangeKeyCmpr(pWinKey, &key) != 0 && winCount == count) {
@ -529,7 +660,7 @@ SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SS
streamStateCurNext(pFileStore, pCur); streamStateCurNext(pFileStore, pCur);
return pCur; return pCur;
} }
winCount = *((COUNT_TYPE*) ((char*)pVal + (resSize - sizeof(COUNT_TYPE)))); winCount = *((COUNT_TYPE*)((char*)pVal + (resSize - sizeof(COUNT_TYPE))));
taosMemoryFreeClear(pVal); taosMemoryFreeClear(pVal);
if (sessionRangeKeyCmpr(pWinKey, &key) == 0 || winCount < count) { if (sessionRangeKeyCmpr(pWinKey, &key) == 0 || winCount < count) {
streamStateCurPrev(pFileStore, pCur); streamStateCurPrev(pFileStore, pCur);
@ -579,6 +710,12 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void
code = TSDB_CODE_SUCCESS; code = TSDB_CODE_SUCCESS;
} else if (code == TSDB_CODE_SUCCESS && pVal) { } else if (code == TSDB_CODE_SUCCESS && pVal) {
SRowBuffPos* pNewPos = getNewRowPosForWrite(pCur->pStreamFileState); SRowBuffPos* pNewPos = getNewRowPosForWrite(pCur->pStreamFileState);
if (!pNewPos || !pNewPos->pRowBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFreeClear(pData);
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
return code;
}
memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey));
pNewPos->needFree = true; pNewPos->needFree = true;
pNewPos->beFlushed = true; pNewPos->beFlushed = true;
@ -590,17 +727,17 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void
return code; return code;
} }
int32_t sessionWinStateMoveToNext(SStreamStateCur* pCur) { void sessionWinStateMoveToNext(SStreamStateCur* pCur) {
qTrace("move cursor to next"); qTrace("move cursor to next");
if (pCur && pCur->buffIndex >= 0) { if (pCur && pCur->buffIndex >= 0) {
pCur->buffIndex++; pCur->buffIndex++;
} else { } else {
streamStateCurNext_rocksdb(NULL, pCur); streamStateCurNext_rocksdb(pCur);
} }
return TSDB_CODE_SUCCESS;
} }
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey, range_cmpr_fn cmpFn) { int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey,
range_cmpr_fn cmpFn) {
SStreamStateCur* pCur = sessionWinStateSeekKeyCurrentPrev(pFileState, key); SStreamStateCur* pCur = sessionWinStateSeekKeyCurrentPrev(pFileState, key);
SSessionKey tmpKey = *key; SSessionKey tmpKey = *key;
int32_t code = sessionWinStateGetKVByCur(pCur, &tmpKey, NULL, NULL); int32_t code = sessionWinStateGetKVByCur(pCur, &tmpKey, NULL, NULL);
@ -639,18 +776,28 @@ _end:
} }
int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
(*pWinCode) = TSDB_CODE_SUCCESS;
SSessionKey* pWinKey = key; SSessionKey* pWinKey = key;
TSKEY gap = 0; TSKEY gap = 0;
int32_t code = TSDB_CODE_SUCCESS;
SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SSHashObj* pSessionBuff = getRowStateBuff(pFileState);
SArray* pWinStates = NULL; SArray* pWinStates = NULL;
void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t));
if (ppBuff) { if (ppBuff) {
pWinStates = (SArray*)(*ppBuff); pWinStates = (SArray*)(*ppBuff);
} else { } else {
pWinStates = taosArrayInit(16, POINTER_BYTES); pWinStates = taosArrayInit(16, POINTER_BYTES);
tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); if (!pWinStates) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
code = tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES);
TSDB_CHECK_CODE(code, lino, _end);
} }
TSKEY startTs = pWinKey->win.skey; TSKEY startTs = pWinKey->win.skey;
@ -663,13 +810,19 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch
int32_t code_file = streamStateStateAddIfNotExist_rocksdb(pFileStore, pWinKey, pKeyData, keyDataLen, fn, &p, pVLen); int32_t code_file = streamStateStateAddIfNotExist_rocksdb(pFileStore, pWinKey, pKeyData, keyDataLen, fn, &p, pVLen);
if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) {
(*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen);
code = code_file; if (!(*pVal)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
(*pWinCode) = code_file;
qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey,
pWinKey->win.ekey, code_file); pWinKey->win.ekey, code_file);
} else { } else {
(*pVal) = addNewSessionWindow(pFileState, pWinStates, key); code = addNewSessionWindow(pFileState, pWinStates, key, (SRowBuffPos**)pVal);
code = TSDB_CODE_FAILED; (*pWinCode) = TSDB_CODE_FAILED;
taosMemoryFree(p); taosMemoryFree(p);
TSDB_CHECK_CODE(code, lino, _end);
} }
goto _end; goto _end;
} }
@ -714,7 +867,12 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch
streamStateStateAddIfNotExist_rocksdb(pFileStore, pWinKey, pKeyData, keyDataLen, fn, &p, pVLen); streamStateStateAddIfNotExist_rocksdb(pFileStore, pWinKey, pKeyData, keyDataLen, fn, &p, pVLen);
if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) {
(*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen);
code = code_file; if (!(*pVal)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
(*pWinCode) = code_file;
qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey,
pWinKey->win.ekey, code_file); pWinKey->win.ekey, code_file);
goto _end; goto _end;
@ -725,14 +883,21 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch
} }
if (index == size - 1) { if (index == size - 1) {
(*pVal) = addNewSessionWindow(pFileState, pWinStates, key); code = addNewSessionWindow(pFileState, pWinStates, key, (SRowBuffPos**)pVal);
code = TSDB_CODE_FAILED; TSDB_CHECK_CODE(code, lino, _end);
(*pWinCode) = TSDB_CODE_FAILED;
goto _end; goto _end;
} }
(*pVal) = insertNewSessionWindow(pFileState, pWinStates, key, index + 1); code = insertNewSessionWindow(pFileState, pWinStates, key, index + 1, (SRowBuffPos**)pVal);
code = TSDB_CODE_FAILED; TSDB_CHECK_CODE(code, lino, _end);
(*pWinCode) = TSDB_CODE_FAILED;
_end: _end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code; return code;
} }
@ -751,10 +916,14 @@ int32_t getCountWinStateFromDisc(SStreamState* pState, SSessionKey* pKey, void**
return code; return code;
} }
int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen) { int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal,
int32_t* pVLen, int32_t* pWinCount) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
(*pWinCount) = TSDB_CODE_SUCCESS;
SSessionKey* pWinKey = pKey; SSessionKey* pWinKey = pKey;
const TSKEY gap = 0; const TSKEY gap = 0;
int32_t code = TSDB_CODE_SUCCESS;
SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SSHashObj* pSessionBuff = getRowStateBuff(pFileState);
SArray* pWinStates = NULL; SArray* pWinStates = NULL;
void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t));
@ -762,7 +931,13 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C
pWinStates = (SArray*)(*ppBuff); pWinStates = (SArray*)(*ppBuff);
} else { } else {
pWinStates = taosArrayInit(16, POINTER_BYTES); pWinStates = taosArrayInit(16, POINTER_BYTES);
tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); if (!pWinStates) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
code = tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES);
TSDB_CHECK_CODE(code, lino, _end);
} }
TSKEY startTs = pWinKey->win.skey; TSKEY startTs = pWinKey->win.skey;
@ -772,15 +947,20 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C
if (size == 0) { if (size == 0) {
void* pFileStore = getStateFileStore(pFileState); void* pFileStore = getStateFileStore(pFileState);
void* pRockVal = NULL; void* pRockVal = NULL;
code = getCountWinStateFromDisc(pFileStore, pWinKey, &pRockVal, pVLen); (*pWinCount) = getCountWinStateFromDisc(pFileStore, pWinKey, &pRockVal, pVLen);
if (code == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { if ((*pWinCount) == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) {
qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey,
pWinKey->win.ekey, code); pWinKey->win.ekey, (*pWinCount));
if (code == TSDB_CODE_SUCCESS) { if ((*pWinCount) == TSDB_CODE_SUCCESS) {
int32_t valSize = *pVLen; int32_t valSize = *pVLen;
COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)((char*)(pRockVal) + (valSize - sizeof(COUNT_TYPE))); COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)((char*)(pRockVal) + (valSize - sizeof(COUNT_TYPE)));
if (inSessionWindow(pWinKey, startTs, gap) || (*pWinStateCout) < winCount) { if (inSessionWindow(pWinKey, startTs, gap) || (*pWinStateCout) < winCount) {
(*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen); (*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen);
if (!(*pVal)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
goto _end; goto _end;
} }
} }
@ -788,9 +968,15 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C
pWinKey->win.ekey = endTs; pWinKey->win.ekey = endTs;
(*pVal) = createSessionWinBuff(pFileState, pWinKey, NULL, NULL); (*pVal) = createSessionWinBuff(pFileState, pWinKey, NULL, NULL);
taosMemoryFree(pRockVal); taosMemoryFree(pRockVal);
if (!(*pVal)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
} else { } else {
(*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey); code = addNewSessionWindow(pFileState, pWinStates, pWinKey, (SRowBuffPos**)pVal);
code = TSDB_CODE_FAILED; TSDB_CHECK_CODE(code, lino, _end);
(*pWinCount) = TSDB_CODE_FAILED;
} }
goto _end; goto _end;
} }
@ -802,8 +988,8 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C
if (index >= 0) { if (index >= 0) {
pPos = taosArrayGetP(pWinStates, index); pPos = taosArrayGetP(pWinStates, index);
COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)( (char*)(pPos->pRowBuff) + (valSize - sizeof(COUNT_TYPE)) ); COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)((char*)(pPos->pRowBuff) + (valSize - sizeof(COUNT_TYPE)));
if (inSessionWindow(pPos->pKey, startTs, gap) || (index == size - 1 && (*pWinStateCout) < winCount) ) { if (inSessionWindow(pPos->pKey, startTs, gap) || (index == size - 1 && (*pWinStateCout) < winCount)) {
(*pVal) = pPos; (*pVal) = pPos;
SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey;
pPos->beUsed = true; pPos->beUsed = true;
@ -825,7 +1011,12 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C
if (tmpKey.win.ekey < pFirstWinKey->win.skey) { if (tmpKey.win.ekey < pFirstWinKey->win.skey) {
*pWinKey = tmpKey; *pWinKey = tmpKey;
(*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen); (*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen);
code = code_file; if (!(*pVal)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
(*pWinCount) = code_file;
qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey,
pWinKey->win.ekey, code_file); pWinKey->win.ekey, code_file);
goto _end; goto _end;
@ -845,10 +1036,15 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C
goto _end; goto _end;
} }
(*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey); code = addNewSessionWindow(pFileState, pWinStates, pWinKey, (SRowBuffPos**)pVal);
code = TSDB_CODE_FAILED; TSDB_CHECK_CODE(code, lino, _end);
(*pWinCount) = TSDB_CODE_FAILED;
_end: _end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code; return code;
} }
@ -856,6 +1052,7 @@ int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey
SSessionKey* pWinKey = pKey; SSessionKey* pWinKey = pKey;
const TSKEY gap = 0; const TSKEY gap = 0;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SSHashObj* pSessionBuff = getRowStateBuff(pFileState);
SArray* pWinStates = NULL; SArray* pWinStates = NULL;
void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t));
@ -863,7 +1060,8 @@ int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey
pWinStates = (SArray*)(*ppBuff); pWinStates = (SArray*)(*ppBuff);
} else { } else {
pWinStates = taosArrayInit(16, POINTER_BYTES); pWinStates = taosArrayInit(16, POINTER_BYTES);
tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); code = tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES);
TSDB_CHECK_CODE(code, lino, _end);
} }
TSKEY startTs = pWinKey->win.skey; TSKEY startTs = pWinKey->win.skey;
@ -877,18 +1075,26 @@ int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey
int32_t code_file = getCountWinStateFromDisc(pFileStore, pWinKey, &p, pVLen); int32_t code_file = getCountWinStateFromDisc(pFileStore, pWinKey, &p, pVLen);
if (code_file == TSDB_CODE_SUCCESS && isFlushedState(pFileState, endTs, 0)) { if (code_file == TSDB_CODE_SUCCESS && isFlushedState(pFileState, endTs, 0)) {
(*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen);
code = code_file; if (!(*pVal)) {
qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); code = TSDB_CODE_OUT_OF_MEMORY;
} else { TSDB_CHECK_CODE(code, lino, _end);
(*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey);
code = TSDB_CODE_FAILED;
taosMemoryFree(p);
} }
goto _end;
qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey,
pWinKey->win.ekey, code_file);
} else { } else {
(*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey); code = addNewSessionWindow(pFileState, pWinStates, pWinKey, (SRowBuffPos**)pVal);
taosMemoryFree(p);
TSDB_CHECK_CODE(code, lino, _end);
}
} else {
code = addNewSessionWindow(pFileState, pWinStates, pWinKey, (SRowBuffPos**)pVal);
TSDB_CHECK_CODE(code, lino, _end);
} }
_end: _end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code; return code;
} }

18
source/libs/stream/src/streamStartHistory.c
View File

@ -212,7 +212,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) {
stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", idStr, hStreamId, hTaskId, stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", idStr, hStreamId, hTaskId,
pStatus.name); pStatus.name);
streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false);
return -1; // todo set the correct error code return -1; // todo set the correct error code
} }
@ -228,11 +228,11 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) {
code = streamMetaAcquireTask(pMeta, hStreamId, hTaskId, &pHisTask); code = streamMetaAcquireTask(pMeta, hStreamId, hTaskId, &pHisTask);
if (pHisTask == NULL) { if (pHisTask == NULL) {
stDebug("s-task:%s failed acquire and start fill-history task, it may have been dropped/stopped", idStr); stDebug("s-task:%s failed acquire and start fill-history task, it may have been dropped/stopped", idStr);
streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false);
} else { } else {
if (pHisTask->status.downstreamReady == 1) { // it's ready now, do nothing if (pHisTask->status.downstreamReady == 1) { // it's ready now, do nothing
stDebug("s-task:%s fill-history task is ready, no need to check downstream", pHisTask->id.idStr); stDebug("s-task:%s fill-history task is ready, no need to check downstream", pHisTask->id.idStr);
streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, true); (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, true);
} else { // exist, but not ready, continue check downstream task status } else { // exist, but not ready, continue check downstream task status
if (pHisTask->pBackend == NULL) { if (pHisTask->pBackend == NULL) {
code = pMeta->expandTaskFn(pHisTask); code = pMeta->expandTaskFn(pHisTask);
@ -289,7 +289,7 @@ void notRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo,
SHistoryTaskInfo* pHTaskInfo = &pTask->hTaskInfo; SHistoryTaskInfo* pHTaskInfo = &pTask->hTaskInfo;
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); (void) streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false);
stError("s-task:%s max retry:%d reached, quit from retrying launch related fill-history task:0x%x, ref:%d", stError("s-task:%s max retry:%d reached, quit from retrying launch related fill-history task:0x%x, ref:%d",
pTask->id.idStr, MAX_RETRY_LAUNCH_HISTORY_TASK, (int32_t)pHTaskInfo->id.taskId, ref); pTask->id.idStr, MAX_RETRY_LAUNCH_HISTORY_TASK, (int32_t)pHTaskInfo->id.taskId, ref);
@ -307,7 +307,7 @@ void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, i
stDebug("s-task:0x%" PRIx64 " stopped, not launch rel history task:0x%" PRIx64 ", ref:%d", pInfo->id.taskId, stDebug("s-task:0x%" PRIx64 " stopped, not launch rel history task:0x%" PRIx64 ", ref:%d", pInfo->id.taskId,
pInfo->hTaskId.taskId, ref); pInfo->hTaskId.taskId, ref);
streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); (void) streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false);
taosMemoryFree(pInfo); taosMemoryFree(pInfo);
} else { } else {
char* p = streamTaskGetStatus(pTask).name; char* p = streamTaskGetStatus(pTask).name;
@ -350,7 +350,7 @@ void tryLaunchHistoryTask(void* param, void* tmrId) {
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
// record the related fill-history task failed // record the related fill-history task failed
streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); (void) streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false);
taosMemoryFree(pInfo); taosMemoryFree(pInfo);
return; return;
} }
@ -407,7 +407,7 @@ void tryLaunchHistoryTask(void* param, void* tmrId) {
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
} else { } else {
streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); (void) streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false);
int32_t ref = atomic_sub_fetch_32(&(*ppTask)->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&(*ppTask)->status.timerActive, 1);
stError("s-task:0x%x rel fill-history task:0x%" PRIx64 " may have been destroyed, not launch, ref:%d", stError("s-task:0x%x rel fill-history task:0x%" PRIx64 " may have been destroyed, not launch, ref:%d",
@ -448,7 +448,7 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) {
SLaunchHTaskInfo* pInfo = createHTaskLaunchInfo(pMeta, &id, hStreamId, hTaskId); SLaunchHTaskInfo* pInfo = createHTaskLaunchInfo(pMeta, &id, hStreamId, hTaskId);
if (pInfo == NULL) { if (pInfo == NULL) {
stError("s-task:%s failed to launch related fill-history task, since Out Of Memory", idStr); stError("s-task:%s failed to launch related fill-history task, since Out Of Memory", idStr);
streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false);
return terrno; return terrno;
} }
@ -465,7 +465,7 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) {
stError("s-task:%s failed to start timer, related fill-history task not launched, ref:%d", idStr, ref); stError("s-task:%s failed to start timer, related fill-history task not launched, ref:%d", idStr, ref);
taosMemoryFree(pInfo); taosMemoryFree(pInfo);
streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false);
return terrno; return terrno;
} }

896
source/libs/stream/src/streamState.c
View File

File diff suppressed because it is too large Load Diff

46
source/libs/stream/src/streamTask.c
View File

@ -197,13 +197,13 @@ void tFreeStreamTask(SStreamTask* pTask) {
STaskExecStatisInfo* pStatis = &pTask->execInfo; STaskExecStatisInfo* pStatis = &pTask->execInfo;
ETaskStatus status1 = TASK_STATUS__UNINIT; ETaskStatus status1 = TASK_STATUS__UNINIT;
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pTask->status.pSM != NULL) { if (pTask->status.pSM != NULL) {
SStreamTaskState pStatus = streamTaskGetStatus(pTask); SStreamTaskState pStatus = streamTaskGetStatus(pTask);
p = pStatus.name; p = pStatus.name;
status1 = pStatus.state; status1 = pStatus.state;
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
stDebug("start to free s-task:0x%x %p, state:%s", taskId, pTask, p); stDebug("start to free s-task:0x%x %p, state:%s", taskId, pTask, p);
@ -284,11 +284,11 @@ void tFreeStreamTask(SStreamTask* pTask) {
streamTaskDestroyUpstreamInfo(&pTask->upstreamInfo); streamTaskDestroyUpstreamInfo(&pTask->upstreamInfo);
taosMemoryFree(pTask->outputInfo.pTokenBucket); taosMemoryFree(pTask->outputInfo.pTokenBucket);
taosThreadMutexDestroy(&pTask->lock); streamMutexDestroy(&pTask->lock);
taosArrayDestroy(pTask->msgInfo.pSendInfo); taosArrayDestroy(pTask->msgInfo.pSendInfo);
pTask->msgInfo.pSendInfo = NULL; pTask->msgInfo.pSendInfo = NULL;
taosThreadMutexDestroy(&pTask->msgInfo.lock); streamMutexDestroy(&pTask->msgInfo.lock);
taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList); taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList);
pTask->outputInfo.pNodeEpsetUpdateList = NULL; pTask->outputInfo.pNodeEpsetUpdateList = NULL;
@ -644,11 +644,7 @@ void streamTaskOpenAllUpstreamInput(SStreamTask* pTask) {
void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) { void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) {
SStreamUpstreamEpInfo* pInfo = NULL; SStreamUpstreamEpInfo* pInfo = NULL;
streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo);
int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) {
return;
}
if ((pInfo != NULL) && pInfo->dataAllowed) { if ((pInfo != NULL) && pInfo->dataAllowed) {
pInfo->dataAllowed = false; pInfo->dataAllowed = false;
@ -659,11 +655,7 @@ void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) {
void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId) { void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId) {
SStreamUpstreamEpInfo* pInfo = NULL; SStreamUpstreamEpInfo* pInfo = NULL;
streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo);
int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) {
return;
}
if (pInfo != NULL && (!pInfo->dataAllowed)) { if (pInfo != NULL && (!pInfo->dataAllowed)) {
int32_t t = atomic_sub_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); int32_t t = atomic_sub_fetch_32(&pTask->upstreamInfo.numOfClosed, 1);
@ -679,34 +671,34 @@ bool streamTaskIsAllUpstreamClosed(SStreamTask* pTask) {
bool streamTaskSetSchedStatusWait(SStreamTask* pTask) { bool streamTaskSetSchedStatusWait(SStreamTask* pTask) {
bool ret = false; bool ret = false;
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pTask->status.schedStatus == TASK_SCHED_STATUS__INACTIVE) { if (pTask->status.schedStatus == TASK_SCHED_STATUS__INACTIVE) {
pTask->status.schedStatus = TASK_SCHED_STATUS__WAITING; pTask->status.schedStatus = TASK_SCHED_STATUS__WAITING;
ret = true; ret = true;
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return ret; return ret;
} }
int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask) { int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask) {
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
int8_t status = pTask->status.schedStatus; int8_t status = pTask->status.schedStatus;
if (status == TASK_SCHED_STATUS__WAITING) { if (status == TASK_SCHED_STATUS__WAITING) {
pTask->status.schedStatus = TASK_SCHED_STATUS__ACTIVE; pTask->status.schedStatus = TASK_SCHED_STATUS__ACTIVE;
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return status; return status;
} }
int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) { int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) {
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
int8_t status = pTask->status.schedStatus; int8_t status = pTask->status.schedStatus;
ASSERT(status == TASK_SCHED_STATUS__WAITING || status == TASK_SCHED_STATUS__ACTIVE || ASSERT(status == TASK_SCHED_STATUS__WAITING || status == TASK_SCHED_STATUS__ACTIVE ||
status == TASK_SCHED_STATUS__INACTIVE); status == TASK_SCHED_STATUS__INACTIVE);
pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return status; return status;
} }
@ -723,7 +715,7 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) {
stDebug("s-task:%s clear the related stream task:0x%x attr to fill-history task", pTask->id.idStr, stDebug("s-task:%s clear the related stream task:0x%x attr to fill-history task", pTask->id.idStr,
(int32_t)sTaskId.taskId); (int32_t)sTaskId.taskId);
taosThreadMutexLock(&(*ppStreamTask)->lock); streamMutexLock(&(*ppStreamTask)->lock);
CLEAR_RELATED_FILLHISTORY_TASK((*ppStreamTask)); CLEAR_RELATED_FILLHISTORY_TASK((*ppStreamTask));
if (resetRelHalt) { if (resetRelHalt) {
@ -734,7 +726,7 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) {
} }
streamMetaSaveTask(pMeta, *ppStreamTask); streamMetaSaveTask(pMeta, *ppStreamTask);
taosThreadMutexUnlock(&(*ppStreamTask)->lock); streamMutexUnlock(&(*ppStreamTask)->lock);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -923,6 +915,7 @@ void streamTaskResume(SStreamTask* pTask) {
bool streamTaskIsSinkTask(const SStreamTask* pTask) { return pTask->info.taskLevel == TASK_LEVEL__SINK; } bool streamTaskIsSinkTask(const SStreamTask* pTask) { return pTask->info.taskLevel == TASK_LEVEL__SINK; }
// this task must success
int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) { int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) {
int32_t code; int32_t code;
int32_t tlen = 0; int32_t tlen = 0;
@ -960,24 +953,23 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) {
return 0; return 0;
} }
int32_t streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo) { void streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo) {
*pEpInfo = NULL; *pEpInfo = NULL;
int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList); int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList);
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
if (pInfo == NULL) { if (pInfo == NULL) {
return TSDB_CODE_FAILED; return;
} }
if (pInfo->taskId == taskId) { if (pInfo->taskId == taskId) {
*pEpInfo = pInfo; *pEpInfo = pInfo;
return TSDB_CODE_SUCCESS; return;
} }
} }
stError("s-task:%s failed to find upstream task:0x%x", pTask->id.idStr, taskId); stError("s-task:%s failed to find upstream task:0x%x", pTask->id.idStr, taskId);
return TSDB_CODE_FAILED;
} }
SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId) { SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId) {
@ -1100,7 +1092,7 @@ void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) {
return; return;
} }
taosThreadMutexDestroy(&pInfo->lock); streamMutexDestroy(&pInfo->lock);
taosArrayDestroy(pInfo->pDispatchTriggerList); taosArrayDestroy(pInfo->pDispatchTriggerList);
pInfo->pDispatchTriggerList = NULL; pInfo->pDispatchTriggerList = NULL;
taosArrayDestroy(pInfo->pReadyMsgList); taosArrayDestroy(pInfo->pReadyMsgList);

46
source/libs/stream/src/streamTaskSm.c
View File

@ -201,7 +201,7 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName,
pSM->pActiveTrans = pNextTrans; pSM->pActiveTrans = pNextTrans;
pSM->startTs = taosGetTimestampMs(); pSM->startTs = taosGetTimestampMs();
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
code = pNextTrans->pAction(pSM->pTask); code = pNextTrans->pAction(pSM->pTask);
if (pNextTrans->autoInvokeEndFn) { if (pNextTrans->autoInvokeEndFn) {
@ -210,7 +210,7 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName,
return code; return code;
} }
} else { } else {
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
stDebug("s-task:%s state:%s event:%s in waiting list, req state:%s not fulfilled, put it back", pTask->id.idStr, stDebug("s-task:%s state:%s event:%s in waiting list, req state:%s not fulfilled, put it back", pTask->id.idStr,
pSM->current.name, GET_EVT_NAME(pEvtInfo->event), pSM->current.name, GET_EVT_NAME(pEvtInfo->event),
StreamTaskStatusList[pEvtInfo->status].name); StreamTaskStatusList[pEvtInfo->status].name);
@ -247,7 +247,7 @@ int32_t streamTaskRestoreStatus(SStreamTask* pTask) {
SStreamTaskSM* pSM = pTask->status.pSM; SStreamTaskSM* pSM = pTask->status.pSM;
int32_t code = 0; int32_t code = 0;
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pSM->current.state == TASK_STATUS__PAUSE && pSM->pActiveTrans == NULL) { if (pSM->current.state == TASK_STATUS__PAUSE && pSM->pActiveTrans == NULL) {
SStreamTaskState state = pSM->current; SStreamTaskState state = pSM->current;
@ -326,13 +326,13 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt
return code; return code;
} }
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
while (1) { while (1) {
// wait for the task to be here // wait for the task to be here
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
ETaskStatus s = streamTaskGetStatus(pTask).state; ETaskStatus s = streamTaskGetStatus(pTask).state;
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {// this event has been handled already if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {// this event has been handled already
stDebug("s-task:%s attached event:%s handled", id, GET_EVT_NAME(pTrans->event)); stDebug("s-task:%s attached event:%s handled", id, GET_EVT_NAME(pTrans->event));
@ -349,7 +349,7 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt
} else { // override current active trans } else { // override current active trans
pSM->pActiveTrans = pTrans; pSM->pActiveTrans = pTrans;
pSM->startTs = taosGetTimestampMs(); pSM->startTs = taosGetTimestampMs();
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
code = pTrans->pAction(pTask); code = pTrans->pAction(pTask);
@ -374,11 +374,11 @@ static int32_t doHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, ST
info.callBackFn = callbackFn; info.callBackFn = callbackFn;
code = attachWaitedEvent(pTask, &info); code = attachWaitedEvent(pTask, &info);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
} else { // override current active trans } else { // override current active trans
pSM->pActiveTrans = pTrans; pSM->pActiveTrans = pTrans;
pSM->startTs = taosGetTimestampMs(); pSM->startTs = taosGetTimestampMs();
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
code = pTrans->pAction(pTask); code = pTrans->pAction(pTask);
// todo handle error code; // todo handle error code;
@ -400,11 +400,11 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) {
STaskStateTrans* pTrans = NULL; STaskStateTrans* pTrans = NULL;
while (1) { while (1) {
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) { if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) {
EStreamTaskEvent evt = pSM->pActiveTrans->event; EStreamTaskEvent evt = pSM->pActiveTrans->event;
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed", stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed",
pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt)); pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt));
@ -414,7 +414,7 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) {
pTrans = streamTaskFindTransform(pSM->current.state, event); pTrans = streamTaskFindTransform(pSM->current.state, event);
if (pTrans == NULL) { if (pTrans == NULL) {
stDebug("s-task:%s failed to handle event:%s", pTask->id.idStr, GET_EVT_NAME(event)); stDebug("s-task:%s failed to handle event:%s", pTask->id.idStr, GET_EVT_NAME(event));
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return TSDB_CODE_STREAM_INVALID_STATETRANS; return TSDB_CODE_STREAM_INVALID_STATETRANS;
} }
@ -439,11 +439,11 @@ int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, _
STaskStateTrans* pTrans = NULL; STaskStateTrans* pTrans = NULL;
while (1) { while (1) {
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) { if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) {
EStreamTaskEvent evt = pSM->pActiveTrans->event; EStreamTaskEvent evt = pSM->pActiveTrans->event;
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed", stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed",
pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt)); pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt));
@ -453,7 +453,7 @@ int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, _
pTrans = streamTaskFindTransform(pSM->current.state, event); pTrans = streamTaskFindTransform(pSM->current.state, event);
if (pTrans == NULL) { if (pTrans == NULL) {
stDebug("s-task:%s failed to handle event:%s, status:%s", pTask->id.idStr, GET_EVT_NAME(event), pSM->current.name); stDebug("s-task:%s failed to handle event:%s, status:%s", pTask->id.idStr, GET_EVT_NAME(event), pSM->current.name);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return TSDB_CODE_STREAM_INVALID_STATETRANS; return TSDB_CODE_STREAM_INVALID_STATETRANS;
} }
@ -485,7 +485,7 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even
int32_t code = 0; int32_t code = 0;
// do update the task status // do update the task status
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
STaskStateTrans* pTrans = pSM->pActiveTrans; STaskStateTrans* pTrans = pSM->pActiveTrans;
if (pTrans == NULL) { if (pTrans == NULL) {
@ -497,14 +497,14 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even
stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", id, GET_EVT_NAME(event), stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", id, GET_EVT_NAME(event),
pSM->current.name, GET_EVT_NAME(pSM->prev.evt)); pSM->current.name, GET_EVT_NAME(pSM->prev.evt));
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return TSDB_CODE_STREAM_INVALID_STATETRANS; return TSDB_CODE_STREAM_INVALID_STATETRANS;
} }
if (pTrans->event != event) { if (pTrans->event != event) {
stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", id, GET_EVT_NAME(event), stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", id, GET_EVT_NAME(event),
pSM->current.name, GET_EVT_NAME(pTrans->event)); pSM->current.name, GET_EVT_NAME(pTrans->event));
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return TSDB_CODE_STREAM_INVALID_STATETRANS; return TSDB_CODE_STREAM_INVALID_STATETRANS;
} }
@ -518,7 +518,7 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even
// on success callback, add into lock if necessary, or maybe we should add an option for this? // on success callback, add into lock if necessary, or maybe we should add an option for this?
code = pTrans->pSuccAction(pTask); code = pTrans->pSuccAction(pTask);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
// todo: add parameter to control lock // todo: add parameter to control lock
// after handling the callback function assigned by invoker, go on handling the waiting tasks // after handling the callback function assigned by invoker, go on handling the waiting tasks
@ -532,13 +532,13 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even
stDebug("s-task:%s handle user-specified callback fn for event:%s completed", id, GET_EVT_NAME(pTrans->event)); stDebug("s-task:%s handle user-specified callback fn for event:%s completed", id, GET_EVT_NAME(pTrans->event));
} }
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
// tasks in waiting list // tasks in waiting list
if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { if (taosArrayGetSize(pSM->pWaitingEventList) > 0) {
code = doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask); code = doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask);
} else { } else {
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
int64_t el = (taosGetTimestampMs() - pSM->startTs); int64_t el = (taosGetTimestampMs() - pSM->startTs);
stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", id, stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", id,
@ -563,14 +563,14 @@ const char* streamTaskGetStatusStr(ETaskStatus status) {
void streamTaskResetStatus(SStreamTask* pTask) { void streamTaskResetStatus(SStreamTask* pTask) {
SStreamTaskSM* pSM = pTask->status.pSM; SStreamTaskSM* pSM = pTask->status.pSM;
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
stDebug("s-task:%s level:%d fill-history:%d vgId:%d set uninit, prev status:%s", pTask->id.idStr, stDebug("s-task:%s level:%d fill-history:%d vgId:%d set uninit, prev status:%s", pTask->id.idStr,
pTask->info.taskLevel, pTask->info.fillHistory, pTask->pMeta->vgId, pSM->current.name); pTask->info.taskLevel, pTask->info.fillHistory, pTask->pMeta->vgId, pSM->current.name);
pSM->current = StreamTaskStatusList[TASK_STATUS__UNINIT]; pSM->current = StreamTaskStatusList[TASK_STATUS__UNINIT];
pSM->pActiveTrans = NULL; pSM->pActiveTrans = NULL;
taosArrayClear(pSM->pWaitingEventList); taosArrayClear(pSM->pWaitingEventList);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
// clear the downstream ready status // clear the downstream ready status
pTask->status.downstreamReady = 0; pTask->status.downstreamReady = 0;

8
source/libs/stream/src/streamTimer.c
View File

@ -41,11 +41,11 @@ tmr_h streamTimerGetInstance() {
void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId,
const char* pMsg) { const char* pMsg) {
while (1) { // while (1) {
bool ret = taosTmrReset(fp, mseconds, param, handle, pTmrId); bool ret = taosTmrReset(fp, mseconds, param, handle, pTmrId);
if (ret) { if (ret) {
break; // break;
}
stError("vgId:%d failed to reset %s, try again", vgId, pMsg);
} }
// stError("vgId:%d failed to reset tmr: %s, try again", vgId, pMsg);
// }
} }

99
source/libs/stream/src/streamUpdate.c
View File

@ -172,7 +172,8 @@ int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, b
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end); TSDB_CHECK_CODE(code, lino, _end);
} }
windowSBfAdd(pInfo, bfSize); code = windowSBfAdd(pInfo, bfSize);
TSDB_CHECK_CODE(code, lino, _end);
pInfo->pTsBuckets = taosArrayInit(DEFAULT_BUCKET_SIZE, sizeof(TSKEY)); pInfo->pTsBuckets = taosArrayInit(DEFAULT_BUCKET_SIZE, sizeof(TSKEY));
if (pInfo->pTsBuckets == NULL) { if (pInfo->pTsBuckets == NULL) {
@ -183,7 +184,11 @@ int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, b
TSKEY dumy = 0; TSKEY dumy = 0;
for (uint64_t i = 0; i < DEFAULT_BUCKET_SIZE; ++i) { for (uint64_t i = 0; i < DEFAULT_BUCKET_SIZE; ++i) {
taosArrayPush(pInfo->pTsBuckets, &dumy); void* tmp = taosArrayPush(pInfo->pTsBuckets, &dumy);
if (!tmp) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
} }
pInfo->numBuckets = DEFAULT_BUCKET_SIZE; pInfo->numBuckets = DEFAULT_BUCKET_SIZE;
pInfo->pCloseWinSBF = NULL; pInfo->pCloseWinSBF = NULL;
@ -242,7 +247,9 @@ static int32_t getSBf(SUpdateInfo* pInfo, TSKEY ts, SScalableBf** ppSBf) {
if (index >= pInfo->numSBFs) { if (index >= pInfo->numSBFs) {
uint64_t count = index + 1 - pInfo->numSBFs; uint64_t count = index + 1 - pInfo->numSBFs;
windowSBfDelete(pInfo, count); windowSBfDelete(pInfo, count);
windowSBfAdd(pInfo, count); code = windowSBfAdd(pInfo, count);
TSDB_CHECK_CODE(code, lino, _end);
index = pInfo->numSBFs - 1; index = pInfo->numSBFs - 1;
} }
SScalableBf* res = taosArrayGetP(pInfo->pTsSBFs, index); SScalableBf* res = taosArrayGetP(pInfo->pTsSBFs, index);
@ -251,7 +258,11 @@ static int32_t getSBf(SUpdateInfo* pInfo, TSKEY ts, SScalableBf** ppSBf) {
code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &res); code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &res);
TSDB_CHECK_CODE(code, lino, _end); TSDB_CHECK_CODE(code, lino, _end);
taosArrayPush(pInfo->pTsSBFs, &res); void* tmp = taosArrayPush(pInfo->pTsSBFs, &res);
if (!tmp) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
} }
(*ppSBf) = res; (*ppSBf) = res;
@ -268,9 +279,14 @@ bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid) {
return false; return false;
} }
TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) { int32_t updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol,
TSKEY* pMaxResTs) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (pBlock == NULL || pBlock->info.rows == 0) return INT64_MIN; int32_t lino = 0;
if (pBlock == NULL || pBlock->info.rows == 0) {
(*pMaxResTs) = INT64_MIN;
goto _end;
}
TSKEY maxTs = INT64_MIN; TSKEY maxTs = INT64_MIN;
void* pPkVal = NULL; void* pPkVal = NULL;
void* pMaxPkVal = NULL; void* pMaxPkVal = NULL;
@ -294,10 +310,9 @@ TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t p
} }
} }
SScalableBf* pSBf = NULL; SScalableBf* pSBf = NULL;
int32_t code = getSBf(pInfo, ts, &pSBf); code = getSBf(pInfo, ts, &pSBf);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _end);
uWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
}
if (pSBf) { if (pSBf) {
if (primaryKeyCol >= 0) { if (primaryKeyCol >= 0) {
pPkVal = colDataGetData(pPkDataInfo, i); pPkVal = colDataGetData(pPkDataInfo, i);
@ -305,18 +320,29 @@ TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t p
} }
int32_t buffLen = getKeyBuff(ts, tbUid, pPkVal, len, pInfo->pKeyBuff); int32_t buffLen = getKeyBuff(ts, tbUid, pPkVal, len, pInfo->pKeyBuff);
// we don't care whether the data is updated or not // we don't care whether the data is updated or not
tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen); int32_t winRes = 0;
code = tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen, &winRes);
TSDB_CHECK_CODE(code, lino, _end);
} }
} }
void* pMaxTs = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); void* pMaxTs = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t));
if (pMaxTs == NULL || pInfo->comparePkRowFn(pMaxTs, &maxTs, pMaxPkVal, pInfo->comparePkCol) == -1) { if (pMaxTs == NULL || pInfo->comparePkRowFn(pMaxTs, &maxTs, pMaxPkVal, pInfo->comparePkCol) == -1) {
int32_t valueLen = getValueBuff(maxTs, pMaxPkVal, maxLen, pInfo->pValueBuff); int32_t valueLen = getValueBuff(maxTs, pMaxPkVal, maxLen, pInfo->pValueBuff);
taosHashPut(pInfo->pMap, &tbUid, sizeof(int64_t), pInfo->pValueBuff, valueLen); code = taosHashPut(pInfo->pMap, &tbUid, sizeof(int64_t), pInfo->pValueBuff, valueLen);
TSDB_CHECK_CODE(code, lino, _end);
} }
return maxTs; (*pMaxResTs) = maxTs;
_end:
if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t res = TSDB_CODE_FAILED; int32_t res = TSDB_CODE_FAILED;
int32_t buffLen = 0; int32_t buffLen = 0;
@ -327,7 +353,8 @@ bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* p
if (ts < maxTs - pInfo->watermark) { if (ts < maxTs - pInfo->watermark) {
// this window has been closed. // this window has been closed.
if (pInfo->pCloseWinSBF) { if (pInfo->pCloseWinSBF) {
res = tScalableBfPut(pInfo->pCloseWinSBF, pInfo->pKeyBuff, buffLen); code = tScalableBfPut(pInfo->pCloseWinSBF, pInfo->pKeyBuff, buffLen, &res);
TSDB_CHECK_CODE(code, lino, _end);
if (res == TSDB_CODE_SUCCESS) { if (res == TSDB_CODE_SUCCESS) {
return false; return false;
} else { } else {
@ -338,16 +365,16 @@ bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* p
} }
SScalableBf* pSBf = NULL; SScalableBf* pSBf = NULL;
int32_t code = getSBf(pInfo, ts, &pSBf); code = getSBf(pInfo, ts, &pSBf);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _end);
uWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
}
int32_t size = taosHashGetSize(pInfo->pMap); int32_t size = taosHashGetSize(pInfo->pMap);
if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) ||
(pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == -1)) { (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == -1)) {
int32_t valueLen = getValueBuff(ts, pPkVal, len, pInfo->pValueBuff); int32_t valueLen = getValueBuff(ts, pPkVal, len, pInfo->pValueBuff);
taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen); code = taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen);
TSDB_CHECK_CODE(code, lino, _end);
// pSBf may be a null pointer // pSBf may be a null pointer
if (pSBf) { if (pSBf) {
res = tScalableBfPutNoCheck(pSBf, pInfo->pKeyBuff, buffLen); res = tScalableBfPutNoCheck(pSBf, pInfo->pKeyBuff, buffLen);
@ -357,7 +384,8 @@ bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* p
// pSBf may be a null pointer // pSBf may be a null pointer
if (pSBf) { if (pSBf) {
res = tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen); code = tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen, &res);
TSDB_CHECK_CODE(code, lino, _end);
} }
if (!pMapMaxTs && maxTs < ts) { if (!pMapMaxTs && maxTs < ts) {
@ -370,6 +398,11 @@ bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* p
} else if (res == TSDB_CODE_SUCCESS) { } else if (res == TSDB_CODE_SUCCESS) {
return false; return false;
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
// check from tsdb api // check from tsdb api
return true; return true;
} }
@ -543,7 +576,11 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) {
TSKEY ts = INT64_MIN; TSKEY ts = INT64_MIN;
for (int32_t i = 0; i < size; i++) { for (int32_t i = 0; i < size; i++) {
if (tDecodeI64(&decoder, &ts) < 0) return -1; if (tDecodeI64(&decoder, &ts) < 0) return -1;
taosArrayPush(pInfo->pTsBuckets, &ts); void* tmp = taosArrayPush(pInfo->pTsBuckets, &ts);
if (!tmp) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
} }
if (tDecodeU64(&decoder, &pInfo->numBuckets) < 0) return -1; if (tDecodeU64(&decoder, &pInfo->numBuckets) < 0) return -1;
@ -556,7 +593,11 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) {
code = tScalableBfDecode(&decoder, &pSBf); code = tScalableBfDecode(&decoder, &pSBf);
TSDB_CHECK_CODE(code, lino, _error); TSDB_CHECK_CODE(code, lino, _error);
taosArrayPush(pInfo->pTsSBFs, &pSBf); void* tmp = taosArrayPush(pInfo->pTsSBFs, &pSBf);
if (!tmp) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
} }
if (tDecodeU64(&decoder, &pInfo->numSBFs) < 0) return -1; if (tDecodeU64(&decoder, &pInfo->numSBFs) < 0) return -1;
@ -580,7 +621,8 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) {
for (int32_t i = 0; i < mapSize; i++) { for (int32_t i = 0; i < mapSize; i++) {
if (tDecodeU64(&decoder, &uid) < 0) return -1; if (tDecodeU64(&decoder, &uid) < 0) return -1;
if (tDecodeBinary(&decoder, (uint8_t**)&pVal, &valSize) < 0) return -1; if (tDecodeBinary(&decoder, (uint8_t**)&pVal, &valSize) < 0) return -1;
taosHashPut(pInfo->pMap, &uid, sizeof(uint64_t), pVal, valSize); code = taosHashPut(pInfo->pMap, &uid, sizeof(uint64_t), pVal, valSize);
TSDB_CHECK_CODE(code, lino, _error);
} }
ASSERT(mapSize == taosHashGetSize(pInfo->pMap)); ASSERT(mapSize == taosHashGetSize(pInfo->pMap));
if (tDecodeU64(&decoder, &pInfo->maxDataVersion) < 0) return -1; if (tDecodeU64(&decoder, &pInfo->maxDataVersion) < 0) return -1;
@ -611,13 +653,22 @@ _error:
} }
bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
TSKEY* pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); TSKEY* pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t));
bool res = true; bool res = true;
if (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == 1) { if (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == 1) {
res = false; res = false;
} else { } else {
int32_t valueLen = getValueBuff(ts, pPkVal, len, pInfo->pValueBuff); int32_t valueLen = getValueBuff(ts, pPkVal, len, pInfo->pValueBuff);
taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen); code = taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen);
TSDB_CHECK_CODE(code, lino, _error);
} }
return res; return res;
_error:
if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return false;
} }

26
source/libs/stream/src/streamUtil.c Normal file
View File

@ -0,0 +1,26 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "streamInt.h"
void streamMutexLock(TdThreadMutex *pMutex) {
(void) taosThreadMutexLock(pMutex);
}
void streamMutexUnlock(TdThreadMutex *pMutex) {
(void) taosThreadMutexUnlock(pMutex);
}
void streamMutexDestroy(TdThreadMutex *pMutex) { (void) taosThreadMutexDestroy(pMutex); }

305
source/libs/stream/src/tstreamFileState.c
View File

@ -57,7 +57,6 @@ struct SStreamFileState {
_state_file_remove_fn stateFileRemoveFn; _state_file_remove_fn stateFileRemoveFn;
_state_file_get_fn stateFileGetFn; _state_file_get_fn stateFileGetFn;
_state_file_clear_fn stateFileClearFn;
_state_fun_get_fn stateFunctionGetFn; _state_fun_get_fn stateFunctionGetFn;
}; };
@ -72,15 +71,15 @@ int32_t stateHashBuffRemoveFn(void* pBuff, const void* pKey, size_t keyLen) {
return tSimpleHashRemove(pBuff, pKey, keyLen); return tSimpleHashRemove(pBuff, pKey, keyLen);
} }
int32_t stateHashBuffRemoveByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos) { void stateHashBuffRemoveByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos) {
size_t keyLen = pFileState->keyLen; size_t keyLen = pFileState->keyLen;
SRowBuffPos** ppPos = tSimpleHashGet(pFileState->rowStateBuff, pPos->pKey, keyLen); SRowBuffPos** ppPos = tSimpleHashGet(pFileState->rowStateBuff, pPos->pKey, keyLen);
if (ppPos) { if (ppPos) {
if ((*ppPos) == pPos) { if ((*ppPos) == pPos) {
return tSimpleHashRemove(pFileState->rowStateBuff, pPos->pKey, keyLen); int32_t tmpRes = tSimpleHashRemove(pFileState->rowStateBuff, pPos->pKey, keyLen);
qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);
} }
} }
return TSDB_CODE_SUCCESS;
} }
void stateHashBuffClearFn(void* pBuff) { tSimpleHashClear(pBuff); } void stateHashBuffClearFn(void* pBuff) { tSimpleHashClear(pBuff); }
@ -125,7 +124,8 @@ static void streamFileStateEncode(TSKEY* pKey, void** pVal, int32_t* pLen) {
*pLen = sizeof(TSKEY); *pLen = sizeof(TSKEY);
(*pVal) = taosMemoryCalloc(1, *pLen); (*pVal) = taosMemoryCalloc(1, *pLen);
void* buff = *pVal; void* buff = *pVal;
taosEncodeFixedI64(&buff, *pKey); int32_t tmp = taosEncodeFixedI64(&buff, *pKey);
ASSERT(tmp == sizeof(TSKEY));
} }
SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize,
@ -157,7 +157,6 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_
pFileState->stateFileRemoveFn = intervalFileRemoveFn; pFileState->stateFileRemoveFn = intervalFileRemoveFn;
pFileState->stateFileGetFn = intervalFileGetFn; pFileState->stateFileGetFn = intervalFileGetFn;
pFileState->stateFileClearFn = streamStateClear_rocksdb;
pFileState->cfName = taosStrdup("state"); pFileState->cfName = taosStrdup("state");
pFileState->stateFunctionGetFn = getRowBuff; pFileState->stateFunctionGetFn = getRowBuff;
} else { } else {
@ -169,7 +168,6 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_
pFileState->stateFileRemoveFn = sessionFileRemoveFn; pFileState->stateFileRemoveFn = sessionFileRemoveFn;
pFileState->stateFileGetFn = sessionFileGetFn; pFileState->stateFileGetFn = sessionFileGetFn;
pFileState->stateFileClearFn = streamStateSessionClear_rocksdb;
pFileState->cfName = taosStrdup("sess"); pFileState->cfName = taosStrdup("sess");
pFileState->stateFunctionGetFn = getSessionRowBuff; pFileState->stateFunctionGetFn = getSessionRowBuff;
} }
@ -258,14 +256,25 @@ void streamFileStateDestroy(SStreamFileState* pFileState) {
taosMemoryFree(pFileState); taosMemoryFree(pFileState);
} }
void putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { int32_t putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pPos->pRowBuff) { if (pPos->pRowBuff) {
tdListAppend(pFileState->freeBuffs, &(pPos->pRowBuff)); code = tdListAppend(pFileState->freeBuffs, &(pPos->pRowBuff));
TSDB_CHECK_CODE(code, lino, _end);
pPos->pRowBuff = NULL; pPos->pRowBuff = NULL;
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
void clearExpiredRowBuff(SStreamFileState* pFileState, TSKEY ts, bool all) { void clearExpiredRowBuff(SStreamFileState* pFileState, TSKEY ts, bool all) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SListIter iter = {0}; SListIter iter = {0};
tdListInitIter(pFileState->usedBuffs, &iter, TD_LIST_FORWARD); tdListInitIter(pFileState->usedBuffs, &iter, TD_LIST_FORWARD);
@ -273,19 +282,27 @@ void clearExpiredRowBuff(SStreamFileState* pFileState, TSKEY ts, bool all) {
while ((pNode = tdListNext(&iter)) != NULL) { while ((pNode = tdListNext(&iter)) != NULL) {
SRowBuffPos* pPos = *(SRowBuffPos**)(pNode->data); SRowBuffPos* pPos = *(SRowBuffPos**)(pNode->data);
if (all || (pFileState->getTs(pPos->pKey) < ts && !pPos->beUsed)) { if (all || (pFileState->getTs(pPos->pKey) < ts && !pPos->beUsed)) {
putFreeBuff(pFileState, pPos); code = putFreeBuff(pFileState, pPos);
TSDB_CHECK_CODE(code, lino, _end);
if (!all) { if (!all) {
pFileState->stateBuffRemoveByPosFn(pFileState, pPos); pFileState->stateBuffRemoveByPosFn(pFileState, pPos);
} }
destroyRowBuffPos(pPos); destroyRowBuffPos(pPos);
tdListPopNode(pFileState->usedBuffs, pNode); SListNode* tmp = tdListPopNode(pFileState->usedBuffs, pNode);
taosMemoryFreeClear(pNode); taosMemoryFreeClear(tmp);
} }
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
} }
void clearFlushedRowBuff(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uint64_t max) { int32_t clearFlushedRowBuff(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uint64_t max) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
uint64_t i = 0; uint64_t i = 0;
SListIter iter = {0}; SListIter iter = {0};
tdListInitIter(pFileState->usedBuffs, &iter, TD_LIST_FORWARD); tdListInitIter(pFileState->usedBuffs, &iter, TD_LIST_FORWARD);
@ -294,16 +311,24 @@ void clearFlushedRowBuff(SStreamFileState* pFileState, SStreamSnapshot* pFlushLi
while ((pNode = tdListNext(&iter)) != NULL && i < max) { while ((pNode = tdListNext(&iter)) != NULL && i < max) {
SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data; SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data;
if (isFlushedState(pFileState, pFileState->getTs(pPos->pKey), 0) && !pPos->beUsed) { if (isFlushedState(pFileState, pFileState->getTs(pPos->pKey), 0) && !pPos->beUsed) {
tdListAppend(pFlushList, &pPos); code = tdListAppend(pFlushList, &pPos);
TSDB_CHECK_CODE(code, lino, _end);
pFileState->flushMark = TMAX(pFileState->flushMark, pFileState->getTs(pPos->pKey)); pFileState->flushMark = TMAX(pFileState->flushMark, pFileState->getTs(pPos->pKey));
pFileState->stateBuffRemoveByPosFn(pFileState, pPos); pFileState->stateBuffRemoveByPosFn(pFileState, pPos);
tdListPopNode(pFileState->usedBuffs, pNode); SListNode* tmp = tdListPopNode(pFileState->usedBuffs, pNode);
taosMemoryFreeClear(pNode); taosMemoryFreeClear(tmp);
if (pPos->pRowBuff) { if (pPos->pRowBuff) {
i++; i++;
} }
} }
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
void streamFileStateClear(SStreamFileState* pFileState) { void streamFileStateClear(SStreamFileState* pFileState) {
@ -317,7 +342,9 @@ bool needClearDiskBuff(SStreamFileState* pFileState) { return pFileState->flushM
void streamFileStateReleaseBuff(SStreamFileState* pFileState, SRowBuffPos* pPos, bool used) { pPos->beUsed = used; } void streamFileStateReleaseBuff(SStreamFileState* pFileState, SRowBuffPos* pPos, bool used) { pPos->beUsed = used; }
void popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uint64_t max, bool used) { int32_t popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uint64_t max, bool used) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
uint64_t i = 0; uint64_t i = 0;
SListIter iter = {0}; SListIter iter = {0};
tdListInitIter(pFileState->usedBuffs, &iter, TD_LIST_FORWARD); tdListInitIter(pFileState->usedBuffs, &iter, TD_LIST_FORWARD);
@ -330,11 +357,13 @@ void popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uin
ASSERT(pPos->needFree == true); ASSERT(pPos->needFree == true);
continue; continue;
} }
tdListAppend(pFlushList, &pPos); code = tdListAppend(pFlushList, &pPos);
TSDB_CHECK_CODE(code, lino, _end);
pFileState->flushMark = TMAX(pFileState->flushMark, pFileState->getTs(pPos->pKey)); pFileState->flushMark = TMAX(pFileState->flushMark, pFileState->getTs(pPos->pKey));
pFileState->stateBuffRemoveByPosFn(pFileState, pPos); pFileState->stateBuffRemoveByPosFn(pFileState, pPos);
tdListPopNode(pFileState->usedBuffs, pNode); SListNode* tmp = tdListPopNode(pFileState->usedBuffs, pNode);
taosMemoryFreeClear(pNode); taosMemoryFreeClear(tmp);
if (pPos->pRowBuff) { if (pPos->pRowBuff) {
i++; i++;
} }
@ -342,22 +371,35 @@ void popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uin
} }
qInfo("stream state flush %d rows to disk. is used:%d", listNEles(pFlushList), used); qInfo("stream state flush %d rows to disk. is used:%d", listNEles(pFlushList), used);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
int32_t flushRowBuff(SStreamFileState* pFileState) { int32_t flushRowBuff(SStreamFileState* pFileState) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamSnapshot* pFlushList = tdListNew(POINTER_BYTES); SStreamSnapshot* pFlushList = tdListNew(POINTER_BYTES);
if (!pFlushList) { if (!pFlushList) {
return TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
} }
uint64_t num = (uint64_t)(pFileState->curRowCount * FLUSH_RATIO); uint64_t num = (uint64_t)(pFileState->curRowCount * FLUSH_RATIO);
num = TMAX(num, FLUSH_NUM); num = TMAX(num, FLUSH_NUM);
clearFlushedRowBuff(pFileState, pFlushList, num); code = clearFlushedRowBuff(pFileState, pFlushList, num);
if (isListEmpty(pFlushList)) { TSDB_CHECK_CODE(code, lino, _end);
popUsedBuffs(pFileState, pFlushList, num, false);
if (isListEmpty(pFlushList)) { if (isListEmpty(pFlushList)) {
popUsedBuffs(pFileState, pFlushList, num, true); code = popUsedBuffs(pFileState, pFlushList, num, false);
TSDB_CHECK_CODE(code, lino, _end);
if (isListEmpty(pFlushList)) {
code = popUsedBuffs(pFileState, pFlushList, num, true);
TSDB_CHECK_CODE(code, lino, _end);
} }
} }
@ -368,11 +410,17 @@ int32_t flushRowBuff(SStreamFileState* pFileState) {
SListNode* pNode = NULL; SListNode* pNode = NULL;
while ((pNode = tdListNext(&fIter)) != NULL) { while ((pNode = tdListNext(&fIter)) != NULL) {
SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data; SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data;
putFreeBuff(pFileState, pPos); code = putFreeBuff(pFileState, pPos);
TSDB_CHECK_CODE(code, lino, _end);
} }
tdListFreeP(pFlushList, destroyRowBuffPosPtr); tdListFreeP(pFlushList, destroyRowBuffPosPtr);
return TSDB_CODE_SUCCESS;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
int32_t clearRowBuff(SStreamFileState* pFileState) { int32_t clearRowBuff(SStreamFileState* pFileState) {
@ -396,17 +444,27 @@ void* getFreeBuff(SStreamFileState* pFileState) {
return ptr; return ptr;
} }
int32_t streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { void streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) {
if (pPos->pRowBuff) { if (pPos->pRowBuff) {
memset(pPos->pRowBuff, 0, pFileState->rowSize); memset(pPos->pRowBuff, 0, pFileState->rowSize);
return TSDB_CODE_SUCCESS;
} }
return TSDB_CODE_FAILED;
} }
SRowBuffPos* getNewRowPos(SStreamFileState* pFileState) { SRowBuffPos* getNewRowPos(SStreamFileState* pFileState) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SRowBuffPos* pPos = taosMemoryCalloc(1, sizeof(SRowBuffPos)); SRowBuffPos* pPos = taosMemoryCalloc(1, sizeof(SRowBuffPos));
if (!pPos) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
pPos->pKey = taosMemoryCalloc(1, pFileState->keyLen); pPos->pKey = taosMemoryCalloc(1, pFileState->keyLen);
if (!pPos->pKey) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
void* pBuff = getFreeBuff(pFileState); void* pBuff = getFreeBuff(pFileState);
if (pBuff) { if (pBuff) {
pPos->pRowBuff = pBuff; pPos->pRowBuff = pBuff;
@ -422,27 +480,51 @@ SRowBuffPos* getNewRowPos(SStreamFileState* pFileState) {
} }
} }
int32_t code = clearRowBuff(pFileState); code = clearRowBuff(pFileState);
ASSERT(code == 0); TSDB_CHECK_CODE(code, lino, _error);
pPos->pRowBuff = getFreeBuff(pFileState); pPos->pRowBuff = getFreeBuff(pFileState);
_end: _end:
tdListAppend(pFileState->usedBuffs, &pPos); code = tdListAppend(pFileState->usedBuffs, &pPos);
TSDB_CHECK_CODE(code, lino, _error);
_error:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL;
}
ASSERT(pPos->pRowBuff != NULL); ASSERT(pPos->pRowBuff != NULL);
return pPos; return pPos;
} }
SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState) { SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SRowBuffPos* newPos = getNewRowPos(pFileState); SRowBuffPos* newPos = getNewRowPos(pFileState);
if (!newPos) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
newPos->beUsed = true; newPos->beUsed = true;
newPos->beFlushed = false; newPos->beFlushed = false;
newPos->needFree = false; newPos->needFree = false;
newPos->beUpdated = true; newPos->beUpdated = true;
return newPos; return newPos;
_error:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return NULL;
} }
int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen) { int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
(*pWinCode) = TSDB_CODE_SUCCESS;
pFileState->maxTs = TMAX(pFileState->maxTs, pFileState->getTs(pKey)); pFileState->maxTs = TMAX(pFileState->maxTs, pFileState->getTs(pKey));
SRowBuffPos** pos = tSimpleHashGet(pFileState->rowStateBuff, pKey, keyLen); SRowBuffPos** pos = tSimpleHashGet(pFileState->rowStateBuff, pKey, keyLen);
if (pos) { if (pos) {
@ -450,40 +532,49 @@ int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, voi
*pVal = *pos; *pVal = *pos;
(*pos)->beUsed = true; (*pos)->beUsed = true;
(*pos)->beFlushed = false; (*pos)->beFlushed = false;
return code; goto _end;
} }
SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState);
ASSERT(pNewPos->pRowBuff); if (!pNewPos || !pNewPos->pRowBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
memcpy(pNewPos->pKey, pKey, keyLen); memcpy(pNewPos->pKey, pKey, keyLen);
code = TSDB_CODE_FAILED; (*pWinCode) = TSDB_CODE_FAILED;
TSKEY ts = pFileState->getTs(pKey); TSKEY ts = pFileState->getTs(pKey);
if (!isDeteled(pFileState, ts) && isFlushedState(pFileState, ts, 0)) { if (!isDeteled(pFileState, ts) && isFlushedState(pFileState, ts, 0)) {
int32_t len = 0; int32_t len = 0;
void* p = NULL; void* p = NULL;
code = streamStateGet_rocksdb(pFileState->pFileStore, pKey, &p, &len); (*pWinCode) = streamStateGet_rocksdb(pFileState->pFileStore, pKey, &p, &len);
qDebug("===stream===get %" PRId64 " from disc, res %d", ts, code); qDebug("===stream===get %" PRId64 " from disc, res %d", ts, (*pWinCode));
if (code == TSDB_CODE_SUCCESS) { if ((*pWinCode) == TSDB_CODE_SUCCESS) {
memcpy(pNewPos->pRowBuff, p, len); memcpy(pNewPos->pRowBuff, p, len);
} }
taosMemoryFree(p); taosMemoryFree(p);
} }
tSimpleHashPut(pFileState->rowStateBuff, pKey, keyLen, &pNewPos, POINTER_BYTES); code = tSimpleHashPut(pFileState->rowStateBuff, pKey, keyLen, &pNewPos, POINTER_BYTES);
TSDB_CHECK_CODE(code, lino, _end);
if (pVal) { if (pVal) {
*pVLen = pFileState->rowSize; *pVLen = pFileState->rowSize;
*pVal = pNewPos; *pVal = pNewPos;
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code; return code;
} }
int32_t deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) { void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) {
int32_t code_buff = pFileState->stateBuffRemoveFn(pFileState->rowStateBuff, pKey, keyLen); int32_t code_buff = pFileState->stateBuffRemoveFn(pFileState->rowStateBuff, pKey, keyLen);
qTrace("%s at line %d res:%d", __func__, __LINE__, code_buff);
int32_t code_file = pFileState->stateFileRemoveFn(pFileState, pKey); int32_t code_file = pFileState->stateFileRemoveFn(pFileState, pKey);
if (code_buff == TSDB_CODE_SUCCESS || code_file == TSDB_CODE_SUCCESS) { qTrace("%s at line %d res:%d", __func__, __LINE__, code_file);
return TSDB_CODE_SUCCESS;
}
return TSDB_CODE_FAILED;
} }
int32_t resetRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) { int32_t resetRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) {
@ -495,42 +586,65 @@ int32_t resetRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t key
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
static void recoverSessionRowBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { static int32_t recoverSessionRowBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t len = 0; int32_t len = 0;
void* pBuff = NULL; void* pBuff = NULL;
pFileState->stateFileGetFn(pFileState, pPos->pKey, &pBuff, &len); code = pFileState->stateFileGetFn(pFileState, pPos->pKey, &pBuff, &len);
TSDB_CHECK_CODE(code, lino, _end);
memcpy(pPos->pRowBuff, pBuff, len); memcpy(pPos->pRowBuff, pBuff, len);
taosMemoryFree(pBuff); taosMemoryFree(pBuff);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** pVal) { int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** pVal) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pPos->pRowBuff) { if (pPos->pRowBuff) {
if (pPos->needFree) { if (pPos->needFree) {
recoverSessionRowBuff(pFileState, pPos); code = recoverSessionRowBuff(pFileState, pPos);
TSDB_CHECK_CODE(code, lino, _end);
} }
(*pVal) = pPos->pRowBuff; (*pVal) = pPos->pRowBuff;
return TSDB_CODE_SUCCESS; goto _end;
} }
pPos->pRowBuff = getFreeBuff(pFileState); pPos->pRowBuff = getFreeBuff(pFileState);
if (!pPos->pRowBuff) { if (!pPos->pRowBuff) {
if (pFileState->curRowCount < pFileState->maxRowCount) { if (pFileState->curRowCount < pFileState->maxRowCount) {
pPos->pRowBuff = taosMemoryCalloc(1, pFileState->rowSize); pPos->pRowBuff = taosMemoryCalloc(1, pFileState->rowSize);
if (!pPos->pRowBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
pFileState->curRowCount++; pFileState->curRowCount++;
} else { } else {
int32_t code = clearRowBuff(pFileState); code = clearRowBuff(pFileState);
ASSERT(code == 0); TSDB_CHECK_CODE(code, lino, _end);
pPos->pRowBuff = getFreeBuff(pFileState); pPos->pRowBuff = getFreeBuff(pFileState);
} }
ASSERT(pPos->pRowBuff); ASSERT(pPos->pRowBuff);
} }
recoverSessionRowBuff(pFileState, pPos); code = recoverSessionRowBuff(pFileState, pPos);
TSDB_CHECK_CODE(code, lino, _end);
(*pVal) = pPos->pRowBuff; (*pVal) = pPos->pRowBuff;
if (!pPos->needFree) { if (!pPos->needFree) {
tdListPrepend(pFileState->usedBuffs, &pPos); code = tdListPrepend(pFileState->usedBuffs, &pPos);
} }
return TSDB_CODE_SUCCESS;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen) { bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen) {
@ -549,8 +663,9 @@ SStreamSnapshot* getSnapshot(SStreamFileState* pFileState) {
return pFileState->usedBuffs; return pFileState->usedBuffs;
} }
int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState) { void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SListIter iter = {0}; SListIter iter = {0};
tdListInitIter(pSnapshot, &iter, TD_LIST_FORWARD); tdListInitIter(pSnapshot, &iter, TD_LIST_FORWARD);
@ -563,8 +678,17 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
int32_t len = pFileState->rowSize + sizeof(uint64_t) + sizeof(int32_t) + 64; int32_t len = pFileState->rowSize + sizeof(uint64_t) + sizeof(int32_t) + 64;
char* buf = taosMemoryCalloc(1, len); char* buf = taosMemoryCalloc(1, len);
if (!buf) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
void* batch = streamStateCreateBatch(); void* batch = streamStateCreateBatch();
if (!batch) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
while ((pNode = tdListNext(&iter)) != NULL && code == TSDB_CODE_SUCCESS) { while ((pNode = tdListNext(&iter)) != NULL && code == TSDB_CODE_SUCCESS) {
SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data; SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data;
if (pPos->beFlushed || !pPos->pRowBuff) { if (pPos->beFlushed || !pPos->pRowBuff) {
@ -575,14 +699,16 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
qDebug("===stream===flushed start:%" PRId64, pFileState->getTs(pPos->pKey)); qDebug("===stream===flushed start:%" PRId64, pFileState->getTs(pPos->pKey));
if (streamStateGetBatchSize(batch) >= BATCH_LIMIT) { if (streamStateGetBatchSize(batch) >= BATCH_LIMIT) {
streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); code = streamStatePutBatch_rocksdb(pFileState->pFileStore, batch);
streamStateClearBatch(batch); streamStateClearBatch(batch);
TSDB_CHECK_CODE(code, lino, _end);
} }
void* pSKey = pFileState->stateBuffCreateStateKeyFn(pPos, ((SStreamState*)pFileState->pFileStore)->number); void* pSKey = pFileState->stateBuffCreateStateKeyFn(pPos, ((SStreamState*)pFileState->pFileStore)->number);
code = streamStatePutBatchOptimize(pFileState->pFileStore, idx, batch, pSKey, pPos->pRowBuff, pFileState->rowSize, code = streamStatePutBatchOptimize(pFileState->pFileStore, idx, batch, pSKey, pPos->pRowBuff, pFileState->rowSize,
0, buf); 0, buf);
taosMemoryFreeClear(pSKey); taosMemoryFreeClear(pSKey);
TSDB_CHECK_CODE(code, lino, _end);
// todo handle failure // todo handle failure
memset(buf, 0, len); memset(buf, 0, len);
} }
@ -590,7 +716,8 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
int32_t numOfElems = streamStateGetBatchSize(batch); int32_t numOfElems = streamStateGetBatchSize(batch);
if (numOfElems > 0) { if (numOfElems > 0) {
streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); code = streamStatePutBatch_rocksdb(pFileState->pFileStore, batch);
TSDB_CHECK_CODE(code, lino, _end);
} else { } else {
goto _end; goto _end;
} }
@ -606,14 +733,19 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
int32_t len = 0; int32_t len = 0;
streamFileStateEncode(&pFileState->flushMark, &valBuf, &len); streamFileStateEncode(&pFileState->flushMark, &valBuf, &len);
qDebug("===stream===flushMark write:%" PRId64, pFileState->flushMark); qDebug("===stream===flushMark write:%" PRId64, pFileState->flushMark);
streamStatePutBatch(pFileState->pFileStore, "default", batch, STREAM_STATE_INFO_NAME, valBuf, len, 0); code = streamStatePutBatch(pFileState->pFileStore, "default", batch, STREAM_STATE_INFO_NAME, valBuf, len, 0);
taosMemoryFree(valBuf); taosMemoryFree(valBuf);
streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); TSDB_CHECK_CODE(code, lino, _end);
code = streamStatePutBatch_rocksdb(pFileState->pFileStore, batch);
TSDB_CHECK_CODE(code, lino, _end);
} }
_end: _end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
streamStateDestroyBatch(batch); streamStateDestroyBatch(batch);
return code;
} }
int32_t forceRemoveCheckpoint(SStreamFileState* pFileState, int64_t checkpointId) { int32_t forceRemoveCheckpoint(SStreamFileState* pFileState, int64_t checkpointId) {
@ -660,26 +792,25 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) {
ts = atol((char*)buf); ts = atol((char*)buf);
if (ts < mark) { if (ts < mark) {
// statekey winkey.ts < mark // statekey winkey.ts < mark
forceRemoveCheckpoint(pFileState, i); int32_t tmpRes = forceRemoveCheckpoint(pFileState, i);
qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);
break; break;
} }
} }
return code; return code;
} }
int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { void recoverSesssion(SStreamFileState* pFileState, int64_t ckId) {
int code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (pFileState->maxTs != INT64_MIN) { if (pFileState->maxTs != INT64_MIN) {
int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs) int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs)
? INT64_MIN ? INT64_MIN
: pFileState->maxTs - pFileState->deleteMark; : pFileState->maxTs - pFileState->deleteMark;
deleteExpiredCheckPoint(pFileState, mark); int32_t tmpRes = deleteExpiredCheckPoint(pFileState, mark);
qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);
} }
SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileState->pFileStore, INT64_MAX); SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileState->pFileStore, INT64_MAX);
if (pCur == NULL) {
return -1;
}
int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount);
while (code == TSDB_CODE_SUCCESS) { while (code == TSDB_CODE_SUCCESS) {
if (pFileState->curRowCount >= recoverNum) { if (pFileState->curRowCount >= recoverNum) {
@ -690,30 +821,32 @@ int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) {
int32_t vlen = 0; int32_t vlen = 0;
SSessionKey key = {0}; SSessionKey key = {0};
code = streamStateSessionGetKVByCur_rocksdb(pCur, &key, &pVal, &vlen); code = streamStateSessionGetKVByCur_rocksdb(pCur, &key, &pVal, &vlen);
if (code != 0) { if (code != TSDB_CODE_SUCCESS) {
break; break;
} }
SRowBuffPos* pPos = createSessionWinBuff(pFileState, &key, pVal, &vlen); SRowBuffPos* pPos = createSessionWinBuff(pFileState, &key, pVal, &vlen);
putSessionWinResultBuff(pFileState, pPos); code = putSessionWinResultBuff(pFileState, pPos);
if (code != TSDB_CODE_SUCCESS) {
break;
}
code = streamStateSessionCurPrev_rocksdb(pCur); code = streamStateSessionCurPrev_rocksdb(pCur);
} }
streamStateFreeCur(pCur); streamStateFreeCur(pCur);
return code;
} }
int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pFileState->maxTs != INT64_MIN) { if (pFileState->maxTs != INT64_MIN) {
int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs) int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs)
? INT64_MIN ? INT64_MIN
: pFileState->maxTs - pFileState->deleteMark; : pFileState->maxTs - pFileState->deleteMark;
deleteExpiredCheckPoint(pFileState, mark); int32_t tmpRes = deleteExpiredCheckPoint(pFileState, mark);
qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);
} }
SStreamStateCur* pCur = streamStateSeekToLast_rocksdb(pFileState->pFileStore); SStreamStateCur* pCur = streamStateSeekToLast_rocksdb(pFileState->pFileStore);
if (pCur == NULL) {
return -1;
}
int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount);
while (code == TSDB_CODE_SUCCESS) { while (code == TSDB_CODE_SUCCESS) {
if (pFileState->curRowCount >= recoverNum) { if (pFileState->curRowCount >= recoverNum) {
@ -723,6 +856,11 @@ int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) {
void* pVal = NULL; void* pVal = NULL;
int32_t vlen = 0; int32_t vlen = 0;
SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState);
if (!pNewPos || !pNewPos->pRowBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
code = streamStateGetKVByCur_rocksdb(pCur, pNewPos->pKey, (const void**)&pVal, &vlen); code = streamStateGetKVByCur_rocksdb(pCur, pNewPos->pKey, (const void**)&pVal, &vlen);
if (code != TSDB_CODE_SUCCESS || pFileState->getTs(pNewPos->pKey) < pFileState->flushMark) { if (code != TSDB_CODE_SUCCESS || pFileState->getTs(pNewPos->pKey) < pFileState->flushMark) {
destroyRowBuffPos(pNewPos); destroyRowBuffPos(pNewPos);
@ -740,11 +878,14 @@ int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) {
destroyRowBuffPos(pNewPos); destroyRowBuffPos(pNewPos);
break; break;
} }
code = streamStateCurPrev_rocksdb(pCur); streamStateCurPrev_rocksdb(pCur);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
} }
streamStateFreeCur(pCur); streamStateFreeCur(pCur);
return TSDB_CODE_SUCCESS;
} }
int32_t streamFileStateGetSelectRowSize(SStreamFileState* pFileState) { return pFileState->selectivityRowSize; } int32_t streamFileStateGetSelectRowSize(SStreamFileState* pFileState) { return pFileState->selectivityRowSize; }
@ -759,7 +900,8 @@ void* getRowStateBuff(SStreamFileState* pFileState) { return pFileState->rowStat
void* getStateFileStore(SStreamFileState* pFileState) { return pFileState->pFileStore; } void* getStateFileStore(SStreamFileState* pFileState) { return pFileState->pFileStore; }
bool isDeteled(SStreamFileState* pFileState, TSKEY ts) { bool isDeteled(SStreamFileState* pFileState, TSKEY ts) {
return pFileState->deleteMark != INT64_MAX && pFileState->maxTs > 0 && ts < (pFileState->maxTs - pFileState->deleteMark); return pFileState->deleteMark != INT64_MAX && pFileState->maxTs > 0 &&
ts < (pFileState->maxTs - pFileState->deleteMark);
} }
bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap) { return ts <= (pFileState->flushMark + gap); } bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap) { return ts <= (pFileState->flushMark + gap); }
@ -767,5 +909,6 @@ bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap) { return
int32_t getRowStateRowSize(SStreamFileState* pFileState) { return pFileState->rowSize; } int32_t getRowStateRowSize(SStreamFileState* pFileState) { return pFileState->rowSize; }
int32_t getFunctionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen) { int32_t getFunctionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen) {
return pFileState->stateFunctionGetFn(pFileState, pKey, keyLen, pVal, pVLen); int32_t winCode = TSDB_CODE_SUCCESS;
return pFileState->stateFunctionGetFn(pFileState, pKey, keyLen, pVal, pVLen, &winCode);
} }

17
source/libs/transport/src/transCli.c
View File

@ -825,18 +825,19 @@ static int32_t allocConnRef(SCliConn* conn, bool update) {
transRemoveExHandle(transGetRefMgt(), conn->refId); transRemoveExHandle(transGetRefMgt(), conn->refId);
conn->refId = -1; conn->refId = -1;
} }
SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
exh->handle = conn;
exh->pThrd = conn->hostThrd;
QUEUE_INIT(&exh->q);
taosInitRWLatch(&exh->latch);
SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
exh->refId = transAddExHandle(transGetRefMgt(), exh); exh->refId = transAddExHandle(transGetRefMgt(), exh);
SExHandle* self = transAcquireExHandle(transGetRefMgt(), exh->refId); SExHandle* self = transAcquireExHandle(transGetRefMgt(), exh->refId);
ASSERT(exh == self); if (self != exh) {
taosMemoryFree(exh);
return TSDB_CODE_REF_INVALID_ID;
}
QUEUE_INIT(&exh->q); QUEUE_INIT(&exh->q);
taosInitRWLatch(&exh->latch); taosInitRWLatch(&exh->latch);
exh->handle = conn;
exh->pThrd = conn->hostThrd;
conn->refId = exh->refId; conn->refId = exh->refId;
if (conn->refId == -1) { if (conn->refId == -1) {
@ -2836,6 +2837,10 @@ int64_t transAllocHandle() {
exh->refId = transAddExHandle(transGetRefMgt(), exh); exh->refId = transAddExHandle(transGetRefMgt(), exh);
SExHandle* self = transAcquireExHandle(transGetRefMgt(), exh->refId); SExHandle* self = transAcquireExHandle(transGetRefMgt(), exh->refId);
ASSERT(exh == self); ASSERT(exh == self);
if (exh != self) {
taosMemoryFree(exh);
return TSDB_CODE_REF_INVALID_ID;
}
QUEUE_INIT(&exh->q); QUEUE_INIT(&exh->q);
taosInitRWLatch(&exh->latch); taosInitRWLatch(&exh->latch);

1
source/util/src/terror.c
View File

@ -435,6 +435,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST, "Stable not exists")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER, "Table schema is old") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER, "Table schema is old")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE, "Table already exists in other stables") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE, "Table already exists in other stables")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INCONSISTENT_DB_ID, "Inconsistent database id")
// query // query
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle")

48
source/util/src/tjson.c
View File

@ -49,8 +49,7 @@ int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t
int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number) { int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number) {
if (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number)) { if (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number)) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -58,8 +57,7 @@ int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double num
int32_t tjsonAddBoolToObject(SJson* pJson, const char* pName, const bool boolean) { int32_t tjsonAddBoolToObject(SJson* pJson, const char* pName, const bool boolean) {
if (NULL == cJSON_AddBoolToObject((cJSON*)pJson, pName, boolean)) { if (NULL == cJSON_AddBoolToObject((cJSON*)pJson, pName, boolean)) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -67,8 +65,7 @@ int32_t tjsonAddBoolToObject(SJson* pJson, const char* pName, const bool boolean
int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal) { int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal) {
if (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal)) { if (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal)) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -87,8 +84,7 @@ int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
} }
int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem) { int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem) {
@ -96,8 +92,7 @@ int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
} }
int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj) { int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj) {
@ -106,18 +101,27 @@ int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void
} }
SJson* pJobj = tjsonCreateObject(); SJson* pJobj = tjsonCreateObject();
if (NULL == pJobj || TSDB_CODE_SUCCESS != func(pObj, pJobj)) { if (NULL == pJobj) {
return terrno;
}
int32_t rc = func(pObj, pJobj);
if (rc != TSDB_CODE_SUCCESS) {
tjsonDelete(pJobj); tjsonDelete(pJobj);
return TSDB_CODE_FAILED; return rc;
} }
return tjsonAddItemToObject(pJson, pName, pJobj); return tjsonAddItemToObject(pJson, pName, pJobj);
} }
int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj) { int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj) {
SJson* pJobj = tjsonCreateObject(); SJson* pJobj = tjsonCreateObject();
if (NULL == pJobj || TSDB_CODE_SUCCESS != func(pObj, pJobj)) { if (pJobj == NULL) {
return terrno;
}
int32_t rc = func(pObj, pJobj);
if (rc != TSDB_CODE_SUCCESS) {
tjsonDelete(pJobj); tjsonDelete(pJobj);
return TSDB_CODE_FAILED; return rc;
} }
return tjsonAddItemToArray(pJson, pJobj); return tjsonAddItemToArray(pJson, pJobj);
} }
@ -156,9 +160,21 @@ int32_t tjsonAddTArray(SJson* pJson, const char* pName, FToJson func, const SArr
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
char* tjsonToString(const SJson* pJson) { return cJSON_Print((cJSON*)pJson); } char* tjsonToString(const SJson* pJson) {
char* p = cJSON_Print((cJSON*)pJson);
if (!p) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
return p;
}
char* tjsonToUnformattedString(const SJson* pJson) { return cJSON_PrintUnformatted((cJSON*)pJson); } char* tjsonToUnformattedString(const SJson* pJson) {
char* p = cJSON_PrintUnformatted((cJSON*)pJson);
if (!p) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
return p;
}
SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName) { return cJSON_GetObjectItem(pJson, pName); } SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName) { return cJSON_GetObjectItem(pJson, pName); }

4
source/util/src/tlist.c
View File

@ -55,13 +55,11 @@ void tdListEmptyP(SList *list, FDelete fp) {
} }
} }
void *tdListFreeP(SList *list, FDelete fp) { void tdListFreeP(SList *list, FDelete fp) {
if (list) { if (list) {
tdListEmptyP(list, fp); tdListEmptyP(list, fp);
taosMemoryFree(list); taosMemoryFree(list);
} }
return NULL;
} }
void tdListPrependNode(SList *list, SListNode *node) { TD_DLIST_PREPEND(list, node); } void tdListPrependNode(SList *list, SListNode *node) { TD_DLIST_PREPEND(list, node); }

19
source/util/src/tscalablebf.c
View File

@ -97,19 +97,20 @@ _error:
return code; return code;
} }
int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len) { int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len, int32_t* winRes) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0; int32_t lino = 0;
if (pSBf->status == SBF_INVALID) { if (pSBf->status == SBF_INVALID) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error); TSDB_CHECK_CODE(code, lino, _end);
} }
uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len);
uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len);
int32_t size = taosArrayGetSize(pSBf->bfArray); int32_t size = taosArrayGetSize(pSBf->bfArray);
for (int32_t i = size - 2; i >= 0; --i) { for (int32_t i = size - 2; i >= 0; --i) {
if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) { if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED; (*winRes) = TSDB_CODE_FAILED;
goto _end;
} }
} }
@ -120,12 +121,12 @@ int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len) {
pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pSBf->status = SBF_INVALID; pSBf->status = SBF_INVALID;
TSDB_CHECK_CODE(code, lino, _error); TSDB_CHECK_CODE(code, lino, _end);
} }
} }
return tBloomFilterPutHash(pNormalBf, h1, h2); (*winRes) = tBloomFilterPutHash(pNormalBf, h1, h2);
_error: _end:
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
} }
@ -233,7 +234,11 @@ int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) {
SBloomFilter* pBF = NULL; SBloomFilter* pBF = NULL;
code = tBloomFilterDecode(pDecoder, &pBF); code = tBloomFilterDecode(pDecoder, &pBF);
TSDB_CHECK_CODE(code, lino, _error); TSDB_CHECK_CODE(code, lino, _error);
taosArrayPush(pSBf->bfArray, &pBF); void* tmpRes = taosArrayPush(pSBf->bfArray, &pBF);
if (!tmpRes) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
} }
if (tDecodeU32(pDecoder, &pSBf->growth) < 0) { if (tDecodeU32(pDecoder, &pSBf->growth) < 0) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_FAILED;

22
source/util/test/bloomFilterTest.cpp
View File

@ -14,7 +14,7 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) {
GTEST_ASSERT_NE(0, tBloomFilterInit(100, -0.1, &pBFTmp)); GTEST_ASSERT_NE(0, tBloomFilterInit(100, -0.1, &pBFTmp));
GTEST_ASSERT_NE(0, tBloomFilterInit(0, 0.01, &pBFTmp)); GTEST_ASSERT_NE(0, tBloomFilterInit(0, 0.01, &pBFTmp));
SBloomFilter *pBF1 = NULL; SBloomFilter* pBF1 = NULL;
int32_t code = tBloomFilterInit(100, 0.005, &pBF1); int32_t code = tBloomFilterInit(100, 0.005, &pBF1);
GTEST_ASSERT_EQ(0, code); GTEST_ASSERT_EQ(0, code);
GTEST_ASSERT_EQ(pBF1->numBits, 1152); GTEST_ASSERT_EQ(pBF1->numBits, 1152);
@ -83,7 +83,9 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) {
int64_t index = 0; int64_t index = 0;
for (; count < 100; index++) { for (; count < 100; index++) {
int64_t ts = index + ts1; int64_t ts = index + ts1;
if (tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS) { int32_t res = TSDB_CODE_SUCCESS;
int32_t code = tScalableBfPut(pSBF1, &ts, sizeof(int64_t), &res);
if (res == TSDB_CODE_SUCCESS) {
count++; count++;
} }
} }
@ -91,7 +93,9 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) {
for (; count < 300; index++) { for (; count < 300; index++) {
int64_t ts = index + ts1; int64_t ts = index + ts1;
if (tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS) { int32_t res = TSDB_CODE_SUCCESS;
int32_t code = tScalableBfPut(pSBF1, &ts, sizeof(int64_t), &res);
if (res == TSDB_CODE_SUCCESS) {
count++; count++;
} }
} }
@ -99,7 +103,9 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) {
for (; count < 700; index++) { for (; count < 700; index++) {
int64_t ts = index + ts1; int64_t ts = index + ts1;
if (tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS) { int32_t res = TSDB_CODE_SUCCESS;
int32_t code = tScalableBfPut(pSBF1, &ts, sizeof(int64_t), &res);
if (res == TSDB_CODE_SUCCESS) {
count++; count++;
} }
} }
@ -107,7 +113,9 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) {
for (; count < 1500; index++) { for (; count < 1500; index++) {
int64_t ts = index + ts1; int64_t ts = index + ts1;
if (tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS) { int32_t res = TSDB_CODE_SUCCESS;
int32_t code = tScalableBfPut(pSBF1, &ts, sizeof(int64_t), &res);
if (res == TSDB_CODE_SUCCESS) {
count++; count++;
} }
} }
@ -132,7 +140,9 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) {
GTEST_ASSERT_EQ(0, tScalableBfInit(size, 0.001, &pSBF4)); GTEST_ASSERT_EQ(0, tScalableBfInit(size, 0.001, &pSBF4));
for (int64_t i = 0; i < 1000; i++) { for (int64_t i = 0; i < 1000; i++) {
int64_t ts = i + ts1; int64_t ts = i + ts1;
GTEST_ASSERT_EQ(tScalableBfPut(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); int32_t res = TSDB_CODE_SUCCESS;
int32_t code = tScalableBfPut(pSBF4, &ts, sizeof(int64_t), &res);
GTEST_ASSERT_EQ(res, TSDB_CODE_SUCCESS);
} }
for (int64_t i = 0; i < 1000; i++) { for (int64_t i = 0; i < 1000; i++) {

8
tests/parallel_test/cases.task
View File

@ -27,7 +27,7 @@
,,y,army,./pytest.sh python3 ./test.py -f insert/insert_basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f insert/insert_basic.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f cluster/splitVgroupByLearner.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f cluster/splitVgroupByLearner.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f authorith/authBasic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f authorith/authBasic.py -N 3
,,n,army,python3 ./test.py -f cmdline/fullopt.py # ,,n,army,python3 ./test.py -f cmdline/fullopt.py
,,n,army,python3 ./test.py -f query/show.py -N 3 ,,n,army,python3 ./test.py -f query/show.py -N 3
,,n,army,python3 ./test.py -f alter/alterConfig.py -N 3 ,,n,army,python3 ./test.py -f alter/alterConfig.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f query/subquery/subqueryBugs.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/subquery/subqueryBugs.py -N 3
@ -55,6 +55,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/window_close_session_ext.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/window_close_session_ext.py
,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/partition_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/partition_interval.py
,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py
,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/state_window_case.py
#,,n,system-test,python3 ./test.py -f 8-stream/vnode_restart.py -N 4 #,,n,system-test,python3 ./test.py -f 8-stream/vnode_restart.py -N 4
#,,n,system-test,python3 ./test.py -f 8-stream/snode_restart.py -N 4 #,,n,system-test,python3 ./test.py -f 8-stream/snode_restart.py -N 4
,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 ,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4
@ -278,8 +279,8 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db-removewal.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db-removewal.py -N 2 -n 1
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb-removewal.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb-removewal.py -N 6 -n 3
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 #,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 #,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3
#,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3 #,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3
@ -300,6 +301,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3581.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3581.py
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3311.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3311.py
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3821.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3821.py
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-5130.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/balance_vgroups_r1.py -N 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/balance_vgroups_r1.py -N 6
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShell.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShell.py

12
tests/pytest/util/common.py
View File

@ -538,21 +538,21 @@ class TDCom:
tdLog.info("cfgPath: %s" % cfgPath) tdLog.info("cfgPath: %s" % cfgPath)
return cfgPath return cfgPath
def newcon(self,host='localhost',port=6030,user='root',password='taosdata'): def newcon(self,host='localhost',port=6030,user='root',password='taosdata', database=None):
con=taos.connect(host=host, user=user, password=password, port=port) con=taos.connect(host=host, user=user, password=password, port=port, database=database)
# print(con) # print(con)
return con return con
def newcur(self,host='localhost',port=6030,user='root',password='taosdata'): def newcur(self,host='localhost',port=6030,user='root',password='taosdata',database=None):
cfgPath = self.getClientCfgPath() cfgPath = self.getClientCfgPath()
con=taos.connect(host=host, user=user, password=password, config=cfgPath, port=port) con=taos.connect(host=host, user=user, password=password, config=cfgPath, port=port,database=database)
cur=con.cursor() cur=con.cursor()
# print(cur) # print(cur)
return cur return cur
def newTdSql(self, host='localhost',port=6030,user='root',password='taosdata'): def newTdSql(self, host='localhost',port=6030,user='root',password='taosdata', database = None):
newTdSql = TDSql() newTdSql = TDSql()
cur = self.newcur(host=host,port=port,user=user,password=password) cur = self.newcur(host=host,port=port,user=user,password=password, database=database)
newTdSql.init(cur, False) newTdSql.init(cur, False)
return newTdSql return newTdSql

2
tests/script/tsim/stream/basic0.sim
View File

@ -37,6 +37,8 @@ sleep 1000
sql create stream s1 trigger at_once into outstb as select _wstart, min(k), max(k), sum(k) as sum_alias from ct1 interval(10m) sql create stream s1 trigger at_once into outstb as select _wstart, min(k), max(k), sum(k) as sum_alias from ct1 interval(10m)
run tsim/stream/checkTaskStatus.sim
sql show stables sql show stables
if $rows != 2 then if $rows != 2 then
return -1 return -1

14
tests/script/tsim/stream/basic1.sim
View File

@ -18,6 +18,9 @@ sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791213000,1,2,3,1.0);
sql insert into t1 values(1648791223001,2,2,3,1.1); sql insert into t1 values(1648791223001,2,2,3,1.1);
sql insert into t1 values(1648791233002,3,2,3,2.1); sql insert into t1 values(1648791233002,3,2,3,2.1);
@ -548,6 +551,8 @@ sql create table t5 using st tags(2,2,2);
sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) c3,max(b) c4 from st partition by tbname interval(10s); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) c3,max(b) c4 from st partition by tbname interval(10s);
sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart, count(*) c1, sum(a) c3,max(b) c4, now c5 from st partition by tbname interval(10s); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart, count(*) c1, sum(a) c3,max(b) c4, now c5 from st partition by tbname interval(10s);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,1,1,1,1.0) t2 values(1648791213000,2,2,2,2.0) t3 values(1648791213000,3,3,3,3.0) t4 values(1648791213000,4,4,4,4.0); sql insert into t1 values(1648791213000,1,1,1,1.0) t2 values(1648791213000,2,2,2,2.0) t3 values(1648791213000,3,3,3,3.0) t4 values(1648791213000,4,4,4,4.0);
$loop_count = 0 $loop_count = 0
@ -669,6 +674,8 @@ sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc i
sql create table ts1 using st tags(1,1,1); sql create table ts1 using st tags(1,1,1);
sql create stream stream_t3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from ts1 interval(10s) ; sql create stream stream_t3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from ts1 interval(10s) ;
run tsim/stream/checkTaskStatus.sim
sql insert into ts1 values(1648791211000,1,2,3); sql insert into ts1 values(1648791211000,1,2,3);
sleep 50 sleep 50
sql insert into ts1 values(1648791222001,2,2,3); sql insert into ts1 values(1648791222001,2,2,3);
@ -702,6 +709,9 @@ sql create database test4 vgroups 1;
sql use test4; sql use test4;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt__4 as select _wstart, count(*) c1 from t1 where a > 5 interval(10s); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt__4 as select _wstart, count(*) c1 from t1 where a > 5 interval(10s);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791213000,1,2,3,1.0);
$loop_count = 0 $loop_count = 0
@ -800,6 +810,8 @@ sql create table ts1 using st tags(1,1,1);
sql create stream streams5 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt5 as select count(*), _wstart, _wend, max(a) from ts1 interval(10s) ; sql create stream streams5 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt5 as select count(*), _wstart, _wend, max(a) from ts1 interval(10s) ;
sql create stream streams6 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt6 as select count(*), _wstart, _wend, max(a), _wstart as ts from ts1 interval(10s) ; sql create stream streams6 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt6 as select count(*), _wstart, _wend, max(a), _wstart as ts from ts1 interval(10s) ;
run tsim/stream/checkTaskStatus.sim
sql_error create stream streams7 trigger at_once into streamt7 as select _wstart, count(*), _wstart, _wend, max(a) from ts1 interval(10s) ; sql_error create stream streams7 trigger at_once into streamt7 as select _wstart, count(*), _wstart, _wend, max(a) from ts1 interval(10s) ;
sql_error create stream streams8 trigger at_once into streamt8 as select count(*), _wstart, _wstart, _wend, max(a) from ts1 interval(10s) ; sql_error create stream streams8 trigger at_once into streamt8 as select count(*), _wstart, _wstart, _wend, max(a) from ts1 interval(10s) ;
sql_error create stream streams9 trigger at_once into streamt9 as select _wstart as ts, count(*), _wstart as ts, _wend, max(a) from ts1 interval(10s) ; sql_error create stream streams9 trigger at_once into streamt9 as select _wstart as ts, count(*), _wstart as ts, _wend, max(a) from ts1 interval(10s) ;
@ -842,6 +854,8 @@ sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc i
sql create table ts1 using st tags(1,1,1); sql create table ts1 using st tags(1,1,1);
sql create stream streams7 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt7 as select _wstart, count(*) from ts1 interval(10s) ; sql create stream streams7 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt7 as select _wstart, count(*) from ts1 interval(10s) ;
run tsim/stream/checkTaskStatus.sim
sql insert into ts1 values(1648791211000,1,2,3); sql insert into ts1 values(1648791211000,1,2,3);
sql_error insert into ts1 values(-1648791211000,1,2,3); sql_error insert into ts1 values(-1648791211000,1,2,3);

2
tests/script/tsim/stream/basic2.sim
View File

@ -35,6 +35,8 @@ endi
sql create stream s1 trigger at_once into outstb as select _wstart, min(k), max(k), sum(k) as sum_alias from ct1 interval(10m) sql create stream s1 trigger at_once into outstb as select _wstart, min(k), max(k), sum(k) as sum_alias from ct1 interval(10m)
run tsim/stream/checkTaskStatus.sim
sql show stables sql show stables
if $rows != 2 then if $rows != 2 then
return -1 return -1

1
tests/script/tsim/stream/basic3.sim
View File

@ -28,6 +28,7 @@ sql create stream streamd4 into streamt4 as select tbname, _wstart,_wend, count(
sql create stream streamd5 into streamt5 as select tbname, _wstart,_wend, count(*), max(ca), min(cb) from st where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by tbname state_window(cc); sql create stream streamd5 into streamt5 as select tbname, _wstart,_wend, count(*), max(ca), min(cb) from st where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by tbname state_window(cc);
sql create stream streamd6 into streamt6 as select ca, _wstart,_wend, count(*), max(ca), min(cb) from t1 where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ca state_window(cc); sql create stream streamd6 into streamt6 as select ca, _wstart,_wend, count(*), max(ca), min(cb) from t1 where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ca state_window(cc);
run tsim/stream/checkTaskStatus.sim
sql alter local 'keepColumnName' '1' sql alter local 'keepColumnName' '1'
sql CREATE STABLE `meters_test_data` (`ts` TIMESTAMP, `close` FLOAT, `parttime` TIMESTAMP, `parttime_str` VARCHAR(32)) TAGS (`id` VARCHAR(32)); sql CREATE STABLE `meters_test_data` (`ts` TIMESTAMP, `close` FLOAT, `parttime` TIMESTAMP, `parttime_str` VARCHAR(32)) TAGS (`id` VARCHAR(32));

7
tests/script/tsim/stream/basic4.sim
View File

@ -15,6 +15,8 @@ sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams0 trigger at_once ignore expired 0 ignore update 0 into streamt as select _wstart, count(*) c1 from t1 interval(1s); sql create stream streams0 trigger at_once ignore expired 0 ignore update 0 into streamt as select _wstart, count(*) c1 from t1 interval(1s);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791211000,1,2,3,1.0); sql insert into t1 values(1648791211000,1,2,3,1.0);
sql insert into t1 values(1648791212001,2,2,3,1.1); sql insert into t1 values(1648791212001,2,2,3,1.1);
sql insert into t1 values(1648791213002,3,2,3,2.1); sql insert into t1 values(1648791213002,3,2,3,2.1);
@ -80,7 +82,8 @@ sql use test2;
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1); sql create table t1 using st tags(1,1,1);
sql create stream streams2 trigger at_once ignore expired 0 ignore update 0 waterMark 200s into streamt2 as select _wstart, count(*) c1 from t1 interval(1s); sql create stream streams2 trigger at_once ignore expired 0 ignore update 0 waterMark 200s into streamt2 as select _wstart, count(*) c1 from t1 interval(1s);
sleep 1000
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791211000,1,2,3,1.0); sql insert into t1 values(1648791211000,1,2,3,1.0);
sql insert into t1 values(1648791212001,2,2,3,1.1); sql insert into t1 values(1648791212001,2,2,3,1.1);
@ -147,6 +150,8 @@ sql use test1;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams1 trigger at_once ignore expired 0 ignore update 0 into streamt1 as select _wstart, count(*) c1 from t1 session(ts, 1s); sql create stream streams1 trigger at_once ignore expired 0 ignore update 0 into streamt1 as select _wstart, count(*) c1 from t1 session(ts, 1s);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791211000,1,2,3,1.0); sql insert into t1 values(1648791211000,1,2,3,1.0);
sql insert into t1 values(1648791213000,1,2,3,1.1); sql insert into t1 values(1648791213000,1,2,3,1.1);
sql insert into t1 values(1648791215000,1,2,3,1.1); sql insert into t1 values(1648791215000,1,2,3,1.1);

8
tests/script/tsim/stream/basic5.sim
View File

@ -15,7 +15,7 @@ sql use test3;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams3 trigger at_once ignore expired 0 ignore update 0 into streamt3 as select _wstart, count(*) c1 from t1 state_window(a); sql create stream streams3 trigger at_once ignore expired 0 ignore update 0 into streamt3 as select _wstart, count(*) c1 from t1 state_window(a);
sleep 1000 run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791211000,1,2,3,1.0); sql insert into t1 values(1648791211000,1,2,3,1.0);
sql insert into t1 values(1648791213000,2,2,3,1.1); sql insert into t1 values(1648791213000,2,2,3,1.1);
@ -225,7 +225,7 @@ sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2); sql create table t2 using st tags(2,2,2);
sql create stream streams4 trigger at_once ignore expired 0 ignore update 0 into streamt4 as select _wstart, first(a), b, c, ta, tb from st interval(1s); sql create stream streams4 trigger at_once ignore expired 0 ignore update 0 into streamt4 as select _wstart, first(a), b, c, ta, tb from st interval(1s);
sleep 1000 run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791211000,1,2,3,1.0); sql insert into t1 values(1648791211000,1,2,3,1.0);
sql insert into t1 values(1648791213000,2,3,4,1.1); sql insert into t1 values(1648791213000,2,3,4,1.1);
@ -301,7 +301,7 @@ sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2); sql create table t2 using st tags(2,2,2);
sql create stream streams5 trigger at_once ignore expired 0 ignore update 0 into streamt5 as select _wstart, b, c, ta, tb, max(b) from t1 interval(1s); sql create stream streams5 trigger at_once ignore expired 0 ignore update 0 into streamt5 as select _wstart, b, c, ta, tb, max(b) from t1 interval(1s);
sleep 1000 run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791211000,1,2,3,1.0); sql insert into t1 values(1648791211000,1,2,3,1.0);
sql insert into t1 values(1648791213000,2,3,4,1.1); sql insert into t1 values(1648791213000,2,3,4,1.1);
@ -382,7 +382,7 @@ sql create stream streams9 trigger at_once ignore expired 0 ignore update 0 into
sql create stream streams10 trigger at_once ignore expired 0 ignore update 0 into streamt10 as select ts, b, c, last(c), ta, tb from st partition by tbname event_window start with d = 0 end with d = 9; sql create stream streams10 trigger at_once ignore expired 0 ignore update 0 into streamt10 as select ts, b, c, last(c), ta, tb from st partition by tbname event_window start with d = 0 end with d = 9;
sql create stream streams11 trigger at_once ignore expired 1 ignore update 0 watermark 100s into streamt11 as select ts, b, c, last(c), ta, tb from st partition by tbname count_window(2); sql create stream streams11 trigger at_once ignore expired 1 ignore update 0 watermark 100s into streamt11 as select ts, b, c, last(c), ta, tb from st partition by tbname count_window(2);
sleep 1000 run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791211000,1,2,3,0); sql insert into t1 values(1648791211000,1,2,3,0);
sql insert into t1 values(1648791213000,2,3,4,0); sql insert into t1 values(1648791213000,2,3,4,0);

21
tests/script/tsim/stream/checkStreamSTable.sim
View File

@ -22,7 +22,8 @@ sql create table t2 using st tags(2,2,2);
sql create stable result.streamt0(ts timestamp,a int,b int) tags(ta int,tb varchar(100),tc int); sql create stable result.streamt0(ts timestamp,a int,b int) tags(ta int,tb varchar(100),tc int);
sql create stream streams0 trigger at_once into result.streamt0 tags(tb) as select _wstart, count(*) c1, max(a) c2 from st partition by tbname tb interval(10s); sql create stream streams0 trigger at_once into result.streamt0 tags(tb) as select _wstart, count(*) c1, max(a) c2 from st partition by tbname tb interval(10s);
sleep 500
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,1,2,3); sql insert into t1 values(1648791213000,1,2,3);
sql insert into t2 values(1648791213000,2,2,3); sql insert into t2 values(1648791213000,2,2,3);
@ -108,7 +109,8 @@ sql create table t2 using st tags(2,2,2);
sql create stable result1.streamt1(ts timestamp,a int,b int,c int) tags(ta varchar(100),tb int,tc int); sql create stable result1.streamt1(ts timestamp,a int,b int,c int) tags(ta varchar(100),tb int,tc int);
sql create stream streams1 trigger at_once into result1.streamt1(ts,c,a,b) tags(ta) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by tbname as ta interval(10s); sql create stream streams1 trigger at_once into result1.streamt1(ts,c,a,b) tags(ta) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by tbname as ta interval(10s);
sleep 500
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,10,20,30); sql insert into t1 values(1648791213000,10,20,30);
sql insert into t2 values(1648791213000,40,50,60); sql insert into t2 values(1648791213000,40,50,60);
@ -198,7 +200,8 @@ sql_error create stream streams2 trigger at_once into result2.streamt2 as selec
# column dest 3, source 2 # column dest 3, source 2
sql create stream streams2 trigger at_once into result2.streamt2(ts, a) tags(ta) as select _wstart, count(*) c1 from st partition by tbname as ta interval(10s); sql create stream streams2 trigger at_once into result2.streamt2(ts, a) tags(ta) as select _wstart, count(*) c1 from st partition by tbname as ta interval(10s);
sleep 500
run tsim/stream/checkTaskStatus.sim
print ===== step5 print ===== step5
@ -215,7 +218,8 @@ sql create table t2 using st tags(4,5,6);
sql create stable result3.streamt3(ts timestamp,a int,b int,c int, d int) tags(ta int,tb int,tc int); sql create stable result3.streamt3(ts timestamp,a int,b int,c int, d int) tags(ta int,tb int,tc int);
sql create stream streams3 trigger at_once into result3.streamt3(ts,c,a,b) as select _wstart, count(*) c1, max(a),min(b) c2 from st interval(10s); sql create stream streams3 trigger at_once into result3.streamt3(ts,c,a,b) as select _wstart, count(*) c1, max(a),min(b) c2 from st interval(10s);
sleep 500
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,10,20,30); sql insert into t1 values(1648791213000,10,20,30);
sql insert into t2 values(1648791213000,40,50,60); sql insert into t2 values(1648791213000,40,50,60);
@ -295,7 +299,8 @@ sql create table t2 using st tags(4,5,6);
sql create stable result4.streamt4(ts timestamp,a int,b int,c int, d int) tags(tg1 int,tg2 int,tg3 int); sql create stable result4.streamt4(ts timestamp,a int,b int,c int, d int) tags(tg1 int,tg2 int,tg3 int);
sql create stream streams4 trigger at_once into result4.streamt4(ts,c,a,b) tags(tg2, tg3, tg1) subtable( concat("tbl-", cast(tg1 as varchar(10)) ) ) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by ta+1 as tg1, cast(tb as bigint) as tg2, tc as tg3 interval(10s); sql create stream streams4 trigger at_once into result4.streamt4(ts,c,a,b) tags(tg2, tg3, tg1) subtable( concat("tbl-", cast(tg1 as varchar(10)) ) ) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by ta+1 as tg1, cast(tb as bigint) as tg2, tc as tg3 interval(10s);
sleep 500
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,10,20,30); sql insert into t1 values(1648791213000,10,20,30);
sql insert into t2 values(1648791213000,40,50,60); sql insert into t2 values(1648791213000,40,50,60);
@ -380,7 +385,8 @@ sql create table t2 using st tags(4,5,6);
sql create stable result5.streamt5(ts timestamp,a int,b int,c int, d int) tags(tg1 int,tg2 int,tg3 int); sql create stable result5.streamt5(ts timestamp,a int,b int,c int, d int) tags(tg1 int,tg2 int,tg3 int);
sql create stream streams5 trigger at_once into result5.streamt5(ts,c,a,b) tags(tg2, tg3, tg1) subtable( concat("tbl-", cast(tg3 as varchar(10)) ) ) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by ta+1 as tg1, cast(tb as bigint) as tg2, a as tg3 session(ts, 10s); sql create stream streams5 trigger at_once into result5.streamt5(ts,c,a,b) tags(tg2, tg3, tg1) subtable( concat("tbl-", cast(tg3 as varchar(10)) ) ) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by ta+1 as tg1, cast(tb as bigint) as tg2, a as tg3 session(ts, 10s);
sleep 500
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL);
@ -465,7 +471,8 @@ sql create stream streams8 trigger at_once into streamt8 as select _wstart as
sql drop stream streams8; sql drop stream streams8;
sql create stream streams71 trigger at_once into streamt8(ts, c2) tags(group_id)as select _wstart, count(*) from t1 partition by tbname as group_id interval(10s); sql create stream streams71 trigger at_once into streamt8(ts, c2) tags(group_id)as select _wstart, count(*) from t1 partition by tbname as group_id interval(10s);
sleep 500
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791233000,1,2,3,1.0); sql insert into t1 values(1648791233000,1,2,3,1.0);

60
tests/script/tsim/stream/checkStreamSTable1.sim
View File

@ -16,36 +16,8 @@ sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2); sql create table t2 using st tags(2,2,2);
sql create stream streams1 trigger at_once into streamt1 as select _wstart, count(*) c1, count(a) c2 from st interval(1s) ; sql create stream streams1 trigger at_once into streamt1 as select _wstart, count(*) c1, count(a) c2 from st interval(1s) ;
print ====check task status start run tsim/stream/checkTaskStatus.sim
$loop_count = 0
loopCheck:
sleep 1000
$loop_count = $loop_count + 1
if $loop_count == 30 then
return -1
endi
print 1 select * from information_schema.ins_stream_tasks;
sql select * from information_schema.ins_stream_tasks;
if $rows == 0 then
print rows=$rows
goto loopCheck
endi
print 1 select * from information_schema.ins_stream_tasks where status != "ready";
sql select * from information_schema.ins_stream_tasks where status != "ready";
if $rows != 0 then
print rows=$rows
goto loopCheck
endi
print ====check task status end
sql insert into t1 values(1648791211000,1,2,3); sql insert into t1 values(1648791211000,1,2,3);
sql insert into t1 values(1648791212000,2,2,3); sql insert into t1 values(1648791212000,2,2,3);
@ -77,36 +49,8 @@ sql alter table streamt1 add column c3 double;
print create stream streams1 trigger at_once into streamt1 as select _wstart, count(*) c1, count(a) c2, avg(b) c3 from st interval(1s) ; print create stream streams1 trigger at_once into streamt1 as select _wstart, count(*) c1, count(a) c2, avg(b) c3 from st interval(1s) ;
sql create stream streams1 trigger at_once into streamt1 as select _wstart, count(*) c1, count(a) c2, avg(b) c3 from st interval(1s) ; sql create stream streams1 trigger at_once into streamt1 as select _wstart, count(*) c1, count(a) c2, avg(b) c3 from st interval(1s) ;
print ====check task status start run tsim/stream/checkTaskStatus.sim
$loop_count = 0
loopCheck1:
sleep 1000
$loop_count = $loop_count + 1
if $loop_count == 30 then
return -1
endi
print 1 select * from information_schema.ins_stream_tasks;
sql select * from information_schema.ins_stream_tasks;
if $rows == 0 then
print rows=$rows
goto loopCheck1
endi
print 1 select * from information_schema.ins_stream_tasks where status != "ready";
sql select * from information_schema.ins_stream_tasks where status != "ready";
if $rows != 0 then
print rows=$rows
goto loopCheck1
endi
print ====check task status end
sql insert into t2 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,1,2,3);
sql insert into t1 values(1648791214000,1,2,3); sql insert into t1 values(1648791214000,1,2,3);

31
tests/script/tsim/stream/checkTaskStatus.sim Normal file
View File

@ -0,0 +1,31 @@
print ====check task status start
$loop_count = 0
loopCheck0:
print loop_count=$loop_count
sleep 1000
$loop_count = $loop_count + 1
if $loop_count == 60 then
return 1
endi
print 1 select * from information_schema.ins_stream_tasks;
sql select * from information_schema.ins_stream_tasks;
if $rows == 0 then
print rows=$rows
goto loopCheck0
endi
print 1 select * from information_schema.ins_stream_tasks where status != "ready";
sql select * from information_schema.ins_stream_tasks where status != "ready";
if $rows != 0 then
print rows=$rows
goto loopCheck0
endi
print ====check task status end

3
tests/script/tsim/stream/checkpointInterval0.sim
View File

@ -15,6 +15,9 @@ sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from t1 interval(10s); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from t1 interval(10s);
sql create stream streams1 trigger window_close IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart, count(*) c1, sum(a) from t1 interval(10s); sql create stream streams1 trigger window_close IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart, count(*) c1, sum(a) from t1 interval(10s);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791213000,1,2,3,1.0);
sql insert into t1 values(1648791213001,2,2,3,1.1); sql insert into t1 values(1648791213001,2,2,3,1.1);

4
tests/script/tsim/stream/checkpointInterval1.sim
View File

@ -15,7 +15,7 @@ sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2); sql create table t2 using st tags(2,2,2);
sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from st interval(10s); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from st interval(10s);
sleep 1000 run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791213000,1,2,3,1.0);
@ -59,6 +59,8 @@ system sh/stop_dnodes.sh
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213002,3,2,3,1.1); sql insert into t1 values(1648791213002,3,2,3,1.1);
sql insert into t2 values(1648791223003,4,2,3,1.1); sql insert into t2 values(1648791223003,4,2,3,1.1);

3
tests/script/tsim/stream/checkpointSession0.sim
View File

@ -14,6 +14,9 @@ sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from t1 session(ts, 10s); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from t1 session(ts, 10s);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791213000,1,2,3,1.0);
sql insert into t1 values(1648791213001,2,2,3,1.1); sql insert into t1 values(1648791213001,2,2,3,1.1);

3
tests/script/tsim/stream/checkpointSession1.sim
View File

@ -15,7 +15,8 @@ sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2); sql create table t2 using st tags(2,2,2);
sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from st session(ts, 10s); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from st session(ts, 10s);
sleep 1000 run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791213000,1,2,3,1.0);

3
tests/script/tsim/stream/checkpointState0.sim
View File

@ -14,6 +14,9 @@ sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from t1 state_window(b); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from t1 state_window(b);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791213000,1,2,3,1.0);
sql insert into t1 values(1648791213001,2,2,3,1.1); sql insert into t1 values(1648791213001,2,2,3,1.1);

12
tests/script/tsim/stream/count0.sim
View File

@ -11,7 +11,9 @@ sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3); sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3);
sleep 1000
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,0,1,1,1.0); sql insert into t1 values(1648791213000,0,1,1,1.0);
sql insert into t1 values(1648791213001,9,2,2,1.1); sql insert into t1 values(1648791213001,9,2,2,1.1);
@ -80,7 +82,9 @@ sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,t
sql create table t1 using st tags(1,1,1); sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2); sql create table t2 using st tags(2,2,2);
sql create stream streams2 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(3) sql create stream streams2 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(3)
sleep 1000
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,0,1,1,1.0); sql insert into t1 values(1648791213000,0,1,1,1.0);
sql insert into t1 values(1648791213001,9,2,2,1.1); sql insert into t1 values(1648791213001,9,2,2,1.1);
@ -192,7 +196,9 @@ sql insert into t1 values(1648791213009,0,3,3,1.0);
sleep 500 sleep 500
sql create stream streams3 trigger at_once FILL_HISTORY 1 IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt3 as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3); sql create stream streams3 trigger at_once FILL_HISTORY 1 IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt3 as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3);
sleep 1000
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791223000,0,1,1,1.0); sql insert into t1 values(1648791223000,0,1,1,1.0);
sql insert into t1 values(1648791223001,9,2,2,1.1); sql insert into t1 values(1648791223001,9,2,2,1.1);

8
tests/script/tsim/stream/count2.sim
View File

@ -11,7 +11,9 @@ sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3); sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3);
sleep 1000
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213001,9,2,2,1.1); sql insert into t1 values(1648791213001,9,2,2,1.1);
sql insert into t1 values(1648791213009,0,3,3,1.0); sql insert into t1 values(1648791213009,0,3,3,1.0);
@ -140,7 +142,9 @@ sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,t
sql create table t1 using st tags(1,1,1); sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2); sql create table t2 using st tags(2,2,2);
sql create stream streams2 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(3) sql create stream streams2 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(3)
sleep 1000
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213001,9,2,2,1.1); sql insert into t1 values(1648791213001,9,2,2,1.1);

4
tests/script/tsim/stream/count3.sim
View File

@ -11,7 +11,9 @@ sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3); sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3);
sleep 1000
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,0,1,1,1.0); sql insert into t1 values(1648791213000,0,1,1,1.0);
sql insert into t1 values(1648791213001,9,2,2,1.1); sql insert into t1 values(1648791213001,9,2,2,1.1);

8
tests/script/tsim/stream/countSliding0.sim
View File

@ -11,7 +11,9 @@ sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2); sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
sleep 1000
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,0,1,1,1.0); sql insert into t1 values(1648791213000,0,1,1,1.0);
sleep 100 sleep 100
@ -241,7 +243,9 @@ sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,t
sql create table t1 using st tags(1,1,1); sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2); sql create table t2 using st tags(2,2,2);
sql create stream streams2 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2); sql create stream streams2 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2);
sleep 1000
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,0,1,1,1.0); sql insert into t1 values(1648791213000,0,1,1,1.0);
sleep 100 sleep 100

4
tests/script/tsim/stream/countSliding1.sim
View File

@ -11,7 +11,9 @@ sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2); sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
sleep 1000
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,0,1,1,1.0); sql insert into t1 values(1648791213000,0,1,1,1.0);
sleep 100 sleep 100

4
tests/script/tsim/stream/countSliding2.sim
View File

@ -11,7 +11,9 @@ sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2); sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2);
sleep 1000
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,0,1,1,1.0); sql insert into t1 values(1648791213000,0,1,1,1.0);
sleep 100 sleep 100

12
tests/script/tsim/stream/deleteInterval.sim
View File

@ -17,7 +17,9 @@ sql create database test vgroups 1;
sql use test; sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 interval(10s); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 interval(10s);
sleep 500
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
sleep 1000 sleep 1000
@ -195,7 +197,9 @@ sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb
sql create table t1 using st tags(1,1,1); sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2); sql create table t2 using st tags(2,2,2);
sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s);
sleep 500
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
@ -422,7 +426,9 @@ sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb
sql create table t1 using st tags(1,1,1); sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2); sql create table t2 using st tags(2,2,2);
sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s);
sleep 500
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);

9
tests/script/tsim/stream/deleteScalar.sim
View File

@ -10,7 +10,8 @@ sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select ts, a, b from t1 partition by a; sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select ts, a, b from t1 partition by a;
sleep 1000 run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,0,2,3,1.0); sql insert into t1 values(1648791213000,0,2,3,1.0);
sql insert into t1 values(1648791213001,1,2,3,1.0); sql insert into t1 values(1648791213001,1,2,3,1.0);
@ -94,7 +95,8 @@ sql use test1;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 subtable(concat("aaa-", cast( a as varchar(10) ))) as select ts, a, b from t1 partition by a; sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 subtable(concat("aaa-", cast( a as varchar(10) ))) as select ts, a, b from t1 partition by a;
sleep 1000 run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,0,2,3,1.0); sql insert into t1 values(1648791213000,0,2,3,1.0);
sql insert into t1 values(1648791213001,1,2,3,1.0); sql insert into t1 values(1648791213001,1,2,3,1.0);
@ -178,7 +180,8 @@ sql use test2;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 subtable("aaa-a") as select ts, a, b from t1; sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 subtable("aaa-a") as select ts, a, b from t1;
sleep 1000 run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,0,2,3,1.0); sql insert into t1 values(1648791213000,0,2,3,1.0);
sql insert into t1 values(1648791213001,1,2,3,1.0); sql insert into t1 values(1648791213001,1,2,3,1.0);

14
tests/script/tsim/stream/deleteSession.sim
View File

@ -18,7 +18,8 @@ sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 session(ts, 5s); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 session(ts, 5s);
sleep 2000 run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
sleep 1000 sleep 1000
@ -195,7 +196,8 @@ sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2); sql create table t2 using st tags(2,2,2);
sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s);
sleep 2000 run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
@ -426,7 +428,9 @@ sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb
sql create table t1 using st tags(1,1,1); sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2); sql create table t2 using st tags(2,2,2);
sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s);
sleep 2000
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791210000,1,1,1,NULL); sql insert into t1 values(1648791210000,1,1,1,NULL);
sql insert into t1 values(1648791210001,2,2,2,NULL); sql insert into t1 values(1648791210001,2,2,2,NULL);
@ -538,7 +542,9 @@ sql create table t2 using st tags(2,2,2);
print create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname session(ts, 2s); print create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname session(ts, 2s);
sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname session(ts, 2s); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname session(ts, 2s);
sleep 1000
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791210000,1,2,3); sql insert into t1 values(1648791210000,1,2,3);
sql insert into t1 values(1648791220000,2,2,3); sql insert into t1 values(1648791220000,2,2,3);

4
tests/script/tsim/stream/deleteState.sim
View File

@ -18,6 +18,8 @@ sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double); sql create table t1(ts timestamp, a int, b int , c int, d double);
sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 state_window(a); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 state_window(a);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
sleep 1000 sleep 1000
sql delete from t1 where ts = 1648791213000; sql delete from t1 where ts = 1648791213000;
@ -195,6 +197,8 @@ sql create table t2 using st tags(2,2,2);
print create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname state_window(c); print create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname state_window(c);
sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname state_window(c); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname state_window(c);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791210000,1,2,1); sql insert into t1 values(1648791210000,1,2,1);
sql insert into t1 values(1648791220000,2,2,2); sql insert into t1 values(1648791220000,2,2,2);
sql insert into t1 values(1648791221000,2,2,2); sql insert into t1 values(1648791221000,2,2,2);

11
tests/script/tsim/stream/distributeInterval0.sim
View File

@ -19,7 +19,8 @@ sql create table ts3 using st tags(3,2,2);
sql create table ts4 using st tags(4,2,2); sql create table ts4 using st tags(4,2,2);
sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s);
sleep 1000 run tsim/stream/checkTaskStatus.sim
sql insert into ts1 values(1648791213001,1,12,3,1.0); sql insert into ts1 values(1648791213001,1,12,3,1.0);
sql insert into ts2 values(1648791213001,1,12,3,1.0); sql insert into ts2 values(1648791213001,1,12,3,1.0);
@ -439,7 +440,8 @@ sql create table ts1 using st tags(1,1,1);
sql create table ts2 using st tags(2,2,2); sql create table ts2 using st tags(2,2,2);
sql create stream stream_t2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 20s into streamtST1 as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ; sql create stream stream_t2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 20s into streamtST1 as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ;
sleep 2000 run tsim/stream/checkTaskStatus.sim
sql insert into ts1 values(1648791211000,1,2,3); sql insert into ts1 values(1648791211000,1,2,3);
sql insert into ts1 values(1648791222001,2,2,3); sql insert into ts1 values(1648791222001,2,2,3);
sql insert into ts2 values(1648791211000,1,2,3); sql insert into ts2 values(1648791211000,1,2,3);
@ -475,7 +477,8 @@ sql create table ts1 using st tags(1,1,1);
sql create table ts2 using st tags(2,2,2); sql create table ts2 using st tags(2,2,2);
sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from st interval(10s) ; sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from st interval(10s) ;
sleep 1000 run tsim/stream/checkTaskStatus.sim
sql insert into ts1 values(1648791211000,1,2,3); sql insert into ts1 values(1648791211000,1,2,3);
sleep 50 sleep 50
@ -527,6 +530,8 @@ sql create table ddd using st tags(4,2,2);
sql create stream streams1 IGNORE EXPIRED 0 IGNORE UPDATE 0 fill_history 0 watermark 3s into streamst subtable(c) as select _wstart, c , count(*) c1, last_row(b) c2 from st partition by c interval(1s) ; sql create stream streams1 IGNORE EXPIRED 0 IGNORE UPDATE 0 fill_history 0 watermark 3s into streamst subtable(c) as select _wstart, c , count(*) c1, last_row(b) c2 from st partition by c interval(1s) ;
run tsim/stream/checkTaskStatus.sim
sql insert into aaa values(1648791221001,2,2,"/a1/aa/aa"); sql insert into aaa values(1648791221001,2,2,"/a1/aa/aa");
sql insert into bbb values(1648791221001,2,2,"/a1/aa/aa"); sql insert into bbb values(1648791221001,2,2,"/a1/aa/aa");
sql insert into ccc values(1648791221001,2,2,"/a1/aa/aa"); sql insert into ccc values(1648791221001,2,2,"/a1/aa/aa");

Some files were not shown because too many files have changed in this diff Show More