Merge branch '3.0' into enh/TD-29367-8

This commit is contained in:
Hongze Cheng 2024-07-19 16:06:53 +08:00
commit 5e79c2afea
51 changed files with 3939 additions and 2403 deletions

View File

@ -253,7 +253,7 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc); int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc);
SSDataBlock* createDataBlock(); SSDataBlock* createDataBlock();
void* blockDataDestroy(SSDataBlock* pBlock); void blockDataDestroy(SSDataBlock* pBlock);
void blockDataFreeRes(SSDataBlock* pBlock); void blockDataFreeRes(SSDataBlock* pBlock);
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
SSDataBlock* createSpecialDataBlock(EStreamType type); SSDataBlock* createSpecialDataBlock(EStreamType type);

View File

@ -58,12 +58,13 @@ extern int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict
extern int32_t filterConverNcharColumns(SFilterInfo *pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterConverNcharColumns(SFilterInfo *pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo *pFilterInfo); extern int32_t filterFreeNcharColumns(SFilterInfo *pFilterInfo);
extern void filterFreeInfo(SFilterInfo *info); extern void filterFreeInfo(SFilterInfo *info);
extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pColsAgg, int32_t numOfCols, int32_t numOfRows); extern int32_t filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows,
bool *keep);
/* condition split interface */ /* condition split interface */
int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond, int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond,
SNode **pOtherCond); SNode **pOtherCond);
bool filterIsMultiTableColsCond(SNode *pCond); int32_t filterIsMultiTableColsCond(SNode *pCond, bool *res);
EConditionType filterClassifyCondition(SNode *pNode); EConditionType filterClassifyCondition(SNode *pNode);
#ifdef __cplusplus #ifdef __cplusplus

View File

@ -629,7 +629,7 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask);
int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg); int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg);
int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code); int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code);
int32_t streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo); void streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo);
#if 0 #if 0
SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId); SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId);
#endif #endif
@ -659,10 +659,10 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask);
int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt); int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt);
int32_t streamExecTask(SStreamTask* pTask); int32_t streamExecTask(SStreamTask* pTask);
int32_t streamResumeTask(SStreamTask* pTask); void streamResumeTask(SStreamTask* pTask);
int32_t streamTrySchedExec(SStreamTask* pTask); int32_t streamTrySchedExec(SStreamTask* pTask);
int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType); int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType);
int32_t streamTaskResumeInFuture(SStreamTask* pTask); void streamTaskResumeInFuture(SStreamTask* pTask);
void streamTaskClearSchedIdleInfo(SStreamTask* pTask); void streamTaskClearSchedIdleInfo(SStreamTask* pTask);
void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime); void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime);
@ -675,8 +675,8 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa
int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp); int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp);
// check downstream status // check downstream status
int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask); void streamTaskStartMonitorCheckRsp(SStreamTask* pTask);
int32_t streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id); void streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id);
void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo); void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo);
// fill-history task // fill-history task

View File

@ -597,6 +597,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR TAOS_DEF_ERROR_CODE(0, 0x0731) #define TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR TAOS_DEF_ERROR_CODE(0, 0x0731)
#define TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0732) #define TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0732)
#define TSDB_CODE_QRY_INVALID_JOIN_CONDITION TAOS_DEF_ERROR_CODE(0, 0x0733) #define TSDB_CODE_QRY_INVALID_JOIN_CONDITION TAOS_DEF_ERROR_CODE(0, 0x0733)
#define TSDB_CODE_QRY_FILTER_NOT_SUPPORT_TYPE TAOS_DEF_ERROR_CODE(0, 0x0734)
// grant // grant
#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) #define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800)

View File

@ -1523,9 +1523,9 @@ void blockDataFreeRes(SSDataBlock* pBlock) {
memset(&pBlock->info, 0, sizeof(SDataBlockInfo)); memset(&pBlock->info, 0, sizeof(SDataBlockInfo));
} }
void* blockDataDestroy(SSDataBlock* pBlock) { void blockDataDestroy(SSDataBlock* pBlock) {
if (pBlock == NULL) { if (pBlock == NULL) {
return NULL; return;
} }
if (IS_VAR_DATA_TYPE(pBlock->info.pks[0].type)) { if (IS_VAR_DATA_TYPE(pBlock->info.pks[0].type)) {
@ -1535,7 +1535,6 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
blockDataFreeRes(pBlock); blockDataFreeRes(pBlock);
taosMemoryFreeClear(pBlock); taosMemoryFreeClear(pBlock);
return NULL;
} }
// todo remove it // todo remove it
@ -2468,19 +2467,18 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) {
int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, char* cname) { int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, char* cname) {
if (stbFullName[0] == 0) { if (stbFullName[0] == 0) {
terrno = TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
return TSDB_CODE_FAILED;
} }
SArray* tags = taosArrayInit(0, sizeof(SSmlKv)); SArray* tags = taosArrayInit(0, sizeof(SSmlKv));
if (tags == NULL) { if (tags == NULL) {
return TSDB_CODE_FAILED; return TSDB_CODE_OUT_OF_MEMORY;
} }
if (cname == NULL) { if (cname == NULL) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
taosArrayDestroy(tags); taosArrayDestroy(tags);
return TSDB_CODE_FAILED; return terrno;
} }
int8_t type = TSDB_DATA_TYPE_UBIGINT; int8_t type = TSDB_DATA_TYPE_UBIGINT;

View File

@ -377,14 +377,14 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
goto _OVER; goto _OVER;
} }
taosThreadMutexLock(&pMgmt->createLock); //taosThreadMutexLock(&pMgmt->createLock);
code = vmWriteVnodeListToFile(pMgmt); code = vmWriteVnodeListToFile(pMgmt);
if (code != 0) { if (code != 0) {
code = terrno != 0 ? terrno : code; code = terrno != 0 ? terrno : code;
taosThreadMutexUnlock(&pMgmt->createLock); //taosThreadMutexUnlock(&pMgmt->createLock);
goto _OVER; goto _OVER;
} }
taosThreadMutexUnlock(&pMgmt->createLock); //taosThreadMutexUnlock(&pMgmt->createLock);
_OVER: _OVER:
if (code != 0) { if (code != 0) {
@ -988,7 +988,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMultiMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;

View File

@ -92,10 +92,12 @@ _exit:
static int32_t tsdbSaveFSToFile(STsdbFS *pFS, const char *fname) { static int32_t tsdbSaveFSToFile(STsdbFS *pFS, const char *fname) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
TdFilePtr pFD = NULL;
uint8_t *pData = NULL;
// encode to binary // encode to binary
int32_t size = tsdbFSToBinary(NULL, pFS) + sizeof(TSCKSUM); int32_t size = tsdbFSToBinary(NULL, pFS) + sizeof(TSCKSUM);
uint8_t *pData = taosMemoryMalloc(size); pData = taosMemoryMalloc(size);
if (pData == NULL) { if (pData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
@ -104,32 +106,26 @@ static int32_t tsdbSaveFSToFile(STsdbFS *pFS, const char *fname) {
taosCalcChecksumAppend(0, pData, size); taosCalcChecksumAppend(0, pData, size);
// save to file // save to file
TdFilePtr pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFD == NULL) { if (pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
int64_t n = taosWriteFile(pFD, pData, size); int64_t n = taosWriteFile(pFD, pData, size);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
taosCloseFile(&pFD);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (taosFsyncFile(pFD) < 0) { if (taosFsyncFile(pFD) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
taosCloseFile(&pFD);
TSDB_CHECK_CODE(code, lino, _exit);
} }
taosCloseFile(&pFD);
_exit: _exit:
if (pData) taosMemoryFree(pData);
if (code) { if (code) {
tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname);
} }
taosMemoryFree(pData);
taosCloseFile(&pFD);
return code; return code;
} }
@ -180,11 +176,6 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
// if (size != tsdbLogicToFileSize(pTsdb->fs.pDelFile->size, pTsdb->pVnode->config.tsdbPageSize)) {
// code = TSDB_CODE_FILE_CORRUPTED;
// TSDB_CHECK_CODE(code, lino, _exit);
// }
} }
// SArray<SDFileSet> // SArray<SDFileSet>
@ -327,17 +318,15 @@ static int32_t load_fs(const char *fname, STsdbFS *pFS) {
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
taosCloseFile(&pFD);
// decode binary
code = tsdbBinaryToFS(pData, size, pFS); code = tsdbBinaryToFS(pData, size, pFS);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (pData) taosMemoryFree(pData);
if (code) { if (code) {
tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname);
} }
taosMemoryFree(pData);
taosCloseFile(&pFD);
return code; return code;
} }
@ -383,7 +372,11 @@ static int32_t tsdbNewFileSet(STsdb *pTsdb, SDFileSet *pSetTo, SDFileSet *pSetFr
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
*pSetTo = (SDFileSet){.diskId = pSetFrom->diskId, .fid = pSetFrom->fid, .nSttF = 0}; *pSetTo = (SDFileSet){
.diskId = pSetFrom->diskId,
.fid = pSetFrom->fid,
.nSttF = 0,
};
// head // head
pSetTo->pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); pSetTo->pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile));

View File

@ -41,7 +41,9 @@ static const char *gCurrentFname[] = {
static int32_t create_fs(STsdb *pTsdb, STFileSystem **fs) { static int32_t create_fs(STsdb *pTsdb, STFileSystem **fs) {
fs[0] = taosMemoryCalloc(1, sizeof(*fs[0])); fs[0] = taosMemoryCalloc(1, sizeof(*fs[0]));
if (fs[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (fs[0] == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
fs[0]->tsdb = pTsdb; fs[0]->tsdb = pTsdb;
tsem_init(&fs[0]->canEdit, 0, 1); tsem_init(&fs[0]->canEdit, 0, 1);
@ -76,68 +78,74 @@ int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype) {
static int32_t save_json(const cJSON *json, const char *fname) { static int32_t save_json(const cJSON *json, const char *fname) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
char *data = NULL;
TdFilePtr fp = NULL;
char *data = cJSON_PrintUnformatted(json); data = cJSON_PrintUnformatted(json);
if (data == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (data == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
TdFilePtr fp = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); fp = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (fp == NULL) { if (fp == NULL) {
code = TAOS_SYSTEM_ERROR(code); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit);
goto _exit;
} }
if (taosWriteFile(fp, data, strlen(data)) < 0) { if (taosWriteFile(fp, data, strlen(data)) < 0) {
code = TAOS_SYSTEM_ERROR(code); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit);
goto _exit;
} }
if (taosFsyncFile(fp) < 0) { if (taosFsyncFile(fp) < 0) {
code = TAOS_SYSTEM_ERROR(code); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit);
goto _exit;
} }
taosCloseFile(&fp);
_exit: _exit:
if (code) {
tsdbError("%s failed at %s:%d since %s", __func__, fname, __LINE__, tstrerror(code));
}
taosMemoryFree(data); taosMemoryFree(data);
taosCloseFile(&fp);
return code; return code;
} }
static int32_t load_json(const char *fname, cJSON **json) { static int32_t load_json(const char *fname, cJSON **json) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
char *data = NULL; char *data = NULL;
TdFilePtr fp = taosOpenFile(fname, TD_FILE_READ); TdFilePtr fp = taosOpenFile(fname, TD_FILE_READ);
if (fp == NULL) return TAOS_SYSTEM_ERROR(code); if (fp == NULL) {
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
int64_t size; int64_t size;
if (taosFStatFile(fp, &size, NULL) < 0) { if (taosFStatFile(fp, &size, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(code); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit);
goto _exit;
} }
data = taosMemoryMalloc(size + 1); data = taosMemoryMalloc(size + 1);
if (data == NULL) { if (data == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _exit;
} }
if (taosReadFile(fp, data, size) < 0) { if (taosReadFile(fp, data, size) < 0) {
code = TAOS_SYSTEM_ERROR(code); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit);
goto _exit;
} }
data[size] = '\0'; data[size] = '\0';
json[0] = cJSON_Parse(data); json[0] = cJSON_Parse(data);
if (json[0] == NULL) { if (json[0] == NULL) {
code = TSDB_CODE_FILE_CORRUPTED; TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
goto _exit;
} }
_exit: _exit:
if (code) {
tsdbError("%s failed at %s:%d since %s", __func__, fname, __LINE__, tstrerror(code));
json[0] = NULL;
}
taosCloseFile(&fp); taosCloseFile(&fp);
if (data) taosMemoryFree(data); taosMemoryFree(data);
if (code) json[0] = NULL;
return code; return code;
} }
@ -146,26 +154,25 @@ int32_t save_fs(const TFileSetArray *arr, const char *fname) {
int32_t lino = 0; int32_t lino = 0;
cJSON *json = cJSON_CreateObject(); cJSON *json = cJSON_CreateObject();
if (!json) return TSDB_CODE_OUT_OF_MEMORY; if (json == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
// fmtv // fmtv
if (cJSON_AddNumberToObject(json, "fmtv", 1) == NULL) { if (cJSON_AddNumberToObject(json, "fmtv", 1) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// fset // fset
cJSON *ajson = cJSON_AddArrayToObject(json, "fset"); cJSON *ajson = cJSON_AddArrayToObject(json, "fset");
if (!ajson) { if (!ajson) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
const STFileSet *fset; const STFileSet *fset;
TARRAY2_FOREACH(arr, fset) { TARRAY2_FOREACH(arr, fset) {
cJSON *item = cJSON_CreateObject(); cJSON *item = cJSON_CreateObject();
if (!item) { if (!item) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
cJSON_AddItemToArray(ajson, item); cJSON_AddItemToArray(ajson, item);
@ -226,14 +233,17 @@ static int32_t load_fs(STsdb *pTsdb, const char *fname, TFileSetArray *arr) {
_exit: _exit:
if (code) { if (code) {
tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); tsdbError("%s failed at %sP%d since %s, fname:%s", __func__, __FILE__, lino, tstrerror(code), fname);
}
if (json) {
cJSON_Delete(json);
} }
if (json) cJSON_Delete(json);
return code; return code;
} }
static int32_t apply_commit(STFileSystem *fs) { static int32_t apply_commit(STFileSystem *fs) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
TFileSetArray *fsetArray1 = fs->fSetArr; TFileSetArray *fsetArray1 = fs->fSetArr;
TFileSetArray *fsetArray2 = fs->fSetArrTmp; TFileSetArray *fsetArray2 = fs->fSetArrTmp;
int32_t i1 = 0, i2 = 0; int32_t i1 = 0, i2 = 0;
@ -250,15 +260,15 @@ static int32_t apply_commit(STFileSystem *fs) {
} else if (fset1->fid > fset2->fid) { } else if (fset1->fid > fset2->fid) {
// create new file set with fid of fset2->fid // create new file set with fid of fset2->fid
code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1); code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1);
if (code) return code; TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn); code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn);
if (code) return code; TSDB_CHECK_CODE(code, lino, _exit);
i1++; i1++;
i2++; i2++;
} else { } else {
// edit // edit
code = tsdbTFileSetApplyEdit(fs->tsdb, fset2, fset1); code = tsdbTFileSetApplyEdit(fs->tsdb, fset2, fset1);
if (code) return code; TSDB_CHECK_CODE(code, lino, _exit);
i1++; i1++;
i2++; i2++;
} }
@ -269,15 +279,19 @@ static int32_t apply_commit(STFileSystem *fs) {
} else { } else {
// create new file set with fid of fset2->fid // create new file set with fid of fset2->fid
code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1); code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1);
if (code) return code; TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn); code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn);
if (code) return code; TSDB_CHECK_CODE(code, lino, _exit);
i1++; i1++;
i2++; i2++;
} }
} }
return 0; _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(fs->tsdb->pVnode), lino, code);
}
return code;
} }
static int32_t commit_edit(STFileSystem *fs) { static int32_t commit_edit(STFileSystem *fs) {
@ -303,7 +317,8 @@ static int32_t commit_edit(STFileSystem *fs) {
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(fs->tsdb->pVnode), __func__, lino, tstrerror(code)); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(fs->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbInfo("vgId:%d %s success, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype); tsdbInfo("vgId:%d %s success, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype);
} }
@ -335,7 +350,8 @@ static int32_t abort_edit(STFileSystem *fs) {
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed since %s", TD_VID(fs->tsdb->pVnode), __func__, tstrerror(code)); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(fs->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbInfo("vgId:%d %s success, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype); tsdbInfo("vgId:%d %s success, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype);
} }
@ -360,13 +376,6 @@ static int32_t tsdbFSDoScanAndFixFile(STFileSystem *fs, const STFileObj *fobj) {
} }
found = true; found = true;
/*
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
long s3_size = s3Size(object_name);
if (s3_size > 0) {
return 0;
}
*/
} }
if (!found) { if (!found) {
@ -376,16 +385,6 @@ static int32_t tsdbFSDoScanAndFixFile(STFileSystem *fs, const STFileObj *fobj) {
} }
} }
{ // TODO: check file size
// int64_t fsize;
// if (taosStatFile(fobj->fname, &fsize, NULL, NULL) < 0) {
// code = TAOS_SYSTEM_ERROR(terrno);
// tsdbError("vgId:%d %s failed since file:%s stat failed, reason:%s", TD_VID(fs->tsdb->pVnode), __func__,
// fobj->fname, tstrerror(code));
// return code;
// }
}
return 0; return 0;
} }
@ -408,6 +407,7 @@ static int32_t tsdbFSAddEntryToFileObjHash(STFileHash *hash, const char *fname)
static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) { static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
// init hash table // init hash table
@ -415,14 +415,13 @@ static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) {
hash->numBucket = 4096; hash->numBucket = 4096;
hash->buckets = taosMemoryCalloc(hash->numBucket, sizeof(STFileHashEntry *)); hash->buckets = taosMemoryCalloc(hash->numBucket, sizeof(STFileHashEntry *));
if (hash->buckets == NULL) { if (hash->buckets == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
return code;
} }
// vnode.json // vnode.json
current_fname(fs->tsdb, fname, TSDB_FCURRENT); current_fname(fs->tsdb, fname, TSDB_FCURRENT);
code = tsdbFSAddEntryToFileObjHash(hash, fname); code = tsdbFSAddEntryToFileObjHash(hash, fname);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
// other // other
STFileSet *fset = NULL; STFileSet *fset = NULL;
@ -431,7 +430,7 @@ static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) {
for (int32_t i = 0; i < TSDB_FTYPE_MAX; i++) { for (int32_t i = 0; i < TSDB_FTYPE_MAX; i++) {
if (fset->farr[i] != NULL) { if (fset->farr[i] != NULL) {
code = tsdbFSAddEntryToFileObjHash(hash, fset->farr[i]->fname); code = tsdbFSAddEntryToFileObjHash(hash, fset->farr[i]->fname);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
@ -441,13 +440,14 @@ static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) {
STFileObj *fobj; STFileObj *fobj;
TARRAY2_FOREACH(lvl->fobjArr, fobj) { TARRAY2_FOREACH(lvl->fobjArr, fobj) {
code = tsdbFSAddEntryToFileObjHash(hash, fobj->fname); code = tsdbFSAddEntryToFileObjHash(hash, fobj->fname);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(fs->tsdb->pVnode), lino, code);
tsdbFSDestroyFileObjHash(hash); tsdbFSDestroyFileObjHash(hash);
} }
return code; return code;

View File

@ -35,7 +35,9 @@ int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **wri
int32_t lino = 0; int32_t lino = 0;
writer[0] = taosMemoryCalloc(1, sizeof(SFSetRAWWriter)); writer[0] = taosMemoryCalloc(1, sizeof(SFSetRAWWriter));
if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (writer[0] == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
writer[0]->config[0] = config[0]; writer[0]->config[0] = config[0];

View File

@ -44,9 +44,10 @@ static int32_t tsdbFSetWriteTableDataBegin(SFSetWriter *writer, const TABLEID *t
writer->ctx->tbid->uid = tbid->uid; writer->ctx->tbid->uid = tbid->uid;
code = tsdbUpdateSkmTb(writer->config->tsdb, writer->ctx->tbid, writer->skmTb); code = tsdbUpdateSkmTb(writer->config->tsdb, writer->ctx->tbid, writer->skmTb);
TSDB_CHECK_CODE(code , lino, _exit);
code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, tbid->suid ? tbid->suid : tbid->uid, &writer->pColCmprObj); code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, tbid->suid ? tbid->suid : tbid->uid, &writer->pColCmprObj);
// TSDB_CHECK_CODE(code, lino, _exit); // TODO: TSDB_CHECK_CODE(code, lino, _exit);
writer->blockDataIdx = 0; writer->blockDataIdx = 0;
for (int32_t i = 0; i < ARRAY_SIZE(writer->blockData); i++) { for (int32_t i = 0; i < ARRAY_SIZE(writer->blockData); i++) {
@ -136,7 +137,9 @@ int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer) {
int32_t lino = 0; int32_t lino = 0;
writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); writer[0] = taosMemoryCalloc(1, sizeof(*writer[0]));
if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (writer[0] == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
writer[0]->config[0] = config[0]; writer[0]->config[0] = config[0];

View File

@ -4648,7 +4648,8 @@ void tsdbReaderClose2(STsdbReader* pReader) {
} }
if (pReader->resBlockInfo.freeBlock) { if (pReader->resBlockInfo.freeBlock) {
pReader->resBlockInfo.pResBlock = blockDataDestroy(pReader->resBlockInfo.pResBlock); blockDataDestroy(pReader->resBlockInfo.pResBlock);
pReader->resBlockInfo.pResBlock = NULL;
} }
taosMemoryFree(pSupInfo->colId); taosMemoryFree(pSupInfo->colId);

View File

@ -16,10 +16,12 @@
#include "cos.h" #include "cos.h"
#include "crypt.h" #include "crypt.h"
#include "tsdb.h" #include "tsdb.h"
#include "tsdbDef.h"
#include "vnd.h" #include "vnd.h"
static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
const char *path = pFD->path; const char *path = pFD->path;
int32_t szPage = pFD->szPage; int32_t szPage = pFD->szPage;
int32_t flag = pFD->flag; int32_t flag = pFD->flag;
@ -38,70 +40,27 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
char *dot = strrchr(lc_path, '.'); char *dot = strrchr(lc_path, '.');
if (!dot) { if (!dot) {
tsdbError("unexpected path: %s", lc_path); tsdbError("unexpected path: %s", lc_path);
code = TAOS_SYSTEM_ERROR(ENOENT); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(ENOENT), lino, _exit);
goto _exit;
} }
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", pFD->lcn); snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", pFD->lcn);
pFD->pFD = taosOpenFile(lc_path, flag); pFD->pFD = taosOpenFile(lc_path, flag);
if (pFD->pFD == NULL) { if (pFD->pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
// taosMemoryFree(pFD);
goto _exit;
} }
if (taosStatFile(lc_path, &lc_size, NULL, NULL) < 0) { if (taosStatFile(lc_path, &lc_size, NULL, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
// taosCloseFile(&pFD->pFD);
// taosMemoryFree(pFD);
goto _exit;
} }
} else { } else {
tsdbInfo("no file: %s", path); tsdbInfo("no file: %s", path);
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
// taosMemoryFree(pFD);
goto _exit;
} }
pFD->s3File = 1; pFD->s3File = 1;
/*
const char *object_name = taosDirEntryBaseName((char *)path);
long s3_size = 0;
if (tsS3Enabled) {
long size = s3Size(object_name);
if (size < 0) {
code = terrno = TSDB_CODE_FAILED_TO_CONNECT_S3;
goto _exit;
}
s3_size = size;
}
if (tsS3Enabled && !strncmp(path + strlen(path) - 5, ".data", 5) && s3_size > 0) {
#ifndef S3_BLOCK_CACHE
s3EvictCache(path, s3_size);
s3Get(object_name, path);
pFD->pFD = taosOpenFile(path, flag);
if (pFD->pFD == NULL) {
code = TAOS_SYSTEM_ERROR(ENOENT);
// taosMemoryFree(pFD);
goto _exit;
}
#else
pFD->pFD = (TdFilePtr)&pFD->s3File;
int32_t vid = 0;
sscanf(object_name, "v%df%dver%" PRId64 ".data", &vid, &pFD->fid, &pFD->cid);
pFD->objName = object_name;
// pFD->szFile = s3_size;
#endif
*/
} }
pFD->pBuf = taosMemoryCalloc(1, szPage); pFD->pBuf = taosMemoryCalloc(1, szPage);
if (pFD->pBuf == NULL) { if (pFD->pBuf == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
// taosCloseFile(&pFD->pFD);
// taosMemoryFree(pFD);
goto _exit;
} }
if (lc_size > 0) { if (lc_size > 0) {
@ -114,11 +73,7 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
// not check file size when reading data files. // not check file size when reading data files.
if (flag != TD_FILE_READ /* && !pFD->s3File*/) { if (flag != TD_FILE_READ /* && !pFD->s3File*/) {
if (!lc_size && taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) { if (!lc_size && taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
// taosMemoryFree(pFD->pBuf);
// taosCloseFile(&pFD->pFD);
// taosMemoryFree(pFD);
goto _exit;
} }
} }
@ -126,12 +81,16 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
pFD->szFile = pFD->szFile / szPage; pFD->szFile = pFD->szFile / szPage;
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
// =============== PAGE-WISE FILE =============== // =============== PAGE-WISE FILE ===============
int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn) { int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
STsdbFD *pFD = NULL; STsdbFD *pFD = NULL;
int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; int32_t szPage = pTsdb->pVnode->config.tsdbPageSize;
@ -139,8 +98,7 @@ int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppF
pFD = (STsdbFD *)taosMemoryCalloc(1, sizeof(*pFD) + strlen(path) + 1); pFD = (STsdbFD *)taosMemoryCalloc(1, sizeof(*pFD) + strlen(path) + 1);
if (pFD == NULL) { if (pFD == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _exit;
} }
pFD->path = (char *)&pFD[1]; pFD->path = (char *)&pFD[1];
@ -155,6 +113,9 @@ int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppF
*ppFD = pFD; *ppFD = pFD;
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pTsdb->pVnode), lino, code);
}
return code; return code;
} }
@ -172,12 +133,11 @@ void tsdbCloseFile(STsdbFD **ppFD) {
static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey) { static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
if (!pFD->pFD) { if (!pFD->pFD) {
code = tsdbOpenFileImpl(pFD); code = tsdbOpenFileImpl(pFD);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _exit;
}
} }
if (pFD->pgno > 0) { if (pFD->pgno > 0) {
@ -193,8 +153,7 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e
int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
goto _exit;
} }
taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage); taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage);
@ -223,8 +182,7 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e
n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage); n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
goto _exit;
} }
if (pFD->szFile < pFD->pgno) { if (pFD->szFile < pFD->pgno) {
@ -234,18 +192,20 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e
pFD->pgno = 0; pFD->pgno = 0;
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgorithm, char *encryptKey) { static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
// ASSERT(pgno <= pFD->szFile); // ASSERT(pgno <= pFD->szFile);
if (!pFD->pFD) { if (!pFD->pFD) {
code = tsdbOpenFileImpl(pFD); code = tsdbOpenFileImpl(pFD);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _exit;
}
} }
int64_t offset = PAGE_OFFSET(pgno, pFD->szPage); int64_t offset = PAGE_OFFSET(pgno, pFD->szPage);
@ -257,43 +217,19 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor
offset -= chunkoffset; offset -= chunkoffset;
} }
ASSERT(offset >= 0); ASSERT(offset >= 0);
/*
if (pFD->s3File) {
LRUHandle *handle = NULL;
pFD->blkno = (pgno + tsS3BlockSize - 1) / tsS3BlockSize;
code = tsdbCacheGetBlockS3(pFD->pTsdb->bCache, pFD, &handle);
if (code != TSDB_CODE_SUCCESS || handle == NULL) {
tsdbCacheRelease(pFD->pTsdb->bCache, handle);
if (code == TSDB_CODE_SUCCESS && !handle) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
goto _exit;
}
uint8_t *pBlock = (uint8_t *)taosLRUCacheValue(pFD->pTsdb->bCache, handle);
int64_t blk_offset = (pFD->blkno - 1) * tsS3BlockSize * pFD->szPage;
memcpy(pFD->pBuf, pBlock + (offset - blk_offset), pFD->szPage);
tsdbCacheRelease(pFD->pTsdb->bCache, handle);
} else {
*/
// seek // seek
int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
goto _exit;
} }
// read // read
n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage); n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
goto _exit;
} else if (n < pFD->szPage) { } else if (n < pFD->szPage) {
code = TSDB_CODE_FILE_CORRUPTED; TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
goto _exit;
} }
//} //}
@ -322,19 +258,22 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor
// check // check
if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
code = TSDB_CODE_FILE_CORRUPTED; TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
goto _exit;
} }
pFD->pgno = pgno; pFD->pgno = pgno;
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm, int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
char *encryptKey) { char *encryptKey) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
int64_t bOffset = fOffset % pFD->szPage; int64_t bOffset = fOffset % pFD->szPage;
@ -343,11 +282,11 @@ int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t
do { do {
if (pFD->pgno != pgno) { if (pFD->pgno != pgno) {
code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey); code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
if (pgno <= pFD->szFile) { if (pgno <= pFD->szFile) {
code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey); code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
pFD->pgno = pgno; pFD->pgno = pgno;
} }
@ -362,12 +301,16 @@ int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t
} while (n < size); } while (n < size);
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm, static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
char *encryptKey) { char *encryptKey) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
int64_t n = 0; int64_t n = 0;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
@ -380,7 +323,7 @@ static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int6
while (n < size) { while (n < size) {
if (pFD->pgno != pgno) { if (pFD->pgno != pgno) {
code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey); code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} }
int64_t nRead = TMIN(szPgCont - bOffset, size - n); int64_t nRead = TMIN(szPgCont - bOffset, size - n);
@ -392,15 +335,20 @@ static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int6
} }
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config; SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config;
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize; int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int64_t cOffset = offset % chunksize; int64_t cOffset = offset % chunksize;
int64_t n = 0; int64_t n = 0;
char *buf = NULL;
char *object_name = taosDirEntryBaseName(pFD->path); char *object_name = taosDirEntryBaseName(pFD->path);
char object_name_prefix[TSDB_FILENAME_LEN]; char object_name_prefix[TSDB_FILENAME_LEN];
@ -410,11 +358,13 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo
char *dot = strrchr(object_name_prefix, '.'); char *dot = strrchr(object_name_prefix, '.');
if (!dot) { if (!dot) {
tsdbError("unexpected path: %s", object_name_prefix); tsdbError("unexpected path: %s", object_name_prefix);
code = TAOS_SYSTEM_ERROR(ENOENT); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(ENOENT), lino, _exit);
goto _exit;
} }
char *buf = taosMemoryCalloc(1, size); buf = taosMemoryCalloc(1, size);
if (buf == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
for (int32_t chunkno = offset / chunksize + 1; n < size; ++chunkno) { for (int32_t chunkno = offset / chunksize + 1; n < size; ++chunkno) {
int64_t nRead = TMIN(chunksize - cOffset, size - n); int64_t nRead = TMIN(chunksize - cOffset, size - n);
@ -423,20 +373,14 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo
// read last chunk // read last chunk
int64_t ret = taosLSeekFile(pFD->pFD, chunksize * (chunkno - pFD->lcn) + cOffset, SEEK_SET); int64_t ret = taosLSeekFile(pFD->pFD, chunksize * (chunkno - pFD->lcn) + cOffset, SEEK_SET);
if (ret < 0) { if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
taosMemoryFree(buf);
goto _exit;
} }
ret = taosReadFile(pFD->pFD, buf + n, nRead); ret = taosReadFile(pFD->pFD, buf + n, nRead);
if (ret < 0) { if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
taosMemoryFree(buf);
goto _exit;
} else if (ret < nRead) { } else if (ret < nRead) {
code = TSDB_CODE_FILE_CORRUPTED; TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
taosMemoryFree(buf);
goto _exit;
} }
} else { } else {
uint8_t *pBlock = NULL; uint8_t *pBlock = NULL;
@ -444,10 +388,7 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", chunkno); snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", chunkno);
code = s3GetObjectBlock(object_name_prefix, cOffset, nRead, check, &pBlock); code = s3GetObjectBlock(object_name_prefix, cOffset, nRead, check, &pBlock);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _exit);
taosMemoryFree(buf);
goto _exit;
}
memcpy(buf + n, pBlock, nRead); memcpy(buf + n, pBlock, nRead);
taosMemoryFree(pBlock); taosMemoryFree(pBlock);
@ -457,14 +398,19 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo
cOffset = 0; cOffset = 0;
} }
*ppBlock = buf;
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
taosMemoryFree(buf);
} else {
*ppBlock = (uint8_t *)buf;
}
return code; return code;
} }
static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) { static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
int64_t n = 0; int64_t n = 0;
int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage); int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage);
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
@ -486,7 +432,7 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
if (handle) { if (handle) {
tsdbCacheRelease(pFD->pTsdb->pgCache, handle); tsdbCacheRelease(pFD->pTsdb->pgCache, handle);
} }
goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} }
if (!handle) { if (!handle) {
@ -499,8 +445,7 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
// check // check
if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
code = TSDB_CODE_FILE_CORRUPTED; TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
goto _exit;
} }
pFD->pgno = pgno; pFD->pgno = pgno;
@ -525,16 +470,9 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
} }
int64_t retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage; int64_t retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage;
/*
code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, 1, &pBlock);
if (code != TSDB_CODE_SUCCESS) {
goto _exit;
}
*/
code = tsdbReadFileBlock(pFD, retrieve_offset, retrieve_size, 1, &pBlock); code = tsdbReadFileBlock(pFD, retrieve_offset, retrieve_size, 1, &pBlock);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _exit);
goto _exit;
}
// 3, Store Pages in Cache // 3, Store Pages in Cache
int nPage = pgnoEnd - pgno + 1; int nPage = pgnoEnd - pgno + 1;
for (int i = 0; i < nPage; ++i) { for (int i = 0; i < nPage; ++i) {
@ -550,8 +488,7 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
// check // check
if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
code = TSDB_CODE_FILE_CORRUPTED; TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
goto _exit;
} }
pFD->pgno = pgno; pFD->pgno = pgno;
@ -568,59 +505,72 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
} }
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint, int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint,
int32_t encryptAlgorithm, char *encryptKey) { int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
if (!pFD->pFD) { if (!pFD->pFD) {
code = tsdbOpenFileImpl(pFD); code = tsdbOpenFileImpl(pFD);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _exit;
}
} }
if (pFD->s3File && pFD->lcn > 1 /* && tsS3BlockSize < 0*/) { if (pFD->s3File && pFD->lcn > 1 /* && tsS3BlockSize < 0*/) {
return tsdbReadFileS3(pFD, offset, pBuf, size, szHint); code = tsdbReadFileS3(pFD, offset, pBuf, size, szHint);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
return tsdbReadFileImp(pFD, offset, pBuf, size, encryptAlgorithm, encryptKey); code = tsdbReadFileImp(pFD, offset, pBuf, size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint, int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint,
int32_t encryptAlgorithm, char *encryptKey) { int32_t encryptAlgorithm, char *encryptKey) {
int32_t code; int32_t code;
int32_t lino;
code = tBufferEnsureCapacity(buffer, buffer->size + size); code = tBufferEnsureCapacity(buffer, buffer->size + size);
if (code) return code; TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadFile(pFD, offset, (uint8_t *)tBufferGetDataEnd(buffer), size, szHint, encryptAlgorithm, encryptKey); code = tsdbReadFile(pFD, offset, (uint8_t *)tBufferGetDataEnd(buffer), size, szHint, encryptAlgorithm, encryptKey);
if (code) return code; TSDB_CHECK_CODE(code, lino, _exit);
buffer->size += size; buffer->size += size;
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey) { int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0; int32_t code = 0;
/* int32_t lino;
if (pFD->s3File) {
tsdbWarn("%s file: %s", __func__, pFD->path);
return code;
}
*/
code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey); code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
if (taosFsyncFile(pFD->pFD) < 0) { if (taosFsyncFile(pFD->pFD) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
goto _exit;
} }
_exit: _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code);
}
return code; return code;
} }
@ -635,8 +585,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
// alloc // alloc
pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader)); pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) { if (pReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
pReader->pTsdb = pTsdb; pReader->pTsdb = pTsdb;
pReader->pSet = pSet; pReader->pSet = pSet;
@ -711,22 +660,25 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) {
int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SHeadFile *pHeadFile = pReader->pSet->pHeadF; SHeadFile *pHeadFile = pReader->pSet->pHeadF;
int64_t offset = pHeadFile->offset; int64_t offset = pHeadFile->offset;
int64_t size = pHeadFile->size - offset; int64_t size = pHeadFile->size - offset;
taosArrayClear(aBlockIdx); taosArrayClear(aBlockIdx);
if (size == 0) return code; if (size == 0) {
return code;
}
// alloc // alloc
code = tRealloc(&pReader->aBuf[0], size); code = tRealloc(&pReader->aBuf[0], size);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// read // read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// decode // decode
int64_t n = 0; int64_t n = 0;
@ -735,37 +687,39 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
n += tGetBlockIdx(pReader->aBuf[0] + n, &blockIdx); n += tGetBlockIdx(pReader->aBuf[0] + n, &blockIdx);
if (taosArrayPush(aBlockIdx, &blockIdx) == NULL) { if (taosArrayPush(aBlockIdx, &blockIdx) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
} }
ASSERT(n == size); ASSERT(n == size);
return code; _exit:
if (code) {
_err: TSDB_ERROR_LOG(TD_VID(pReader->pTsdb->pVnode), lino, code);
tsdbError("vgId:%d, read block idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) { int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SSttFile *pSttFile = pReader->pSet->aSttF[iStt]; SSttFile *pSttFile = pReader->pSet->aSttF[iStt];
int64_t offset = pSttFile->offset; int64_t offset = pSttFile->offset;
int64_t size = pSttFile->size - offset; int64_t size = pSttFile->size - offset;
taosArrayClear(aSttBlk); taosArrayClear(aSttBlk);
if (size == 0) return code; if (size == 0) {
return 0;
}
// alloc // alloc
code = tRealloc(&pReader->aBuf[0], size); code = tRealloc(&pReader->aBuf[0], size);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// read // read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// decode // decode
int64_t n = 0; int64_t n = 0;
@ -774,33 +728,33 @@ int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) {
n += tGetSttBlk(pReader->aBuf[0] + n, &sttBlk); n += tGetSttBlk(pReader->aBuf[0] + n, &sttBlk);
if (taosArrayPush(aSttBlk, &sttBlk) == NULL) { if (taosArrayPush(aSttBlk, &sttBlk) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
} }
ASSERT(n == size); ASSERT(n == size);
return code; _exit:
if (code) {
_err: TSDB_ERROR_LOG(TD_VID(pReader->pTsdb->pVnode), lino, code);
tsdbError("vgId:%d, read stt blk failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mDataBlk) { int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mDataBlk) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
int64_t offset = pBlockIdx->offset; int64_t offset = pBlockIdx->offset;
int64_t size = pBlockIdx->size; int64_t size = pBlockIdx->size;
// alloc // alloc
code = tRealloc(&pReader->aBuf[0], size); code = tRealloc(&pReader->aBuf[0], size);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// read // read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// decode // decode
int32_t n; int32_t n;
@ -808,10 +762,10 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m
if (code) goto _err; if (code) goto _err;
ASSERT(n == size); ASSERT(n == size);
return code; _exit:
if (code) {
_err: TSDB_ERROR_LOG(TD_VID(pReader->pTsdb->pVnode), lino, code);
tsdbError("vgId:%d, read block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
@ -832,8 +786,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb
// alloc // alloc
pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader)); pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader));
if (pDelFReader == NULL) { if (pDelFReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _exit;
} }
// open impl // open impl
@ -842,15 +795,13 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb
tsdbDelFileName(pTsdb, pFile, fname); tsdbDelFileName(pTsdb, pFile, fname);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pDelFReader->pReadH, 0); code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pDelFReader->pReadH, 0);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
taosMemoryFree(pDelFReader);
goto _exit;
}
_exit: _exit:
if (code) { if (code) {
*ppReader = NULL; *ppReader = NULL;
tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); TSDB_ERROR_LOG(TD_VID(pTsdb->pVnode), lino, code);
taosMemoryFree(pDelFReader);
} else { } else {
*ppReader = pDelFReader; *ppReader = pDelFReader;
} }
@ -879,6 +830,7 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData
int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer) { int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
int64_t offset = pDelIdx->offset; int64_t offset = pDelIdx->offset;
int64_t size = pDelIdx->size; int64_t size = pDelIdx->size;
int64_t n; int64_t n;
@ -887,13 +839,13 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa
// alloc // alloc
code = tRealloc(&pReader->aBuf[0], size); code = tRealloc(&pReader->aBuf[0], size);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// read // read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// // decode // // decode
n = 0; n = 0;
@ -905,22 +857,22 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa
continue; continue;
} }
if (taosArrayPush(aDelData, &delData) == NULL) { if (taosArrayPush(aDelData, &delData) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
} }
ASSERT(n == size); ASSERT(n == size);
return code; _exit:
if (code) {
_err: TSDB_ERROR_LOG(TD_VID(pReader->pTsdb->pVnode), lino, code);
tsdbError("vgId:%d, read del data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
int32_t n; int32_t n;
int64_t offset = pReader->fDel.offset; int64_t offset = pReader->fDel.offset;
int64_t size = pReader->fDel.size - offset; int64_t size = pReader->fDel.size - offset;
@ -929,13 +881,13 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
// alloc // alloc
code = tRealloc(&pReader->aBuf[0], size); code = tRealloc(&pReader->aBuf[0], size);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// read // read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// decode // decode
n = 0; n = 0;
@ -945,16 +897,15 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx); n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx);
if (taosArrayPush(aDelIdx, &delIdx) == NULL) { if (taosArrayPush(aDelIdx, &delIdx) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
} }
ASSERT(n == size); ASSERT(n == size);
return code; _exit:
if (code) {
_err: TSDB_ERROR_LOG(TD_VID(pReader->pTsdb->pVnode), lino, code);
tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }

View File

@ -607,7 +607,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pCo
blockDataAppendColInfo(pBlock, &idata); blockDataAppendColInfo(pBlock, &idata);
} }
blockDecode(pBlock, pStart); (void) blockDecode(pBlock, pStart);
blockDataEnsureCapacity(pRes, pBlock->info.rows); blockDataEnsureCapacity(pRes, pBlock->info.rows);
// data from mnode // data from mnode

View File

@ -970,7 +970,7 @@ void cleanupExprSupp(SExprSupp* pSupp) {
taosMemoryFree(pSupp->rowEntryInfoOffset); taosMemoryFree(pSupp->rowEntryInfoOffset);
} }
void cleanupBasicInfo(SOptrBasicInfo* pInfo) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } void cleanupBasicInfo(SOptrBasicInfo* pInfo) { blockDataDestroy(pInfo->pRes); pInfo->pRes = NULL;}
bool groupbyTbname(SNodeList* pGroupList) { bool groupbyTbname(SNodeList* pGroupList) {
bool bytbname = false; bool bytbname = false;

View File

@ -310,8 +310,10 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
void destroyFillOperatorInfo(void* param) { void destroyFillOperatorInfo(void* param) {
SFillOperatorInfo* pInfo = (SFillOperatorInfo*)param; SFillOperatorInfo* pInfo = (SFillOperatorInfo*)param;
pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo);
pInfo->pRes = blockDataDestroy(pInfo->pRes); blockDataDestroy(pInfo->pRes);
pInfo->pFinalRes = blockDataDestroy(pInfo->pFinalRes); pInfo->pRes = NULL;
blockDataDestroy(pInfo->pFinalRes);
pInfo->pFinalRes = NULL;
cleanupExprSupp(&pInfo->noFillExprSupp); cleanupExprSupp(&pInfo->noFillExprSupp);

View File

@ -1065,7 +1065,8 @@ static void destroyHashJoinOperator(void* param) {
hJoinFreeTableInfo(&pJoinOperator->tbs[0]); hJoinFreeTableInfo(&pJoinOperator->tbs[0]);
hJoinFreeTableInfo(&pJoinOperator->tbs[1]); hJoinFreeTableInfo(&pJoinOperator->tbs[1]);
pJoinOperator->finBlk = blockDataDestroy(pJoinOperator->finBlk); blockDataDestroy(pJoinOperator->finBlk);
pJoinOperator->finBlk = NULL;
taosMemoryFreeClear(pJoinOperator->pResColMap); taosMemoryFreeClear(pJoinOperator->pResColMap);
taosArrayDestroyEx(pJoinOperator->pRowBufs, hJoinFreeBufPage); taosArrayDestroyEx(pJoinOperator->pRowBufs, hJoinFreeBufPage);

View File

@ -3305,8 +3305,10 @@ void mJoinDestroyWindowCtx(SMJoinOperatorInfo* pJoin) {
mWinJoinResetWindowCache(pCtx, &pCtx->cache); mWinJoinResetWindowCache(pCtx, &pCtx->cache);
pCtx->finBlk = blockDataDestroy(pCtx->finBlk); blockDataDestroy(pCtx->finBlk);
pCtx->cache.outBlk = blockDataDestroy(pCtx->cache.outBlk); pCtx->finBlk = NULL;
blockDataDestroy(pCtx->cache.outBlk);
pCtx->cache.outBlk = NULL;
taosArrayDestroy(pCtx->cache.grps); taosArrayDestroy(pCtx->cache.grps);
} }
@ -3378,9 +3380,11 @@ int32_t mJoinInitWindowCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* p
void mJoinDestroyMergeCtx(SMJoinOperatorInfo* pJoin) { void mJoinDestroyMergeCtx(SMJoinOperatorInfo* pJoin) {
SMJoinMergeCtx* pCtx = &pJoin->ctx.mergeCtx; SMJoinMergeCtx* pCtx = &pJoin->ctx.mergeCtx;
blockDataDestroy(pCtx->finBlk);
blockDataDestroy(pCtx->midBlk);
pCtx->finBlk = blockDataDestroy(pCtx->finBlk); pCtx->finBlk = NULL;
pCtx->midBlk = blockDataDestroy(pCtx->midBlk); pCtx->midBlk = NULL;
} }

View File

@ -232,8 +232,11 @@ int32_t getSortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, u
void destroySortMergeOperatorInfo(void* param) { void destroySortMergeOperatorInfo(void* param) {
SSortMergeInfo* pSortMergeInfo = param; SSortMergeInfo* pSortMergeInfo = param;
pSortMergeInfo->pInputBlock = blockDataDestroy(pSortMergeInfo->pInputBlock); blockDataDestroy(pSortMergeInfo->pInputBlock);
pSortMergeInfo->pIntermediateBlock = blockDataDestroy(pSortMergeInfo->pIntermediateBlock); pSortMergeInfo->pInputBlock = NULL;
blockDataDestroy(pSortMergeInfo->pIntermediateBlock);
pSortMergeInfo->pIntermediateBlock = NULL;
taosArrayDestroy(pSortMergeInfo->matchInfo.pList); taosArrayDestroy(pSortMergeInfo->matchInfo.pList);
@ -429,7 +432,8 @@ SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) {
void destroyMultiwayMergeOperatorInfo(void* param) { void destroyMultiwayMergeOperatorInfo(void* param) {
SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)param; SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)param;
pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); blockDataDestroy(pInfo->binfo.pRes);
pInfo->binfo.pRes = NULL;
if (NULL != gMultiwayMergeFps[pInfo->type].closeFn) { if (NULL != gMultiwayMergeFps[pInfo->type].closeFn) {
(*gMultiwayMergeFps[pInfo->type].closeFn)(&pInfo->sortMergeInfo); (*gMultiwayMergeFps[pInfo->type].closeFn)(&pInfo->sortMergeInfo);

View File

@ -220,14 +220,14 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo*
return code; return code;
} }
static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg* pColsAgg, int32_t numOfCols, static int32_t doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg* pColsAgg, int32_t numOfCols,
int32_t numOfRows) { int32_t numOfRows, bool *keep) {
if (pColsAgg == NULL || pFilterInfo == NULL) { if (pColsAgg == NULL || pFilterInfo == NULL) {
return true; *keep = true;
return TSDB_CODE_SUCCESS;
} }
bool keep = filterRangeExecute(pFilterInfo, pColsAgg, numOfCols, numOfRows); return filterRangeExecute(pFilterInfo, pColsAgg, numOfCols, numOfRows, keep);
return keep;
} }
static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) {
@ -351,7 +351,11 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo);
if (success) { if (success) {
size_t size = taosArrayGetSize(pBlock->pDataBlock); size_t size = taosArrayGetSize(pBlock->pDataBlock);
bool keep = doFilterByBlockSMA(pOperator->exprSupp.pFilterInfo, pBlock->pBlockAgg, size, pBlockInfo->rows); bool keep = false;
int32_t code = doFilterByBlockSMA(pOperator->exprSupp.pFilterInfo, pBlock->pBlockAgg, size, pBlockInfo->rows, &keep);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
if (!keep) { if (!keep) {
qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64,
GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
@ -3785,7 +3789,8 @@ static void destroyTagScanOperatorInfo(void* param) {
taosArrayDestroy(pInfo->aFilterIdxs); taosArrayDestroy(pInfo->aFilterIdxs);
taosArrayDestroyEx(pInfo->aUidTags, tagScanFreeUidTag); taosArrayDestroyEx(pInfo->aUidTags, tagScanFreeUidTag);
pInfo->pRes = blockDataDestroy(pInfo->pRes); blockDataDestroy(pInfo->pRes);
pInfo->pRes = NULL;
taosArrayDestroy(pInfo->matchInfo.pList); taosArrayDestroy(pInfo->matchInfo.pList);
pInfo->pTableListInfo = tableListDestroy(pInfo->pTableListInfo); pInfo->pTableListInfo = tableListDestroy(pInfo->pTableListInfo);
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
@ -4738,15 +4743,19 @@ void destroyTableMergeScanOperatorInfo(void* param) {
pTableScanInfo->pSortHandle = NULL; pTableScanInfo->pSortHandle = NULL;
taosHashCleanup(pTableScanInfo->mSkipTables); taosHashCleanup(pTableScanInfo->mSkipTables);
pTableScanInfo->mSkipTables = NULL; pTableScanInfo->mSkipTables = NULL;
pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); blockDataDestroy(pTableScanInfo->pSortInputBlock);
pTableScanInfo->pSortInputBlock = NULL;
// end one reader variable // end one reader variable
cleanupQueryTableDataCond(&pTableScanInfo->base.cond); cleanupQueryTableDataCond(&pTableScanInfo->base.cond);
destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI);
pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); blockDataDestroy(pTableScanInfo->pResBlock);
pTableScanInfo->pResBlock = NULL;
// remove it from the task->result list // remove it from the task->result list
pTableScanInfo->pReaderBlock = blockDataDestroy(pTableScanInfo->pReaderBlock); blockDataDestroy(pTableScanInfo->pReaderBlock);
pTableScanInfo->pReaderBlock = NULL;
taosArrayDestroy(pTableScanInfo->pSortInfo); taosArrayDestroy(pTableScanInfo->pSortInfo);
stopSubTablesTableMergeScan(pTableScanInfo); stopSubTablesTableMergeScan(pTableScanInfo);

View File

@ -368,7 +368,8 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
void destroySortOperatorInfo(void* param) { void destroySortOperatorInfo(void* param) {
SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param; SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param;
pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); blockDataDestroy(pInfo->binfo.pRes);
pInfo->binfo.pRes = NULL;
tsortDestroySortHandle(pInfo->pSortHandle); tsortDestroySortHandle(pInfo->pSortHandle);
taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->pSortInfo);
@ -611,7 +612,8 @@ int32_t getGroupSortExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, u
void destroyGroupSortOperatorInfo(void* param) { void destroyGroupSortOperatorInfo(void* param) {
SGroupSortOperatorInfo* pInfo = (SGroupSortOperatorInfo*)param; SGroupSortOperatorInfo* pInfo = (SGroupSortOperatorInfo*)param;
pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); blockDataDestroy(pInfo->binfo.pRes);
pInfo->binfo.pRes = NULL;
taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->pSortInfo);
taosArrayDestroy(pInfo->matchInfo.pList); taosArrayDestroy(pInfo->matchInfo.pList);

View File

@ -126,9 +126,12 @@ static void destroyStreamFillOperatorInfo(void* param) {
SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param; SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param;
pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo); pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo);
pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup); pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup);
pInfo->pRes = blockDataDestroy(pInfo->pRes); blockDataDestroy(pInfo->pRes);
pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); pInfo->pRes = NULL;
pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); blockDataDestroy(pInfo->pSrcBlock);
pInfo->pSrcBlock = NULL;
blockDataDestroy(pInfo->pDelRes);
pInfo->pDelRes = NULL;
taosArrayDestroy(pInfo->matchInfo.pList); taosArrayDestroy(pInfo->matchInfo.pList);
pInfo->matchInfo.pList = NULL; pInfo->matchInfo.pList = NULL;
taosMemoryFree(pInfo); taosMemoryFree(pInfo);

View File

@ -1087,7 +1087,8 @@ _error:
void destroyTimeSliceOperatorInfo(void* param) { void destroyTimeSliceOperatorInfo(void* param) {
STimeSliceOperatorInfo* pInfo = (STimeSliceOperatorInfo*)param; STimeSliceOperatorInfo* pInfo = (STimeSliceOperatorInfo*)param;
pInfo->pRes = blockDataDestroy(pInfo->pRes); blockDataDestroy(pInfo->pRes);
pInfo->pRes = NULL;
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pPrevRow); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pPrevRow); ++i) {
SGroupKeys* pKey = taosArrayGet(pInfo->pPrevRow, i); SGroupKeys* pKey = taosArrayGet(pInfo->pPrevRow, i);

View File

@ -555,7 +555,8 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT
(*numOfCompleted) += 1; (*numOfCompleted) += 1;
pSource->src.rowIndex = -1; pSource->src.rowIndex = -1;
pSource->pageIndex = -1; pSource->pageIndex = -1;
pSource->src.pBlock = blockDataDestroy(pSource->src.pBlock); blockDataDestroy(pSource->src.pBlock);
pSource->src.pBlock = NULL;
} else { } else {
if (pSource->pageIndex % 512 == 0) { if (pSource->pageIndex % 512 == 0) {
qDebug("begin source %p page %d", pSource, pSource->pageIndex); qDebug("begin source %p page %d", pSource, pSource->pageIndex);

View File

@ -143,12 +143,24 @@ int32_t executeMakePointFunc(SColumnInfoData *pInputData[], int32_t iLeft, int32
getDoubleValueFn[1]= getVectorDoubleValueFn(pInputData[1]->info.type); getDoubleValueFn[1]= getVectorDoubleValueFn(pInputData[1]->info.type);
unsigned char *output = NULL; unsigned char *output = NULL;
code = doMakePointFunc(getDoubleValueFn[0](pInputData[0]->pData, iLeft), getDoubleValueFn[1](pInputData[1]->pData, iRight), &output); double leftRes = 0;
if (code != TSDB_CODE_SUCCESS) { double rightRes = 0;
code = getDoubleValueFn[0](pInputData[0]->pData, iLeft, &leftRes);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
code = getDoubleValueFn[1](pInputData[1]->pData, iRight, &rightRes);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
code = doMakePointFunc(leftRes, rightRes, &output);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
code = colDataSetVal(pOutputData, TMAX(iLeft, iRight), output, (output == NULL));
if (TSDB_CODE_SUCCESS != code) {
goto _exit; goto _exit;
} }
colDataSetVal(pOutputData, TMAX(iLeft, iRight), output, (output == NULL));
_exit: _exit:
if (output) { if (output) {

View File

@ -5045,7 +5045,13 @@ static int32_t hashJoinOptSplitPrimFromLogicCond(SNode **pCondition, SNode **pPr
SNodeList *pPrimaryKeyConds = NULL; SNodeList *pPrimaryKeyConds = NULL;
SNode *pCond = NULL; SNode *pCond = NULL;
WHERE_EACH(pCond, pLogicCond->pParameterList) { WHERE_EACH(pCond, pLogicCond->pParameterList) {
if (filterIsMultiTableColsCond(pCond) || COND_TYPE_PRIMARY_KEY != filterClassifyCondition(pCond)) { bool result = false;
code = filterIsMultiTableColsCond(pCond, &result);
if (TSDB_CODE_SUCCESS != code) {
break;
}
if (result || COND_TYPE_PRIMARY_KEY != filterClassifyCondition(pCond)) {
WHERE_NEXT; WHERE_NEXT;
continue; continue;
} }
@ -5089,7 +5095,12 @@ int32_t hashJoinOptSplitPrimCond(SNode **pCondition, SNode **pPrimaryKeyCond) {
} }
bool needOutput = false; bool needOutput = false;
if (filterIsMultiTableColsCond(*pCondition)) { bool result = false;
int32_t code = filterIsMultiTableColsCond(*pCondition, &result);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
if (result) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

View File

@ -98,7 +98,7 @@ typedef struct SFilterRange {
typedef bool (*rangeCompFunc)(const void *, const void *, const void *, const void *, __compar_fn_t); typedef bool (*rangeCompFunc)(const void *, const void *, const void *, const void *, __compar_fn_t);
typedef int32_t (*filter_desc_compare_func)(const void *, const void *); typedef int32_t (*filter_desc_compare_func)(const void *, const void *);
typedef bool (*filter_exec_func)(void *, int32_t, SColumnInfoData *, SColumnDataAgg *, int16_t, int32_t *); typedef int32_t (*filter_exec_func)(void *, int32_t, SColumnInfoData *, SColumnDataAgg *, int16_t, int32_t *, bool *);
typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char *, void **); typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char *, void **);
typedef struct SFilterDataInfo { typedef struct SFilterDataInfo {
@ -363,7 +363,8 @@ struct SFilterInfo {
} while (0) } while (0)
#define INSERT_RANGE(ctx, r, ra) \ #define INSERT_RANGE(ctx, r, ra) \
do { \ do { \
SFilterRangeNode *n = filterNewRange(ctx, ra); \ SFilterRangeNode *n = NULL; \
FLT_ERR_RET(filterNewRange(ctx, ra, &n)); \
n->prev = (r)->prev; \ n->prev = (r)->prev; \
if ((r)->prev) { \ if ((r)->prev) { \
(r)->prev->next = n; \ (r)->prev->next = n; \
@ -375,7 +376,8 @@ struct SFilterInfo {
} while (0) } while (0)
#define APPEND_RANGE(ctx, r, ra) \ #define APPEND_RANGE(ctx, r, ra) \
do { \ do { \
SFilterRangeNode *n = filterNewRange(ctx, ra); \ SFilterRangeNode *n = NULL; \
FLT_ERR_RET(filterNewRange(ctx, ra, &n)); \
n->prev = (r); \ n->prev = (r); \
if (r) { \ if (r) { \
(r)->next = n; \ (r)->next = n; \
@ -462,7 +464,9 @@ struct SFilterInfo {
do { \ do { \
(colInfo).type = RANGE_TYPE_UNIT; \ (colInfo).type = RANGE_TYPE_UNIT; \
(colInfo).dataType = FILTER_UNIT_DATA_TYPE(u); \ (colInfo).dataType = FILTER_UNIT_DATA_TYPE(u); \
taosArrayPush((SArray *)((colInfo).info), &u); \ if (taosArrayPush((SArray *)((colInfo).info), &u) == NULL) { \
FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); \
} \
} while (0) } while (0)
#define FILTER_PUSH_VAR_HASH(colInfo, ha) \ #define FILTER_PUSH_VAR_HASH(colInfo, ha) \
do { \ do { \
@ -494,7 +498,7 @@ struct SFilterInfo {
#define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY) #define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY)
extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right); extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right);
extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr); extern int32_t filterGetCompFunc(__compar_fn_t *func, int32_t type, int32_t optr);
extern __compar_fn_t filterGetCompFuncEx(int32_t lType, int32_t rType, int32_t optr); extern __compar_fn_t filterGetCompFuncEx(int32_t lType, int32_t rType, int32_t optr);
#ifdef __cplusplus #ifdef __cplusplus

View File

@ -95,11 +95,11 @@ int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode);
#define GET_PARAM_PRECISON(_c) ((_c)->columnData->info.precision) #define GET_PARAM_PRECISON(_c) ((_c)->columnData->info.precision)
void sclFreeParam(SScalarParam* param); void sclFreeParam(SScalarParam* param);
void doVectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, int32_t doVectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows,
int32_t _ord, int32_t optr); int32_t _ord, int32_t optr);
void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, int32_t vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows,
int32_t _ord, int32_t optr); int32_t _ord, int32_t optr);
void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr); int32_t vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -29,43 +29,54 @@ typedef struct SSclVectorConvCtx {
int16_t outType; int16_t outType;
} SSclVectorConvCtx; } SSclVectorConvCtx;
typedef double (*_getDoubleValue_fn_t)(void *src, int32_t index); typedef int32_t (*_getDoubleValue_fn_t)(void *src, int32_t index, double *out);
static FORCE_INLINE double getVectorDoubleValue_TINYINT(void *src, int32_t index) { static FORCE_INLINE int32_t getVectorDoubleValue_TINYINT(void *src, int32_t index, double *out) {
return (double)*((int8_t *)src + index); *out = (double)*((int8_t *)src + index);
return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE double getVectorDoubleValue_UTINYINT(void *src, int32_t index) { static FORCE_INLINE int32_t getVectorDoubleValue_UTINYINT(void *src, int32_t index, double *out) {
return (double)*((uint8_t *)src + index); *out = (double)*((uint8_t *)src + index);
return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE double getVectorDoubleValue_SMALLINT(void *src, int32_t index) { static FORCE_INLINE int32_t getVectorDoubleValue_SMALLINT(void *src, int32_t index, double *out) {
return (double)*((int16_t *)src + index); *out = (double)*((int16_t *)src + index);
return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE double getVectorDoubleValue_USMALLINT(void *src, int32_t index) { static FORCE_INLINE int32_t getVectorDoubleValue_USMALLINT(void *src, int32_t index, double *out) {
return (double)*((uint16_t *)src + index); *out = (double)*((uint16_t *)src + index);
return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE double getVectorDoubleValue_INT(void *src, int32_t index) { static FORCE_INLINE int32_t getVectorDoubleValue_INT(void *src, int32_t index, double *out) {
return (double)*((int32_t *)src + index); *out = (double)*((int32_t *)src + index);
return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE double getVectorDoubleValue_UINT(void *src, int32_t index) { static FORCE_INLINE int32_t getVectorDoubleValue_UINT(void *src, int32_t index, double *out) {
return (double)*((uint32_t *)src + index); *out = (double)*((uint32_t *)src + index);
return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE double getVectorDoubleValue_BIGINT(void *src, int32_t index) { static FORCE_INLINE int32_t getVectorDoubleValue_BIGINT(void *src, int32_t index, double *out) {
return (double)*((int64_t *)src + index); *out = (double)*((int64_t *)src + index);
return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE double getVectorDoubleValue_UBIGINT(void *src, int32_t index) { static FORCE_INLINE int32_t getVectorDoubleValue_UBIGINT(void *src, int32_t index, double *out) {
return (double)*((uint64_t *)src + index); *out = (double)*((uint64_t *)src + index);
return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE double getVectorDoubleValue_FLOAT(void *src, int32_t index) { static FORCE_INLINE int32_t getVectorDoubleValue_FLOAT(void *src, int32_t index, double *out) {
return (double)*((float *)src + index); *out = (double)*((float *)src + index);
return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE double getVectorDoubleValue_DOUBLE(void *src, int32_t index) { static FORCE_INLINE int32_t getVectorDoubleValue_DOUBLE(void *src, int32_t index, double *out) {
return (double)*((double *)src + index); *out = (double)*((double *)src + index);
return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE double getVectorDoubleValue_BOOL(void *src, int32_t index) { static FORCE_INLINE int32_t getVectorDoubleValue_BOOL(void *src, int32_t index, double *out) {
return (double)*((bool *)src + index); *out = (double)*((bool *)src + index);
return TSDB_CODE_SUCCESS;
} }
double getVectorDoubleValue_JSON(void *src, int32_t index); int32_t getVectorDoubleValue_JSON(void *src, int32_t index, double *out);
static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) {
_getDoubleValue_fn_t p = NULL; _getDoubleValue_fn_t p = NULL;
@ -103,8 +114,8 @@ static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType)
return p; return p;
} }
typedef void (*_bufConverteFunc)(char *buf, SScalarParam *pOut, int32_t outType, int32_t *overflow); typedef int32_t (*_bufConverteFunc)(char *buf, SScalarParam *pOut, int32_t outType, int32_t *overflow);
typedef void (*_bin_scalar_fn_t)(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *output, int32_t order); typedef int32_t (*_bin_scalar_fn_t)(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *output, int32_t order);
_bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator); _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator);
#ifdef __cplusplus #ifdef __cplusplus

File diff suppressed because it is too large Load Diff

View File

@ -72,9 +72,16 @@ int32_t sclConvertValueToSclParam(SValueNode *pValueNode, SScalarParam *out, int
return code; return code;
} }
colDataSetVal(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); code = colDataSetVal(in.columnData, 0, nodesGetValueFromNode(pValueNode), false);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = colInfoDataEnsureCapacity(out->columnData, 1, true);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
colInfoDataEnsureCapacity(out->columnData, 1, true);
code = vectorConvertSingleColImpl(&in, out, overflow, -1, -1); code = vectorConvertSingleColImpl(&in, out, overflow, -1, -1);
sclFreeParam(&in); sclFreeParam(&in);
@ -84,6 +91,7 @@ int32_t sclConvertValueToSclParam(SValueNode *pValueNode, SScalarParam *out, int
int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockList) { int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockList) {
SSDataBlock *pb = taosArrayGetP(pBlockList, 0); SSDataBlock *pb = taosArrayGetP(pBlockList, 0);
SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam)); SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam));
int32_t code = TSDB_CODE_SUCCESS;
if (NULL == pLeft) { if (NULL == pLeft) {
sclError("calloc %d failed", (int32_t)sizeof(SScalarParam)); sclError("calloc %d failed", (int32_t)sizeof(SScalarParam));
SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
@ -92,15 +100,16 @@ int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockL
pLeft->numOfRows = pb->info.rows; pLeft->numOfRows = pb->info.rows;
if (pDst->numOfRows < pb->info.rows) { if (pDst->numOfRows < pb->info.rows) {
colInfoDataEnsureCapacity(pDst->columnData, pb->info.rows, true); SCL_ERR_JRET(colInfoDataEnsureCapacity(pDst->columnData, pb->info.rows, true));
} }
_bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(OP_TYPE_ASSIGN); _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(OP_TYPE_ASSIGN);
OperatorFn(pLeft, pSrc, pDst, TSDB_ORDER_ASC); SCL_ERR_JRET(OperatorFn(pLeft, pSrc, pDst, TSDB_ORDER_ASC));
_return:
taosMemoryFree(pLeft); taosMemoryFree(pLeft);
return TSDB_CODE_SUCCESS; SCL_RET(code);
} }
int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) {
@ -116,7 +125,9 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) {
SNodeListNode *nodeList = (SNodeListNode *)pNode; SNodeListNode *nodeList = (SNodeListNode *)pNode;
SListCell *cell = nodeList->pNodeList->pHead; SListCell *cell = nodeList->pNodeList->pHead;
SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
if (out.columnData == NULL) {
SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
int32_t len = 0; int32_t len = 0;
void *buf = NULL; void *buf = NULL;
@ -228,7 +239,7 @@ int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) {
SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
memcpy(*res, nodesGetValueFromNode(pNode), pNode->node.resType.bytes); (void)memcpy(*res, nodesGetValueFromNode(pNode), pNode->node.resType.bytes);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -343,7 +354,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t
if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type || valueNode->isNull) { if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type || valueNode->isNull) {
colDataSetNULL(param->columnData, 0); colDataSetNULL(param->columnData, 0);
} else { } else {
colDataSetVal(param->columnData, 0, nodesGetValueFromNode(valueNode), false); SCL_ERR_RET(colDataSetVal(param->columnData, 0, nodesGetValueFromNode(valueNode), false));
} }
break; break;
} }
@ -502,23 +513,27 @@ _return:
SCL_RET(code); SCL_RET(code);
} }
int32_t sclGetNodeType(SNode *pNode, SScalarCtx *ctx) { int32_t sclGetNodeType(SNode *pNode, SScalarCtx *ctx, int32_t *type) {
if (NULL == pNode) { if (NULL == pNode) {
return -1; *type = -1;
return TSDB_CODE_SUCCESS;
} }
switch ((int)nodeType(pNode)) { switch ((int)nodeType(pNode)) {
case QUERY_NODE_VALUE: { case QUERY_NODE_VALUE: {
SValueNode *valueNode = (SValueNode *)pNode; SValueNode *valueNode = (SValueNode *)pNode;
return valueNode->node.resType.type; *type = valueNode->node.resType.type;
return TSDB_CODE_SUCCESS;
} }
case QUERY_NODE_NODE_LIST: { case QUERY_NODE_NODE_LIST: {
SNodeListNode *nodeList = (SNodeListNode *)pNode; SNodeListNode *nodeList = (SNodeListNode *)pNode;
return nodeList->node.resType.type; *type = nodeList->node.resType.type;
return TSDB_CODE_SUCCESS;
} }
case QUERY_NODE_COLUMN: { case QUERY_NODE_COLUMN: {
SColumnNode *colNode = (SColumnNode *)pNode; SColumnNode *colNode = (SColumnNode *)pNode;
return colNode->node.resType.type; *type = colNode->node.resType.type;
return TSDB_CODE_SUCCESS;
} }
case QUERY_NODE_FUNCTION: case QUERY_NODE_FUNCTION:
case QUERY_NODE_OPERATOR: case QUERY_NODE_OPERATOR:
@ -526,19 +541,22 @@ int32_t sclGetNodeType(SNode *pNode, SScalarCtx *ctx) {
SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &pNode, POINTER_BYTES); SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &pNode, POINTER_BYTES);
if (NULL == res) { if (NULL == res) {
sclError("no result for node, type:%d, node:%p", nodeType(pNode), pNode); sclError("no result for node, type:%d, node:%p", nodeType(pNode), pNode);
return -1; SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
return res->columnData->info.type; *type = (int32_t)(res->columnData->info.type);
return TSDB_CODE_SUCCESS;
} }
} }
return -1; *type = -1;
return TSDB_CODE_SUCCESS;
} }
void sclSetOperatorValueType(SOperatorNode *node, SScalarCtx *ctx) { int32_t sclSetOperatorValueType(SOperatorNode *node, SScalarCtx *ctx) {
ctx->type.opResType = node->node.resType.type; ctx->type.opResType = node->node.resType.type;
ctx->type.selfType = sclGetNodeType(node->pLeft, ctx); SCL_ERR_RET(sclGetNodeType(node->pLeft, ctx, &(ctx->type.selfType)));
ctx->type.peerType = sclGetNodeType(node->pRight, ctx); SCL_ERR_RET(sclGetNodeType(node->pRight, ctx, &(ctx->type.peerType)));
SCL_RET(TSDB_CODE_SUCCESS);
} }
int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScalarCtx *ctx, int32_t *rowNum) { int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScalarCtx *ctx, int32_t *rowNum) {
@ -555,7 +573,7 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal
SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
sclSetOperatorValueType(node, ctx); SCL_ERR_JRET(sclSetOperatorValueType(node, ctx));
SCL_ERR_JRET(sclInitParam(node->pLeft, &paramList[0], ctx, rowNum)); SCL_ERR_JRET(sclInitParam(node->pLeft, &paramList[0], ctx, rowNum));
if (paramNum > 1) { if (paramNum > 1) {
@ -603,13 +621,13 @@ int32_t sclWalkCaseWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell
SCL_ERR_RET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); SCL_ERR_RET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen));
SCL_ERR_RET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); SCL_ERR_RET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen));
vectorCompareImpl(pCase, pWhen, pComp, rowIdx, 1, TSDB_ORDER_ASC, OP_TYPE_EQUAL); SCL_ERR_JRET(vectorCompareImpl(pCase, pWhen, pComp, rowIdx, 1, TSDB_ORDER_ASC, OP_TYPE_EQUAL));
bool *equal = (bool *)colDataGetData(pComp->columnData, rowIdx); bool *equal = (bool *)colDataGetData(pComp->columnData, rowIdx);
if (*equal) { if (*equal) {
bool isNull = colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0)); bool isNull = colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0));
char *pData = isNull ? NULL : colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0)); char *pData = isNull ? NULL : colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0));
colDataSetVal(output->columnData, rowIdx, pData, isNull); SCL_ERR_JRET(colDataSetVal(output->columnData, rowIdx, pData, isNull));
if (0 == rowIdx && 1 == pCase->numOfRows && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && totalRows > 1) { if (0 == rowIdx && 1 == pCase->numOfRows && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && totalRows > 1) {
SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList));
@ -623,7 +641,7 @@ int32_t sclWalkCaseWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell
if (pElse) { if (pElse) {
bool isNull = colDataIsNull_s(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0)); bool isNull = colDataIsNull_s(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0));
char *pData = isNull ? NULL : colDataGetData(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0)); char *pData = isNull ? NULL : colDataGetData(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0));
colDataSetVal(output->columnData, rowIdx, pData, isNull); SCL_ERR_JRET(colDataSetVal(output->columnData, rowIdx, pData, isNull));
if (0 == rowIdx && 1 == pCase->numOfRows && 1 == pElse->numOfRows && totalRows > 1) { if (0 == rowIdx && 1 == pCase->numOfRows && 1 == pElse->numOfRows && totalRows > 1) {
SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList));
@ -633,7 +651,7 @@ int32_t sclWalkCaseWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell
goto _return; goto _return;
} }
colDataSetVal(output->columnData, rowIdx, NULL, true); SCL_ERR_JRET(colDataSetVal(output->columnData, rowIdx, NULL, true));
if (0 == rowIdx && 1 == pCase->numOfRows && totalRows > 1) { if (0 == rowIdx && 1 == pCase->numOfRows && totalRows > 1) {
SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList));
@ -672,7 +690,7 @@ int32_t sclWalkWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell *pCe
if (*whenValue) { if (*whenValue) {
bool isNull = colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0)); bool isNull = colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0));
char *pData = isNull ? NULL : colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0)); char *pData = isNull ? NULL : colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0));
colDataSetVal(output->columnData, rowIdx, pData, isNull); SCL_ERR_JRET(colDataSetVal(output->columnData, rowIdx, pData, isNull));
if (preSingle && 0 == rowIdx && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && totalRows > 1) { if (preSingle && 0 == rowIdx && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && totalRows > 1) {
SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList));
@ -691,7 +709,7 @@ int32_t sclWalkWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell *pCe
if (pElse) { if (pElse) {
bool isNull = colDataIsNull_s(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0)); bool isNull = colDataIsNull_s(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0));
char *pData = isNull ? NULL : colDataGetData(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0)); char *pData = isNull ? NULL : colDataGetData(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0));
colDataSetVal(output->columnData, rowIdx, pData, isNull); SCL_ERR_JRET(colDataSetVal(output->columnData, rowIdx, pData, isNull));
if (preSingle && 0 == rowIdx && 1 == pElse->numOfRows && totalRows > 1) { if (preSingle && 0 == rowIdx && 1 == pElse->numOfRows && totalRows > 1) {
SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList));
@ -701,7 +719,7 @@ int32_t sclWalkWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell *pCe
goto _return; goto _return;
} }
colDataSetVal(output->columnData, rowIdx, NULL, true); SCL_ERR_JRET(colDataSetVal(output->columnData, rowIdx, NULL, true));
if (preSingle && 0 == rowIdx && totalRows > 1) { if (preSingle && 0 == rowIdx && totalRows > 1) {
SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList));
@ -827,7 +845,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o
} }
if (complete) { if (complete) {
colDataSetVal(output->columnData, i, (char *)&value, false); SCL_ERR_JRET(colDataSetVal(output->columnData, i, (char *)&value, false));
if (value) { if (value) {
numOfQualified++; numOfQualified++;
} }
@ -876,9 +894,8 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp
SScalarParam *pRight = paramNum > 1 ? &params[1] : NULL; SScalarParam *pRight = paramNum > 1 ? &params[1] : NULL;
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
OperatorFn(pLeft, pRight, output, TSDB_ORDER_ASC); SCL_ERR_JRET(OperatorFn(pLeft, pRight, output, TSDB_ORDER_ASC));
code = terrno; SCL_ERR_JRET(terrno);
_return: _return:
sclFreeParamList(params, paramNum); sclFreeParamList(params, paramNum);
@ -928,13 +945,14 @@ int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *outp
} }
if (pCase) { if (pCase) {
vectorCompare(pCase, pWhen, &comp, TSDB_ORDER_ASC, OP_TYPE_EQUAL); SCL_ERR_JRET(vectorCompare(pCase, pWhen, &comp, TSDB_ORDER_ASC, OP_TYPE_EQUAL));
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
bool *equal = (bool *)colDataGetData(comp.columnData, (comp.numOfRows > 1 ? i : 0)); bool *equal = (bool *)colDataGetData(comp.columnData, (comp.numOfRows > 1 ? i : 0));
if (*equal) { if (*equal) {
colDataSetVal(output->columnData, i, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? i : 0)), SCL_ERR_JRET(colDataSetVal(output->columnData, i,
colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? i : 0))); colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? i : 0)),
colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? i : 0))));
if (0 == i && 1 == pCase->numOfRows && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && rowNum > 1) { if (0 == i && 1 == pCase->numOfRows && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && rowNum > 1) {
SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList));
break; break;
@ -951,8 +969,9 @@ int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *outp
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
bool *whenValue = (bool *)colDataGetData(pWhen->columnData, (pWhen->numOfRows > 1 ? i : 0)); bool *whenValue = (bool *)colDataGetData(pWhen->columnData, (pWhen->numOfRows > 1 ? i : 0));
if (*whenValue) { if (*whenValue) {
colDataSetVal(output->columnData, i, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? i : 0)), SCL_ERR_JRET(colDataSetVal(output->columnData, i,
colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? i : 0))); colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? i : 0)),
colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? i : 0))));
if (0 == i && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && rowNum > 1) { if (0 == i && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && rowNum > 1) {
SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList));
break; break;
@ -1063,42 +1082,34 @@ static uint8_t sclGetOpValueNodeTsPrecision(SNode *pLeft, SNode *pRight) {
return 0; return 0;
} }
int32_t sclConvertOpValueNodeTs(SOperatorNode *node, SScalarCtx *ctx) {
int32_t code = 0;
int32_t sclConvertOpValueNodeTs(SOperatorNode *node) {
if (node->pLeft && SCL_IS_VAR_VALUE_NODE(node->pLeft)) { if (node->pLeft && SCL_IS_VAR_VALUE_NODE(node->pLeft)) {
if (node->pRight && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pRight)->resType.type)) { if (node->pRight && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pRight)->resType.type)) {
SCL_ERR_JRET( SCL_ERR_RET(
sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode *)node->pLeft)); sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode *)node->pLeft));
} }
} else if (node->pRight && SCL_IS_NOTNULL_CONST_NODE(node->pRight)) { } else if (node->pRight && SCL_IS_NOTNULL_CONST_NODE(node->pRight)) {
if (node->pLeft && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pLeft)->resType.type)) { if (node->pLeft && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pLeft)->resType.type)) {
if (SCL_IS_VAR_VALUE_NODE(node->pRight)) { if (SCL_IS_VAR_VALUE_NODE(node->pRight)) {
SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), SCL_ERR_RET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight),
(SValueNode *)node->pRight)); (SValueNode *)node->pRight));
} else if (QUERY_NODE_NODE_LIST == node->pRight->type) { } else if (QUERY_NODE_NODE_LIST == node->pRight->type) {
SNode *pNode; SNode *pNode;
FOREACH(pNode, ((SNodeListNode *)node->pRight)->pNodeList) { FOREACH(pNode, ((SNodeListNode *)node->pRight)->pNodeList) {
if (SCL_IS_VAR_VALUE_NODE(pNode)) { if (SCL_IS_VAR_VALUE_NODE(pNode)) {
SCL_ERR_JRET( SCL_ERR_RET(
sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, pNode), (SValueNode *)pNode)); sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, pNode), (SValueNode *)pNode));
} }
} }
} }
} }
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return:
ctx->code = code;
return DEAL_RES_ERROR;
} }
int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) {
int32_t code = 0;
int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node) {
if (NULL == node->pCase) { if (NULL == node->pCase) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1108,7 +1119,7 @@ int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) {
FOREACH(pNode, node->pWhenThenList) { FOREACH(pNode, node->pWhenThenList) {
SExprNode *pExpr = (SExprNode *)((SWhenThenNode *)pNode)->pWhen; SExprNode *pExpr = (SExprNode *)((SWhenThenNode *)pNode)->pWhen;
if (TSDB_DATA_TYPE_TIMESTAMP == pExpr->resType.type) { if (TSDB_DATA_TYPE_TIMESTAMP == pExpr->resType.type) {
SCL_ERR_JRET(sclConvertToTsValueNode(pExpr->resType.precision, (SValueNode *)node->pCase)); SCL_ERR_RET(sclConvertToTsValueNode(pExpr->resType.precision, (SValueNode *)node->pCase));
break; break;
} }
} }
@ -1116,18 +1127,13 @@ int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) {
SNode *pNode; SNode *pNode;
FOREACH(pNode, node->pWhenThenList) { FOREACH(pNode, node->pWhenThenList) {
if (SCL_IS_VAR_VALUE_NODE(((SWhenThenNode *)pNode)->pWhen)) { if (SCL_IS_VAR_VALUE_NODE(((SWhenThenNode *)pNode)->pWhen)) {
SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pCase)->resType.precision, SCL_ERR_RET(sclConvertToTsValueNode(((SExprNode *)node->pCase)->resType.precision,
(SValueNode *)((SWhenThenNode *)pNode)->pWhen)); (SValueNode *)((SWhenThenNode *)pNode)->pWhen));
} }
} }
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return:
ctx->code = code;
return DEAL_RES_ERROR;
} }
EDealRes sclRewriteNonConstOperator(SNode **pNode, SScalarCtx *ctx) { EDealRes sclRewriteNonConstOperator(SNode **pNode, SScalarCtx *ctx) {
@ -1213,7 +1219,7 @@ EDealRes sclRewriteFunction(SNode **pNode, SScalarCtx *ctx) {
res->translate = true; res->translate = true;
strcpy(res->node.aliasName, node->node.aliasName); (void)strcpy(res->node.aliasName, node->node.aliasName);
res->node.resType.type = output.columnData->info.type; res->node.resType.type = output.columnData->info.type;
res->node.resType.bytes = output.columnData->info.bytes; res->node.resType.bytes = output.columnData->info.bytes;
res->node.resType.scale = output.columnData->info.scale; res->node.resType.scale = output.columnData->info.scale;
@ -1225,14 +1231,33 @@ EDealRes sclRewriteFunction(SNode **pNode, SScalarCtx *ctx) {
if (type == TSDB_DATA_TYPE_JSON) { if (type == TSDB_DATA_TYPE_JSON) {
int32_t len = getJsonValueLen(output.columnData->pData); int32_t len = getJsonValueLen(output.columnData->pData);
res->datum.p = taosMemoryCalloc(len, 1); res->datum.p = taosMemoryCalloc(len, 1);
memcpy(res->datum.p, output.columnData->pData, len); if (NULL == res->datum.p) {
sclError("calloc %d failed", len);
sclFreeParam(&output);
nodesDestroyNode((SNode *)res);
ctx->code = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}
(void)memcpy(res->datum.p, output.columnData->pData, len);
} else if (IS_VAR_DATA_TYPE(type)) { } else if (IS_VAR_DATA_TYPE(type)) {
// res->datum.p = taosMemoryCalloc(res->node.resType.bytes + VARSTR_HEADER_SIZE + 1, 1); // res->datum.p = taosMemoryCalloc(res->node.resType.bytes + VARSTR_HEADER_SIZE + 1, 1);
res->datum.p = taosMemoryCalloc(varDataTLen(output.columnData->pData) + 1, 1); res->datum.p = taosMemoryCalloc(varDataTLen(output.columnData->pData) + 1, 1);
if (NULL == res->datum.p) {
sclError("calloc %d failed", (int)(varDataTLen(output.columnData->pData) + 1));
sclFreeParam(&output);
nodesDestroyNode((SNode *)res);
ctx->code = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}
res->node.resType.bytes = varDataTLen(output.columnData->pData); res->node.resType.bytes = varDataTLen(output.columnData->pData);
memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData)); (void)memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData));
} else { } else {
nodesSetValueNodeValue(res, output.columnData->pData); ctx->code = nodesSetValueNodeValue(res, output.columnData->pData);
if (ctx->code) {
sclFreeParam(&output);
nodesDestroyNode((SNode *)res);
return DEAL_RES_ERROR;
}
} }
} }
@ -1269,13 +1294,17 @@ EDealRes sclRewriteLogic(SNode **pNode, SScalarCtx *ctx) {
res->node.resType = node->node.resType; res->node.resType = node->node.resType;
res->translate = true; res->translate = true;
strcpy(res->node.aliasName, node->node.aliasName); (void)strcpy(res->node.aliasName, node->node.aliasName);
int32_t type = output.columnData->info.type; int32_t type = output.columnData->info.type;
if (IS_VAR_DATA_TYPE(type)) { if (IS_VAR_DATA_TYPE(type)) {
res->datum.p = output.columnData->pData; res->datum.p = output.columnData->pData;
output.columnData->pData = NULL; output.columnData->pData = NULL;
} else { } else {
nodesSetValueNodeValue(res, output.columnData->pData); ctx->code = nodesSetValueNodeValue(res, output.columnData->pData);
if (ctx->code) {
sclFreeParam(&output);
return DEAL_RES_ERROR;
}
} }
nodesDestroyNode(*pNode); nodesDestroyNode(*pNode);
@ -1288,7 +1317,10 @@ EDealRes sclRewriteLogic(SNode **pNode, SScalarCtx *ctx) {
EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) { EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) {
SOperatorNode *node = (SOperatorNode *)*pNode; SOperatorNode *node = (SOperatorNode *)*pNode;
SCL_ERR_RET(sclConvertOpValueNodeTs(node, ctx)); ctx->code = sclConvertOpValueNodeTs(node);
if (ctx->code) {
return DEAL_RES_ERROR;
}
if ((!SCL_IS_CONST_NODE(node->pLeft)) || (!SCL_IS_CONST_NODE(node->pRight))) { if ((!SCL_IS_CONST_NODE(node->pLeft)) || (!SCL_IS_CONST_NODE(node->pRight))) {
return sclRewriteNonConstOperator(pNode, ctx); return sclRewriteNonConstOperator(pNode, ctx);
@ -1311,7 +1343,7 @@ EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) {
res->translate = true; res->translate = true;
strcpy(res->node.aliasName, node->node.aliasName); (void)strcpy(res->node.aliasName, node->node.aliasName);
res->node.resType = node->node.resType; res->node.resType = node->node.resType;
if (colDataIsNull_s(output.columnData, 0)) { if (colDataIsNull_s(output.columnData, 0)) {
res->isNull = true; res->isNull = true;
@ -1322,7 +1354,11 @@ EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) {
res->datum.p = output.columnData->pData; res->datum.p = output.columnData->pData;
output.columnData->pData = NULL; output.columnData->pData = NULL;
} else { } else {
nodesSetValueNodeValue(res, output.columnData->pData); ctx->code = nodesSetValueNodeValue(res, output.columnData->pData);
if (ctx->code) {
sclFreeParam(&output);
return DEAL_RES_ERROR;
}
} }
} }
@ -1336,7 +1372,10 @@ EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) {
EDealRes sclRewriteCaseWhen(SNode **pNode, SScalarCtx *ctx) { EDealRes sclRewriteCaseWhen(SNode **pNode, SScalarCtx *ctx) {
SCaseWhenNode *node = (SCaseWhenNode *)*pNode; SCaseWhenNode *node = (SCaseWhenNode *)*pNode;
SCL_ERR_RET(sclConvertCaseWhenValueNodeTs(node, ctx)); ctx->code = sclConvertCaseWhenValueNodeTs(node);
if (ctx->code) {
return DEAL_RES_ERROR;
}
if ((!SCL_IS_CONST_NODE(node->pCase)) || (!SCL_IS_CONST_NODE(node->pElse))) { if ((!SCL_IS_CONST_NODE(node->pCase)) || (!SCL_IS_CONST_NODE(node->pElse))) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
@ -1367,7 +1406,7 @@ EDealRes sclRewriteCaseWhen(SNode **pNode, SScalarCtx *ctx) {
res->translate = true; res->translate = true;
strcpy(res->node.aliasName, node->node.aliasName); (void)strcpy(res->node.aliasName, node->node.aliasName);
res->node.resType = node->node.resType; res->node.resType = node->node.resType;
if (colDataIsNull_s(output.columnData, 0)) { if (colDataIsNull_s(output.columnData, 0)) {
res->isNull = true; res->isNull = true;
@ -1376,9 +1415,21 @@ EDealRes sclRewriteCaseWhen(SNode **pNode, SScalarCtx *ctx) {
int32_t type = output.columnData->info.type; int32_t type = output.columnData->info.type;
if (IS_VAR_DATA_TYPE(type)) { // todo refactor if (IS_VAR_DATA_TYPE(type)) { // todo refactor
res->datum.p = taosMemoryCalloc(varDataTLen(output.columnData->pData) + 1, sizeof(char)); // add \0 to the end for print json value res->datum.p = taosMemoryCalloc(varDataTLen(output.columnData->pData) + 1, sizeof(char)); // add \0 to the end for print json value
memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData)); if (NULL == res->datum.p) {
sclError("calloc %d failed", (int)(varDataTLen(output.columnData->pData) + 1));
sclFreeParam(&output);
nodesDestroyNode((SNode *)res);
ctx->code = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}
(void)memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData));
} else { } else {
nodesSetValueNodeValue(res, output.columnData->pData); ctx->code = nodesSetValueNodeValue(res, output.columnData->pData);
if (ctx->code) {
sclFreeParam(&output);
nodesDestroyNode((SNode *)res);
return DEAL_RES_ERROR;
}
} }
} }
@ -1513,11 +1564,14 @@ EDealRes sclWalkTarget(SNode *pNode, SScalarCtx *ctx) {
return DEAL_RES_ERROR; return DEAL_RES_ERROR;
} }
colDataAssign(col, res->columnData, res->numOfRows, NULL); ctx->code = colDataAssign(col, res->columnData, res->numOfRows, NULL);
if (ctx->code) {
return DEAL_RES_ERROR;
}
block->info.rows = res->numOfRows; block->info.rows = res->numOfRows;
sclFreeParam(res); sclFreeParam(res);
taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); (void)taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
@ -1733,14 +1787,14 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) {
if (1 == res->numOfRows && pb->info.rows > 0) { if (1 == res->numOfRows && pb->info.rows > 0) {
SCL_ERR_JRET(sclExtendResRows(pDst, res, pBlockList)); SCL_ERR_JRET(sclExtendResRows(pDst, res, pBlockList));
} else { } else {
colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows, true); SCL_ERR_JRET(colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows, true));
colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL); SCL_ERR_JRET(colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL));
pDst->numOfRows = res->numOfRows; pDst->numOfRows = res->numOfRows;
pDst->numOfQualified = res->numOfQualified; pDst->numOfQualified = res->numOfQualified;
} }
sclFreeParam(res); sclFreeParam(res);
taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); (void)taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES);
} }
_return: _return:

View File

@ -124,7 +124,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
} }
default: { default: {
colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL); SCL_ERR_RET(colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL));
} }
} }
@ -147,7 +147,9 @@ static int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SS
colDataSetNULL(pOutputData, i); colDataSetNULL(pOutputData, i);
continue; continue;
} }
double result = valFn(getValueFn(pInputData->pData, i)); double tmp = 0;
SCL_ERR_RET(getValueFn(pInputData->pData, i, &tmp));
double result = valFn(tmp);
if (isinf(result) || isnan(result)) { if (isinf(result) || isnan(result)) {
colDataSetNULL(pOutputData, i); colDataSetNULL(pOutputData, i);
} else { } else {
@ -182,7 +184,11 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S
colDataSetNULL(pOutputData, i); colDataSetNULL(pOutputData, i);
continue; continue;
} }
result = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, i)); double val1 = 0;
double val2 = 0;
SCL_ERR_RET(getValueFn[0](pInputData[0]->pData, i, &val1));
SCL_ERR_RET(getValueFn[1](pInputData[1]->pData, i, &val2));
result = valFn(val1, val2);
if (isinf(result) || isnan(result)) { if (isinf(result) || isnan(result)) {
colDataSetNULL(pOutputData, i); colDataSetNULL(pOutputData, i);
} else { } else {
@ -198,8 +204,11 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S
colDataSetNULL(pOutputData, i); colDataSetNULL(pOutputData, i);
continue; continue;
} }
double val1 = 0;
result = valFn(getValueFn[0](pInputData[0]->pData, 0), getValueFn[1](pInputData[1]->pData, i)); double val2 = 0;
SCL_ERR_RET(getValueFn[0](pInputData[0]->pData, 0, &val1));
SCL_ERR_RET(getValueFn[1](pInputData[1]->pData, i, &val2));
result = valFn(val1, val2);
if (isinf(result) || isnan(result)) { if (isinf(result) || isnan(result)) {
colDataSetNULL(pOutputData, i); colDataSetNULL(pOutputData, i);
continue; continue;
@ -217,8 +226,11 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S
colDataSetNULL(pOutputData, i); colDataSetNULL(pOutputData, i);
continue; continue;
} }
double val1 = 0;
result = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, 0)); double val2 = 0;
SCL_ERR_RET(getValueFn[0](pInputData[0]->pData, i, &val1));
SCL_ERR_RET(getValueFn[1](pInputData[1]->pData, 0, &val2));
result = valFn(val1, val2);
if (isinf(result) || isnan(result)) { if (isinf(result) || isnan(result)) {
colDataSetNULL(pOutputData, i); colDataSetNULL(pOutputData, i);
continue; continue;
@ -277,7 +289,7 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
} }
default: { default: {
colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL); SCL_ERR_RET(colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL));
} }
} }
@ -380,6 +392,9 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, VarDataLenT *dataLen) { static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, VarDataLenT *dataLen) {
if (hasNchar && type == TSDB_DATA_TYPE_VARCHAR) { if (hasNchar && type == TSDB_DATA_TYPE_VARCHAR) {
TdUcs4 *newBuf = taosMemoryCalloc((varDataLen(input) + 1) * TSDB_NCHAR_SIZE, 1); TdUcs4 *newBuf = taosMemoryCalloc((varDataLen(input) + 1) * TSDB_NCHAR_SIZE, 1);
if (NULL == newBuf) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t len = varDataLen(input); int32_t len = varDataLen(input);
bool ret = taosMbsToUcs4(varDataVal(input), len, newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, &len); bool ret = taosMbsToUcs4(varDataVal(input), len, newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, &len);
if (!ret) { if (!ret) {
@ -410,12 +425,19 @@ static int32_t getNumOfNullEntries(SColumnInfoData *pColumnInfoData, int32_t num
} }
int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
int32_t ret = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *));
SColumnInfoData *pOutputData = pOutput->columnData; SColumnInfoData *pOutputData = pOutput->columnData;
char **input = taosMemoryCalloc(inputNum, POINTER_BYTES); char **input = taosMemoryCalloc(inputNum, POINTER_BYTES);
char *outputBuf = NULL; char *outputBuf = NULL;
if (NULL == pInputData) {
SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
if (NULL == input) {
SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
int32_t inputLen = 0; int32_t inputLen = 0;
int32_t numOfRows = 0; int32_t numOfRows = 0;
bool hasNchar = (GET_PARAM_TYPE(pOutput) == TSDB_DATA_TYPE_NCHAR) ? true : false; bool hasNchar = (GET_PARAM_TYPE(pOutput) == TSDB_DATA_TYPE_NCHAR) ? true : false;
@ -441,6 +463,9 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE;
outputBuf = taosMemoryCalloc(outputLen, 1); outputBuf = taosMemoryCalloc(outputLen, 1);
if (NULL == outputBuf) {
SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
char *output = outputBuf; char *output = outputBuf;
for (int32_t k = 0; k < numOfRows; ++k) { for (int32_t k = 0; k < numOfRows; ++k) {
@ -462,33 +487,37 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k; int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k;
input[i] = colDataGetData(pInputData[i], rowIdx); input[i] = colDataGetData(pInputData[i], rowIdx);
ret = concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen); SCL_ERR_JRET(concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen));
if (ret != TSDB_CODE_SUCCESS) {
goto DONE;
}
} }
varDataSetLen(output, dataLen); varDataSetLen(output, dataLen);
colDataSetVal(pOutputData, k, output, false); SCL_ERR_JRET(colDataSetVal(pOutputData, k, output, false));
output += varDataTLen(output); output += varDataTLen(output);
} }
pOutput->numOfRows = numOfRows; pOutput->numOfRows = numOfRows;
DONE: _return:
taosMemoryFree(input); taosMemoryFree(input);
taosMemoryFree(outputBuf); taosMemoryFree(outputBuf);
taosMemoryFree(pInputData); taosMemoryFree(pInputData);
return ret; SCL_RET(code);
} }
int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
int32_t ret = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *));
SColumnInfoData *pOutputData = pOutput->columnData; SColumnInfoData *pOutputData = pOutput->columnData;
char **input = taosMemoryCalloc(inputNum, POINTER_BYTES); char **input = taosMemoryCalloc(inputNum, POINTER_BYTES);
char *outputBuf = NULL; char *outputBuf = NULL;
if (NULL == pInputData) {
SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
if (NULL == input) {
SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
int32_t inputLen = 0; int32_t inputLen = 0;
int32_t numOfRows = 0; int32_t numOfRows = 0;
bool hasNchar = (GET_PARAM_TYPE(pOutput) == TSDB_DATA_TYPE_NCHAR) ? true : false; bool hasNchar = (GET_PARAM_TYPE(pOutput) == TSDB_DATA_TYPE_NCHAR) ? true : false;
@ -518,6 +547,10 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE;
outputBuf = taosMemoryCalloc(outputLen, 1); outputBuf = taosMemoryCalloc(outputLen, 1);
if (NULL == outputBuf) {
SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
char *output = outputBuf; char *output = outputBuf;
for (int32_t k = 0; k < numOfRows; ++k) { for (int32_t k = 0; k < numOfRows; ++k) {
@ -537,18 +570,12 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k; int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k;
input[i] = colDataGetData(pInputData[i], rowIdx); input[i] = colDataGetData(pInputData[i], rowIdx);
ret = concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen); SCL_ERR_JRET(concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen));
if (ret != TSDB_CODE_SUCCESS) {
goto DONE;
}
if (i < inputNum - 1) { if (i < inputNum - 1) {
// insert the separator // insert the separator
char *sep = (pInput[0].numOfRows == 1) ? colDataGetData(pInputData[0], 0) : colDataGetData(pInputData[0], k); char *sep = (pInput[0].numOfRows == 1) ? colDataGetData(pInputData[0], 0) : colDataGetData(pInputData[0], k);
ret = concatCopyHelper(sep, output, hasNchar, GET_PARAM_TYPE(&pInput[0]), &dataLen); SCL_ERR_JRET(concatCopyHelper(sep, output, hasNchar, GET_PARAM_TYPE(&pInput[0]), &dataLen));
if (ret != TSDB_CODE_SUCCESS) {
goto DONE;
}
} }
} }
@ -557,19 +584,19 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
memset(output, 0, dataLen); memset(output, 0, dataLen);
} else { } else {
varDataSetLen(output, dataLen); varDataSetLen(output, dataLen);
colDataSetVal(pOutputData, k, output, false); SCL_ERR_JRET(colDataSetVal(pOutputData, k, output, false));
output += varDataTLen(output); output += varDataTLen(output);
} }
} }
pOutput->numOfRows = numOfRows; pOutput->numOfRows = numOfRows;
DONE: _return:
taosMemoryFree(input); taosMemoryFree(input);
taosMemoryFree(outputBuf); taosMemoryFree(outputBuf);
taosMemoryFree(pInputData); taosMemoryFree(pInputData);
return ret; return code;
} }
static int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _conv_fn convFn) { static int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _conv_fn convFn) {
@ -580,6 +607,10 @@ static int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScala
int32_t outputLen = pInputData->varmeta.length; int32_t outputLen = pInputData->varmeta.length;
char *outputBuf = taosMemoryCalloc(outputLen, 1); char *outputBuf = taosMemoryCalloc(outputLen, 1);
if (outputBuf == NULL) {
SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
char *output = outputBuf; char *output = outputBuf;
for (int32_t i = 0; i < pInput->numOfRows; ++i) { for (int32_t i = 0; i < pInput->numOfRows; ++i) {
@ -600,7 +631,11 @@ static int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScala
} }
} }
varDataSetLen(output, len); varDataSetLen(output, len);
colDataSetVal(pOutputData, i, output, false); int32_t code = colDataSetVal(pOutputData, i, output, false);
if (TSDB_CODE_SUCCESS != code) {
taosMemoryFree(outputBuf);
SCL_ERR_RET(code);
}
output += varDataTLen(output); output += varDataTLen(output);
} }
@ -618,6 +653,10 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar
int32_t outputLen = pInputData->varmeta.length; int32_t outputLen = pInputData->varmeta.length;
char *outputBuf = taosMemoryCalloc(outputLen, 1); char *outputBuf = taosMemoryCalloc(outputLen, 1);
if (outputBuf == NULL) {
SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
char *output = outputBuf; char *output = outputBuf;
for (int32_t i = 0; i < pInput->numOfRows; ++i) { for (int32_t i = 0; i < pInput->numOfRows; ++i) {
@ -631,7 +670,11 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar
int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE; int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE;
trimFn(input, output, type, charLen); trimFn(input, output, type, charLen);
colDataSetVal(pOutputData, i, output, false); int32_t code = colDataSetVal(pOutputData, i, output, false);
if (TSDB_CODE_SUCCESS != code) {
taosMemoryFree(outputBuf);
SCL_ERR_RET(code);
}
output += varDataTLen(output); output += varDataTLen(output);
} }
@ -688,7 +731,11 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
varDataSetLen(output, 0); varDataSetLen(output, 0);
} }
colDataSetVal(pOutputData, i, output, false); int32_t code = colDataSetVal(pOutputData, i, output, false);
if (TSDB_CODE_SUCCESS != code) {
taosMemoryFree(outputBuf);
SCL_ERR_RET(code);
}
} }
pOutput->numOfRows = pInput->numOfRows; pOutput->numOfRows = pInput->numOfRows;
@ -715,12 +762,21 @@ int32_t md5Function(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutpu
if (bufLen < varDataLen(input) + VARSTR_HEADER_SIZE) { if (bufLen < varDataLen(input) + VARSTR_HEADER_SIZE) {
bufLen = varDataLen(input) + VARSTR_HEADER_SIZE; bufLen = varDataLen(input) + VARSTR_HEADER_SIZE;
pOutputBuf = taosMemoryRealloc(pOutputBuf, bufLen); pOutputBuf = taosMemoryRealloc(pOutputBuf, bufLen);
if (!pOutputBuf) {
qError("md5 function alloc memory failed");
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
char *output = pOutputBuf; char *output = pOutputBuf;
memcpy(varDataVal(output), varDataVal(input), varDataLen(input)); memcpy(varDataVal(output), varDataVal(input), varDataLen(input));
int32_t len = taosCreateMD5Hash(varDataVal(output), varDataLen(input)); int32_t len = taosCreateMD5Hash(varDataVal(output), varDataLen(input));
varDataSetLen(output, len); varDataSetLen(output, len);
colDataSetVal(pOutputData, i, output, false); int32_t code = colDataSetVal(pOutputData, i, output, false);
if (TSDB_CODE_SUCCESS != code) {
taosMemoryFree(pOutputBuf);
SCL_ERR_RET(code);
}
} }
pOutput->numOfRows = pInput->numOfRows; pOutput->numOfRows = pInput->numOfRows;
taosMemoryFree(pOutputBuf); taosMemoryFree(pOutputBuf);
@ -1062,7 +1118,10 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
} }
} }
colDataSetVal(pOutput->columnData, i, output, false); code = colDataSetVal(pOutput->columnData, i, output, false);
if (TSDB_CODE_SUCCESS != code) {
goto _end;
}
} }
pOutput->numOfRows = pInput->numOfRows; pOutput->numOfRows = pInput->numOfRows;
@ -1167,7 +1226,7 @@ _end:
memmove(buf + VARSTR_HEADER_SIZE, buf, len); memmove(buf + VARSTR_HEADER_SIZE, buf, len);
varDataSetLen(buf, len); varDataSetLen(buf, len);
colDataSetVal(pOutput->columnData, i, buf, false); SCL_ERR_RET(colDataSetVal(pOutput->columnData, i, buf, false));
} }
pOutput->numOfRows = pInput->numOfRows; pOutput->numOfRows = pInput->numOfRows;
@ -1193,7 +1252,7 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
colDataSetNULL(pOutput->columnData, i); colDataSetNULL(pOutput->columnData, i);
} else { } else {
colDataSetVal(pOutput->columnData, i, (char *)&timeVal, false); SCL_ERR_RET(colDataSetVal(pOutput->columnData, i, (char *)&timeVal, false));
} }
} }
@ -1206,12 +1265,21 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
int32_t type = GET_PARAM_TYPE(pInput); int32_t type = GET_PARAM_TYPE(pInput);
char tmp[TSDB_MAX_JSON_TAG_LEN] = {0}; char tmp[TSDB_MAX_JSON_TAG_LEN] = {0};
int32_t code = TSDB_CODE_SUCCESS;
for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
SArray *pTagVals = taosArrayInit(8, sizeof(STagVal)); SArray *pTagVals = taosArrayInit(8, sizeof(STagVal));
if (NULL == pTagVals) {
return TSDB_CODE_OUT_OF_MEMORY;
}
STag *pTag = NULL; STag *pTag = NULL;
if (colDataIsNull_s(pInput[0].columnData, i)) { if (colDataIsNull_s(pInput[0].columnData, i)) {
tTagNew(pTagVals, 1, true, &pTag); code = tTagNew(pTagVals, 1, true, &pTag);
if (TSDB_CODE_SUCCESS != code) {
tTagFree(pTag);
taosArrayDestroy(pTagVals);
SCL_ERR_RET(code);
}
} else { } else {
char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[i]; char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[i];
if (varDataLen(input) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { if (varDataLen(input) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
@ -1221,13 +1289,21 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
memcpy(tmp, varDataVal(input), varDataLen(input)); memcpy(tmp, varDataVal(input), varDataLen(input));
tmp[varDataLen(input)] = 0; tmp[varDataLen(input)] = 0;
if (parseJsontoTagData(tmp, pTagVals, &pTag, NULL)) { if (parseJsontoTagData(tmp, pTagVals, &pTag, NULL)) {
tTagNew(pTagVals, 1, true, &pTag); code = tTagNew(pTagVals, 1, true, &pTag);
if (TSDB_CODE_SUCCESS != code) {
tTagFree(pTag);
taosArrayDestroy(pTagVals);
SCL_ERR_RET(code);
}
} }
} }
colDataSetVal(pOutput->columnData, i, (const char *)pTag, false); code = colDataSetVal(pOutput->columnData, i, (const char *)pTag, false);
tTagFree(pTag); tTagFree(pTag);
taosArrayDestroy(pTagVals); taosArrayDestroy(pTagVals);
if (TSDB_CODE_SUCCESS != code ) {
SCL_ERR_RET(code);
}
} }
pOutput->numOfRows = pInput->numOfRows; pOutput->numOfRows = pInput->numOfRows;
@ -1242,6 +1318,9 @@ int32_t toTimestampFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam
int32_t len, code = TSDB_CODE_SUCCESS; int32_t len, code = TSDB_CODE_SUCCESS;
SArray *formats = NULL; SArray *formats = NULL;
if (tsStr == NULL || format == NULL) {
SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) { if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) {
colDataSetNULL(pOutput->columnData, i); colDataSetNULL(pOutput->columnData, i);
@ -1251,11 +1330,11 @@ int32_t toTimestampFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam
char *tsData = colDataGetData(pInput[0].columnData, i); char *tsData = colDataGetData(pInput[0].columnData, i);
char *formatData = colDataGetData(pInput[1].columnData, pInput[1].numOfRows > 1 ? i : 0); char *formatData = colDataGetData(pInput[1].columnData, pInput[1].numOfRows > 1 ? i : 0);
len = TMIN(TS_FORMAT_MAX_LEN - 1, varDataLen(tsData)); len = TMIN(TS_FORMAT_MAX_LEN - 1, varDataLen(tsData));
strncpy(tsStr, varDataVal(tsData), len); (void)strncpy(tsStr, varDataVal(tsData), len); // No need to handle the return value.
tsStr[len] = '\0'; tsStr[len] = '\0';
len = TMIN(TS_FORMAT_MAX_LEN - 1, varDataLen(formatData)); len = TMIN(TS_FORMAT_MAX_LEN - 1, varDataLen(formatData));
if (pInput[1].numOfRows > 1 || i == 0) { if (pInput[1].numOfRows > 1 || i == 0) {
strncpy(format, varDataVal(formatData), len); (void)strncpy(format, varDataVal(formatData), len); // No need to handle the return value.
format[len] = '\0'; format[len] = '\0';
if (formats) { if (formats) {
taosArrayDestroy(formats); taosArrayDestroy(formats);
@ -1267,10 +1346,12 @@ int32_t toTimestampFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam
code = taosChar2Ts(format, &formats, tsStr, &ts, precision, errMsg, 128); code = taosChar2Ts(format, &formats, tsStr, &ts, precision, errMsg, 128);
if (code) { if (code) {
qError("func to_timestamp failed %s", errMsg); qError("func to_timestamp failed %s", errMsg);
break; SCL_ERR_JRET(code);
} }
colDataSetVal(pOutput->columnData, i, (char *)&ts, false); SCL_ERR_JRET(colDataSetVal(pOutput->columnData, i, (char *)&ts, false));
} }
_return:
if (formats) taosArrayDestroy(formats); if (formats) taosArrayDestroy(formats);
taosMemoryFree(tsStr); taosMemoryFree(tsStr);
taosMemoryFree(format); taosMemoryFree(format);
@ -1283,6 +1364,10 @@ int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOu
int32_t len; int32_t len;
SArray *formats = NULL; SArray *formats = NULL;
int32_t code = 0; int32_t code = 0;
if (format == NULL || out == NULL) {
SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) { if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) {
colDataSetNULL(pOutput->columnData, i); colDataSetNULL(pOutput->columnData, i);
@ -1293,7 +1378,7 @@ int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOu
char *formatData = colDataGetData(pInput[1].columnData, pInput[1].numOfRows > 1 ? i : 0); char *formatData = colDataGetData(pInput[1].columnData, pInput[1].numOfRows > 1 ? i : 0);
len = TMIN(TS_FORMAT_MAX_LEN - 1, varDataLen(formatData)); len = TMIN(TS_FORMAT_MAX_LEN - 1, varDataLen(formatData));
if (pInput[1].numOfRows > 1 || i == 0) { if (pInput[1].numOfRows > 1 || i == 0) {
strncpy(format, varDataVal(formatData), len); (void)strncpy(format, varDataVal(formatData), len);
format[len] = '\0'; format[len] = '\0';
if (formats) { if (formats) {
taosArrayDestroy(formats); taosArrayDestroy(formats);
@ -1301,11 +1386,12 @@ int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOu
} }
} }
int32_t precision = pInput[0].columnData->info.precision; int32_t precision = pInput[0].columnData->info.precision;
code = taosTs2Char(format, &formats, *(int64_t *)ts, precision, varDataVal(out), TS_FORMAT_MAX_LEN); SCL_ERR_JRET(taosTs2Char(format, &formats, *(int64_t *)ts, precision, varDataVal(out), TS_FORMAT_MAX_LEN));
if (code) break;
varDataSetLen(out, strlen(varDataVal(out))); varDataSetLen(out, strlen(varDataVal(out)));
colDataSetVal(pOutput->columnData, i, out, false); SCL_ERR_JRET(colDataSetVal(pOutput->columnData, i, out, false));
} }
_return:
if (formats) taosArrayDestroy(formats); if (formats) taosArrayDestroy(formats);
taosMemoryFree(format); taosMemoryFree(format);
taosMemoryFree(out); taosMemoryFree(out);
@ -1341,7 +1427,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
} }
GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[timePrecIdx]), pInput[timePrecIdx].columnData->pData); GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[timePrecIdx]), pInput[timePrecIdx].columnData->pData);
memcpy(timezone, varDataVal(pInput[timeZoneIdx].columnData->pData), varDataLen(pInput[timeZoneIdx].columnData->pData)); (void)memcpy(timezone, varDataVal(pInput[timeZoneIdx].columnData->pData), varDataLen(pInput[timeZoneIdx].columnData->pData));
for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
if (colDataIsNull_s(pInput[0].columnData, i)) { if (colDataIsNull_s(pInput[0].columnData, i)) {
@ -1373,7 +1459,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
} else { } else {
timeVal = timeVal / timeUnit * timeUnit; timeVal = timeVal / timeUnit * timeUnit;
} }
colDataSetVal(pOutput->columnData, i, (char *)&timeVal, false); SCL_ERR_RET(colDataSetVal(pOutput->columnData, i, (char *)&timeVal, false));
} }
pOutput->numOfRows = pInput->numOfRows; pOutput->numOfRows = pInput->numOfRows;
@ -1507,7 +1593,7 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
} }
} }
colDataSetVal(pOutput->columnData, i, (char *)&result, false); SCL_ERR_RET(colDataSetVal(pOutput->columnData, i, (char *)&result, false));
} }
pOutput->numOfRows = numOfRows; pOutput->numOfRows = numOfRows;
@ -1541,10 +1627,10 @@ int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
char output[TD_TIMEZONE_LEN + VARSTR_HEADER_SIZE] = {0}; char output[TD_TIMEZONE_LEN + VARSTR_HEADER_SIZE] = {0};
memcpy(varDataVal(output), tsTimezoneStr, TD_TIMEZONE_LEN); (void)memcpy(varDataVal(output), tsTimezoneStr, TD_TIMEZONE_LEN);
varDataSetLen(output, strlen(tsTimezoneStr)); varDataSetLen(output, strlen(tsTimezoneStr));
for (int32_t i = 0; i < pInput->numOfRows; ++i) { for (int32_t i = 0; i < pInput->numOfRows; ++i) {
colDataSetVal(pOutput->columnData, i, output, false); SCL_ERR_RET(colDataSetVal(pOutput->columnData, i, output, false));
} }
pOutput->numOfRows = pInput->numOfRows; pOutput->numOfRows = pInput->numOfRows;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -2259,16 +2345,16 @@ int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPa
char interceptBuf[64] = {0}; char interceptBuf[64] = {0};
int n = snprintf(slopBuf, 64, "%.6lf", matrix02); int n = snprintf(slopBuf, 64, "%.6lf", matrix02);
if (n > LEASTSQUARES_DOUBLE_ITEM_LENGTH) { if (n > LEASTSQUARES_DOUBLE_ITEM_LENGTH) {
snprintf(slopBuf, 64, "%." DOUBLE_PRECISION_DIGITS, matrix02); (void)snprintf(slopBuf, 64, "%." DOUBLE_PRECISION_DIGITS, matrix02);
} }
n = snprintf(interceptBuf, 64, "%.6lf", matrix12); n = snprintf(interceptBuf, 64, "%.6lf", matrix12);
if (n > LEASTSQUARES_DOUBLE_ITEM_LENGTH) { if (n > LEASTSQUARES_DOUBLE_ITEM_LENGTH) {
snprintf(interceptBuf, 64, "%." DOUBLE_PRECISION_DIGITS, matrix12); (void) snprintf(interceptBuf, 64, "%." DOUBLE_PRECISION_DIGITS, matrix12);
} }
size_t len = size_t len =
snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{slop:%s, intercept:%s}", slopBuf, interceptBuf); snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{slop:%s, intercept:%s}", slopBuf, interceptBuf);
varDataSetLen(buf, len); varDataSetLen(buf, len);
colDataSetVal(pOutputData, 0, buf, false); SCL_ERR_RET(colDataSetVal(pOutputData, 0, buf, false));
} }
pOutput->numOfRows = 1; pOutput->numOfRows = 1;
@ -2295,7 +2381,7 @@ int32_t percentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalar
if (hasNull) { if (hasNull) {
colDataSetNULL(pOutputData, 0); colDataSetNULL(pOutputData, 0);
} else { } else {
colDataSetVal(pOutputData, 0, (char *)&val, false); SCL_ERR_RET(colDataSetVal(pOutputData, 0, (char *)&val, false));
} }
pOutput->numOfRows = 1; pOutput->numOfRows = 1;
@ -2341,7 +2427,7 @@ int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
colDataSetNULL(pOutputData, 0); colDataSetNULL(pOutputData, 0);
} else { } else {
double result = max - min; double result = max - min;
colDataSetVal(pOutputData, 0, (char *)&result, false); SCL_ERR_RET(colDataSetVal(pOutputData, 0, (char *)&result, false));
} }
pOutput->numOfRows = 1; pOutput->numOfRows = 1;
@ -2544,7 +2630,7 @@ int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalar
} else { } else {
count = 0; count = 0;
} }
colDataSetVal(pOutputData, i, (char *)&out, false); SCL_ERR_RET(colDataSetVal(pOutputData, i, (char *)&out, false));
} }
pOutput->numOfRows = pInput->numOfRows; pOutput->numOfRows = pInput->numOfRows;
@ -2568,7 +2654,7 @@ int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SSca
if (ret) { if (ret) {
out = 0; out = 0;
} }
colDataSetVal(pOutputData, i, (char *)&out, false); SCL_ERR_RET(colDataSetVal(pOutputData, i, (char *)&out, false));
} }
pOutput->numOfRows = pInput->numOfRows; pOutput->numOfRows = pInput->numOfRows;
@ -2599,7 +2685,7 @@ typedef struct SHistoFuncBin {
double percentage; double percentage;
} SHistoFuncBin; } SHistoFuncBin;
static bool getHistogramBinDesc(SHistoFuncBin **bins, int32_t *binNum, char *binDescStr, int8_t binType, static int32_t getHistogramBinDesc(SHistoFuncBin **bins, int32_t *binNum, char *binDescStr, int8_t binType,
bool normalized) { bool normalized) {
cJSON *binDesc = cJSON_Parse(binDescStr); cJSON *binDesc = cJSON_Parse(binDescStr);
int32_t numOfBins; int32_t numOfBins;
@ -2609,7 +2695,7 @@ static bool getHistogramBinDesc(SHistoFuncBin **bins, int32_t *binNum, char *bin
int32_t startIndex; int32_t startIndex;
if (numOfParams != 4) { if (numOfParams != 4) {
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; SCL_ERR_RET(TSDB_CODE_FAILED);
} }
cJSON *start = cJSON_GetObjectItem(binDesc, "start"); cJSON *start = cJSON_GetObjectItem(binDesc, "start");
@ -2620,18 +2706,18 @@ static bool getHistogramBinDesc(SHistoFuncBin **bins, int32_t *binNum, char *bin
if (!cJSON_IsNumber(start) || !cJSON_IsNumber(count) || !cJSON_IsBool(infinity)) { if (!cJSON_IsNumber(start) || !cJSON_IsNumber(count) || !cJSON_IsBool(infinity)) {
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; SCL_RET(TSDB_CODE_SUCCESS);
} }
if (count->valueint <= 0 || count->valueint > 1000) { // limit count to 1000 if (count->valueint <= 0 || count->valueint > 1000) { // limit count to 1000
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; SCL_ERR_RET(TSDB_CODE_FAILED);
} }
if (isinf(start->valuedouble) || (width != NULL && isinf(width->valuedouble)) || if (isinf(start->valuedouble) || (width != NULL && isinf(width->valuedouble)) ||
(factor != NULL && isinf(factor->valuedouble)) || (count != NULL && isinf(count->valuedouble))) { (factor != NULL && isinf(factor->valuedouble)) || (count != NULL && isinf(count->valuedouble))) {
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; SCL_ERR_RET(TSDB_CODE_FAILED);
} }
int32_t counter = (int32_t)count->valueint; int32_t counter = (int32_t)count->valueint;
@ -2644,19 +2730,23 @@ static bool getHistogramBinDesc(SHistoFuncBin **bins, int32_t *binNum, char *bin
} }
intervals = taosMemoryCalloc(numOfBins, sizeof(double)); intervals = taosMemoryCalloc(numOfBins, sizeof(double));
if (NULL == intervals) {
cJSON_Delete(binDesc);
SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
if (cJSON_IsNumber(width) && factor == NULL && binType == LINEAR_BIN) { if (cJSON_IsNumber(width) && factor == NULL && binType == LINEAR_BIN) {
// linear bin process // linear bin process
if (width->valuedouble == 0) { if (width->valuedouble == 0) {
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; SCL_ERR_RET(TSDB_CODE_FAILED);
} }
for (int i = 0; i < counter + 1; ++i) { for (int i = 0; i < counter + 1; ++i) {
intervals[startIndex] = start->valuedouble + i * width->valuedouble; intervals[startIndex] = start->valuedouble + i * width->valuedouble;
if (isinf(intervals[startIndex])) { if (isinf(intervals[startIndex])) {
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; SCL_ERR_RET(TSDB_CODE_FAILED);
} }
startIndex++; startIndex++;
} }
@ -2665,26 +2755,26 @@ static bool getHistogramBinDesc(SHistoFuncBin **bins, int32_t *binNum, char *bin
if (start->valuedouble == 0) { if (start->valuedouble == 0) {
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; SCL_ERR_RET(TSDB_CODE_FAILED);
} }
if (factor->valuedouble < 0 || factor->valuedouble == 0 || factor->valuedouble == 1) { if (factor->valuedouble < 0 || factor->valuedouble == 0 || factor->valuedouble == 1) {
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; SCL_ERR_RET(TSDB_CODE_FAILED);
} }
for (int i = 0; i < counter + 1; ++i) { for (int i = 0; i < counter + 1; ++i) {
intervals[startIndex] = start->valuedouble * pow(factor->valuedouble, i * 1.0); intervals[startIndex] = start->valuedouble * pow(factor->valuedouble, i * 1.0);
if (isinf(intervals[startIndex])) { if (isinf(intervals[startIndex])) {
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; SCL_ERR_RET(TSDB_CODE_FAILED);
} }
startIndex++; startIndex++;
} }
} else { } else {
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; SCL_ERR_RET(TSDB_CODE_FAILED);
} }
if (infinity->valueint == true) { if (infinity->valueint == true) {
@ -2692,7 +2782,7 @@ static bool getHistogramBinDesc(SHistoFuncBin **bins, int32_t *binNum, char *bin
intervals[numOfBins - 1] = INFINITY; intervals[numOfBins - 1] = INFINITY;
// in case of desc bin orders, -inf/inf should be swapped // in case of desc bin orders, -inf/inf should be swapped
if (numOfBins < 4) { if (numOfBins < 4) {
return false; SCL_ERR_RET(TSDB_CODE_FAILED);
} }
if (intervals[1] > intervals[numOfBins - 2]) { if (intervals[1] > intervals[numOfBins - 2]) {
TSWAP(intervals[0], intervals[numOfBins - 1]); TSWAP(intervals[0], intervals[numOfBins - 1]);
@ -2701,15 +2791,19 @@ static bool getHistogramBinDesc(SHistoFuncBin **bins, int32_t *binNum, char *bin
} else if (cJSON_IsArray(binDesc)) { /* user input bins */ } else if (cJSON_IsArray(binDesc)) { /* user input bins */
if (binType != USER_INPUT_BIN) { if (binType != USER_INPUT_BIN) {
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; SCL_ERR_RET(TSDB_CODE_FAILED);
} }
numOfBins = cJSON_GetArraySize(binDesc); numOfBins = cJSON_GetArraySize(binDesc);
intervals = taosMemoryCalloc(numOfBins, sizeof(double)); intervals = taosMemoryCalloc(numOfBins, sizeof(double));
if (NULL == intervals) {
cJSON_Delete(binDesc);
SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
cJSON *bin = binDesc->child; cJSON *bin = binDesc->child;
if (bin == NULL) { if (bin == NULL) {
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; SCL_ERR_RET(TSDB_CODE_FAILED);
} }
int i = 0; int i = 0;
while (bin) { while (bin) {
@ -2717,23 +2811,26 @@ static bool getHistogramBinDesc(SHistoFuncBin **bins, int32_t *binNum, char *bin
if (!cJSON_IsNumber(bin)) { if (!cJSON_IsNumber(bin)) {
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; SCL_ERR_RET(TSDB_CODE_FAILED);
} }
if (i != 0 && intervals[i] <= intervals[i - 1]) { if (i != 0 && intervals[i] <= intervals[i - 1]) {
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; SCL_ERR_RET(TSDB_CODE_FAILED);
} }
bin = bin->next; bin = bin->next;
i++; i++;
} }
} else { } else {
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; SCL_RET(TSDB_CODE_FAILED);
} }
*binNum = numOfBins - 1; *binNum = numOfBins - 1;
*bins = taosMemoryCalloc(numOfBins, sizeof(SHistoFuncBin)); *bins = taosMemoryCalloc(numOfBins, sizeof(SHistoFuncBin));
if (NULL == bins) {
SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
for (int32_t i = 0; i < *binNum; ++i) { for (int32_t i = 0; i < *binNum; ++i) {
(*bins)[i].lower = intervals[i] < intervals[i + 1] ? intervals[i] : intervals[i + 1]; (*bins)[i].lower = intervals[i] < intervals[i + 1] ? intervals[i] : intervals[i + 1];
(*bins)[i].upper = intervals[i + 1] > intervals[i] ? intervals[i + 1] : intervals[i]; (*bins)[i].upper = intervals[i + 1] > intervals[i] ? intervals[i + 1] : intervals[i];
@ -2743,7 +2840,7 @@ static bool getHistogramBinDesc(SHistoFuncBin **bins, int32_t *binNum, char *bin
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return true; SCL_RET(TSDB_CODE_SUCCESS);
} }
int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
@ -2762,9 +2859,10 @@ int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
int64_t normalized = *(int64_t *)(pInput[3].columnData->pData); int64_t normalized = *(int64_t *)(pInput[3].columnData->pData);
int32_t type = GET_PARAM_TYPE(pInput); int32_t type = GET_PARAM_TYPE(pInput);
if (!getHistogramBinDesc(&bins, &numOfBins, binDesc, binType, (bool)normalized)) { int32_t code = getHistogramBinDesc(&bins, &numOfBins, binDesc, binType, (bool)normalized);
if (TSDB_CODE_SUCCESS != code) {
taosMemoryFree(binDesc); taosMemoryFree(binDesc);
return TSDB_CODE_FAILED; return code;
} }
taosMemoryFree(binDesc); taosMemoryFree(binDesc);
@ -2796,7 +2894,7 @@ int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
} }
} }
colInfoDataEnsureCapacity(pOutputData, numOfBins, false); SCL_ERR_JRET(colInfoDataEnsureCapacity(pOutputData, numOfBins, false));
for (int32_t k = 0; k < numOfBins; ++k) { for (int32_t k = 0; k < numOfBins; ++k) {
int32_t len; int32_t len;
@ -2809,11 +2907,12 @@ int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
bins[k].upper, bins[k].percentage); bins[k].upper, bins[k].percentage);
} }
varDataSetLen(buf, len); varDataSetLen(buf, len);
colDataSetVal(pOutputData, k, buf, false); SCL_ERR_JRET(colDataSetVal(pOutputData, k, buf, false));
} }
taosMemoryFree(bins);
pOutput->numOfRows = numOfBins; pOutput->numOfRows = numOfBins;
_return:
taosMemoryFree(bins);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -2830,7 +2929,7 @@ int32_t selectScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
} }
char *data = colDataGetData(pInputData, i); char *data = colDataGetData(pInputData, i);
colDataSetVal(pOutputData, i, data, false); SCL_ERR_RET(colDataSetVal(pOutputData, i, data, false));
} }
pOutput->numOfRows = 1; pOutput->numOfRows = 1;

File diff suppressed because it is too large Load Diff

View File

@ -55,21 +55,27 @@ void flttInitLogFile() {
tsAsyncLog = 0; tsAsyncLog = 0;
qDebugFlag = 159; qDebugFlag = 159;
strcpy(tsLogDir, TD_LOG_DIR_PATH); (void)strcpy(tsLogDir, TD_LOG_DIR_PATH);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir); printf("failed to open log file in directory:%s\n", tsLogDir);
} }
} }
void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) { int32_t flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
SNode *node = (SNode *)nodesMakeNode(QUERY_NODE_VALUE); SNode *node = (SNode *)nodesMakeNode(QUERY_NODE_VALUE);
if (NULL == node) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SValueNode *vnode = (SValueNode *)node; SValueNode *vnode = (SValueNode *)node;
vnode->node.resType.type = dataType; vnode->node.resType.type = dataType;
if (IS_VAR_DATA_TYPE(dataType)) { if (IS_VAR_DATA_TYPE(dataType)) {
vnode->datum.p = (char *)taosMemoryMalloc(varDataTLen(value)); vnode->datum.p = (char *)taosMemoryMalloc(varDataTLen(value));
varDataCopy(vnode->datum.p, value); if (NULL == vnode->datum.p) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
(void)varDataCopy(vnode->datum.p, value);
vnode->node.resType.bytes = varDataLen(value); vnode->node.resType.bytes = varDataLen(value);
} else { } else {
vnode->node.resType.bytes = tDataTypes[dataType].bytes; vnode->node.resType.bytes = tDataTypes[dataType].bytes;
@ -77,13 +83,17 @@ void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
} }
*pNode = (SNode *)vnode; *pNode = (SNode *)vnode;
FLT_RET(TSDB_CODE_SUCCESS);
} }
void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, int32_t flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum,
void *value) { void *value) {
static uint64_t dbidx = 0; static uint64_t dbidx = 0;
SNode *node = (SNode *)nodesMakeNode(QUERY_NODE_COLUMN); SNode *node = (SNode *)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == node) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SColumnNode *rnode = (SColumnNode *)node; SColumnNode *rnode = (SColumnNode *)node;
rnode->node.resType.type = dataType; rnode->node.resType.type = dataType;
rnode->node.resType.bytes = dataBytes; rnode->node.resType.bytes = dataBytes;
@ -96,23 +106,27 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
rnode->colId = 3; rnode->colId = 3;
*pNode = (SNode *)rnode; *pNode = (SNode *)rnode;
return; FLT_RET(TSDB_CODE_SUCCESS);
} }
if (NULL == *block) { if (NULL == *block) {
SSDataBlock *res = createDataBlock(); SSDataBlock *res = createDataBlock();
for (int32_t i = 0; i < 2; ++i) { for (int32_t i = 0; i < 2; ++i) {
SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_NULL, 10, 1 + i); SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_NULL, 10, 1 + i);
blockDataAppendColInfo(res, &idata); FLT_ERR_RET(blockDataAppendColInfo(res, &idata));
} }
SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3); SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3);
blockDataAppendColInfo(res, &idata); FLT_ERR_RET(blockDataAppendColInfo(res, &idata));
blockDataEnsureCapacity(res, rowNum); FLT_ERR_RET(blockDataEnsureCapacity(res, rowNum));
SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
if (NULL == pColumn) {
fltError("fail to get the last task, num:%d", (int32_t)taosArrayGetSize(res->pDataBlock));
FLT_ERR_RET(TSDB_CODE_QRY_SYS_ERROR);
}
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
colDataSetVal(pColumn, i, (const char *)value, false); FLT_ERR_RET(colDataSetVal(pColumn, i, (const char *)value, false));
if (IS_VAR_DATA_TYPE(dataType)) { if (IS_VAR_DATA_TYPE(dataType)) {
value = (char *)value + varDataTLen(value); value = (char *)value + varDataTLen(value);
} else { } else {
@ -130,13 +144,16 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
int32_t idx = taosArrayGetSize(res->pDataBlock); int32_t idx = taosArrayGetSize(res->pDataBlock);
SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1 + idx); SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1 + idx);
blockDataAppendColInfo(res, &idata); FLT_ERR_RET(blockDataAppendColInfo(res, &idata));
blockDataEnsureCapacity(res, rowNum); FLT_ERR_RET(blockDataEnsureCapacity(res, rowNum));
SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
if (NULL == pColumn) {
fltError("fail to get the last task, num:%d", (int32_t)taosArrayGetSize(res->pDataBlock));
FLT_ERR_RET(TSDB_CODE_QRY_SYS_ERROR);
}
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
colDataSetVal(pColumn, i, (const char *)value, false); FLT_ERR_RET(colDataSetVal(pColumn, i, (const char *)value, false));
if (IS_VAR_DATA_TYPE(dataType)) { if (IS_VAR_DATA_TYPE(dataType)) {
value = (char *)value + varDataTLen(value); value = (char *)value + varDataTLen(value);
} else { } else {
@ -149,10 +166,14 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
} }
*pNode = (SNode *)rnode; *pNode = (SNode *)rnode;
FLT_RET(TSDB_CODE_SUCCESS);
} }
void flttMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) { int32_t flttMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) {
SNode *node = (SNode *)nodesMakeNode(QUERY_NODE_OPERATOR); SNode *node = (SNode *)nodesMakeNode(QUERY_NODE_OPERATOR);
if (NULL == node) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SOperatorNode *onode = (SOperatorNode *)node; SOperatorNode *onode = (SOperatorNode *)node;
onode->node.resType.type = resType; onode->node.resType.type = resType;
onode->node.resType.bytes = tDataTypes[resType].bytes; onode->node.resType.bytes = tDataTypes[resType].bytes;
@ -162,10 +183,14 @@ void flttMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode
onode->pRight = pRight; onode->pRight = pRight;
*pNode = (SNode *)onode; *pNode = (SNode *)onode;
FLT_RET(TSDB_CODE_SUCCESS);
} }
void flttMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeList, int32_t nodeNum) { int32_t flttMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeList, int32_t nodeNum) {
SNode *node = (SNode *)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); SNode *node = (SNode *)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
if (NULL == node) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SLogicConditionNode *onode = (SLogicConditionNode *)node; SLogicConditionNode *onode = (SLogicConditionNode *)node;
onode->condType = opType; onode->condType = opType;
onode->node.resType.type = TSDB_DATA_TYPE_BOOL; onode->node.resType.type = TSDB_DATA_TYPE_BOOL;
@ -173,14 +198,18 @@ void flttMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeLi
onode->pParameterList = nodesMakeList(); onode->pParameterList = nodesMakeList();
for (int32_t i = 0; i < nodeNum; ++i) { for (int32_t i = 0; i < nodeNum; ++i) {
nodesListAppend(onode->pParameterList, nodeList[i]); FLT_ERR_RET(nodesListAppend(onode->pParameterList, nodeList[i]));
} }
*pNode = (SNode *)onode; *pNode = (SNode *)onode;
FLT_RET(TSDB_CODE_SUCCESS);
} }
void flttMakeLogicNodeFromList(SNode **pNode, ELogicConditionType opType, SNodeList *nodeList) { int32_t flttMakeLogicNodeFromList(SNode **pNode, ELogicConditionType opType, SNodeList *nodeList) {
SNode *node = (SNode *)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); SNode *node = (SNode *)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
if (NULL == node) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SLogicConditionNode *onode = (SLogicConditionNode *)node; SLogicConditionNode *onode = (SLogicConditionNode *)node;
onode->condType = opType; onode->condType = opType;
onode->node.resType.type = TSDB_DATA_TYPE_BOOL; onode->node.resType.type = TSDB_DATA_TYPE_BOOL;
@ -189,15 +218,20 @@ void flttMakeLogicNodeFromList(SNode **pNode, ELogicConditionType opType, SNodeL
onode->pParameterList = nodeList; onode->pParameterList = nodeList;
*pNode = (SNode *)onode; *pNode = (SNode *)onode;
FLT_RET(TSDB_CODE_SUCCESS);
} }
void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { int32_t flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) {
SNode *node = (SNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); SNode *node = (SNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
if (NULL == node) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SNodeListNode *lnode = (SNodeListNode *)node; SNodeListNode *lnode = (SNodeListNode *)node;
lnode->node.resType.type = resType; lnode->node.resType.type = resType;
lnode->pNodeList = list; lnode->pNodeList = list;
*pNode = (SNode *)lnode; *pNode = (SNode *)lnode;
FLT_RET(TSDB_CODE_SUCCESS);
} }
void initScalarParam(SScalarParam *pParam) { void initScalarParam(SScalarParam *pParam) {
@ -213,17 +247,21 @@ TEST(timerangeTest, greater) {
SScalarParam res; SScalarParam res;
initScalarParam(&res); initScalarParam(&res);
int32_t code = TSDB_CODE_SUCCESS;
int64_t tsmall = 222, tbig = 333; int64_t tsmall = 222, tbig = 333;
flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); code = flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL);
flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); ASSERT_EQ(code, 0);
flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); code = flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall);
ASSERT_EQ(code, 0);
code = flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval);
ASSERT_EQ(code, 0);
// SFilterInfo *filter = NULL; // SFilterInfo *filter = NULL;
// int32_t code = filterInitFromNode(opNode1, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); // int32_t code = filterInitFromNode(opNode1, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP);
// ASSERT_EQ(code, 0); // ASSERT_EQ(code, 0);
STimeWindow win = {0}; STimeWindow win = {0};
bool isStrict = false; bool isStrict = false;
int32_t code = filterGetTimeRange(opNode1, &win, &isStrict); code = filterGetTimeRange(opNode1, &win, &isStrict);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(isStrict, true); ASSERT_EQ(isStrict, true);
ASSERT_EQ(win.skey, tsmall + 1); ASSERT_EQ(win.skey, tsmall + 1);
@ -237,25 +275,33 @@ TEST(timerangeTest, greater_and_lower) {
bool eRes[5] = {false, false, true, true, true}; bool eRes[5] = {false, false, true, true, true};
SScalarParam res; SScalarParam res;
initScalarParam(&res); initScalarParam(&res);
int32_t code = TSDB_CODE_SUCCESS;
int64_t tsmall = 222, tbig = 333; int64_t tsmall = 222, tbig = 333;
flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); code = flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL);
flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); ASSERT_EQ(code, 0);
flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); code = flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall);
flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); ASSERT_EQ(code, 0);
flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig); code = flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval);
flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); ASSERT_EQ(code, 0);
code = flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL);
ASSERT_EQ(code, 0);
code = flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig);
ASSERT_EQ(code, 0);
code = flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval);
ASSERT_EQ(code, 0);
SNode *list[2] = {0}; SNode *list[2] = {0};
list[0] = opNode1; list[0] = opNode1;
list[1] = opNode2; list[1] = opNode2;
flttMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); code = flttMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2);
ASSERT_EQ(code, 0);
// SFilterInfo *filter = NULL; // SFilterInfo *filter = NULL;
// int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); // int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP);
// ASSERT_EQ(code, 0); // ASSERT_EQ(code, 0);
STimeWindow win = {0}; STimeWindow win = {0};
bool isStrict = false; bool isStrict = false;
int32_t code = filterGetTimeRange(logicNode, &win, &isStrict); code = filterGetTimeRange(logicNode, &win, &isStrict);
ASSERT_EQ(isStrict, true); ASSERT_EQ(isStrict, true);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(win.skey, tsmall + 1); ASSERT_EQ(win.skey, tsmall + 1);
@ -269,25 +315,33 @@ TEST(timerangeTest, greater_equal_and_lower_equal) {
bool eRes[5] = {false, false, true, true, true}; bool eRes[5] = {false, false, true, true, true};
SScalarParam res; SScalarParam res;
initScalarParam(&res); initScalarParam(&res);
int32_t code = TSDB_CODE_SUCCESS;
int64_t tsmall = 222, tbig = 333; int64_t tsmall = 222, tbig = 333;
flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); code = flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL);
flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); ASSERT_EQ(code, 0);
flttMakeOpNode(&opNode1, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pcol, pval); code = flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall);
flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); ASSERT_EQ(code, 0);
flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig); code = flttMakeOpNode(&opNode1, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pcol, pval);
flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pcol, pval); ASSERT_EQ(code, 0);
code = flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL);
ASSERT_EQ(code, 0);
code = flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig);
ASSERT_EQ(code, 0);
code = flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pcol, pval);
ASSERT_EQ(code, 0);
SNode *list[2] = {0}; SNode *list[2] = {0};
list[0] = opNode1; list[0] = opNode1;
list[1] = opNode2; list[1] = opNode2;
flttMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); code = flttMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2);
ASSERT_EQ(code, 0);
// SFilterInfo *filter = NULL; // SFilterInfo *filter = NULL;
// int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); // int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP);
// ASSERT_EQ(code, 0); // ASSERT_EQ(code, 0);
STimeWindow win = {0}; STimeWindow win = {0};
bool isStrict = false; bool isStrict = false;
int32_t code = filterGetTimeRange(logicNode, &win, &isStrict); code = filterGetTimeRange(logicNode, &win, &isStrict);
ASSERT_EQ(isStrict, true); ASSERT_EQ(isStrict, true);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(win.skey, tsmall); ASSERT_EQ(win.skey, tsmall);
@ -301,42 +355,58 @@ TEST(timerangeTest, greater_and_lower_not_strict) {
bool eRes[5] = {false, false, true, true, true}; bool eRes[5] = {false, false, true, true, true};
SScalarParam res; SScalarParam res;
initScalarParam(&res); initScalarParam(&res);
int32_t code = TSDB_CODE_SUCCESS;
int64_t tsmall1 = 222, tbig1 = 333; int64_t tsmall1 = 222, tbig1 = 333;
int64_t tsmall2 = 444, tbig2 = 555; int64_t tsmall2 = 444, tbig2 = 555;
SNode *list[2] = {0}; SNode *list[2] = {0};
flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); code = flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL);
flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall1); ASSERT_EQ(code, 0);
flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); code = flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall1);
flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); ASSERT_EQ(code, 0);
flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig1); code = flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval);
flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); ASSERT_EQ(code, 0);
code = flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL);
ASSERT_EQ(code, 0);
code = flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig1);
ASSERT_EQ(code, 0);
code = flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval);
ASSERT_EQ(code, 0);
list[0] = opNode1; list[0] = opNode1;
list[1] = opNode2; list[1] = opNode2;
flttMakeLogicNode(&logicNode1, LOGIC_COND_TYPE_AND, list, 2); code = flttMakeLogicNode(&logicNode1, LOGIC_COND_TYPE_AND, list, 2);
ASSERT_EQ(code, 0);
flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); code = flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL);
flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall2); ASSERT_EQ(code, 0);
flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); code = flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall2);
flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); ASSERT_EQ(code, 0);
flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig2); code = flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval);
flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); ASSERT_EQ(code, 0);
code = flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL);
ASSERT_EQ(code, 0);
code = flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig2);
ASSERT_EQ(code, 0);
code = flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval);
ASSERT_EQ(code, 0);
list[0] = opNode1; list[0] = opNode1;
list[1] = opNode2; list[1] = opNode2;
flttMakeLogicNode(&logicNode2, LOGIC_COND_TYPE_AND, list, 2); code = flttMakeLogicNode(&logicNode2, LOGIC_COND_TYPE_AND, list, 2);
ASSERT_EQ(code, 0);
list[0] = logicNode1; list[0] = logicNode1;
list[1] = logicNode2; list[1] = logicNode2;
flttMakeLogicNode(&logicNode1, LOGIC_COND_TYPE_OR, list, 2); code = flttMakeLogicNode(&logicNode1, LOGIC_COND_TYPE_OR, list, 2);
ASSERT_EQ(code, 0);
// SFilterInfo *filter = NULL; // SFilterInfo *filter = NULL;
// int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); // int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP);
// ASSERT_EQ(code, 0); // ASSERT_EQ(code, 0);
STimeWindow win = {0}; STimeWindow win = {0};
bool isStrict = false; bool isStrict = false;
int32_t code = filterGetTimeRange(logicNode1, &win, &isStrict); code = filterGetTimeRange(logicNode1, &win, &isStrict);
ASSERT_EQ(isStrict, false); ASSERT_EQ(isStrict, false);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(win.skey, tsmall1 + 1); ASSERT_EQ(win.skey, tsmall1 + 1);

File diff suppressed because it is too large Load Diff

View File

@ -208,7 +208,7 @@ int32_t streamQueueOpen(int64_t cap, SStreamQueue** pQ);
void streamQueueClose(SStreamQueue* pQueue, int32_t taskId); void streamQueueClose(SStreamQueue* pQueue, int32_t taskId);
void streamQueueProcessSuccess(SStreamQueue* queue); void streamQueueProcessSuccess(SStreamQueue* queue);
void streamQueueProcessFail(SStreamQueue* queue); void streamQueueProcessFail(SStreamQueue* queue);
void* streamQueueNextItem(SStreamQueue* pQueue); void streamQueueNextItem(SStreamQueue* pQueue, SStreamQueueItem** pItem);
void streamFreeQitem(SStreamQueueItem* data); void streamFreeQitem(SStreamQueueItem* data);
int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); int32_t streamQueueGetItemSize(const SStreamQueue* pQueue);
@ -226,7 +226,7 @@ int32_t streamTaskDownloadCheckpointData(const char* id, char* path);
int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask); int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask);
int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask); int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask);
int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId, void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId,
int32_t childId, SEpSet* pEpset, int64_t checkpointId); int32_t childId, SEpSet* pEpset, int64_t checkpointId);
int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId, int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId,
int64_t checkpointId, SRpcMsg* pMsg); int64_t checkpointId, SRpcMsg* pMsg);
@ -236,6 +236,10 @@ typedef int32_t (*__stream_async_exec_fn_t)(void* param);
int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code); int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code);
void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock); void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock);
void streamMutexLock(TdThreadMutex *pMutex);
void streamMutexUnlock(TdThreadMutex *pMutex);
void streamMutexDestroy(TdThreadMutex *pMutex);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -887,8 +887,8 @@ _EXIT:
rocksdb_options_destroy(opts); rocksdb_options_destroy(opts);
rocksdb_cache_destroy(cache); rocksdb_cache_destroy(cache);
rocksdb_env_destroy(env); rocksdb_env_destroy(env);
taosThreadMutexDestroy(&pHandle->mutex); streamMutexDestroy(&pHandle->mutex);
taosThreadMutexDestroy(&pHandle->cfMutex); streamMutexDestroy(&pHandle->cfMutex);
taosHashCleanup(pHandle->cfInst); taosHashCleanup(pHandle->cfInst);
tdListFree(pHandle->list); tdListFree(pHandle->list);
taosMemoryFree(pHandle); taosMemoryFree(pHandle);
@ -923,9 +923,9 @@ void streamBackendCleanup(void* arg) {
} }
tdListFree(pHandle->list); tdListFree(pHandle->list);
taosThreadMutexDestroy(&pHandle->mutex); streamMutexDestroy(&pHandle->mutex);
taosThreadMutexDestroy(&pHandle->cfMutex); streamMutexDestroy(&pHandle->cfMutex);
stDebug("destroy stream backend :%p", pHandle); stDebug("destroy stream backend :%p", pHandle);
taosMemoryFree(pHandle); taosMemoryFree(pHandle);
return; return;
@ -1393,7 +1393,7 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) {
// vnode task->db // vnode task->db
SStreamMeta* pMeta = arg; SStreamMeta* pMeta = arg;
taosThreadMutexLock(&pMeta->backendMutex); streamMutexLock(&pMeta->backendMutex);
void* pIter = taosHashIterate(pMeta->pTaskDbUnique, NULL); void* pIter = taosHashIterate(pMeta->pTaskDbUnique, NULL);
int32_t code = 0; int32_t code = 0;
@ -1434,14 +1434,14 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) {
pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter); pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter);
} }
taosThreadMutexUnlock(&pMeta->backendMutex); streamMutexUnlock(&pMeta->backendMutex);
return code; return code;
} }
int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) { int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) {
if (pSnapInfo == NULL) return 0; if (pSnapInfo == NULL) return 0;
SStreamMeta* pMeta = arg; SStreamMeta* pMeta = arg;
int32_t code = 0; int32_t code = 0;
taosThreadMutexLock(&pMeta->backendMutex); streamMutexLock(&pMeta->backendMutex);
char buf[128] = {0}; char buf[128] = {0};
for (int i = 0; i < taosArrayGetSize(pSnapInfo); i++) { for (int i = 0; i < taosArrayGetSize(pSnapInfo); i++) {
@ -1457,7 +1457,7 @@ int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) {
taskDbUnRefChkp(*pTaskDb, pSnap->chkpId); taskDbUnRefChkp(*pTaskDb, pSnap->chkpId);
} }
taosThreadMutexUnlock(&pMeta->backendMutex); streamMutexUnlock(&pMeta->backendMutex);
return 0; return 0;
} }
#ifdef BUILD_NO_CALL #ifdef BUILD_NO_CALL
@ -1697,17 +1697,17 @@ int32_t streamBackendDoCheckpoint(void* arg, int64_t chkpId, int64_t processVer)
SListNode* streamBackendAddCompare(void* backend, void* arg) { SListNode* streamBackendAddCompare(void* backend, void* arg) {
SBackendWrapper* pHandle = (SBackendWrapper*)backend; SBackendWrapper* pHandle = (SBackendWrapper*)backend;
SListNode* node = NULL; SListNode* node = NULL;
taosThreadMutexLock(&pHandle->mutex); streamMutexLock(&pHandle->mutex);
node = tdListAdd(pHandle->list, arg); node = tdListAdd(pHandle->list, arg);
taosThreadMutexUnlock(&pHandle->mutex); streamMutexUnlock(&pHandle->mutex);
return node; return node;
} }
void streamBackendDelCompare(void* backend, void* arg) { void streamBackendDelCompare(void* backend, void* arg) {
SBackendWrapper* pHandle = (SBackendWrapper*)backend; SBackendWrapper* pHandle = (SBackendWrapper*)backend;
SListNode* node = NULL; SListNode* node = NULL;
taosThreadMutexLock(&pHandle->mutex); streamMutexLock(&pHandle->mutex);
node = tdListPopNode(pHandle->list, arg); node = tdListPopNode(pHandle->list, arg);
taosThreadMutexUnlock(&pHandle->mutex); streamMutexUnlock(&pHandle->mutex);
if (node) { if (node) {
streamStateDestroyCompar(node->data); streamStateDestroyCompar(node->data);
taosMemoryFree(node); taosMemoryFree(node);
@ -2461,9 +2461,9 @@ int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** sta
void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) { void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) {
STaskDbWrapper* p = pTaskDb; STaskDbWrapper* p = pTaskDb;
taosThreadMutexLock(&p->mutex); streamMutexLock(&p->mutex);
p->chkpId = chkpId; p->chkpId = chkpId;
taosThreadMutexUnlock(&p->mutex); streamMutexUnlock(&p->mutex);
} }
STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) { STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) {
@ -2622,7 +2622,7 @@ void taskDbDestroy(void* pDb, bool flush) {
taosMemoryFree(wrapper->pCfOpts); taosMemoryFree(wrapper->pCfOpts);
taosMemoryFree(wrapper->pCfParams); taosMemoryFree(wrapper->pCfParams);
taosThreadMutexDestroy(&wrapper->mutex); streamMutexDestroy(&wrapper->mutex);
taskDbDestroyChkpOpt(wrapper); taskDbDestroyChkpOpt(wrapper);
@ -2957,7 +2957,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
SBackendWrapper* handle = backend; SBackendWrapper* handle = backend;
SBackendCfWrapper* pBackendCfWrapper = taosMemoryCalloc(1, sizeof(SBackendCfWrapper)); SBackendCfWrapper* pBackendCfWrapper = taosMemoryCalloc(1, sizeof(SBackendCfWrapper));
taosThreadMutexLock(&handle->cfMutex); streamMutexLock(&handle->cfMutex);
RocksdbCfInst** ppInst = taosHashGet(handle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); RocksdbCfInst** ppInst = taosHashGet(handle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1);
if (ppInst != NULL && *ppInst != NULL) { if (ppInst != NULL && *ppInst != NULL) {
RocksdbCfInst* inst = *ppInst; RocksdbCfInst* inst = *ppInst;
@ -2970,7 +2970,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
pBackendCfWrapper->param = inst->param; pBackendCfWrapper->param = inst->param;
pBackendCfWrapper->pBackend = handle; pBackendCfWrapper->pBackend = handle;
pBackendCfWrapper->pComparNode = inst->pCompareNode; pBackendCfWrapper->pComparNode = inst->pCompareNode;
taosThreadMutexUnlock(&handle->cfMutex); streamMutexUnlock(&handle->cfMutex);
pBackendCfWrapper->backendId = pState->streamBackendRid; pBackendCfWrapper->backendId = pState->streamBackendRid;
memcpy(pBackendCfWrapper->idstr, pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr)); memcpy(pBackendCfWrapper->idstr, pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr));
@ -2987,7 +2987,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
inst->rOpt = NULL; inst->rOpt = NULL;
return 0; return 0;
} }
taosThreadMutexUnlock(&handle->cfMutex); streamMutexUnlock(&handle->cfMutex);
char* err = NULL; char* err = NULL;
int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]);
@ -3046,14 +3046,14 @@ void streamStateCloseBackend(SStreamState* pState, bool remove) {
stInfo("start to close state on backend: %p", pHandle); stInfo("start to close state on backend: %p", pHandle);
taosThreadMutexLock(&pHandle->cfMutex); streamMutexLock(&pHandle->cfMutex);
RocksdbCfInst** ppInst = taosHashGet(pHandle->cfInst, wrapper->idstr, strlen(pState->pTdbState->idstr) + 1); RocksdbCfInst** ppInst = taosHashGet(pHandle->cfInst, wrapper->idstr, strlen(pState->pTdbState->idstr) + 1);
if (ppInst != NULL && *ppInst != NULL) { if (ppInst != NULL && *ppInst != NULL) {
RocksdbCfInst* inst = *ppInst; RocksdbCfInst* inst = *ppInst;
taosMemoryFree(inst); taosMemoryFree(inst);
taosHashRemove(pHandle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); taosHashRemove(pHandle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1);
} }
taosThreadMutexUnlock(&pHandle->cfMutex); streamMutexUnlock(&pHandle->cfMutex);
char* status[] = {"close", "drop"}; char* status[] = {"close", "drop"};
stInfo("start to %s state %p on backendWrapper %p %s", status[remove == false ? 0 : 1], pState, wrapper, stInfo("start to %s state %p on backendWrapper %p %s", status[remove == false ? 0 : 1], pState, wrapper,
@ -3085,7 +3085,7 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) {
return -1; return -1;
} }
taosThreadMutexLock(&wrapper->mutex); streamMutexLock(&wrapper->mutex);
rocksdb_column_family_handle_t* cf = wrapper->pCf[idx]; rocksdb_column_family_handle_t* cf = wrapper->pCf[idx];
if (cf == NULL) { if (cf == NULL) {
@ -3100,7 +3100,7 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) {
wrapper->pCf[idx] = cf; wrapper->pCf[idx] = cf;
} }
} }
taosThreadMutexUnlock(&wrapper->mutex); streamMutexUnlock(&wrapper->mutex);
} }
return idx; return idx;

View File

@ -23,10 +23,10 @@
static void processDownstreamReadyRsp(SStreamTask* pTask); static void processDownstreamReadyRsp(SStreamTask* pTask);
static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId); static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId);
static void rspMonitorFn(void* param, void* tmrId); static void rspMonitorFn(void* param, void* tmrId);
static int32_t streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs); static void streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs);
static int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id); static int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id);
static int32_t streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* id); static void streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* id);
static int32_t streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, int32_t vgId, const char* id); static void streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, int32_t vgId, const char* id);
static void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p); static void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p);
static void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList); static void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList);
static void handleNotReadyDownstreamTask(SStreamTask* pTask, SArray* pNotReadyList); static void handleNotReadyDownstreamTask(SStreamTask* pTask, SArray* pNotReadyList);
@ -41,9 +41,9 @@ static SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t
int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage, int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage,
int64_t* oldStage) { int64_t* oldStage) {
SStreamUpstreamEpInfo* pInfo = NULL; SStreamUpstreamEpInfo* pInfo = NULL;
int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo); streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) { if (pInfo == NULL) {
return code; return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
*oldStage = pInfo->stage; *oldStage = pInfo->stage;
@ -65,21 +65,21 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_
", prev:%" PRId64, ", prev:%" PRId64,
id, upstreamTaskId, vgId, stage, pInfo->stage); id, upstreamTaskId, vgId, stage, pInfo->stage);
// record the checkpoint failure id and sent to mnode // record the checkpoint failure id and sent to mnode
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
ETaskStatus status = streamTaskGetStatus(pTask).state; ETaskStatus status = streamTaskGetStatus(pTask).state;
if (status == TASK_STATUS__CK) { if (status == TASK_STATUS__CK) {
streamTaskSetFailedCheckpointId(pTask); streamTaskSetFailedCheckpointId(pTask);
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
} }
if (pInfo->stage != stage) { if (pInfo->stage != stage) {
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
ETaskStatus status = streamTaskGetStatus(pTask).state; ETaskStatus status = streamTaskGetStatus(pTask).state;
if (status == TASK_STATUS__CK) { if (status == TASK_STATUS__CK) {
streamTaskSetFailedCheckpointId(pTask); streamTaskSetFailedCheckpointId(pTask);
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return TASK_UPSTREAM_NEW_STAGE; return TASK_UPSTREAM_NEW_STAGE;
} else if (pTask->status.downstreamReady != 1) { } else if (pTask->status.downstreamReady != 1) {
@ -120,7 +120,7 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) {
idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, pRange->range.minVer, idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, pRange->range.minVer,
pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId); pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId);
streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, &pTask->outputInfo.fixedDispatcher.epSet); (void) streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, &pTask->outputInfo.fixedDispatcher.epSet);
} else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
streamTaskStartMonitorCheckRsp(pTask); streamTaskStartMonitorCheckRsp(pTask);
@ -140,7 +140,7 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) {
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 stDebug("s-task:%s (vgId:%d) stage:%" PRId64
" check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, reqId:0x%" PRIx64, " check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, reqId:0x%" PRIx64,
idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, req.reqId); idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, req.reqId);
streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); (void) streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
} }
} else { // for sink task, set it ready directly. } else { // for sink task, set it ready directly.
stDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", idstr, pTask->info.nodeId); stDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", idstr, pTask->info.nodeId);
@ -265,7 +265,7 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tEncoderInit(&encoder, (uint8_t*)abuf, len); tEncoderInit(&encoder, (uint8_t*)abuf, len);
tEncodeStreamTaskCheckRsp(&encoder, pRsp); (void) tEncodeStreamTaskCheckRsp(&encoder, pRsp);
tEncoderClear(&encoder); tEncoderClear(&encoder);
SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = *pRpcInfo}; SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = *pRpcInfo};
@ -274,14 +274,16 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa
return 0; return 0;
} }
int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask) { void streamTaskStartMonitorCheckRsp(SStreamTask* pTask) {
int32_t vgId = pTask->pMeta->vgId;
STaskCheckInfo* pInfo = &pTask->taskCheckInfo; STaskCheckInfo* pInfo = &pTask->taskCheckInfo;
taosThreadMutexLock(&pInfo->checkInfoLock);
streamMutexLock(&pInfo->checkInfoLock);
int32_t code = streamTaskStartCheckDownstream(pInfo, pTask->id.idStr); int32_t code = streamTaskStartCheckDownstream(pInfo, pTask->id.idStr);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
return TSDB_CODE_FAILED; return;
} }
/*SStreamTask* p = */ streamMetaAcquireOneTask(pTask); // add task ref here /*SStreamTask* p = */ streamMetaAcquireOneTask(pTask); // add task ref here
@ -293,20 +295,18 @@ int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask) {
if (pInfo->checkRspTmr == NULL) { if (pInfo->checkRspTmr == NULL) {
pInfo->checkRspTmr = taosTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer); pInfo->checkRspTmr = taosTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer);
} else { } else {
taosTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr); streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, "check-status-monitor");
} }
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
return 0;
} }
int32_t streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id) { void streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id) {
taosThreadMutexLock(&pInfo->checkInfoLock); streamMutexLock(&pInfo->checkInfoLock);
pInfo->stopCheckProcess = 1; pInfo->stopCheckProcess = 1;
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
stDebug("s-task:%s set stop check-rsp monitor flag", id); stDebug("s-task:%s set stop check-rsp monitor flag", id);
return TSDB_CODE_SUCCESS;
} }
void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) {
@ -316,21 +316,21 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) {
pInfo->pList = NULL; pInfo->pList = NULL;
if (pInfo->checkRspTmr != NULL) { if (pInfo->checkRspTmr != NULL) {
/*bool ret = */ taosTmrStop(pInfo->checkRspTmr); (void) taosTmrStop(pInfo->checkRspTmr);
pInfo->checkRspTmr = NULL; pInfo->checkRspTmr = NULL;
} }
taosThreadMutexDestroy(&pInfo->checkInfoLock); streamMutexDestroy(&pInfo->checkInfoLock);
} }
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void processDownstreamReadyRsp(SStreamTask* pTask) { void processDownstreamReadyRsp(SStreamTask* pTask) {
EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST; EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST;
streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); (void) streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL);
int64_t checkTs = pTask->execInfo.checkTs; int64_t checkTs = pTask->execInfo.checkTs;
int64_t readyTs = pTask->execInfo.readyTs; int64_t readyTs = pTask->execInfo.readyTs;
streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true); (void) streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true);
if (pTask->status.taskStatus == TASK_STATUS__HALT) { if (pTask->status.taskStatus == TASK_STATUS__HALT) {
ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask) && (pTask->info.fillHistory == 0)); ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask) && (pTask->info.fillHistory == 0));
@ -338,21 +338,25 @@ void processDownstreamReadyRsp(SStreamTask* pTask) {
// halt it self for count window stream task until the related fill history task completed. // halt it self for count window stream task until the related fill history task completed.
stDebug("s-task:%s level:%d initial status is %s from mnode, set it to be halt", pTask->id.idStr, stDebug("s-task:%s level:%d initial status is %s from mnode, set it to be halt", pTask->id.idStr,
pTask->info.taskLevel, streamTaskGetStatusStr(pTask->status.taskStatus)); pTask->info.taskLevel, streamTaskGetStatusStr(pTask->status.taskStatus));
streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT); int32_t code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT);
if (code != 0) {
// todo: handle error
}
} }
// start the related fill-history task, when current task is ready // start the related fill-history task, when current task is ready
// not invoke in success callback due to the deadlock. // not invoke in success callback due to the deadlock.
// todo: let's retry
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
stDebug("s-task:%s try to launch related fill-history task", pTask->id.idStr); stDebug("s-task:%s try to launch related fill-history task", pTask->id.idStr);
streamLaunchFillHistoryTask(pTask); (void) streamLaunchFillHistoryTask(pTask);
} }
} }
void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) {
int32_t vgId = pTask->pMeta->vgId; int32_t vgId = pTask->pMeta->vgId;
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList); int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList);
bool existed = false; bool existed = false;
for (int i = 0; i < num; ++i) { for (int i = 0; i < num; ++i) {
@ -365,16 +369,18 @@ void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) {
if (!existed) { if (!existed) {
SDownstreamTaskEpset t = {.nodeId = nodeId}; SDownstreamTaskEpset t = {.nodeId = nodeId};
taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t); void* p = taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t);
if (p == NULL) {
// todo let's retry
}
stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", pTask->id.idStr, vgId, stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", pTask->id.idStr, vgId,
t.nodeId, (num + 1)); t.nodeId, (num + 1));
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
} }
int32_t streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs) { void streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs) {
taosArrayClear(pInfo->pList); taosArrayClear(pInfo->pList);
if (pOutputInfo->type == TASK_OUTPUT__FIXED_DISPATCH) { if (pOutputInfo->type == TASK_OUTPUT__FIXED_DISPATCH) {
@ -387,7 +393,6 @@ int32_t streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOut
pInfo->startTs = startTs; pInfo->startTs = startTs;
pInfo->timeoutStartTs = startTs; pInfo->timeoutStartTs = startTs;
pInfo->stopCheckProcess = 0; pInfo->stopCheckProcess = 0;
return TSDB_CODE_SUCCESS;
} }
SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId) { SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId) {
@ -403,7 +408,7 @@ SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId)
int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t status, int64_t rspTs, int64_t reqId, int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t status, int64_t rspTs, int64_t reqId,
int32_t* pNotReady, const char* id) { int32_t* pNotReady, const char* id) {
taosThreadMutexLock(&pInfo->checkInfoLock); streamMutexLock(&pInfo->checkInfoLock);
SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId); SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId);
if (p != NULL) { if (p != NULL) {
@ -411,7 +416,7 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t
stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64 stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64
" expired check-rsp recv from downstream task:0x%x, discarded", " expired check-rsp recv from downstream task:0x%x, discarded",
id, reqId, p->reqId, taskId); id, reqId, p->reqId, taskId);
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -425,11 +430,11 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t
p->status = status; p->status = status;
p->rspTs = rspTs; p->rspTs = rspTs;
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
stError("s-task:%s unexpected check rsp msg, invalid downstream task:0x%x, reqId:%" PRIx64 " discarded", id, taskId, stError("s-task:%s unexpected check rsp msg, invalid downstream task:0x%x, reqId:%" PRIx64 " discarded", id, taskId,
reqId); reqId);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
@ -450,9 +455,9 @@ int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* id) { void streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* id) {
if (lock) { if (lock) {
taosThreadMutexLock(&pInfo->checkInfoLock); streamMutexLock(&pInfo->checkInfoLock);
} }
if (pInfo->inCheckProcess) { if (pInfo->inCheckProcess) {
@ -474,27 +479,28 @@ int32_t streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char*
} }
if (lock) { if (lock) {
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
} }
return 0;
} }
int32_t streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, int32_t vgId, const char* id) { // todo: retry until success
void streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, int32_t vgId, const char* id) {
SDownstreamStatusInfo info = {.taskId = taskId, .status = -1, .vgId = vgId, .reqId = reqId, .rspTs = 0}; SDownstreamStatusInfo info = {.taskId = taskId, .status = -1, .vgId = vgId, .reqId = reqId, .rspTs = 0};
taosThreadMutexLock(&pInfo->checkInfoLock); streamMutexLock(&pInfo->checkInfoLock);
SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId); SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId);
if (p != NULL) { if (p != NULL) {
stDebug("s-task:%s check info to task:0x%x already sent", id, taskId); stDebug("s-task:%s check info to task:0x%x already sent", id, taskId);
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
return TSDB_CODE_SUCCESS; return;
} }
taosArrayPush(pInfo->pList, &info); void* px = taosArrayPush(pInfo->pList, &info);
if (px == NULL) {
// todo: retry
}
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
return TSDB_CODE_SUCCESS;
} }
void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) {
@ -519,7 +525,7 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) {
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64, id, stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64, id,
pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId); pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId);
streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet); (void) streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet);
} else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) { } else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
SArray* vgInfo = pOutputInfo->shuffleDispatcher.dbInfo.pVgroupInfos; SArray* vgInfo = pOutputInfo->shuffleDispatcher.dbInfo.pVgroupInfos;
int32_t numOfVgs = taosArrayGetSize(vgInfo); int32_t numOfVgs = taosArrayGetSize(vgInfo);
@ -533,7 +539,7 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) {
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 stDebug("s-task:%s (vgId:%d) stage:%" PRId64
" re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d reqId:0x%" PRIx64, " re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d reqId:0x%" PRIx64,
id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, p->reqId); id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, p->reqId);
streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); (void) streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
break; break;
} }
} }
@ -556,12 +562,12 @@ void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, i
if (p->rspTs == 0) { // not response yet if (p->rspTs == 0) { // not response yet
ASSERT(p->status == -1); ASSERT(p->status == -1);
if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec. if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec.
taosArrayPush(pTimeoutList, &p->taskId); (void) taosArrayPush(pTimeoutList, &p->taskId);
} else { // el < CHECK_NOT_RSP_DURATION } else { // el < CHECK_NOT_RSP_DURATION
(*numOfNotRsp) += 1; // do nothing and continue waiting for their rsp (*numOfNotRsp) += 1; // do nothing and continue waiting for their rsp
} }
} else { } else {
taosArrayPush(pNotReadyList, &p->taskId); (void) taosArrayPush(pNotReadyList, &p->taskId);
} }
} }
} }
@ -676,7 +682,7 @@ void rspMonitorFn(void* param, void* tmrId) {
// not record the failed of the current task if try to close current vnode // not record the failed of the current task if try to close current vnode
// otherwise, the put of message operation may incur invalid read of message queue. // otherwise, the put of message operation may incur invalid read of message queue.
if (!pMeta->closeFlag) { if (!pMeta->closeFlag) {
addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); (void) addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId);
} }
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
@ -692,14 +698,14 @@ void rspMonitorFn(void* param, void* tmrId) {
return; return;
} }
taosThreadMutexLock(&pInfo->checkInfoLock); streamMutexLock(&pInfo->checkInfoLock);
if (pInfo->notReadyTasks == 0) { if (pInfo->notReadyTasks == 0) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, pStat.name, stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, pStat.name,
vgId, ref); vgId, ref);
streamTaskCompleteCheckRsp(pInfo, false, id); streamTaskCompleteCheckRsp(pInfo, false, id);
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return; return;
} }
@ -723,7 +729,7 @@ void rspMonitorFn(void* param, void* tmrId) {
id, pStat.name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); id, pStat.name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref);
streamTaskCompleteCheckRsp(pInfo, false, id); streamTaskCompleteCheckRsp(pInfo, false, id);
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
taosArrayDestroy(pNotReadyList); taosArrayDestroy(pNotReadyList);
@ -743,9 +749,9 @@ void rspMonitorFn(void* param, void* tmrId) {
id, pStat.name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); id, pStat.name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref);
streamTaskCompleteCheckRsp(pInfo, false, id); streamTaskCompleteCheckRsp(pInfo, false, id);
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); (void) addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
taosArrayDestroy(pNotReadyList); taosArrayDestroy(pNotReadyList);
@ -761,8 +767,8 @@ void rspMonitorFn(void* param, void* tmrId) {
handleTimeoutDownstreamTasks(pTask, pTimeoutList); handleTimeoutDownstreamTasks(pTask, pTimeoutList);
} }
taosTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr); streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, "check-status-monitor");
taosThreadMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
stDebug( stDebug(
"s-task:%s vgId:%d continue checking rsp in 300ms, total:%d, notRsp:%d, notReady:%d, fault:%d, timeout:%d, " "s-task:%s vgId:%d continue checking rsp in 300ms, total:%d, notRsp:%d, notReady:%d, fault:%d, timeout:%d, "

View File

@ -77,6 +77,7 @@ int32_t createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int6
} }
// this message must be put into inputq successfully, continue retrying until it succeeds // this message must be put into inputq successfully, continue retrying until it succeeds
// todo must be success
int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId, int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId,
int32_t srcTaskId) { int32_t srcTaskId) {
SStreamDataBlock* pCheckpoint = NULL; SStreamDataBlock* pCheckpoint = NULL;
@ -119,7 +120,7 @@ int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTri
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER, pRsp->checkpointId, pRsp->transId, (void)appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER, pRsp->checkpointId, pRsp->transId,
pRsp->upstreamTaskId); pRsp->upstreamTaskId);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -164,7 +165,7 @@ int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStream
int32_t code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock); int32_t code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock);
if (code == 0) { if (code == 0) {
ASSERT(pTask->chkInfo.pActiveInfo->dispatchTrigger == false); ASSERT(pTask->chkInfo.pActiveInfo->dispatchTrigger == false);
streamDispatchStreamBlock(pTask); code = streamDispatchStreamBlock(pTask);
} else { } else {
stError("s-task:%s failed to put checkpoint into outputQ, code:%s", pTask->id.idStr, tstrerror(code)); stError("s-task:%s failed to put checkpoint into outputQ, code:%s", pTask->id.idStr, tstrerror(code));
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
@ -184,12 +185,12 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pTask->chkInfo.checkpointId > checkpointId) { if (pTask->chkInfo.checkpointId > checkpointId) {
stError("s-task:%s vgId:%d current checkpointId:%" PRId64 stError("s-task:%s vgId:%d current checkpointId:%" PRId64
" recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard", " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard",
id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId); id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId);
code = taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
return code; return code;
@ -199,13 +200,16 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
{ // send checkpoint-ready msg to upstream { // send checkpoint-ready msg to upstream
SRpcMsg msg = {0}; SRpcMsg msg = {0};
SStreamUpstreamEpInfo* pInfo = NULL; SStreamUpstreamEpInfo* pInfo = NULL;
code = streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId, &pInfo); streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) { if (pInfo == NULL) {
return code; streamMutexUnlock(&pTask->lock);
return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg); code = initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg);
tmsgSendReq(&pInfo->epSet, &msg); if (code == TSDB_CODE_SUCCESS) {
(void)tmsgSendReq(&pInfo->epSet, &msg);
}
} }
stWarn( stWarn(
@ -214,7 +218,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
id, vgId, pBlock->srcTaskId); id, vgId, pBlock->srcTaskId);
streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId); streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId);
code = taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
return code; return code;
@ -225,7 +229,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64 stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64
" discard", " discard",
id, vgId, pActiveInfo->activeId, checkpointId); id, vgId, pActiveInfo->activeId, checkpointId);
code = taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
return code; return code;
@ -235,7 +239,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
"s-task:%s vgId:%d all upstream checkpoint-trigger recv, discard this checkpoint-trigger, " "s-task:%s vgId:%d all upstream checkpoint-trigger recv, discard this checkpoint-trigger, "
"checkpointId:%" PRId64 " transId:%d", "checkpointId:%" PRId64 " transId:%d",
id, vgId, checkpointId, transId); id, vgId, checkpointId, transId);
code = taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
return code; return code;
} }
@ -250,7 +254,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
", prev recvTs:%" PRId64 " discard", ", prev recvTs:%" PRId64 " discard",
pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs); pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs);
code = taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
return code; return code;
} }
@ -259,7 +263,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
} }
} }
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
stDebug("s-task:%s vgId:%d start to handle the checkpoint-trigger block, checkpointId:%" PRId64 " ver:%" PRId64 stDebug("s-task:%s vgId:%d start to handle the checkpoint-trigger block, checkpointId:%" PRId64 " ver:%" PRId64
", transId:%d current active checkpointId:%" PRId64, ", transId:%d current active checkpointId:%" PRId64,
@ -284,7 +288,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
if (pActiveInfo->pChkptTriggerTmr == NULL) { if (pActiveInfo->pChkptTriggerTmr == NULL) {
pActiveInfo->pChkptTriggerTmr = taosTmrStart(checkpointTriggerMonitorFn, 100, pTask, streamTimer); pActiveInfo->pChkptTriggerTmr = taosTmrStart(checkpointTriggerMonitorFn, 100, pTask, streamTimer);
} else { } else {
taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr); streamTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr, vgId, "trigger-recv-monitor");
} }
} }
@ -300,9 +304,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) { if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId); stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId);
continueDispatchCheckpointTriggerBlock(pBlock, pTask); (void)continueDispatchCheckpointTriggerBlock(pBlock, pTask); // todo handle this failure
} else { // only one task exists, no need to dispatch downstream info } else { // only one task exists, no need to dispatch downstream info
appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId, -1); (void)appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId,
-1);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
} }
} else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) { } else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) {
@ -312,8 +317,9 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
pTask->execInfo.checkpoint += 1; pTask->execInfo.checkpoint += 1;
} }
// todo: handle this
// update the child Id for downstream tasks // update the child Id for downstream tasks
streamAddCheckpointReadyMsg(pTask, pBlock->srcTaskId, pTask->info.selfChildId, checkpointId); (void) streamAddCheckpointReadyMsg(pTask, pBlock->srcTaskId, pTask->info.selfChildId, checkpointId);
// there are still some upstream tasks not send checkpoint request, do nothing and wait for then // there are still some upstream tasks not send checkpoint request, do nothing and wait for then
if (pActiveInfo->allUpstreamTriggerRecv != 1) { if (pActiveInfo->allUpstreamTriggerRecv != 1) {
@ -325,7 +331,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
if (taskLevel == TASK_LEVEL__SINK) { if (taskLevel == TASK_LEVEL__SINK) {
stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, send ready msg to upstream", id, num); stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, send ready msg to upstream", id, num);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
streamTaskBuildCheckpoint(pTask); (void)streamTaskBuildCheckpoint(pTask); // todo: not handle error yet
} else { // source & agg tasks need to forward the checkpoint msg downwards } else { // source & agg tasks need to forward the checkpoint msg downwards
stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, forwards to downstream", id, num); stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, forwards to downstream", id, num);
@ -369,7 +375,7 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId
return -1; return -1;
} }
(void) taosThreadMutexLock(&pInfo->lock); streamMutexLock(&pInfo->lock);
// only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task // only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task
int32_t size = taosArrayGetSize(pInfo->pCheckpointReadyRecvList); int32_t size = taosArrayGetSize(pInfo->pCheckpointReadyRecvList);
@ -391,16 +397,16 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId
.transId = pInfo->transId, .transId = pInfo->transId,
.streamId = pTask->id.streamId, .streamId = pTask->id.streamId,
.downstreamNodeId = downstreamNodeId}; .downstreamNodeId = downstreamNodeId};
taosArrayPush(pInfo->pCheckpointReadyRecvList, &info); (void)taosArrayPush(pInfo->pCheckpointReadyRecvList, &info);
} }
int32_t notReady = total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList); int32_t notReady = total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList);
int32_t transId = pInfo->transId; int32_t transId = pInfo->transId;
(void) taosThreadMutexUnlock(&pInfo->lock); streamMutexUnlock(&pInfo->lock);
if (notReady == 0) { if (notReady == 0) {
stDebug("s-task:%s all downstream tasks have completed build checkpoint, do checkpoint for current task", id); stDebug("s-task:%s all downstream tasks have completed build checkpoint, do checkpoint for current task", id);
appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, checkpointId, transId, -1); (void)appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, checkpointId, transId, -1);
} }
return 0; return 0;
@ -411,7 +417,7 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream
int64_t now = taosGetTimestampMs(); int64_t now = taosGetTimestampMs();
int32_t numOfConfirmed = 0; int32_t numOfConfirmed = 0;
(void) taosThreadMutexLock(&pInfo->lock); streamMutexLock(&pInfo->lock);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) {
STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i); STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i);
if (pReadyInfo->upstreamTaskId == upstreamTaskId && pReadyInfo->checkpointId == checkpointId) { if (pReadyInfo->upstreamTaskId == upstreamTaskId && pReadyInfo->checkpointId == checkpointId) {
@ -432,7 +438,7 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream
stDebug("s-task:%s send checkpoint-ready msg to %d upstream confirmed, checkpointId:%" PRId64, pTask->id.idStr, stDebug("s-task:%s send checkpoint-ready msg to %d upstream confirmed, checkpointId:%" PRId64, pTask->id.idStr,
numOfConfirmed, checkpointId); numOfConfirmed, checkpointId);
(void) taosThreadMutexUnlock(&pInfo->lock); streamMutexUnlock(&pInfo->lock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -440,12 +446,12 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) {
pTask->chkInfo.startTs = 0; // clear the recorded start time pTask->chkInfo.startTs = 0; // clear the recorded start time
streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks
(void) taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock); streamMutexLock(&pTask->chkInfo.pActiveInfo->lock);
streamTaskClearActiveInfo(pTask->chkInfo.pActiveInfo); streamTaskClearActiveInfo(pTask->chkInfo.pActiveInfo);
if (clearChkpReadyMsg) { if (clearChkpReadyMsg) {
streamClearChkptReadyMsg(pTask->chkInfo.pActiveInfo); streamClearChkptReadyMsg(pTask->chkInfo.pActiveInfo);
} }
(void) taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock); streamMutexUnlock(&pTask->chkInfo.pActiveInfo->lock);
} }
int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq) { int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq) {
@ -455,7 +461,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
SCheckpointInfo* pInfo = &pTask->chkInfo; SCheckpointInfo* pInfo = &pTask->chkInfo;
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pReq->checkpointId <= pInfo->checkpointId) { if (pReq->checkpointId <= pInfo->checkpointId) {
stDebug("s-task:%s vgId:%d latest checkpointId:%" PRId64 " checkpointVer:%" PRId64 stDebug("s-task:%s vgId:%d latest checkpointId:%" PRId64 " checkpointVer:%" PRId64
@ -463,7 +469,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
" transId:%d ignored", " transId:%d ignored",
id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer, id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer,
pReq->transId); pReq->transId);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
{ // destroy the related fill-history tasks { // destroy the related fill-history tasks
// drop task should not in the meta-lock, and drop the related fill-history task now // drop task should not in the meta-lock, and drop the related fill-history task now
@ -525,13 +531,14 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
pTask->status.taskStatus = TASK_STATUS__READY; pTask->status.taskStatus = TASK_STATUS__READY;
code = streamMetaSaveTask(pMeta, pTask); code = streamMetaSaveTask(pMeta, pTask);
streamMutexUnlock(&pTask->lock);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s vgId:%d failed to save task info after do checkpoint, checkpointId:%" PRId64 ", since %s", id, stError("s-task:%s vgId:%d failed to save task info after do checkpoint, checkpointId:%" PRId64 ", since %s", id,
vgId, pReq->checkpointId, terrstr()); vgId, pReq->checkpointId, terrstr());
return code; return code;
} }
(void) taosThreadMutexUnlock(&pTask->lock);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
// drop task should not in the meta-lock, and drop the related fill-history task now // drop task should not in the meta-lock, and drop the related fill-history task now
@ -730,9 +737,9 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
code = streamSendChkptReportMsg(pTask, &pTask->chkInfo, dropRelHTask); code = streamSendChkptReportMsg(pTask, &pTask->chkInfo, dropRelHTask);
} }
} else { // clear the checkpoint info if failed } else { // clear the checkpoint info if failed
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
streamTaskSetFailedCheckpointId(pTask); // set failed checkpoint id before clear the checkpoint info streamTaskSetFailedCheckpointId(pTask); // set failed checkpoint id before clear the checkpoint info
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE); code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE);
stDebug("s-task:%s clear checkpoint flag since gen checkpoint failed, checkpointId:%" PRId64, id, ckId); stDebug("s-task:%s clear checkpoint flag since gen checkpoint failed, checkpointId:%" PRId64, id, ckId);
@ -763,20 +770,20 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
} }
if (++pActiveInfo->checkCounter < 100) { if (++pActiveInfo->checkCounter < 100) {
taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr); streamTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr, vgId, "trigger-recv-monitor");
return; return;
} }
pActiveInfo->checkCounter = 0; pActiveInfo->checkCounter = 0;
stDebug("s-task:%s vgId:%d checkpoint-trigger monitor in tmr, ts:%" PRId64, id, vgId, now); stDebug("s-task:%s vgId:%d checkpoint-trigger monitor in tmr, ts:%" PRId64, id, vgId, now);
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state != TASK_STATUS__CK) { if (pState.state != TASK_STATUS__CK) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
return; return;
} }
@ -787,14 +794,14 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
stDebug("s-task:%s vgId:%d all checkpoint-trigger recv, quit from monitor checkpoint-trigger, ref:%d", id, vgId, stDebug("s-task:%s vgId:%d all checkpoint-trigger recv, quit from monitor checkpoint-trigger, ref:%d", id, vgId,
ref); ref);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
return; return;
} }
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
(void) taosThreadMutexLock(&pActiveInfo->lock); streamMutexLock(&pActiveInfo->lock);
// send msg to retrieve checkpoint trigger msg // send msg to retrieve checkpoint trigger msg
SArray* pList = pTask->upstreamInfo.pList; SArray* pList = pTask->upstreamInfo.pList;
@ -820,19 +827,19 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
if (!recved) { // make sure the inputQ is opened for not recv upstream checkpoint-trigger message if (!recved) { // make sure the inputQ is opened for not recv upstream checkpoint-trigger message
streamTaskOpenUpstreamInput(pTask, pInfo->taskId); streamTaskOpenUpstreamInput(pTask, pInfo->taskId);
taosArrayPush(pNotSendList, pInfo); (void)taosArrayPush(pNotSendList, pInfo);
} }
} }
// do send retrieve checkpoint trigger msg to upstream // do send retrieve checkpoint trigger msg to upstream
int32_t size = taosArrayGetSize(pNotSendList); int32_t size = taosArrayGetSize(pNotSendList);
doSendRetrieveTriggerMsg(pTask, pNotSendList); (void)doSendRetrieveTriggerMsg(pTask, pNotSendList);
(void) taosThreadMutexUnlock(&pActiveInfo->lock); streamMutexUnlock(&pActiveInfo->lock);
// check every 100ms // check every 100ms
if (size > 0) { if (size > 0) {
stDebug("s-task:%s start to monitor checkpoint-trigger in 10s", id); stDebug("s-task:%s start to monitor checkpoint-trigger in 10s", id);
taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr); streamTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr, vgId, "trigger-recv-monitor");
} else { } else {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s all checkpoint-trigger recved, quit from monitor checkpoint-trigger tmr, ref:%d", id, ref); stDebug("s-task:%s all checkpoint-trigger recved, quit from monitor checkpoint-trigger tmr, ref:%d", id, ref);
@ -863,7 +870,7 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) {
SRetrieveChkptTriggerReq* pReq = rpcMallocCont(sizeof(SRetrieveChkptTriggerReq)); SRetrieveChkptTriggerReq* pReq = rpcMallocCont(sizeof(SRetrieveChkptTriggerReq));
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
stError("vgId:%d failed to create msg to retrieve trigger msg for task:%s exec, code:out of memory", vgId, pId); stError("vgId:%d failed to create msg to retrieve trigger msg for task:%s exec, code:out of memory", vgId, pId);
continue; continue;
} }
@ -880,11 +887,16 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) {
initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE_TRIGGER, pReq, sizeof(SRetrieveChkptTriggerReq)); initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE_TRIGGER, pReq, sizeof(SRetrieveChkptTriggerReq));
code = tmsgSendReq(&pUpstreamTask->epSet, &rpcMsg); code = tmsgSendReq(&pUpstreamTask->epSet, &rpcMsg);
stDebug("s-task:%s vgId:%d send checkpoint-trigger retrieve msg to 0x%x(vgId:%d) checkpointId:%" PRId64, pId, vgId, if (code == TSDB_CODE_SUCCESS) {
pUpstreamTask->taskId, pUpstreamTask->nodeId, checkpointId); stDebug("s-task:%s vgId:%d send checkpoint-trigger retrieve msg to 0x%x(vgId:%d) checkpointId:%" PRId64, pId,
vgId, pUpstreamTask->taskId, pUpstreamTask->nodeId, checkpointId);
} else {
stError("s-task:%s vgId:%d failed to send checkpoint-trigger retrieve msg to 0x%x(vgId:%d) checkpointId:%" PRId64,
pId, vgId, pUpstreamTask->taskId, pUpstreamTask->nodeId, checkpointId);
}
} }
return TSDB_CODE_SUCCESS; return code;
} }
bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) { bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) {
@ -897,9 +909,9 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId)
return false; return false;
} }
(void) taosThreadMutexLock(&pInfo->lock); streamMutexLock(&pInfo->lock);
if (!pInfo->dispatchTrigger) { if (!pInfo->dispatchTrigger) {
(void) taosThreadMutexUnlock(&pInfo->lock); streamMutexUnlock(&pTask->lock);
return false; return false;
} }
@ -921,7 +933,7 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId)
id, pSendInfo->sendTs, before, pInfo->activeId, pInfo->transId); id, pSendInfo->sendTs, before, pInfo->activeId, pInfo->transId);
} }
(void) taosThreadMutexUnlock(&pInfo->lock); streamMutexUnlock(&pTask->lock);
return true; return true;
} }
@ -945,7 +957,7 @@ void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) {
SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
int64_t now = taosGetTimestampMs(); int64_t now = taosGetTimestampMs();
(void) taosThreadMutexLock(&pInfo->lock); streamMutexLock(&pInfo->lock);
// outputQ should be empty here // outputQ should be empty here
ASSERT(streamQueueGetNumOfUnAccessedItems(pTask->outputq.queue) == 0); ASSERT(streamQueueGetNumOfUnAccessedItems(pTask->outputq.queue) == 0);
@ -971,21 +983,21 @@ void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) {
} }
} }
(void) taosThreadMutexUnlock(&pInfo->lock); streamMutexUnlock(&pInfo->lock);
} }
int32_t streamTaskGetNumOfConfirmed(SStreamTask* pTask) { int32_t streamTaskGetNumOfConfirmed(SStreamTask* pTask) {
SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
int32_t num = 0; int32_t num = 0;
(void) taosThreadMutexLock(&pInfo->lock); streamMutexLock(&pInfo->lock);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) {
STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i);
if (p->recved) { if (p->recved) {
num++; num++;
} }
} }
(void) taosThreadMutexUnlock(&pInfo->lock); streamMutexUnlock(&pInfo->lock);
return num; return num;
} }
@ -993,7 +1005,7 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) {
SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
int32_t taskId = 0; int32_t taskId = 0;
(void) taosThreadMutexLock(&pInfo->lock); streamMutexLock(&pInfo->lock);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) {
STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i);
@ -1007,7 +1019,7 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) {
} }
} }
(void) taosThreadMutexUnlock(&pInfo->lock); streamMutexUnlock(&pInfo->lock);
int32_t numOfConfirmed = streamTaskGetNumOfConfirmed(pTask); int32_t numOfConfirmed = streamTaskGetNumOfConfirmed(pTask);
int32_t total = streamTaskGetNumOfDownstream(pTask); int32_t total = streamTaskGetNumOfDownstream(pTask);
@ -1184,16 +1196,16 @@ int32_t deleteCheckpointFile(const char* id, const char* name) {
int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask) { int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask) {
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pTask->status.sendConsensusChkptId == true) { if (pTask->status.sendConsensusChkptId == true) {
stDebug("s-task:%s already start to consensus-checkpointId, not start again before it completed", id); stDebug("s-task:%s already start to consensus-checkpointId, not start again before it completed", id);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else { } else {
pTask->status.sendConsensusChkptId = true; pTask->status.sendConsensusChkptId = true;
} }
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
ASSERT(pTask->pBackend == NULL); ASSERT(pTask->pBackend == NULL);
pTask->status.requireConsensusChkptId = true; pTask->status.requireConsensusChkptId = true;
@ -1207,12 +1219,12 @@ int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask) {
return code; return code;
} }
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState p = streamTaskGetStatus(pTask); SStreamTaskState p = streamTaskGetStatus(pTask);
if (p.state == TASK_STATUS__CK) { if (p.state == TASK_STATUS__CK) {
code = streamTaskSendCheckpointSourceRsp(pTask); code = streamTaskSendCheckpointSourceRsp(pTask);
} }
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return code; return code;
} }

View File

@ -49,7 +49,7 @@ int32_t createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t
pInput = p; pInput = p;
} }
blockDecode(pDataBlock, pInput); (void) blockDecode(pDataBlock, pInput);
if (pRetrieve->compressed && compLen < fullLen) { if (pRetrieve->compressed && compLen < fullLen) {
taosMemoryFree(pInput); taosMemoryFree(pInput);
@ -116,11 +116,11 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock
return terrno; return terrno;
} }
taosArrayPush(pArray, &(SSDataBlock){0}); (void) taosArrayPush(pArray, &(SSDataBlock){0});
SRetrieveTableRsp* pRetrieve = pReq->pRetrieve; SRetrieveTableRsp* pRetrieve = pReq->pRetrieve;
SSDataBlock* pDataBlock = taosArrayGet(pArray, 0); SSDataBlock* pDataBlock = taosArrayGet(pArray, 0);
blockDecode(pDataBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN); (void) blockDecode(pDataBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN);
// TODO: refactor // TODO: refactor
pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.skey = be64toh(pRetrieve->skey);
@ -156,28 +156,31 @@ void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit) {
taosFreeQitem(pDataSubmit); taosFreeQitem(pDataSubmit);
} }
SStreamMergedSubmit* streamMergedSubmitNew() { int32_t streamMergedSubmitNew(SStreamMergedSubmit** pSubmit) {
SStreamMergedSubmit* pMerged; *pSubmit = NULL;
int32_t code = taosAllocateQitem(sizeof(SStreamMergedSubmit), DEF_QITEM, 0, (void**)&pMerged); int32_t code = taosAllocateQitem(sizeof(SStreamMergedSubmit), DEF_QITEM, 0, (void**)pSubmit);
if (code) { if (code) {
terrno = code; return TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
pMerged->submits = taosArrayInit(0, sizeof(SPackedData)); (*pSubmit)->submits = taosArrayInit(0, sizeof(SPackedData));
if (pMerged->submits == NULL) { if ((*pSubmit)->submits == NULL) {
taosArrayDestroy(pMerged->submits); taosFreeQitem(*pSubmit);
taosFreeQitem(pMerged); *pSubmit = NULL;
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
pMerged->type = STREAM_INPUT__MERGED_SUBMIT; (*pSubmit)->type = STREAM_INPUT__MERGED_SUBMIT;
return pMerged; return TSDB_CODE_SUCCESS;
} }
int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubmit) { int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubmit) {
taosArrayPush(pMerged->submits, &pSubmit->submit); void* p = taosArrayPush(pMerged->submits, &pSubmit->submit);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (pSubmit->ver > pMerged->ver) { if (pSubmit->ver > pMerged->ver) {
pMerged->ver = pSubmit->ver; pMerged->ver = pSubmit->ver;
} }
@ -187,11 +190,12 @@ int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubm
// todo handle memory error // todo handle memory error
int32_t streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem, SStreamQueueItem** pRes) { int32_t streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem, SStreamQueueItem** pRes) {
*pRes = NULL; *pRes = NULL;
int32_t code = 0;
if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) { if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) {
SStreamDataBlock* pBlock = (SStreamDataBlock*)dst; SStreamDataBlock* pBlock = (SStreamDataBlock*)dst;
SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)pElem; SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)pElem;
taosArrayAddAll(pBlock->blocks, pBlockSrc->blocks); (void) taosArrayAddAll(pBlock->blocks, pBlockSrc->blocks);
taosArrayDestroy(pBlockSrc->blocks); taosArrayDestroy(pBlockSrc->blocks);
streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem)); streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem));
@ -201,33 +205,38 @@ int32_t streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem
} else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) {
SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)dst; SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)dst;
SStreamDataSubmit* pBlockSrc = (SStreamDataSubmit*)pElem; SStreamDataSubmit* pBlockSrc = (SStreamDataSubmit*)pElem;
streamMergeSubmit(pMerged, pBlockSrc);
code = streamMergeSubmit(pMerged, pBlockSrc);
streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem)); streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem));
taosFreeQitem(pElem); taosFreeQitem(pElem);
*pRes = dst; *pRes = dst;
*pRes = dst; *pRes = dst;
return TSDB_CODE_SUCCESS; return code;
} else if (dst->type == STREAM_INPUT__DATA_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) {
SStreamMergedSubmit* pMerged = streamMergedSubmitNew(); SStreamMergedSubmit* pMerged = NULL;
if (pMerged == NULL) { code = streamMergedSubmitNew(&pMerged);
return TSDB_CODE_OUT_OF_MEMORY; if (code != 0) {
return code;
} }
streamQueueItemIncSize((SStreamQueueItem*)pMerged, streamQueueItemGetSize(pElem)); streamQueueItemIncSize((SStreamQueueItem*)pMerged, streamQueueItemGetSize(pElem));
streamMergeSubmit(pMerged, (SStreamDataSubmit*)dst); code = streamMergeSubmit(pMerged, (SStreamDataSubmit*)dst);
streamMergeSubmit(pMerged, (SStreamDataSubmit*)pElem); if (code == 0) {
code = streamMergeSubmit(pMerged, (SStreamDataSubmit*)pElem);
}
taosFreeQitem(dst); taosFreeQitem(dst);
taosFreeQitem(pElem); taosFreeQitem(pElem);
*pRes = (SStreamQueueItem*)pMerged; *pRes = (SStreamQueueItem*)pMerged;
return TSDB_CODE_SUCCESS; return code;
} else { } else {
code = TSDB_CODE_FAILED;
stDebug("block type:%s not merged with existed blocks list, type:%d", streamQueueItemGetTypeStr(pElem->type), stDebug("block type:%s not merged with existed blocks list, type:%d", streamQueueItemGetTypeStr(pElem->type),
dst->type); dst->type);
return TSDB_CODE_FAILED; return code;
} }
} }

View File

@ -121,7 +121,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, abuf, len); tEncoderInit(&encoder, abuf, len);
tEncodeStreamRetrieveReq(&encoder, req); (void) tEncodeStreamRetrieveReq(&encoder, req);
tEncoderClear(&encoder); tEncoderClear(&encoder);
SRpcMsg rpcMsg = {0}; SRpcMsg rpcMsg = {0};
@ -189,6 +189,7 @@ int32_t streamBroadcastToUpTasks(SStreamTask* pTask, const SSDataBlock* pBlock)
return code; return code;
} }
// no need to do anything if failed
int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet) { int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet) {
void* buf = NULL; void* buf = NULL;
int32_t code = -1; int32_t code = -1;
@ -202,7 +203,7 @@ int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq,
buf = rpcMallocCont(sizeof(SMsgHead) + tlen); buf = rpcMallocCont(sizeof(SMsgHead) + tlen);
if (buf == NULL) { if (buf == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
((SMsgHead*)buf)->vgId = htonl(nodeId); ((SMsgHead*)buf)->vgId = htonl(nodeId);
@ -220,8 +221,7 @@ int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq,
stDebug("s-task:%s (level:%d) send check msg to s-task:0x%" PRIx64 ":0x%x (vgId:%d)", pTask->id.idStr, stDebug("s-task:%s (level:%d) send check msg to s-task:0x%" PRIx64 ":0x%x (vgId:%d)", pTask->id.idStr,
pTask->info.taskLevel, pReq->streamId, pReq->downstreamTaskId, nodeId); pTask->info.taskLevel, pReq->streamId, pReq->downstreamTaskId, nodeId);
tmsgSendReq(pEpSet, &msg); return tmsgSendReq(pEpSet, &msg);
return 0;
} }
void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups) { void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups) {
@ -239,7 +239,7 @@ void clearBufferedDispatchMsg(SStreamTask* pTask) {
destroyDispatchMsg(pMsgInfo->pData, streamTaskGetNumOfDownstream(pTask)); destroyDispatchMsg(pMsgInfo->pData, streamTaskGetNumOfDownstream(pTask));
} }
taosThreadMutexLock(&pMsgInfo->lock); streamMutexLock(&pMsgInfo->lock);
pMsgInfo->checkpointId = -1; pMsgInfo->checkpointId = -1;
pMsgInfo->transId = -1; pMsgInfo->transId = -1;
@ -249,7 +249,7 @@ void clearBufferedDispatchMsg(SStreamTask* pTask) {
clearDispatchInfo(pMsgInfo); clearDispatchInfo(pMsgInfo);
taosArrayClear(pTask->msgInfo.pSendInfo); taosArrayClear(pTask->msgInfo.pSendInfo);
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
} }
static SStreamDispatchReq* createDispatchDataReq(SStreamTask* pTask, const SStreamDataBlock* pData) { static SStreamDispatchReq* createDispatchDataReq(SStreamTask* pTask, const SStreamDataBlock* pData) {
@ -424,9 +424,9 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch
} }
static void setNotInDispatchMonitor(SDispatchMsgInfo* pMsgInfo) { static void setNotInDispatchMonitor(SDispatchMsgInfo* pMsgInfo) {
taosThreadMutexLock(&pMsgInfo->lock); streamMutexLock(&pMsgInfo->lock);
pMsgInfo->inMonitor = 0; pMsgInfo->inMonitor = 0;
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
} }
static void setResendInfo(SDispatchEntry* pEntry, int64_t now) { static void setResendInfo(SDispatchEntry* pEntry, int64_t now) {
@ -440,13 +440,13 @@ static void addDispatchEntry(SDispatchMsgInfo* pMsgInfo, int32_t nodeId, int64_t
SDispatchEntry entry = {.nodeId = nodeId, .rspTs = -1, .status = 0, .sendTs = now}; SDispatchEntry entry = {.nodeId = nodeId, .rspTs = -1, .status = 0, .sendTs = now};
if (lock) { if (lock) {
taosThreadMutexLock(&pMsgInfo->lock); streamMutexLock(&pMsgInfo->lock);
} }
taosArrayPush(pMsgInfo->pSendInfo, &entry); (void) taosArrayPush(pMsgInfo->pSendInfo, &entry);
if (lock) { if (lock) {
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
} }
} }
@ -496,16 +496,16 @@ static void doMonitorDispatchData(void* param, void* tmrId) {
return; return;
} }
taosThreadMutexLock(&pMsgInfo->lock); streamMutexLock(&pMsgInfo->lock);
if (pTask->outputq.status == TASK_OUTPUT_STATUS__NORMAL) { if (pTask->outputq.status == TASK_OUTPUT_STATUS__NORMAL) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s not in dispatch procedure, abort from timer, ref:%d", pTask->id.idStr, ref); stDebug("s-task:%s not in dispatch procedure, abort from timer, ref:%d", pTask->id.idStr, ref);
pTask->msgInfo.inMonitor = 0; pTask->msgInfo.inMonitor = 0;
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
return; return;
} }
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
int32_t numOfFailed = getFailedDispatchInfo(pMsgInfo, now); int32_t numOfFailed = getFailedDispatchInfo(pMsgInfo, now);
if (numOfFailed == 0) { if (numOfFailed == 0) {
@ -577,8 +577,9 @@ static void doMonitorDispatchData(void* param, void* tmrId) {
} }
void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) { void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) {
int32_t vgId = pTask->pMeta->vgId;
if (pTask->msgInfo.pRetryTmr != NULL) { if (pTask->msgInfo.pRetryTmr != NULL) {
taosTmrReset(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr); streamTmrReset(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr, vgId, "dispatch-monitor-tmr");
} else { } else {
pTask->msgInfo.pRetryTmr = taosTmrStart(doMonitorDispatchData, waitDuration, pTask, streamTimer); pTask->msgInfo.pRetryTmr = taosTmrStart(doMonitorDispatchData, waitDuration, pTask, streamTimer);
} }
@ -612,7 +613,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
} }
} }
} else { } else {
buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName); (void) buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName);
} }
snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->outputInfo.shuffleDispatcher.dbInfo.db, snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->outputInfo.shuffleDispatcher.dbInfo.db,
@ -624,8 +625,10 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
SBlockName bln = {0}; SBlockName bln = {0};
bln.hashValue = hashValue; bln.hashValue = hashValue;
memcpy(bln.parTbName, pDataBlock->info.parTbName, strlen(pDataBlock->info.parTbName)); memcpy(bln.parTbName, pDataBlock->info.parTbName, strlen(pDataBlock->info.parTbName));
// failed to put into name buffer, no need to do anything
if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) { if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) {
tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName)); (void) tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName));
} }
} }
@ -633,14 +636,14 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
int32_t numOfVgroups = taosArrayGetSize(vgInfo); int32_t numOfVgroups = taosArrayGetSize(vgInfo);
// TODO: optimize search // TODO: optimize search
taosThreadMutexLock(&pTask->msgInfo.lock); streamMutexLock(&pTask->msgInfo.lock);
for (int32_t j = 0; j < numOfVgroups; j++) { for (int32_t j = 0; j < numOfVgroups; j++) {
SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j);
if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) {
if (streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]) < 0) { if (streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]) < 0) {
taosThreadMutexUnlock(&pTask->msgInfo.lock); streamMutexUnlock(&pTask->msgInfo.lock);
return -1; return -1;
} }
@ -655,7 +658,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
} }
} }
taosThreadMutexUnlock(&pTask->msgInfo.lock); streamMutexUnlock(&pTask->msgInfo.lock);
ASSERT(found); ASSERT(found);
return 0; return 0;
} }
@ -690,7 +693,8 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
ASSERT(pTask->msgInfo.pData == NULL); ASSERT(pTask->msgInfo.pData == NULL);
stDebug("s-task:%s start to dispatch msg, set output status:%d", id, pTask->outputq.status); stDebug("s-task:%s start to dispatch msg, set output status:%d", id, pTask->outputq.status);
SStreamDataBlock* pBlock = streamQueueNextItem(pTask->outputq.queue); SStreamDataBlock* pBlock = NULL;
streamQueueNextItem(pTask->outputq.queue, (SStreamQueueItem**)&pBlock);
if (pBlock == NULL) { if (pBlock == NULL) {
atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL); atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL);
stDebug("s-task:%s not dispatch since no elems in outputQ, output status:%d", id, pTask->outputq.status); stDebug("s-task:%s not dispatch since no elems in outputQ, output status:%d", id, pTask->outputq.status);
@ -703,9 +707,9 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
pTask->execInfo.dispatch += 1; pTask->execInfo.dispatch += 1;
taosThreadMutexLock(&pTask->msgInfo.lock); streamMutexLock(&pTask->msgInfo.lock);
initDispatchInfo(&pTask->msgInfo, pTask->execInfo.dispatch); initDispatchInfo(&pTask->msgInfo, pTask->execInfo.dispatch);
taosThreadMutexUnlock(&pTask->msgInfo.lock); streamMutexUnlock(&pTask->msgInfo.lock);
int32_t code = doBuildDispatchMsg(pTask, pBlock); int32_t code = doBuildDispatchMsg(pTask, pBlock);
if (code == 0) { if (code == 0) {
@ -719,7 +723,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
code = sendDispatchMsg(pTask, pTask->msgInfo.pData); code = sendDispatchMsg(pTask, pTask->msgInfo.pData);
taosThreadMutexLock(&pTask->msgInfo.lock); streamMutexLock(&pTask->msgInfo.lock);
if (pTask->msgInfo.inMonitor == 0) { if (pTask->msgInfo.inMonitor == 0) {
int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s start dispatch monitor tmr in %dms, ref:%d, dispatch code:%s", id, DISPATCH_RETRY_INTERVAL_MS, stDebug("s-task:%s start dispatch monitor tmr in %dms, ref:%d, dispatch code:%s", id, DISPATCH_RETRY_INTERVAL_MS,
@ -730,7 +734,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
stDebug("s-task:%s already in dispatch monitor tmr", id); stDebug("s-task:%s already in dispatch monitor tmr", id);
} }
taosThreadMutexUnlock(&pTask->msgInfo.lock); streamMutexUnlock(&pTask->msgInfo.lock);
// this block can not be deleted until it has been sent to downstream task successfully. // this block can not be deleted until it has been sent to downstream task successfully.
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -758,7 +762,7 @@ int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32
buf = rpcMallocCont(sizeof(SMsgHead) + tlen); buf = rpcMallocCont(sizeof(SMsgHead) + tlen);
if (buf == NULL) { if (buf == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId); ((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId);
@ -791,33 +795,33 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
if (++pActiveInfo->sendReadyCheckCounter < 100) { if (++pActiveInfo->sendReadyCheckCounter < 100) {
taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr); streamTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr, vgId, "chkpt-ready-monitor");
return; return;
} }
pActiveInfo->sendReadyCheckCounter = 0; pActiveInfo->sendReadyCheckCounter = 0;
stDebug("s-task:%s in sending checkpoint-ready msg monitor timer", id); stDebug("s-task:%s in sending checkpoint-ready msg monitor timer", id);
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state != TASK_STATUS__CK) { if (pState.state != TASK_STATUS__CK) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s vgId:%d status:%s not in checkpoint, quit from monitor checkpoint-ready send, ref:%d", id, vgId, stDebug("s-task:%s vgId:%d status:%s not in checkpoint, quit from monitor checkpoint-ready send, ref:%d", id, vgId,
pState.name, ref); pState.name, ref);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
return; return;
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
taosThreadMutexLock(&pActiveInfo->lock); streamMutexLock(&pActiveInfo->lock);
SArray* pList = pActiveInfo->pReadyMsgList; SArray* pList = pActiveInfo->pReadyMsgList;
int32_t num = taosArrayGetSize(pList); int32_t num = taosArrayGetSize(pList);
// active checkpoint info is cleared for now // active checkpoint info is cleared for now
if ((pActiveInfo->activeId == 0) && (pActiveInfo->transId == 0) && (num == 0) && (pTask->chkInfo.startTs == 0)) { if ((pActiveInfo->activeId == 0) && (pActiveInfo->transId == 0) && (num == 0) && (pTask->chkInfo.startTs == 0)) {
taosThreadMutexUnlock(&pActiveInfo->lock); streamMutexUnlock(&pActiveInfo->lock);
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stWarn("s-task:%s vgId:%d active checkpoint may be cleared, quit from readyMsg send tmr, ref:%d", id, vgId, ref); stWarn("s-task:%s vgId:%d active checkpoint may be cleared, quit from readyMsg send tmr, ref:%d", id, vgId, ref);
@ -835,7 +839,7 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
continue; continue;
} }
taosArrayPush(pNotRspList, &pInfo->upstreamTaskId); (void) taosArrayPush(pNotRspList, &pInfo->upstreamTaskId);
stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId,
pTask->info.taskLevel, pInfo->upstreamTaskId); pTask->info.taskLevel, pInfo->upstreamTaskId);
} }
@ -852,17 +856,25 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
if (taskId == pReadyInfo->upstreamTaskId) { // send msg again if (taskId == pReadyInfo->upstreamTaskId) { // send msg again
SRpcMsg msg = {0}; SRpcMsg msg = {0};
initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, pReadyInfo->childId, int32_t code = initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, pReadyInfo->childId,
checkpointId, &msg); checkpointId, &msg);
tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg); if (code == TSDB_CODE_SUCCESS) {
code = tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg);
if (code == TSDB_CODE_SUCCESS) {
stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x again", id, pTask->info.taskLevel, stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x again", id, pTask->info.taskLevel,
pReadyInfo->upstreamTaskId); pReadyInfo->upstreamTaskId);
} else {
stError("s-task:%s failed to send checkpoint-ready msg, try nex time in 10s", id);
}
} else {
stError("s-task:%s failed to prepare the checkpoint-ready msg, try nex time in 10s", id);
}
} }
} }
} }
taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr); streamTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr, vgId, "chkpt-ready-monitor");
taosThreadMutexUnlock(&pActiveInfo->lock); streamMutexUnlock(&pActiveInfo->lock);
} else { } else {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug( stDebug(
@ -871,7 +883,7 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
id, vgId, ref); id, vgId, ref);
streamClearChkptReadyMsg(pActiveInfo); streamClearChkptReadyMsg(pActiveInfo);
taosThreadMutexUnlock(&pActiveInfo->lock); streamMutexUnlock(&pActiveInfo->lock);
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
} }
@ -883,9 +895,10 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int32_t vgId = pTask->pMeta->vgId;
SArray* pList = pActiveInfo->pReadyMsgList; SArray* pList = pActiveInfo->pReadyMsgList;
taosThreadMutexLock(&pActiveInfo->lock); streamMutexLock(&pActiveInfo->lock);
int32_t num = taosArrayGetSize(pList); int32_t num = taosArrayGetSize(pList);
ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num); ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num);
@ -894,15 +907,22 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i); STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i);
SRpcMsg msg = {0}; SRpcMsg msg = {0};
initCheckpointReadyMsg(pTask, pInfo->upstreamNodeId, pInfo->upstreamTaskId, pInfo->childId, pInfo->checkpointId, int32_t code = initCheckpointReadyMsg(pTask, pInfo->upstreamNodeId, pInfo->upstreamTaskId, pInfo->childId, pInfo->checkpointId,
&msg); &msg);
tmsgSendReq(&pInfo->upstreamNodeEpset, &msg); if (code == TSDB_CODE_SUCCESS) {
code = tmsgSendReq(&pInfo->upstreamNodeEpset, &msg);
if (code == TSDB_CODE_SUCCESS) {
stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x", id, pTask->info.taskLevel, stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x", id, pTask->info.taskLevel,
pInfo->upstreamTaskId); pInfo->upstreamTaskId);
} else {
stError("s-task:%s failed to send checkpoint-ready msg, try nex time in 10s", id);
}
} else {
stError("s-task:%s failed to prepare the checkpoint-ready msg, try nex time in 10s", id);
}
} }
taosThreadMutexUnlock(&pActiveInfo->lock); streamMutexUnlock(&pActiveInfo->lock);
stDebug("s-task:%s level:%d checkpoint-ready msg sent to all %d upstreams", id, pTask->info.taskLevel, num); stDebug("s-task:%s level:%d checkpoint-ready msg sent to all %d upstreams", id, pTask->info.taskLevel, num);
// start to check if checkpoint ready msg has successfully received by upstream tasks. // start to check if checkpoint ready msg has successfully received by upstream tasks.
@ -914,7 +934,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
if (pActiveInfo->pSendReadyMsgTmr == NULL) { if (pActiveInfo->pSendReadyMsgTmr == NULL) {
pActiveInfo->pSendReadyMsgTmr = taosTmrStart(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer); pActiveInfo->pSendReadyMsgTmr = taosTmrStart(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer);
} else { } else {
taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr); streamTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr, vgId, "chkpt-ready-monitor");
} }
} }
@ -925,7 +945,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) {
SArray* pList = pTask->chkInfo.pActiveInfo->pReadyMsgList; SArray* pList = pTask->chkInfo.pActiveInfo->pReadyMsgList;
taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock); streamMutexLock(&pTask->chkInfo.pActiveInfo->lock);
ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE);
if (taosArrayGetSize(pList) == 1) { if (taosArrayGetSize(pList) == 1) {
@ -940,7 +960,7 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) {
pTask->info.taskLevel); pTask->info.taskLevel);
} }
taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock); streamMutexUnlock(&pTask->chkInfo.pActiveInfo->lock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -978,8 +998,8 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch
payloadLen += sizeof(SRetrieveTableRsp); payloadLen += sizeof(SRetrieveTableRsp);
taosArrayPush(pReq->dataLen, &payloadLen); (void) taosArrayPush(pReq->dataLen, &payloadLen);
taosArrayPush(pReq->data, &buf); (void) taosArrayPush(pReq->data, &buf);
pReq->totalLen += dataStrLen; pReq->totalLen += dataStrLen;
return 0; return 0;
@ -1056,7 +1076,7 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp
void* abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead));
tEncoderInit(&encoder, (uint8_t*)abuf, len); tEncoderInit(&encoder, (uint8_t*)abuf, len);
tEncodeStreamCheckpointSourceRsp(&encoder, &rsp); (void) tEncodeStreamCheckpointSourceRsp(&encoder, &rsp);
tEncoderClear(&encoder); tEncoderClear(&encoder);
initRpcMsg(pMsg, 0, pBuf, sizeof(SMsgHead) + len); initRpcMsg(pMsg, 0, pBuf, sizeof(SMsgHead) + len);
@ -1071,10 +1091,11 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
STaskCheckpointReadyInfo info = { STaskCheckpointReadyInfo info = {
.recvTs = taosGetTimestampMs(), .transId = pReq->transId, .checkpointId = pReq->checkpointId}; .recvTs = taosGetTimestampMs(), .transId = pReq->transId, .checkpointId = pReq->checkpointId};
streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, TSDB_CODE_SUCCESS); // todo retry until it success
(void) streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, TSDB_CODE_SUCCESS);
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
taosThreadMutexLock(&pActiveInfo->lock); streamMutexLock(&pActiveInfo->lock);
int32_t size = taosArrayGetSize(pActiveInfo->pReadyMsgList); int32_t size = taosArrayGetSize(pActiveInfo->pReadyMsgList);
if (size > 0) { if (size > 0) {
@ -1091,15 +1112,15 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
ASSERT(0); // failed to handle it ASSERT(0); // failed to handle it
} }
} else { } else {
taosArrayPush(pActiveInfo->pReadyMsgList, &info); (void) taosArrayPush(pActiveInfo->pReadyMsgList, &info);
stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size + 1); stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size + 1);
} }
taosThreadMutexUnlock(&pActiveInfo->lock); streamMutexUnlock(&pActiveInfo->lock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId, void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId,
int32_t childId, SEpSet* pEpset, int64_t checkpointId) { int32_t childId, SEpSet* pEpset, int64_t checkpointId) {
ASSERT(upstreamTaskId != 0); ASSERT(upstreamTaskId != 0);
@ -1109,8 +1130,6 @@ int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t up
pReadyInfo->recvTs = taosGetTimestampMs(); pReadyInfo->recvTs = taosGetTimestampMs();
pReadyInfo->checkpointId = checkpointId; pReadyInfo->checkpointId = checkpointId;
pReadyInfo->childId = childId; pReadyInfo->childId = childId;
return TSDB_CODE_SUCCESS;
} }
int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, int32_t index, int64_t checkpointId) { int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, int32_t index, int64_t checkpointId) {
@ -1119,9 +1138,9 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
} }
SStreamUpstreamEpInfo* pInfo = NULL; SStreamUpstreamEpInfo* pInfo = NULL;
int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo); streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) { if (pInfo == NULL) {
return code; return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
STaskCheckpointReadyInfo info = {0}; STaskCheckpointReadyInfo info = {0};
@ -1132,8 +1151,8 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
taosThreadMutexLock(&pActiveInfo->lock); streamMutexLock(&pActiveInfo->lock);
taosArrayPush(pActiveInfo->pReadyMsgList, &info); (void) taosArrayPush(pActiveInfo->pReadyMsgList, &info);
int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList); int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList);
int32_t total = streamTaskGetNumOfUpstream(pTask); int32_t total = streamTaskGetNumOfUpstream(pTask);
@ -1145,7 +1164,7 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
stDebug("s-task:%s %d/%d checkpoint-trigger recv", pTask->id.idStr, numOfRecv, total); stDebug("s-task:%s %d/%d checkpoint-trigger recv", pTask->id.idStr, numOfRecv, total);
} }
taosThreadMutexUnlock(&pActiveInfo->lock); streamMutexUnlock(&pActiveInfo->lock);
return 0; return 0;
} }
@ -1187,10 +1206,8 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId
if (delayDispatch) { if (delayDispatch) {
return 0; return 0;
} else { } else {
streamDispatchStreamBlock(pTask); return streamDispatchStreamBlock(pTask);
} }
return 0;
} }
static bool setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t code, int64_t now, int32_t* pNotRsp, const char* id) { static bool setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t code, int64_t now, int32_t* pNotRsp, const char* id) {
@ -1200,7 +1217,7 @@ static bool setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t
bool allRsp = false; bool allRsp = false;
*pNotRsp = 0; *pNotRsp = 0;
taosThreadMutexLock(&pMsgInfo->lock); streamMutexLock(&pMsgInfo->lock);
int32_t numOfDispatchBranch = taosArrayGetSize(pMsgInfo->pSendInfo); int32_t numOfDispatchBranch = taosArrayGetSize(pMsgInfo->pSendInfo);
for(int32_t i = 0; i < numOfDispatchBranch; ++i) { for(int32_t i = 0; i < numOfDispatchBranch; ++i) {
@ -1228,7 +1245,7 @@ static bool setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t
*pNotRsp = numOfDispatchBranch - numOfRsp; *pNotRsp = numOfDispatchBranch - numOfRsp;
allRsp = (numOfRsp == numOfDispatchBranch); allRsp = (numOfRsp == numOfDispatchBranch);
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
ASSERT(updated); ASSERT(updated);
return allRsp; return allRsp;
@ -1240,7 +1257,7 @@ bool isDispatchRspTimeout(SDispatchEntry* pEntry, int64_t now) {
int32_t getFailedDispatchInfo(SDispatchMsgInfo* pMsgInfo, int64_t now) { int32_t getFailedDispatchInfo(SDispatchMsgInfo* pMsgInfo, int64_t now) {
int32_t numOfFailed = 0; int32_t numOfFailed = 0;
taosThreadMutexLock(&pMsgInfo->lock); streamMutexLock(&pMsgInfo->lock);
for (int32_t j = 0; j < taosArrayGetSize(pMsgInfo->pSendInfo); ++j) { for (int32_t j = 0; j < taosArrayGetSize(pMsgInfo->pSendInfo); ++j) {
SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, j); SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, j);
@ -1248,7 +1265,7 @@ int32_t getFailedDispatchInfo(SDispatchMsgInfo* pMsgInfo, int64_t now) {
numOfFailed += 1; numOfFailed += 1;
} }
} }
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
return numOfFailed; return numOfFailed;
} }
@ -1260,9 +1277,9 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
bool allRsp = false; bool allRsp = false;
int32_t notRsp = 0; int32_t notRsp = 0;
taosThreadMutexLock(&pMsgInfo->lock); streamMutexLock(&pMsgInfo->lock);
int32_t msgId = pMsgInfo->msgId; int32_t msgId = pMsgInfo->msgId;
taosThreadMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
// follower not handle the dispatch rsp // follower not handle the dispatch rsp
if ((pTask->pMeta->role == NODE_ROLE_FOLLOWER) || (pTask->status.downstreamReady != 1)) { if ((pTask->pMeta->role == NODE_ROLE_FOLLOWER) || (pTask->status.downstreamReady != 1)) {
@ -1315,7 +1332,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
{ {
bool delayDispatch = (pMsgInfo->dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); bool delayDispatch = (pMsgInfo->dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER);
if (delayDispatch) { if (delayDispatch) {
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
// we only set the dispatch msg info for current checkpoint trans // we only set the dispatch msg info for current checkpoint trans
if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK && if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK &&
pTask->chkInfo.pActiveInfo->activeId == pMsgInfo->checkpointId) { pTask->chkInfo.pActiveInfo->activeId == pMsgInfo->checkpointId) {
@ -1329,7 +1346,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
" transId:%d discard, since expired", " transId:%d discard, since expired",
pTask->id.idStr, pMsgInfo->checkpointId, pMsgInfo->transId); pTask->id.idStr, pMsgInfo->checkpointId, pMsgInfo->transId);
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
} }
} }
} }
@ -1374,12 +1391,12 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
// now ready for next data output // now ready for next data output
atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL); atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL);
} else { } else {
handleDispatchSuccessRsp(pTask, pRsp->downstreamTaskId, pRsp->downstreamNodeId); code = handleDispatchSuccessRsp(pTask, pRsp->downstreamTaskId, pRsp->downstreamNodeId);
} }
} }
} }
return 0; return code;
} }
static int32_t buildDispatchRsp(const SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t status, void** pBuf) { static int32_t buildDispatchRsp(const SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t status, void** pBuf) {
@ -1438,9 +1455,9 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->totalLen, pReq->msgId); pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->totalLen, pReq->msgId);
SStreamUpstreamEpInfo* pInfo = NULL; SStreamUpstreamEpInfo* pInfo = NULL;
int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId, &pInfo); streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) { if (pInfo == NULL) {
return code; return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
if (pMeta->role == NODE_ROLE_FOLLOWER) { if (pMeta->role == NODE_ROLE_FOLLOWER) {
@ -1474,7 +1491,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
{ {
// do send response with the input status // do send response with the input status
code = buildDispatchRsp(pTask, pReq, status, &pRsp->pCont); int32_t code = buildDispatchRsp(pTask, pReq, status, &pRsp->pCont);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed to build dispatch rsp, msgId:%d, code:%s", id, pReq->msgId, tstrerror(code)); stError("s-task:%s failed to build dispatch rsp, msgId:%d, code:%s", id, pReq->msgId, tstrerror(code));
return code; return code;
@ -1484,6 +1501,5 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
tmsgSendRsp(pRsp); tmsgSendRsp(pRsp);
} }
streamTrySchedExec(pTask); return streamTrySchedExec(pTask);
return code;
} }

View File

@ -24,7 +24,7 @@
#define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec #define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec
static int32_t streamTransferStateDoPrepare(SStreamTask* pTask); static int32_t streamTransferStateDoPrepare(SStreamTask* pTask);
static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks); static void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks);
bool streamTaskShouldStop(const SStreamTask* pTask) { bool streamTaskShouldStop(const SStreamTask* pTask) {
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
@ -52,8 +52,9 @@ static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBl
return code; return code;
} }
// not handle error, if dispatch failed, try next time.
// checkpoint trigger will be checked // checkpoint trigger will be checked
streamDispatchStreamBlock(pTask); (void) streamDispatchStreamBlock(pTask);
} }
return code; return code;
@ -90,7 +91,7 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray*
return code; return code;
} }
int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) { void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
void* pExecutor = pTask->exec.pExecutor; void* pExecutor = pTask->exec.pExecutor;
@ -108,7 +109,7 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t*
if (streamTaskShouldStop(pTask)) { if (streamTaskShouldStop(pTask)) {
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
return 0; return;
} }
SSDataBlock* output = NULL; SSDataBlock* output = NULL;
@ -128,11 +129,16 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t*
const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem; const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem;
ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1);
assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); (void) assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0));
block.info.type = STREAM_PULL_OVER; block.info.type = STREAM_PULL_OVER;
block.info.childId = pTask->info.selfChildId; block.info.childId = pTask->info.selfChildId;
taosArrayPush(pRes, &block);
void* p = taosArrayPush(pRes, &block);
if (p != NULL) {
numOfBlocks += 1; numOfBlocks += 1;
} else {
stError("s-task:%s failed to add retrieve block", pTask->id.idStr);
}
stDebug("s-task:%s(child %d) retrieve process completed, reqId:0x%" PRIx64 " dump results", pTask->id.idStr, stDebug("s-task:%s(child %d) retrieve process completed, reqId:0x%" PRIx64 " dump results", pTask->id.idStr,
pTask->info.selfChildId, pRetrieveBlock->reqId); pTask->info.selfChildId, pRetrieveBlock->reqId);
@ -151,13 +157,21 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t*
} }
SSDataBlock block = {0}; SSDataBlock block = {0};
assignOneDataBlock(&block, output); code = assignOneDataBlock(&block, output);
if (code) {
stError("s-task:%s failed to build result block due to out of memory", pTask->id.idStr);
continue;
}
block.info.childId = pTask->info.selfChildId; block.info.childId = pTask->info.selfChildId;
size += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block); size += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block);
numOfBlocks += 1; numOfBlocks += 1;
taosArrayPush(pRes, &block); void* p = taosArrayPush(pRes, &block);
if (p == NULL) {
stError("s-task:%s failed to add computing results, the final res may be incorrect", pTask->id.idStr);
}
stDebug("s-task:%s (child %d) executed and get %d result blocks, size:%.2fMiB", pTask->id.idStr, stDebug("s-task:%s (child %d) executed and get %d result blocks, size:%.2fMiB", pTask->id.idStr,
pTask->info.selfChildId, numOfBlocks, SIZE_IN_MiB(size)); pTask->info.selfChildId, numOfBlocks, SIZE_IN_MiB(size));
@ -166,8 +180,9 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t*
if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || size >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) { if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || size >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) {
ASSERT(numOfBlocks == taosArrayGetSize(pRes)); ASSERT(numOfBlocks == taosArrayGetSize(pRes));
code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks); code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks);
// todo: here we need continue retry to put it into output buffer
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return;
} }
pRes = NULL; pRes = NULL;
@ -182,16 +197,16 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t*
} else { } else {
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
} }
return code;
} }
static int32_t handleSanhistoryResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) { // todo contiuous try to create result blocks
static int32_t handleScanhistoryResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (taosArrayGetSize(pRes) > 0) { if (taosArrayGetSize(pRes) > 0) {
SStreamDataBlock* pStreamBlocks = NULL; SStreamDataBlock* pStreamBlocks = NULL;
code = createStreamBlockFromResults(NULL, pTask, size, pRes, &pStreamBlocks); code = createStreamBlockFromResults(NULL, pTask, size, pRes, &pStreamBlocks);
if (code) { if (code) {
stError("s-task:%s failed to build history result blocks", pTask->id.idStr);
return code; return code;
} }
@ -236,9 +251,13 @@ static void streamScanHistoryDataImpl(SStreamTask* pTask, SArray* pRes, int32_t*
} }
SSDataBlock block = {0}; SSDataBlock block = {0};
assignOneDataBlock(&block, output); (void) assignOneDataBlock(&block, output);
block.info.childId = pTask->info.selfChildId; block.info.childId = pTask->info.selfChildId;
taosArrayPush(pRes, &block);
void* p = taosArrayPush(pRes, &block);
if (p == NULL) {
stError("s-task:%s failed to add computing results, the final res may be incorrect", pTask->id.idStr);
}
(*pSize) += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block); (*pSize) += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block);
numOfBlocks += 1; numOfBlocks += 1;
@ -264,7 +283,7 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) {
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
if (!pTask->hTaskInfo.operatorOpen) { if (!pTask->hTaskInfo.operatorOpen) {
qSetStreamOpOpen(exec); (void) qSetStreamOpOpen(exec);
pTask->hTaskInfo.operatorOpen = true; pTask->hTaskInfo.operatorOpen = true;
} }
@ -301,8 +320,8 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) {
return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0); return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0);
} }
// dispatch the generated results // dispatch the generated results, todo fix error
/*int32_t code = */handleSanhistoryResultBlocks(pTask, pRes, size); (void) handleScanhistoryResultBlocks(pTask, pRes, size);
if (finished) { if (finished) {
return buildScanhistoryExecRet(TASK_SCANHISTORY_CONT, 0); return buildScanhistoryExecRet(TASK_SCANHISTORY_CONT, 0);
@ -323,14 +342,15 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
SStreamTask* pStreamTask = NULL; SStreamTask* pStreamTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId, &pStreamTask); int32_t code = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId, &pStreamTask);
if (pStreamTask == NULL) { if (pStreamTask == NULL || code != TSDB_CODE_SUCCESS) {
stError( stError(
"s-task:%s failed to find related stream task:0x%x, it may have been destroyed or closed, destroy the related " "s-task:%s failed to find related stream task:0x%x, it may have been destroyed or closed, destroy the related "
"fill-history task", "fill-history task",
id, (int32_t) pTask->streamTaskId.taskId); id, (int32_t) pTask->streamTaskId.taskId);
// 1. free it and remove fill-history task from disk meta-store // 1. free it and remove fill-history task from disk meta-store
streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0); // todo: this function should never be failed.
(void) streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0);
// 2. save to disk // 2. save to disk
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
@ -388,14 +408,14 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN, pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN,
pTimeWindow->ekey, p, pStreamTask->status.schedStatus); pTimeWindow->ekey, p, pStreamTask->status.schedStatus);
streamTaskResetTimewindowFilter(pStreamTask); (void) streamTaskResetTimewindowFilter(pStreamTask);
} else { } else {
stDebug("s-task:%s no need to update/reset filter time window for non-source tasks", pStreamTask->id.idStr); stDebug("s-task:%s no need to update/reset filter time window for non-source tasks", pStreamTask->id.idStr);
} }
// NOTE: transfer the ownership of executor state before handle the checkpoint block during stream exec // NOTE: transfer the ownership of executor state before handle the checkpoint block during stream exec
// 2. send msg to mnode to launch a checkpoint to keep the state for current stream // 2. send msg to mnode to launch a checkpoint to keep the state for current stream
streamTaskSendCheckpointReq(pStreamTask); (void) streamTaskSendCheckpointReq(pStreamTask);
// 3. assign the status to the value that will be kept in disk // 3. assign the status to the value that will be kept in disk
pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask).state; pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask).state;
@ -409,7 +429,7 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
static int32_t haltCallback(SStreamTask* pTask, void* param) { static int32_t haltCallback(SStreamTask* pTask, void* param) {
streamTaskOpenAllUpstreamInput(pTask); streamTaskOpenAllUpstreamInput(pTask);
streamTaskSendCheckpointReq(pTask); (void) streamTaskSendCheckpointReq(pTask);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -445,18 +465,19 @@ int32_t streamTransferStatePrepare(SStreamTask* pTask) {
} }
// set input // set input
static void doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_t* pVer, const char* id) { static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_t* pVer, const char* id) {
void* pExecutor = pTask->exec.pExecutor; void* pExecutor = pTask->exec.pExecutor;
int32_t code = 0;
const SStreamQueueItem* pItem = pInput; const SStreamQueueItem* pItem = pInput;
if (pItem->type == STREAM_INPUT__GET_RES) { if (pItem->type == STREAM_INPUT__GET_RES) {
const SStreamTrigger* pTrigger = (const SStreamTrigger*)pInput; const SStreamTrigger* pTrigger = (const SStreamTrigger*)pInput;
qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); code = qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
} else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE);
const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput; const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput;
qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); code = qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT);
stDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, id, pSubmit, stDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, id, pSubmit,
pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver);
ASSERT((*pVer) <= pSubmit->submit.ver); ASSERT((*pVer) <= pSubmit->submit.ver);
@ -468,7 +489,7 @@ static void doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_
SArray* pBlockList = pBlock->blocks; SArray* pBlockList = pBlock->blocks;
int32_t numOfBlocks = taosArrayGetSize(pBlockList); int32_t numOfBlocks = taosArrayGetSize(pBlockList);
stDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, id, numOfBlocks, pBlock->sourceVer); stDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, id, numOfBlocks, pBlock->sourceVer);
qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK); code = qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK);
} else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) {
const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)pInput; const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)pInput;
@ -477,24 +498,26 @@ static void doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_
int32_t numOfBlocks = taosArrayGetSize(pBlockList); int32_t numOfBlocks = taosArrayGetSize(pBlockList);
stDebug("s-task:%s %p set (merged) submit blocks as a batch, numOfBlocks:%d, ver:%" PRId64, id, pTask, numOfBlocks, stDebug("s-task:%s %p set (merged) submit blocks as a batch, numOfBlocks:%d, ver:%" PRId64, id, pTask, numOfBlocks,
pMerged->ver); pMerged->ver);
qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); code = qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT);
ASSERT((*pVer) <= pMerged->ver); ASSERT((*pVer) <= pMerged->ver);
(*pVer) = pMerged->ver; (*pVer) = pMerged->ver;
} else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) {
const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)pInput; const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)pInput;
qSetMultiStreamInput(pExecutor, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK); code = qSetMultiStreamInput(pExecutor, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
} else if (pItem->type == STREAM_INPUT__CHECKPOINT || pItem->type == STREAM_INPUT__CHECKPOINT_TRIGGER) { } else if (pItem->type == STREAM_INPUT__CHECKPOINT || pItem->type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
const SStreamDataBlock* pCheckpoint = (const SStreamDataBlock*)pInput; const SStreamDataBlock* pCheckpoint = (const SStreamDataBlock*)pInput;
qSetMultiStreamInput(pExecutor, pCheckpoint->blocks, 1, pItem->type); code = qSetMultiStreamInput(pExecutor, pCheckpoint->blocks, 1, pItem->type);
} else { } else {
ASSERT(0); ASSERT(0);
} }
return code;
} }
int32_t streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { void streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) {
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t level = pTask->info.taskLevel; int32_t level = pTask->info.taskLevel;
@ -505,7 +528,7 @@ int32_t streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBloc
if (remain > 0) { if (remain > 0) {
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
stDebug("s-task:%s receive upstream trans-state msg, not sent remain:%d", id, remain); stDebug("s-task:%s receive upstream trans-state msg, not sent remain:%d", id, remain);
return 0; return;
} }
} }
@ -526,7 +549,7 @@ int32_t streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBloc
pBlock->srcVgId = pTask->pMeta->vgId; pBlock->srcVgId = pTask->pMeta->vgId;
code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock); code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock);
if (code == 0) { if (code == 0) {
streamDispatchStreamBlock(pTask); (void) streamDispatchStreamBlock(pTask);
} else { // todo put into queue failed, retry } else { // todo put into queue failed, retry
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
} }
@ -540,11 +563,9 @@ int32_t streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBloc
code = streamTransferStatePrepare(pTask); code = streamTransferStatePrepare(pTask);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
/*int8_t status = */ streamTaskSetSchedStatusInactive(pTask); (void)streamTaskSetSchedStatusInactive(pTask);
} }
} }
return code;
} }
//static void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; } //static void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; }
@ -559,7 +580,11 @@ static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, i
stDebug("s-task:%s start to process batch blocks, num:%d, type:%s", id, num, streamQueueItemGetTypeStr(pBlock->type)); stDebug("s-task:%s start to process batch blocks, num:%d, type:%s", id, num, streamQueueItemGetTypeStr(pBlock->type));
doSetStreamInputBlock(pTask, pBlock, &ver, id); int32_t code = doSetStreamInputBlock(pTask, pBlock, &ver, id);
if(code) {
stError("s-task:%s failed to set input block, not exec for these blocks", id);
return;
}
int64_t totalSize = 0; int64_t totalSize = 0;
int32_t totalBlocks = 0; int32_t totalBlocks = 0;
@ -601,9 +626,9 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB
STaskId* pHTaskId = &pTask->hTaskInfo.id; STaskId* pHTaskId = &pTask->hTaskInfo.id;
SStreamTask* pHTask = NULL; SStreamTask* pHTask = NULL;
int32_t code = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask); int32_t code = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask);
if (pHTask != NULL) { if (code == TSDB_CODE_SUCCESS) { // ignore the error code.
streamTaskReleaseState(pHTask); (void) streamTaskReleaseState(pHTask);
streamTaskReloadState(pTask); (void) streamTaskReloadState(pTask);
stDebug("s-task:%s transfer state from fill-history task:%s, status:%s completed", id, pHTask->id.idStr, stDebug("s-task:%s transfer state from fill-history task:%s, status:%s completed", id, pHTask->id.idStr,
streamTaskGetStatus(pHTask).name); streamTaskGetStatus(pHTask).name);
@ -676,7 +701,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
// dispatch checkpoint msg to all downstream tasks // dispatch checkpoint msg to all downstream tasks
int32_t type = pInput->type; int32_t type = pInput->type;
if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) { if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput); (void) streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput);
continue; continue;
} }
@ -694,7 +719,10 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
if (type == STREAM_INPUT__DATA_BLOCK) { if (type == STREAM_INPUT__DATA_BLOCK) {
pTask->execInfo.sink.dataSize += blockSize; pTask->execInfo.sink.dataSize += blockSize;
stDebug("s-task:%s sink task start to sink %d blocks, size:%.2fKiB", id, numOfBlocks, SIZE_IN_KiB(blockSize)); stDebug("s-task:%s sink task start to sink %d blocks, size:%.2fKiB", id, numOfBlocks, SIZE_IN_KiB(blockSize));
doOutputResultBlockImpl(pTask, (SStreamDataBlock*)pInput); int32_t code = doOutputResultBlockImpl(pTask, (SStreamDataBlock*)pInput);
if (code != TSDB_CODE_SUCCESS) {
// todo handle error.
}
double el = (taosGetTimestampMs() - st) / 1000.0; double el = (taosGetTimestampMs() - st) / 1000.0;
if (fabs(el - 0.0) <= DBL_EPSILON) { if (fabs(el - 0.0) <= DBL_EPSILON) {
@ -712,11 +740,11 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
streamFreeQitem(pInput); streamFreeQitem(pInput);
} else { // todo other thread may change the status } else { // todo other thread may change the status
// do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed.
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state == TASK_STATUS__CK) { if (pState.state == TASK_STATUS__CK) {
stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name); stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name);
streamTaskBuildCheckpoint(pTask); (void) streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue
} else { // todo refactor } else { // todo refactor
int32_t code = 0; int32_t code = 0;
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
@ -732,7 +760,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
} }
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamFreeQitem(pInput); streamFreeQitem(pInput);
return 0; return 0;
} }
@ -765,21 +793,21 @@ bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) {
} }
} }
int32_t streamResumeTask(SStreamTask* pTask) { void streamResumeTask(SStreamTask* pTask) {
ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__ACTIVE); ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__ACTIVE);
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
while (1) { while (1) {
/*int32_t code = */ doStreamExecTask(pTask); (void) doStreamExecTask(pTask);
// check if continue // check if continue
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue); int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue);
if ((numOfItems == 0) || streamTaskShouldStop(pTask) || streamTaskShouldPause(pTask)) { if ((numOfItems == 0) || streamTaskShouldStop(pTask) || streamTaskShouldPause(pTask)) {
atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE);
streamTaskClearSchedIdleInfo(pTask); streamTaskClearSchedIdleInfo(pTask);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
setLastExecTs(pTask, taosGetTimestampMs()); setLastExecTs(pTask, taosGetTimestampMs());
@ -787,19 +815,19 @@ int32_t streamResumeTask(SStreamTask* pTask) {
stDebug("s-task:%s exec completed, status:%s, sched-status:%d, lastExecTs:%" PRId64, id, p, stDebug("s-task:%s exec completed, status:%s, sched-status:%d, lastExecTs:%" PRId64, id, p,
pTask->status.schedStatus, pTask->status.lastExecTs); pTask->status.schedStatus, pTask->status.lastExecTs);
return 0; return;
} else { } else {
// check if this task needs to be idle for a while // check if this task needs to be idle for a while
if (pTask->status.schedIdleTime > 0) { if (pTask->status.schedIdleTime > 0) {
streamTaskResumeInFuture(pTask); streamTaskResumeInFuture(pTask);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
setLastExecTs(pTask, taosGetTimestampMs()); setLastExecTs(pTask, taosGetTimestampMs());
return 0; return;
} }
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
} }
} }

View File

@ -54,7 +54,7 @@ static bool existInHbMsg(SStreamHbMsg* pMsg, SDownstreamTaskEpset* pTaskEpset) {
static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) { static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) {
SStreamMeta* pMeta = pTask->pMeta; SStreamMeta* pMeta = pTask->pMeta;
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList); int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList);
for (int j = 0; j < num; ++j) { for (int j = 0; j < num; ++j) {
@ -73,7 +73,7 @@ static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) {
} }
taosArrayClear(pTask->outputInfo.pNodeEpsetUpdateList); taosArrayClear(pTask->outputInfo.pNodeEpsetUpdateList);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
} }
static int32_t doSendHbMsgInfo(SStreamHbMsg* pMsg, SStreamMeta* pMeta, SEpSet* pEpset) { static int32_t doSendHbMsgInfo(SStreamHbMsg* pMsg, SStreamMeta* pMeta, SEpSet* pEpset) {
@ -170,9 +170,9 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) {
continue; continue;
} }
(void) taosThreadMutexLock(&(*pTask)->lock); streamMutexLock(&(*pTask)->lock);
STaskStatusEntry entry = streamTaskGetStatusEntry(*pTask); STaskStatusEntry entry = streamTaskGetStatusEntry(*pTask);
(void) taosThreadMutexUnlock(&(*pTask)->lock); streamMutexUnlock(&(*pTask)->lock);
entry.inputRate = entry.inputQUsed * 100.0 / (2 * STREAM_TASK_QUEUE_CAPACITY_IN_SIZE); entry.inputRate = entry.inputQUsed * 100.0 / (2 * STREAM_TASK_QUEUE_CAPACITY_IN_SIZE);
if ((*pTask)->info.taskLevel == TASK_LEVEL__SINK) { if ((*pTask)->info.taskLevel == TASK_LEVEL__SINK) {
@ -190,9 +190,9 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) {
stInfo("s-task:%s set kill checkpoint trans in hbMsg, transId:%d, clear the active checkpointInfo", stInfo("s-task:%s set kill checkpoint trans in hbMsg, transId:%d, clear the active checkpointInfo",
(*pTask)->id.idStr, p->transId); (*pTask)->id.idStr, p->transId);
(void) taosThreadMutexLock(&(*pTask)->lock); streamMutexLock(&(*pTask)->lock);
streamTaskClearCheckInfo((*pTask), true); streamTaskClearCheckInfo((*pTask), true);
(void) taosThreadMutexUnlock(&(*pTask)->lock); streamMutexUnlock(&(*pTask)->lock);
} }
} }

View File

@ -57,10 +57,13 @@ static void streamMetaEnvInit() {
streamMetaId = taosOpenRef(64, streamMetaCloseImpl); streamMetaId = taosOpenRef(64, streamMetaCloseImpl);
metaRefMgtInit(); metaRefMgtInit();
streamTimerInit(); int32_t code = streamTimerInit();
if (code != 0) {
stError("failed to init stream meta env, start failed");
}
} }
void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } void streamMetaInit() { (void) taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); }
void streamMetaCleanup() { void streamMetaCleanup() {
taosCloseRef(streamBackendId); taosCloseRef(streamBackendId);
@ -89,11 +92,11 @@ void metaRefMgtCleanup() {
} }
taosHashCleanup(gMetaRefMgt.pTable); taosHashCleanup(gMetaRefMgt.pTable);
taosThreadMutexDestroy(&gMetaRefMgt.mutex); streamMutexDestroy(&gMetaRefMgt.mutex);
} }
int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) { int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) {
taosThreadMutexLock(&gMetaRefMgt.mutex); streamMutexLock(&gMetaRefMgt.mutex);
void* p = taosHashGet(gMetaRefMgt.pTable, &vgId, sizeof(vgId)); void* p = taosHashGet(gMetaRefMgt.pTable, &vgId, sizeof(vgId));
if (p == NULL) { if (p == NULL) {
@ -105,7 +108,7 @@ int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) {
taosArrayPush(list, &rid); taosArrayPush(list, &rid);
} }
taosThreadMutexUnlock(&gMetaRefMgt.mutex); streamMutexUnlock(&gMetaRefMgt.mutex);
return 0; return 0;
} }
@ -238,7 +241,7 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) {
int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key) { int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key) {
int64_t chkpId = pTask->chkInfo.checkpointId; int64_t chkpId = pTask->chkInfo.checkpointId;
taosThreadMutexLock(&pMeta->backendMutex); streamMutexLock(&pMeta->backendMutex);
void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key)); void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key));
if ((ppBackend != NULL) && (*ppBackend != NULL)) { if ((ppBackend != NULL) && (*ppBackend != NULL)) {
taskDbAddRef(*ppBackend); taskDbAddRef(*ppBackend);
@ -247,7 +250,7 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
pBackend->pMeta = pMeta; pBackend->pMeta = pMeta;
pTask->pBackend = pBackend; pTask->pBackend = pBackend;
taosThreadMutexUnlock(&pMeta->backendMutex); streamMutexUnlock(&pMeta->backendMutex);
stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend); stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend);
return 0; return 0;
} }
@ -260,11 +263,11 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
break; break;
} }
taosThreadMutexUnlock(&pMeta->backendMutex); streamMutexUnlock(&pMeta->backendMutex);
taosMsleep(1000); taosMsleep(1000);
stDebug("backend held by other task, restart later, path:%s, key:%s", pMeta->path, key); stDebug("backend held by other task, restart later, path:%s, key:%s", pMeta->path, key);
taosThreadMutexLock(&pMeta->backendMutex); streamMutexLock(&pMeta->backendMutex);
} }
int64_t tref = taosAddRef(taskDbWrapperId, pBackend); int64_t tref = taosAddRef(taskDbWrapperId, pBackend);
@ -276,7 +279,7 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
if (processVer != -1) pTask->chkInfo.processedVer = processVer; if (processVer != -1) pTask->chkInfo.processedVer = processVer;
taosHashPut(pMeta->pTaskDbUnique, key, strlen(key), &pBackend, sizeof(void*)); taosHashPut(pMeta->pTaskDbUnique, key, strlen(key), &pBackend, sizeof(void*));
taosThreadMutexUnlock(&pMeta->backendMutex); streamMutexUnlock(&pMeta->backendMutex);
stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend); stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend);
return 0; return 0;
@ -286,10 +289,10 @@ void streamMetaRemoveDB(void* arg, char* key) {
if (arg == NULL || key == NULL) return; if (arg == NULL || key == NULL) return;
SStreamMeta* pMeta = arg; SStreamMeta* pMeta = arg;
taosThreadMutexLock(&pMeta->backendMutex); streamMutexLock(&pMeta->backendMutex);
taosHashRemove(pMeta->pTaskDbUnique, key, strlen(key)); taosHashRemove(pMeta->pTaskDbUnique, key, strlen(key));
taosThreadMutexUnlock(&pMeta->backendMutex); streamMutexUnlock(&pMeta->backendMutex);
} }
int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, FTaskExpand expandTaskFn, int32_t vgId, int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, FTaskExpand expandTaskFn, int32_t vgId,
@ -541,7 +544,7 @@ void streamMetaCloseImpl(void* arg) {
pMeta->pHbInfo = NULL; pMeta->pHbInfo = NULL;
taosMemoryFree(pMeta->path); taosMemoryFree(pMeta->path);
taosThreadMutexDestroy(&pMeta->backendMutex); streamMutexDestroy(&pMeta->backendMutex);
taosCleanUpScheduler(pMeta->qHandle); taosCleanUpScheduler(pMeta->qHandle);
taosMemoryFree(pMeta->qHandle); taosMemoryFree(pMeta->qHandle);
@ -1132,7 +1135,7 @@ int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) {
continue; continue;
} }
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state == TASK_STATUS__CK) { if (pState.state == TASK_STATUS__CK) {
@ -1141,7 +1144,7 @@ int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) {
stDebug("s-task:%s status:%s not reset the checkpoint", pTask->id.idStr, pState.name); stDebug("s-task:%s status:%s not reset the checkpoint", pTask->id.idStr, pState.name);
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
} }
@ -1271,7 +1274,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) {
streamLaunchFillHistoryTask(pTask); streamLaunchFillHistoryTask(pTask);
} }
streamMetaAddTaskLaunchResult(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->checkTs, pInfo->readyTs, true); (void) streamMetaAddTaskLaunchResult(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->checkTs, pInfo->readyTs, true);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
continue; continue;
} }
@ -1388,16 +1391,16 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas
ASSERT(pTask->status.downstreamReady == 0); ASSERT(pTask->status.downstreamReady == 0);
// avoid initialization and destroy running concurrently. // avoid initialization and destroy running concurrently.
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pTask->pBackend == NULL) { if (pTask->pBackend == NULL) {
code = pMeta->expandTaskFn(pTask); code = pMeta->expandTaskFn(pTask);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs); streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs);
} }
} else { } else {
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
} }
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
@ -1440,7 +1443,6 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3
STaskId id = {.streamId = streamId, .taskId = taskId}; STaskId id = {.streamId = streamId, .taskId = taskId};
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
SStreamTask** p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); SStreamTask** p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (p == NULL) { // task does not exists in current vnode, not record the complete info if (p == NULL) { // task does not exists in current vnode, not record the complete info
stError("vgId:%d s-task:0x%x not exists discard the check downstream info", pMeta->vgId, taskId); stError("vgId:%d s-task:0x%x not exists discard the check downstream info", pMeta->vgId, taskId);
@ -1449,9 +1451,9 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3
} }
// clear the send consensus-checkpointId flag // clear the send consensus-checkpointId flag
taosThreadMutexLock(&(*p)->lock); streamMutexLock(&(*p)->lock);
(*p)->status.sendConsensusChkptId = false; (*p)->status.sendConsensusChkptId = false;
taosThreadMutexUnlock(&(*p)->lock); streamMutexUnlock(&(*p)->lock);
if (pStartInfo->startAllTasks != 1) { if (pStartInfo->startAllTasks != 1) {
int64_t el = endTs - startTs; int64_t el = endTs - startTs;
@ -1537,9 +1539,9 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t ta
streamMetaRUnLock(pMeta); streamMetaRUnLock(pMeta);
// add the failed task info, along with the related fill-history task info into tasks list. // add the failed task info, along with the related fill-history task info into tasks list.
streamMetaAddTaskLaunchResult(pMeta, streamId, taskId, startTs, now, false); (void) streamMetaAddTaskLaunchResult(pMeta, streamId, taskId, startTs, now, false);
if (hasFillhistoryTask) { if (hasFillhistoryTask) {
streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false); (void) streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false);
} }
} else { } else {
streamMetaRUnLock(pMeta); streamMetaRUnLock(pMeta);
@ -1554,12 +1556,12 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t ta
void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs) { void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs) {
int32_t startTs = pTask->execInfo.checkTs; int32_t startTs = pTask->execInfo.checkTs;
streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, startTs, failedTs, false); (void) streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, startTs, failedTs, false);
// automatically set the related fill-history task to be failed. // automatically set the related fill-history task to be failed.
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
STaskId* pId = &pTask->hTaskInfo.id; STaskId* pId = &pTask->hTaskInfo.id;
streamMetaAddTaskLaunchResult(pTask->pMeta, pId->streamId, pId->taskId, startTs, failedTs, false); (void) streamMetaAddTaskLaunchResult(pTask->pMeta, pId->streamId, pId->taskId, startTs, failedTs, false);
} }
} }

View File

@ -27,20 +27,24 @@ typedef struct SQueueReader {
int32_t waitDuration; // maximum wait time to format several block into a batch to process, unit: ms int32_t waitDuration; // maximum wait time to format several block into a batch to process, unit: ms
} SQueueReader; } SQueueReader;
#define streamQueueCurItem(_q) ((_q)->qItem)
static bool streamTaskExtractAvailableToken(STokenBucket* pBucket, const char* id); static bool streamTaskExtractAvailableToken(STokenBucket* pBucket, const char* id);
static void streamTaskPutbackToken(STokenBucket* pBucket); static void streamTaskPutbackToken(STokenBucket* pBucket);
static void streamTaskConsumeQuota(STokenBucket* pBucket, int32_t bytes); static void streamTaskConsumeQuota(STokenBucket* pBucket, int32_t bytes);
static void streamQueueCleanup(SStreamQueue* pQueue) { static void streamQueueCleanup(SStreamQueue* pQueue) {
void* qItem = NULL; SStreamQueueItem* qItem = NULL;
while ((qItem = streamQueueNextItem(pQueue)) != NULL) { while (1) {
streamQueueNextItem(pQueue, &qItem);
if (qItem == NULL) {
break;
}
streamFreeQitem(qItem); streamFreeQitem(qItem);
} }
pQueue->status = STREAM_QUEUE__SUCESS; pQueue->status = STREAM_QUEUE__SUCESS;
} }
static void* streamQueueCurItem(SStreamQueue* queue) { return queue->qItem; }
int32_t streamQueueOpen(int64_t cap, SStreamQueue** pQ) { int32_t streamQueueOpen(int64_t cap, SStreamQueue** pQ) {
*pQ = NULL; *pQ = NULL;
int32_t code = 0; int32_t code = 0;
@ -81,21 +85,22 @@ void streamQueueClose(SStreamQueue* pQueue, int32_t taskId) {
taosMemoryFree(pQueue); taosMemoryFree(pQueue);
} }
void* streamQueueNextItem(SStreamQueue* pQueue) { void streamQueueNextItem(SStreamQueue* pQueue, SStreamQueueItem** pItem) {
*pItem = NULL;
int8_t flag = atomic_exchange_8(&pQueue->status, STREAM_QUEUE__PROCESSING); int8_t flag = atomic_exchange_8(&pQueue->status, STREAM_QUEUE__PROCESSING);
if (flag == STREAM_QUEUE__FAILED) { if (flag == STREAM_QUEUE__FAILED) {
ASSERT(pQueue->qItem != NULL); ASSERT(pQueue->qItem != NULL);
return streamQueueCurItem(pQueue); *pItem = streamQueueCurItem(pQueue);
} else { } else {
pQueue->qItem = NULL; pQueue->qItem = NULL;
taosGetQitem(pQueue->qall, &pQueue->qItem); (void) taosGetQitem(pQueue->qall, &pQueue->qItem);
if (pQueue->qItem == NULL) { if (pQueue->qItem == NULL) {
taosReadAllQitems(pQueue->pQueue, pQueue->qall); (void) taosReadAllQitems(pQueue->pQueue, pQueue->qall);
taosGetQitem(pQueue->qall, &pQueue->qItem); (void) taosGetQitem(pQueue->qall, &pQueue->qItem);
} }
return streamQueueCurItem(pQueue); *pItem = streamQueueCurItem(pQueue);
} }
} }
@ -181,7 +186,8 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIte
return EXEC_CONTINUE; return EXEC_CONTINUE;
} }
SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputq.queue); SStreamQueueItem* qItem = NULL;
streamQueueNextItem(pTask->inputq.queue, (SStreamQueueItem**)&qItem);
if (qItem == NULL) { if (qItem == NULL) {
// restore the token to bucket // restore the token to bucket
if (*numOfBlocks > 0) { if (*numOfBlocks > 0) {
@ -338,7 +344,8 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem)
if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && type != STREAM_INPUT__CHECKPOINT_TRIGGER && if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && type != STREAM_INPUT__CHECKPOINT_TRIGGER &&
(pTask->info.delaySchedParam != 0)) { (pTask->info.delaySchedParam != 0)) {
atomic_val_compare_exchange_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE); (void)atomic_val_compare_exchange_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE,
TASK_TRIGGER_STATUS__ACTIVE);
stDebug("s-task:%s new data arrived, active the sched-trigger, triggerStatus:%d", pTask->id.idStr, stDebug("s-task:%s new data arrived, active the sched-trigger, triggerStatus:%d", pTask->id.idStr,
pTask->schedInfo.status); pTask->schedInfo.status);
} }
@ -347,18 +354,19 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem)
} }
int32_t streamTaskPutTranstateIntoInputQ(SStreamTask* pTask) { int32_t streamTaskPutTranstateIntoInputQ(SStreamTask* pTask) {
int32_t code; int32_t code = 0;
SStreamDataBlock* pTranstate; SStreamDataBlock* pTranstate = NULL;
SSDataBlock* pBlock = NULL;
code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock), (void**)&pTranstate); code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock), (void**)&pTranstate);
if (code) { if (code) {
return code; return code;
} }
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
if (pBlock == NULL) { if (pBlock == NULL) {
taosFreeQitem(pTranstate); code = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_OUT_OF_MEMORY; goto _err;
} }
pTranstate->type = STREAM_INPUT__TRANS_STATE; pTranstate->type = STREAM_INPUT__TRANS_STATE;
@ -368,15 +376,30 @@ int32_t streamTaskPutTranstateIntoInputQ(SStreamTask* pTask) {
pBlock->info.childId = pTask->info.selfChildId; pBlock->info.childId = pTask->info.selfChildId;
pTranstate->blocks = taosArrayInit(4, sizeof(SSDataBlock)); // pBlock; pTranstate->blocks = taosArrayInit(4, sizeof(SSDataBlock)); // pBlock;
taosArrayPush(pTranstate->blocks, pBlock); if (pTranstate->blocks == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
void* p = taosArrayPush(pTranstate->blocks, pBlock);
if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
taosMemoryFree(pBlock); taosMemoryFree(pBlock);
if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTranstate) < 0) { if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTranstate) < 0) {
return TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
} }
pTask->status.appendTranstateBlock = true; pTask->status.appendTranstateBlock = true;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_err:
taosMemoryFree(pBlock);
taosFreeQitem(pTranstate);
return code;
} }
// the result should be put into the outputQ in any cases, the result may be lost otherwise. // the result should be put into the outputQ in any cases, the result may be lost otherwise.

View File

@ -36,7 +36,7 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask) {
int32_t streamTrySchedExec(SStreamTask* pTask) { int32_t streamTrySchedExec(SStreamTask* pTask) {
if (streamTaskSetSchedStatusWait(pTask)) { if (streamTaskSetSchedStatusWait(pTask)) {
streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pTask->id.streamId, pTask->id.taskId, 0); return streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pTask->id.streamId, pTask->id.taskId, 0);
} else { } else {
stTrace("s-task:%s not launch task since sched status:%d", pTask->id.idStr, pTask->status.schedStatus); stTrace("s-task:%s not launch task since sched status:%d", pTask->id.idStr, pTask->status.schedStatus);
} }
@ -47,10 +47,9 @@ int32_t streamTrySchedExec(SStreamTask* pTask) {
int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType) { int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType) {
SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq));
if (pRunReq == NULL) { if (pRunReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
stError("vgId:%d failed to create msg to start stream task:0x%x exec, type:%d, code:%s", vgId, taskId, execType, stError("vgId:%d failed to create msg to start stream task:0x%x exec, type:%d, code:%s", vgId, taskId, execType,
terrstr()); terrstr());
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
if (streamId != 0) { if (streamId != 0) {
@ -65,29 +64,27 @@ int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int3
pRunReq->reqType = execType; pRunReq->reqType = execType;
SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)};
tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg); return tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg);
return TSDB_CODE_SUCCESS;
} }
void streamTaskClearSchedIdleInfo(SStreamTask* pTask) { pTask->status.schedIdleTime = 0; } void streamTaskClearSchedIdleInfo(SStreamTask* pTask) { pTask->status.schedIdleTime = 0; }
void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; } void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; }
int32_t streamTaskResumeInFuture(SStreamTask* pTask) { void streamTaskResumeInFuture(SStreamTask* pTask) {
int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s task should idle, add into timer to retry in %dms, ref:%d", pTask->id.idStr, stDebug("s-task:%s task should idle, add into timer to retry in %dms, ref:%d", pTask->id.idStr,
pTask->status.schedIdleTime, ref); pTask->status.schedIdleTime, ref);
// add one ref count for task // add one ref count for task
/*SStreamTask* pAddRefTask = */ streamMetaAcquireOneTask(pTask); streamMetaAcquireOneTask(pTask);
if (pTask->schedInfo.pIdleTimer == NULL) { if (pTask->schedInfo.pIdleTimer == NULL) {
pTask->schedInfo.pIdleTimer = taosTmrStart(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer); pTask->schedInfo.pIdleTimer = taosTmrStart(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer);
} else { } else {
taosTmrReset(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer, &pTask->schedInfo.pIdleTimer); streamTmrReset(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer,
&pTask->schedInfo.pIdleTimer, pTask->pMeta->vgId, "resume-task-tmr");
} }
return TSDB_CODE_SUCCESS;
} }
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -97,7 +94,7 @@ void streamTaskResumeHelper(void* param, void* tmrId) {
SStreamTaskState p = streamTaskGetStatus(pTask); SStreamTaskState p = streamTaskGetStatus(pTask);
if (p.state == TASK_STATUS__DROPPING || p.state == TASK_STATUS__STOP) { if (p.state == TASK_STATUS__DROPPING || p.state == TASK_STATUS__STOP) {
streamTaskSetSchedStatusInactive(pTask); (void) streamTaskSetSchedStatusInactive(pTask);
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s status:%s not resume task, ref:%d", pId->idStr, p.name, ref); stDebug("s-task:%s status:%s not resume task, ref:%d", pId->idStr, p.name, ref);
@ -106,8 +103,10 @@ void streamTaskResumeHelper(void* param, void* tmrId) {
return; return;
} }
streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pId->streamId, pId->taskId, STREAM_EXEC_T_RESUME_TASK); int32_t code = streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pId->streamId, pId->taskId, STREAM_EXEC_T_RESUME_TASK);
if (code) {
stError("s-task:%s sched task failed, code:%s", pTask->id.idStr, strerror(code));
} else {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("trigger to resume s-task:%s after being idled for %dms, ref:%d", pId->idStr, pTask->status.schedIdleTime, stDebug("trigger to resume s-task:%s after being idled for %dms, ref:%d", pId->idStr, pTask->status.schedIdleTime,
ref); ref);
@ -115,12 +114,14 @@ void streamTaskResumeHelper(void* param, void* tmrId) {
// release the task ref count // release the task ref count
streamTaskClearSchedIdleInfo(pTask); streamTaskClearSchedIdleInfo(pTask);
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
}
} }
void streamTaskSchedHelper(void* param, void* tmrId) { void streamTaskSchedHelper(void* param, void* tmrId) {
SStreamTask* pTask = (void*)param; SStreamTask* pTask = (void*)param;
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int32_t nextTrigger = (int32_t)pTask->info.delaySchedParam; int32_t nextTrigger = (int32_t)pTask->info.delaySchedParam;
int32_t vgId = pTask->pMeta->vgId;
int8_t status = atomic_load_8(&pTask->schedInfo.status); int8_t status = atomic_load_8(&pTask->schedInfo.status);
stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger); stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger);
@ -140,7 +141,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) {
if (code) { if (code) {
stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory", stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory",
nextTrigger); nextTrigger);
taosTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr");
terrno = code; terrno = code;
return; return;
} }
@ -152,7 +153,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) {
stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory", stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory",
nextTrigger); nextTrigger);
taosTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr");
return; return;
} }
@ -161,13 +162,16 @@ void streamTaskSchedHelper(void* param, void* tmrId) {
code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger); code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr");
return; return;
} }
streamTrySchedExec(pTask); code = streamTrySchedExec(pTask);
if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed to sched to run, wait for next time", pTask->id.idStr);
}
} }
} }
taosTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr");
} }

View File

@ -212,7 +212,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) {
stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", idStr, hStreamId, hTaskId, stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", idStr, hStreamId, hTaskId,
pStatus.name); pStatus.name);
streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false);
return -1; // todo set the correct error code return -1; // todo set the correct error code
} }
@ -228,11 +228,11 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) {
code = streamMetaAcquireTask(pMeta, hStreamId, hTaskId, &pHisTask); code = streamMetaAcquireTask(pMeta, hStreamId, hTaskId, &pHisTask);
if (pHisTask == NULL) { if (pHisTask == NULL) {
stDebug("s-task:%s failed acquire and start fill-history task, it may have been dropped/stopped", idStr); stDebug("s-task:%s failed acquire and start fill-history task, it may have been dropped/stopped", idStr);
streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false);
} else { } else {
if (pHisTask->status.downstreamReady == 1) { // it's ready now, do nothing if (pHisTask->status.downstreamReady == 1) { // it's ready now, do nothing
stDebug("s-task:%s fill-history task is ready, no need to check downstream", pHisTask->id.idStr); stDebug("s-task:%s fill-history task is ready, no need to check downstream", pHisTask->id.idStr);
streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, true); (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, true);
} else { // exist, but not ready, continue check downstream task status } else { // exist, but not ready, continue check downstream task status
if (pHisTask->pBackend == NULL) { if (pHisTask->pBackend == NULL) {
code = pMeta->expandTaskFn(pHisTask); code = pMeta->expandTaskFn(pHisTask);
@ -289,7 +289,7 @@ void notRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo,
SHistoryTaskInfo* pHTaskInfo = &pTask->hTaskInfo; SHistoryTaskInfo* pHTaskInfo = &pTask->hTaskInfo;
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); (void) streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false);
stError("s-task:%s max retry:%d reached, quit from retrying launch related fill-history task:0x%x, ref:%d", stError("s-task:%s max retry:%d reached, quit from retrying launch related fill-history task:0x%x, ref:%d",
pTask->id.idStr, MAX_RETRY_LAUNCH_HISTORY_TASK, (int32_t)pHTaskInfo->id.taskId, ref); pTask->id.idStr, MAX_RETRY_LAUNCH_HISTORY_TASK, (int32_t)pHTaskInfo->id.taskId, ref);
@ -307,7 +307,7 @@ void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, i
stDebug("s-task:0x%" PRIx64 " stopped, not launch rel history task:0x%" PRIx64 ", ref:%d", pInfo->id.taskId, stDebug("s-task:0x%" PRIx64 " stopped, not launch rel history task:0x%" PRIx64 ", ref:%d", pInfo->id.taskId,
pInfo->hTaskId.taskId, ref); pInfo->hTaskId.taskId, ref);
streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); (void) streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false);
taosMemoryFree(pInfo); taosMemoryFree(pInfo);
} else { } else {
char* p = streamTaskGetStatus(pTask).name; char* p = streamTaskGetStatus(pTask).name;
@ -350,7 +350,7 @@ void tryLaunchHistoryTask(void* param, void* tmrId) {
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
// record the related fill-history task failed // record the related fill-history task failed
streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); (void) streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false);
taosMemoryFree(pInfo); taosMemoryFree(pInfo);
return; return;
} }
@ -407,7 +407,7 @@ void tryLaunchHistoryTask(void* param, void* tmrId) {
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
} else { } else {
streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); (void) streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false);
int32_t ref = atomic_sub_fetch_32(&(*ppTask)->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&(*ppTask)->status.timerActive, 1);
stError("s-task:0x%x rel fill-history task:0x%" PRIx64 " may have been destroyed, not launch, ref:%d", stError("s-task:0x%x rel fill-history task:0x%" PRIx64 " may have been destroyed, not launch, ref:%d",
@ -448,7 +448,7 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) {
SLaunchHTaskInfo* pInfo = createHTaskLaunchInfo(pMeta, &id, hStreamId, hTaskId); SLaunchHTaskInfo* pInfo = createHTaskLaunchInfo(pMeta, &id, hStreamId, hTaskId);
if (pInfo == NULL) { if (pInfo == NULL) {
stError("s-task:%s failed to launch related fill-history task, since Out Of Memory", idStr); stError("s-task:%s failed to launch related fill-history task, since Out Of Memory", idStr);
streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false);
return terrno; return terrno;
} }
@ -465,7 +465,7 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) {
stError("s-task:%s failed to start timer, related fill-history task not launched, ref:%d", idStr, ref); stError("s-task:%s failed to start timer, related fill-history task not launched, ref:%d", idStr, ref);
taosMemoryFree(pInfo); taosMemoryFree(pInfo);
streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false);
return terrno; return terrno;
} }

View File

@ -197,13 +197,13 @@ void tFreeStreamTask(SStreamTask* pTask) {
STaskExecStatisInfo* pStatis = &pTask->execInfo; STaskExecStatisInfo* pStatis = &pTask->execInfo;
ETaskStatus status1 = TASK_STATUS__UNINIT; ETaskStatus status1 = TASK_STATUS__UNINIT;
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pTask->status.pSM != NULL) { if (pTask->status.pSM != NULL) {
SStreamTaskState pStatus = streamTaskGetStatus(pTask); SStreamTaskState pStatus = streamTaskGetStatus(pTask);
p = pStatus.name; p = pStatus.name;
status1 = pStatus.state; status1 = pStatus.state;
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
stDebug("start to free s-task:0x%x %p, state:%s", taskId, pTask, p); stDebug("start to free s-task:0x%x %p, state:%s", taskId, pTask, p);
@ -284,11 +284,11 @@ void tFreeStreamTask(SStreamTask* pTask) {
streamTaskDestroyUpstreamInfo(&pTask->upstreamInfo); streamTaskDestroyUpstreamInfo(&pTask->upstreamInfo);
taosMemoryFree(pTask->outputInfo.pTokenBucket); taosMemoryFree(pTask->outputInfo.pTokenBucket);
taosThreadMutexDestroy(&pTask->lock); streamMutexDestroy(&pTask->lock);
taosArrayDestroy(pTask->msgInfo.pSendInfo); taosArrayDestroy(pTask->msgInfo.pSendInfo);
pTask->msgInfo.pSendInfo = NULL; pTask->msgInfo.pSendInfo = NULL;
taosThreadMutexDestroy(&pTask->msgInfo.lock); streamMutexDestroy(&pTask->msgInfo.lock);
taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList); taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList);
pTask->outputInfo.pNodeEpsetUpdateList = NULL; pTask->outputInfo.pNodeEpsetUpdateList = NULL;
@ -644,11 +644,7 @@ void streamTaskOpenAllUpstreamInput(SStreamTask* pTask) {
void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) { void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) {
SStreamUpstreamEpInfo* pInfo = NULL; SStreamUpstreamEpInfo* pInfo = NULL;
streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo);
int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) {
return;
}
if ((pInfo != NULL) && pInfo->dataAllowed) { if ((pInfo != NULL) && pInfo->dataAllowed) {
pInfo->dataAllowed = false; pInfo->dataAllowed = false;
@ -659,11 +655,7 @@ void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) {
void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId) { void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId) {
SStreamUpstreamEpInfo* pInfo = NULL; SStreamUpstreamEpInfo* pInfo = NULL;
streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo);
int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) {
return;
}
if (pInfo != NULL && (!pInfo->dataAllowed)) { if (pInfo != NULL && (!pInfo->dataAllowed)) {
int32_t t = atomic_sub_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); int32_t t = atomic_sub_fetch_32(&pTask->upstreamInfo.numOfClosed, 1);
@ -679,34 +671,34 @@ bool streamTaskIsAllUpstreamClosed(SStreamTask* pTask) {
bool streamTaskSetSchedStatusWait(SStreamTask* pTask) { bool streamTaskSetSchedStatusWait(SStreamTask* pTask) {
bool ret = false; bool ret = false;
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pTask->status.schedStatus == TASK_SCHED_STATUS__INACTIVE) { if (pTask->status.schedStatus == TASK_SCHED_STATUS__INACTIVE) {
pTask->status.schedStatus = TASK_SCHED_STATUS__WAITING; pTask->status.schedStatus = TASK_SCHED_STATUS__WAITING;
ret = true; ret = true;
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return ret; return ret;
} }
int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask) { int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask) {
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
int8_t status = pTask->status.schedStatus; int8_t status = pTask->status.schedStatus;
if (status == TASK_SCHED_STATUS__WAITING) { if (status == TASK_SCHED_STATUS__WAITING) {
pTask->status.schedStatus = TASK_SCHED_STATUS__ACTIVE; pTask->status.schedStatus = TASK_SCHED_STATUS__ACTIVE;
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return status; return status;
} }
int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) { int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) {
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
int8_t status = pTask->status.schedStatus; int8_t status = pTask->status.schedStatus;
ASSERT(status == TASK_SCHED_STATUS__WAITING || status == TASK_SCHED_STATUS__ACTIVE || ASSERT(status == TASK_SCHED_STATUS__WAITING || status == TASK_SCHED_STATUS__ACTIVE ||
status == TASK_SCHED_STATUS__INACTIVE); status == TASK_SCHED_STATUS__INACTIVE);
pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return status; return status;
} }
@ -723,7 +715,7 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) {
stDebug("s-task:%s clear the related stream task:0x%x attr to fill-history task", pTask->id.idStr, stDebug("s-task:%s clear the related stream task:0x%x attr to fill-history task", pTask->id.idStr,
(int32_t)sTaskId.taskId); (int32_t)sTaskId.taskId);
taosThreadMutexLock(&(*ppStreamTask)->lock); streamMutexLock(&(*ppStreamTask)->lock);
CLEAR_RELATED_FILLHISTORY_TASK((*ppStreamTask)); CLEAR_RELATED_FILLHISTORY_TASK((*ppStreamTask));
if (resetRelHalt) { if (resetRelHalt) {
@ -734,7 +726,7 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) {
} }
streamMetaSaveTask(pMeta, *ppStreamTask); streamMetaSaveTask(pMeta, *ppStreamTask);
taosThreadMutexUnlock(&(*ppStreamTask)->lock); streamMutexUnlock(&(*ppStreamTask)->lock);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -923,6 +915,7 @@ void streamTaskResume(SStreamTask* pTask) {
bool streamTaskIsSinkTask(const SStreamTask* pTask) { return pTask->info.taskLevel == TASK_LEVEL__SINK; } bool streamTaskIsSinkTask(const SStreamTask* pTask) { return pTask->info.taskLevel == TASK_LEVEL__SINK; }
// this task must success
int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) { int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) {
int32_t code; int32_t code;
int32_t tlen = 0; int32_t tlen = 0;
@ -960,24 +953,23 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) {
return 0; return 0;
} }
int32_t streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo) { void streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo) {
*pEpInfo = NULL; *pEpInfo = NULL;
int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList); int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList);
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
if (pInfo == NULL) { if (pInfo == NULL) {
return TSDB_CODE_FAILED; return;
} }
if (pInfo->taskId == taskId) { if (pInfo->taskId == taskId) {
*pEpInfo = pInfo; *pEpInfo = pInfo;
return TSDB_CODE_SUCCESS; return;
} }
} }
stError("s-task:%s failed to find upstream task:0x%x", pTask->id.idStr, taskId); stError("s-task:%s failed to find upstream task:0x%x", pTask->id.idStr, taskId);
return TSDB_CODE_FAILED;
} }
SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId) { SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId) {
@ -1100,7 +1092,7 @@ void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) {
return; return;
} }
taosThreadMutexDestroy(&pInfo->lock); streamMutexDestroy(&pInfo->lock);
taosArrayDestroy(pInfo->pDispatchTriggerList); taosArrayDestroy(pInfo->pDispatchTriggerList);
pInfo->pDispatchTriggerList = NULL; pInfo->pDispatchTriggerList = NULL;
taosArrayDestroy(pInfo->pReadyMsgList); taosArrayDestroy(pInfo->pReadyMsgList);

View File

@ -201,7 +201,7 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName,
pSM->pActiveTrans = pNextTrans; pSM->pActiveTrans = pNextTrans;
pSM->startTs = taosGetTimestampMs(); pSM->startTs = taosGetTimestampMs();
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
code = pNextTrans->pAction(pSM->pTask); code = pNextTrans->pAction(pSM->pTask);
if (pNextTrans->autoInvokeEndFn) { if (pNextTrans->autoInvokeEndFn) {
@ -210,7 +210,7 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName,
return code; return code;
} }
} else { } else {
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
stDebug("s-task:%s state:%s event:%s in waiting list, req state:%s not fulfilled, put it back", pTask->id.idStr, stDebug("s-task:%s state:%s event:%s in waiting list, req state:%s not fulfilled, put it back", pTask->id.idStr,
pSM->current.name, GET_EVT_NAME(pEvtInfo->event), pSM->current.name, GET_EVT_NAME(pEvtInfo->event),
StreamTaskStatusList[pEvtInfo->status].name); StreamTaskStatusList[pEvtInfo->status].name);
@ -247,7 +247,7 @@ int32_t streamTaskRestoreStatus(SStreamTask* pTask) {
SStreamTaskSM* pSM = pTask->status.pSM; SStreamTaskSM* pSM = pTask->status.pSM;
int32_t code = 0; int32_t code = 0;
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pSM->current.state == TASK_STATUS__PAUSE && pSM->pActiveTrans == NULL) { if (pSM->current.state == TASK_STATUS__PAUSE && pSM->pActiveTrans == NULL) {
SStreamTaskState state = pSM->current; SStreamTaskState state = pSM->current;
@ -326,13 +326,13 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt
return code; return code;
} }
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
while (1) { while (1) {
// wait for the task to be here // wait for the task to be here
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
ETaskStatus s = streamTaskGetStatus(pTask).state; ETaskStatus s = streamTaskGetStatus(pTask).state;
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {// this event has been handled already if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {// this event has been handled already
stDebug("s-task:%s attached event:%s handled", id, GET_EVT_NAME(pTrans->event)); stDebug("s-task:%s attached event:%s handled", id, GET_EVT_NAME(pTrans->event));
@ -349,7 +349,7 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt
} else { // override current active trans } else { // override current active trans
pSM->pActiveTrans = pTrans; pSM->pActiveTrans = pTrans;
pSM->startTs = taosGetTimestampMs(); pSM->startTs = taosGetTimestampMs();
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
code = pTrans->pAction(pTask); code = pTrans->pAction(pTask);
@ -374,11 +374,11 @@ static int32_t doHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, ST
info.callBackFn = callbackFn; info.callBackFn = callbackFn;
code = attachWaitedEvent(pTask, &info); code = attachWaitedEvent(pTask, &info);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
} else { // override current active trans } else { // override current active trans
pSM->pActiveTrans = pTrans; pSM->pActiveTrans = pTrans;
pSM->startTs = taosGetTimestampMs(); pSM->startTs = taosGetTimestampMs();
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
code = pTrans->pAction(pTask); code = pTrans->pAction(pTask);
// todo handle error code; // todo handle error code;
@ -400,11 +400,11 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) {
STaskStateTrans* pTrans = NULL; STaskStateTrans* pTrans = NULL;
while (1) { while (1) {
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) { if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) {
EStreamTaskEvent evt = pSM->pActiveTrans->event; EStreamTaskEvent evt = pSM->pActiveTrans->event;
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed", stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed",
pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt)); pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt));
@ -414,7 +414,7 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) {
pTrans = streamTaskFindTransform(pSM->current.state, event); pTrans = streamTaskFindTransform(pSM->current.state, event);
if (pTrans == NULL) { if (pTrans == NULL) {
stDebug("s-task:%s failed to handle event:%s", pTask->id.idStr, GET_EVT_NAME(event)); stDebug("s-task:%s failed to handle event:%s", pTask->id.idStr, GET_EVT_NAME(event));
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return TSDB_CODE_STREAM_INVALID_STATETRANS; return TSDB_CODE_STREAM_INVALID_STATETRANS;
} }
@ -439,11 +439,11 @@ int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, _
STaskStateTrans* pTrans = NULL; STaskStateTrans* pTrans = NULL;
while (1) { while (1) {
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) { if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) {
EStreamTaskEvent evt = pSM->pActiveTrans->event; EStreamTaskEvent evt = pSM->pActiveTrans->event;
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed", stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed",
pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt)); pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt));
@ -453,7 +453,7 @@ int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, _
pTrans = streamTaskFindTransform(pSM->current.state, event); pTrans = streamTaskFindTransform(pSM->current.state, event);
if (pTrans == NULL) { if (pTrans == NULL) {
stDebug("s-task:%s failed to handle event:%s, status:%s", pTask->id.idStr, GET_EVT_NAME(event), pSM->current.name); stDebug("s-task:%s failed to handle event:%s, status:%s", pTask->id.idStr, GET_EVT_NAME(event), pSM->current.name);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return TSDB_CODE_STREAM_INVALID_STATETRANS; return TSDB_CODE_STREAM_INVALID_STATETRANS;
} }
@ -485,7 +485,7 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even
int32_t code = 0; int32_t code = 0;
// do update the task status // do update the task status
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
STaskStateTrans* pTrans = pSM->pActiveTrans; STaskStateTrans* pTrans = pSM->pActiveTrans;
if (pTrans == NULL) { if (pTrans == NULL) {
@ -497,14 +497,14 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even
stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", id, GET_EVT_NAME(event), stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", id, GET_EVT_NAME(event),
pSM->current.name, GET_EVT_NAME(pSM->prev.evt)); pSM->current.name, GET_EVT_NAME(pSM->prev.evt));
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return TSDB_CODE_STREAM_INVALID_STATETRANS; return TSDB_CODE_STREAM_INVALID_STATETRANS;
} }
if (pTrans->event != event) { if (pTrans->event != event) {
stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", id, GET_EVT_NAME(event), stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", id, GET_EVT_NAME(event),
pSM->current.name, GET_EVT_NAME(pTrans->event)); pSM->current.name, GET_EVT_NAME(pTrans->event));
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
return TSDB_CODE_STREAM_INVALID_STATETRANS; return TSDB_CODE_STREAM_INVALID_STATETRANS;
} }
@ -518,7 +518,7 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even
// on success callback, add into lock if necessary, or maybe we should add an option for this? // on success callback, add into lock if necessary, or maybe we should add an option for this?
code = pTrans->pSuccAction(pTask); code = pTrans->pSuccAction(pTask);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
// todo: add parameter to control lock // todo: add parameter to control lock
// after handling the callback function assigned by invoker, go on handling the waiting tasks // after handling the callback function assigned by invoker, go on handling the waiting tasks
@ -532,13 +532,13 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even
stDebug("s-task:%s handle user-specified callback fn for event:%s completed", id, GET_EVT_NAME(pTrans->event)); stDebug("s-task:%s handle user-specified callback fn for event:%s completed", id, GET_EVT_NAME(pTrans->event));
} }
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
// tasks in waiting list // tasks in waiting list
if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { if (taosArrayGetSize(pSM->pWaitingEventList) > 0) {
code = doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask); code = doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask);
} else { } else {
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
int64_t el = (taosGetTimestampMs() - pSM->startTs); int64_t el = (taosGetTimestampMs() - pSM->startTs);
stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", id, stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", id,
@ -563,14 +563,14 @@ const char* streamTaskGetStatusStr(ETaskStatus status) {
void streamTaskResetStatus(SStreamTask* pTask) { void streamTaskResetStatus(SStreamTask* pTask) {
SStreamTaskSM* pSM = pTask->status.pSM; SStreamTaskSM* pSM = pTask->status.pSM;
(void) taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
stDebug("s-task:%s level:%d fill-history:%d vgId:%d set uninit, prev status:%s", pTask->id.idStr, stDebug("s-task:%s level:%d fill-history:%d vgId:%d set uninit, prev status:%s", pTask->id.idStr,
pTask->info.taskLevel, pTask->info.fillHistory, pTask->pMeta->vgId, pSM->current.name); pTask->info.taskLevel, pTask->info.fillHistory, pTask->pMeta->vgId, pSM->current.name);
pSM->current = StreamTaskStatusList[TASK_STATUS__UNINIT]; pSM->current = StreamTaskStatusList[TASK_STATUS__UNINIT];
pSM->pActiveTrans = NULL; pSM->pActiveTrans = NULL;
taosArrayClear(pSM->pWaitingEventList); taosArrayClear(pSM->pWaitingEventList);
(void) taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
// clear the downstream ready status // clear the downstream ready status
pTask->status.downstreamReady = 0; pTask->status.downstreamReady = 0;

View File

@ -41,11 +41,11 @@ tmr_h streamTimerGetInstance() {
void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId,
const char* pMsg) { const char* pMsg) {
while (1) { // while (1) {
bool ret = taosTmrReset(fp, mseconds, param, handle, pTmrId); bool ret = taosTmrReset(fp, mseconds, param, handle, pTmrId);
if (ret) { if (ret) {
break; // break;
}
stError("vgId:%d failed to reset %s, try again", vgId, pMsg);
} }
// stError("vgId:%d failed to reset tmr: %s, try again", vgId, pMsg);
// }
} }

View File

@ -0,0 +1,26 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "streamInt.h"
void streamMutexLock(TdThreadMutex *pMutex) {
(void) taosThreadMutexLock(pMutex);
}
void streamMutexUnlock(TdThreadMutex *pMutex) {
(void) taosThreadMutexUnlock(pMutex);
}
void streamMutexDestroy(TdThreadMutex *pMutex) { (void) taosThreadMutexDestroy(pMutex); }

View File

@ -471,6 +471,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR, "Geometry not support
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_WINDOW_CONDITION, "The time pseudo column is illegally used in the condition of the event window.") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_WINDOW_CONDITION, "The time pseudo column is illegally used in the condition of the event window.")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR, "Executor internal error") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR, "Executor internal error")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_JOIN_CONDITION, "Not supported join on condition") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_JOIN_CONDITION, "Not supported join on condition")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_FILTER_NOT_SUPPORT_TYPE, "Not supported range type")
// grant // grant
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, "License expired") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, "License expired")