xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #26720 from taosdata/fix/TD-31059.2 

fix: function return code validation
This commit is contained in:
dapan1121 2024-07-23 08:49:51 +08:00 committed by GitHub
parent 5d1d0dd290 14383a32a3
commit 53c386526c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 1308 additions and 502 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/os/osString.h
View File

@ -61,6 +61,7 @@ typedef enum { M2C = 0, C2M } ConvType;
#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src))
#define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size))
#define TAOS_STRCAT(_dst, _src) ((void)strcat(_dst, _src))
char *tstrdup(const char *src);
int32_t taosUcs4len(TdUcs4 *ucs4);

11
source/libs/executor/inc/mergejoin.h
View File

@ -432,6 +432,15 @@ typedef struct SMJoinOperatorInfo {
} \
} while (0)
#define MJ_RET(c) \
do { \
int32_t _code = c; \
if (_code != TSDB_CODE_SUCCESS) { \
terrno = _code; \
} \
return _code; \
} while (0)
void mJoinDestroyMergeCtx(SMJoinOperatorInfo* pJoin);
@ -459,7 +468,7 @@ int32_t mJoinCreateFullBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pT
int32_t mJoinCreateBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable);
int32_t mJoinSetKeyColsData(SSDataBlock* pBlock, SMJoinTableCtx* pTable);
int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastBuildGrp);
bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build);
int32_t mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build, bool* cont);
int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pFirst, SMJoinGrpRows* pSecond);
int32_t mJoinHandleMidRemains(SMJoinMergeCtx* pCtx);
int32_t mJoinNonEqGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pGrp, bool probeGrp);

92
source/libs/executor/src/dataDeleter.c
View File

@ -53,7 +53,7 @@ typedef struct SDataDeleterHandle {
TdThreadMutex mutex;
} SDataDeleterHandle;
static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInput, SDataDeleterBuf* pBuf) {
static int32_t toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInput, SDataDeleterBuf* pBuf) {
int32_t numOfCols = LIST_LENGTH(pHandle->pSchema->pSlots);
SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData;
@ -65,14 +65,23 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp
pBuf->useSize = sizeof(SDataCacheEntry);
SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 0);
if (NULL == pColRes) {
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pColSKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 1);
if (NULL == pColSKey) {
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pColEKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 2);
if (NULL == pColEKey) {
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SDeleterRes* pRes = (SDeleterRes*)pEntry->data;
pRes->suid = pHandle->pParam->suid;
pRes->uidList = pHandle->pParam->pUidList;
strcpy(pRes->tableName, pHandle->pDeleter->tableFName);
strcpy(pRes->tsColName, pHandle->pDeleter->tsColName);
TAOS_STRCPY(pRes->tableName, pHandle->pDeleter->tableFName);
TAOS_STRCPY(pRes->tsColName, pHandle->pDeleter->tsColName);
pRes->affectedRows = *(int64_t*)pColRes->pData;
if (pRes->affectedRows) {
@ -88,16 +97,18 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp
pBuf->useSize += pEntry->dataLen;
atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
(void)atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
(void)atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
return TSDB_CODE_SUCCESS;
}
static bool allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, SDataDeleterBuf* pBuf) {
static int32_t allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, SDataDeleterBuf* pBuf) {
uint32_t capacity = pDeleter->pManager->cfg.maxDataBlockNumPerQuery;
if (taosQueueItemSize(pDeleter->pDataBlocks) > capacity) {
qError("SinkNode queue is full, no capacity, max:%d, current:%d, no capacity", capacity,
taosQueueItemSize(pDeleter->pDataBlocks));
return false;
return TSDB_CODE_OUT_OF_MEMORY;
}
pBuf->allocSize = sizeof(SDataCacheEntry) + sizeof(SDeleterRes);
@ -105,55 +116,66 @@ static bool allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, SDa
pBuf->pData = taosMemoryMalloc(pBuf->allocSize);
if (pBuf->pData == NULL) {
qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno));
return terrno;
}
return NULL != pBuf->pData;
return TSDB_CODE_SUCCESS;
}
static int32_t updateStatus(SDataDeleterHandle* pDeleter) {
taosThreadMutexLock(&pDeleter->mutex);
(void)taosThreadMutexLock(&pDeleter->mutex);
int32_t blockNums = taosQueueItemSize(pDeleter->pDataBlocks);
int32_t status =
(0 == blockNums ? DS_BUF_EMPTY
: (blockNums < pDeleter->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL));
pDeleter->status = status;
taosThreadMutexUnlock(&pDeleter->mutex);
(void)taosThreadMutexUnlock(&pDeleter->mutex);
return status;
}
static int32_t getStatus(SDataDeleterHandle* pDeleter) {
taosThreadMutexLock(&pDeleter->mutex);
(void)taosThreadMutexLock(&pDeleter->mutex);
int32_t status = pDeleter->status;
taosThreadMutexUnlock(&pDeleter->mutex);
(void)taosThreadMutexUnlock(&pDeleter->mutex);
return status;
}
static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) {
SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle;
SDataDeleterBuf* pBuf;
SDataDeleterBuf* pBuf = NULL;
int32_t code = taosAllocateQitem(sizeof(SDataDeleterBuf), DEF_QITEM, 0, (void**)&pBuf);
if (code) {
return code;
}
if (!allocBuf(pDeleter, pInput, pBuf)) {
code = allocBuf(pDeleter, pInput, pBuf);
if (code) {
taosFreeQitem(pBuf);
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
toDataCacheEntry(pDeleter, pInput, pBuf);
taosWriteQitem(pDeleter->pDataBlocks, pBuf);
QRY_ERR_JRET(toDataCacheEntry(pDeleter, pInput, pBuf));
QRY_ERR_JRET(taosWriteQitem(pDeleter->pDataBlocks, pBuf));
*pContinue = (DS_BUF_LOW == updateStatus(pDeleter) ? true : false);
return TSDB_CODE_SUCCESS;
_return:
taosFreeQitem(pBuf);
return code;
}
static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle;
taosThreadMutexLock(&pDeleter->mutex);
(void)taosThreadMutexLock(&pDeleter->mutex);
pDeleter->queryEnd = true;
pDeleter->useconds = useconds;
taosThreadMutexUnlock(&pDeleter->mutex);
(void)taosThreadMutexUnlock(&pDeleter->mutex);
}
static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) {
@ -165,9 +187,9 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRaw
}
SDataDeleterBuf* pBuf = NULL;
taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
(void)taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
if (pBuf != NULL) {
memcpy(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf));
TAOS_MEMCPY(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf));
taosFreeQitem(pBuf);
}
@ -192,35 +214,35 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
}
SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDeleter->nextOutput.pData);
memcpy(pOutput->pData, pEntry->data, pEntry->dataLen);
TAOS_MEMCPY(pOutput->pData, pEntry->data, pEntry->dataLen);
pDeleter->pParam->pUidList = NULL;
pOutput->numOfRows = pEntry->numOfRows;
pOutput->numOfCols = pEntry->numOfCols;
pOutput->compressed = pEntry->compressed;
atomic_sub_fetch_64(&pDeleter->cachedSize, pEntry->dataLen);
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
(void)atomic_sub_fetch_64(&pDeleter->cachedSize, pEntry->dataLen);
(void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
taosMemoryFreeClear(pDeleter->nextOutput.pData); // todo persistent
pOutput->bufStatus = updateStatus(pDeleter);
taosThreadMutexLock(&pDeleter->mutex);
(void)taosThreadMutexLock(&pDeleter->mutex);
pOutput->queryEnd = pDeleter->queryEnd;
pOutput->useconds = pDeleter->useconds;
pOutput->precision = pDeleter->pSchema->precision;
taosThreadMutexUnlock(&pDeleter->mutex);
(void)taosThreadMutexUnlock(&pDeleter->mutex);
return TSDB_CODE_SUCCESS;
}
static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle;
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDeleter->cachedSize);
(void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDeleter->cachedSize);
taosMemoryFreeClear(pDeleter->nextOutput.pData);
taosArrayDestroy(pDeleter->pParam->pUidList);
taosMemoryFree(pDeleter->pParam);
while (!taosQueueEmpty(pDeleter->pDataBlocks)) {
SDataDeleterBuf* pBuf = NULL;
taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
(void)taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
if (pBuf != NULL) {
taosMemoryFreeClear(pBuf->pData);
@ -228,9 +250,10 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
}
}
taosCloseQueue(pDeleter->pDataBlocks);
taosThreadMutexDestroy(&pDeleter->mutex);
(void)taosThreadMutexDestroy(&pDeleter->mutex);
taosMemoryFree(pDeleter->pManager);
return TSDB_CODE_SUCCESS;
}
@ -247,8 +270,8 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData
SDataDeleterHandle* deleter = taosMemoryCalloc(1, sizeof(SDataDeleterHandle));
if (NULL == deleter) {
code = terrno;
taosMemoryFree(pParam);
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
@ -276,17 +299,22 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData
if (code) {
goto _end;
}
taosThreadMutexInit(&deleter->mutex, NULL);
code = taosThreadMutexInit(&deleter->mutex, NULL);
if (code) {
goto _end;
}
*pHandle = deleter;
return code;
_end:
if (deleter != NULL) {
destroyDataSinker((SDataSinkHandle*)deleter);
(void)destroyDataSinker((SDataSinkHandle*)deleter);
taosMemoryFree(deleter);
} else {
taosMemoryFree(pManager);
}
return code;
}

89
source/libs/executor/src/dataDispatcher.c
View File

@ -63,7 +63,7 @@ typedef struct SDataDispatchHandle {
// The length of bitmap is decided by number of rows of this data block, and the length of each column data is
// recorded in the first segment, next to the struct header
// clang-format on
static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) {
static int32_t toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) {
int32_t numOfCols = 0;
SNode* pNode;
@ -88,6 +88,9 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
if (pHandle->pCompressBuf == NULL) {
// allocate additional 8 bytes to avoid invalid write if compress failed to reduce the size
pHandle->pCompressBuf = taosMemoryMalloc(pBuf->allocSize + 8);
if (NULL == pHandle->pCompressBuf) {
QRY_RET(terrno);
}
pHandle->bufSize = pBuf->allocSize + 8;
} else {
if (pHandle->bufSize < pBuf->allocSize + 8) {
@ -96,9 +99,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
if (p != NULL) {
pHandle->pCompressBuf = p;
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;
qError("failed to prepare compress buf:%d, code: out of memory", pHandle->bufSize);
return;
qError("failed to prepare compress buf:%d, code: %x", pHandle->bufSize, terrno);
return terrno;
}
}
}
@ -114,7 +116,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
pEntry->compressed = 0;
pEntry->dataLen = dataLen;
pEntry->rawLen = dataLen;
memcpy(pEntry->data, pHandle->pCompressBuf, dataLen);
TAOS_MEMCPY(pEntry->data, pHandle->pCompressBuf, dataLen);
}
} else {
pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols);
@ -124,11 +126,13 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
pBuf->useSize += pEntry->dataLen;
atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
(void)atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
(void)atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
return TSDB_CODE_SUCCESS;
}
static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) {
static int32_t allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) {
/*
uint32_t capacity = pDispatcher->pManager->cfg.maxDataBlockNumPerQuery;
if (taosQueueItemSize(pDispatcher->pDataBlocks) > capacity) {
@ -142,69 +146,73 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput,
pBuf->pData = taosMemoryMalloc(pBuf->allocSize);
if (pBuf->pData == NULL) {
qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno));
qError("SinkNode failed to malloc memory, size:%d, code:%x", pBuf->allocSize, terrno);
return terrno;
}
return NULL != pBuf->pData;
return TSDB_CODE_SUCCESS;
}
static int32_t updateStatus(SDataDispatchHandle* pDispatcher) {
taosThreadMutexLock(&pDispatcher->mutex);
(void)taosThreadMutexLock(&pDispatcher->mutex);
int32_t blockNums = taosQueueItemSize(pDispatcher->pDataBlocks);
int32_t status =
(0 == blockNums ? DS_BUF_EMPTY
: (blockNums < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL));
pDispatcher->status = status;
taosThreadMutexUnlock(&pDispatcher->mutex);
(void)taosThreadMutexUnlock(&pDispatcher->mutex);
return status;
}
static int32_t getStatus(SDataDispatchHandle* pDispatcher) {
taosThreadMutexLock(&pDispatcher->mutex);
(void)taosThreadMutexLock(&pDispatcher->mutex);
int32_t status = pDispatcher->status;
taosThreadMutexUnlock(&pDispatcher->mutex);
(void)taosThreadMutexUnlock(&pDispatcher->mutex);
return status;
}
static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) {
int32_t code = 0;
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
SDataDispatchBuf* pBuf;
SDataDispatchBuf* pBuf = NULL;
code = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM, 0, (void**)&pBuf);
if (code) {
return code;
}
if (!allocBuf(pDispatcher, pInput, pBuf)) {
code = allocBuf(pDispatcher, pInput, pBuf);
if (code) {
taosFreeQitem(pBuf);
return TSDB_CODE_OUT_OF_MEMORY;
}
toDataCacheEntry(pDispatcher, pInput, pBuf);
code = taosWriteQitem(pDispatcher->pDataBlocks, pBuf);
if (code != 0) {
return code;
}
QRY_ERR_JRET(toDataCacheEntry(pDispatcher, pInput, pBuf));
QRY_ERR_JRET(taosWriteQitem(pDispatcher->pDataBlocks, pBuf));
int32_t status = updateStatus(pDispatcher);
*pContinue = (status == DS_BUF_LOW || status == DS_BUF_EMPTY);
return TSDB_CODE_SUCCESS;
_return:
taosFreeQitem(pBuf);
return code;
}
static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
taosThreadMutexLock(&pDispatcher->mutex);
(void)taosThreadMutexLock(&pDispatcher->mutex);
pDispatcher->queryEnd = true;
pDispatcher->useconds = useconds;
taosThreadMutexUnlock(&pDispatcher->mutex);
(void)taosThreadMutexUnlock(&pDispatcher->mutex);
}
static void resetDispatcher(struct SDataSinkHandle* pHandle) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
taosThreadMutexLock(&pDispatcher->mutex);
(void)taosThreadMutexLock(&pDispatcher->mutex);
pDispatcher->queryEnd = false;
taosThreadMutexUnlock(&pDispatcher->mutex);
(void)taosThreadMutexUnlock(&pDispatcher->mutex);
}
static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRowLen, bool* pQueryEnd) {
@ -216,9 +224,9 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRow
}
SDataDispatchBuf* pBuf = NULL;
taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf);
(void)taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf);
if (pBuf != NULL) {
memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf));
TAOS_MEMCPY(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf));
taosFreeQitem(pBuf);
}
@ -243,33 +251,34 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
}
SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData);
memcpy(pOutput->pData, pEntry->data, pEntry->dataLen);
TAOS_MEMCPY(pOutput->pData, pEntry->data, pEntry->dataLen);
pOutput->numOfRows = pEntry->numOfRows;
pOutput->numOfCols = pEntry->numOfCols;
pOutput->compressed = pEntry->compressed;
atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen);
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
(void)atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen);
(void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
taosMemoryFreeClear(pDispatcher->nextOutput.pData); // todo persistent
pOutput->bufStatus = updateStatus(pDispatcher);
taosThreadMutexLock(&pDispatcher->mutex);
(void)taosThreadMutexLock(&pDispatcher->mutex);
pOutput->queryEnd = pDispatcher->queryEnd;
pOutput->useconds = pDispatcher->useconds;
pOutput->precision = pDispatcher->pSchema->precision;
taosThreadMutexUnlock(&pDispatcher->mutex);
(void)taosThreadMutexUnlock(&pDispatcher->mutex);
return TSDB_CODE_SUCCESS;
}
static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDispatcher->cachedSize);
(void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDispatcher->cachedSize);
taosMemoryFreeClear(pDispatcher->nextOutput.pData);
while (!taosQueueEmpty(pDispatcher->pDataBlocks)) {
SDataDispatchBuf* pBuf = NULL;
taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf);
(void)taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf);
if (pBuf != NULL) {
taosMemoryFreeClear(pBuf->pData);
taosFreeQitem(pBuf);
@ -280,7 +289,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
taosMemoryFreeClear(pDispatcher->pCompressBuf);
pDispatcher->bufSize = 0;
taosThreadMutexDestroy(&pDispatcher->mutex);
(void)taosThreadMutexDestroy(&pDispatcher->mutex);
taosMemoryFree(pDispatcher->pManager);
return TSDB_CODE_SUCCESS;
}
@ -297,7 +306,6 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle));
if (NULL == dispatcher) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _return;
}
@ -318,7 +326,11 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
terrno = code;
goto _return;
}
taosThreadMutexInit(&dispatcher->mutex, NULL);
code = taosThreadMutexInit(&dispatcher->mutex, NULL);
if (code) {
terrno = code;
goto _return;
}
if (NULL == dispatcher->pDataBlocks) {
taosMemoryFree(dispatcher);
@ -330,6 +342,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
return TSDB_CODE_SUCCESS;
_return:
taosMemoryFree(pManager);
return terrno;
}

95
source/libs/executor/src/dataInserter.c
View File

@ -58,10 +58,17 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
SSubmitRspParam* pParam = (SSubmitRspParam*)param;
SDataInserterHandle* pInserter = pParam->pInserter;
pInserter->submitRes.code = code;
if (code) {
pInserter->submitRes.code = code;
}
if (code == TSDB_CODE_SUCCESS) {
pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp2));
if (NULL == pInserter->submitRes.pRsp) {
pInserter->submitRes.code = terrno;
goto _return;
}
SDecoder coder = {0};
tDecoderInit(&coder, pMsg->pData, pMsg->len);
code = tDecodeSSubmitRsp2(&coder, pInserter->submitRes.pRsp);
@ -77,6 +84,10 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
for (int32_t i = 0; i < numOfTables; ++i) {
SVCreateTbRsp* pRsp = taosArrayGet(pCreateTbList, i);
if (NULL == pRsp) {
pInserter->submitRes.code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
goto _return;
}
if (TSDB_CODE_SUCCESS != pRsp->code) {
code = pRsp->code;
taosMemoryFree(pInserter->submitRes.pRsp);
@ -94,8 +105,10 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
}
_return:
tsem_post(&pInserter->ready);
(void)tsem_post(&pInserter->ready);
taosMemoryFree(pMsg->pData);
return TSDB_CODE_SUCCESS;
}
@ -105,11 +118,15 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, void* pMsg, int
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (NULL == pMsgSendInfo) {
taosMemoryFreeClear(pMsg);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
SSubmitRspParam* pParam = taosMemoryCalloc(1, sizeof(SSubmitRspParam));
if (NULL == pParam) {
taosMemoryFreeClear(pMsg);
taosMemoryFreeClear(pMsgSendInfo);
return terrno;
}
pParam->pInserter = pInserter;
pMsgSendInfo->param = pParam;
@ -133,7 +150,7 @@ static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int
len += sizeof(SSubmitReq2Msg);
pBuf = taosMemoryMalloc(len);
if (NULL == pBuf) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
((SSubmitReq2Msg*)pBuf)->header.vgId = htonl(vgId);
((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len);
@ -149,6 +166,7 @@ static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int
} else {
taosMemoryFree(pBuf);
}
return code;
}
@ -162,12 +180,10 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
if (NULL == pReq) {
if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
}
@ -208,6 +224,10 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
}
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, colIdx);
if (NULL == pColInfoData) {
terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
goto _end;
}
void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
switch (pColInfoData->info.type) {
@ -217,13 +237,17 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
ASSERT(pColInfoData->info.type == pCol->type);
if (colDataIsNull_s(pColInfoData, j)) {
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type);
taosArrayPush(pVals, &cv);
if (NULL == taosArrayPush(pVals, &cv)) {
goto _end;
}
} else {
void* data = colDataGetVarData(pColInfoData, j);
SValue sv = (SValue){
.type = pCol->type, .nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value
SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
taosArrayPush(pVals, &cv);
if (NULL == taosArrayPush(pVals, &cv)) {
goto _end;
}
}
break;
}
@ -245,7 +269,9 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
}
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); // should use pCol->type
taosArrayPush(pVals, &cv);
if (NULL == taosArrayPush(pVals, &cv)) {
goto _end;
}
} else {
if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId && !needSortMerge) {
if (*(int64_t*)var <= lastTs) {
@ -256,9 +282,11 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
}
SValue sv = {.type = pCol->type};
memcpy(&sv.val, var, tDataTypes[pCol->type].bytes);
TAOS_MEMCPY(&sv.val, var, tDataTypes[pCol->type].bytes);
SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
taosArrayPush(pVals, &cv);
if (NULL == taosArrayPush(pVals, &cv)) {
goto _end;
}
}
} else {
uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type);
@ -274,7 +302,9 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
goto _end;
}
taosArrayPush(tbData.aRowP, &pRow);
if (NULL == taosArrayPush(tbData.aRowP, &pRow)) {
goto _end;
}
}
if (needSortMerge) {
@ -284,9 +314,12 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
}
}
taosArrayPush(pReq->aSubmitTbData, &tbData);
if (NULL == taosArrayPush(pReq->aSubmitTbData, &tbData)) {
goto _end;
}
_end:
taosArrayDestroy(pVals);
if (terrno != 0) {
*ppReq = NULL;
@ -294,9 +327,11 @@ _end:
tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
taosMemoryFree(pReq);
}
return terrno;
}
*ppReq = pReq;
return TSDB_CODE_SUCCESS;
}
@ -312,7 +347,9 @@ int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32
for (int32_t i = 0; i < sz; i++) {
SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i);
if (NULL == pDataBlock) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
code = buildSubmitReqFromBlock(pInserter, &pReq, pDataBlock, pTSchema, uid, vgId, suid);
if (code) {
if (pReq) {
@ -334,7 +371,9 @@ int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32
static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) {
SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle;
if (!pInserter->explain) {
taosArrayPush(pInserter->pDataBlocks, &pInput->pData);
if (NULL == taosArrayPush(pInserter->pDataBlocks, &pInput->pData)) {
return terrno;
}
void* pMsg = NULL;
int32_t msgLen = 0;
int32_t code = dataBlocksToSubmitReq(pInserter, &pMsg, &msgLen);
@ -350,7 +389,7 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput,
return code;
}
tsem_wait(&pInserter->ready);
QRY_ERR_RET(tsem_wait(&pInserter->ready));
if (pInserter->submitRes.code) {
return pInserter->submitRes.code;
@ -364,10 +403,10 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput,
static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle;
taosThreadMutexLock(&pInserter->mutex);
(void)taosThreadMutexLock(&pInserter->mutex);
pInserter->queryEnd = true;
pInserter->useconds = useconds;
taosThreadMutexUnlock(&pInserter->mutex);
(void)taosThreadMutexUnlock(&pInserter->mutex);
}
static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) {
@ -378,12 +417,12 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRaw
static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle;
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pInserter->cachedSize);
(void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pInserter->cachedSize);
taosArrayDestroy(pInserter->pDataBlocks);
taosMemoryFree(pInserter->pSchema);
taosMemoryFree(pInserter->pParam);
taosHashCleanup(pInserter->pCols);
taosThreadMutexDestroy(&pInserter->mutex);
(void)taosThreadMutexDestroy(&pInserter->mutex);
taosMemoryFree(pInserter->pManager);
return TSDB_CODE_SUCCESS;
@ -401,7 +440,6 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle));
if (NULL == inserter) {
taosMemoryFree(pParam);
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _return;
}
@ -432,28 +470,31 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
}
inserter->pDataBlocks = taosArrayInit(1, POINTER_BYTES);
taosThreadMutexInit(&inserter->mutex, NULL);
if (NULL == inserter->pDataBlocks) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _return;
}
QRY_ERR_JRET(taosThreadMutexInit(&inserter->mutex, NULL));
inserter->fullOrderColList = pInserterNode->pCols->length == inserter->pSchema->numOfCols;
inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT),
false, HASH_NO_LOCK);
if (NULL == inserter->pCols) {
goto _return;
}
SNode* pNode = NULL;
int32_t i = 0;
FOREACH(pNode, pInserterNode->pCols) {
SColumnNode* pCol = (SColumnNode*)pNode;
taosHashPut(inserter->pCols, &pCol->colId, sizeof(pCol->colId), &pCol->slotId, sizeof(pCol->slotId));
QRY_ERR_JRET(taosHashPut(inserter->pCols, &pCol->colId, sizeof(pCol->colId), &pCol->slotId, sizeof(pCol->slotId)));
if (inserter->fullOrderColList && pCol->colId != inserter->pSchema->columns[i].colId) {
inserter->fullOrderColList = false;
}
++i;
}
tsem_init(&inserter->ready, 0, 0);
QRY_ERR_JRET(tsem_init(&inserter->ready, 0, 0));
*pHandle = inserter;
return TSDB_CODE_SUCCESS;
@ -461,7 +502,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
_return:
if (inserter) {
destroyDataSinker((SDataSinkHandle*)inserter);
(void)destroyDataSinker((SDataSinkHandle*)inserter);
taosMemoryFree(inserter);
} else {
taosMemoryFree(pManager);

9
source/libs/executor/src/dataSinkMgt.c
View File

@ -23,20 +23,20 @@ SDataSinkStat gDataSinkStat = {0};
int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI, void** ppSinkManager) {
SDataSinkManager* pSinkManager = taosMemoryMalloc(sizeof(SDataSinkManager));
if (NULL == pSinkManager) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pSinkManager->cfg = *cfg;
pSinkManager->pAPI = pAPI;
*ppSinkManager = pSinkManager;
return 0; // to avoid compiler eror
return TSDB_CODE_SUCCESS; // to avoid compiler eror
}
int32_t dsDataSinkGetCacheSize(SDataSinkStat* pStat) {
pStat->cachedSize = atomic_load_64(&gDataSinkStat.cachedSize);
return 0;
return TSDB_CODE_SUCCESS;
}
int32_t dsCreateDataSinker(void* pSinkManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void* pParam, const char* id) {
@ -56,6 +56,7 @@ int32_t dsCreateDataSinker(void* pSinkManager, const SDataSinkNode* pDataSink, D
taosMemoryFree(pSinkManager);
qError("invalid input node type:%d, %s", nodeType(pDataSink), id);
return TSDB_CODE_QRY_INVALID_INPUT;
}
@ -97,6 +98,6 @@ void dsScheduleProcess(void* ahandle, void* pItem) {
void dsDestroyDataSinker(DataSinkHandle handle) {
SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle;
pHandleImpl->fDestroy(pHandleImpl);
(void)pHandleImpl->fDestroy(pHandleImpl);
taosMemoryFree(pHandleImpl);
}

222
source/libs/executor/src/dynqueryctrloperator.c
View File

@ -105,7 +105,7 @@ static FORCE_INLINE bool tableNeedCache(int64_t uid, SStbJoinPrevJoinCtx* pPrev,
return (NULL == num) ? false : true;
}
static void updatePostJoinCurrTableInfo(SStbJoinDynCtrlInfo* pStbJoin) {
static int32_t updatePostJoinCurrTableInfo(SStbJoinDynCtrlInfo* pStbJoin) {
SStbJoinPrevJoinCtx* pPrev = &pStbJoin->ctx.prev;
SStbJoinPostJoinCtx* pPost = &pStbJoin->ctx.post;
SStbJoinTableList* pNode = pPrev->pListHead;
@ -141,32 +141,38 @@ static void updatePostJoinCurrTableInfo(SStbJoinDynCtrlInfo* pStbJoin)
pPost->leftNeedCache = tableNeedCache(*leftUid, pPrev, pPost, false, pStbJoin->basic.batchFetch);
pPost->rightNeedCache = tableNeedCache(*rightUid, pPrev, pPost, true, pStbJoin->basic.batchFetch);
if (pPost->rightNeedCache && rightPrevUid != pPost->rightCurrUid) {
tSimpleHashPut(pPrev->rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid), NULL, 0);
if (!pStbJoin->basic.batchFetch && pPost->rightNeedCache && rightPrevUid != pPost->rightCurrUid) {
QRY_ERR_RET(tSimpleHashPut(pPrev->rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid), NULL, 0));
pStbJoin->execInfo.rightCacheNum++;
}
return TSDB_CODE_SUCCESS;
}
static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t vgId, int64_t tbUid, bool needCache, SOperatorParam* pChild) {
int32_t code = TSDB_CODE_SUCCESS;
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
code = terrno;
freeOperatorParam(pChild, OP_GET_PARAM);
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
if (pChild) {
(*ppRes)->pChildren = taosArrayInit(1, POINTER_BYTES);
if (NULL == (*ppRes)->pChildren) {
code = terrno;
freeOperatorParam(pChild, OP_GET_PARAM);
freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild)) {
code = terrno;
freeOperatorParam(pChild, OP_GET_PARAM);
freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
} else {
(*ppRes)->pChildren = NULL;
@ -174,9 +180,10 @@ static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t down
SGcOperatorParam* pGc = taosMemoryMalloc(sizeof(SGcOperatorParam));
if (NULL == pGc) {
code = terrno;
freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
pGc->sessionId = atomic_add_fetch_64(&gSessionId, 1);
@ -194,16 +201,18 @@ static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t down
static int32_t buildGroupCacheNotifyOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t vgId, int64_t tbUid) {
int32_t code = TSDB_CODE_SUCCESS;
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
(*ppRes)->pChildren = NULL;
SGcNotifyOperatorParam* pGc = taosMemoryMalloc(sizeof(SGcNotifyOperatorParam));
if (NULL == pGc) {
code = terrno;
freeOperatorParam(*ppRes, OP_NOTIFY_PARAM);
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
pGc->downstreamIdx = downstreamIdx;
@ -221,13 +230,13 @@ static int32_t buildGroupCacheNotifyOperatorParam(SOperatorParam** ppRes, int32_
static int32_t buildExchangeOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t* pVgId, int64_t* pUid) {
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
(*ppRes)->pChildren = NULL;
SExchangeOperatorParam* pExc = taosMemoryMalloc(sizeof(SExchangeOperatorParam));
if (NULL == pExc) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pExc->multiParams = false;
@ -237,7 +246,7 @@ static int32_t buildExchangeOperatorParam(SOperatorParam** ppRes, int32_t downst
pExc->basic.uidList = taosArrayInit(1, sizeof(int64_t));
if (NULL == pExc->basic.uidList) {
taosMemoryFree(pExc);
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
taosArrayPush(pExc->basic.uidList, pUid);
@ -252,14 +261,14 @@ static int32_t buildExchangeOperatorParam(SOperatorParam** ppRes, int32_t downst
static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, SSHashObj* pVg) {
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
(*ppRes)->pChildren = NULL;
SExchangeOperatorBatchParam* pExc = taosMemoryMalloc(sizeof(SExchangeOperatorBatchParam));
if (NULL == pExc) {
taosMemoryFreeClear(*ppRes);
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pExc->multiParams = true;
@ -267,7 +276,7 @@ static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t d
if (NULL == pExc->pBatchs) {
taosMemoryFree(pExc);
taosMemoryFreeClear(*ppRes);
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
tSimpleHashSetFreeFp(pExc->pBatchs, freeExchangeGetBasicOperatorParam);
@ -283,7 +292,7 @@ static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t d
basic.uidList = pUidList;
basic.tableSeq = false;
tSimpleHashPut(pExc->pBatchs, pVgId, sizeof(*pVgId), &basic, sizeof(basic));
QRY_ERR_RET(tSimpleHashPut(pExc->pBatchs, pVgId, sizeof(*pVgId), &basic, sizeof(basic)));
qTrace("build downstreamIdx %d batch scan, vgId:%d, uidNum:%" PRId64, downstreamIdx, *pVgId, (int64_t)taosArrayGetSize(pUidList));
*(SArray**)p = NULL;
@ -298,39 +307,45 @@ static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t d
static int32_t buildMergeJoinOperatorParam(SOperatorParam** ppRes, bool initParam, SOperatorParam* pChild0, SOperatorParam* pChild1) {
int32_t code = TSDB_CODE_SUCCESS;
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
code = terrno;
freeOperatorParam(pChild0, OP_GET_PARAM);
freeOperatorParam(pChild1, OP_GET_PARAM);
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
(*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES);
if (NULL == *ppRes) {
code = terrno;
freeOperatorParam(pChild0, OP_GET_PARAM);
freeOperatorParam(pChild1, OP_GET_PARAM);
freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild0)) {
code = terrno;
freeOperatorParam(pChild0, OP_GET_PARAM);
freeOperatorParam(pChild1, OP_GET_PARAM);
freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild1)) {
code = terrno;
freeOperatorParam(pChild1, OP_GET_PARAM);
freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
SSortMergeJoinOperatorParam* pJoin = taosMemoryMalloc(sizeof(SSortMergeJoinOperatorParam));
if (NULL == pJoin) {
code = terrno;
freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
pJoin->initDownstream = initParam;
@ -343,31 +358,36 @@ static int32_t buildMergeJoinOperatorParam(SOperatorParam** ppRes, bool initPara
static int32_t buildMergeJoinNotifyOperatorParam(SOperatorParam** ppRes, SOperatorParam* pChild0, SOperatorParam* pChild1) {
int32_t code = TSDB_CODE_SUCCESS;
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) {
code = terrno;
freeOperatorParam(pChild0, OP_NOTIFY_PARAM);
freeOperatorParam(pChild1, OP_NOTIFY_PARAM);
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
(*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES);
if (NULL == *ppRes) {
code = terrno;
taosMemoryFreeClear(*ppRes);
freeOperatorParam(pChild0, OP_NOTIFY_PARAM);
freeOperatorParam(pChild1, OP_NOTIFY_PARAM);
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
if (pChild0 && NULL == taosArrayPush((*ppRes)->pChildren, &pChild0)) {
code = terrno;
freeOperatorParam(*ppRes, OP_NOTIFY_PARAM);
freeOperatorParam(pChild0, OP_NOTIFY_PARAM);
freeOperatorParam(pChild1, OP_NOTIFY_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
if (pChild1 && NULL == taosArrayPush((*ppRes)->pChildren, &pChild1)) {
code = terrno;
freeOperatorParam(*ppRes, OP_NOTIFY_PARAM);
freeOperatorParam(pChild1, OP_NOTIFY_PARAM);
*ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
(*ppRes)->opType = QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN;
@ -407,9 +427,11 @@ static int32_t buildBatchTableScanOperatorParam(SOperatorParam** ppRes, int32_t
static int32_t buildSingleTableScanOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t* pVgId, int64_t* pUid) {
SArray* pUidList = taosArrayInit(1, sizeof(int64_t));
if (NULL == pUidList) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
if (NULL == taosArrayPush(pUidList, pUid)) {
return terrno;
}
taosArrayPush(pUidList, pUid);
int32_t code = buildTableScanOperatorParam(ppRes, pUidList, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, true);
taosArrayDestroy(pUidList);
@ -435,7 +457,7 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS
qDebug("start %" PRId64 ":%" PRId64 "th stbJoin, left:%d,%" PRIu64 " - right:%d,%" PRIu64,
rowIdx, pPrev->tableNum, *leftVg, *leftUid, *rightVg, *rightUid);
updatePostJoinCurrTableInfo(&pInfo->stbJoin);
QRY_ERR_RET(updatePostJoinCurrTableInfo(&pInfo->stbJoin));
if (pInfo->stbJoin.basic.batchFetch) {
if (pPrev->leftHash) {
@ -538,53 +560,56 @@ static int32_t notifySeqJoinTableCacheEnd(SOperatorInfo* pOperator, SStbJoinPost
return optrDefaultNotifyFn(pOperator->pDownstream[1], pMergeJoinParam);
}
static void handleSeqJoinCurrRetrieveEnd(SOperatorInfo* pOperator, SStbJoinDynCtrlInfo* pStbJoin) {
static int32_t handleSeqJoinCurrRetrieveEnd(SOperatorInfo* pOperator, SStbJoinDynCtrlInfo* pStbJoin) {
SStbJoinPostJoinCtx* pPost = &pStbJoin->ctx.post;
pPost->isStarted = false;
if (pStbJoin->basic.batchFetch) {
return;
return TSDB_CODE_SUCCESS;
}
if (pPost->leftNeedCache) {
uint32_t* num = tSimpleHashGet(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid));
if (num && --(*num) <= 0) {
tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid));
notifySeqJoinTableCacheEnd(pOperator, pPost, true);
(void)tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid));
QRY_ERR_RET(notifySeqJoinTableCacheEnd(pOperator, pPost, true));
}
}
if (!pPost->rightNeedCache) {
void* v = tSimpleHashGet(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid));
if (NULL != v) {
tSimpleHashRemove(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid));
notifySeqJoinTableCacheEnd(pOperator, pPost, false);
(void)tSimpleHashRemove(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid));
QRY_ERR_RET(notifySeqJoinTableCacheEnd(pOperator, pPost, false));
}
}
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE void seqJoinContinueCurrRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
static FORCE_INLINE int32_t seqJoinContinueCurrRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinPostJoinCtx* pPost = &pInfo->stbJoin.ctx.post;
SStbJoinPrevJoinCtx* pPrev = &pInfo->stbJoin.ctx.prev;
if (!pPost->isStarted) {
return;
return TSDB_CODE_SUCCESS;
}
qDebug("%s dynQueryCtrl continue to retrieve block from post op", GET_TASKID(pOperator->pTaskInfo));
*ppRes = getNextBlockFromDownstream(pOperator, 1);
if (NULL == *ppRes) {
handleSeqJoinCurrRetrieveEnd(pOperator, &pInfo->stbJoin);
QRY_ERR_RET(handleSeqJoinCurrRetrieveEnd(pOperator, &pInfo->stbJoin));
pPrev->pListHead->readIdx++;
} else {
pInfo->stbJoin.execInfo.postBlkNum++;
pInfo->stbJoin.execInfo.postBlkRows += (*ppRes)->info.rows;
return;
}
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE int32_t addToJoinVgroupHash(SSHashObj* pHash, void* pKey, int32_t keySize, void* pVal, int32_t valSize) {
@ -592,35 +617,38 @@ static FORCE_INLINE int32_t addToJoinVgroupHash(SSHashObj* pHash, void* pKey, in
if (NULL == ppArray) {
SArray* pArray = taosArrayInit(10, valSize);
if (NULL == pArray) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
if (NULL == taosArrayPush(pArray, pVal)) {
taosArrayDestroy(pArray);
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
if (tSimpleHashPut(pHash, pKey, keySize, &pArray, POINTER_BYTES)) {
taosArrayDestroy(pArray);
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
return TSDB_CODE_SUCCESS;
}
if (NULL == taosArrayPush(*ppArray, pVal)) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE int32_t addToJoinTableHash(SSHashObj* pHash, SSHashObj* pOnceHash, void* pKey, int32_t keySize) {
int32_t code = TSDB_CODE_SUCCESS;
uint32_t* pNum = tSimpleHashGet(pHash, pKey, keySize);
if (NULL == pNum) {
uint32_t n = 1;
if (tSimpleHashPut(pHash, pKey, keySize, &n, sizeof(n))) {
return TSDB_CODE_OUT_OF_MEMORY;
code = tSimpleHashPut(pHash, pKey, keySize, &n, sizeof(n));
if (code) {
return code;
}
if (tSimpleHashPut(pOnceHash, pKey, keySize, NULL, 0)) {
return TSDB_CODE_OUT_OF_MEMORY;
code = tSimpleHashPut(pOnceHash, pKey, keySize, NULL, 0);
if (code) {
return code;
}
return TSDB_CODE_SUCCESS;
}
@ -633,7 +661,7 @@ static FORCE_INLINE int32_t addToJoinTableHash(SSHashObj* pHash, SSHashObj* pOnc
break;
default:
if (1 == (*pNum)) {
tSimpleHashRemove(pOnceHash, pKey, keySize);
(void)tSimpleHashRemove(pOnceHash, pKey, keySize);
}
(*pNum)++;
break;
@ -655,23 +683,40 @@ static void freeStbJoinTableList(SStbJoinTableList* pList) {
}
static int32_t appendStbJoinTableList(SStbJoinPrevJoinCtx* pCtx, int64_t rows, int32_t* pLeftVg, int64_t* pLeftUid, int32_t* pRightVg, int64_t* pRightUid) {
int32_t code = TSDB_CODE_SUCCESS;
SStbJoinTableList* pNew = taosMemoryMalloc(sizeof(SStbJoinTableList));
if (NULL == pNew) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pNew->pLeftVg = taosMemoryMalloc(rows * sizeof(*pLeftVg));
pNew->pLeftUid = taosMemoryMalloc(rows * sizeof(*pLeftUid));
pNew->pRightVg = taosMemoryMalloc(rows * sizeof(*pRightVg));
pNew->pRightUid = taosMemoryMalloc(rows * sizeof(*pRightUid));
if (NULL == pNew->pLeftVg || NULL == pNew->pLeftUid || NULL == pNew->pRightVg || NULL == pNew->pRightUid) {
if (NULL == pNew->pLeftVg) {
code = terrno;
freeStbJoinTableList(pNew);
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
pNew->pLeftUid = taosMemoryMalloc(rows * sizeof(*pLeftUid));
if (NULL == pNew->pLeftUid) {
code = terrno;
freeStbJoinTableList(pNew);
return code;
}
pNew->pRightVg = taosMemoryMalloc(rows * sizeof(*pRightVg));
if (NULL == pNew->pRightVg) {
code = terrno;
freeStbJoinTableList(pNew);
return code;
}
pNew->pRightUid = taosMemoryMalloc(rows * sizeof(*pRightUid));
if (NULL == pNew->pRightUid) {
code = terrno;
freeStbJoinTableList(pNew);
return code;
}
memcpy(pNew->pLeftVg, pLeftVg, rows * sizeof(*pLeftVg));
memcpy(pNew->pLeftUid, pLeftUid, rows * sizeof(*pLeftUid));
memcpy(pNew->pRightVg, pRightVg, rows * sizeof(*pRightVg));
memcpy(pNew->pRightUid, pRightUid, rows * sizeof(*pRightUid));
TAOS_MEMCPY(pNew->pLeftVg, pLeftVg, rows * sizeof(*pLeftVg));
TAOS_MEMCPY(pNew->pLeftUid, pLeftUid, rows * sizeof(*pLeftUid));
TAOS_MEMCPY(pNew->pRightVg, pRightVg, rows * sizeof(*pRightVg));
TAOS_MEMCPY(pNew->pRightUid, pRightUid, rows * sizeof(*pRightUid));
pNew->readIdx = 0;
pNew->uidNum = rows;
@ -693,9 +738,21 @@ static void doBuildStbJoinTableHash(SOperatorInfo* pOperator, SSDataBlock* pBloc
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin;
SColumnInfoData* pVg0 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.vgSlot[0]);
if (NULL == pVg0) {
QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pVg1 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.vgSlot[1]);
if (NULL == pVg1) {
QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pUid0 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.uidSlot[0]);
if (NULL == pUid0) {
QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pUid1 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.uidSlot[1]);
if (NULL == pUid1) {
QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pStbJoin->basic.batchFetch) {
for (int32_t i = 0; i < pBlock->info.rows; ++i) {
@ -731,6 +788,8 @@ static void doBuildStbJoinTableHash(SOperatorInfo* pOperator, SSDataBlock* pBloc
}
}
_return:
if (TSDB_CODE_SUCCESS != code) {
pOperator->pTaskInfo->code = code;
T_LONG_JMP(pOperator->pTaskInfo->env, pOperator->pTaskInfo->code);
@ -754,7 +813,7 @@ static void postProcessStbJoinTableHash(SOperatorInfo* pOperator) {
uint64_t* pUid = NULL;
int32_t iter = 0;
while (NULL != (pUid = tSimpleHashIterate(pStbJoin->ctx.prev.onceTable, pUid, &iter))) {
tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, pUid, sizeof(*pUid));
(void)tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, pUid, sizeof(*pUid));
}
pStbJoin->execInfo.leftCacheNum = tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache);
@ -789,7 +848,7 @@ static void buildStbJoinTableList(SOperatorInfo* pOperator) {
pStbJoin->ctx.prev.joinBuild = true;
}
static void seqJoinLaunchNewRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
static int32_t seqJoinLaunchNewRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin;
SStbJoinPrevJoinCtx* pPrev = &pStbJoin->ctx.prev;
@ -805,17 +864,17 @@ static void seqJoinLaunchNewRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppR
seqJoinLaunchNewRetrieveImpl(pOperator, ppRes);
if (*ppRes) {
return;
return TSDB_CODE_SUCCESS;
}
handleSeqJoinCurrRetrieveEnd(pOperator, pStbJoin);
QRY_ERR_RET(handleSeqJoinCurrRetrieveEnd(pOperator, pStbJoin));
pPrev->pListHead->readIdx++;
}
*ppRes = NULL;
setOperatorCompleted(pOperator);
return;
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE SSDataBlock* seqStableJoinComposeRes(SStbJoinDynCtrlInfo* pStbJoin, SSDataBlock* pBlock) {
@ -825,6 +884,7 @@ static FORCE_INLINE SSDataBlock* seqStableJoinComposeRes(SStbJoinDynCtrlInfo*
SSDataBlock* seqStableJoin(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin;
SSDataBlock* pRes = NULL;
@ -846,18 +906,23 @@ SSDataBlock* seqStableJoin(SOperatorInfo* pOperator) {
}
}
seqJoinContinueCurrRetrieve(pOperator, &pRes);
QRY_ERR_JRET(seqJoinContinueCurrRetrieve(pOperator, &pRes));
if (pRes) {
goto _return;
}
seqJoinLaunchNewRetrieve(pOperator, &pRes);
QRY_ERR_JRET(seqJoinLaunchNewRetrieve(pOperator, &pRes));
_return:
if (pOperator->cost.openCost == 0) {
pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
}
if (code) {
pOperator->pTaskInfo->code = code;
T_LONG_JMP(pOperator->pTaskInfo->env, pOperator->pTaskInfo->code);
}
return pRes ? seqStableJoinComposeRes(pStbJoin, pRes) : NULL;
}
@ -866,24 +931,24 @@ int32_t initSeqStbJoinTableHash(SStbJoinPrevJoinCtx* pPrev, bool batchFetch) {
if (batchFetch) {
pPrev->leftHash = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
if (NULL == pPrev->leftHash) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pPrev->rightHash = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
if (NULL == pPrev->rightHash) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
} else {
pPrev->leftCache = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (NULL == pPrev->leftCache) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pPrev->rightCache = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (NULL == pPrev->rightCache) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pPrev->onceTable = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (NULL == pPrev->onceTable) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
}
@ -893,13 +958,16 @@ int32_t initSeqStbJoinTableHash(SStbJoinPrevJoinCtx* pPrev, bool batchFetch) {
SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream,
SDynQueryCtrlPhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo) {
SDynQueryCtrlOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SDynQueryCtrlOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
__optr_fn_t nextFp = NULL;
int32_t code = TSDB_CODE_SUCCESS;
if (pOperator == NULL || pInfo == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
__optr_fn_t nextFp = NULL;
SDynQueryCtrlOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SDynQueryCtrlOperatorInfo));
if (pInfo == NULL) {
code = terrno;
goto _error;
}
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pOperator == NULL) {
code = terrno;
goto _error;
}
@ -913,7 +981,7 @@ SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32
pInfo->qType = pPhyciNode->qType;
switch (pInfo->qType) {
case DYN_QTYPE_STB_HASH:
memcpy(&pInfo->stbJoin.basic, &pPhyciNode->stbJoin, sizeof(pPhyciNode->stbJoin));
TAOS_MEMCPY(&pInfo->stbJoin.basic, &pPhyciNode->stbJoin, sizeof(pPhyciNode->stbJoin));
pInfo->stbJoin.outputBlkId = pPhyciNode->node.pOutputDataBlockDesc->dataBlockId;
code = initSeqStbJoinTableHash(&pInfo->stbJoin.ctx.prev, pInfo->stbJoin.basic.batchFetch);
if (TSDB_CODE_SUCCESS != code) {
@ -934,12 +1002,14 @@ SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32
return pOperator;
_error:
if (pInfo != NULL) {
destroyDynQueryCtrlOperator(pInfo);
}
taosMemoryFree(pOperator);
pTaskInfo->code = code;
return NULL;
}

201
source/libs/executor/src/groupcacheoperator.c
View File

@ -30,9 +30,9 @@
static void removeGroupCacheFile(SGroupCacheFileInfo* pFileInfo) {
if (pFileInfo->fd.fd) {
taosCloseFile(&pFileInfo->fd.fd);
(void)taosCloseFile(&pFileInfo->fd.fd);
pFileInfo->fd.fd = NULL;
taosThreadMutexDestroy(&pFileInfo->fd.mutex);
(void)taosThreadMutexDestroy(&pFileInfo->fd.mutex);
}
pFileInfo->deleted = true;
}
@ -88,7 +88,7 @@ static void logGroupCacheExecInfo(SGroupCacheOperatorInfo* pGrpCacheOperator) {
static void freeSGcSessionCtx(void* p) {
SGcSessionCtx* pSession = p;
if (pSession->semInit) {
tsem_destroy(&pSession->waitSem);
(void)tsem_destroy(&pSession->waitSem);
}
}
@ -148,6 +148,10 @@ void blockDataDeepCleanup(SSDataBlock* pDataBlock) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
if (NULL == p) {
qError("fail to get %dth col in dataBlock, numOfCols:%d", i, (int32_t)numOfCols);
continue;
}
taosMemoryFreeClear(p->pData);
if (IS_VAR_DATA_TYPE(p->info.type)) {
taosMemoryFreeClear(p->varmeta.offset);
@ -196,10 +200,14 @@ static FORCE_INLINE int32_t initOpenCacheFile(SGroupCacheFileFd* pFileFd, char*
TdFilePtr newFd = taosOpenFile(filename, TD_FILE_CREATE|TD_FILE_READ|TD_FILE_WRITE|TD_FILE_AUTO_DEL);
//TdFilePtr newFd = taosOpenFile(filename, TD_FILE_CREATE|TD_FILE_READ|TD_FILE_WRITE);
if (NULL == newFd) {
return TAOS_SYSTEM_ERROR(errno);
QRY_ERR_RET(TAOS_SYSTEM_ERROR(errno));
}
pFileFd->fd = newFd;
taosThreadMutexInit(&pFileFd->mutex, NULL);
int32_t code = taosThreadMutexInit(&pFileFd->mutex, NULL);
if (code) {
qError("taosThreadMutexInit failed, code:%x", code);
QRY_ERR_RET(code);
}
qTrace("file path %s created", filename);
@ -218,13 +226,16 @@ static int32_t acquireFdFromFileCtx(SGcFileCacheCtx* pFileCtx, int32_t fileId, S
SGroupCacheFileInfo* pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId));
if (NULL == pTmp) {
sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%d", fileId);
(void)sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%d", fileId);
SGroupCacheFileInfo newFile = {0};
taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile));
if (taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile))) {
QRY_ERR_RET(terrno);
}
pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId));
if (NULL == pTmp) {
return TSDB_CODE_OUT_OF_MEMORY;
qError("fail to get file %d from pCacheFile", fileId);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
}
@ -240,14 +251,14 @@ static int32_t acquireFdFromFileCtx(SGcFileCacheCtx* pFileCtx, int32_t fileId, S
}
}
taosThreadMutexLock(&pTmp->fd.mutex);
(void)taosThreadMutexLock(&pTmp->fd.mutex);
*ppFd = &pTmp->fd;
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE void releaseFdToFileCtx(SGroupCacheFileFd* pFd) {
taosThreadMutexUnlock(&pFd->mutex);
(void)taosThreadMutexUnlock(&pFd->mutex);
}
static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamCtx* pCtx, SGcBlkBufInfo* pHead) {
@ -275,7 +286,7 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC
int64_t blkId = pHead->basic.blkId;
pHead = pHead->next;
taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId));
(void)taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId));
continue;
}
@ -295,7 +306,7 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC
int64_t blkId = pHead->basic.blkId;
pHead = pHead->next;
taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId));
(void)taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId));
continue;
}
@ -321,7 +332,7 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC
int64_t blkId = pHead->basic.blkId;
pHead = pHead->next;
taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId));
(void)taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId));
}
_return:
@ -330,7 +341,7 @@ _return:
taosHashRelease(pGrpHash, pGroup);
}
atomic_val_compare_exchange_32(&pGCache->blkCache.writeDownstreamId, pCtx->id, -1);
(void)atomic_val_compare_exchange_32(&pGCache->blkCache.writeDownstreamId, pCtx->id, -1);
return code;
}
@ -341,7 +352,8 @@ static int32_t addBlkToDirtyBufList(SGroupCacheOperatorInfo* pGCache, SGcDownstr
}
pBufInfo = taosHashGet(pCache->pDirtyBlk, &pBufInfo->basic.blkId, sizeof(pBufInfo->basic.blkId));
if (NULL == pBufInfo) {
return TSDB_CODE_OUT_OF_MEMORY;
qError("fail to get blk %" PRId64 " from pCache->pDirtyBlk", pBufInfo->basic.blkId);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t code = TSDB_CODE_SUCCESS;
SGcBlkBufInfo* pWriteHead = NULL;
@ -424,7 +436,7 @@ static int32_t addBlkToBufCache(struct SOperatorInfo* pOperator, SSDataBlock* pB
qError("group cache add block to cache failed, size:%" PRId64, bufSize);
return TSDB_CODE_OUT_OF_MEMORY;
}
blockDataToBuf(pBufInfo->pBuf, pBlock);
QRY_ERR_RET(blockDataToBuf(pBufInfo->pBuf, pBlock));
SGcFileCacheCtx* pFileCtx = pGCache->batchFetch ? &pCtx->fileCtx : &pGroup->pVgCtx->fileCtx;
@ -449,6 +461,10 @@ void blockDataDeepClear(SSDataBlock* pDataBlock) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
if (NULL == p) {
qError("fail to get %d col from pDataBlock, numOfCols:%d", i, (int32_t)numOfCols);
continue;
}
p->pData = NULL;
if (IS_VAR_DATA_TYPE(p->info.type)) {
p->varmeta.offset = NULL;
@ -473,8 +489,9 @@ static int32_t buildGroupCacheBaseBlock(SSDataBlock** ppDst, SSDataBlock* pSrc)
taosMemoryFree(*ppDst);
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy(&(*ppDst)->info, &pSrc->info, sizeof(pSrc->info));
TAOS_MEMCPY(&(*ppDst)->info, &pSrc->info, sizeof(pSrc->info));
blockDataDeepClear(*ppDst);
return TSDB_CODE_SUCCESS;
}
@ -490,11 +507,17 @@ static int32_t acquireBaseBlockFromList(SGcDownstreamCtx* pCtx, SSDataBlock** pp
return TSDB_CODE_SUCCESS;
}
static void releaseBaseBlockToList(SGcDownstreamCtx* pCtx, SSDataBlock* pBlock) {
static int32_t releaseBaseBlockToList(SGcDownstreamCtx* pCtx, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
blockDataDeepCleanup(pBlock);
taosWLockLatch(&pCtx->blkLock);
taosArrayPush(pCtx->pFreeBlock, &pBlock);
if (NULL == taosArrayPush(pCtx->pFreeBlock, &pBlock)) {
code = terrno;
}
taosWUnLockLatch(&pCtx->blkLock);
return code;
}
@ -573,14 +596,15 @@ static int32_t retrieveBlkFromBufCache(SGroupCacheOperatorInfo* pGCache, SGroupC
return code;
}
taosHashPut(pCache->pReadBlk, &sessionId, sizeof(sessionId), ppRes, POINTER_BYTES);
QRY_ERR_RET(taosHashPut(pCache->pReadBlk, &sessionId, sizeof(sessionId), ppRes, POINTER_BYTES));
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE void initGcVgroupCtx(SOperatorInfo* pOperator, SGcVgroupCtx* pVgCtx, int32_t downstreamId, int32_t vgId, SArray* pTbList) {
pVgCtx->pTbList = pTbList;
pVgCtx->id = vgId;
snprintf(pVgCtx->fileCtx.baseFilename, sizeof(pVgCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d_%d",
(void)snprintf(pVgCtx->fileCtx.baseFilename, sizeof(pVgCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d_%d",
tsTempDir, getpid(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, downstreamId, vgId);
pVgCtx->fileCtx.baseFilename[sizeof(pVgCtx->fileCtx.baseFilename) - 1] = 0;
@ -594,15 +618,27 @@ static int32_t addNewGroupToVgHash(SOperatorInfo* pOperator, SSHashObj* pHash, S
if (NULL == pList) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(pList, pNew);
if (NULL == taosArrayPush(pList, pNew)) {
QRY_ERR_RET(terrno);
}
SGcVgroupCtx vgCtx = {0};
initGcVgroupCtx(pOperator, &vgCtx, pNew->pGroup->downstreamIdx, pNew->vgId, pList);
tSimpleHashPut(pHash, &pNew->vgId, sizeof(pNew->vgId), &vgCtx, sizeof(vgCtx));
QRY_ERR_RET(tSimpleHashPut(pHash, &pNew->vgId, sizeof(pNew->vgId), &vgCtx, sizeof(vgCtx)));
pNew->pGroup->pVgCtx = tSimpleHashGet(pHash, &pNew->vgId, sizeof(pNew->vgId));
if (NULL == pNew->pGroup->pVgCtx) {
qError("fail to get vg %d ctx from vgHash", pNew->vgId);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
return TSDB_CODE_SUCCESS;
}
taosArrayPush(pVgCtx->pTbList, pNew);
if (NULL == taosArrayPush(pVgCtx->pTbList, pNew)) {
QRY_ERR_RET(terrno);
}
return TSDB_CODE_SUCCESS;
}
@ -620,6 +656,11 @@ static FORCE_INLINE int32_t appendNewGroupToDownstream(struct SOperatorInfo* pOp
for (int32_t i = 0; i < num; ++i) {
SGcNewGroupInfo* pNew = taosArrayGet(pCtx->pNewGrpList, i);
if (NULL == pNew) {
qError("fail to get vg %d SGcNewGroupInfo from pNewGrpList", i);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (!pGCache->batchFetch) {
code = addNewGroupToVgHash(pOperator, pCtx->pVgTbHash, pNew);
if (code) {
@ -672,7 +713,9 @@ static FORCE_INLINE int32_t getBlkFromDownstreamOperator(struct SOperatorInfo* p
if (code) {
return code;
}
taosArrayPush(pGCache->pDownstreams[downstreamIdx].pFreeBlock, &pGCache->pDownstreams[downstreamIdx].pBaseBlock);
if (NULL == taosArrayPush(pGCache->pDownstreams[downstreamIdx].pFreeBlock, &pGCache->pDownstreams[downstreamIdx].pBaseBlock)) {
QRY_ERR_RET(terrno);
}
}
}
@ -681,26 +724,36 @@ static FORCE_INLINE int32_t getBlkFromDownstreamOperator(struct SOperatorInfo* p
return code;
}
static void notifyWaitingSessions(SArray* pWaitQueue) {
static int32_t notifyWaitingSessions(SArray* pWaitQueue) {
if (NULL == pWaitQueue || taosArrayGetSize(pWaitQueue) <= 0) {
return;
return TSDB_CODE_SUCCESS;
}
int32_t n = taosArrayGetSize(pWaitQueue);
for (int32_t i = 0; i < n; ++i) {
SGcSessionCtx* pSession = taosArrayGetP(pWaitQueue, i);
tsem_post(&pSession->waitSem);
if (NULL == pSession) {
qError("fail to get %d SGcSessionCtx in pWaitQueue, total:%d", i, n);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
QRY_ERR_RET(tsem_post(&pSession->waitSem));
}
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE void handleGroupFetchDone(SGroupCacheData* pGroup) {
static FORCE_INLINE int32_t handleGroupFetchDone(SGroupCacheData* pGroup) {
int32_t code = TSDB_CODE_SUCCESS;
pGroup->pBlock = NULL;
atomic_store_8((int8_t*)&pGroup->fetchDone, true);
taosThreadMutexLock(&pGroup->mutex);
notifyWaitingSessions(pGroup->waitQueue);
(void)taosThreadMutexLock(&pGroup->mutex);
code = notifyWaitingSessions(pGroup->waitQueue);
taosArrayClear(pGroup->waitQueue);
taosThreadMutexUnlock(&pGroup->mutex);
(void)taosThreadMutexUnlock(&pGroup->mutex);
return code;
}
static int32_t addFileRefTableNum(SGcFileCacheCtx* pFileCtx, int32_t fileId, int32_t downstreamId, int32_t vgId) {
@ -714,14 +767,15 @@ static int32_t addFileRefTableNum(SGcFileCacheCtx* pFileCtx, int32_t fileId, int
SGroupCacheFileInfo* pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId));
if (NULL == pTmp) {
sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%u", fileId);
(void)sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%u", fileId);
SGroupCacheFileInfo newFile = {0};
newFile.groupNum = 1;
taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile));
QRY_ERR_RET(taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile)));
pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId));
if (NULL == pTmp) {
return TSDB_CODE_OUT_OF_MEMORY;
qError("fail to get file %d in pCacheFile", fileId);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
} else {
pTmp->groupNum++;
@ -746,7 +800,7 @@ static int32_t handleVgroupTableFetchDone(SGcDownstreamCtx* pCtx, SGroupCacheDat
if (NULL == pNew || pNew->uid == uid) {
break;
}
handleGroupFetchDone(pNew->pGroup);
QRY_ERR_RET(handleGroupFetchDone(pNew->pGroup));
}
groupCacheSwitchNewFile(&pGroup->pVgCtx->fileCtx, pGroup->downstreamIdx, pGroup->vgId, true);
@ -764,15 +818,21 @@ static int32_t handleVgroupTableFetchDone(SGcDownstreamCtx* pCtx, SGroupCacheDat
}
static FORCE_INLINE void initNewGroupData(SGcDownstreamCtx* pCtx, SGroupCacheData* pGroup, int32_t downstreamIdx, int32_t vgId, bool batchFetch, bool needCache) {
taosThreadMutexInit(&pGroup->mutex, NULL);
static FORCE_INLINE int32_t initNewGroupData(SGcDownstreamCtx* pCtx, SGroupCacheData* pGroup, int32_t downstreamIdx, int32_t vgId, bool batchFetch, bool needCache) {
QRY_ERR_RET(taosThreadMutexInit(&pGroup->mutex, NULL));
pGroup->downstreamIdx = downstreamIdx;
pGroup->vgId = vgId;
pGroup->fileId = -1;
pGroup->blkList.pList = taosArrayInit(10, sizeof(SGcBlkBufBasic));
if (NULL == pGroup->blkList.pList) {
QRY_ERR_RET(terrno);
}
pGroup->startOffset = -1;
pGroup->needCache = needCache;
pGroup->pVgCtx = tSimpleHashGet(pCtx->pVgTbHash, &pGroup->vgId, sizeof(pGroup->vgId));
return TSDB_CODE_SUCCESS;
}
static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam* pParam, SGroupCacheData** ppGrp, int32_t vgId, int64_t uid) {
@ -801,7 +861,7 @@ static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam*
if (NULL == *ppGrp) {
return TSDB_CODE_OUT_OF_MEMORY;
}
initNewGroupData(pCtx, *ppGrp, pParam->downstreamIdx, vgId, pGCache->batchFetch, pGcParam->needCache);
QRY_ERR_RET(initNewGroupData(pCtx, *ppGrp, pParam->downstreamIdx, vgId, pGCache->batchFetch, pGcParam->needCache));
qDebug("new group %" PRIu64 " initialized, downstreamIdx:%d, vgId:%d, needCache:%d", uid, pParam->downstreamIdx, vgId, pGcParam->needCache);
@ -829,7 +889,9 @@ static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam*
static int32_t addBlkToGroupCache(bool batchFetch, SGroupCacheData* pGroup, SGcBlkBufInfo* pNewBlk, int64_t* pIdx) {
taosWLockLatch(&pGroup->blkList.lock);
taosArrayPush(pGroup->blkList.pList, &pNewBlk->basic);
if (NULL == taosArrayPush(pGroup->blkList.pList, &pNewBlk->basic)) {
QRY_ERR_RET(terrno);
}
*pIdx = taosArrayGetSize(pGroup->blkList.pList) - 1;
taosWUnLockLatch(&pGroup->blkList.lock);
@ -889,7 +951,7 @@ static int32_t handleGroupCacheRetrievedBlk(struct SOperatorInfo* pOperator, SSD
pGroup->pBlock = pBlock;
}
notifyWaitingSessions(pGroup->waitQueue);
QRY_ERR_RET(notifyWaitingSessions(pGroup->waitQueue));
if (pGroup == pSession->pGroupData) {
if (pGroup->needCache) {
pSession->lastBlkId = newBlkIdx;
@ -909,7 +971,7 @@ static int32_t handleDownstreamFetchDone(struct SOperatorInfo* pOperator, SGcSes
SHashObj* pGrpHash = pGCache->globalGrp ? pGCache->pGrpHash : pCtx->pGrpHash;
SGroupCacheData* pGroup = NULL;
while (NULL != (pGroup = taosHashIterate(pGrpHash, pGroup))) {
handleGroupFetchDone(pGroup);
QRY_ERR_RET(handleGroupFetchDone(pGroup));
}
pCtx->fetchDone = true;
} else {
@ -920,7 +982,7 @@ static int32_t handleDownstreamFetchDone(struct SOperatorInfo* pOperator, SGcSes
uidNum = taosArrayGetSize(pVgCtx->pTbList);
for (int32_t i = 0; i < uidNum; ++i) {
SGcNewGroupInfo* pNew = taosArrayGet(pVgCtx->pTbList, i);
handleGroupFetchDone(pNew->pGroup);
QRY_ERR_RET(handleGroupFetchDone(pNew->pGroup));
}
taosArrayClear(pVgCtx->pTbList);
}
@ -961,8 +1023,8 @@ static int32_t getCacheBlkFromDownstreamOperator(struct SOperatorInfo* pOperator
}
SGcSessionCtx* pWaitCtx = *ppWaitCtx;
pWaitCtx->newFetch = true;
taosHashRemove(pCtx->pWaitSessions, pSessionId, sizeof(*pSessionId));
tsem_post(&pWaitCtx->waitSem);
(void)taosHashRemove(pCtx->pWaitSessions, pSessionId, sizeof(*pSessionId));
QRY_ERR_RET(tsem_post(&pWaitCtx->waitSem));
return code;
}
@ -1028,36 +1090,47 @@ static int32_t groupCacheSessionWait(struct SOperatorInfo* pOperator, SGcDownstr
SGroupCacheOperatorInfo* pGCache = pOperator->info;
SGroupCacheData* pGroup = pSession->pGroupData;
int32_t code = TSDB_CODE_SUCCESS;
bool inLock = true;
if (NULL == pGroup->waitQueue) {
pGroup->waitQueue = taosArrayInit(1, POINTER_BYTES);
if (NULL == pGroup->waitQueue) {
taosThreadMutexUnlock(&pSession->pGroupData->mutex);
return TSDB_CODE_OUT_OF_MEMORY;
QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
}
taosArrayPush(pGroup->waitQueue, &pSession);
if (NULL == taosArrayPush(pGroup->waitQueue, &pSession)) {
QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
if (!pSession->semInit) {
tsem_init(&pSession->waitSem, 0, 0);
QRY_ERR_JRET(tsem_init(&pSession->waitSem, 0, 0));
pSession->semInit = true;
}
taosThreadMutexUnlock(&pSession->pGroupData->mutex);
(void)taosThreadMutexUnlock(&pSession->pGroupData->mutex);
inLock = false;
taosHashPut(pCtx->pWaitSessions, &sessionId, sizeof(sessionId), &pSession, POINTER_BYTES);
QRY_ERR_JRET(taosHashPut(pCtx->pWaitSessions, &sessionId, sizeof(sessionId), &pSession, POINTER_BYTES));
tsem_wait(&pSession->waitSem);
(void)tsem_wait(&pSession->waitSem);
if (pSession->newFetch) {
pSession->newFetch = false;
return getCacheBlkFromDownstreamOperator(pOperator, pCtx, sessionId, pSession, ppRes);
}
taosHashRemove(pCtx->pWaitSessions, &sessionId, sizeof(sessionId));
(void)taosHashRemove(pCtx->pWaitSessions, &sessionId, sizeof(sessionId));
bool got = false;
return getBlkFromSessionCacheImpl(pOperator, sessionId, pSession, ppRes, &got);
_return:
if (inLock) {
(void)taosThreadMutexUnlock(&pSession->pGroupData->mutex);
}
return code;
}
@ -1077,7 +1150,7 @@ static int32_t getBlkFromSessionCache(struct SOperatorInfo* pOperator, int64_t s
if ((atomic_load_64(&pCtx->fetchSessionId) == sessionId)
|| (-1 == atomic_val_compare_exchange_64(&pCtx->fetchSessionId, -1, sessionId))) {
if (locked) {
taosThreadMutexUnlock(&pSession->pGroupData->mutex);
(void)taosThreadMutexUnlock(&pSession->pGroupData->mutex);
locked = false;
}
@ -1095,7 +1168,7 @@ static int32_t getBlkFromSessionCache(struct SOperatorInfo* pOperator, int64_t s
break;
}
taosThreadMutexLock(&pSession->pGroupData->mutex);
(void)taosThreadMutexLock(&pSession->pGroupData->mutex);
locked = true;
};
@ -1103,7 +1176,7 @@ static int32_t getBlkFromSessionCache(struct SOperatorInfo* pOperator, int64_t s
_return:
if (locked) {
taosThreadMutexUnlock(&pSession->pGroupData->mutex);
(void)taosThreadMutexUnlock(&pSession->pGroupData->mutex);
}
return code;
@ -1165,6 +1238,10 @@ static int32_t initGroupCacheSession(struct SOperatorInfo* pOperator, SOperatorP
}
*ppSession = taosHashGet(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
if (NULL == *ppSession) {
qError("fail to get session %" PRId64 " from pSessions", pGcParam->sessionId);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
return TSDB_CODE_SUCCESS;
}
@ -1187,15 +1264,15 @@ static int32_t getBlkFromGroupCache(struct SOperatorInfo* pOperator, SSDataBlock
} else if (pSession->pGroupData->needCache) {
SSDataBlock** ppBlock = taosHashGet(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
if (ppBlock) {
releaseBaseBlockToList(pCtx, *ppBlock);
taosHashRemove(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
QRY_ERR_RET(releaseBaseBlockToList(pCtx, *ppBlock));
(void)taosHashRemove(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
}
}
code = getBlkFromSessionCache(pOperator, pGcParam->sessionId, pSession, ppRes);
if (NULL == *ppRes) {
qDebug("session %" PRId64 " in downstream %d total got %" PRId64 " rows", pGcParam->sessionId, pCtx->id, pSession->resRows);
taosHashRemove(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
(void)taosHashRemove(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
} else {
pSession->resRows += (*ppRes)->info.rows;
qDebug("session %" PRId64 " in downstream %d got %" PRId64 " rows in one block", pGcParam->sessionId, pCtx->id, (*ppRes)->info.rows);
@ -1241,7 +1318,7 @@ static void freeRemoveGroupCacheData(void* p) {
taosArrayDestroy(pGroup->waitQueue);
taosArrayDestroy(pGroup->blkList.pList);
taosThreadMutexDestroy(&pGroup->mutex);
(void)taosThreadMutexDestroy(&pGroup->mutex);
qTrace("group removed");
}
@ -1271,7 +1348,7 @@ static int32_t initGroupCacheDownstreamCtx(SOperatorInfo* pOperator) {
int32_t defaultVg = 0;
SGcVgroupCtx vgCtx = {0};
initGcVgroupCtx(pOperator, &vgCtx, pCtx->id, defaultVg, NULL);
tSimpleHashPut(pCtx->pVgTbHash, &defaultVg, sizeof(defaultVg), &vgCtx, sizeof(vgCtx));
QRY_ERR_RET(tSimpleHashPut(pCtx->pVgTbHash, &defaultVg, sizeof(defaultVg), &vgCtx, sizeof(vgCtx)));
}
pCtx->pNewGrpList = taosArrayInit(10, sizeof(SGcNewGroupInfo));
@ -1302,7 +1379,7 @@ static int32_t initGroupCacheDownstreamCtx(SOperatorInfo* pOperator) {
return TSDB_CODE_OUT_OF_MEMORY;
}
snprintf(pCtx->fileCtx.baseFilename, sizeof(pCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d",
(void)snprintf(pCtx->fileCtx.baseFilename, sizeof(pCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d",
tsTempDir, getpid(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, pCtx->id);
pCtx->fileCtx.baseFilename[sizeof(pCtx->fileCtx.baseFilename) - 1] = 0;
pCtx->fileCtx.baseNameLen = strlen(pCtx->fileCtx.baseFilename);

4
source/libs/executor/src/hashjoin.c
View File

@ -90,7 +90,7 @@ int32_t hLeftJoinHandleSeqRowRemains(struct SOperatorInfo* pOperator, SHJoinOper
while (!allFetched) {
hJoinAppendResToBlock(pOperator, pJoin->midBlk, &allFetched);
if (pJoin->midBlk->info.rows > 0) {
doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL);
HJ_ERR_RET(doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL));
if (pJoin->midBlk->info.rows > 0) {
pCtx->readMatch = true;
HJ_ERR_RET(hJoinCopyMergeMidBlk(pCtx, &pJoin->midBlk, &pJoin->finBlk));
@ -170,7 +170,7 @@ int32_t hLeftJoinHandleSeqProbeRows(struct SOperatorInfo* pOperator, SHJoinOpera
while (!allFetched) {
hJoinAppendResToBlock(pOperator, pJoin->midBlk, &allFetched);
if (pJoin->midBlk->info.rows > 0) {
doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL);
HJ_ERR_RET(doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL));
if (pJoin->midBlk->info.rows > 0) {
pCtx->readMatch = true;
HJ_ERR_RET(hJoinCopyMergeMidBlk(pCtx, &pJoin->midBlk, &pJoin->finBlk));

84
source/libs/executor/src/hashjoinoperator.c
View File

@ -238,10 +238,12 @@ static int32_t hJoinInitValColsInfo(SHJoinTableCtx* pTable, SNodeList* pList) {
if (NULL == pTable->valVarCols) {
pTable->valVarCols = taosArrayInit(pTable->valNum, sizeof(int32_t));
if (NULL == pTable->valVarCols) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
}
taosArrayPush(pTable->valVarCols, &i);
if (NULL == taosArrayPush(pTable->valVarCols, &i)) {
return terrno;
}
}
pTable->valCols[i].bytes = pColNode->node.resType.bytes;
if (!pTable->valCols[i].keyCol && !pTable->valCols[i].vardata) {
@ -332,7 +334,7 @@ static int32_t hJoinInitTableInfo(SHJoinOperatorInfo* pJoin, SHashJoinPhysiNode*
return code;
}
memcpy(&pTable->inputStat, pStat, sizeof(*pStat));
TAOS_MEMCPY(&pTable->inputStat, pStat, sizeof(*pStat));
HJ_ERR_RET(hJoinInitPrimExprCtx(pTable->primExpr, &pTable->primCtx, pTable));
@ -416,7 +418,9 @@ static FORCE_INLINE int32_t hJoinAddPageToBufs(SArray* pRowBufs) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(pRowBufs, &page);
if (NULL == taosArrayPush(pRowBufs, &page)) {
return terrno;
}
return TSDB_CODE_SUCCESS;
}
@ -464,12 +468,21 @@ static void hJoinDestroyKeyHash(SSHashObj** ppHash) {
*ppHash = NULL;
}
static FORCE_INLINE char* hJoinRetrieveColDataFromRowBufs(SArray* pRowBufs, SBufRowInfo* pRow) {
static FORCE_INLINE int32_t hJoinRetrieveColDataFromRowBufs(SArray* pRowBufs, SBufRowInfo* pRow, char** ppData) {
*ppData = NULL;
if ((uint16_t)-1 == pRow->pageId) {
return NULL;
return TSDB_CODE_SUCCESS;
}
SBufPageInfo *pPage = taosArrayGet(pRowBufs, pRow->pageId);
return pPage->data + pRow->offset;
if (NULL == pPage) {
qError("fail to get %d page, total:%d", pRow->pageId, (int32_t)taosArrayGetSize(pRowBufs));
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
*ppData = pPage->data + pRow->offset;
return TSDB_CODE_SUCCESS;
}
static int32_t hJoinCopyResRowsToBlock(SHJoinOperatorInfo* pJoin, int32_t rowNum, SBufRowInfo* pStart, SSDataBlock* pRes) {
@ -479,9 +492,11 @@ static int32_t hJoinCopyResRowsToBlock(SHJoinOperatorInfo* pJoin, int32_t rowNum
int32_t probeIdx = 0;
SBufRowInfo* pRow = pStart;
int32_t code = 0;
char* pData = NULL;
for (int32_t r = 0; r < rowNum; ++r) {
char* pData = hJoinRetrieveColDataFromRowBufs(pJoin->pRowBufs, pRow);
HJ_ERR_RET(hJoinRetrieveColDataFromRowBufs(pJoin->pRowBufs, pRow, &pData));
char* pValData = pData + pBuild->valBitMapSize;
char* pKeyData = pProbe->keyData;
buildIdx = buildValIdx = probeIdx = 0;
@ -544,7 +559,7 @@ int32_t hJoinCopyNMatchRowsToBlock(SHJoinOperatorInfo* pJoin, SSDataBlock* pRes,
SColumnInfoData* pSrc = taosArrayGet(pJoin->ctx.pProbeData->pDataBlock, pProbe->valCols[probeIdx].srcSlot);
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pProbe->valCols[probeIdx].dstSlot);
colDataAssignNRows(pDst, pRes->info.rows, pSrc, startIdx, rows);
QRY_ERR_RET(colDataAssignNRows(pDst, pRes->info.rows, pSrc, startIdx, rows));
probeIdx++;
}
@ -606,11 +621,11 @@ bool hJoinCopyKeyColsDataToBuf(SHJoinTableCtx* pTable, int32_t rowIdx, size_t *p
}
if (pTable->keyCols[i].vardata) {
pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx];
memcpy(pTable->keyBuf + bufLen, pData, varDataTLen(pData));
TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, varDataTLen(pData));
bufLen += varDataTLen(pData);
} else {
pData = pTable->keyCols[i].data + pTable->keyCols[i].bytes * rowIdx;
memcpy(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes);
TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes);
bufLen += pTable->keyCols[i].bytes;
}
}
@ -627,6 +642,10 @@ bool hJoinCopyKeyColsDataToBuf(SHJoinTableCtx* pTable, int32_t rowIdx, size_t *p
static int32_t hJoinSetKeyColsData(SSDataBlock* pBlock, SHJoinTableCtx* pTable) {
for (int32_t i = 0; i < pTable->keyNum; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->keyCols[i].srcSlot);
if (NULL == pCol) {
qError("fail to get %d col, total:%d", pTable->keyCols[i].srcSlot, (int32_t)taosArrayGetSize(pBlock->pDataBlock));
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pTable->keyCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) {
qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->keyCols[i].srcSlot, pCol->info.type, pTable->keyCols[i].vardata);
return TSDB_CODE_INVALID_PARA;
@ -654,6 +673,10 @@ static int32_t hJoinSetValColsData(SSDataBlock* pBlock, SHJoinTableCtx* pTable)
continue;
}
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->valCols[i].srcSlot);
if (NULL == pCol) {
qError("fail to get %d col, total:%d", pTable->valCols[i].srcSlot, (int32_t)taosArrayGetSize(pBlock->pDataBlock));
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pTable->valCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) {
qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->valCols[i].srcSlot, pCol->info.type, pTable->valCols[i].vardata);
return TSDB_CODE_INVALID_PARA;
@ -683,7 +706,7 @@ static FORCE_INLINE void hJoinCopyValColsDataToBuf(SHJoinTableCtx* pTable, int32
char *pData = NULL;
size_t bufLen = pTable->valBitMapSize;
memset(pTable->valData, 0, pTable->valBitMapSize);
TAOS_MEMSET(pTable->valData, 0, pTable->valBitMapSize);
for (int32_t i = 0, m = 0; i < pTable->valNum; ++i) {
if (pTable->valCols[i].keyCol) {
continue;
@ -693,7 +716,7 @@ static FORCE_INLINE void hJoinCopyValColsDataToBuf(SHJoinTableCtx* pTable, int32
colDataSetNull_f(pTable->valData, m);
} else {
pData = pTable->valCols[i].data + pTable->valCols[i].offset[rowIdx];
memcpy(pTable->valData + bufLen, pData, varDataTLen(pData));
TAOS_MEMCPY(pTable->valData + bufLen, pData, varDataTLen(pData));
bufLen += varDataTLen(pData);
}
} else {
@ -701,7 +724,7 @@ static FORCE_INLINE void hJoinCopyValColsDataToBuf(SHJoinTableCtx* pTable, int32
colDataSetNull_f(pTable->valData, m);
} else {
pData = pTable->valCols[i].data + pTable->valCols[i].bytes * rowIdx;
memcpy(pTable->valData + bufLen, pData, pTable->valCols[i].bytes);
TAOS_MEMCPY(pTable->valData + bufLen, pData, pTable->valCols[i].bytes);
bufLen += pTable->valCols[i].bytes;
}
}
@ -1009,7 +1032,11 @@ static SSDataBlock* hJoinMainProcess(struct SOperatorInfo* pOperator) {
}
if (pRes->info.rows > 0 && pJoin->pFinFilter != NULL) {
doFilter(pRes, pJoin->pFinFilter, NULL);
code = doFilter(pRes, pJoin->pFinFilter, NULL);
if (code) {
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
}
if (pRes->info.rows > 0) {
return pRes;
@ -1037,7 +1064,11 @@ static SSDataBlock* hJoinMainProcess(struct SOperatorInfo* pOperator) {
}
if (pRes->info.rows > 0 && pJoin->pFinFilter != NULL) {
doFilter(pRes, pJoin->pFinFilter, NULL);
code = doFilter(pRes, pJoin->pFinFilter, NULL);
if (code) {
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
}
if (pRes->info.rows > 0) {
@ -1119,13 +1150,25 @@ static uint32_t hJoinGetFinBlkCapacity(SHJoinOperatorInfo* pJoin, SHashJoinPhysi
int32_t hJoinInitResBlocks(SHJoinOperatorInfo* pJoin, SHashJoinPhysiNode* pJoinNode) {
pJoin->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc);
if (NULL == pJoin->finBlk) {
QRY_ERR_RET(terrno);
}
ASSERT(pJoinNode->node.pOutputDataBlockDesc->totalRowSize > 0);
blockDataEnsureCapacity(pJoin->finBlk, hJoinGetFinBlkCapacity(pJoin, pJoinNode));
int32_t code = blockDataEnsureCapacity(pJoin->finBlk, hJoinGetFinBlkCapacity(pJoin, pJoinNode));
if (TSDB_CODE_SUCCESS != code) {
QRY_ERR_RET(terrno);
}
if (NULL != pJoin->pPreFilter) {
pJoin->midBlk = createOneDataBlock(pJoin->finBlk, false);
blockDataEnsureCapacity(pJoin->midBlk, pJoin->finBlk->info.capacity);
if (NULL == pJoin->finBlk) {
QRY_ERR_RET(terrno);
}
code = blockDataEnsureCapacity(pJoin->midBlk, pJoin->finBlk->info.capacity);
if (TSDB_CODE_SUCCESS != code) {
QRY_ERR_RET(terrno);
}
}
pJoin->blkThreshold = pJoin->finBlk->info.capacity * HJOIN_BLK_THRESHOLD_RATIO;
@ -1152,8 +1195,8 @@ SOperatorInfo* createHashJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t n
setOperatorInfo(pOperator, "HashJoinOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, false, OP_NOT_OPENED, pInfo, pTaskInfo);
hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0]);
hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1]);
HJ_ERR_JRET(hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0]));
HJ_ERR_JRET(hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1]));
hJoinSetBuildAndProbeTable(pInfo, pJoinNode);
@ -1183,6 +1226,7 @@ SOperatorInfo* createHashJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t n
return pOperator;
_return:
if (pInfo != NULL) {
destroyHashJoinOperator(pInfo);
}

333
source/libs/executor/src/mergejoin.c
View File

@ -72,9 +72,15 @@ int32_t mWinJoinDumpGrpCache(SMJoinWindowCtx* pCtx) {
if ((!pCtx->seqWinGrp) && 0 == cache->grpIdx && probeRows * buildTotalRows <= rowsLeft) {
SMJoinGrpRows* pFirstBuild = taosArrayGet(cache->grps, 0);
if (NULL == pFirstBuild) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pFirstBuild->readIdx == pFirstBuild->beginIdx) {
for (; cache->grpIdx < buildGrpNum; ++cache->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(cache->grps, cache->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
buildGrp->readIdx = buildGrp->beginIdx;
}
@ -89,6 +95,9 @@ int32_t mWinJoinDumpGrpCache(SMJoinWindowCtx* pCtx) {
probeGrp->endIdx = probeGrp->readIdx;
for (; cache->grpIdx < buildGrpNum && rowsLeft > 0; ++cache->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(cache->grps, cache->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
@ -134,7 +143,8 @@ static int32_t mOuterJoinHashFullCart(SMJoinMergeCtx* pCtx) {
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (build->grpRowIdx >= 0) {
bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build);
bool contLoop = false;
MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop));
if (build->grpRowIdx < 0) {
probeGrp->readIdx++;
}
@ -171,7 +181,8 @@ static int32_t mOuterJoinHashFullCart(SMJoinMergeCtx* pCtx) {
}
build->grpRowIdx = 0;
bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build);
bool contLoop = false;
MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop));
if (!contLoop) {
if (build->grpRowIdx < 0) {
probeGrp->readIdx++;
@ -193,15 +204,27 @@ static int32_t mOuterJoinMergeFullCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t buildGrpNum = taosArrayGetSize(build->eqGrps);
int32_t probeRows = GRP_REMAIN_ROWS(probeGrp);
int32_t probeEndIdx = probeGrp->endIdx;
if (0 == build->grpIdx && probeRows * build->grpTotalRows <= rowsLeft) {
SMJoinGrpRows* pFirstBuild = taosArrayGet(build->eqGrps, 0);
if (NULL == pFirstBuild) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pFirstBuild->readIdx == pFirstBuild->beginIdx) {
for (; build->grpIdx < buildGrpNum; ++build->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
buildGrp->readIdx = buildGrp->beginIdx;
}
@ -215,6 +238,9 @@ static int32_t mOuterJoinMergeFullCart(SMJoinMergeCtx* pCtx) {
probeGrp->endIdx = probeGrp->readIdx;
for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
@ -254,6 +280,10 @@ static int32_t mOuterJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t buildGrpNum = taosArrayGetSize(build->eqGrps);
int32_t probeEndIdx = probeGrp->endIdx;
int32_t rowsLeft = pCtx->midBlk->info.capacity;
@ -274,6 +304,10 @@ static int32_t mOuterJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (startRowIdx < 0) {
startRowIdx = buildGrp->readIdx;
}
@ -351,13 +385,17 @@ static int32_t mOuterJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, bool* contLoop)
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t startRowIdx = 0;
//blockDataCleanup(pCtx->midBlk);
do {
startRowIdx = build->grpRowIdx;
mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build);
MJ_ERR_RET(mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build, NULL));
if (pCtx->midBlk->info.rows > 0) {
if (build->rowBitmapSize > 0) {
@ -406,6 +444,10 @@ static int32_t mOuterJoinHashSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
bool contLoop = false;
if (build->grpRowIdx >= 0) {
@ -500,7 +542,15 @@ static bool mLeftJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoi
if (buildGot) {
SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId);
if (NULL == pProbeCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId);
if (NULL == pBuildCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) {
pJoin->build->blkRowIdx = pJoin->build->blk->info.rows;
buildGot = false;
@ -649,15 +699,27 @@ static int32_t mInnerJoinMergeCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t buildGrpNum = taosArrayGetSize(build->eqGrps);
int32_t probeRows = GRP_REMAIN_ROWS(probeGrp);
int32_t probeEndIdx = probeGrp->endIdx;
if (0 == build->grpIdx && probeRows * build->grpTotalRows <= rowsLeft) {
SMJoinGrpRows* pFirstBuild = taosArrayGet(build->eqGrps, 0);
if (NULL == pFirstBuild) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pFirstBuild->readIdx == pFirstBuild->beginIdx) {
for (; build->grpIdx < buildGrpNum; ++build->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
buildGrp->readIdx = buildGrp->beginIdx;
}
@ -671,6 +733,9 @@ static int32_t mInnerJoinMergeCart(SMJoinMergeCtx* pCtx) {
probeGrp->endIdx = probeGrp->readIdx;
for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
@ -711,9 +776,13 @@ static int32_t mInnerJoinHashCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (build->grpRowIdx >= 0) {
bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build);
bool contLoop = false;
MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop));
if (build->grpRowIdx < 0) {
probeGrp->readIdx++;
}
@ -734,7 +803,8 @@ static int32_t mInnerJoinHashCart(SMJoinMergeCtx* pCtx) {
if (NULL != pGrp) {
build->pHashCurGrp = *pGrp;
build->grpRowIdx = 0;
bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build);
bool contLoop = false;
MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop));
if (!contLoop) {
if (build->grpRowIdx < 0) {
probeGrp->readIdx++;
@ -772,7 +842,15 @@ static bool mInnerJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJo
if (buildGot) {
SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId);
if (NULL == pProbeCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId);
if (NULL == pBuildCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) {
pJoin->build->blkRowIdx = pJoin->build->blk->info.rows;
buildGot = false;
@ -903,6 +981,10 @@ static int32_t mFullJoinMergeCart(SMJoinMergeCtx* pCtx) {
static FORCE_INLINE int32_t mFullJoinOutputHashRow(SMJoinMergeCtx* pCtx, SMJoinHashGrpRows* pGrpRows, int32_t idx) {
SMJoinGrpRows grp = {0};
SMJoinRowPos* pPos = taosArrayGet(pGrpRows->pRows, idx);
if (NULL == pPos) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
grp.blk = pPos->pBlk;
grp.readIdx = pPos->pos;
grp.endIdx = pPos->pos;
@ -1042,6 +1124,10 @@ static int32_t mFullJoinHandleMergeGrpRemains(SMJoinMergeCtx* pCtx) {
grpDone = false;
SMJoinGrpRows* pGrpRows = taosArrayGet(build->eqGrps, pNMatch->grpIdx);
if (NULL == pGrpRows) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pGrpRows->allRowsMatch) {
continue;
}
@ -1283,7 +1369,7 @@ static int32_t mSemiJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, SMJoinGrpRows* p
do {
blockDataCleanup(pCtx->midBlk);
mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build);
MJ_ERR_RET(mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build, NULL));
if (pCtx->midBlk->info.rows > 0) {
MJ_ERR_RET(mJoinFilterAndKeepSingleRow(pCtx->midBlk, pCtx->pJoin->pPreFilter));
@ -1312,6 +1398,9 @@ static int32_t mSemiJoinHashSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
size_t bufLen = 0;
int32_t probeEndIdx = probeGrp->endIdx;
@ -1343,6 +1432,10 @@ static int32_t mSemiJoinHashFullCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
size_t bufLen = 0;
for (; !GRP_DONE(probeGrp) && !BLK_IS_FULL(pCtx->finBlk); ++probeGrp->readIdx) {
@ -1358,7 +1451,7 @@ static int32_t mSemiJoinHashFullCart(SMJoinMergeCtx* pCtx) {
build->pHashCurGrp = *(SArray**)pGrp;
ASSERT(1 == taosArrayGetSize(build->pHashCurGrp));
build->grpRowIdx = 0;
mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build);
MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, NULL));
ASSERT(build->grpRowIdx < 0);
}
@ -1372,6 +1465,10 @@ static int32_t mSemiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SMJoinGrpRows* buildGrp = NULL;
int32_t buildGrpNum = taosArrayGetSize(build->eqGrps);
int32_t probeEndIdx = probeGrp->endIdx;
@ -1387,6 +1484,9 @@ static int32_t mSemiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
blockDataCleanup(pCtx->midBlk);
for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) {
buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp));
@ -1448,6 +1548,10 @@ static int32_t mSemiJoinMergeFullCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, 0);
if (NULL == buildGrp || NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t probeRows = GRP_REMAIN_ROWS(probeGrp);
int32_t probeEndIdx = probeGrp->endIdx;
@ -1584,6 +1688,10 @@ static int32_t mAntiJoinHashFullCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
size_t bufLen = 0;
int32_t probeEndIdx = probeGrp->endIdx;
@ -1616,7 +1724,7 @@ static int32_t mAntiJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, SMJoinGrpRows* p
do {
blockDataCleanup(pCtx->midBlk);
mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build);
MJ_ERR_RET(mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build, NULL));
if (pCtx->midBlk->info.rows > 0) {
MJ_ERR_RET(mJoinFilterAndNoKeepRows(pCtx->midBlk, pCtx->pJoin->pPreFilter));
@ -1642,6 +1750,10 @@ static int32_t mAntiJoinHashSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
size_t bufLen = 0;
int32_t probeEndIdx = probeGrp->endIdx;
@ -1682,6 +1794,10 @@ static int32_t mAntiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SMJoinGrpRows* buildGrp = NULL;
int32_t buildGrpNum = taosArrayGetSize(build->eqGrps);
int32_t probeEndIdx = probeGrp->endIdx;
@ -1697,6 +1813,10 @@ static int32_t mAntiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
blockDataCleanup(pCtx->midBlk);
for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) {
buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp));
rowsLeft -= GRP_REMAIN_ROWS(buildGrp);
@ -1889,6 +2009,9 @@ int32_t mAsofBackwardAddEqRowsToCache(struct SOperatorInfo* pOperator, SMJoinWin
grp.blk = pTable->blk;
SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*(int64_t*)colDataGetNumData(pCol, pTable->blkRowIdx) != timestamp) {
return TSDB_CODE_SUCCESS;
@ -2102,7 +2225,9 @@ int32_t mAsofBackwardHandleGrpRemains(SMJoinWindowCtx* pCtx) {
return (pCtx->lastEqGrp) ? mAsofBackwardDumpUpdateEqRows(pCtx, pCtx->pJoin, false, true) : mAsofBackwardDumpGrpCache(pCtx);
}
static bool mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx) {
static int32_t mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx, bool* newBlock) {
*newBlock = false;
bool probeGot = mJoinRetrieveBlk(pJoin, &pJoin->probe->blkRowIdx, &pJoin->probe->blk, pJoin->probe);
bool buildGot = false;
@ -2116,7 +2241,7 @@ static bool mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo*
mJoinSetDone(pOperator);
}
return false;
return TSDB_CODE_SUCCESS;
}
break;
@ -2124,13 +2249,18 @@ static bool mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo*
if (buildGot && NULL == pCtx->cache.outBlk) {
pCtx->cache.outBlk = createOneDataBlock(pJoin->build->blk, false);
blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit);
if (NULL == pCtx->cache.outBlk) {
MJ_ERR_RET(terrno);
}
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit));
}
pCtx->probeGrp.blk = pJoin->probe->blk;
pCtx->buildGrp.blk = pJoin->build->blk;
return true;
*newBlock = true;
return TSDB_CODE_SUCCESS;
}
@ -2142,6 +2272,7 @@ SSDataBlock* mAsofBackwardJoinDo(struct SOperatorInfo* pOperator) {
int64_t buildTs = 0;
SColumnInfoData* pBuildCol = NULL;
SColumnInfoData* pProbeCol = NULL;
bool newBlock = false;
blockDataCleanup(pCtx->finBlk);
@ -2154,7 +2285,8 @@ SSDataBlock* mAsofBackwardJoinDo(struct SOperatorInfo* pOperator) {
}
do {
if (!mAsofBackwardRetrieve(pOperator, pJoin, pCtx)) {
MJ_ERR_JRET(mAsofBackwardRetrieve(pOperator, pJoin, pCtx, &newBlock));
if (!newBlock) {
if (pCtx->groupJoin && pCtx->finBlk->info.rows <= 0 && !mJoinIsDone(pOperator)) {
continue;
}
@ -2240,6 +2372,10 @@ int32_t mAsofForwardTrimCacheBlk(SMJoinWindowCtx* pCtx) {
}
SMJoinGrpRows* pGrp = taosArrayGet(pCtx->cache.grps, 0);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pGrp->blk == pCtx->cache.outBlk && pCtx->pJoin->build->blkRowIdx > 0) {
MJ_ERR_RET(blockDataTrimFirstRows(pGrp->blk, pCtx->pJoin->build->blkRowIdx));
pCtx->pJoin->build->blkRowIdx = 0;
@ -2262,6 +2398,10 @@ int32_t mAsofForwardChkFillGrpCache(SMJoinWindowCtx* pCtx) {
int32_t grpNum = taosArrayGetSize(pCache->grps);
if (grpNum >= 1) {
SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, grpNum - 1);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pGrp->blk != pCache->outBlk) {
int32_t beginIdx = (1 == grpNum) ? build->blkRowIdx : 0;
MJ_ERR_RET(blockDataMergeNRows(pCache->outBlk, pGrp->blk, beginIdx, pGrp->blk->info.rows - beginIdx));
@ -2271,8 +2411,12 @@ int32_t mAsofForwardChkFillGrpCache(SMJoinWindowCtx* pCtx) {
pGrp->readIdx = 0;
//pGrp->endIdx = pGrp->blk->info.rows - 1;
} else {
taosArrayPop(pCache->grps);
(void)taosArrayPop(pCache->grps);
pGrp = taosArrayGet(pCache->grps, 0);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
ASSERT(pGrp->blk == pCache->outBlk);
//pGrp->endIdx = pGrp->blk->info.rows - pGrp->beginIdx;
}
@ -2311,16 +2455,20 @@ int32_t mAsofForwardChkFillGrpCache(SMJoinWindowCtx* pCtx) {
return TSDB_CODE_SUCCESS;
}
void mAsofForwardUpdateBuildGrpEndIdx(SMJoinWindowCtx* pCtx) {
int32_t mAsofForwardUpdateBuildGrpEndIdx(SMJoinWindowCtx* pCtx) {
int32_t grpNum = taosArrayGetSize(pCtx->cache.grps);
if (grpNum <= 0) {
return;
return TSDB_CODE_SUCCESS;
}
SMJoinGrpRows* pGrp = taosArrayGet(pCtx->cache.grps, 0);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (1 == grpNum) {
pGrp->endIdx = pGrp->beginIdx + TMIN(pGrp->blk->info.rows - pGrp->beginIdx, pCtx->jLimit) - 1;
return;
return TSDB_CODE_SUCCESS;
}
ASSERT(pCtx->jLimit > (pGrp->blk->info.rows - pGrp->beginIdx));
@ -2329,7 +2477,13 @@ void mAsofForwardUpdateBuildGrpEndIdx(SMJoinWindowCtx* pCtx) {
int64_t remainRows = pCtx->jLimit - (pGrp->endIdx - pGrp->beginIdx + 1);
pGrp = taosArrayGet(pCtx->cache.grps, 1);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
pGrp->endIdx = pGrp->beginIdx + TMIN(pGrp->blk->info.rows, remainRows) - 1;
return TSDB_CODE_SUCCESS;
}
int32_t mAsofForwardFillDumpGrpCache(SMJoinWindowCtx* pCtx, bool lastBuildGrp) {
@ -2338,13 +2492,16 @@ int32_t mAsofForwardFillDumpGrpCache(SMJoinWindowCtx* pCtx, bool lastBuildGrp) {
MJ_ERR_RET(mAsofForwardChkFillGrpCache(pCtx));
}
mAsofForwardUpdateBuildGrpEndIdx(pCtx);
MJ_ERR_RET(mAsofForwardUpdateBuildGrpEndIdx(pCtx));
return mWinJoinDumpGrpCache(pCtx);
}
int32_t mAsofForwardSkipEqRows(SMJoinWindowCtx* pCtx, SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk) {
SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*(int64_t*)colDataGetNumData(pCol, pTable->blkRowIdx) != timestamp) {
*wholeBlk = false;
@ -2479,7 +2636,9 @@ int32_t mAsofForwardSkipBuildGrp(SMJoinWindowCtx* pCtx, SMJoinOperatorInfo* pJoi
return TSDB_CODE_SUCCESS;
}
static bool mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx) {
static int32_t mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx, bool* newBlock) {
*newBlock = false;
bool probeGot = mJoinRetrieveBlk(pJoin, &pJoin->probe->blkRowIdx, &pJoin->probe->blk, pJoin->probe);
bool buildGot = false;
@ -2496,12 +2655,16 @@ static bool mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* p
mJoinSetDone(pOperator);
}
return false;
return TSDB_CODE_SUCCESS;
}
if (buildGot) {
SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId);
SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId);
if (NULL == pProbeCol || NULL == pBuildCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) {
pJoin->build->blkRowIdx = pJoin->build->blk->info.rows;
MJOIN_POP_TB_BLK(&pCtx->cache);
@ -2516,7 +2679,10 @@ static bool mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* p
if (buildGot && pJoin->build->newBlk) {
if (NULL == pCtx->cache.outBlk) {
pCtx->cache.outBlk = createOneDataBlock(pJoin->build->blk, false);
blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit);
if (NULL == pCtx->cache.outBlk) {
MJ_ERR_RET(terrno);
}
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit));
}
MJOIN_PUSH_BLK_TO_CACHE(&pCtx->cache, pJoin->build->blk);
@ -2524,8 +2690,9 @@ static bool mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* p
}
pCtx->probeGrp.blk = pJoin->probe->blk;
*newBlock = true;
return true;
return TSDB_CODE_SUCCESS;
}
@ -2537,6 +2704,7 @@ SSDataBlock* mAsofForwardJoinDo(struct SOperatorInfo* pOperator) {
int64_t buildTs = 0;
SColumnInfoData* pBuildCol = NULL;
SColumnInfoData* pProbeCol = NULL;
bool newBlock = false;
blockDataCleanup(pCtx->finBlk);
@ -2549,7 +2717,8 @@ SSDataBlock* mAsofForwardJoinDo(struct SOperatorInfo* pOperator) {
}
do {
if (!mAsofForwardRetrieve(pOperator, pJoin, pCtx)) {
MJ_ERR_JRET(mAsofForwardRetrieve(pOperator, pJoin, pCtx, &newBlock));
if (!newBlock) {
if (pCtx->groupJoin && pCtx->finBlk->info.rows <= 0 && !mJoinIsDone(pOperator)) {
continue;
}
@ -2643,7 +2812,7 @@ static FORCE_INLINE void mWinJoinPopFrontGroup(SMJoinWindowCtx* pCtx, SMJoinGrpR
if (pGrp->blk == pCtx->cache.outBlk) {
blockDataCleanup(pGrp->blk);
} else if (pGrp->clonedBlk) {
blockDataDestroy(pGrp->blk);
(void)blockDataDestroy(pGrp->blk);
}
taosArrayPopFrontBatch(pCtx->cache.grps, 1);
@ -2658,6 +2827,10 @@ static int32_t mWinJoinCloneCacheBlk(SMJoinWindowCtx* pCtx) {
}
SMJoinGrpRows* pGrp = (SMJoinGrpRows*)taosArrayGetLast(pGrpArray);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (!pGrp->clonedBlk) {
if (0 == pGrp->beginIdx) {
pGrp->blk = createOneDataBlock(pGrp->blk, true);
@ -2667,6 +2840,10 @@ static int32_t mWinJoinCloneCacheBlk(SMJoinWindowCtx* pCtx) {
pGrp->beginIdx = 0;
pGrp->readIdx = 0;
}
if (NULL == pGrp->blk) {
MJ_ERR_RET(terrno);
}
pGrp->clonedBlk = true;
}
@ -2674,14 +2851,16 @@ static int32_t mWinJoinCloneCacheBlk(SMJoinWindowCtx* pCtx) {
return TSDB_CODE_SUCCESS;
}
static bool mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx) {
static int32_t mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx, bool* newBlock) {
*newBlock = false;
bool probeGot = mJoinRetrieveBlk(pJoin, &pJoin->probe->blkRowIdx, &pJoin->probe->blk, pJoin->probe);
bool buildGot = false;
do {
if (probeGot || MJOIN_DS_NEED_INIT(pOperator, pJoin->build)) {
if (NULL == pJoin->build->blk) {
mWinJoinCloneCacheBlk(pCtx);
MJ_ERR_RET(mWinJoinCloneCacheBlk(pCtx));
}
buildGot = mJoinRetrieveBlk(pJoin, &pJoin->build->blkRowIdx, &pJoin->build->blk, pJoin->build);
@ -2692,12 +2871,16 @@ static bool mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin
mJoinSetDone(pOperator);
}
return false;
return TSDB_CODE_SUCCESS;
}
if (buildGot && pCtx->forwardRowsAcq) {
SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId);
SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId);
if (NULL == pProbeCol || NULL == pBuildCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) {
pJoin->build->blkRowIdx = pJoin->build->blk->info.rows;
buildGot = false;
@ -2709,13 +2892,18 @@ static bool mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin
} while (true);
pCtx->probeGrp.blk = pJoin->probe->blk;
return true;
*newBlock = true;
return TSDB_CODE_SUCCESS;
}
int32_t mWinJoinTryAddWinBeginBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, SMJoinTableCtx* build, bool* winEnd) {
SSDataBlock* pBlk = build->blk;
SColumnInfoData* pCol = taosArrayGet(pBlk->pDataBlock, build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pCtx->ascTs) {
if (*((int64_t*)pCol->pData + pBlk->info.rows - 1) < pCtx->winBeginTs) {
*winEnd = false;
@ -2736,6 +2924,9 @@ int32_t mWinJoinTryAddWinBeginBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache,
if (*((int64_t*)pCol->pData + build->blkRowIdx) <= pCtx->winEndTs) {
SMJoinGrpRows grp = {.blk = pBlk, .beginIdx = build->blkRowIdx};
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = pGrp->beginIdx;
@ -2772,7 +2963,10 @@ int32_t mWinJoinTryAddWinBeginBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache,
if (*((int64_t*)pCol->pData + build->blkRowIdx) >= pCtx->winBeginTs) {
SMJoinGrpRows grp = {.blk = pBlk, .beginIdx = build->blkRowIdx};
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = pGrp->beginIdx;
@ -2841,7 +3035,15 @@ int32_t mWinJoinMoveAscWinBegin(SMJoinWindowCtx* pCtx) {
int32_t grpNum = taosArrayGetSize(pCache->grps);
for (int32_t i = 0; i < grpNum; ++i) {
SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, i);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) < pCtx->winBeginTs) {
mWinJoinPopFrontGroup(pCtx, pGrp);
grpNum--;
@ -2877,6 +3079,7 @@ int32_t mWinJoinMoveAscWinBegin(SMJoinWindowCtx* pCtx) {
pCache->grps = pCache->grpsQueue;
pCache->rowNum = 1;
pCache->grpsQueue = NULL;
continue;
}
@ -2893,7 +3096,15 @@ int32_t mWinJoinMoveDescWinBegin(SMJoinWindowCtx* pCtx) {
int32_t grpNum = taosArrayGetSize(pCache->grps);
for (int32_t i = 0; i < grpNum; ++i) {
SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, i);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) > pCtx->winEndTs) {
mWinJoinPopFrontGroup(pCtx, pGrp);
@ -2930,6 +3141,7 @@ int32_t mWinJoinMoveDescWinBegin(SMJoinWindowCtx* pCtx) {
pCache->grps = pCache->grpsQueue;
pCache->rowNum = 1;
pCache->grpsQueue = NULL;
continue;
}
@ -2963,6 +3175,10 @@ void mWinJoinRemoveOverflowGrp(SMJoinWindowCtx* pCtx) {
int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, SMJoinTableCtx* build, bool* winEnd) {
SSDataBlock* pBlk = build->blk;
SColumnInfoData* pCol = taosArrayGet(pBlk->pDataBlock, build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SMJoinGrpRows grp = {.blk = pBlk, .beginIdx = build->blkRowIdx};
if (pCtx->ascTs) {
@ -2978,6 +3194,9 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S
if (*((int64_t*)pCol->pData + pBlk->info.rows - 1) <= pCtx->winEndTs) {
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = pBlk->info.rows - 1;
@ -3005,6 +3224,9 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S
}
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = build->blkRowIdx - 1;
@ -3025,6 +3247,9 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S
if (*((int64_t*)pCol->pData + pBlk->info.rows - 1) >= pCtx->winBeginTs) {
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = pBlk->info.rows - 1;
@ -3047,7 +3272,10 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S
}
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = build->blkRowIdx - 1;
@ -3107,7 +3335,15 @@ int32_t mWinJoinMoveAscWinEnd(SMJoinWindowCtx* pCtx) {
}
SMJoinGrpRows* pGrp = taosArrayGetLast(pCache->grps);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) <= pCtx->winEndTs) {
pCache->rowNum += pGrp->blk->info.rows - pGrp->endIdx - 1;
if (pCache->rowNum >= pCtx->jLimit) {
@ -3150,7 +3386,15 @@ int32_t mWinJoinMoveDescWinEnd(SMJoinWindowCtx* pCtx) {
}
SMJoinGrpRows* pGrp = taosArrayGetLast(pCache->grps);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) >= pCtx->winBeginTs) {
pCache->rowNum += pGrp->blk->info.rows - pGrp->endIdx - 1;
pGrp->endIdx = pGrp->blk->info.rows - 1;
@ -3194,6 +3438,10 @@ int32_t mWinJoinTrimDumpGrpCache(SMJoinWindowCtx* pCtx) {
int32_t buildGrpNum = taosArrayGetSize(cache->grps);
for (int32_t i = 0; i < buildGrpNum && skipRows > 0; ++i) {
SMJoinGrpRows* buildGrp = taosArrayGet(cache->grps, i);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (skipRows >= GRP_REMAIN_ROWS(buildGrp)) {
skipRows -= GRP_REMAIN_ROWS(buildGrp);
mWinJoinPopFrontGroup(pCtx, buildGrp);
@ -3220,6 +3468,7 @@ SSDataBlock* mWinJoinDo(struct SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int64_t probeTs = 0;
SColumnInfoData* pProbeCol = NULL;
bool newBlock = false;
blockDataCleanup(pCtx->finBlk);
@ -3232,7 +3481,8 @@ SSDataBlock* mWinJoinDo(struct SOperatorInfo* pOperator) {
}
do {
if (!mWinJoinRetrieve(pOperator, pJoin, pCtx)) {
MJ_ERR_JRET(mWinJoinRetrieve(pOperator, pJoin, pCtx, &newBlock));
if (!newBlock) {
if (pCtx->groupJoin && pCtx->finBlk->info.rows <= 0 && !mJoinIsDone(pOperator)) {
continue;
}
@ -3293,7 +3543,7 @@ int32_t mJoinInitWindowCache(SMJoinWinCache* pCache, SMJoinOperatorInfo* pJoin,
pCache->grps = taosArrayInit(2, sizeof(SMJoinGrpRows));
if (NULL == pCache->grps) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
//taosArrayReserve(pTable->eqGrps, 1);
@ -3369,7 +3619,11 @@ int32_t mJoinInitWindowCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* p
}
pCtx->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc);
blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode));
if (NULL == pCtx->finBlk) {
MJ_ERR_RET(terrno);
}
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode)));
pCtx->blkThreshold = pCtx->finBlk->info.capacity * MJOIN_BLK_THRESHOLD_RATIO;
@ -3409,13 +3663,20 @@ int32_t mJoinInitMergeCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* pJ
}
pCtx->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc);
if (NULL == pCtx->finBlk) {
MJ_ERR_RET(terrno);
}
ASSERT(pJoinNode->node.pOutputDataBlockDesc->totalRowSize > 0);
blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode));
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode)));
if (pJoin->pFPreFilter) {
pCtx->midBlk = createOneDataBlock(pCtx->finBlk, false);
blockDataEnsureCapacity(pCtx->midBlk, pCtx->finBlk->info.capacity);
if (NULL == pCtx->midBlk) {
MJ_ERR_RET(terrno);
}
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->midBlk, pCtx->finBlk->info.capacity));
}
pCtx->blkThreshold = pCtx->finBlk->info.capacity * MJOIN_BLK_THRESHOLD_RATIO;

321
source/libs/executor/src/mergejoinoperator.c
View File

@ -30,6 +30,9 @@
int32_t mJoinBuildEqGrp(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk, SMJoinGrpRows* pGrp) {
SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
pGrp->beginIdx = pTable->blkRowIdx;
pGrp->readIdx = pTable->blkRowIdx;
@ -59,12 +62,16 @@ int32_t mJoinBuildEqGrp(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBl
}
void mJoinTrimKeepFirstRow(SSDataBlock* pBlock) {
int32_t mJoinTrimKeepFirstRow(SSDataBlock* pBlock) {
int32_t bmLen = BitmapLen(pBlock->info.rows);
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
if (NULL == pDst) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
// it is a reserved column for scalar function, and no data in this column yet.
if (pDst->pData == NULL || (IS_VAR_DATA_TYPE(pDst->info.type) && pDst->varmeta.length == 0)) {
continue;
@ -86,7 +93,7 @@ void mJoinTrimKeepFirstRow(SSDataBlock* pBlock) {
} else {
bool isNull = colDataIsNull_f(pDst->nullbitmap, 0);
memset(pDst->nullbitmap, 0, bmLen);
TAOS_MEMSET(pDst->nullbitmap, 0, bmLen);
if (isNull) {
colDataSetNull_f(pDst->nullbitmap, 0);
}
@ -94,11 +101,14 @@ void mJoinTrimKeepFirstRow(SSDataBlock* pBlock) {
}
pBlock->info.rows = 1;
return TSDB_CODE_SUCCESS;
}
void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList) {
int32_t mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList) {
// int32_t totalRows = pBlock->info.rows;
int32_t code = TSDB_CODE_SUCCESS;
int32_t bmLen = BitmapLen(totalRows);
char* pBitmap = NULL;
int32_t maxRows = 0;
@ -106,6 +116,10 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
if (NULL == pDst) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
// it is a reserved column for scalar function, and no data in this column yet.
if (pDst->pData == NULL || (IS_VAR_DATA_TYPE(pDst->info.type) && pDst->varmeta.length == 0)) {
continue;
@ -135,8 +149,12 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo
len = varDataTLen(p1);
}
char* p2 = taosMemoryMalloc(len);
memcpy(p2, p1, len);
colDataSetVal(pDst, numOfRows, p2, false);
TAOS_MEMCPY(p2, p1, len);
code = colDataSetVal(pDst, numOfRows, p2, false);
if (code) {
taosMemoryFreeClear(p2);
MJ_ERR_RET(terrno);
}
taosMemoryFree(p2);
}
numOfRows += 1;
@ -150,10 +168,13 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo
} else {
if (pBitmap == NULL) {
pBitmap = taosMemoryCalloc(1, bmLen);
if (NULL == pBitmap) {
MJ_ERR_RET(terrno);
}
}
memcpy(pBitmap, pDst->nullbitmap, bmLen);
memset(pDst->nullbitmap, 0, bmLen);
TAOS_MEMCPY(pBitmap, pDst->nullbitmap, bmLen);
TAOS_MEMSET(pDst->nullbitmap, 0, bmLen);
int32_t j = 0;
@ -243,6 +264,8 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo
if (pBitmap != NULL) {
taosMemoryFree(pBitmap);
}
return TSDB_CODE_SUCCESS;
}
@ -290,8 +313,10 @@ int32_t mJoinFilterAndMarkHashRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo
code = TSDB_CODE_SUCCESS;
_err:
colDataDestroy(p);
taosMemoryFree(p);
return code;
}
@ -300,18 +325,19 @@ int32_t mJoinFilterAndMarkRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SM
return TSDB_CODE_SUCCESS;
}
int32_t code = TSDB_CODE_SUCCESS;
SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock};
SColumnInfoData* p = NULL;
int32_t code = filterSetDataFromSlotId(pFilterInfo, &param1);
code = filterSetDataFromSlotId(pFilterInfo, &param1);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
goto _return;
}
int32_t status = 0;
code = filterExecute(pFilterInfo, pBlock, &p, NULL, param1.numOfCols, &status);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
goto _return;
}
int32_t rowNum = 0;
@ -320,6 +346,9 @@ int32_t mJoinFilterAndMarkRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SM
if (status == FILTER_RESULT_ALL_QUALIFIED || status == FILTER_RESULT_PARTIAL_QUALIFIED) {
for (int32_t i = startGrpIdx; i < grpNum && rowNum < pBlock->info.rows; startRowIdx = 0, ++i) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, i);
if (NULL == buildGrp) {
MJ_ERR_JRET(terrno);
}
if (buildGrp->allRowsMatch) {
rowNum += buildGrp->endIdx - startRowIdx + 1;
continue;
@ -350,9 +379,11 @@ int32_t mJoinFilterAndMarkRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SM
code = TSDB_CODE_SUCCESS;
_err:
_return:
colDataDestroy(p);
taosMemoryFree(p);
return code;
}
@ -366,29 +397,31 @@ int32_t mJoinFilterAndKeepSingleRow(SSDataBlock* pBlock, SFilterInfo* pFilterInf
int32_t code = filterSetDataFromSlotId(pFilterInfo, &param1);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
goto _return;
}
int32_t status = 0;
code = filterExecute(pFilterInfo, pBlock, &p, NULL, param1.numOfCols, &status);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
goto _return;
}
if (status == FILTER_RESULT_ALL_QUALIFIED) {
pBlock->info.rows = 1;
mJoinTrimKeepFirstRow(pBlock);
MJ_ERR_JRET(mJoinTrimKeepFirstRow(pBlock));
} else if (status == FILTER_RESULT_NONE_QUALIFIED) {
pBlock->info.rows = 0;
} else if (status == FILTER_RESULT_PARTIAL_QUALIFIED) {
mJoinTrimKeepOneRow(pBlock, pBlock->info.rows, (bool*)p->pData);
MJ_ERR_JRET(mJoinTrimKeepOneRow(pBlock, pBlock->info.rows, (bool*)p->pData));
}
code = TSDB_CODE_SUCCESS;
_err:
_return:
colDataDestroy(p);
taosMemoryFree(p);
return code;
}
@ -418,8 +451,10 @@ int32_t mJoinFilterAndNoKeepRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo)
code = TSDB_CODE_SUCCESS;
_err:
colDataDestroy(p);
taosMemoryFree(p);
return code;
}
@ -480,12 +515,20 @@ int32_t mJoinNonEqGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app
SMJoinColMap* pFirstCol = probe->finCols + c;
SColumnInfoData* pInCol = taosArrayGet(pGrp->blk->pDataBlock, pFirstCol->srcSlot);
SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pFirstCol->dstSlot);
colDataAssignNRows(pOutCol, currRows, pInCol, pGrp->readIdx, firstRows);
if (NULL == pInCol || NULL == pOutCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(colDataAssignNRows(pOutCol, currRows, pInCol, pGrp->readIdx, firstRows));
}
for (int32_t c = 0; c < build->finNum; ++c) {
SMJoinColMap* pSecondCol = build->finCols + c;
SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pSecondCol->dstSlot);
if (NULL == pOutCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
colDataSetNItemsNull(pOutCol, currRows, firstRows);
}
@ -536,6 +579,10 @@ int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app
SMJoinColMap* pFirstCol = probe->finCols + c;
SColumnInfoData* pInCol = taosArrayGet(pFirst->blk->pDataBlock, pFirstCol->srcSlot);
SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pFirstCol->dstSlot);
if (NULL == pInCol || NULL == pOutCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
for (int32_t r = 0; r < firstRows; ++r) {
if (colDataIsNull_s(pInCol, pFirst->readIdx + r)) {
colDataSetNItemsNull(pOutCol, currRows + r * secondRows, secondRows);
@ -543,7 +590,7 @@ int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app
ASSERT(pRes->info.capacity >= (pRes->info.rows + firstRows * secondRows));
uint32_t startOffset = (IS_VAR_DATA_TYPE(pOutCol->info.type)) ? pOutCol->varmeta.length : ((currRows + r * secondRows) * pOutCol->info.bytes);
ASSERT((startOffset + 1 * pOutCol->info.bytes) <= pRes->info.capacity * pOutCol->info.bytes);
colDataSetNItems(pOutCol, currRows + r * secondRows, colDataGetData(pInCol, pFirst->readIdx + r), secondRows, true);
MJ_ERR_RET(colDataSetNItems(pOutCol, currRows + r * secondRows, colDataGetData(pInCol, pFirst->readIdx + r), secondRows, true));
}
}
}
@ -552,28 +599,40 @@ int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app
SMJoinColMap* pSecondCol = build->finCols + c;
SColumnInfoData* pInCol = taosArrayGet(pSecond->blk->pDataBlock, pSecondCol->srcSlot);
SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pSecondCol->dstSlot);
if (NULL == pInCol || NULL == pOutCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
for (int32_t r = 0; r < firstRows; ++r) {
colDataAssignNRows(pOutCol, currRows + r * secondRows, pInCol, pSecond->readIdx, secondRows);
MJ_ERR_RET(colDataAssignNRows(pOutCol, currRows + r * secondRows, pInCol, pSecond->readIdx, secondRows));
}
}
pRes->info.rows = append ? (pRes->info.rows + firstRows * secondRows) : firstRows * secondRows;
return TSDB_CODE_SUCCESS;
}
bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build) {
int32_t mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build, bool* cont) {
if (NULL != cont) {
*cont = false;
}
int32_t rowsLeft = append ? (pBlk->info.capacity - pBlk->info.rows) : pBlk->info.capacity;
if (rowsLeft <= 0) {
return false;
return TSDB_CODE_SUCCESS;
}
int32_t buildGrpRows = taosArrayGetSize(build->pHashCurGrp);
int32_t grpRows = buildGrpRows - build->grpRowIdx;
if (grpRows <= 0 || build->grpRowIdx < 0) {
build->grpRowIdx = -1;
return true;
if (NULL != cont) {
*cont = true;
}
return TSDB_CODE_SUCCESS;
}
int32_t actRows = TMIN(grpRows, rowsLeft);
@ -583,10 +642,14 @@ bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, S
SMJoinColMap* pFirstCol = probe->finCols + c;
SColumnInfoData* pInCol = taosArrayGet(probeGrp->blk->pDataBlock, pFirstCol->srcSlot);
SColumnInfoData* pOutCol = taosArrayGet(pBlk->pDataBlock, pFirstCol->dstSlot);
if (NULL == pInCol || NULL == pOutCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (colDataIsNull_s(pInCol, probeGrp->readIdx)) {
colDataSetNItemsNull(pOutCol, currRows, actRows);
} else {
colDataSetNItems(pOutCol, currRows, colDataGetData(pInCol, probeGrp->readIdx), actRows, true);
MJ_ERR_RET(colDataSetNItems(pOutCol, currRows, colDataGetData(pInCol, probeGrp->readIdx), actRows, true));
}
}
@ -595,8 +658,16 @@ bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, S
SColumnInfoData* pOutCol = taosArrayGet(pBlk->pDataBlock, pSecondCol->dstSlot);
for (int32_t r = 0; r < actRows; ++r) {
SMJoinRowPos* pRow = taosArrayGet(build->pHashCurGrp, build->grpRowIdx + r);
if (NULL == pRow) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pInCol = taosArrayGet(pRow->pBlk->pDataBlock, pSecondCol->srcSlot);
colDataAssignNRows(pOutCol, currRows + r, pInCol, pRow->pos, 1);
if (NULL == pInCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(colDataAssignNRows(pOutCol, currRows + r, pInCol, pRow->pos, 1));
}
}
@ -609,16 +680,24 @@ bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, S
}
if (actRows == rowsLeft) {
return false;
return TSDB_CODE_SUCCESS;
}
return true;
if (NULL != cont) {
*cont = true;
}
return TSDB_CODE_SUCCESS;
}
int32_t mJoinAllocGrpRowBitmap(SMJoinTableCtx* pTb) {
int32_t grpNum = taosArrayGetSize(pTb->eqGrps);
for (int32_t i = 0; i < grpNum; ++i) {
SMJoinGrpRows* pGrp = (SMJoinGrpRows*)taosArrayGet(pTb->eqGrps, i);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(mJoinGetRowBitmapOffset(pTb, pGrp->endIdx - pGrp->beginIdx + 1, &pGrp->rowBitmapOffset));
pGrp->rowMatchNum = 0;
}
@ -632,9 +711,9 @@ int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastB
pCtx->lastEqGrp = true;
mJoinBuildEqGroups(pJoin->probe, timestamp, NULL, true);
MJ_ERR_RET(mJoinBuildEqGroups(pJoin->probe, timestamp, NULL, true));
if (!lastBuildGrp) {
mJoinRetrieveEqGrpRows(pJoin, pJoin->build, timestamp);
MJ_ERR_RET(mJoinRetrieveEqGrpRows(pJoin, pJoin->build, timestamp));
} else {
pJoin->build->grpIdx = 0;
}
@ -661,7 +740,7 @@ int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastB
pCtx->hashJoin = false;
if (!lastBuildGrp && pJoin->build->rowBitmapSize > 0) {
mJoinAllocGrpRowBitmap(pJoin->build);
MJ_ERR_RET(mJoinAllocGrpRowBitmap(pJoin->build));
}
return (*pCtx->mergeCartFp)(pCtx);
@ -721,7 +800,7 @@ int32_t mJoinInitDownstreamInfo(SMJoinOperatorInfo* pInfo, SOperatorInfo*** pDow
*newDownstreams = true;
*pDownstream = mJoinBuildDownstreams(pInfo, *pDownstream);
if (NULL == *pDownstream) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
*numOfDownstream = 2;
}
@ -732,7 +811,7 @@ int32_t mJoinInitDownstreamInfo(SMJoinOperatorInfo* pInfo, SOperatorInfo*** pDow
static int32_t mJoinInitPrimKeyInfo(SMJoinTableCtx* pTable, int32_t slotId) {
pTable->primCol = taosMemoryMalloc(sizeof(SMJoinColInfo));
if (NULL == pTable->primCol) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pTable->primCol->srcSlot = slotId;
@ -745,7 +824,7 @@ static int32_t mJoinInitColsInfo(int32_t* colNum, int64_t* rowSize, SMJoinColInf
*pCols = taosMemoryMalloc((*colNum) * sizeof(SMJoinColInfo));
if (NULL == *pCols) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
*rowSize = 0;
@ -779,7 +858,7 @@ static int32_t mJoinInitKeyColsInfo(SMJoinTableCtx* pTable, SNodeList* pList, bo
pTable->keyBuf = taosMemoryMalloc(TMAX(rowSize, pTable->keyNullSize));
if (NULL == pTable->keyBuf) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
}
@ -790,7 +869,7 @@ static int32_t mJoinInitKeyColsInfo(SMJoinTableCtx* pTable, SNodeList* pList, bo
static int32_t mJoinInitFinColsInfo(SMJoinTableCtx* pTable, SNodeList* pList) {
pTable->finCols = taosMemoryMalloc(LIST_LENGTH(pList) * sizeof(SMJoinColMap));
if (NULL == pTable->finCols) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
int32_t i = 0;
@ -837,8 +916,20 @@ static int32_t mJoinInitPrimExprCtx(SNode* pNode, SMJoinPrimExprCtx* pCtx, SMJoi
}
SValueNode* pUnit = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1);
SValueNode* pCurrTz = (5 == pFunc->pParameterList->length) ? (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2) : NULL;
if (NULL == pUnit) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
SValueNode* pCurrTz = NULL;
if (5 == pFunc->pParameterList->length){
pCurrTz = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2);
if (NULL == pCurrTz) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
}
SValueNode* pTimeZone = (5 == pFunc->pParameterList->length) ? (SValueNode*)nodesListGetNode(pFunc->pParameterList, 4) : (SValueNode*)nodesListGetNode(pFunc->pParameterList, 3);
if (NULL == pTimeZone) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
pCtx->truncateUnit = pUnit->typeData;
if ((NULL == pCurrTz || 1 == pCurrTz->typeData) && pCtx->truncateUnit >= (86400 * TSDB_TICK_PER_SECOND(pFunc->node.resType.precision))) {
@ -859,24 +950,32 @@ static int32_t mJoinInitTableInfo(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysi
MJ_ERR_RET(mJoinInitKeyColsInfo(pTable, (0 == idx) ? pJoinNode->pEqLeft : pJoinNode->pEqRight, JOIN_TYPE_FULL == pJoin->joinType));
MJ_ERR_RET(mJoinInitFinColsInfo(pTable, pJoinNode->pTargets));
memcpy(&pTable->inputStat, pStat, sizeof(*pStat));
TAOS_MEMCPY(&pTable->inputStat, pStat, sizeof(*pStat));
pTable->eqGrps = taosArrayInit(8, sizeof(SMJoinGrpRows));
//taosArrayReserve(pTable->eqGrps, 1);
if (NULL == pTable->eqGrps) {
return terrno;
}
if (E_JOIN_TB_BUILD == pTable->type) {
pTable->createdBlks = taosArrayInit(8, POINTER_BYTES);
if (NULL == pTable->createdBlks) {
return terrno;
}
pTable->pGrpArrays = taosArrayInit(32, POINTER_BYTES);
if (NULL == pTable->pGrpArrays) {
return terrno;
}
pTable->pGrpHash = tSimpleHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY));
if (NULL == pTable->createdBlks || NULL == pTable->pGrpArrays || NULL == pTable->pGrpHash) {
return TSDB_CODE_OUT_OF_MEMORY;
if (NULL == pTable->pGrpHash) {
return terrno;
}
if (pJoin->pFPreFilter && IS_FULL_OUTER_JOIN(pJoin->joinType, pJoin->subType)) {
pTable->rowBitmapSize = MJOIN_ROW_BITMAP_SIZE;
pTable->pRowBitmap = taosMemoryMalloc(pTable->rowBitmapSize);
if (NULL == pTable->pRowBitmap) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
}
@ -945,7 +1044,15 @@ int32_t mJoinLaunchPrimExpr(SSDataBlock* pBlock, SMJoinTableCtx* pTable) {
SMJoinPrimExprCtx* pCtx = &pTable->primCtx;
SColumnInfoData* pPrimIn = taosArrayGet(pBlock->pDataBlock, pTable->primCol->srcSlot);
if (NULL == pPrimIn) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
SColumnInfoData* pPrimOut = taosArrayGet(pBlock->pDataBlock, pTable->primCtx.targetSlotId);
if (NULL == pPrimOut) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
if (0 != pCtx->timezoneUnit) {
for (int32_t i = 0; i < pBlock->info.rows; ++i) {
((int64_t*)pPrimOut->pData)[i] = ((int64_t*)pPrimIn->pData)[i] - (((int64_t*)pPrimIn->pData)[i] + pCtx->timezoneUnit) % pCtx->truncateUnit;
@ -961,6 +1068,7 @@ int32_t mJoinLaunchPrimExpr(SSDataBlock* pBlock, SMJoinTableCtx* pTable) {
SSDataBlock* mJoinGrpRetrieveImpl(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable) {
SSDataBlock* pTmp = NULL;
int32_t code = TSDB_CODE_SUCCESS;
int32_t dsIdx = pTable->downStreamIdx;
if (E_JOIN_TB_PROBE == pTable->type) {
if (pTable->remainInBlk) {
@ -1028,7 +1136,12 @@ SSDataBlock* mJoinGrpRetrieveImpl(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTa
_return:
mJoinLaunchPrimExpr(pTmp, pTable);
code = mJoinLaunchPrimExpr(pTmp, pTable);
if (code) {
pJoin->errCode = code;
T_LONG_JMP(pJoin->pOperator->pTaskInfo->env, pJoin->errCode);
}
return pTmp;
}
@ -1041,7 +1154,11 @@ static FORCE_INLINE SSDataBlock* mJoinRetrieveImpl(SMJoinOperatorInfo* pJoin, SM
if (NULL == pTmp) {
pTable->dsFetchDone = true;
} else {
mJoinLaunchPrimExpr(pTmp, pTable);
int32_t code = mJoinLaunchPrimExpr(pTmp, pTable);
if (code) {
pJoin->errCode = code;
T_LONG_JMP(pJoin->pOperator->pTaskInfo->env, pJoin->errCode);
}
}
return pTmp;
@ -1107,7 +1224,7 @@ bool mJoinRetrieveBlk(SMJoinOperatorInfo* pJoin, int32_t* pIdx, SSDataBlock** pp
static void mJoinDestroyCreatedBlks(SArray* pCreatedBlks) {
int32_t blkNum = taosArrayGetSize(pCreatedBlks);
for (int32_t i = 0; i < blkNum; ++i) {
blockDataDestroy(*(SSDataBlock**)TARRAY_GET_ELEM(pCreatedBlks, i));
(void)blockDataDestroy(*(SSDataBlock**)TARRAY_GET_ELEM(pCreatedBlks, i));
}
taosArrayClear(pCreatedBlks);
}
@ -1119,12 +1236,12 @@ int32_t mJoinGetRowBitmapOffset(SMJoinTableCtx* pTable, int32_t rowNum, int32_t
int64_t newSize = reqSize * 1.1;
pTable->pRowBitmap = taosMemoryRealloc(pTable->pRowBitmap, newSize);
if (NULL == pTable->pRowBitmap) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pTable->rowBitmapSize = newSize;
}
memset(pTable->pRowBitmap + pTable->rowBitmapOffset, 0xFFFFFFFF, bitmapLen);
TAOS_MEMSET(pTable->pRowBitmap + pTable->rowBitmapOffset, 0xFFFFFFFF, bitmapLen);
*rowBitmapOffset = pTable->rowBitmapOffset;
pTable->rowBitmapOffset += bitmapLen;
@ -1140,13 +1257,18 @@ void mJoinResetForBuildTable(SMJoinTableCtx* pTable) {
taosArrayClear(pTable->eqGrps);
if (pTable->rowBitmapSize > 0) {
pTable->rowBitmapOffset = 1;
memset(&pTable->nMatchCtx, 0, sizeof(pTable->nMatchCtx));
TAOS_MEMSET(&pTable->nMatchCtx, 0, sizeof(pTable->nMatchCtx));
}
}
int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk, bool restart) {
SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SMJoinGrpRows* pGrp = NULL;
int32_t code = TSDB_CODE_SUCCESS;
if (*(int64_t*)colDataGetNumData(pCol, pTable->blkRowIdx) != timestamp) {
return TSDB_CODE_SUCCESS;
@ -1158,7 +1280,10 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol
bool keepGrp = true;
pGrp = taosArrayReserve(pTable->eqGrps, 1);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->beginIdx = pTable->blkRowIdx++;
pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = pGrp->beginIdx;
@ -1207,7 +1332,12 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol
if (0 == pGrp->beginIdx && pTable->multiEqGrpRows && 0 == pTable->eqRowLimit) {
pGrp->blk = createOneDataBlock(pTable->blk, true);
taosArrayPush(pTable->createdBlks, &pGrp->blk);
if (NULL == pGrp->blk) {
MJ_ERR_RET(terrno);
}
if (NULL == taosArrayPush(pTable->createdBlks, &pGrp->blk)) {
MJ_ERR_RET(terrno);
}
} else {
if (!pTable->multiEqGrpRows) {
pGrp->endIdx = pGrp->beginIdx;
@ -1230,10 +1360,16 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol
pTable->eqRowNum += rowNum;
pGrp->blk = blockDataExtractBlock(pTable->blk, pGrp->beginIdx, rowNum);
if (NULL == pGrp->blk) {
MJ_ERR_RET(terrno);
}
pGrp->endIdx -= pGrp->beginIdx;
pGrp->beginIdx = 0;
pGrp->readIdx = 0;
taosArrayPush(pTable->createdBlks, &pGrp->blk);
if (NULL == taosArrayPush(pTable->createdBlks, &pGrp->blk)) {
MJ_ERR_RET(terrno);
}
}
}
@ -1242,19 +1378,19 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol
_return:
if (pTable->noKeepEqGrpRows || !keepGrp || (!pTable->multiEqGrpRows && !restart)) {
taosArrayPop(pTable->eqGrps);
(void)taosArrayPop(pTable->eqGrps);
} else {
pTable->grpTotalRows += pGrp->endIdx - pGrp->beginIdx + 1;
}
return TSDB_CODE_SUCCESS;
return code;
}
int32_t mJoinRetrieveEqGrpRows(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable, int64_t timestamp) {
bool wholeBlk = false;
mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, true);
MJ_ERR_RET(mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, true));
while (wholeBlk && !pTable->dsFetchDone) {
pTable->blk = (*pJoin->retrieveFp)(pJoin, pTable);
@ -1267,7 +1403,7 @@ int32_t mJoinRetrieveEqGrpRows(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable
}
wholeBlk = false;
mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, false);
MJ_ERR_RET(mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, false));
}
return TSDB_CODE_SUCCESS;
@ -1276,6 +1412,10 @@ int32_t mJoinRetrieveEqGrpRows(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable
int32_t mJoinSetKeyColsData(SSDataBlock* pBlock, SMJoinTableCtx* pTable) {
for (int32_t i = 0; i < pTable->keyNum; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->keyCols[i].srcSlot);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pTable->keyCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) {
qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->keyCols[i].srcSlot, pCol->info.type, pTable->keyCols[i].vardata);
return TSDB_CODE_INVALID_PARA;
@ -1320,15 +1460,15 @@ bool mJoinCopyKeyColsDataToBuf(SMJoinTableCtx* pTable, int32_t rowIdx, size_t *p
}
if (pTable->keyCols[0].jsonData) {
pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx];
memcpy(pTable->keyBuf + bufLen, pData, getJsonValueLen(pData));
TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, getJsonValueLen(pData));
bufLen += getJsonValueLen(pData);
} else if (pTable->keyCols[i].vardata) {
pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx];
memcpy(pTable->keyBuf + bufLen, pData, varDataTLen(pData));
TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, varDataTLen(pData));
bufLen += varDataTLen(pData);
} else {
pData = pTable->keyCols[i].data + pTable->keyCols[i].bytes * rowIdx;
memcpy(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes);
TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes);
bufLen += pTable->keyCols[i].bytes;
}
}
@ -1346,15 +1486,20 @@ static int32_t mJoinGetAvailableGrpArray(SMJoinTableCtx* pTable, SArray** ppRes)
do {
if (pTable->grpArrayIdx < taosArrayGetSize(pTable->pGrpArrays)) {
*ppRes = taosArrayGetP(pTable->pGrpArrays, pTable->grpArrayIdx++);
if (NULL == *ppRes) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
taosArrayClear(*ppRes);
return TSDB_CODE_SUCCESS;
}
SArray* pNew = taosArrayInit(4, sizeof(SMJoinRowPos));
if (NULL == pNew) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
if (NULL == taosArrayPush(pTable->pGrpArrays, &pNew)) {
return terrno;
}
taosArrayPush(pTable->pGrpArrays, &pNew);
} while (true);
return TSDB_CODE_SUCCESS;
@ -1368,10 +1513,14 @@ static int32_t mJoinAddRowToHash(SMJoinOperatorInfo* pJoin, size_t keyLen, SSDat
SArray* pNewGrp = NULL;
MJ_ERR_RET(mJoinGetAvailableGrpArray(pBuild, &pNewGrp));
taosArrayPush(pNewGrp, &pos);
tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, POINTER_BYTES);
if (NULL == taosArrayPush(pNewGrp, &pos)) {
return terrno;
}
MJ_ERR_RET(tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, POINTER_BYTES));
} else if (pBuild->multiRowsGrp) {
taosArrayPush(*pGrpRows, &pos);
if (NULL == taosArrayPush(*pGrpRows, &pos)) {
return terrno;
}
}
return TSDB_CODE_SUCCESS;
@ -1386,10 +1535,14 @@ static int32_t mJoinAddRowToFullHash(SMJoinOperatorInfo* pJoin, size_t keyLen, S
SMJoinHashGrpRows pNewGrp = {0};
MJ_ERR_RET(mJoinGetAvailableGrpArray(pBuild, &pNewGrp.pRows));
taosArrayPush(pNewGrp.pRows, &pos);
tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, sizeof(pNewGrp));
if (NULL == taosArrayPush(pNewGrp.pRows, &pos)) {
return terrno;
}
MJ_ERR_RET(tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, sizeof(pNewGrp)));
} else {
taosArrayPush(pGrpRows->pRows, &pos);
if (NULL == taosArrayPush(pGrpRows->pRows, &pos)) {
return terrno;
}
}
return TSDB_CODE_SUCCESS;
@ -1406,6 +1559,9 @@ int32_t mJoinCreateFullBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pT
int32_t grpNum = taosArrayGetSize(pTable->eqGrps);
for (int32_t g = 0; g < grpNum; ++g) {
SMJoinGrpRows* pGrp = taosArrayGet(pTable->eqGrps, g);
if (NULL == pGrp) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
MJ_ERR_RET(mJoinSetKeyColsData(pGrp->blk, pTable));
int32_t grpRows = GRP_REMAIN_ROWS(pGrp);
@ -1434,6 +1590,10 @@ int32_t mJoinCreateBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable
int32_t grpNum = taosArrayGetSize(pTable->eqGrps);
for (int32_t g = 0; g < grpNum; ++g) {
SMJoinGrpRows* pGrp = taosArrayGet(pTable->eqGrps, g);
if (NULL == pGrp) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
MJ_ERR_RET(mJoinSetKeyColsData(pGrp->blk, pTable));
int32_t grpRows = GRP_REMAIN_ROWS(pGrp);
@ -1490,8 +1650,11 @@ void mWinJoinResetWindowCache(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache) {
for (int32_t i = 0; i < grpNum; ++i) {
SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, i);
if (NULL == pGrp) {
continue;
}
if (pGrp->blk != pCtx->cache.outBlk && pGrp->clonedBlk) {
blockDataDestroy(pGrp->blk);
(void)blockDataDestroy(pGrp->blk);
}
}
@ -1533,6 +1696,7 @@ void mJoinResetOperator(struct SOperatorInfo* pOperator) {
SSDataBlock* mJoinMainProcess(struct SOperatorInfo* pOperator) {
SMJoinOperatorInfo* pJoin = pOperator->info;
int32_t code = TSDB_CODE_SUCCESS;
if (pOperator->status == OP_EXEC_DONE) {
if (NULL == pOperator->pDownstreamGetParams || NULL == pOperator->pDownstreamGetParams[0] || NULL == pOperator->pDownstreamGetParams[1]) {
qDebug("%s merge join done", GET_TASKID(pOperator->pTaskInfo));
@ -1553,7 +1717,6 @@ SSDataBlock* mJoinMainProcess(struct SOperatorInfo* pOperator) {
pBlock = (*pJoin->joinFp)(pOperator);
if (NULL == pBlock) {
if (pJoin->errCode) {
ASSERT(0);
T_LONG_JMP(pOperator->pTaskInfo->env, pJoin->errCode);
}
break;
@ -1561,7 +1724,11 @@ SSDataBlock* mJoinMainProcess(struct SOperatorInfo* pOperator) {
pBlock->info.id.blockId = pJoin->outBlkId;
if (pJoin->pFinFilter != NULL) {
doFilter(pBlock, pJoin->pFinFilter, NULL);
code = doFilter(pBlock, pJoin->pFinFilter, NULL);
if (code) {
pJoin->errCode = code;
T_LONG_JMP(pOperator->pTaskInfo->env, pJoin->errCode);
}
}
if (pBlock->info.rows > 0 || pOperator->status == OP_EXEC_DONE) {
@ -1697,13 +1864,17 @@ int32_t mJoinSetImplFp(SMJoinOperatorInfo* pJoin) {
SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream,
SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) {
SMJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMJoinOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
bool newDownstreams = false;
int32_t code = TSDB_CODE_SUCCESS;
if (pOperator == NULL || pInfo == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
SMJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMJoinOperatorInfo));
if (pInfo == NULL) {
code = terrno;
goto _return;
}
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pOperator == NULL) {
code = terrno;
goto _return;
}
@ -1716,8 +1887,8 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
MJ_ERR_JRET(mJoinHandleConds(pInfo, pJoinNode));
mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0], newDownstreams);
mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1], newDownstreams);
MJ_ERR_JRET(mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0], newDownstreams));
MJ_ERR_JRET(mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1], newDownstreams));
MJ_ERR_JRET(mJoinInitCtx(pInfo, pJoinNode));
MJ_ERR_JRET(mJoinSetImplFp(pInfo));
@ -1736,6 +1907,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
return pOperator;
_return:
if (pInfo != NULL) {
destroyMergeJoinOperator(pInfo);
}
@ -1745,6 +1917,7 @@ _return:
taosMemoryFree(pOperator);
pTaskInfo->code = code;
return NULL;
}

348
source/libs/executor/test/joinTests.cpp
View File

File diff suppressed because it is too large Load Diff