diff --git a/include/util/tutil.h b/include/util/tutil.h index 87710b091d..aa3b774e84 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -120,6 +120,18 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, } } +/* + * LIKELY and UNLIKELY macros for branch predication hints. Use them judiciously + * only in very hot code paths. Misuse or abuse can lead to performance degradation. + */ +#if __GNUC__ >= 3 +#define LIKELY(x) __builtin_expect((x) != 0, 1) +#define UNLIKELY(x) __builtin_expect((x) != 0, 0) +#else +#define LIKELY(x) ((x) != 0) +#define UNLIKELY(x) ((x) != 0) +#endif + #define TAOS_CHECK_ERRNO(CODE) \ do { \ terrno = (CODE); \ @@ -129,25 +141,27 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, } \ } while (0) -#define TSDB_CHECK_CODE(CODE, LINO, LABEL) \ - do { \ - if (TSDB_CODE_SUCCESS != (CODE)) { \ - LINO = __LINE__; \ - goto LABEL; \ - } \ +#define TSDB_CHECK_CODE(CODE, LINO, LABEL) \ + do { \ + if (UNLIKELY(TSDB_CODE_SUCCESS != (CODE))) { \ + LINO = __LINE__; \ + goto LABEL; \ + } \ } while (0) #define QUERY_CHECK_CODE TSDB_CHECK_CODE -#define QUERY_CHECK_CONDITION(condition, CODE, LINO, LABEL, ERRNO) \ - if (!condition) { \ - (CODE) = (ERRNO); \ - (LINO) = __LINE__; \ - goto LABEL; \ +#define TSDB_CHECK_CONDITION(condition, CODE, LINO, LABEL, ERRNO) \ + if (UNLIKELY(!(condition))) { \ + (CODE) = (ERRNO); \ + (LINO) = __LINE__; \ + goto LABEL; \ } +#define QUERY_CHECK_CONDITION TSDB_CHECK_CONDITION + #define TSDB_CHECK_NULL(ptr, CODE, LINO, LABEL, ERRNO) \ - if ((ptr) == NULL) { \ + if (UNLIKELY((ptr) == NULL)) { \ (CODE) = (ERRNO); \ (LINO) = __LINE__; \ goto LABEL; \ diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 610ba43673..743bfd36d0 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -172,7 +172,7 @@ void tsdbReleaseDataBlock2(STsdbReader *pReader); int32_t tsdbRetrieveDataBlock2(STsdbReader *pReader, SSDataBlock **pBlock, SArray *pIdList); int32_t tsdbReaderReset2(STsdbReader *pReader, SQueryTableDataCond *pCond); int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); -int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle); +int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle, uint32_t *rows); void *tsdbGetIdx2(SMeta *pMeta); void *tsdbGetIvtIdx2(SMeta *pMeta); uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader); diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index c7626dcf36..0f524e22d7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -25,82 +25,109 @@ #define HASTYPE(_type, _t) (((_type) & (_t)) == (_t)) static int32_t setFirstLastResColToNull(SColumnInfoData* pCol, int32_t row) { - char* buf = taosMemoryCalloc(1, pCol->info.bytes); - if (buf == NULL) { - return terrno; - } + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + char* buf = NULL; + SFirstLastRes* pRes = NULL; - SFirstLastRes* pRes = (SFirstLastRes*)((char*)buf + VARSTR_HEADER_SIZE); + TSDB_CHECK_NULL(pCol, code, lino, _end, TSDB_CODE_INVALID_PARA); + + buf = taosMemoryCalloc(1, pCol->info.bytes); + TSDB_CHECK_NULL(buf, code, lino, _end, terrno); + + pRes = (SFirstLastRes*)((char*)buf + VARSTR_HEADER_SIZE); pRes->bytes = 0; pRes->hasResult = true; pRes->isNull = true; varDataSetLen(buf, pCol->info.bytes - VARSTR_HEADER_SIZE); - int32_t code = colDataSetVal(pCol, row, buf, false); - taosMemoryFree(buf); + code = colDataSetVal(pCol, row, buf, false); + TSDB_CHECK_CODE(code, lino, _end); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (buf != NULL) { + taosMemoryFreeClear(buf); + } return code; } static int32_t saveOneRowForLastRaw(SLastCol* pColVal, SCacheRowsReader* pReader, const int32_t slotId, - SColumnInfoData* pColInfoData, int32_t numOfRows) { - SColVal* pVal = &pColVal->colVal; - int32_t code = 0; + SColumnInfoData* pColInfoData, int32_t numOfRows) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SColVal* pVal = NULL; + + TSDB_CHECK_NULL(pColVal, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pVal = &pColVal->colVal; // allNullRow = false; if (IS_VAR_DATA_TYPE(pColVal->colVal.value.type)) { if (!COL_VAL_IS_VALUE(&pColVal->colVal)) { colDataSetNULL(pColInfoData, numOfRows); } else { + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); varDataSetLen(pReader->transferBuf[slotId], pVal->value.nData); memcpy(varDataVal(pReader->transferBuf[slotId]), pVal->value.pData, pVal->value.nData); code = colDataSetVal(pColInfoData, numOfRows, pReader->transferBuf[slotId], false); + TSDB_CHECK_CODE(code, lino, _end); } } else { code = colDataSetVal(pColInfoData, numOfRows, (const char*)&pVal->value.val, !COL_VAL_IS_VALUE(pVal)); + TSDB_CHECK_CODE(code, lino, _end); } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* pReader, const int32_t* slotIds, const int32_t* dstSlotIds, void** pRes, const char* idStr) { - int32_t numOfRows = pBlock->info.rows; - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t numOfRows = 0; + SArray* funcTypeBlockArray = NULL; + + TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + if (pReader->numOfCols > 0) { + TSDB_CHECK_NULL(slotIds, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(dstSlotIds, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pRes, code, lino, _end, TSDB_CODE_INVALID_PARA); + } + + numOfRows = pBlock->info.rows; if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST)) { uint64_t ts = TSKEY_MIN; SFirstLastRes* p = NULL; col_id_t colId = -1; - SArray* funcTypeBlockArray = taosArrayInit(pReader->numOfCols, sizeof(int32_t)); - if (funcTypeBlockArray == NULL) { - return terrno; - } + funcTypeBlockArray = taosArrayInit(pReader->numOfCols, sizeof(int32_t)); + TSDB_CHECK_NULL(funcTypeBlockArray, code, lino, _end, terrno); for (int32_t i = 0; i < pReader->numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]); - if (pColInfoData == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA); int32_t funcType = FUNCTION_TYPE_CACHE_LAST; if (pReader->pFuncTypeList != NULL && taosArrayGetSize(pReader->pFuncTypeList) > i) { void* pVal = taosArrayGet(pReader->pFuncTypeList, i); - if (pVal == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pVal, code, lino, _end, TSDB_CODE_INVALID_PARA); - funcType = *(int32_t*) pVal; + funcType = *(int32_t*)pVal; pVal = taosArrayGet(pReader->pFuncTypeList, i); - if (pVal == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pVal, code, lino, _end, TSDB_CODE_INVALID_PARA); void* px = taosArrayInsert(funcTypeBlockArray, dstSlotIds[i], pVal); - if (px == NULL) { - return terrno; - } + TSDB_CHECK_NULL(px, code, lino, _end, terrno); } if (slotIds[i] == -1) { @@ -110,24 +137,18 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p } code = setFirstLastResColToNull(pColInfoData, numOfRows); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); continue; } int32_t slotId = slotIds[i]; SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); - if (pColVal == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pColVal, code, lino, _end, TSDB_CODE_INVALID_PARA); colId = pColVal->colVal.cid; if (FUNCTION_TYPE_CACHE_LAST_ROW == funcType) { code = saveOneRowForLastRaw(pColVal, pReader, slotId, pColInfoData, numOfRows); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); continue; } @@ -154,22 +175,16 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p p->hasResult = true; varDataSetLen(pRes[i], pColInfoData->info.bytes - VARSTR_HEADER_SIZE); code = colDataSetVal(pColInfoData, numOfRows, (const char*)pRes[i], false); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } for (int32_t idx = 0; idx < taosArrayGetSize(pBlock->pDataBlock); ++idx) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, idx); - if (pCol == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pCol, code, lino, _end, TSDB_CODE_INVALID_PARA); if (idx < funcTypeBlockArray->size) { void* pVal = taosArrayGet(funcTypeBlockArray, idx); - if (pVal == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pVal, code, lino, _end, TSDB_CODE_INVALID_PARA); int32_t funcType = *(int32_t*)pVal; if (FUNCTION_TYPE_CACHE_LAST_ROW == funcType) { @@ -182,17 +197,13 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p colDataSetNULL(pCol, numOfRows); } else { code = colDataSetVal(pCol, numOfRows, (const char*)&ts, false); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } continue; } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && (pCol->info.colId == colId || colId == -1)) { if (p && !p->isNull) { code = colDataSetVal(pCol, numOfRows, p->buf, false); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } else { colDataSetNULL(pCol, numOfRows); } @@ -201,13 +212,10 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p // pBlock->info.rows += allNullRow ? 0 : 1; ++pBlock->info.rows; - taosArrayDestroy(funcTypeBlockArray); } else if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST_ROW)) { for (int32_t i = 0; i < pReader->numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]); - if (pColInfoData == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA); int32_t slotId = slotIds[i]; if (slotId == -1) { @@ -216,47 +224,53 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p } SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); - if (pColVal == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pColVal, code, lino, _end, TSDB_CODE_INVALID_PARA); code = saveOneRowForLastRaw(pColVal, pReader, slotId, pColInfoData, numOfRows); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } // pBlock->info.rows += allNullRow ? 0 : 1; ++pBlock->info.rows; } else { tsdbError("invalid retrieve type:%d, %s", pReader->type, idStr); - return TSDB_CODE_INVALID_PARA; + code = TSDB_CODE_INVALID_PARA; + TSDB_CHECK_CODE(code, lino, _end); } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (funcTypeBlockArray != NULL) { + taosArrayDestroy(funcTypeBlockArray); + } return code; } static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* idstr) { - int32_t numOfTables = p->numOfTables; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t numOfTables = 0; + + TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA); + + numOfTables = p->numOfTables; if (suid != 0) { code = metaGetTbTSchemaNotNull(p->pVnode->pMeta, suid, -1, 1, &p->pSchema); if (TSDB_CODE_SUCCESS != code) { tsdbWarn("stable:%" PRIu64 " has been dropped, failed to retrieve cached rows, %s", suid, idstr); - if(code == TSDB_CODE_NOT_FOUND) { - return TSDB_CODE_PAR_TABLE_NOT_EXIST; - } else { - return code; + if (code == TSDB_CODE_NOT_FOUND) { + code = TSDB_CODE_PAR_TABLE_NOT_EXIST; } + TSDB_CHECK_CODE(code, lino, _end); } } else { for (int32_t i = 0; i < numOfTables; ++i) { uint64_t uid = p->pTableList[i].uid; code = metaGetTbTSchemaMaybeNull(p->pVnode->pMeta, uid, -1, 1, &p->pSchema); - if(code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (p->pSchema != NULL) { break; } @@ -267,33 +281,52 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id // all queried tables have been dropped already, return immediately. if (p->pSchema == NULL) { tsdbWarn("all queried tables has been dropped, try next group, %s", idstr); - return TSDB_CODE_PAR_TABLE_NOT_EXIST; + code = TSDB_CODE_PAR_TABLE_NOT_EXIST; + TSDB_CHECK_CODE(code, lino, _end); } } - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOfTables) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SCacheRowsReader* pReader = (SCacheRowsReader*)reader; + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + pReader->pTableList = pTableIdList; pReader->numOfTables = numOfTables; pReader->lastTs = INT64_MIN; destroySttBlockReader(pReader->pLDataIterArray, NULL); pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + TSDB_CHECK_NULL(pReader->pLDataIterArray, code, lino, _end, terrno); - return (pReader->pLDataIterArray != NULL) ? TSDB_CODE_SUCCESS : terrno; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables, int32_t numOfCols, SArray* pCidList, int32_t* pSlotIds, uint64_t suid, void** pReader, const char* idstr, SArray* pFuncTypeList, SColumnInfo* pPkCol, int32_t numOfPks) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SCacheRowsReader* p = NULL; + + TSDB_CHECK_NULL(pVnode, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + *pReader = NULL; - SCacheRowsReader* p = taosMemoryCalloc(1, sizeof(SCacheRowsReader)); - if (p == NULL) { - return terrno; - } + p = taosMemoryCalloc(1, sizeof(SCacheRowsReader)); + TSDB_CHECK_NULL(p, code, lino, _end, terrno); p->type = type; p->pVnode = pVnode; @@ -307,12 +340,13 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, p->rowKey.numOfPKs = numOfPks; if (numOfPks > 0) { + TSDB_CHECK_NULL(pPkCol, code, lino, _end, TSDB_CODE_INVALID_PARA); p->rowKey.pks[0].type = pPkCol->type; if (IS_VAR_DATA_TYPE(pPkCol->type)) { p->rowKey.pks[0].pData = taosMemoryCalloc(1, pPkCol->bytes); if (p->rowKey.pks[0].pData == NULL) { - taosMemoryFree(p); - return terrno; + taosMemoryFreeClear(p); + TSDB_CHECK_NULL(p->rowKey.pks[0].pData, code, lino, _end, terrno); } } @@ -321,48 +355,46 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, if (numOfTables == 0) { *pReader = p; - return TSDB_CODE_SUCCESS; + p = NULL; + goto _end; } p->pTableList = pTableIdList; p->numOfTables = numOfTables; - int32_t code = setTableSchema(p, suid, idstr); - if (code != TSDB_CODE_SUCCESS) { - tsdbCacherowsReaderClose(p); - return code; - } + code = setTableSchema(p, suid, idstr); + TSDB_CHECK_CODE(code, lino, _end); p->transferBuf = taosMemoryCalloc(p->pSchema->numOfCols, POINTER_BYTES); - if (p->transferBuf == NULL) { - tsdbCacherowsReaderClose(p); - return terrno; - } + TSDB_CHECK_NULL(p->transferBuf, code, lino, _end, terrno); for (int32_t i = 0; i < p->pSchema->numOfCols; ++i) { if (IS_VAR_DATA_TYPE(p->pSchema->columns[i].type)) { p->transferBuf[i] = taosMemoryMalloc(p->pSchema->columns[i].bytes); - if (p->transferBuf[i] == NULL) { - tsdbCacherowsReaderClose(p); - return terrno; - } + TSDB_CHECK_NULL(p->transferBuf[i], code, lino, _end, terrno); } } - p->idstr = taosStrdup(idstr); - if (idstr != NULL && p->idstr == NULL) { - tsdbCacherowsReaderClose(p); - return terrno; + if (idstr != NULL) { + p->idstr = taosStrdup(idstr); + TSDB_CHECK_NULL(p->idstr, code, lino, _end, terrno); } code = taosThreadMutexInit(&p->readerMutex, NULL); - if (code) { - tsdbCacherowsReaderClose(p); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); p->lastTs = INT64_MIN; *pReader = p; + p = NULL; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + *pReader = NULL; + } + if (p != NULL) { + tsdbCacherowsReaderClose(p); + } return code; } @@ -393,6 +425,7 @@ void tsdbCacherowsReaderClose(void* pReader) { if (p->pLDataIterArray) { destroySttBlockReader(p->pLDataIterArray, NULL); + p->pLDataIterArray = NULL; } if (p->pFileReader) { @@ -401,7 +434,7 @@ void tsdbCacherowsReaderClose(void* pReader) { } taosMemoryFree((void*)p->idstr); - (void) taosThreadMutexDestroy(&p->readerMutex); + (void)taosThreadMutexDestroy(&p->readerMutex); if (p->pTableMap) { void* pe = NULL; @@ -443,39 +476,32 @@ static int32_t tsdbCacheQueryReseek(void* pQHandle) { int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32_t* slotIds, const int32_t* dstSlotIds, SArray* pTableUidList, bool* pGotAll) { - if (pReader == NULL || pResBlock == NULL) { - return TSDB_CODE_INVALID_PARA; - } - int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; bool hasRes = false; SArray* pRow = NULL; void** pRes = NULL; - SCacheRowsReader* pr = pReader; + SCacheRowsReader* pr = NULL; int32_t pkBufLen = 0; + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pResBlock, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pr = pReader; + pr->pReadSnap = NULL; pRow = taosArrayInit(TARRAY_SIZE(pr->pCidList), sizeof(SLastCol)); - if (pRow == NULL) { - code = terrno; - goto _end; - } + TSDB_CHECK_NULL(pRow, code, lino, _end, terrno); pRes = taosMemoryCalloc(pr->numOfCols, POINTER_BYTES); - if (pRes == NULL) { - code = terrno; - goto _end; - } + TSDB_CHECK_NULL(pRes, code, lino, _end, terrno); pkBufLen = (pr->rowKey.numOfPKs > 0) ? pr->pkColumn.bytes : 0; for (int32_t j = 0; j < pr->numOfCols; ++j) { int32_t bytes = (slotIds[j] == -1) ? 1 : pr->pSchema->columns[slotIds[j]].bytes; pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + bytes + pkBufLen + VARSTR_HEADER_SIZE); - if (pRes[j] == NULL) { - code = terrno; - goto _end; - } + TSDB_CHECK_NULL(pRes[j], code, lino, _end, terrno); SFirstLastRes* p = (SFirstLastRes*)varDataVal(pRes[j]); p->ts = INT64_MIN; @@ -483,9 +509,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 (void)taosThreadMutexLock(&pr->readerMutex); code = tsdbTakeReadSnap2((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap, pr->idstr); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } + TSDB_CHECK_CODE(code, lino, _end); int8_t ltype = (pr->type & CACHESCAN_RETRIEVE_LAST) >> 3; @@ -494,20 +518,14 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 // retrieve the only one last row of all tables in the uid list. if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_SINGLE)) { SArray* pLastCols = taosArrayInit(pr->numOfCols, sizeof(SLastCol)); - if (pLastCols == NULL) { - code = terrno; - goto _end; - } + TSDB_CHECK_NULL(pLastCols, code, lino, _end, terrno); for (int32_t i = 0; i < pr->numOfCols; ++i) { int32_t slotId = slotIds[i]; if (slotId == -1) { SLastCol p = {.rowKey.ts = INT64_MIN, .colVal.value.type = TSDB_DATA_TYPE_BOOL, .colVal.flag = CV_FLAG_NULL}; void* px = taosArrayPush(pLastCols, &p); - if (px == NULL) { - code = terrno; - goto _end; - } + TSDB_CHECK_NULL(px, code, lino, _end, terrno); continue; } struct STColumn* pCol = &pr->pSchema->columns[slotId]; @@ -518,29 +536,19 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 for (int32_t j = 0; j < pr->rowKey.numOfPKs; j++) { p.rowKey.pks[j].type = pr->pkColumn.type; if (IS_VAR_DATA_TYPE(pr->pkColumn.type)) { - p.rowKey.pks[j].pData = taosMemoryCalloc(1, pr->pkColumn.bytes); - if (p.rowKey.pks[j].pData == NULL) { - code = terrno; - goto _end; - } + TSDB_CHECK_NULL(p.rowKey.pks[j].pData, code, lino, _end, terrno); } } } if (IS_VAR_DATA_TYPE(pCol->type)) { p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char)); - if (p.colVal.value.pData == NULL) { - code = terrno; - goto _end; - } + TSDB_CHECK_NULL(p.colVal.value.pData, code, lino, _end, terrno); } void* px = taosArrayPush(pLastCols, &p); - if (px == NULL) { - code = terrno; - goto _end; - } + TSDB_CHECK_NULL(px, code, lino, _end, terrno); } int64_t st = taosGetTimestampUs(); @@ -549,11 +557,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 tb_uid_t uid = pTableList[i].uid; code = tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); - if (code == -1) {// fix the invalid return code + if (code == -1) { // fix the invalid return code code = 0; - } else if (code != 0) { - goto _end; } + TSDB_CHECK_CODE(code, lino, _end); if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { taosArrayClearEx(pRow, tsdbCacheFreeSLastColItem); @@ -600,10 +607,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 if (k == 0) { if (TARRAY_SIZE(pTableUidList) == 0) { void* px = taosArrayPush(pTableUidList, &uid); - if (px == NULL) { - code = terrno; - goto _end; - } + TSDB_CHECK_NULL(px, code, lino, _end, terrno); } else { taosArraySet(pTableUidList, 0, &uid); } @@ -654,9 +658,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 if (hasRes) { code = saveOneRow(pLastCols, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); - if (code) { - goto _end; - } + TSDB_CHECK_CODE(code, lino, _end); } taosArrayDestroyEx(pLastCols, tsdbCacheFreeSLastColItem); @@ -666,11 +668,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 tb_uid_t uid = pTableList[i].uid; if ((code = tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype)) != 0) { - if (code == -1) {// fix the invalid return code + if (code == -1) { // fix the invalid return code code = 0; - } else if (code != 0) { - goto _end; } + TSDB_CHECK_CODE(code, lino, _end); } if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { @@ -679,17 +680,12 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } code = saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); - if (code) { - goto _end; - } + TSDB_CHECK_CODE(code, lino, _end); taosArrayClearEx(pRow, tsdbCacheFreeSLastColItem); void* px = taosArrayPush(pTableUidList, &uid); - if (px == NULL) { - code = terrno; - goto _end; - } + TSDB_CHECK_NULL(px, code, lino, _end, terrno); ++pr->tableIndex; if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -702,6 +698,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } } else { code = TSDB_CODE_INVALID_PARA; + TSDB_CHECK_CODE(code, lino, _end); } _end: @@ -723,5 +720,8 @@ _end: taosMemoryFree(pRes); taosArrayDestroy(pRow); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index c4971e27cf..f30f7eb310 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -24,13 +24,14 @@ #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define getCurrentKeyInSttBlock(_r) (&((_r)->currentKey)) -#define tColRowGetKeyDeepCopy(_pBlock, _irow, _slotId, _pKey) \ - do { \ - (_pKey)->ts = (_pBlock)->aTSKEY[(_irow)]; \ - (_pKey)->numOfPKs = 0; \ - if ((_slotId) != -1) { \ - tColRowGetPriamyKeyDeepCopy(_pBlock, _irow, _slotId, _pKey); \ - } \ +#define tColRowGetKeyDeepCopy(_pBlock, _irow, _slotId, _pKey) \ + do { \ + (_pKey)->ts = (_pBlock)->aTSKEY[(_irow)]; \ + (_pKey)->numOfPKs = 0; \ + if ((_slotId) != -1) { \ + code = tColRowGetPriamyKeyDeepCopy(_pBlock, _irow, _slotId, _pKey); \ + TSDB_CHECK_CODE(code, lino, _end); \ + } \ } while (0) #define outOfTimeWindow(_ts, _window) (((_ts) > (_window)->ekey) || ((_ts) < (_window)->skey)) @@ -45,23 +46,23 @@ typedef struct { bool moreThanCapcity; } SDataBlockToLoadInfo; -static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo, const char* idStr); -static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, - STsdbReader* pReader); -static void getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRes); -static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey, - STsdbReader* pReader); -static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, - SRowMerger* pMerger, int32_t pkSrcSlot, SVersionRange* pVerRange, const char* id); -static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList, - STsdbReader* pReader); -static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, - STableBlockScanInfo* pScanInfo); -static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, - int32_t rowIndex); -static void setComposedBlockFlag(STsdbReader* pReader, bool composed); -static bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, - SVersionRange* pVerRange, bool hasPk); +static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo, const char* idStr); +static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, + STsdbReader* pReader); +static int32_t getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRes); +static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey, + STsdbReader* pReader); +static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, + SRowMerger* pMerger, int32_t pkSrcSlot, SVersionRange* pVerRange, const char* id); +static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList, + STsdbReader* pReader); +static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, + STableBlockScanInfo* pScanInfo); +static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, + int32_t rowIndex); +static void setComposedBlockFlag(STsdbReader* pReader, bool composed); +static int32_t hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, + SVersionRange* pVerRange, bool hasPk, bool* dropped); static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIterInfo* pIter, SArray* pDelList, TSDBROW* pResRow, STsdbReader* pReader, bool* freeTSRow); @@ -77,10 +78,10 @@ static void getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRet int8_t* pLevel, STsdb** pTsdb); static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); static int32_t doBuildDataBlock(STsdbReader* pReader); -static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); +static int32_t getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader, TSDBKEY* key); static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo); static bool hasDataInSttBlock(STableBlockScanInfo* pInfo); -static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter); +static int32_t initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter); static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order); static int32_t resetTableListIndex(SReaderStatus* pStatus, const char* id); static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t* pMinKey); @@ -111,9 +112,17 @@ FORCE_INLINE int32_t pkCompEx(SRowKey* p1, SRowKey* p2) { } } -static void tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_t slotId, SRowKey* pKey) { - SColData* pColData = &pBlock->aColData[slotId]; +static int32_t tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_t slotId, SRowKey* pKey) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SColData* pColData = NULL; SColVal cv; + + TSDB_CHECK_CONDITION((pBlock != NULL) && (pBlock->aColData != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pColData = &pBlock->aColData[slotId]; + tColDataGetValue(pColData, irow, &cv); pKey->numOfPKs = 1; @@ -123,8 +132,14 @@ static void tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_ pKey->pks[0].val = cv.value.val; } else { pKey->pks[0].nData = cv.value.nData; - (void)memcpy(pKey->pks[0].pData, cv.value.pData, cv.value.nData); + TAOS_MEMCPY(pKey->pks[0].pData, cv.value.pData, cv.value.nData); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } // for test purpose, todo remove it @@ -135,10 +150,20 @@ static int32_t tGetPrimaryKeyIndex(uint8_t* p, SPrimaryKeyIndex* index) { return n; } -static void tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) { +static int32_t tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SPrimaryKeyIndex indices[TD_MAX_PK_COLS]; + uint8_t* data = NULL; + int32_t len = 0; - uint8_t* data = pRow->data; + TSDB_CHECK_NULL(pRow, code, lino, _end, TSDB_CODE_INVALID_PARA); + if (pRow->numOfPKs > 0) { + TSDB_CHECK_NULL(pRow->data, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA); + } + + data = pRow->data; for (int32_t i = 0; i < pRow->numOfPKs; i++) { data += tGetPrimaryKeyIndex(data, &indices[i]); } @@ -154,16 +179,29 @@ static void tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) { if (IS_VAR_DATA_TYPE(indices[i].type)) { tdata += tGetU32v(tdata, &pKey->pks[i].nData); - (void)memcpy(pKey->pks[i].pData, tdata, pKey->pks[i].nData); + TAOS_MEMCPY(pKey->pks[i].pData, tdata, pKey->pks[i].nData); } else { - (void)memcpy(&pKey->pks[i].val, data + indices[i].offset, tDataTypes[pKey->pks[i].type].bytes); + TAOS_MEMCPY(&pKey->pks[i].val, data + indices[i].offset, tDataTypes[pKey->pks[i].type].bytes); } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) { - bool initSucc = true; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + TSDB_CHECK_NULL(pSupInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + if (numOfCols > 0) { + TSDB_CHECK_NULL(pSlotIdList, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pCols, code, lino, _end, TSDB_CODE_INVALID_PARA); + } pSupInfo->pk.pk = 0; pSupInfo->numOfPks = 0; @@ -173,10 +211,7 @@ static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pC pSupInfo->numOfCols = numOfCols; pSupInfo->colId = taosMemoryMalloc(numOfCols * (sizeof(int16_t) * 2 + POINTER_BYTES)); - if (pSupInfo->colId == NULL) { - taosMemoryFree(pSupInfo->colId); - return terrno; - } + TSDB_CHECK_NULL(pSupInfo->colId, code, lino, _end, terrno); pSupInfo->slotId = (int16_t*)((char*)pSupInfo->colId + (sizeof(int16_t) * numOfCols)); pSupInfo->buildBuf = (char**)((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols)); @@ -184,36 +219,47 @@ static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pC pSupInfo->colId[i] = pCols[i].colId; pSupInfo->slotId[i] = pSlotIdList[i]; - if (IS_VAR_DATA_TYPE(pCols[i].type)) { - pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes); - if (pSupInfo->buildBuf[i] == NULL) { - tsdbError("failed to prepare memory for set columnId slot list, size:%d, code:out of memory", pCols[i].bytes); - initSucc = false; - } - } else { - pSupInfo->buildBuf[i] = NULL; - } - if (pCols[i].pk) { pSupInfo->pk = pCols[i]; pSupInfo->pkSrcSlot = i - 1; pSupInfo->pkDstSlot = pSlotIdList[i]; pSupInfo->numOfPks += 1; } + + if (IS_VAR_DATA_TYPE(pCols[i].type)) { + pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes); + if (pSupInfo->buildBuf[i] == NULL) { + tsdbError("failed to prepare memory for set columnId slot list, size:%d, code: %s", pCols[i].bytes, + tstrerror(terrno)); + } + TSDB_CHECK_NULL(pSupInfo->buildBuf[i], code, lino, _end, terrno); + } else { + pSupInfo->buildBuf[i] = NULL; + } } - return (initSucc)? TSDB_CODE_SUCCESS:TSDB_CODE_OUT_OF_MEMORY; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) { - int32_t i = 0, j = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t i = 0, j = 0; + STColumn* pTCol = NULL; + + TSDB_CHECK_NULL(pSchema, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pSupInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); while (i < pSchema->numOfCols && j < pSupInfo->numOfCols) { - STColumn* pTCol = &pSchema->columns[i]; + pTCol = &pSchema->columns[i]; if (pTCol->colId == pSupInfo->colId[j]) { if (!IS_BSMA_ON(pTCol) && (PRIMARYKEY_TIMESTAMP_COL_ID != pTCol->colId)) { pSupInfo->smaValid = false; - return TSDB_CODE_SUCCESS; + goto _end; } i += 1; @@ -221,33 +267,62 @@ static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInf } else if (pTCol->colId < pSupInfo->colId[j]) { // do nothing i += 1; } else { - return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + TSDB_CHECK_CODE(code, lino, _end); } } - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) { return pWindow->skey > pWindow->ekey; } +static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) { + return (pWindow == NULL) || (pWindow->skey > pWindow->ekey); +} // Update the query time window according to the data time to live(TTL) information, in order to avoid to return // the expired data to client, even it is queried already. -static STimeWindow updateQueryTimeWindow(STsdb* pTsdb, STimeWindow* pWindow) { - int64_t earlyTs = tsdbGetEarliestTs(pTsdb); - STimeWindow win = *pWindow; - if (win.skey < earlyTs) { - win.skey = earlyTs; +static int32_t updateQueryTimeWindow(STsdb* pTsdb, STimeWindow* pWindow, STimeWindow* out) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int64_t earlyTs = 0; + + TSDB_CHECK_NULL(pTsdb, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pWindow, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(out, code, lino, _end, TSDB_CODE_INVALID_PARA); + + earlyTs = tsdbGetEarliestTs(pTsdb); + *out = *pWindow; + if (out->skey < earlyTs) { + out->skey = earlyTs; } - return win; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } // init file iterator static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetArray, STsdbReader* pReader) { - SBlockLoadSuppInfo* pInfo = &pReader->suppInfo; - size_t numOfFileset = TARRAY2_SIZE(pFileSetArray); - bool asc = ASCENDING_TRAVERSE(pReader->info.order); int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SBlockLoadSuppInfo* pInfo = NULL; + SSttBlockReader* pSttReader = NULL; + size_t numOfFileset = 0; + bool asc = false; + + TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pFileSetArray, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pInfo = &pReader->suppInfo; + numOfFileset = TARRAY2_SIZE(pFileSetArray); + asc = ASCENDING_TRAVERSE(pReader->info.order); pIter->index = asc ? -1 : numOfFileset; pIter->order = pReader->info.order; @@ -258,11 +333,11 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetA pIter->pSttBlockReader = taosMemoryCalloc(1, sizeof(struct SSttBlockReader)); if (pIter->pSttBlockReader == NULL) { tsdbError("failed to prepare the last block iterator, since:%s %s", tstrerror(terrno), pReader->idStr); - return terrno; } + TSDB_CHECK_NULL(pIter->pSttBlockReader, code, lino, _end, terrno); } - SSttBlockReader* pSttReader = pIter->pSttBlockReader; + pSttReader = pIter->pSttBlockReader; pSttReader->order = pReader->info.order; pSttReader->window = pReader->info.window; pSttReader->verRange = pReader->info.verRange; @@ -276,31 +351,46 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetA } else { tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); } + TSDB_CHECK_CODE(code, lino, _end); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bool* hasNext) { - bool asc = ASCENDING_TRAVERSE(pIter->order); - int32_t step = asc ? 1 : -1; - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + bool asc = false; + int32_t step = 0; + SReadCostSummary* pCost = NULL; + STFileObj** pFileObj = NULL; + + TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(hasNext, code, lino, _end, TSDB_CODE_INVALID_PARA); + + asc = ASCENDING_TRAVERSE(pIter->order); + step = asc ? 1 : -1; + *hasNext = false; pIter->index += step; if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) { *hasNext = false; - return TSDB_CODE_SUCCESS; + goto _end; } - SReadCostSummary* pCost = &pReader->cost; + pCost = &pReader->cost; + TSDB_CHECK_NULL(pIter->pSttBlockReader, code, lino, _end, TSDB_CODE_INVALID_PARA); pIter->pSttBlockReader->uid = 0; tMergeTreeClose(&pIter->pSttBlockReader->mergeTree); destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); - if (pReader->status.pLDataIterArray == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pReader->status.pLDataIterArray, code, lino, _end, terrno); // check file the time range of coverage STimeWindow win = {0}; @@ -310,9 +400,10 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo tsdbDataFileReaderClose(&pReader->pFileReader); } + TSDB_CHECK_CONDITION(pIter->index < pIter->pFilesetList->size, code, lino, _end, TSDB_CODE_INTERNAL_ERROR); pReader->status.pCurrentFileset = pIter->pFilesetList->data[pIter->index]; - STFileObj** pFileObj = pReader->status.pCurrentFileset->farr; + pFileObj = pReader->status.pCurrentFileset->farr; if (pFileObj[0] != NULL || pFileObj[3] != NULL) { SDataFileReaderConfig conf = {.tsdb = pReader->pTsdb, .szPage = pReader->pTsdb->pVnode->config.tsdbPageSize}; @@ -339,9 +430,7 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo } code = tsdbDataFileReaderOpen(filesName, &conf, &pReader->pFileReader); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _end); pReader->cost.headFileLoad += 1; } @@ -354,14 +443,14 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pReader, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); *hasNext = false; - return TSDB_CODE_SUCCESS; + break; } if ((asc && (win.ekey < pReader->info.window.skey)) || ((!asc) && (win.skey > pReader->info.window.ekey))) { pIter->index += step; if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) { *hasNext = false; - return TSDB_CODE_SUCCESS; + break; } continue; } @@ -370,17 +459,26 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo pReader->info.window.ekey, pReader->idStr); *hasNext = true; - return TSDB_CODE_SUCCESS; + break; } -_err: - *hasNext = false; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } -bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp) { return (pSupp->numOfPks > 0) && IS_VAR_DATA_TYPE(pSupp->pk.type); } +bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp) { + return (pSupp != NULL) && (pSupp->numOfPks > 0) && IS_VAR_DATA_TYPE(pSupp->pk.type); +} int32_t resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool needFree, const char* id) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA); + pIter->order = order; pIter->index = -1; pIter->numOfBlocks = 0; @@ -389,100 +487,165 @@ int32_t resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool needFr pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); if (pIter->blockList == NULL) { tsdbError("%s failed to reset block iter, func:%s at line:%d code:%s", id, __func__, __LINE__, tstrerror(terrno)); - return terrno; } + TSDB_CHECK_NULL(pIter->blockList, code, lino, _end, terrno); } else { clearDataBlockIterator(pIter, needFree); } - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void initReaderStatus(SReaderStatus* pStatus) { + if (pStatus == NULL) { + return; + } pStatus->pTableIter = NULL; pStatus->loadFromFile = true; } static int32_t createResBlock(SQueryTableDataCond* pCond, int32_t capacity, SSDataBlock** pResBlock) { - QRY_PARAM_CHECK(pResBlock); - + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SSDataBlock* pBlock = NULL; - int32_t code = createDataBlock(&pBlock); - if (code != 0) { - return code; - } + + TSDB_CHECK_NULL(pCond, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_CONDITION(capacity >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pResBlock, code, lino, _end, TSDB_CODE_INVALID_PARA); + + code = createDataBlock(&pBlock); + TSDB_CHECK_CODE(code, lino, _end); for (int32_t i = 0; i < pCond->numOfCols; ++i) { SColumnInfoData colInfo = {0}; colInfo.info = pCond->colList[i]; code = blockDataAppendColInfo(pBlock, &colInfo); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFree(pBlock); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } code = blockDataEnsureCapacity(pBlock, capacity); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFree(pBlock); - } + TSDB_CHECK_CODE(code, lino, _end); *pResBlock = pBlock; + pBlock = NULL; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (pBlock) { + taosArrayDestroy(pBlock->pDataBlock); + taosMemoryFreeClear(pBlock); + } return code; } static int32_t tsdbInitReaderLock(STsdbReader* pReader) { - int32_t code = taosThreadMutexInit(&pReader->readerMutex, NULL); - tsdbTrace("tsdb/read: %p, post-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + code = taosThreadMutexInit(&pReader->readerMutex, NULL); + tsdbTrace("tsdb/read: %p, post-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbUninitReaderLock(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; - tsdbTrace("tsdb/read: %p, pre-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + int32_t lino = 0; + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + tsdbTrace("tsdb/read: %p, pre-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); code = taosThreadMutexDestroy(&pReader->readerMutex); tsdbTrace("tsdb/read: %p, post-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + TSDB_CHECK_CODE(code, lino, _end); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbAcquireReader(STsdbReader* pReader) { - int32_t code = -1; - tsdbTrace("tsdb/read: %s, pre-take read mutex: %p, code: %d", pReader->idStr, &pReader->readerMutex, code); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + TSDB_CHECK_CONDITION((pReader != NULL) && (pReader->idStr != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA); + + tsdbTrace("tsdb/read: %s, pre-take read mutex: %p, code: %d", pReader->idStr, &pReader->readerMutex, code); code = taosThreadMutexLock(&pReader->readerMutex); - if (code != 0) { + if (code != TSDB_CODE_SUCCESS) { tsdbError("tsdb/read:%p, failed to lock reader mutex, code:%s", pReader->idStr, tstrerror(code)); } else { tsdbTrace("tsdb/read: %s, post-take read mutex: %p, code: %d", pReader->idStr, &pReader->readerMutex, code); } + TSDB_CHECK_CODE(code, lino, _end); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbTryAcquireReader(STsdbReader* pReader) { - int32_t code = taosThreadMutexTryLock(&pReader->readerMutex); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + code = taosThreadMutexTryLock(&pReader->readerMutex); if (code != TSDB_CODE_SUCCESS) { tsdbError("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); } else { tsdbTrace("tsdb/read: %p, post-trytask read mutex: %p", pReader, &pReader->readerMutex); } + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbReleaseReader(STsdbReader* pReader) { - int32_t code = taosThreadMutexUnlock(&pReader->readerMutex); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + code = taosThreadMutexUnlock(&pReader->readerMutex); if (code != TSDB_CODE_SUCCESS) { tsdbError("tsdb/read: %p post-untake read mutex:%p failed, code:%d", pReader, &pReader->readerMutex, code); } else { tsdbTrace("tsdb/read: %p, post-untake read mutex: %p", pReader, &pReader->readerMutex); } + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } void tsdbReleaseDataBlock2(STsdbReader* pReader) { + if (pReader == NULL) return; + SReaderStatus* pStatus = &pReader->status; if (!pStatus->composedDataBlock) { (void) tsdbReleaseReader(pReader); @@ -491,35 +654,34 @@ void tsdbReleaseDataBlock2(STsdbReader* pReader) { static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacity, SSDataBlock* pResBlock, SQueryTableDataCond* pCond, SBlockLoadSuppInfo* pSup) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SSDataBlock* p = NULL; + + TSDB_CHECK_NULL(pResBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_CONDITION((pResBlockInfo->pResBlock != NULL) || (pSup != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA); + pResBlockInfo->capacity = capacity; pResBlockInfo->pResBlock = pResBlock; - int32_t code = 0; if (pResBlockInfo->pResBlock == NULL) { pResBlockInfo->freeBlock = true; pResBlockInfo->pResBlock = NULL; code = createResBlock(pCond, pResBlockInfo->capacity, &pResBlockInfo->pResBlock); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pSup->numOfPks > 0) { - SSDataBlock* p = pResBlockInfo->pResBlock; + p = pResBlockInfo->pResBlock; p->info.pks[0].type = pSup->pk.type; p->info.pks[1].type = pSup->pk.type; if (IS_VAR_DATA_TYPE(pSup->pk.type)) { p->info.pks[0].pData = taosMemoryCalloc(1, pSup->pk.bytes); - if (p->info.pks[0].pData == NULL) { - return terrno; - } + TSDB_CHECK_NULL(p->info.pks[0].pData, code, lino, _end, terrno); p->info.pks[1].pData = taosMemoryCalloc(1, pSup->pk.bytes); - if (p->info.pks[1].pData == NULL) { - taosMemoryFreeClear(p->info.pks[0].pData); - return terrno; - } + TSDB_CHECK_NULL(p->info.pks[0].pData, code, lino, _end, terrno); p->info.pks[0].nData = pSup->pk.bytes; p->info.pks[1].nData = pSup->pk.bytes; @@ -529,18 +691,28 @@ static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacit pResBlockInfo->freeBlock = false; } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void** ppReader, int32_t capacity, SSDataBlock* pResBlock, const char* idstr) { - int32_t code = 0; - int8_t level = 0; - STsdbReader* pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader)); - if (pReader == NULL) { - code = terrno; - goto _end; - } + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int8_t level = 0; + STsdbReader* pReader = NULL; + SBlockLoadSuppInfo* pSup = NULL; + + TSDB_CHECK_NULL(pVnode, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pCond, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(ppReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + *ppReader = NULL; + pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader)); + TSDB_CHECK_NULL(pReader, code, lino, _end, terrno); if (VND_IS_TSMA(pVnode)) { tsdbDebug("vgId:%d, tsma is selected to query, %s", TD_VID(pVnode), idstr); @@ -552,13 +724,14 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void pReader->info.suid = pCond->suid; pReader->info.order = pCond->order; pReader->info.verRange = getQueryVerRange(pVnode, pCond, level); - pReader->info.window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); + code = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows, &pReader->info.window); + TSDB_CHECK_CODE(code, lino, _end); - pReader->idStr = (idstr != NULL) ? taosStrdup(idstr) : NULL; - if (idstr != NULL && pReader->idStr == NULL) { - code = terrno; - goto _end; + if (idstr == NULL) { + idstr = ""; } + pReader->idStr = taosStrdup(idstr); + TSDB_CHECK_NULL(pReader->idStr, code, lino, _end, terrno); pReader->type = pCond->type; pReader->bFilesetDelimited = false; @@ -566,81 +739,80 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void if (pCond->numOfCols <= 0) { tsdbError("vgId:%d, invalid column number %d in query cond, %s", TD_VID(pVnode), pCond->numOfCols, idstr); - code = TSDB_CODE_INVALID_PARA; - goto _end; + TSDB_CHECK_CONDITION(pCond->numOfCols > 0, code, lino, _end, TSDB_CODE_INVALID_PARA); } // allocate buffer in order to load data blocks from file - SBlockLoadSuppInfo* pSup = &pReader->suppInfo; + pSup = &pReader->suppInfo; pSup->tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; code = setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } + TSDB_CHECK_CODE(code, lino, _end); code = initResBlockInfo(&pReader->resBlockInfo, capacity, pResBlock, pCond, pSup); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } + TSDB_CHECK_CODE(code, lino, _end); code = tBlockDataCreate(&pReader->status.fileBlockData); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } + TSDB_CHECK_CODE(code, lino, _end); if (pReader->suppInfo.colId[0] != PRIMARYKEY_TIMESTAMP_COL_ID) { tsdbError("the first column isn't primary timestamp, %d, %s", pReader->suppInfo.colId[0], pReader->idStr); - code = TSDB_CODE_INVALID_PARA; - goto _end; + TSDB_CHECK_CONDITION(pReader->suppInfo.colId[0] == PRIMARYKEY_TIMESTAMP_COL_ID, code, lino, _end, + TSDB_CODE_INVALID_PARA); } pReader->status.pPrimaryTsCol = taosArrayGet(pReader->resBlockInfo.pResBlock->pDataBlock, pSup->slotId[0]); - if (pReader->status.pPrimaryTsCol == NULL) { - code = terrno; - goto _end; - } + TSDB_CHECK_NULL(pReader->status.pPrimaryTsCol, code, lino, _end, terrno); int32_t type = pReader->status.pPrimaryTsCol->info.type; if (type != TSDB_DATA_TYPE_TIMESTAMP) { tsdbError("the first column isn't primary timestamp in result block, actual: %s, %s", tDataTypes[type].name, pReader->idStr); - code = TSDB_CODE_INVALID_PARA; - goto _end; + TSDB_CHECK_CONDITION(type == TSDB_DATA_TYPE_TIMESTAMP, code, lino, _end, TSDB_CODE_INVALID_PARA); } code = tsdbInitReaderLock(pReader); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } + TSDB_CHECK_CODE(code, lino, _end); code = tsem_init(&pReader->resumeAfterSuspend, 0, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } + TSDB_CHECK_CODE(code, lino, _end); *ppReader = pReader; - return code; + pReader = NULL; _end: - tsdbReaderClose2(pReader); - *ppReader = NULL; + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (pReader) { + tsdbReaderClose2(pReader); + } return code; } static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileReader, SArray* pIndexList) { - int64_t st = taosGetTimestampUs(); - int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); - if (pFileReader == NULL) { - return TSDB_CODE_SUCCESS; - } - + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t st = 0; + int32_t et1 = 0; + int32_t et2 = 0; + int32_t numOfTables = 0; const TBrinBlkArray* pBlkArray = NULL; + STableUidList* pList = NULL; + SBrinBlk* pBrinBlk = NULL; - int32_t code = tsdbDataFileReadBrinBlk(pFileReader, &pBlkArray); - if (code != TSDB_CODE_SUCCESS) { - return code; + if (pFileReader == NULL) { + goto _end; } + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pIndexList, code, lino, _end, TSDB_CODE_INVALID_PARA); + + st = taosGetTimestampUs(); + numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + + code = tsdbDataFileReadBrinBlk(pFileReader, &pBlkArray); + TSDB_CHECK_CODE(code, lino, _end); + #if 0 LRUHandle* handle = NULL; @@ -659,10 +831,9 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead #endif // todo binary search to the start position - int64_t et1 = taosGetTimestampUs(); + et1 = taosGetTimestampUs(); - SBrinBlk* pBrinBlk = NULL; - STableUidList* pList = &pReader->status.uidList; + pList = &pReader->status.uidList; int32_t i = 0; while (i < TARRAY2_SIZE(pBlkArray)) { @@ -676,10 +847,10 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead break; } - if (!(pBrinBlk->minTbid.suid <= pReader->info.suid && pBrinBlk->maxTbid.suid >= pReader->info.suid)) { - tsdbError("tsdb failed at: %s %d", __func__, __LINE__); - return TSDB_CODE_INTERNAL_ERROR; - } + TSDB_CHECK_CONDITION( + (pBrinBlk->minTbid.suid <= pReader->info.suid) && (pBrinBlk->maxTbid.suid >= pReader->info.suid), code, lino, + _end, TSDB_CODE_INTERNAL_ERROR); + if (pBrinBlk->maxTbid.suid == pReader->info.suid && pBrinBlk->maxTbid.uid < pList->tableUidList[0]) { i += 1; continue; @@ -689,47 +860,55 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead break; } - void* p1 = taosArrayPush(pIndexList, pBrinBlk); - if (p1 == NULL) { - return terrno; - } + const void* p1 = taosArrayPush(pIndexList, pBrinBlk); + TSDB_CHECK_NULL(p1, code, lino, _end, terrno); i += 1; } - int64_t et2 = taosGetTimestampUs(); + et2 = taosGetTimestampUs(); tsdbDebug("load block index for %d/%d tables completed, elapsed time:%.2f ms, set BrinBlk:%.2f ms, size:%.2f Kb %s", numOfTables, (int32_t)pBlkArray->size, (et1 - st) / 1000.0, (et2 - et1) / 1000.0, pBlkArray->size * sizeof(SBrinBlk) / 1024.0, pReader->idStr); pReader->cost.headFileLoadTime += (et1 - st) / 1000.0; -//_end: - // tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum, SArray* pTableScanInfoList) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int64_t st = 0; + bool asc = false; + STimeWindow w = {0}; + SBrinRecordIter iter = {0}; + int32_t numOfTables = 0; + SBrinRecord* pRecord = NULL; int32_t k = 0; size_t sizeInDisk = 0; - int64_t st = taosGetTimestampUs(); - bool asc = ASCENDING_TRAVERSE(pReader->info.order); - STimeWindow w = pReader->info.window; - SBrinRecord* pRecord = NULL; - int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); - SBrinRecordIter iter = {0}; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockNum, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pTableScanInfoList, code, lino, _end, TSDB_CODE_INVALID_PARA); + + st = taosGetTimestampUs(); + asc = ASCENDING_TRAVERSE(pReader->info.order); + w = pReader->info.window; + numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); // clear info for the new file cleanupInfoForNextFileset(pReader->status.pTableMap); initBrinRecordIter(&iter, pReader->pFileReader, pIndexList); while (1) { - int32_t code = getNextBrinRecord(&iter, &pRecord); - if (code != TSDB_CODE_SUCCESS) { - clearBrinBlockIter(&iter); - return code; - } + code = getNextBrinRecord(&iter, &pRecord); + TSDB_CHECK_CODE(code, lino, _end); if (pRecord == NULL) { break; @@ -760,18 +939,12 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S continue; } - if (!(pRecord->suid == pReader->info.suid && uid == pRecord->uid)) { - tsdbError("tsdb failed at: %s:%d", __func__, __LINE__); - clearBrinBlockIter(&iter); - return TSDB_CODE_INTERNAL_ERROR; - } + TSDB_CHECK_CONDITION((pRecord->suid == pReader->info.suid) && (uid == pRecord->uid), code, lino, _end, + TSDB_CODE_INTERNAL_ERROR); STableBlockScanInfo* pScanInfo = NULL; code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - clearBrinBlockIter(&iter); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); // here we should find the first timestamp that is greater than the lastProcKey // the window is an open interval NOW. @@ -813,31 +986,21 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S if (pScanInfo->pBlockList == NULL) { pScanInfo->pBlockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); - if (pScanInfo->pBlockList == NULL) { - clearBrinBlockIter(&iter); - return terrno; - } + TSDB_CHECK_NULL(pScanInfo->pBlockList, code, lino, _end, terrno); } if (pScanInfo->pBlockIdxList == NULL) { pScanInfo->pBlockIdxList = taosArrayInit(4, sizeof(STableDataBlockIdx)); - if (pScanInfo->pBlockIdxList == NULL) { - clearBrinBlockIter(&iter); - return terrno; - } + TSDB_CHECK_NULL(pScanInfo->pBlockIdxList, code, lino, _end, terrno); } SFileDataBlockInfo blockInfo = {.tbBlockIdx = TARRAY_SIZE(pScanInfo->pBlockList)}; code = recordToBlockInfo(&blockInfo, pRecord); - if (code != TSDB_CODE_SUCCESS) { - clearBrinBlockIter(&iter); - return code; - } - void* p1 = taosArrayPush(pScanInfo->pBlockList, &blockInfo); - if (p1 == NULL) { - clearBrinBlockIter(&iter); - return terrno; - } + TSDB_CHECK_CODE(code, lino, _end); + sizeInDisk += blockInfo.blockSize; + + const void* p1 = taosArrayPush(pScanInfo->pBlockList, &blockInfo); + TSDB_CHECK_NULL(p1, code, lino, _end, terrno); // todo: refactor to record the fileset skey/ekey if (pScanInfo->filesetWindow.skey > pRecord->firstKey.key.ts) { @@ -851,27 +1014,18 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S pBlockNum->numOfBlocks += 1; if (taosArrayGetSize(pTableScanInfoList) == 0) { p1 = taosArrayPush(pTableScanInfoList, &pScanInfo); + TSDB_CHECK_NULL(p1, code, lino, _end, terrno); } else { STableBlockScanInfo** p = taosArrayGetLast(pTableScanInfoList); - if (p == NULL) { - clearBrinBlockIter(&iter); - tsdbError("invalid param, empty in tablescanInfoList, %s", pReader->idStr); - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA); if ((*p)->uid != uid) { p1 = taosArrayPush(pTableScanInfoList, &pScanInfo); + TSDB_CHECK_NULL(p1, code, lino, _end, terrno); } } - - if (p1 == NULL) { - clearBrinBlockIter(&iter); - return terrno; - } } - clearBrinBlockIter(&iter); - pBlockNum->numOfSttFiles = pReader->status.pCurrentFileset->lvlArr->size; int32_t total = pBlockNum->numOfSttFiles + pBlockNum->numOfBlocks; @@ -885,43 +1039,69 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S pReader->cost.numOfBlocks += total; pReader->cost.headFileLoadTime += el; - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + clearBrinBlockIter(&iter); + return code; } static void setBlockAllDumped(SFileBlockDumpInfo* pDumpInfo, int64_t maxKey, int32_t order) { - pDumpInfo->allDumped = true; + if (pDumpInfo != NULL) { + pDumpInfo->allDumped = true; + } } -static void updateLastKeyInfo(SRowKey* pKey, SFileDataBlockInfo* pBlockInfo, SDataBlockInfo* pInfo, int32_t numOfPks, - bool asc) { +static int32_t updateLastKeyInfo(SRowKey* pKey, SFileDataBlockInfo* pBlockInfo, SDataBlockInfo* pInfo, int32_t numOfPks, + bool asc) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + pKey->ts = asc ? pInfo->window.ekey : pInfo->window.skey; pKey->numOfPKs = numOfPks; if (pKey->numOfPKs <= 0) { - return; + goto _end; } + TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); if (IS_NUMERIC_TYPE(pKey->pks[0].type)) { pKey->pks[0].val = asc ? pBlockInfo->lastPk.val : pBlockInfo->firstPk.val; } else { uint8_t* p = asc ? pBlockInfo->lastPk.pData : pBlockInfo->firstPk.pData; pKey->pks[0].nData = asc ? varDataLen(pBlockInfo->lastPk.pData) : varDataLen(pBlockInfo->firstPk.pData); - (void)memcpy(pKey->pks[0].pData, p, pKey->pks[0].nData); + TAOS_MEMCPY(pKey->pks[0].pData, p, pKey->pks[0].nData); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal, SBlockLoadSuppInfo* pSup) { - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pColVal, code, lino, _end, TSDB_CODE_INVALID_PARA); if (IS_VAR_DATA_TYPE(pColVal->value.type)) { if (!COL_VAL_IS_VALUE(pColVal)) { colDataSetNULL(pColInfoData, rowIndex); } else { + TSDB_CHECK_NULL(pSup, code, lino, _end, TSDB_CODE_INVALID_PARA); varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData); if ((pColVal->value.nData + VARSTR_HEADER_SIZE) > pColInfoData->info.bytes) { tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pColVal->cid, pColVal->value.nData, pColInfoData->info.bytes); - return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + TSDB_CHECK_CODE(code, lino, _end); } if (pColVal->value.nData > 0) { // pData may be null, if nData is 0 @@ -929,31 +1109,41 @@ static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int } code = colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); + TSDB_CHECK_CODE(code, lino, _end); } } else { code = colDataSetVal(pColInfoData, rowIndex, (const char*)&pColVal->value.val, !COL_VAL_IS_VALUE(pColVal)); + TSDB_CHECK_CODE(code, lino, _end); } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo, const char* id) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + TSDB_CHECK_NULL(pBlockIter, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockIter->blockList, code, lino, _end, TSDB_CODE_INVALID_PARA); + *pInfo = NULL; - if (pBlockIter->blockList == NULL) { - return TSDB_CODE_FAILED; - } - size_t num = TARRAY_SIZE(pBlockIter->blockList); - if (num == 0) { - if (num != pBlockIter->numOfBlocks) { - tsdbError("tsdb read failed at: %s:%d %s", __func__, __LINE__, id); - } - return TSDB_CODE_FAILED; - } + TSDB_CHECK_CONDITION(num != 0, code, lino, _end, TSDB_CODE_INVALID_PARA); *pInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); - return (*pInfo) != NULL? TSDB_CODE_SUCCESS:TSDB_CODE_FAILED; + TSDB_CHECK_NULL(*pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t doBinarySearchKey(const TSKEY* keyList, int num, int pos, TSKEY key, int order) { @@ -962,7 +1152,7 @@ static int32_t doBinarySearchKey(const TSKEY* keyList, int num, int pos, TSKEY k s = pos; // check - if (!(pos >= 0 && pos < num && num > 0)) { + if (!(keyList != NULL && pos >= 0 && pos < num && num > 0)) { return -1; } if (order == TSDB_ORDER_ASC) { @@ -1025,8 +1215,15 @@ static int32_t findFirstPos(const int64_t* pTsList, int32_t num, int32_t startPo static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SBrinRecord* pRecord, int32_t pos) { // NOTE: reverse the order to find the end position in data block + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t endPos = -1; - bool asc = ASCENDING_TRAVERSE(pReader->info.order); + bool asc = false; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pRecord, code, lino, _end, TSDB_CODE_INVALID_PARA); + + asc = ASCENDING_TRAVERSE(pReader->info.order); if (asc && pReader->info.window.ekey >= pRecord->lastKey.key.ts) { endPos = pRecord->numRow - 1; @@ -1036,7 +1233,7 @@ static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData int64_t key = asc ? pReader->info.window.ekey : pReader->info.window.skey; endPos = doBinarySearchKey(pBlockData->aTSKEY, pRecord->numRow, pos, key, pReader->info.order); if (endPos == -1) { - return endPos; + goto _end; } endPos = findFirstPos(pBlockData->aTSKEY, pRecord->numRow, endPos, asc); @@ -1063,16 +1260,28 @@ static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData endPos = i; } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return endPos; } -static void copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, - int32_t dumpedRows, bool asc) { +static int32_t copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, + int32_t dumpedRows, bool asc) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + TSDB_CHECK_CONDITION((pBlockData != NULL) && (pBlockData->aTSKEY != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pDumpInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_CONDITION(dumpedRows >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA); + if (asc) { - (void)memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t)); + TAOS_MEMCPY(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t)); } else { int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1; - (void)memcpy(pColData->pData, &pBlockData->aTSKEY[startIndex], dumpedRows * sizeof(int64_t)); + TAOS_MEMCPY(pColData->pData, &pBlockData->aTSKEY[startIndex], dumpedRows * sizeof(int64_t)); // todo: opt perf by extract the loop // reverse the array list @@ -1084,12 +1293,28 @@ static void copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* p pts[dumpedRows - j - 1] = t; } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } // a faster version of copy procedure. -static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, - int32_t dumpedRows, bool asc) { +static int32_t copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, + int32_t dumpedRows, bool asc) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; uint8_t* p = NULL; + int32_t step = asc ? 1 : -1; + + TSDB_CHECK_NULL(pData, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pDumpInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_CONDITION(dumpedRows >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA); + + TSDB_CHECK_CONDITION(pData->type < TSDB_DATA_TYPE_MAX, code, lino, _end, TSDB_CODE_INVALID_PARA); if (asc) { p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; } else { @@ -1097,12 +1322,10 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo p = pData->pData + tDataTypes[pData->type].bytes * startIndex; } - int32_t step = asc ? 1 : -1; - // make sure it is aligned to 8bit, the allocated memory address is aligned to 256bit // 1. copy data in a batch model - (void)memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); + TAOS_MEMCPY(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); // 2. reverse the array list in case of descending order scan data block if (!asc) { @@ -1173,6 +1396,12 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo } } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void blockInfoToRecord(SBrinRecord* record, SFileDataBlockInfo* pBlockInfo, SBlockLoadSuppInfo* pSupp) { @@ -1211,38 +1440,54 @@ static void blockInfoToRecord(SBrinRecord* record, SFileDataBlockInfo* pBlockInf } static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastProcKey) { - SReaderStatus* pStatus = &pReader->status; - SDataBlockIter* pBlockIter = &pStatus->blockIter; - SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - - SBlockData* pBlockData = &pStatus->fileBlockData; - SFileDataBlockInfo* pBlockInfo = NULL; - SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; - int32_t numOfOutputCols = pSupInfo->numOfCols; int32_t code = TSDB_CODE_SUCCESS; - int64_t st = taosGetTimestampUs(); - bool asc = ASCENDING_TRAVERSE(pReader->info.order); - int32_t step = asc ? 1 : -1; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + SDataBlockIter* pBlockIter = NULL; + SBlockLoadSuppInfo* pSupInfo = NULL; + SFileBlockDumpInfo* pDumpInfo = NULL; + SBlockData* pBlockData = NULL; + SFileDataBlockInfo* pBlockInfo = NULL; + SSDataBlock* pResBlock = NULL; + int32_t numOfOutputCols = 0; + int64_t st = 0; + bool asc = false; + int32_t step = 0; + SColVal cv = {0}; + SBrinRecord tmp; + SBrinRecord* pRecord = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pStatus = &pReader->status; + pBlockIter = &pStatus->blockIter; + pSupInfo = &pReader->suppInfo; + pDumpInfo = &pReader->status.fBlockDumpInfo; + + pBlockData = &pStatus->fileBlockData; + pResBlock = pReader->resBlockInfo.pResBlock; + numOfOutputCols = pSupInfo->numOfCols; + st = taosGetTimestampUs(); + asc = ASCENDING_TRAVERSE(pReader->info.order); + step = asc ? 1 : -1; code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); - SColVal cv = {0}; - SBrinRecord tmp; blockInfoToRecord(&tmp, pBlockInfo, pSupInfo); - SBrinRecord* pRecord = &tmp; + pRecord = &tmp; // no data exists, return directly. if (pBlockData->nRow == 0 || pBlockData->aTSKEY == 0) { tsdbWarn("%p no need to copy since no data in blockData, table uid:%" PRIu64 " has been dropped, %s", pReader, pBlockInfo->uid, pReader->idStr); pResBlock->info.rows = 0; - return 0; + goto _end; } + TSDB_CHECK_CONDITION((pDumpInfo->rowIndex >= 0) && (pDumpInfo->rowIndex < pRecord->numRow), code, lino, _end, + TSDB_CODE_INVALID_PARA); + // row index of dump info remain the initial position, let's find the appropriate start position. if (((pDumpInfo->rowIndex == 0) && asc) || ((pDumpInfo->rowIndex == (pRecord->numRow - 1)) && (!asc))) { if (asc && pReader->info.window.skey <= pRecord->firstKey.key.ts && @@ -1263,14 +1508,15 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro "-%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 " %s", pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pRecord->firstKey.key.ts, pRecord->lastKey.key.ts, pRecord->minVer, pRecord->maxVer, pReader->idStr); - return TSDB_CODE_INVALID_PARA; + code = TSDB_CODE_INVALID_PARA; + TSDB_CHECK_CODE(code, lino, _end); } pDumpInfo->rowIndex = findFirstPos(pBlockData->aTSKEY, pRecord->numRow, pDumpInfo->rowIndex, (!asc)); - if (!(pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.maxVer >= pRecord->minVer)) { - tsdbError("tsdb failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_INVALID_PARA; - } + + TSDB_CHECK_CONDITION( + (pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.maxVer >= pRecord->minVer), code, + lino, _end, TSDB_CODE_INVALID_PARA); // find the appropriate start position that satisfies the version requirement. if ((pReader->info.verRange.maxVer >= pRecord->minVer && pReader->info.verRange.maxVer < pRecord->maxVer) || @@ -1299,7 +1545,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro int32_t endIndex = getEndPosInDataBlock(pReader, pBlockData, pRecord, pDumpInfo->rowIndex); if (endIndex == -1) { setBlockAllDumped(pDumpInfo, pReader->info.window.ekey, pReader->info.order); - return TSDB_CODE_SUCCESS; + goto _end; } endIndex += step; @@ -1308,19 +1554,18 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro dumpedRows = pReader->resBlockInfo.capacity; } else if (dumpedRows <= 0) { // no qualified rows in current data block, quit directly. setBlockAllDumped(pDumpInfo, pReader->info.window.ekey, pReader->info.order); - return TSDB_CODE_SUCCESS; + goto _end; } int32_t i = 0; int32_t rowIndex = 0; SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); - if (pColData == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA); if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { - copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc); + code = copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc); + TSDB_CHECK_CODE(code, lino, _end); i += 1; } @@ -1334,22 +1579,19 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro colIndex += 1; } else if (pData->cid == pSupInfo->colId[i]) { pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); - if (pColData == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA); if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) { colDataSetNNULL(pColData, 0, dumpedRows); } else { if (IS_MATHABLE_TYPE(pColData->info.type)) { - copyNumericCols(pData, pDumpInfo, pColData, dumpedRows, asc); + code = copyNumericCols(pData, pDumpInfo, pColData, dumpedRows, asc); + TSDB_CHECK_CODE(code, lino, _end); } else { // varchar/nchar type for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step) { tColDataGetValue(pData, j, &cv); code = doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } } } @@ -1358,9 +1600,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro i += 1; } else { // the specified column does not exist in file block, fill with null data pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); - if (pColData == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA); colDataSetNNULL(pColData, 0, dumpedRows); i += 1; @@ -1370,9 +1610,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro // fill the mis-matched columns with null value while (i < numOfOutputCols) { pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); - if (pColData == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA); colDataSetNNULL(pColData, 0, dumpedRows); i += 1; @@ -1406,63 +1644,76 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pRecord->firstKey.key.ts, pRecord->lastKey.key.ts, dumpedRows, unDumpedRows, pRecord->minVer, pRecord->maxVer, pBlockInfo->uid, elapsedTime, pReader->idStr); - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static FORCE_INLINE STSchema* getTableSchemaImpl(STsdbReader* pReader, uint64_t uid) { - if (pReader->info.pSchema != NULL) { - terrno = TSDB_CODE_INVALID_PARA; - tsdbError("tsdb invalid input param at: %s:%d", __func__, __LINE__); - return NULL; - } + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; - int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, -1, &pReader->info.pSchema); + TSDB_CHECK_CONDITION((pReader != NULL) && (pReader->info.pSchema == NULL), code, lino, _end, TSDB_CODE_INVALID_PARA); + + code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, -1, &pReader->info.pSchema); if (code != TSDB_CODE_SUCCESS || pReader->info.pSchema == NULL) { terrno = code; tsdbError("failed to get table schema, uid:%" PRIu64 ", it may have been dropped, ver:-1, %s", uid, pReader->idStr); - return NULL; } + TSDB_CHECK_CODE(code, lino, _end); + TSDB_CHECK_NULL(pReader->info.pSchema, code, lino, _end, TSDB_CODE_INTERNAL_ERROR); code = tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema); + TSDB_CHECK_CODE(code, lino, _end); + +_end: if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); terrno = code; - tsdbError("failed to init merger, code:%s, %s", tstrerror(code), pReader->idStr); return NULL; } - return pReader->info.pSchema; } static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData, uint64_t uid) { - int32_t code = 0; - STSchema* pSchema = pReader->info.pSchema; - int64_t st = taosGetTimestampUs(); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + STSchema* pSchema = NULL; SFileDataBlockInfo* pBlockInfo = NULL; - SBlockLoadSuppInfo* pSup = &pReader->suppInfo; + SBlockLoadSuppInfo* pSup = NULL; + SFileBlockDumpInfo* pDumpInfo = NULL; + int64_t st = 0; + SBrinRecord tmp; + SBrinRecord* pRecord = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pSchema = pReader->info.pSchema; + st = taosGetTimestampUs(); + pSup = &pReader->suppInfo; tBlockDataReset(pBlockData); if (pReader->info.pSchema == NULL) { pSchema = getTableSchemaImpl(pReader, uid); if (pSchema == NULL) { - code = terrno; tsdbError("%p table uid:%" PRIu64 " failed to get tableschema, code:%s, %s", pReader, uid, tstrerror(code), pReader->idStr); - return code; + TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno); } } code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + pDumpInfo = &pReader->status.fBlockDumpInfo; - SBrinRecord tmp; blockInfoToRecord(&tmp, pBlockInfo, pSup); - SBrinRecord* pRecord = &tmp; + pRecord = &tmp; code = tsdbDataFileReadBlockDataByColumn(pReader->pFileReader, pRecord, pBlockData, pSchema, &pSup->colId[1], pSup->numOfCols - 1); if (code != TSDB_CODE_SUCCESS) { @@ -1470,7 +1721,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI ", rows:%d, code:%s %s", pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->firstKey, pBlockInfo->lastKey, pBlockInfo->numRow, tstrerror(code), pReader->idStr); - return code; + TSDB_CHECK_CODE(code, lino, _end); } double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; @@ -1483,6 +1734,10 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI pReader->cost.blockLoadTime += elapsedTime; pDumpInfo->allDumped = false; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } @@ -1496,69 +1751,83 @@ static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* p (pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer); } -static bool getNeighborBlockOfTable(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pBlockInfo, - STableBlockScanInfo* pScanInfo, int32_t* nextIndex, int32_t order, - SBrinRecord* pRecord, SBlockLoadSuppInfo* pSupInfo) { - bool asc = ASCENDING_TRAVERSE(order); - int32_t step = asc ? 1 : -1; +static int32_t getNeighborBlockOfTable(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pBlockInfo, + STableBlockScanInfo* pScanInfo, int32_t* nextIndex, int32_t order, + SBrinRecord* pRecord, SBlockLoadSuppInfo* pSupInfo, bool* res) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + bool asc = false; + int32_t step = 0; + STableDataBlockIdx* pTableDataBlockIdx = NULL; + SFileDataBlockInfo* p = NULL; + + TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(res, code, lino, _end, TSDB_CODE_INVALID_PARA); + + *res = false; + asc = ASCENDING_TRAVERSE(order); + step = asc ? 1 : -1; if (asc && pBlockInfo->tbBlockIdx >= taosArrayGetSize(pScanInfo->pBlockIdxList) - 1) { - return false; + *res = false; + } else if (!asc && pBlockInfo->tbBlockIdx == 0) { + *res = false; + } else { + TSDB_CHECK_NULL(pBlockIter, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(nextIndex, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pRecord, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pSupInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pTableDataBlockIdx = taosArrayGet(pScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step); + TSDB_CHECK_NULL(pTableDataBlockIdx, code, lino, _end, TSDB_CODE_INVALID_PARA); + + p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex); + TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA); + + blockInfoToRecord(pRecord, p, pSupInfo); + + *nextIndex = pBlockInfo->tbBlockIdx + step; + *res = true; } - if (!asc && pBlockInfo->tbBlockIdx == 0) { - return false; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } - - STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step); - if (pTableDataBlockIdx == NULL) { - return TSDB_CODE_INVALID_PARA; - } - - SFileDataBlockInfo* p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex); - if (p == NULL) { - return TSDB_CODE_INVALID_PARA; - } - - blockInfoToRecord(pRecord, p, pSupInfo); - - *nextIndex = pBlockInfo->tbBlockIdx + step; - return true; + return code; } static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t index, int32_t step) { - int32_t code = TSDB_CODE_SUCCESS; - if (index < 0 || index >= pBlockIter->numOfBlocks) { - return TSDB_CODE_FAILED; - } + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + const void* p = NULL; + SFileDataBlockInfo fblock; + SFileDataBlockInfo* pBlockInfo = NULL; + STableBlockScanInfo* pBlockScanInfo = NULL; + STableDataBlockIdx* pTableDataBlockIdx = NULL; - void* p = taosArrayGet(pBlockIter->blockList, index); - if (p == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pBlockIter, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_CONDITION((index >= 0) && (index < pBlockIter->numOfBlocks), code, lino, _end, TSDB_CODE_INVALID_PARA); - SFileDataBlockInfo fblock = *(SFileDataBlockInfo*) p; + p = taosArrayGet(pBlockIter->blockList, index); + TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA); + + fblock = *(SFileDataBlockInfo*)p; pBlockIter->index += step; if (index != pBlockIter->index) { if (index > pBlockIter->index) { for (int32_t i = index - 1; i >= pBlockIter->index; --i) { - SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i); - if (pBlockInfo == NULL) { - return TSDB_CODE_INVALID_PARA; - } + pBlockInfo = taosArrayGet(pBlockIter->blockList, i); + TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); - STableBlockScanInfo* pBlockScanInfo = NULL; code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); - STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx); - if (pTableDataBlockIdx == NULL) { - return TSDB_CODE_INVALID_PARA; - } + pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx); + TSDB_CHECK_NULL(pTableDataBlockIdx, code, lino, _end, TSDB_CODE_INVALID_PARA); pTableDataBlockIdx->globalIndex = i + 1; @@ -1566,21 +1835,14 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte } } else if (index < pBlockIter->index) { for (int32_t i = index + 1; i <= pBlockIter->index; ++i) { - SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i); - if (pBlockInfo == NULL) { - return TSDB_CODE_INVALID_PARA; - } + pBlockInfo = taosArrayGet(pBlockIter->blockList, i); + TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); - STableBlockScanInfo* pBlockScanInfo = NULL; code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); - STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx); - if (pTableDataBlockIdx == NULL) { - return TSDB_CODE_INVALID_PARA; - } + pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx); + TSDB_CHECK_NULL(pTableDataBlockIdx, code, lino, _end, TSDB_CODE_INVALID_PARA); pTableDataBlockIdx->globalIndex = i - 1; taosArraySet(pBlockIter->blockList, i - 1, pBlockInfo); @@ -1588,21 +1850,21 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte } taosArraySet(pBlockIter->blockList, pBlockIter->index, &fblock); - STableBlockScanInfo* pBlockScanInfo = NULL; + pBlockScanInfo = NULL; code = getTableBlockScanInfo(pReader->status.pTableMap, fblock.uid, &pBlockScanInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); - STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, fblock.tbBlockIdx); - if (pTableDataBlockIdx == NULL) { - return TSDB_CODE_INVALID_PARA; - } + pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, fblock.tbBlockIdx); + TSDB_CHECK_NULL(pTableDataBlockIdx, code, lino, _end, TSDB_CODE_INVALID_PARA); pTableDataBlockIdx->globalIndex = pBlockIter->index; } - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } // todo: this attribute could be acquired during extractin the global ordered block list. @@ -1672,15 +1934,26 @@ static bool keyOverlapFileBlock(TSDBKEY key, SFileDataBlockInfo* pBlock, SVersio (pBlock->minVer <= pVerRange->maxVer); } -static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo, - STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, STsdbReader* pReader) { +static int32_t getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo, + STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SBrinRecord rec = {0}; int32_t neighborIndex = 0; - int32_t order = pReader->info.order; - SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; + int32_t order = 0; + SBlockLoadSuppInfo* pSupInfo = NULL; + SBrinRecord pRecord; + bool hasNeighbor = false; - bool hasNeighbor = - getNeighborBlockOfTable(&pReader->status.blockIter, pBlockInfo, pScanInfo, &neighborIndex, order, &rec, pSupInfo); + TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + order = pReader->info.order; + pSupInfo = &pReader->suppInfo; + + code = getNeighborBlockOfTable(&pReader->status.blockIter, pBlockInfo, pScanInfo, &neighborIndex, order, &rec, + pSupInfo, &hasNeighbor); + TSDB_CHECK_CODE(code, lino, _end); // overlap with neighbor if (hasNeighbor) { @@ -1688,7 +1961,6 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* overlapWithNeighborBlock2(pBlockInfo, &rec, order, pSupInfo->pk.type, pSupInfo->numOfPks); } - SBrinRecord pRecord; blockInfoToRecord(&pRecord, pBlockInfo, pSupInfo); // has duplicated ts of different version in this block @@ -1704,6 +1976,12 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pInfo->moreThanCapcity = pBlockInfo->numRow > pReader->resBlockInfo.capacity; pInfo->partiallyRequired = dataBlockPartiallyRequired(&pReader->info.window, &pReader->info.verRange, pBlockInfo); pInfo->overlapWithKeyInBuf = keyOverlapFileBlock(keyInBuf, pBlockInfo, &pReader->info.verRange); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } // 1. the version of all rows should be less than the endVersion @@ -1712,17 +1990,23 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* // 4. output buffer should be large enough to hold all rows in current block // 5. delete info should not overlap with current block data // 6. current block should not contain the duplicated ts -static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, - TSDBKEY keyInBuf) { +static int32_t fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, + TSDBKEY keyInBuf, bool* load) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SDataBlockToLoadInfo info = {0}; - getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pReader); - bool loadDataBlock = - (info.overlapWithNeighborBlock || info.hasDupTs || info.partiallyRequired || info.overlapWithKeyInBuf || - info.moreThanCapcity || info.overlapWithDelInfo || info.overlapWithSttBlock); + TSDB_CHECK_NULL(load, code, lino, _end, TSDB_CODE_INVALID_PARA); + + *load = false; + code = getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pReader); + TSDB_CHECK_CODE(code, lino, _end); + + *load = (info.overlapWithNeighborBlock || info.hasDupTs || info.partiallyRequired || info.overlapWithKeyInBuf || + info.moreThanCapcity || info.overlapWithDelInfo || info.overlapWithSttBlock); // log the reason why load the datablock for profile - if (loadDataBlock) { + if (*load) { tsdbDebug("%p uid:%" PRIu64 " need to load the datablock, overlapneighbor:%d, hasDup:%d, partiallyRequired:%d, " "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithSttBlock:%d, %s", @@ -1731,45 +2015,76 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlock pReader->idStr); } - return loadDataBlock; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, - TSDBKEY keyInBuf) { +static int32_t isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, + STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, bool* res) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SDataBlockToLoadInfo info = {0}; - getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pReader); - bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf || - info.overlapWithDelInfo || info.overlapWithSttBlock); - return isCleanFileBlock; + + TSDB_CHECK_NULL(res, code, lino, _end, TSDB_CODE_INVALID_PARA); + + *res = false; + code = getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pReader); + TSDB_CHECK_CODE(code, lino, _end); + + *res = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf || info.overlapWithDelInfo || + info.overlapWithSttBlock); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t initRowMergeIfNeeded(STsdbReader* pReader, int64_t uid) { - SRowMerger* pMerger = &pReader->status.merger; - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SRowMerger* pMerger = NULL; + STSchema* ps = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pMerger = &pReader->status.merger; if (pMerger->pArray == NULL) { - STSchema* ps = getTableSchemaImpl(pReader, uid); - if (ps == NULL) { - return terrno; - } + ps = getTableSchemaImpl(pReader, uid); + TSDB_CHECK_NULL(ps, code, lino, _end, terrno); } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, int64_t endKey) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int64_t st = 0; + SSDataBlock* pBlock = NULL; + + TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + if (!(pBlockScanInfo->iiter.hasVal || pBlockScanInfo->iter.hasVal)) { - return TSDB_CODE_SUCCESS; + goto _end; } - int32_t code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid); - if (code != 0) { - return code; - } + code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid); + TSDB_CHECK_CODE(code, lino, _end); - int64_t st = taosGetTimestampUs(); - SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + st = taosGetTimestampUs(); + pBlock = pReader->resBlockInfo.pResBlock; code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->resBlockInfo.capacity, pReader); + TSDB_CHECK_CODE(code, lino, _end); double el = (taosGetTimestampUs() - st) / 1000.0; updateComposedBlockInfo(pReader, el, pBlockScanInfo); @@ -1780,43 +2095,65 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* pReader->idStr); pReader->cost.buildmemBlock += el; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } -static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pBlockData, SRowKey* pKey, - SFileBlockDumpInfo* pDumpInfo, bool* copied) { +static int32_t tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pBlockData, SRowKey* pKey, + SFileBlockDumpInfo* pDumpInfo, bool* copied) { // opt version // 1. it is not a border point // 2. the direct next point is not an duplicated timestamp int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + bool asc = false; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pDumpInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(copied, code, lino, _end, TSDB_CODE_INVALID_PARA); *copied = false; - bool asc = (pReader->info.order == TSDB_ORDER_ASC); + asc = (pReader->info.order == TSDB_ORDER_ASC); if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) { int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; SRowKey nextRowKey; + TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA); tColRowGetKey(pBlockData, pDumpInfo->rowIndex + step, &nextRowKey); if (pkCompEx(pKey, &nextRowKey) != 0) { // merge is not needed code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); pDumpInfo->rowIndex += step; *copied = true; } } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, int32_t pkSrcSlot, SVersionRange* pVerRange) { - int32_t code = 0; - int32_t order = pSttBlockReader->order; - int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1; - SRowKey* pNextProc = &pScanInfo->sttKeyInfo.nextProcKey; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t order = 0; + int32_t step = 0; + SRowKey* pNextProc = NULL; + + TSDB_CHECK_NULL(pSttBlockReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + + order = pSttBlockReader->order; + step = ASCENDING_TRAVERSE(order) ? 1 : -1; + pNextProc = &pScanInfo->sttKeyInfo.nextProcKey; while (1) { bool hasVal = false; @@ -1824,7 +2161,7 @@ static int32_t nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBloc if (code) { tsdbError("failed to iter the next row in stt-file merge tree, code:%s, %s", tstrerror(code), pSttBlockReader->mergeTree.idStr); - return code; + TSDB_CHECK_CODE(code, lino, _end); } if (!hasVal) { // the next value will be the accessed key in stt @@ -1839,7 +2176,7 @@ static int32_t nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBloc memset(pNextProc->pks[0].pData, 0, pNextProc->pks[0].nData); } } - return code; + goto _end; } TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree); @@ -1855,17 +2192,24 @@ static int32_t nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBloc tColRowGetKeyDeepCopy(pRow->pBlockData, pRow->iRow, pkSrcSlot, pNextProc); if (pScanInfo->delSkyline != NULL && TARRAY_SIZE(pScanInfo->delSkyline) > 0) { - if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, order, pVerRange, - pSttBlockReader->numOfPks > 0)) { + bool dropped = false; + code = hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, order, pVerRange, + pSttBlockReader->numOfPks > 0, &dropped); + TSDB_CHECK_CODE(code, lino, _end); + if (!dropped) { pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; - return code; + goto _end; } } else { pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; - return code; + goto _end; } } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } @@ -1876,89 +2220,113 @@ static void doUnpinSttBlock(SSttBlockReader* pSttBlockReader) { tMergeTreeUnpinS static int32_t tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, SRowKey* pSttKey, STsdbReader* pReader, bool* copied) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SRowKey* pNext = NULL; + + TSDB_CHECK_NULL(pSttBlockReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(copied, code, lino, _end, TSDB_CODE_INVALID_PARA); + *copied = false; // avoid the fetch next row replace the referenced stt block in buffer doPinSttBlock(pSttBlockReader); code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange); doUnpinSttBlock(pSttBlockReader); - - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (hasDataInSttBlock(pScanInfo)) { - SRowKey* pNext = getCurrentKeyInSttBlock(pSttBlockReader); + pNext = getCurrentKeyInSttBlock(pSttBlockReader); if (pkCompEx(pSttKey, pNext) != 0) { code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow); *copied = (code == TSDB_CODE_SUCCESS); - return code; + TSDB_CHECK_CODE(code, lino, _end); } } else { code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow); *copied = (code == TSDB_CODE_SUCCESS); - return code; + TSDB_CHECK_CODE(code, lino, _end); } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + STSchema* ps = NULL; + void** p = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + // always set the newest schema version in pReader->info.pSchema if (pReader->info.pSchema == NULL) { - STSchema* ps = getTableSchemaImpl(pReader, uid); - if (ps == NULL) { - return NULL; - } + ps = getTableSchemaImpl(pReader, uid); + TSDB_CHECK_NULL(ps, code, lino, _end, terrno); } if (pReader->info.pSchema && sversion == pReader->info.pSchema->version) { - return pReader->info.pSchema; + ps = pReader->info.pSchema; + goto _end; } - void** p = tSimpleHashGet(pReader->pSchemaMap, &sversion, sizeof(sversion)); + p = tSimpleHashGet(pReader->pSchemaMap, &sversion, sizeof(sversion)); if (p != NULL) { - return *(STSchema**)p; + ps = *(STSchema**)p; + goto _end; } - STSchema* ptr = NULL; - int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, sversion, &ptr); + code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, sversion, &ps); + TSDB_CHECK_CODE(code, lino, _end); + + code = tSimpleHashPut(pReader->pSchemaMap, &sversion, sizeof(sversion), &ps, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); + +_end: if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); terrno = code; return NULL; - } else { - code = tSimpleHashPut(pReader->pSchemaMap, &sversion, sizeof(sversion), &ptr, POINTER_BYTES); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return NULL; - } - return ptr; } + return ps; } static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, SIterInfo* pIter, SSttBlockReader* pSttBlockReader) { - SRowMerger* pMerger = &pReader->status.merger; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SRowMerger* pMerger = NULL; SRow* pTSRow = NULL; - SBlockData* pBlockData = &pReader->status.fileBlockData; - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - int32_t pkSrcSlot = pReader->suppInfo.pkSrcSlot; + SBlockData* pBlockData = NULL; + SFileBlockDumpInfo* pDumpInfo = NULL; + SRowKey* pSttKey = NULL; + int32_t pkSrcSlot = 0; + SRowKey k = {0}; + STSchema* pSchema = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pRow, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pMerger = &pReader->status.merger; + pBlockData = &pReader->status.fileBlockData; + pDumpInfo = &pReader->status.fBlockDumpInfo; + pkSrcSlot = pReader->suppInfo.pkSrcSlot; - SRowKey* pSttKey = NULL; if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { pSttKey = getCurrentKeyInSttBlock(pSttBlockReader); } - SRowKey k = {0}; tRowGetKeyEx(pRow, &k); - STSchema* pSchema = NULL; if (pRow->type == TSDBROW_ROW_FMT) { pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - if (pSchema == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno); } SRowKey* pfKey = &(SRowKey){0}; @@ -1971,10 +2339,8 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized - int32_t code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid); - if (code != 0) { - return code; - } + code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid); + TSDB_CHECK_CODE(code, lino, _end); SRowKey minKey = k; if (pReader->info.order == TSDB_ORDER_ASC) { @@ -2002,79 +2368,83 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* // file block ---> stt block -----> mem if (pkCompEx(&minKey, pfKey) == 0) { code = tsdbRowMergerAdd(pMerger, &fRow, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } if (pkCompEx(&minKey, pSttKey) == 0) { TSDBROW* fRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); code = tsdbRowMergerAdd(pMerger, fRow1, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); + code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, + pReader->idStr); + TSDB_CHECK_CODE(code, lino, _end); } if (pkCompEx(&minKey, &k) == 0) { code = tsdbRowMergerAdd(pMerger, pRow, pSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, &k, pBlockScanInfo->delSkyline, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } code = tsdbRowMergerGetRow(pMerger, &pTSRow); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); - taosMemoryFree(pTSRow); + taosMemoryFreeClear(pTSRow); tsdbRowMergerClear(pMerger); + TSDB_CHECK_CODE(code, lino, _end); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* pSttBlockReader, SRowKey* pKey, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SRowMerger* pMerger = &pReader->status.merger; int32_t code = TSDB_CODE_SUCCESS; - int32_t pkSrcSlot = pReader->suppInfo.pkSrcSlot; + int32_t lino = 0; + SFileBlockDumpInfo* pDumpInfo = NULL; + SRowMerger* pMerger = NULL; + SRow* pTSRow = NULL; + int32_t pkSrcSlot = 0; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pDumpInfo = &pReader->status.fBlockDumpInfo; + pMerger = &pReader->status.merger; + pkSrcSlot = pReader->suppInfo.pkSrcSlot; // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid); - if (code != 0) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); bool dataInDataFile = hasDataInFileBlock(pBlockData, pDumpInfo); bool dataInSttFile = hasDataInSttBlock(pBlockScanInfo); if (dataInDataFile && (!dataInSttFile)) { // no stt file block available, only data block exists - return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + code = mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + TSDB_CHECK_CODE(code, lino, _end); } else if ((!dataInDataFile) && dataInSttFile) { // no data in data file exists - return mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader); + code = mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader); + TSDB_CHECK_CODE(code, lino, _end); } else if (pBlockScanInfo->cleanSttBlocks && pReader->info.execMode == READER_EXEC_ROWS) { // opt model for count data in stt file, which is not overlap with data blocks in files. - return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + code = mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + TSDB_CHECK_CODE(code, lino, _end); } else { + TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA); // row in both stt file blocks and data file blocks TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); SRowKey* pSttKey = getCurrentKeyInSttBlock(pSttBlockReader); @@ -2083,74 +2453,90 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* if (ASCENDING_TRAVERSE(pReader->info.order)) { if (ret < 0) { // asc - return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + code = mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + TSDB_CHECK_CODE(code, lino, _end); } else if (ret > 0) { - return mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader); + code = mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader); + TSDB_CHECK_CODE(code, lino, _end); } } else { // desc if (ret > 0) { - return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + code = mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + TSDB_CHECK_CODE(code, lino, _end); } else if (ret < 0) { - return mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader); + code = mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader); + TSDB_CHECK_CODE(code, lino, _end); } } + if (ret != 0) { + goto _end; + } // pKey == pSttKey tRowKeyAssign(&pBlockScanInfo->lastProcKey, pKey); // the following for key == sttKey->key.ts // file block ------> stt block - SRow* pTSRow = NULL; + code = tsdbRowMergerAdd(pMerger, &fRow, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); code = tsdbRowMergerAdd(pMerger, pRow1, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); // pSttKey will be changed when sttBlockReader iterates to the next row, so use pKey instead. - code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, + pReader->idStr); + TSDB_CHECK_CODE(code, lino, _end); code = tsdbRowMergerGetRow(pMerger, &pTSRow); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); - taosMemoryFree(pTSRow); + taosMemoryFreeClear(pTSRow); tsdbRowMergerClear(pMerger); - return code; + TSDB_CHECK_CODE(code, lino, _end); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SSttBlockReader* pSttBlockReader) { - SRowMerger* pMerger = &pReader->status.merger; - SRow* pTSRow = NULL; int32_t code = TSDB_CODE_SUCCESS; - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SArray* pDelList = pBlockScanInfo->delSkyline; - int32_t pkSrcSlot = pReader->suppInfo.pkSrcSlot; + int32_t lino = 0; + SRowMerger* pMerger = NULL; + SRow* pTSRow = NULL; + SFileBlockDumpInfo* pDumpInfo = NULL; + SArray* pDelList = NULL; + int32_t pkSrcSlot = 0; TSDBROW* pRow = NULL; TSDBROW* piRow = NULL; + SRowKey* pSttKey = NULL; - getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader, &pRow); - getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader, &piRow); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pMerger = &pReader->status.merger; + pDumpInfo = &pReader->status.fBlockDumpInfo; + pDelList = pBlockScanInfo->delSkyline; + pkSrcSlot = pReader->suppInfo.pkSrcSlot; + + code = getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader, &pRow); + TSDB_CHECK_CODE(code, lino, _end); + code = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader, &piRow); + TSDB_CHECK_CODE(code, lino, _end); - SRowKey* pSttKey = NULL; if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { pSttKey = getCurrentKeyInSttBlock(pSttBlockReader); } @@ -2169,24 +2555,18 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* STSchema* pSchema = NULL; if (pRow->type == TSDBROW_ROW_FMT) { pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - if (pSchema == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno); } STSchema* piSchema = NULL; if (piRow->type == TSDBROW_ROW_FMT) { piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - if (piSchema == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno); } // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid); - if (code != 0) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); SRowKey minKey = k; if (ASCENDING_TRAVERSE(pReader->info.order)) { @@ -2221,72 +2601,70 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (pkCompEx(&minKey, pfKey) == 0) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); code = tsdbRowMergerAdd(pMerger, &fRow, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } if (pkCompEx(&minKey, pSttKey) == 0) { TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); code = tsdbRowMergerAdd(pMerger, pRow1, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); - code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, + pReader->idStr); + TSDB_CHECK_CODE(code, lino, _end); } if (pkCompEx(&minKey, &ik) == 0) { code = tsdbRowMergerAdd(pMerger, piRow, piSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, &ik, pBlockScanInfo->delSkyline, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } if (pkCompEx(&minKey, &k) == 0) { code = tsdbRowMergerAdd(pMerger, pRow, pSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, &k, pBlockScanInfo->delSkyline, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } code = tsdbRowMergerGetRow(pMerger, &pTSRow); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); - taosMemoryFree(pTSRow); + taosMemoryFreeClear(pTSRow); tsdbRowMergerClear(pMerger); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } int32_t doInitMemDataIter(STsdbReader* pReader, STbData** pData, STableBlockScanInfo* pBlockScanInfo, STsdbRowKey* pKey, SMemTable* pMem, SIterInfo* pIter, const char* type) { int32_t code = TSDB_CODE_SUCCESS; - int32_t backward = (!ASCENDING_TRAVERSE(pReader->info.order)); + int32_t lino = 0; + int32_t backward = false; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(type, code, lino, _end, TSDB_CODE_INVALID_PARA); + + backward = (!ASCENDING_TRAVERSE(pReader->info.order)); pIter->hasVal = false; if (pMem != NULL) { + TSDB_CHECK_NULL(pData, code, lino, _end, TSDB_CODE_INVALID_PARA); *pData = tsdbGetTbDataFromMemTable(pMem, pReader->info.suid, pBlockScanInfo->uid); if ((*pData) != NULL) { @@ -2301,23 +2679,30 @@ int32_t doInitMemDataIter(STsdbReader* pReader, STbData** pData, STableBlockScan } else { tsdbError("%p uid:%" PRIu64 ", failed to create iterator for %s, code:%s, %s", pReader, pBlockScanInfo->uid, type, tstrerror(code), pReader->idStr); - return code; + TSDB_CHECK_CODE(code, lino, _end); } } } else { tsdbDebug("%p uid:%" PRIu64 ", no data in %s, %s", pReader, pBlockScanInfo->uid, type, pReader->idStr); } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } -static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo, - STsdbReader* pReader) { - SRowKey rowKey = {0}; +static int32_t doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo, + STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SRowKey rowKey = {0}; TSDBROW* pRow = NULL; while (1) { - getValidMemRow(pIter, pBlockScanInfo->delSkyline, pReader, &pRow); + code = getValidMemRow(pIter, pBlockScanInfo->delSkyline, pReader, &pRow); + TSDB_CHECK_CODE(code, lino, _end); if (!pIter->hasVal) { break; } @@ -2330,27 +2715,56 @@ static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanIn break; } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } // handle the open interval issue. Find the first row key that is greater than the given one. -static void forwardDataIter(SRowKey* pKey, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { - doForwardDataIter(pKey, &pBlockScanInfo->iter, pBlockScanInfo, pReader); - doForwardDataIter(pKey, &pBlockScanInfo->iiter, pBlockScanInfo, pReader); +static int32_t forwardDataIter(SRowKey* pKey, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + + code = doForwardDataIter(pKey, &pBlockScanInfo->iter, pBlockScanInfo, pReader); + TSDB_CHECK_CODE(code, lino, _end); + code = doForwardDataIter(pKey, &pBlockScanInfo->iiter, pBlockScanInfo, pReader); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; STbData* d = NULL; STbData* di = NULL; - bool asc = ASCENDING_TRAVERSE(pReader->info.order); + bool asc = false; bool forward = true; - STsdbReadSnap* pSnap = pReader->pReadSnap; - STimeWindow* pWindow = &pReader->info.window; + STsdbReadSnap* pSnap = NULL; + STimeWindow* pWindow = NULL; + STsdbRowKey startKey; + + TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + asc = ASCENDING_TRAVERSE(pReader->info.order); + pSnap = pReader->pReadSnap; + pWindow = &pReader->info.window; if (pBlockScanInfo->iterInit) { - return TSDB_CODE_SUCCESS; + goto _end; } - STsdbRowKey startKey; startKey.key = pBlockScanInfo->lastProcKey; startKey.version = asc ? pReader->info.verRange.minVer : pReader->info.verRange.maxVer; if ((asc && (startKey.key.ts < pWindow->skey)) || ((!asc) && startKey.key.ts > pWindow->ekey)) { @@ -2358,80 +2772,114 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea forward = false; } - int32_t code = doInitMemDataIter(pReader, &d, pBlockScanInfo, &startKey, pSnap->pMem, &pBlockScanInfo->iter, "mem"); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = doInitMemDataIter(pReader, &d, pBlockScanInfo, &startKey, pSnap->pMem, &pBlockScanInfo->iter, "mem"); + TSDB_CHECK_CODE(code, lino, _end); code = doInitMemDataIter(pReader, &di, pBlockScanInfo, &startKey, pSnap->pIMem, &pBlockScanInfo->iiter, "imem"); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = loadMemTombData(&pBlockScanInfo->pMemDelData, d, di, pReader->info.verRange.maxVer); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (forward) { - forwardDataIter(&startKey.key, pBlockScanInfo, pReader); + code = forwardDataIter(&startKey.key, pBlockScanInfo, pReader); + TSDB_CHECK_CODE(code, lino, _end); } pBlockScanInfo->iterInit = true; - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static bool isValidFileBlockRow(SBlockData* pBlockData, int32_t rowIndex, STableBlockScanInfo* pBlockScanInfo, bool asc, - STsdbReaderInfo* pInfo, STsdbReader* pReader) { +static int32_t isValidFileBlockRow(SBlockData* pBlockData, int32_t rowIndex, STableBlockScanInfo* pBlockScanInfo, + bool asc, STsdbReaderInfo* pInfo, STsdbReader* pReader, bool* valid) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(valid, code, lino, _end, TSDB_CODE_INVALID_PARA); + + *valid = false; // it is an multi-table data block if (pBlockData->aUid != NULL) { uint64_t uid = pBlockData->aUid[rowIndex]; if (uid != pBlockScanInfo->uid) { // move to next row - return false; + *valid = false; + goto _end; } } + TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); // check for version and time range int64_t ver = pBlockData->aVersion[rowIndex]; if (ver > pInfo->verRange.maxVer || ver < pInfo->verRange.minVer) { - return false; + *valid = false; + goto _end; } int64_t ts = pBlockData->aTSKEY[rowIndex]; if (ts > pInfo->window.ekey || ts < pInfo->window.skey) { - return false; + *valid = false; + goto _end; } + TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); if ((asc && (ts < pBlockScanInfo->lastProcKey.ts)) || ((!asc) && (ts > pBlockScanInfo->lastProcKey.ts))) { - return false; + *valid = false; + goto _end; } if (ts == pBlockScanInfo->lastProcKey.ts) { // todo opt perf SRowKey nextRowKey; // lazy eval tColRowGetKey(pBlockData, rowIndex, &nextRowKey); if (pkCompEx(&pBlockScanInfo->lastProcKey, &nextRowKey) == 0) { - return false; + *valid = false; + goto _end; } } if (pBlockScanInfo->delSkyline != NULL && TARRAY_SIZE(pBlockScanInfo->delSkyline) > 0) { - bool dropped = hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, pInfo->order, - &pInfo->verRange, pReader->suppInfo.numOfPks > 0); + bool dropped = false; + code = hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, pInfo->order, + &pInfo->verRange, pReader->suppInfo.numOfPks > 0, &dropped); + TSDB_CHECK_CODE(code, lino, _end); if (dropped) { - return false; + *valid = false; + goto _end; } } - return true; + *valid = true; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { - int32_t order = pReader->info.order; - bool asc = ASCENDING_TRAVERSE(order); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t order = 0; + bool asc = false; + int64_t st = 0; + SSttDataInfoForTable info = (SSttDataInfoForTable){0}; + + TSDB_CHECK_NULL(pSttBlockReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + order = pReader->info.order; + asc = ASCENDING_TRAVERSE(order); // the stt block reader has been initialized for this table. if (pSttBlockReader->uid == pScanInfo->uid) { - return; + goto _end; } if (pSttBlockReader->uid != 0) { @@ -2447,7 +2895,7 @@ static void initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA; tsdbDebug("uid:%" PRIu64 " set no stt-file data after stt-block retrieved, %s", pScanInfo->uid, pReader->idStr); } - return; + goto _end; } STimeWindow w = pSttBlockReader->window; @@ -2457,7 +2905,7 @@ static void initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan w.ekey = pScanInfo->sttKeyInfo.nextProcKey.ts; } - int64_t st = taosGetTimestampUs(); + st = taosGetTimestampUs(); tsdbDebug("init stt block reader, window:%" PRId64 "-%" PRId64 ", uid:%" PRIu64 ", %s", w.skey, w.ekey, pScanInfo->uid, pReader->idStr); @@ -2481,32 +2929,17 @@ static void initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan .rspRows = (pReader->info.execMode == READER_EXEC_ROWS), }; - SSttDataInfoForTable info = {.pKeyRangeList = taosArrayInit(4, sizeof(SSttKeyRange))}; - if (info.pKeyRangeList == NULL) { - pReader->code = terrno; - return; - } + info.pKeyRangeList = taosArrayInit(4, sizeof(SSttKeyRange)); + TSDB_CHECK_NULL(info.pKeyRangeList, code, lino, _end, terrno); - int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(info.pKeyRangeList); - pReader->code = code; - return; - } + code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info); + TSDB_CHECK_CODE(code, lino, _end); code = initMemDataIterator(pScanInfo, pReader); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(info.pKeyRangeList); - pReader->code = code; - return; - } + TSDB_CHECK_CODE(code, lino, _end); code = initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(info.pKeyRangeList); - pReader->code = code; - return; - } + TSDB_CHECK_CODE(code, lino, _end); if (conf.rspRows) { pScanInfo->cleanSttBlocks = isCleanSttBlock(info.pKeyRangeList, &pReader->info.window, pScanInfo, order); @@ -2536,23 +2969,35 @@ static void initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan } else { // not clean stt blocks INIT_KEYRANGE(&pScanInfo->sttRange); // reset the time window code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange); + if (code != TSDB_CODE_SUCCESS) { + pScanInfo->sttBlockReturned = false; + TSDB_CHECK_CODE(code, lino, _end); + } } } else { pScanInfo->cleanSttBlocks = false; INIT_KEYRANGE(&pScanInfo->sttRange); // reset the time window code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange); + if (code != TSDB_CODE_SUCCESS) { + pScanInfo->sttBlockReturned = false; + TSDB_CHECK_CODE(code, lino, _end); + } } pScanInfo->sttBlockReturned = false; - taosArrayDestroy(info.pKeyRangeList); - int64_t el = taosGetTimestampUs() - st; pReader->cost.initSttBlockReader += (el / 1000.0); tsdbDebug("init stt block reader completed, elapsed time:%" PRId64 "us %s", el, pReader->idStr); - if (code != 0) { - pReader->code = code; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + if (pReader) { + pReader->code = code; + } } + taosArrayDestroy(info.pKeyRangeList); } static bool hasDataInSttBlock(STableBlockScanInfo* pInfo) { return pInfo->sttKeyInfo.status == STT_FILE_HAS_DATA; } @@ -2566,57 +3011,71 @@ bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, SRowKey* pKey, STsdbReader* pReader) { - SRowMerger* pMerger = &pReader->status.merger; - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SRow* pTSRow = NULL; + SRowMerger* pMerger = NULL; + SFileBlockDumpInfo* pDumpInfo = NULL; bool copied = false; - int32_t code = tryCopyDistinctRowFromFileBlock(pReader, pBlockData, pKey, pDumpInfo, &copied); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pMerger = &pReader->status.merger; + pDumpInfo = &pReader->status.fBlockDumpInfo; + + code = tryCopyDistinctRowFromFileBlock(pReader, pBlockData, pKey, pDumpInfo, &copied); + TSDB_CHECK_CODE(code, lino, _end); // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid); - if (code != 0) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); tRowKeyAssign(&pBlockScanInfo->lastProcKey, pKey); if (copied) { - return TSDB_CODE_SUCCESS; - } else { - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - - SRow* pTSRow = NULL; - code = tsdbRowMergerAdd(pMerger, &fRow, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = tsdbRowMergerGetRow(pMerger, &pTSRow); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); - - taosMemoryFree(pTSRow); - tsdbRowMergerClear(pMerger); - return code; + goto _end; } + + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + + code = tsdbRowMergerAdd(pMerger, &fRow, NULL); + TSDB_CHECK_CODE(code, lino, _end); + + code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader); + TSDB_CHECK_CODE(code, lino, _end); + + code = tsdbRowMergerGetRow(pMerger, &pTSRow); + TSDB_CHECK_CODE(code, lino, _end); + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); + + taosMemoryFreeClear(pTSRow); + tsdbRowMergerClear(pMerger); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { - bool copied = false; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SRow* pTSRow = NULL; - int32_t pkSrcSlot = pReader->suppInfo.pkSrcSlot; - SRowMerger* pMerger = &pReader->status.merger; + int32_t pkSrcSlot = 0; + SRowMerger* pMerger = NULL; + bool copied = false; + + TSDB_CHECK_NULL(pSttBlockReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pkSrcSlot = pReader->suppInfo.pkSrcSlot; + pMerger = &pReader->status.merger; // let's record the last processed key tRowKeyAssign(&pScanInfo->lastProcKey, getCurrentKeyInSttBlock(pSttBlockReader)); @@ -2630,51 +3089,54 @@ int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanIn pReader->idStr); } - int32_t code = - tryCopyDistinctRowFromSttBlock(&fRow, pSttBlockReader, pScanInfo, &pScanInfo->lastProcKey, pReader, &copied); - if (code) { - return code; - } + code = tryCopyDistinctRowFromSttBlock(&fRow, pSttBlockReader, pScanInfo, &pScanInfo->lastProcKey, pReader, &copied); + TSDB_CHECK_CODE(code, lino, _end); if (copied) { - return TSDB_CODE_SUCCESS; - } else { - code = tsdbRowMergerAdd(pMerger, &fRow, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); - code = tsdbRowMergerAdd(pMerger, pRow1, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = doMergeRowsInSttBlock(pSttBlockReader, pScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = tsdbRowMergerGetRow(pMerger, &pTSRow); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pScanInfo); - - taosMemoryFree(pTSRow); - tsdbRowMergerClear(pMerger); - return code; + goto _end; } + + code = tsdbRowMergerAdd(pMerger, &fRow, NULL); + TSDB_CHECK_CODE(code, lino, _end); + + TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); + code = tsdbRowMergerAdd(pMerger, pRow1, NULL); + TSDB_CHECK_CODE(code, lino, _end); + + code = doMergeRowsInSttBlock(pSttBlockReader, pScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr); + TSDB_CHECK_CODE(code, lino, _end); + + code = tsdbRowMergerGetRow(pMerger, &pTSRow); + TSDB_CHECK_CODE(code, lino, _end); + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pScanInfo); + + taosMemoryFreeClear(pTSRow); + tsdbRowMergerClear(pMerger); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SSttBlockReader* pSttBlockReader) { - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SFileBlockDumpInfo* pDumpInfo = NULL; + TSDBROW* pRow = NULL; + TSDBROW* piRow = NULL; + SRowKey* pKey = &(SRowKey){0}; - TSDBROW *pRow = NULL, *piRow = NULL; + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pDumpInfo = &pReader->status.fBlockDumpInfo; - SRowKey* pKey = &(SRowKey){0}; if (hasDataInFileBlock(pBlockData, pDumpInfo)) { tColRowGetKey(pBlockData, pDumpInfo->rowIndex, pKey); } else { @@ -2682,46 +3144,65 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI } if (pBlockScanInfo->iter.hasVal) { - getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader, &pRow); + code = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader, &pRow); + TSDB_CHECK_CODE(code, lino, _end); } if (pBlockScanInfo->iiter.hasVal) { - getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader, &piRow); + code = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader, &piRow); + TSDB_CHECK_CODE(code, lino, _end); } - // two levels of mem-table does contain the valid rows if (pRow != NULL && piRow != NULL) { - return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pSttBlockReader); + // two levels of mem-table does contain the valid rows + code = doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pSttBlockReader); + TSDB_CHECK_CODE(code, lino, _end); + } else if (pBlockScanInfo->iiter.hasVal) { + // imem + file + stt block + code = doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, pSttBlockReader); + TSDB_CHECK_CODE(code, lino, _end); + } else if (pBlockScanInfo->iter.hasVal) { + // mem + file + stt block + code = doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, pSttBlockReader); + TSDB_CHECK_CODE(code, lino, _end); + } else { + // files data blocks + stt block + code = mergeFileBlockAndSttBlock(pReader, pSttBlockReader, pKey, pBlockScanInfo, pBlockData); + TSDB_CHECK_CODE(code, lino, _end); } - // imem + file + stt block - if (pBlockScanInfo->iiter.hasVal) { - return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, pSttBlockReader); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } - - // mem + file + stt block - if (pBlockScanInfo->iter.hasVal) { - return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, pSttBlockReader); - } - - // files data blocks + stt block - return mergeFileBlockAndSttBlock(pReader, pSttBlockReader, pKey, pBlockScanInfo, pBlockData); + return code; } static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, bool* loadNeighbor) { int32_t code = TSDB_CODE_SUCCESS; - int32_t order = pReader->info.order; - SDataBlockIter* pIter = &pReader->status.blockIter; - SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1; + int32_t lino = 0; + int32_t order = 0; + SDataBlockIter* pIter = NULL; + SBlockLoadSuppInfo* pSupInfo = NULL; + int32_t step = 0; int32_t nextIndex = -1; SBrinRecord rec = {0}; + bool hasNeighbor = false; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(loadNeighbor, code, lino, _end, TSDB_CODE_INVALID_PARA); + + order = pReader->info.order; + pIter = &pReader->status.blockIter; + pSupInfo = &pReader->suppInfo; + step = ASCENDING_TRAVERSE(order) ? 1 : -1; *loadNeighbor = false; - bool hasNeighbor = getNeighborBlockOfTable(pIter, pBlockInfo, pBlockScanInfo, &nextIndex, order, &rec, pSupInfo); + code = getNeighborBlockOfTable(pIter, pBlockInfo, pBlockScanInfo, &nextIndex, order, &rec, pSupInfo, &hasNeighbor); + TSDB_CHECK_CODE(code, lino, _end); if (!hasNeighbor) { // do nothing - return code; + goto _end; } // load next block @@ -2731,28 +3212,27 @@ static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlock // 1. find the next neighbor block in the scan block list STableDataBlockIdx* tableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, nextIndex); - if (tableDataBlockIdx == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(tableDataBlockIdx, code, lino, _end, TSDB_CODE_INVALID_PARA); // 2. remove it from the scan block list int32_t neighborIndex = tableDataBlockIdx->globalIndex; code = setFileBlockActiveInBlockIter(pReader, pBlockIter, neighborIndex, step); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); // 3. load the neighbor block, and set it to be the currently accessed file data block code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pBlockInfo->uid); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); // 4. check the data values - initBlockDumpInfo(pReader, pBlockIter); + code = initBlockDumpInfo(pReader, pBlockIter); + TSDB_CHECK_CODE(code, lino, _end); *loadNeighbor = true; } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } @@ -2772,15 +3252,27 @@ void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInf } static int32_t buildComposedDataBlock(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; - bool asc = ASCENDING_TRAVERSE(pReader->info.order); - int64_t st = taosGetTimestampUs(); - int32_t step = asc ? 1 : -1; - double el = 0; - SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; - SFileDataBlockInfo* pBlockInfo = NULL; - SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader; - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + bool asc = false; + int64_t st = 0; + double el = 0; + int32_t step = 0; + SSDataBlock* pResBlock = NULL; + SFileDataBlockInfo* pBlockInfo = NULL; + SSttBlockReader* pSttBlockReader = NULL; + SFileBlockDumpInfo* pDumpInfo = NULL; + STableBlockScanInfo* pBlockScanInfo = NULL; + TSDBKEY keyInBuf; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + asc = ASCENDING_TRAVERSE(pReader->info.order); + st = taosGetTimestampUs(); + step = asc ? 1 : -1; + pResBlock = pReader->resBlockInfo.pResBlock; + pSttBlockReader = pReader->status.fileIter.pSttBlockReader; + pDumpInfo = &pReader->status.fBlockDumpInfo; code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo, pReader->idStr); if (code != TSDB_CODE_SUCCESS) { @@ -2792,38 +3284,41 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { return code; } - STableBlockScanInfo* pBlockScanInfo = NULL; code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } + TSDB_CHECK_CODE(code, lino, _end); - TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); + code = getCurrentKeyInBuf(pBlockScanInfo, pReader, &keyInBuf); + TSDB_CHECK_CODE(code, lino, _end); // it is a clean block, load it directly int64_t cap = pReader->resBlockInfo.capacity; - bool directCopy = isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf) && - (pBlockInfo->numRow <= cap) && (pBlockScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA) && + bool isClean = false; + code = isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf, &isClean); + TSDB_CHECK_CODE(code, lino, _end); + bool directCopy = isClean && (pBlockInfo->numRow <= cap) && (pBlockScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA) && ((asc && ((pBlockInfo->lastKey < keyInBuf.ts) || (keyInBuf.ts == INT64_MIN))) || (!asc && (pBlockInfo->lastKey > keyInBuf.ts))); if (directCopy) { code = copyBlockDataToSDataBlock(pReader, &pBlockScanInfo->lastProcKey); + TSDB_CHECK_CODE(code, lino, _end); goto _end; } SBlockData* pBlockData = &pReader->status.fileBlockData; initSttBlockReader(pSttBlockReader, pBlockScanInfo, pReader); - if (pReader->code != 0) { - code = pReader->code; - goto _end; - } + code = pReader->code; + TSDB_CHECK_CODE(code, lino, _end); while (1) { bool hasBlockData = false; { while (pBlockData->nRow > 0 && pBlockData->uid == pBlockScanInfo->uid) { // find the first qualified row in data block - if (isValidFileBlockRow(pBlockData, pDumpInfo->rowIndex, pBlockScanInfo, asc, &pReader->info, pReader)) { + bool valid = false; + code = + isValidFileBlockRow(pBlockData, pDumpInfo->rowIndex, pBlockScanInfo, asc, &pReader->info, pReader, &valid); + TSDB_CHECK_CODE(code, lino, _end); + if (valid) { hasBlockData = true; break; } @@ -2833,15 +3328,14 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { if (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0) { // NOTE: get the new block info code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } + TSDB_CHECK_CODE(code, lino, _end); // continue check for the next file block if the last ts in the current block // is overlapped with the next neighbor block bool loadNeighbor = false; code = loadNeighborIfOverlap(pBlockInfo, pBlockScanInfo, pReader, &loadNeighbor); if ((!loadNeighbor) || (code != 0)) { + lino = __LINE__; setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order); break; } @@ -2855,9 +3349,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } code = buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pSttBlockReader); - if (code) { - goto _end; - } + TSDB_CHECK_CODE(code, lino, _end); // currently loaded file data block is consumed if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { @@ -2871,16 +3363,21 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } _end: - el = (taosGetTimestampUs() - st) / 1000.0; - updateComposedBlockInfo(pReader, el, pBlockScanInfo); + if (pReader) { + el = (taosGetTimestampUs() - st) / 1000.0; + updateComposedBlockInfo(pReader, el, pBlockScanInfo); + } - if (pResBlock->info.rows > 0) { + if (pResBlock && pResBlock->info.rows > 0) { tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); } + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } @@ -2895,32 +3392,35 @@ int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order) { } int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost) { - int32_t code = 0; - int32_t newDelDataInFile = taosArrayGetSize(pBlockScanInfo->pFileDelData); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t newDelDataInFile = 0; + int64_t st = 0; + SArray* pSource = NULL; + + TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + + newDelDataInFile = taosArrayGetSize(pBlockScanInfo->pFileDelData); if (newDelDataInFile == 0 && ((pBlockScanInfo->delSkyline != NULL) || (TARRAY_SIZE(pBlockScanInfo->pMemDelData) == 0))) { - return code; + goto _end; } - int64_t st = taosGetTimestampUs(); + st = taosGetTimestampUs(); if (pBlockScanInfo->delSkyline != NULL) { taosArrayClear(pBlockScanInfo->delSkyline); } else { pBlockScanInfo->delSkyline = taosArrayInit(4, sizeof(TSDBKEY)); - if (pBlockScanInfo->delSkyline == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pBlockScanInfo->delSkyline, code, lino, _end, terrno); } - SArray* pSource = pBlockScanInfo->pFileDelData; + pSource = pBlockScanInfo->pFileDelData; if (pSource == NULL) { pSource = pBlockScanInfo->pMemDelData; } else { - void* p1 = taosArrayAddAll(pSource, pBlockScanInfo->pMemDelData); - if (p1 == NULL) { - return terrno; - } + const void* p1 = taosArrayAddAll(pSource, pBlockScanInfo->pMemDelData); + TSDB_CHECK_NULL(p1, code, lino, _end, terrno); } code = tsdbBuildDeleteSkyline(pSource, 0, taosArrayGetSize(pSource) - 1, pBlockScanInfo->delSkyline); @@ -2932,28 +3432,45 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t orde pBlockScanInfo->iiter.index = index; pBlockScanInfo->fileDelIndex = index; pBlockScanInfo->sttBlockDelIndex = index; + TSDB_CHECK_CODE(code, lino, _end); + TSDB_CHECK_NULL(pCost, code, lino, _end, TSDB_CODE_INVALID_PARA); double el = taosGetTimestampUs() - st; pCost->createSkylineIterTime = el / 1000.0; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } -TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { - bool asc = ASCENDING_TRAVERSE(pReader->info.order); - TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}, ikey = {.ts = TSKEY_INITIAL_VAL}; - - bool hasKey = false, hasIKey = false; +int32_t getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader, TSDBKEY* pkey) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + bool asc = false; + bool hasKey = false; + bool hasIKey = false; TSDBROW* pRow = NULL; TSDBROW* pIRow = NULL; + TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}; + TSDBKEY ikey = {.ts = TSKEY_INITIAL_VAL}; - getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader, &pRow); + TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pkey, code, lino, _end, TSDB_CODE_INVALID_PARA); + + asc = ASCENDING_TRAVERSE(pReader->info.order); + + code = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader, &pRow); + TSDB_CHECK_CODE(code, lino, _end); if (pRow != NULL) { hasKey = true; key = TSDBROW_KEY(pRow); } - getValidMemRow(&pScanInfo->iiter, pScanInfo->delSkyline, pReader, &pIRow); + code = getValidMemRow(&pScanInfo->iiter, pScanInfo->delSkyline, pReader, &pIRow); + TSDB_CHECK_CODE(code, lino, _end); if (pIRow != NULL) { hasIKey = true; ikey = TSDBROW_KEY(pIRow); @@ -2962,18 +3479,24 @@ TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) if (hasKey) { if (hasIKey) { // has data in mem & imem if (asc) { - return key.ts <= ikey.ts ? key : ikey; + *pkey = key.ts <= ikey.ts ? key : ikey; } else { - return key.ts <= ikey.ts ? ikey : key; + *pkey = key.ts <= ikey.ts ? ikey : key; } } else { // no data in imem - return key; + *pkey = key; } } else { // no data in mem & imem, return the initial value // only imem has data, return ikey - return ikey; + *pkey = ikey; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void prepareDurationForNextFileSet(STsdbReader* pReader) { @@ -3015,29 +3538,34 @@ static void prepareDurationForNextFileSet(STsdbReader* pReader) { } static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pTableList) { - SReaderStatus* pStatus = &pReader->status; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + SArray* pIndexList = NULL; + size_t numOfTables = 0; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockNum, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pStatus = &pReader->status; pBlockNum->numOfBlocks = 0; pBlockNum->numOfSttFiles = 0; - size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); - SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBrinBlk)); - if (pIndexList == NULL) { - return terrno; - } + numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + pIndexList = taosArrayInit(numOfTables, sizeof(SBrinBlk)); + TSDB_CHECK_NULL(pIndexList, code, lino, _end, terrno); while (1) { // only check here, since the iterate data in memory is very fast. if (pReader->code != TSDB_CODE_SUCCESS) { tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr); - return pReader->code; + code = pReader->code; + TSDB_CHECK_CODE(code, lino, _end); } - bool hasNext = false; - int32_t code = filesetIteratorNext(&pStatus->fileIter, pReader, &hasNext); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(pIndexList); - return code; - } + bool hasNext = false; + code = filesetIteratorNext(&pStatus->fileIter, pReader, &hasNext); + TSDB_CHECK_CODE(code, lino, _end); if (!hasNext) { // no data files on disk break; @@ -3045,17 +3573,11 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr taosArrayClear(pIndexList); code = doLoadBlockIndex(pReader, pReader->pFileReader, pIndexList); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(pIndexList); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (taosArrayGetSize(pIndexList) > 0 || pReader->status.pCurrentFileset->lvlArr->size > 0) { code = loadFileBlockBrinInfo(pReader, pIndexList, pBlockNum, pTableList); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(pIndexList); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pBlockNum->numOfBlocks + pBlockNum->numOfSttFiles > 0) { if (pReader->bFilesetDelimited) { @@ -3068,23 +3590,42 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr // no blocks in current file, try next files } - taosArrayDestroy(pIndexList); - return loadDataFileTombDataForAll(pReader); + code = loadDataFileTombDataForAll(pReader); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (pIndexList) { + taosArrayDestroy(pIndexList); + } + return code; } // pTableIter can be NULL, no need to handle the return value static int32_t resetTableListIndex(SReaderStatus* pStatus, const char* id) { - STableUidList* pList = &pStatus->uidList; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + STableUidList* pList = NULL; + + TSDB_CHECK_NULL(pStatus, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pList = &pStatus->uidList; pList->currentIndex = 0; uint64_t uid = pList->tableUidList[0]; pStatus->pTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid)); if (pStatus->pTableIter == NULL) { - tsdbError("%s failed to load tableBlockScanInfo for uid:%"PRId64", code: internal error", id, uid); - return TSDB_CODE_INTERNAL_ERROR; + tsdbError("%s failed to load tableBlockScanInfo for uid:%" PRId64 ", code: internal error", id, uid); + TSDB_CHECK_NULL(pStatus->pTableIter, code, lino, _end, TSDB_CODE_INTERNAL_ERROR); } - return 0; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void resetPreFilesetMemTableListIndex(SReaderStatus* pStatus) { @@ -3121,18 +3662,26 @@ static bool moveToNextTableForPreFileSetMem(SReaderStatus* pStatus) { } static int32_t buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo) { - SReaderStatus* pStatus = &pReader->status; - SSttBlockReader* pSttBlockReader = pStatus->fileIter.pSttBlockReader; - SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + SSttBlockReader* pSttBlockReader = NULL; + SSDataBlock* pResBlock = NULL; + SDataBlockInfo* pInfo = NULL; + bool asc = false; - bool asc = ASCENDING_TRAVERSE(pReader->info.order); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); - SDataBlockInfo* pInfo = &pResBlock->info; - int32_t code = blockDataEnsureCapacity(pResBlock, pScanInfo->numOfRowsInStt); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + pStatus = &pReader->status; + pSttBlockReader = pStatus->fileIter.pSttBlockReader; + pResBlock = pReader->resBlockInfo.pResBlock; + asc = ASCENDING_TRAVERSE(pReader->info.order); + code = blockDataEnsureCapacity(pResBlock, pScanInfo->numOfRowsInStt); + TSDB_CHECK_CODE(code, lino, _end); + + pInfo = &pResBlock->info; pInfo->rows = pScanInfo->numOfRowsInStt; pInfo->id.uid = pScanInfo->uid; pInfo->dataLoad = 1; @@ -3156,15 +3705,30 @@ static int32_t buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScan tsdbDebug("%p uid:%" PRId64 " return clean stt block as one, brange:%" PRId64 "-%" PRId64 " rows:%" PRId64 " %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, pReader->idStr); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } -static void buildCleanBlockFromDataFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, - SFileDataBlockInfo* pBlockInfo, int32_t blockIndex) { +static int32_t buildCleanBlockFromDataFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, + SFileDataBlockInfo* pBlockInfo, int32_t blockIndex) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + SDataBlockInfo* pInfo = NULL; + bool asc = false; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + // whole data block is required, return it directly - SReaderStatus* pStatus = &pReader->status; - SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info; - bool asc = ASCENDING_TRAVERSE(pReader->info.order); + pStatus = &pReader->status; + pInfo = &pReader->resBlockInfo.pResBlock->info; + asc = ASCENDING_TRAVERSE(pReader->info.order); pInfo->rows = pBlockInfo->numRow; pInfo->id.uid = pScanInfo->uid; @@ -3188,66 +3752,81 @@ static void buildCleanBlockFromDataFiles(STsdbReader* pReader, STableBlockScanIn // update the last key for the corresponding table setComposedBlockFlag(pReader, false); setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order); - updateLastKeyInfo(&pScanInfo->lastProcKey, pBlockInfo, pInfo, pReader->suppInfo.numOfPks, asc); + code = updateLastKeyInfo(&pScanInfo->lastProcKey, pBlockInfo, pInfo, pReader->suppInfo.numOfPks, asc); + TSDB_CHECK_CODE(code, lino, _end); tsdbDebug("%p uid:%" PRIu64 " clean file block retrieved from file, global index:%d, " "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s", pReader, pScanInfo->uid, blockIndex, pBlockInfo->tbBlockIdx, pBlockInfo->numRow, pBlockInfo->firstKey, pBlockInfo->lastKey, pReader->idStr); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { - SReaderStatus* pStatus = &pReader->status; - SSttBlockReader* pSttBlockReader = pStatus->fileIter.pSttBlockReader; - STableUidList* pUidList = &pStatus->uidList; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + SSttBlockReader* pSttBlockReader = NULL; + STableUidList* pUidList = NULL; + SSDataBlock* pResBlock = NULL; + STableBlockScanInfo* pScanInfo = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pStatus = &pReader->status; + pSttBlockReader = pStatus->fileIter.pSttBlockReader; + pUidList = &pStatus->uidList; if (tSimpleHashGetSize(pStatus->pTableMap) == 0) { - return TSDB_CODE_SUCCESS; + goto _end; } - SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + pResBlock = pReader->resBlockInfo.pResBlock; while (1) { if (pReader->code != TSDB_CODE_SUCCESS) { tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr); - return pReader->code; + code = pReader->code; + TSDB_CHECK_CODE(code, lino, _end); } // load the last data block of current table - STableBlockScanInfo* pScanInfo = NULL; if (pStatus->pTableIter == NULL) { tsdbError("table Iter is null, invalid pScanInfo, try next table %s", pReader->idStr); bool hasNexTable = moveToNextTable(pUidList, pStatus); if (!hasNexTable) { - return TSDB_CODE_SUCCESS; + break; } continue; } else { - pScanInfo = *(STableBlockScanInfo**) pStatus->pTableIter; + pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; } if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pScanInfo->uid, sizeof(pScanInfo->uid))) { // reset the index in last block when handing a new file bool hasNexTable = moveToNextTable(pUidList, pStatus); if (!hasNexTable) { - return TSDB_CODE_SUCCESS; + break; } continue; } initSttBlockReader(pSttBlockReader, pScanInfo, pReader); - if (pReader->code != TSDB_CODE_SUCCESS) { - return pReader->code; - } - + code = pReader->code; + TSDB_CHECK_CODE(code, lino, _end); + if (!hasDataInSttBlock(pScanInfo)) { bool hasNexTable = moveToNextTable(pUidList, pStatus); if (!hasNexTable) { - return TSDB_CODE_SUCCESS; + break; } continue; @@ -3256,7 +3835,8 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { // if only require the total rows, no need to load data from stt file if it is clean stt blocks if (pReader->info.execMode == READER_EXEC_ROWS && pScanInfo->cleanSttBlocks) { code = buildCleanBlockFromSttFiles(pReader, pScanInfo); - return code; + TSDB_CHECK_CODE(code, lino, _end); + break; } int64_t st = taosGetTimestampUs(); @@ -3267,9 +3847,7 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { } code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pSttBlockReader); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) { break; @@ -3284,15 +3862,21 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); - return TSDB_CODE_SUCCESS; + break; } // current table is exhausted, let's try next table bool hasNexTable = moveToNextTable(pUidList, pStatus); if (!hasNexTable) { - return TSDB_CODE_SUCCESS; + break; } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } // current active data block not overlap with the stt-files/stt-blocks @@ -3306,54 +3890,61 @@ static bool notOverlapWithFiles(SFileDataBlockInfo* pBlockInfo, STableBlockScanI } static int32_t doBuildDataBlock(STsdbReader* pReader) { - SReaderStatus* pStatus = &pReader->status; - SDataBlockIter* pBlockIter = &pStatus->blockIter; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + SDataBlockIter* pBlockIter = NULL; STableBlockScanInfo* pScanInfo = NULL; SFileDataBlockInfo* pBlockInfo = NULL; - SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader; - bool asc = ASCENDING_TRAVERSE(pReader->info.order); - int32_t code = TSDB_CODE_SUCCESS; + SSttBlockReader* pSttBlockReader = NULL; + TSDBKEY keyInBuf; + bool asc = false; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pStatus = &pReader->status; + pBlockIter = &pStatus->blockIter; + pSttBlockReader = pReader->status.fileIter.pSttBlockReader; + asc = ASCENDING_TRAVERSE(pReader->info.order); code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order); - return code; + goto _end; } - if (pReader->code != TSDB_CODE_SUCCESS) { - return pReader->code; - } + code = pReader->code; + TSDB_CHECK_CODE(code, lino, _end); code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pScanInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pScanInfo->sttKeyInfo.status == STT_FILE_READER_UNINIT) { initSttBlockReader(pSttBlockReader, pScanInfo, pReader); - if (pReader->code != 0) { - return pReader->code; - } + code = pReader->code; + TSDB_CHECK_CODE(code, lino, _end); } - TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); - if (fileBlockShouldLoad(pReader, pBlockInfo, pScanInfo, keyInBuf)) { + code = getCurrentKeyInBuf(pScanInfo, pReader, &keyInBuf); + TSDB_CHECK_CODE(code, lino, _end); + bool load = false; + code = fileBlockShouldLoad(pReader, pBlockInfo, pScanInfo, keyInBuf, &load); + TSDB_CHECK_CODE(code, lino, _end); + if (load) { code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); // build composed data block code = buildComposedDataBlock(pReader); + TSDB_CHECK_CODE(code, lino, _end); } else if (bufferDataInFileBlockGap(keyInBuf, pBlockInfo, pScanInfo, pReader->info.order)) { // data in memory that are earlier than current file block and stt blocks // rows in buffer should be less than the file block in asc, greater than file block in desc int64_t endKey = getBoarderKeyInFiles(pBlockInfo, pScanInfo, pReader->info.order); code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); + TSDB_CHECK_CODE(code, lino, _end); } else { if (notOverlapWithFiles(pBlockInfo, pScanInfo, asc)) { int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts; @@ -3362,14 +3953,13 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { (!asc && pBlockInfo->firstKey > keyInStt)) { // the stt blocks may located in the gap of different data block, but the whole sttRange may overlap with the // data block, so the overlap check is invalid actually. - buildCleanBlockFromDataFiles(pReader, pScanInfo, pBlockInfo, pBlockIter->index); + code = buildCleanBlockFromDataFiles(pReader, pScanInfo, pBlockInfo, pBlockIter->index); + TSDB_CHECK_CODE(code, lino, _end); } else { // clean stt block - if (!(pReader->info.execMode == READER_EXEC_ROWS && pSttBlockReader->mergeTree.pIter == NULL)) { - tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_INTERNAL_ERROR; - } + TSDB_CHECK_CONDITION((pReader->info.execMode == READER_EXEC_ROWS) && (pSttBlockReader->mergeTree.pIter == NULL), + code, lino, _end, TSDB_CODE_INTERNAL_ERROR); code = buildCleanBlockFromSttFiles(pReader, pScanInfo); - return code; + TSDB_CHECK_CODE(code, lino, _end); } } else { SBlockData* pBData = &pReader->status.fileBlockData; @@ -3382,21 +3972,16 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // let's load data from stt files, make sure clear the cleanStt block flag before load the data from stt files initSttBlockReader(pSttBlockReader, pScanInfo, pReader); - if (pReader->code != 0) { - return pReader->code; - } + code = pReader->code; + TSDB_CHECK_CODE(code, lino, _end); // no data in stt block, no need to proceed. while (hasDataInSttBlock(pScanInfo)) { - if (pScanInfo->sttKeyInfo.status != STT_FILE_HAS_DATA) { - tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_INTERNAL_ERROR; - } + TSDB_CHECK_CONDITION(pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA, code, lino, _end, + TSDB_CODE_INTERNAL_ERROR); code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pSttBlockReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) { break; @@ -3423,11 +4008,24 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { } } - return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code; + code = pReader->code; + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t buildBlockFromBufferSeqForPreFileset(STsdbReader* pReader, int64_t endKey) { - SReaderStatus* pStatus = &pReader->status; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pStatus = &pReader->status; tsdbDebug("seq load data blocks from cache that preceeds fileset %d, %s", pReader->status.pCurrentFileset->fid, pReader->idStr); @@ -3435,7 +4033,8 @@ static int32_t buildBlockFromBufferSeqForPreFileset(STsdbReader* pReader, int64_ while (1) { if (pReader->code != TSDB_CODE_SUCCESS) { tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr); - return pReader->code; + code = pReader->code; + TSDB_CHECK_CODE(code, lino, _end); } STableBlockScanInfo** pBlockScanInfo = pStatus->pProcMemTableIter; @@ -3443,150 +4042,172 @@ static int32_t buildBlockFromBufferSeqForPreFileset(STsdbReader* pReader, int64_ taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) { bool hasNexTable = moveToNextTableForPreFileSetMem(pStatus); if (!hasNexTable) { - return TSDB_CODE_SUCCESS; + break; } continue; } - int32_t code = initMemDataIterator(*pBlockScanInfo, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = initMemDataIterator(*pBlockScanInfo, pReader); + TSDB_CHECK_CODE(code, lino, _end); code = initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pReader->resBlockInfo.pResBlock->info.rows > 0) { - return TSDB_CODE_SUCCESS; + break; } // current table is exhausted, let's try next table bool hasNexTable = moveToNextTableForPreFileSetMem(pStatus); if (!hasNexTable) { - return TSDB_CODE_SUCCESS; + break; } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t endKey) { - SReaderStatus* pStatus = &pReader->status; - STableUidList* pUidList = &pStatus->uidList; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + STableUidList* pUidList = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pStatus = &pReader->status; + pUidList = &pStatus->uidList; tsdbDebug("seq load data blocks from cache, %s", pReader->idStr); while (1) { if (pReader->code != TSDB_CODE_SUCCESS) { tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr); - return pReader->code; + code = pReader->code; + TSDB_CHECK_CODE(code, lino, _end); } STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter; if (pBlockScanInfo == NULL || *pBlockScanInfo == NULL) { - return TSDB_CODE_SUCCESS; + break; } if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) { bool hasNexTable = moveToNextTable(pUidList, pStatus); if (!hasNexTable) { - return TSDB_CODE_SUCCESS; + break; } continue; } - int32_t code = initMemDataIterator(*pBlockScanInfo, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = initMemDataIterator(*pBlockScanInfo, pReader); + TSDB_CHECK_CODE(code, lino, _end); code = initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pReader->resBlockInfo.pResBlock->info.rows > 0) { - return TSDB_CODE_SUCCESS; + break; } // current table is exhausted, let's try next table bool hasNexTable = moveToNextTable(pUidList, pStatus); if (!hasNexTable) { - return TSDB_CODE_SUCCESS; + break; } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } // set the correct start position in case of the first/last file block, according to the query time window -static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { +static int32_t initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SFileDataBlockInfo* pBlockInfo = NULL; - SReaderStatus* pStatus = &pReader->status; - SFileBlockDumpInfo* pDumpInfo = &pStatus->fBlockDumpInfo; + SReaderStatus* pStatus = NULL; + SFileBlockDumpInfo* pDumpInfo = NULL; - int32_t code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pStatus = &pReader->status; + pDumpInfo = &pStatus->fBlockDumpInfo; + + code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr); if (code == TSDB_CODE_SUCCESS) { pDumpInfo->totalRows = pBlockInfo->numRow; pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->numRow - 1; } else { pDumpInfo->totalRows = 0; pDumpInfo->rowIndex = 0; + code = TSDB_CODE_SUCCESS; } pDumpInfo->allDumped = false; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SBlockNumber num = {0}; - SArray* pTableList = taosArrayInit(40, POINTER_BYTES); - if (pTableList == NULL) { - return terrno; - } + SArray* pTableList = NULL; - int32_t code = moveToNextFile(pReader, &num, pTableList); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(pTableList); - return code; - } + pTableList = taosArrayInit(40, POINTER_BYTES); + TSDB_CHECK_NULL(pTableList, code, lino, _end, terrno); + + code = moveToNextFile(pReader, &num, pTableList); + TSDB_CHECK_CODE(code, lino, _end); // all data files are consumed, try data in buffer if (num.numOfBlocks + num.numOfSttFiles == 0) { pReader->status.loadFromFile = false; - taosArrayDestroy(pTableList); - return code; + goto _end; } // initialize the block iterator for a new fileset if (num.numOfBlocks > 0) { code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks, pTableList); + TSDB_CHECK_CODE(code, lino, _end); } else { // no block data, only last block exists tBlockDataReset(&pReader->status.fileBlockData); code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo), pReader->idStr); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = resetTableListIndex(&pReader->status, pReader->idStr); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } - if (code == TSDB_CODE_SUCCESS) { // set the correct start position according to the query time window - initBlockDumpInfo(pReader, pBlockIter); - } + code = initBlockDumpInfo(pReader, pBlockIter); + TSDB_CHECK_CODE(code, lino, _end); - taosArrayDestroy(pTableList); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (pTableList) { + taosArrayDestroy(pTableList); + } return code; } @@ -3602,8 +4223,15 @@ typedef enum { static int32_t doReadDataFromSttFiles(STsdbReader* pReader, ERetrieveType* pReturnType) { int32_t code = TSDB_CODE_SUCCESS; - SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; - SDataBlockIter* pBlockIter = &pReader->status.blockIter; + int32_t lino = 0; + SSDataBlock* pResBlock = NULL; + SDataBlockIter* pBlockIter = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReturnType, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pResBlock = pReader->resBlockInfo.pResBlock; + pBlockIter = &pReader->status.blockIter; *pReturnType = TSDB_READ_RETURN; @@ -3611,89 +4239,96 @@ static int32_t doReadDataFromSttFiles(STsdbReader* pReader, ERetrieveType* pRetu while (1) { code = doLoadSttBlockSequentially(pReader); - if (code != TSDB_CODE_SUCCESS) { - *pReturnType = TSDB_READ_RETURN; - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pResBlock->info.rows > 0) { - *pReturnType = TSDB_READ_RETURN; - return code; + goto _end; } // all data blocks are checked in this stt file, now let's try the next file set - if (pReader->status.pTableIter != NULL) { - code = TSDB_CODE_INTERNAL_ERROR; - tsdbError("tsdb reader failed at: %s:%d, code:%s", __func__, __LINE__, tstrerror(code)); - return code; - } + TSDB_CHECK_CONDITION(pReader->status.pTableIter == NULL, code, lino, _end, TSDB_CODE_INTERNAL_ERROR); code = initForFirstBlockInFile(pReader, pBlockIter); + TSDB_CHECK_CODE(code, lino, _end); - // error happens or all the data files are completely checked - if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { - *pReturnType = TSDB_READ_RETURN; - return code; + // all the data files are completely checked + if (pReader->status.loadFromFile == false) { + goto _end; } if (pReader->status.bProcMemPreFileset) { code = buildFromPreFilesetBuffer(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pResBlock->info.rows > 0) { pReader->status.processingMemPreFileSet = true; - *pReturnType = TSDB_READ_RETURN; - return code; + goto _end; } } if (pBlockIter->numOfBlocks > 0) { // there are data blocks existed. *pReturnType = TSDB_READ_CONTINUE; - return code; + goto _end; } else { // all blocks in data file are checked, let's check the data in stt-files code = resetTableListIndex(&pReader->status, pReader->idStr); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t buildBlockFromFiles(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; - bool asc = ASCENDING_TRAVERSE(pReader->info.order); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SDataBlockIter* pBlockIter = NULL; + SSDataBlock* pResBlock = NULL; + SFileBlockDumpInfo* pDumpInfo = NULL; + SBlockData* pBlockData = NULL; + const char* id = NULL; + bool asc = false; - SDataBlockIter* pBlockIter = &pReader->status.blockIter; - SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBlockData* pBlockData = &pReader->status.fileBlockData; - const char* id = pReader->idStr; + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + asc = ASCENDING_TRAVERSE(pReader->info.order); + + pBlockIter = &pReader->status.blockIter; + pResBlock = pReader->resBlockInfo.pResBlock; + pDumpInfo = &pReader->status.fBlockDumpInfo; + pBlockData = &pReader->status.fileBlockData; + id = pReader->idStr; if (pBlockIter->numOfBlocks == 0) { // let's try to extract data from stt files. ERetrieveType type = 0; code = doReadDataFromSttFiles(pReader, &type); - if (code != 0 || type == TSDB_READ_RETURN) { - return code; + TSDB_CHECK_CODE(code, lino, _end); + if (type == TSDB_READ_RETURN) { + goto _end; } code = doBuildDataBlock(pReader); - if (code != TSDB_CODE_SUCCESS || pResBlock->info.rows > 0) { - return code; + TSDB_CHECK_CODE(code, lino, _end); + if (pResBlock->info.rows > 0) { + goto _end; } } while (1) { if (fileBlockPartiallyRead(pDumpInfo, asc)) { // file data block is partially loaded code = buildComposedDataBlock(pReader); + TSDB_CHECK_CODE(code, lino, _end); } else { // current block are exhausted, try the next file block if (pDumpInfo->allDumped) { // try next data block in current file - bool hasNext = blockIteratorNext(&pReader->status.blockIter, pReader->idStr); + bool hasNext = blockIteratorNext(&pReader->status.blockIter); if (hasNext) { // check for the next block in the block accessed order list - initBlockDumpInfo(pReader, pBlockIter); + code = initBlockDumpInfo(pReader, pBlockIter); + TSDB_CHECK_CODE(code, lino, _end); } else { // all data blocks in files are checked, let's check the data in last files. // data blocks in current file are exhausted, let's try the next file now @@ -3703,30 +4338,34 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { tBlockDataReset(pBlockData); code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo), id); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = resetTableListIndex(&pReader->status, id); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); ERetrieveType type = 0; code = doReadDataFromSttFiles(pReader, &type); - if (code != 0 || type == TSDB_READ_RETURN) { - return code; + TSDB_CHECK_CODE(code, lino, _end); + if (type == TSDB_READ_RETURN) { + break; } } } code = doBuildDataBlock(pReader); + TSDB_CHECK_CODE(code, lino, _end); } - if (code != TSDB_CODE_SUCCESS || pResBlock->info.rows > 0) { - return code; + if (pResBlock->info.rows > 0) { + break; } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idStr, @@ -3768,7 +4407,7 @@ static void getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRet } else if (level == TSDB_RETENTION_L1) { *pLevel = TSDB_RETENTION_L1; tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L1, str); - *pTsdb = VND_RSMA1(pVnode); + *pTsdb = VND_RSMA1(pVnode); return; } else { *pLevel = TSDB_RETENTION_L2; @@ -3795,86 +4434,100 @@ SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_ return (SVersionRange){.minVer = startVer, .maxVer = endVer}; } -static int32_t reverseSearchStartPos(const SArray* pDelList, int32_t index, int64_t key, bool asc) { - size_t num = taosArrayGetSize(pDelList); - int32_t start = index; +static int32_t reverseSearchStartPos(const SArray* pDelList, int32_t index, int64_t key, bool asc, int32_t* start) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + size_t num = 0; + + num = taosArrayGetSize(pDelList); + *start = index; if (asc) { - if (start >= num - 1) { - start = num - 1; + if (*start >= num - 1) { + *start = num - 1; } - TSDBKEY* p = taosArrayGet(pDelList, start); - if (p == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDBKEY* p = taosArrayGet(pDelList, *start); + TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA); - while (p->ts >= key && start > 0) { - start -= 1; + while (p->ts >= key && *start > 0) { + *start -= 1; } } else { if (index <= 0) { - start = 0; + *start = 0; } - TSDBKEY* p = taosArrayGet(pDelList, start); - if (p == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDBKEY* p = taosArrayGet(pDelList, *start); + TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA); - while (p->ts <= key && start < num - 1) { - start += 1; + while (p->ts <= key && *start < num - 1) { + *start += 1; } } - return start; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, - SVersionRange* pVerRange, bool hasPk) { +int32_t hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, + SVersionRange* pVerRange, bool hasPk, bool* dropped) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + size_t num = 0; + int32_t step = 0; + bool asc = false; + + *dropped = false; + if (pDelList == NULL || (TARRAY_SIZE(pDelList) == 0)) { - return false; + goto _end; } - size_t num = taosArrayGetSize(pDelList); - bool asc = ASCENDING_TRAVERSE(order); - int32_t step = asc ? 1 : -1; + num = taosArrayGetSize(pDelList); + asc = ASCENDING_TRAVERSE(order); + step = asc ? 1 : -1; if (hasPk) { // handle the case where duplicated timestamps existed. - *index = reverseSearchStartPos(pDelList, *index, key, asc); + code = reverseSearchStartPos(pDelList, *index, key, asc, index); + TSDB_CHECK_CODE(code, lino, _end); } if (asc) { if (*index >= num - 1) { TSDBKEY* last = taosArrayGetLast(pDelList); if (last == NULL) { - return false; + goto _end; } if (key > last->ts) { - return false; + goto _end; } else if (key == last->ts) { TSDBKEY* prev = taosArrayGet(pDelList, num - 2); if (prev == NULL) { - return false; + goto _end; } - return (prev->version >= ver && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer); + *dropped = (prev->version >= ver && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer); } } else { TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); TSDBKEY* pNext = taosArrayGet(pDelList, (*index) + 1); if (pCurrent == NULL || pNext == NULL) { - return false; + goto _end; } if (key < pCurrent->ts) { - return false; + goto _end; } if (pCurrent->ts <= key && pNext->ts >= key && pCurrent->version >= ver && pVerRange->maxVer >= pCurrent->version) { - return true; + *dropped = true; + goto _end; } while (pNext->ts <= key && (*index) < num - 1) { @@ -3884,7 +4537,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t pCurrent = taosArrayGet(pDelList, *index); pNext = taosArrayGet(pDelList, (*index) + 1); if (pCurrent == NULL || pNext == NULL) { - return false; + break; } // it is not a consecutive deletion range, ignore it @@ -3894,24 +4547,23 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t if (pCurrent->ts <= key && pNext->ts >= key && pCurrent->version >= ver && pVerRange->maxVer >= pCurrent->version) { - return true; + *dropped = true; + break; } } } - - return false; } } else { if (*index <= 0) { TSDBKEY* pFirst = taosArrayGet(pDelList, 0); if (pFirst == NULL) { - return false; + goto _end; } if (key < pFirst->ts) { - return false; + goto _end; } else if (key == pFirst->ts) { - return pFirst->version >= ver; + *dropped = pFirst->version >= ver; } else { tsdbError("unexpected error, key:%" PRId64 ", first:%" PRId64, key, pFirst->ts); } @@ -3919,15 +4571,16 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); TSDBKEY* pPrev = taosArrayGet(pDelList, (*index) - 1); if (pCurrent == NULL || pPrev == NULL) { - return false; + goto _end; } if (key > pCurrent->ts) { - return false; + goto _end; } if (pPrev->ts <= key && pCurrent->ts >= key && pPrev->version >= ver) { - return true; + *dropped = true; + goto _end; } while (pPrev->ts >= key && (*index) > 1) { @@ -3937,7 +4590,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t pCurrent = taosArrayGet(pDelList, *index); pPrev = taosArrayGet(pDelList, (*index) - 1); if (pCurrent == NULL || pPrev == NULL) { - return false; + break; } // it is not a consecutive deletion range, ignore it @@ -3946,46 +4599,60 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t } if (pPrev->ts <= key && pCurrent->ts >= key && pPrev->version >= ver) { - return true; + *dropped = true; + break; } } } - - return false; } } - return false; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -FORCE_INLINE void getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRes) { +FORCE_INLINE int32_t getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRes) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t order = 0; + TSDBROW* pRow = NULL; + TSDBKEY key; + + TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pRes, code, lino, _end, TSDB_CODE_INVALID_PARA); + *pRes = NULL; if (!pIter->hasVal) { - return; + goto _end; } - int32_t order = pReader->info.order; - TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + order = pReader->info.order; + pRow = tsdbTbDataIterGet(pIter->iter); - TSDBKEY key; TSDBROW_INIT_KEY(pRow, key); if (outOfTimeWindow(key.ts, &pReader->info.window)) { pIter->hasVal = false; - return; + goto _end; } // it is a valid data version if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) { if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) { *pRes = pRow; - return; + goto _end; } else { - bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, - pReader->suppInfo.numOfPks > 0); + bool dropped = false; + code = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, + pReader->suppInfo.numOfPks > 0, &dropped); + TSDB_CHECK_CODE(code, lino, _end); if (!dropped) { *pRes = pRow; - return; + goto _end; } } } @@ -3993,7 +4660,7 @@ FORCE_INLINE void getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdb while (1) { pIter->hasVal = tsdbTbDataIterNext(pIter->iter); if (!pIter->hasVal) { - return; + goto _end; } pRow = tsdbTbDataIterGet(pIter->iter); @@ -4001,28 +4668,42 @@ FORCE_INLINE void getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdb TSDBROW_INIT_KEY(pRow, key); if (outOfTimeWindow(key.ts, &pReader->info.window)) { pIter->hasVal = false; - return; + goto _end; } if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) { if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) { *pRes = pRow; - return; + goto _end; } else { - bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, - pReader->suppInfo.numOfPks > 0); + bool dropped = false; + code = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, + pReader->suppInfo.numOfPks > 0, &dropped); + TSDB_CHECK_CODE(code, lino, _end); if (!dropped) { *pRes = pRow; - return; + goto _end; } } } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList, STsdbReader* pReader) { - SRowMerger* pMerger = &pReader->status.merger; - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SRowMerger* pMerger = NULL; + + TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pMerger = &pReader->status.merger; while (1) { pIter->hasVal = tsdbTbDataIterNext(pIter->iter); @@ -4032,7 +4713,8 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArra // data exists but not valid TSDBROW* pRow = NULL; - getValidMemRow(pIter, pDelList, pReader, &pRow); + code = getValidMemRow(pIter, pDelList, pReader, &pRow); + TSDB_CHECK_CODE(code, lino, _end); if (pRow == NULL) { break; } @@ -4053,23 +4735,28 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArra STSchema* pTSchema = NULL; if (pRow->type == TSDBROW_ROW_FMT) { pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); - if (pTSchema == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pTSchema, code, lino, _end, terrno); } code = tsdbRowMergerAdd(pMerger, pRow, pTSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t* rowIndex, SRowKey* pKey, SRowMerger* pMerger, SVersionRange* pVerRange, int32_t step) { - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(rowIndex, code, lino, _end, TSDB_CODE_INVALID_PARA); + while ((*rowIndex) < pBlockData->nRow && (*rowIndex) >= 0) { SRowKey cur; tColRowGetKey(pBlockData, (*rowIndex), &cur); @@ -4085,11 +4772,14 @@ static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t* rowIn TSDBROW fRow = tsdbRowFromBlockData(pBlockData, (*rowIndex)); code = tsdbRowMergerAdd(pMerger, &fRow, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); (*rowIndex) += step; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } @@ -4101,92 +4791,120 @@ typedef enum { static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, SFileDataBlockInfo* pFBlock, SRowMerger* pMerger, SRowKey* pKey, CHECK_FILEBLOCK_STATE* state) { - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBlockData* pBlockData = &pReader->status.fileBlockData; - bool asc = ASCENDING_TRAVERSE(pReader->info.order); - SVersionRange* pVerRange = &pReader->info.verRange; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SFileBlockDumpInfo* pDumpInfo = NULL; + SBlockData* pBlockData = NULL; + bool asc = false; + SVersionRange* pVerRange = NULL; bool loadNeighbor = true; - int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; + int32_t step = 0; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(state, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pDumpInfo = &pReader->status.fBlockDumpInfo; + pBlockData = &pReader->status.fileBlockData; + asc = ASCENDING_TRAVERSE(pReader->info.order); + pVerRange = &pReader->info.verRange; + ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; - int32_t code = loadNeighborIfOverlap(pFBlock, pScanInfo, pReader, &loadNeighbor); *state = CHECK_FILEBLOCK_QUIT; + code = loadNeighborIfOverlap(pFBlock, pScanInfo, pReader, &loadNeighbor); + TSDB_CHECK_CODE(code, lino, _end); - if (loadNeighbor && (code == TSDB_CODE_SUCCESS)) { + if (loadNeighbor) { code = doMergeRowsInFileBlockImpl(pBlockData, &pDumpInfo->rowIndex, pKey, pMerger, pVerRange, step); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if ((pDumpInfo->rowIndex >= pDumpInfo->totalRows && asc) || (pDumpInfo->rowIndex < 0 && !asc)) { *state = CHECK_FILEBLOCK_CONT; } } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey, STsdbReader* pReader) { - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SRowMerger* pMerger = &pReader->status.merger; - bool asc = ASCENDING_TRAVERSE(pReader->info.order); - int32_t step = asc ? 1 : -1; - SVersionRange* pRange = &pReader->info.verRange; - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SFileBlockDumpInfo* pDumpInfo = NULL; + SRowMerger* pMerger = NULL; + bool asc = false; + int32_t step = 0; + SVersionRange* pRange = NULL; + + TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pDumpInfo = &pReader->status.fBlockDumpInfo; + pMerger = &pReader->status.merger; + asc = ASCENDING_TRAVERSE(pReader->info.order); + step = asc ? 1 : -1; + pRange = &pReader->info.verRange; pDumpInfo->rowIndex += step; if ((pDumpInfo->rowIndex <= pBlockData->nRow - 1 && asc) || (pDumpInfo->rowIndex >= 0 && !asc)) { code = doMergeRowsInFileBlockImpl(pBlockData, &pDumpInfo->rowIndex, pKey, pMerger, pRange, step); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } // all rows are consumed, let's try next file block if ((pDumpInfo->rowIndex >= pBlockData->nRow && asc) || (pDumpInfo->rowIndex < 0 && !asc)) { while (1) { - CHECK_FILEBLOCK_STATE st; - SFileDataBlockInfo* pFileBlockInfo = NULL; code = getCurrentBlockInfo(&pReader->status.blockIter, &pFileBlockInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pFileBlockInfo == NULL) { - st = CHECK_FILEBLOCK_QUIT; break; } + CHECK_FILEBLOCK_STATE st = CHECK_FILEBLOCK_QUIT; code = checkForNeighborFileBlock(pReader, pScanInfo, pFileBlockInfo, pMerger, pKey, &st); - if (st == CHECK_FILEBLOCK_QUIT || code != TSDB_CODE_SUCCESS) { + TSDB_CHECK_CODE(code, lino, _end); + if (st == CHECK_FILEBLOCK_QUIT) { break; } } } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, SRowMerger* pMerger, int32_t pkSrcSlot, SVersionRange* pVerRange, const char* idStr) { - SRowKey* pRowKey = &pScanInfo->lastProcKey; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SRowKey* pRowKey = NULL; + SRowKey* pNextKey = NULL; + + TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pRowKey = &pScanInfo->lastProcKey; while (1) { code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pkSrcSlot, pVerRange); - if (code || (!hasDataInSttBlock(pScanInfo))) { - return code; + TSDB_CHECK_CODE(code, lino, _end); + if (!hasDataInSttBlock(pScanInfo)) { + break; } - SRowKey* pNextKey = getCurrentKeyInSttBlock(pSttBlockReader); + pNextKey = getCurrentKeyInSttBlock(pSttBlockReader); int32_t ret = pkCompEx(pRowKey, pNextKey); if (ret == 0) { TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); code = tsdbRowMergerAdd(pMerger, pRow1, NULL); - if (code != TSDB_CODE_SUCCESS) { - break; - } + TSDB_CHECK_CODE(code, lino, _end); } else { tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid, pScanInfo->sttBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), @@ -4195,14 +4913,31 @@ int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanI } } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIterInfo* pIter, SArray* pDelList, TSDBROW* pResRow, STsdbReader* pReader, bool* freeTSRow) { - SRowMerger* pMerger = &pReader->status.merger; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SRowMerger* pMerger = NULL; TSDBROW* pNextRow = NULL; - TSDBROW current = *pRow; + STSchema* pTSchema = NULL; + STSchema* pTSchema1 = NULL; + TSDBROW current; + + TSDB_CHECK_NULL(pRow, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pResRow, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(freeTSRow, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pMerger = &pReader->status.merger; + current = *pRow; { // if the timestamp of the next valid row has a different ts, return current row directly pIter->hasVal = tsdbTbDataIterNext(pIter->iter); @@ -4210,19 +4945,20 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIt if (!pIter->hasVal) { *pResRow = *pRow; *freeTSRow = false; - return TSDB_CODE_SUCCESS; + goto _end; } else { // has next point in mem/imem - getValidMemRow(pIter, pDelList, pReader, &pNextRow); + code = getValidMemRow(pIter, pDelList, pReader, &pNextRow); + TSDB_CHECK_CODE(code, lino, _end); if (pNextRow == NULL) { *pResRow = current; *freeTSRow = false; - return TSDB_CODE_SUCCESS; + goto _end; } if (TSDBROW_TS(¤t) != TSDBROW_TS(pNextRow)) { *pResRow = current; *freeTSRow = false; - return TSDB_CODE_SUCCESS; + goto _end; } if (pKey->numOfPKs > 0) { @@ -4231,145 +4967,140 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIt if (pkCompEx(pKey, &nextRowKey) != 0) { *pResRow = current; *freeTSRow = false; - return TSDB_CODE_SUCCESS; + goto _end; } } } } - terrno = 0; - int32_t code = 0; - // start to merge duplicated rows - STSchema* pTSchema = NULL; if (current.type == TSDBROW_ROW_FMT) { // get the correct schema for row-wise data in memory pTSchema = doGetSchemaForTSRow(current.pTSRow->sver, pReader, uid); - if (pTSchema == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pTSchema, code, lino, _end, terrno); } code = tsdbRowMergerAdd(pMerger, ¤t, pTSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); - STSchema* pTSchema1 = NULL; if (pNextRow->type == TSDBROW_ROW_FMT) { // get the correct schema for row-wise data in memory pTSchema1 = doGetSchemaForTSRow(pNextRow->pTSRow->sver, pReader, uid); - if (pTSchema1 == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pTSchema1, code, lino, _end, terrno); } code = tsdbRowMergerAdd(pMerger, pNextRow, pTSchema1); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doMergeRowsInBuf(pIter, uid, pKey, pDelList, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = tsdbRowMergerGetRow(pMerger, &pResRow->pTSRow); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); pResRow->type = TSDBROW_ROW_FMT; tsdbRowMergerClear(pMerger); *freeTSRow = true; - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t doMergeMemIMemRows(TSDBROW* pRow, SRowKey* pRowKey, TSDBROW* piRow, SRowKey* piRowKey, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, SRow** pTSRow) { - SRowMerger* pMerger = &pReader->status.merger; int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SRowMerger* pMerger = NULL; + STSchema* pSchema = NULL; + STSchema* piSchema = NULL; + + TSDB_CHECK_NULL(pRow, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(piRow, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pMerger = &pReader->status.merger; - STSchema* pSchema = NULL; if (pRow->type == TSDBROW_ROW_FMT) { pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - if (pSchema == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno); } - STSchema* piSchema = NULL; if (piRow->type == TSDBROW_ROW_FMT) { piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - if (piSchema == NULL) { - return terrno; - } + TSDB_CHECK_NULL(piSchema, code, lino, _end, terrno); } if (ASCENDING_TRAVERSE(pReader->info.order)) { // ascending order imem --> mem code = tsdbRowMergerAdd(pMerger, piRow, piSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, piRowKey, pBlockScanInfo->delSkyline, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = tsdbRowMergerAdd(pMerger, pRow, pSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, pRowKey, pBlockScanInfo->delSkyline, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } else { code = tsdbRowMergerAdd(pMerger, pRow, pSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, pRowKey, pBlockScanInfo->delSkyline, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = tsdbRowMergerAdd(pMerger, piRow, piSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, piRowKey, pBlockScanInfo->delSkyline, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } tRowKeyAssign(&pBlockScanInfo->lastProcKey, pRowKey); code = tsdbRowMergerGetRow(pMerger, pTSRow); tsdbRowMergerClear(pMerger); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow, int64_t endKey, bool* freeTSRow) { - TSDBROW* pRow = NULL; - TSDBROW* piRow = NULL; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + TSDBROW* pRow = NULL; + TSDBROW* piRow = NULL; + SArray* pDelList = NULL; + uint64_t uid = 0; + SIterInfo* piter = NULL; + SIterInfo* piiter = NULL; + SRowKey rowKey = {0}; + SRowKey irowKey = {0}; + bool asc = false; - getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader, &pRow); - getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader, &piRow); + TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pResRow, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(freeTSRow, code, lino, _end, TSDB_CODE_INVALID_PARA); - SArray* pDelList = pBlockScanInfo->delSkyline; - uint64_t uid = pBlockScanInfo->uid; - SIterInfo* piter = &pBlockScanInfo->iter; - SIterInfo* piiter = &pBlockScanInfo->iiter; - SRowKey rowKey = {0}, irowKey = {0}; + code = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader, &pRow); + TSDB_CHECK_CODE(code, lino, _end); + code = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader, &piRow); + TSDB_CHECK_CODE(code, lino, _end); + + pDelList = pBlockScanInfo->delSkyline; + uid = pBlockScanInfo->uid; + piter = &pBlockScanInfo->iter; + piiter = &pBlockScanInfo->iiter; // todo refactor - bool asc = ASCENDING_TRAVERSE(pReader->info.order); + asc = ASCENDING_TRAVERSE(pReader->info.order); if (piter->hasVal) { tRowGetKeyEx(pRow, &rowKey); if ((rowKey.ts >= endKey && asc) || (rowKey.ts <= endKey && !asc)) { @@ -4385,71 +5116,76 @@ static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbRea } if (pRow != NULL && piRow != NULL) { - int32_t code = TSDB_CODE_SUCCESS; if (rowKey.numOfPKs == 0) { if ((rowKey.ts > irowKey.ts && asc) || (rowKey.ts < irowKey.ts && (!asc))) { // ik.ts < k.ts code = doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow); + TSDB_CHECK_CODE(code, lino, _end); } else if ((rowKey.ts < irowKey.ts && asc) || (rowKey.ts > irowKey.ts && (!asc))) { code = doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow); + TSDB_CHECK_CODE(code, lino, _end); } else { // ik.ts == k.ts - *freeTSRow = true; pResRow->type = TSDBROW_ROW_FMT; code = doMergeMemIMemRows(pRow, &rowKey, piRow, &irowKey, pBlockScanInfo, pReader, &pResRow->pTSRow); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); + *freeTSRow = true; } } else { int32_t ret = pkCompEx(&rowKey, &irowKey); if (ret != 0) { if ((ret > 0 && asc) || (ret < 0 && (!asc))) { // ik.ts < k.ts code = doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow); + TSDB_CHECK_CODE(code, lino, _end); } else if ((ret < 0 && asc) || (ret > 0 && (!asc))) { code = doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow); + TSDB_CHECK_CODE(code, lino, _end); } } else { // ik.ts == k.ts - *freeTSRow = true; pResRow->type = TSDBROW_ROW_FMT; code = doMergeMemIMemRows(pRow, &rowKey, piRow, &irowKey, pBlockScanInfo, pReader, &pResRow->pTSRow); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); + *freeTSRow = true; } } - - return code; + } else if (piter->hasVal && pRow != NULL) { + code = doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow); + TSDB_CHECK_CODE(code, lino, _end); + } else if (piiter->hasVal && piRow != NULL) { + code = doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow); + TSDB_CHECK_CODE(code, lino, _end); } - if (piter->hasVal && pRow != NULL) { - return doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } - - if (piiter->hasVal && piRow != NULL) { - return doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow); - } - - return TSDB_CODE_SUCCESS; + return code; } int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo) { - int32_t outputRowIndex = pBlock->info.rows; - int64_t uid = pScanInfo->uid; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t outputRowIndex = 0; + int64_t uid = 0; + SBlockLoadSuppInfo* pSupInfo = NULL; + STSchema* pSchema = NULL; + SColVal colVal = {0}; + int32_t i = 0, j = 0; - SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); - if (pSchema == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pTSRow, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); - SColVal colVal = {0}; - int32_t i = 0, j = 0; + outputRowIndex = pBlock->info.rows; + uid = pScanInfo->uid; + + pSupInfo = &pReader->suppInfo; + pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); + TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno); if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); - if (pColData == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA); ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts; i += 1; @@ -4460,26 +5196,18 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT if (colId == pSchema->columns[j].colId) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); - if (pColInfoData == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA); code = tRowGet(pTSRow, pSchema, j, &colVal); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); i += 1; j += 1; } else if (colId < pSchema->columns[j].colId) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); - if (pColInfoData == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA); colDataSetNULL(pColInfoData, outputRowIndex); i += 1; @@ -4491,9 +5219,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT // set null value since current column does not exist in the "pSchema" while (i < pSupInfo->numOfCols) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); - if (pColInfoData == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA); colDataSetNULL(pColInfoData, outputRowIndex); i += 1; @@ -4501,22 +5227,37 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT pBlock->info.dataLoad = 1; pBlock->info.rows += 1; - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex) { - int32_t i = 0, j = 0; - int32_t outputRowIndex = pResBlock->info.rows; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t i = 0, j = 0; + int32_t outputRowIndex = 0; + SBlockLoadSuppInfo* pSupInfo = NULL; + SColVal cv = {0}; + int32_t numOfInputCols = 0; + int32_t numOfOutputCols = 0; - SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; + TSDB_CHECK_NULL(pResBlock, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA); + + outputRowIndex = pResBlock->info.rows; + + pSupInfo = &pReader->suppInfo; ((int64_t*)pReader->status.pPrimaryTsCol->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; i += 1; - SColVal cv = {0}; - int32_t numOfInputCols = pBlockData->nColData; - int32_t numOfOutputCols = pSupInfo->numOfCols; + numOfInputCols = pBlockData->nColData; + numOfOutputCols = pSupInfo->numOfCols; while (i < numOfOutputCols && j < numOfInputCols) { SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); @@ -4529,9 +5270,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S if (pData->cid == pSupInfo->colId[i]) { tColDataGetValue(pData, rowIndex, &cv); code = doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); j += 1; } else if (pData->cid > pCol->info.colId) { // the specified column does not exist in file block, fill with null data @@ -4543,9 +5282,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S while (i < numOfOutputCols) { SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); - if (pCol == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pCol, code, lino, _end, TSDB_CODE_INVALID_PARA); colDataSetNULL(pCol, outputRowIndex); i += 1; @@ -4553,21 +5290,29 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S pResBlock->info.dataLoad = 1; pResBlock->info.rows += 1; - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, STsdbReader* pReader) { - SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SSDataBlock* pBlock = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pBlock = pReader->resBlockInfo.pResBlock; do { TSDBROW row = {.type = -1}; bool freeTSRow = false; code = tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (row.type == -1) { break; @@ -4575,26 +5320,30 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e if (row.type == TSDBROW_ROW_FMT) { code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo); - if (code == TSDB_CODE_SUCCESS) { - pBlockScanInfo->lastProcKey.ts = row.pTSRow->ts; - pBlockScanInfo->lastProcKey.numOfPKs = row.pTSRow->numOfPKs; - if (row.pTSRow->numOfPKs > 0) { - tRowGetPrimaryKeyDeepCopy(row.pTSRow, &pBlockScanInfo->lastProcKey); + if (code != TSDB_CODE_SUCCESS) { + if (freeTSRow) { + taosMemoryFreeClear(row.pTSRow); + } + TSDB_CHECK_CODE(code, lino, _end); + } + pBlockScanInfo->lastProcKey.ts = row.pTSRow->ts; + pBlockScanInfo->lastProcKey.numOfPKs = row.pTSRow->numOfPKs; + if (row.pTSRow->numOfPKs > 0) { + code = tRowGetPrimaryKeyDeepCopy(row.pTSRow, &pBlockScanInfo->lastProcKey); + if (code != TSDB_CODE_SUCCESS) { + if (freeTSRow) { + taosMemoryFreeClear(row.pTSRow); + } + TSDB_CHECK_CODE(code, lino, _end); } } if (freeTSRow) { - taosMemoryFree(row.pTSRow); - } - - if (code) { - return code; + taosMemoryFreeClear(row.pTSRow); } } else { code = doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); tColRowGetKeyDeepCopy(row.pBlockData, row.iRow, pReader->suppInfo.pkSrcSlot, &pBlockScanInfo->lastProcKey); } @@ -4609,21 +5358,30 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e } } while (1); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } // TODO refactor: with createDataBlockScanInfo int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t num) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t size = tSimpleHashGetSize(pReader->status.pTableMap); - + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t size = 0; STableBlockScanInfo** p = NULL; + STableUidList* pUidList = NULL; int32_t iter = 0; + bool acquired = false; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + size = tSimpleHashGetSize(pReader->status.pTableMap); code = tsdbAcquireReader(pReader); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); + acquired = true; while ((p = tSimpleHashIterate(pReader->status.pTableMap, p, &iter)) != NULL) { clearBlockScanInfo(*p); @@ -4633,21 +5391,15 @@ int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t if (size < num) { code = ensureBlockScanInfoBuf(&pReader->blockInfoBuf, num); - if (code) { - (void) tsdbReleaseReader(pReader); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); char* p1 = taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num); - if (p1 == NULL) { - (void) tsdbReleaseReader(pReader); - return terrno; - } + TSDB_CHECK_NULL(p1, code, lino, _end, terrno); pReader->status.uidList.tableUidList = (uint64_t*)p1; } - STableUidList* pUidList = &pReader->status.uidList; + pUidList = &pReader->status.uidList; pUidList->currentIndex = 0; STableKeyInfo* pList = (STableKeyInfo*)pTableList; @@ -4656,28 +5408,34 @@ int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t STableBlockScanInfo* pInfo = NULL; code = getPosInBlockInfoBuf(&pReader->blockInfoBuf, i, &pInfo); - if (code != TSDB_CODE_SUCCESS) { - (void) tsdbReleaseReader(pReader); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = initTableBlockScanInfo(pInfo, pList[i].uid, pReader->status.pTableMap, pReader); - if (code != TSDB_CODE_SUCCESS) { - (void) tsdbReleaseReader(pReader); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } - (void) tsdbReleaseReader(pReader); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (acquired) { + (void)tsdbReleaseReader(pReader); + } return code; } uint64_t tsdbGetReaderMaxVersion2(STsdbReader* pReader) { return pReader->info.verRange.maxVer; } static int32_t doOpenReaderImpl(STsdbReader* pReader) { - SReaderStatus* pStatus = &pReader->status; - SDataBlockIter* pBlockIter = &pStatus->blockIter; int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + SDataBlockIter* pBlockIter = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pStatus = &pReader->status; + pBlockIter = &pStatus->blockIter; if (pReader->bFilesetDelimited) { getMemTableTimeRange(pReader, &pReader->status.memTableMaxKey, &pReader->status.memTableMinKey); @@ -4685,15 +5443,11 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { } code = initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = resetDataBlockIterator(&pStatus->blockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo), pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pStatus->fileIter.numOfFiles == 0) { pStatus->loadFromFile = false; @@ -4703,8 +5457,13 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { if (!pStatus->loadFromFile) { code = resetTableListIndex(pStatus, pReader->idStr); + TSDB_CHECK_CODE(code, lino, _end); } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } @@ -4741,25 +5500,30 @@ static int32_t setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables) { - STimeWindow window = pCond->twindows; - SVnodeCfg* pConf = &(((SVnode*)pVnode)->config); - int32_t code = 0; - int32_t lino = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + STimeWindow window = {0}; + SVnodeCfg* pConf = NULL; + STsdbReader* pReader = NULL; + int32_t capacity = 0; - int32_t capacity = pConf->tsdbCfg.maxRows; + window = pCond->twindows; + pConf = &(((SVnode*)pVnode)->config); + + capacity = pConf->tsdbCfg.maxRows; if (pResBlock != NULL) { code = blockDataEnsureCapacity(pResBlock, capacity); - TSDB_CHECK_CODE(code, lino, _err); + TSDB_CHECK_CODE(code, lino, _end); } code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); - TSDB_CHECK_CODE(code, lino, _err); + TSDB_CHECK_CODE(code, lino, _end); // check for query time window - STsdbReader* pReader = *ppReader; + pReader = *ppReader; if (isEmptyQueryTimeWindow(&pReader->info.window) && pCond->type == TIMEWINDOW_RANGE_CONTAINED) { tsdbDebug("%p query window not overlaps with the data set, no result returned, %s", pReader, pReader->idStr); - return TSDB_CODE_SUCCESS; + goto _end; } if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) { @@ -4777,7 +5541,7 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi // here we only need one more row, so the capacity is set to be ONE. code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[0], 1, pResBlock, idstr); - TSDB_CHECK_CODE(code, lino, _err); + TSDB_CHECK_CODE(code, lino, _end); if (order == TSDB_ORDER_ASC) { pCond->twindows.skey = window.ekey + 1; @@ -4789,7 +5553,7 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi pCond->order = order; code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[1], 1, pResBlock, idstr); - TSDB_CHECK_CODE(code, lino, _err); + TSDB_CHECK_CODE(code, lino, _end); } // NOTE: the endVersion in pCond is the data version not schema version, so pCond->endVersion is not correct here. @@ -4797,14 +5561,14 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi // we should proceed in case of tmq processing. if (pCond->suid != 0) { code = metaGetTbTSchemaMaybeNull(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, -1, 1, &pReader->info.pSchema); - TSDB_CHECK_CODE(code, lino, _err); + TSDB_CHECK_CODE(code, lino, _end); if (pReader->info.pSchema == NULL) { tsdbWarn("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->info.suid, pReader->idStr); } } else if (numOfTables > 0) { STableKeyInfo* pKey = pTableList; code = metaGetTbTSchemaMaybeNull(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1, &pReader->info.pSchema); - TSDB_CHECK_CODE(code, lino, _err); + TSDB_CHECK_CODE(code, lino, _end); if (pReader->info.pSchema == NULL) { tsdbWarn("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr); } @@ -4812,28 +5576,29 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi if (pReader->info.pSchema != NULL) { code = tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema); - TSDB_CHECK_CODE(code, lino, _err); + TSDB_CHECK_CODE(code, lino, _end); } pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash); if (pReader->pSchemaMap == NULL) { tsdbError("failed init schema hash for reader %s", pReader->idStr); - TSDB_CHECK_NULL(pReader->pSchemaMap, code, lino, _err, terrno); + TSDB_CHECK_NULL(pReader->pSchemaMap, code, lino, _end, terrno); } tSimpleHashSetFreeFp(pReader->pSchemaMap, freeSchemaFunc); if (pReader->info.pSchema != NULL) { code = updateBlockSMAInfo(pReader->info.pSchema, &pReader->suppInfo); - TSDB_CHECK_CODE(code, lino, _err); + TSDB_CHECK_CODE(code, lino, _end); } STsdbReader* p = (pReader->innerReader[0] != NULL) ? pReader->innerReader[0] : pReader; - code = createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables, &pReader->status.pTableMap); - TSDB_CHECK_CODE(code, lino, _err); + code = createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables, + &pReader->status.pTableMap); + TSDB_CHECK_CODE(code, lino, _end); pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); - TSDB_CHECK_NULL(pReader->status.pLDataIterArray, code, lino, _err, terrno); + TSDB_CHECK_NULL(pReader->status.pLDataIterArray, code, lino, _end, terrno); pReader->flag = READER_STATUS_SUSPEND; pReader->info.execMode = pCond->notLoadData ? READER_EXEC_ROWS : READER_EXEC_DATA; @@ -4844,12 +5609,12 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi pReader, numOfTables, pReader->info.window.skey, pReader->info.window.ekey, pReader->info.verRange.minVer, pReader->info.verRange.maxVer, pReader->idStr); - return code; - -_err: - tsdbError("failed to create data reader, error at:%d code:%s %s", lino, tstrerror(code), idstr); - tsdbReaderClose2(*ppReader); - *ppReader = NULL; // reset the pointer value. +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s, %s", __func__, lino, tstrerror(code), idstr); + tsdbReaderClose2(*ppReader); + *ppReader = NULL; // reset the pointer value. + } return code; } @@ -4898,8 +5663,7 @@ void tsdbReaderClose2(STsdbReader* pReader) { size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); if (pReader->status.pTableMap != NULL) { - destroyAllBlockScanInfo(pReader->status.pTableMap); - pReader->status.pTableMap = NULL; + destroyAllBlockScanInfo(&pReader->status.pTableMap); } clearBlockScanInfoBuf(&pReader->blockInfoBuf); @@ -4918,6 +5682,7 @@ void tsdbReaderClose2(STsdbReader* pReader) { } destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); + pReader->status.pLDataIterArray = NULL; taosMemoryFreeClear(pReader->status.uidList.tableUidList); tsdbTrace("tsdb/reader-close: %p, untake snapshot", pReader); @@ -4955,7 +5720,14 @@ void tsdbReaderClose2(STsdbReader* pReader) { } static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) { - SReaderStatus* pStatus = &pCurrentReader->status; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + STableBlockScanInfo** p = NULL; + + TSDB_CHECK_NULL(pCurrentReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pStatus = &pCurrentReader->status; if (pStatus->loadFromFile) { tsdbDataFileReaderClose(&pCurrentReader->pFileReader); @@ -4963,14 +5735,10 @@ static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) { SReadCostSummary* pCost = &pCurrentReader->cost; destroySttBlockReader(pStatus->pLDataIterArray, &pCost->sttCost); pStatus->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); - if (pStatus->pLDataIterArray == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pStatus->pLDataIterArray, code, lino, _end, terrno); } // resetDataBlockScanInfo excluding lastKey - STableBlockScanInfo** p = NULL; - int32_t step = ASCENDING_TRAVERSE(pCurrentReader->info.order) ? 1 : -1; int32_t iter = 0; while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { @@ -4983,12 +5751,20 @@ static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) { pStatus->uidList.currentIndex = 0; initReaderStatus(pStatus); - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t tsdbReaderSuspend2(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + // save reader's base state & reset top state to be reconstructed from base state - int32_t code = 0; pReader->status.suspendInvoked = true; // record the suspend status if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { @@ -5005,6 +5781,7 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { // make sure only release once void* p = pReader->pReadSnap; + TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA); if ((p == atomic_val_compare_exchange_ptr((void**)&pReader->pReadSnap, p, NULL)) && (p != NULL)) { tsdbUntakeReadSnap2(pReader, p, false); pReader->pReadSnap = NULL; @@ -5026,6 +5803,11 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { #endif tsdbDebug("reader: %p suspended in this query %s, step:%d", pReader, pReader->idStr, pReader->step); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } @@ -5041,7 +5823,7 @@ static int32_t tsdbSetQueryReseek(void* pQHandle) { } code = tsdbReaderSuspend2(pReader); - (void) tsdbReleaseReader(pReader); + (void)tsdbReleaseReader(pReader); return code; } else if (code == EBUSY) { return TSDB_CODE_VND_QUERY_BUSY; @@ -5052,17 +5834,21 @@ static int32_t tsdbSetQueryReseek(void* pQHandle) { } int32_t tsdbReaderResume2(STsdbReader* pReader) { - int32_t code = 0; - STableBlockScanInfo** pBlockScanInfo = pReader->status.pTableIter; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + STableBlockScanInfo** pBlockScanInfo = NULL; + int32_t numOfTables = 0; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pBlockScanInfo = pReader->status.pTableIter; // restore reader's state, task snapshot - int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); if (numOfTables > 0) { tsdbTrace("tsdb/reader: %p, take snapshot", pReader); code = tsdbTakeReadSnap2(pReader, tsdbSetQueryReseek, &pReader->pReadSnap, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _end); // open reader failure may cause the flag still to be READER_STATUS_SUSPEND, which may cause suspend reader failure. // So we need to set it A.S.A.P @@ -5070,9 +5856,7 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { code = doOpenReaderImpl(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } else { STsdbReader* pPrevReader = pReader->innerReader[0]; STsdbReader* pNextReader = pReader->innerReader[1]; @@ -5080,54 +5864,55 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { // we need only one row pPrevReader->resBlockInfo.capacity = 1; code = setSharedPtr(pPrevReader, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); pNextReader->resBlockInfo.capacity = 1; code = setSharedPtr(pNextReader, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pReader->step == 0 || pReader->step == EXTERNAL_ROWS_PREV) { code = doOpenReaderImpl(pPrevReader); + TSDB_CHECK_CODE(code, lino, _end); } else if (pReader->step == EXTERNAL_ROWS_MAIN) { code = doOpenReaderImpl(pReader); + TSDB_CHECK_CODE(code, lino, _end); } else { code = doOpenReaderImpl(pNextReader); - } - - if (code != TSDB_CODE_SUCCESS) { - return code; + TSDB_CHECK_CODE(code, lino, _end); } } } tsdbDebug("reader: %p resumed uid %" PRIu64 ", numOfTable:%" PRId32 ", in this query %s", pReader, pBlockScanInfo ? (*pBlockScanInfo)->uid : 0, numOfTables, pReader->idStr); - return code; -_err: - tsdbError("failed to resume data reader, code:%s %s", tstrerror(code), pReader->idStr); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s, %s", __func__, lino, tstrerror(code), + (pReader && pReader->idStr) ? pReader->idStr : ""); + } return code; } static int32_t buildFromPreFilesetBuffer(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; - SReaderStatus* pStatus = &pReader->status; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + SSDataBlock* pBlock = NULL; + int32_t fid = 0; + STimeWindow win = {0}; - SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); - int32_t fid = pReader->status.pCurrentFileset->fid; - STimeWindow win = {0}; + pStatus = &pReader->status; + pBlock = pReader->resBlockInfo.pResBlock; + fid = pReader->status.pCurrentFileset->fid; tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey); int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? win.skey : win.ekey; code = buildBlockFromBufferSeqForPreFileset(pReader, endKey); - if (code != TSDB_CODE_SUCCESS || pBlock->info.rows > 0) { - return code; - } else { + TSDB_CHECK_CODE(code, lino, _end); + if (pBlock->info.rows <= 0) { tsdbDebug("finished pre-fileset %d buffer processing. %s", fid, pReader->idStr); pStatus->bProcMemPreFileset = false; pStatus->processingMemPreFileSet = false; @@ -5138,26 +5923,36 @@ static int32_t buildFromPreFilesetBuffer(STsdbReader* pReader) { tsdbDebug("new duration %d start notification when buffer pre-fileset, %s", fid, pReader->idStr); } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) { - SReaderStatus* pStatus = &pReader->status; int32_t code = TSDB_CODE_SUCCESS; - SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + SSDataBlock* pBlock = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pStatus = &pReader->status; + pBlock = pReader->resBlockInfo.pResBlock; if (pStatus->loadFromFile) { if (pStatus->bProcMemPreFileset) { code = buildFromPreFilesetBuffer(pReader); - if (code != TSDB_CODE_SUCCESS || pBlock->info.rows > 0) { - return code; + TSDB_CHECK_CODE(code, lino, _end); + if (pBlock->info.rows > 0) { + goto _end; } } code = buildBlockFromFiles(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); tsdbTrace("block from file rows: %" PRId64 ", will process pre-file set buffer: %d. %s", pBlock->info.rows, pStatus->bProcMemFirstFileset, pReader->idStr); @@ -5176,59 +5971,77 @@ static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) { if (pBlock->info.rows <= 0) { code = resetTableListIndex(&pReader->status, pReader->idStr); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; code = buildBlockFromBufferSequentially(pReader, endKey); + TSDB_CHECK_CODE(code, lino, _end); } } else { // no data in files, let's try the buffer int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; code = buildBlockFromBufferSequentially(pReader, endKey); + TSDB_CHECK_CODE(code, lino, _end); + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } return code; } static int32_t doTsdbNextDataBlockFilesFirst(STsdbReader* pReader) { - SReaderStatus* pStatus = &pReader->status; int32_t code = TSDB_CODE_SUCCESS; - SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + SSDataBlock* pBlock = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pStatus = &pReader->status; + pBlock = pReader->resBlockInfo.pResBlock; if (pStatus->loadFromFile) { code = buildBlockFromFiles(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pBlock->info.rows <= 0) { code = resetTableListIndex(&pReader->status, pReader->idStr); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; code = buildBlockFromBufferSequentially(pReader, endKey); + TSDB_CHECK_CODE(code, lino, _end); } } else { // no data in files, let's try the buffer int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; code = buildBlockFromBufferSequentially(pReader, endKey); + TSDB_CHECK_CODE(code, lino, _end); + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } return code; } static int32_t doTsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SSDataBlock* pBlock = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); // cleanup the data that belongs to the previous data block - SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + pBlock = pReader->resBlockInfo.pResBlock; blockDataCleanup(pBlock); *hasNext = false; SReaderStatus* pStatus = &pReader->status; if (tSimpleHashGetSize(pStatus->pTableMap) == 0) { - return code; + goto _end; } if (!pReader->bFilesetDelimited) { @@ -5238,21 +6051,34 @@ static int32_t doTsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { } *hasNext = pBlock->info.rows > 0; + TSDB_CHECK_CODE(code, lino, _end); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + bool acquired = false; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(hasNext, code, lino, _end, TSDB_CODE_INVALID_PARA); *hasNext = false; - if (isEmptyQueryTimeWindow(&pReader->info.window) || pReader->step == EXTERNAL_ROWS_NEXT || - pReader->code != TSDB_CODE_SUCCESS) { - return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code; + code = pReader->code; + TSDB_CHECK_CODE(code, lino, _end); + + if (isEmptyQueryTimeWindow(&pReader->info.window) || pReader->step == EXTERNAL_ROWS_NEXT) { + goto _end; } - SReaderStatus* pStatus = &pReader->status; + pStatus = &pReader->status; // NOTE: the following codes is used to perform test for suspend/resume for tsdbReader when it blocks the commit // the data should be ingested in round-robin and all the child tables should be createted before ingesting data @@ -5265,27 +6091,19 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { #endif code = tsdbAcquireReader(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); + acquired = true; tsdbTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code); if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); - if (code != TSDB_CODE_SUCCESS) { - // release reader failure should be suppressed here, to avoid over-write the original error code - (void) tsdbReleaseReader(pReader); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } if (pReader->innerReader[0] != NULL && pReader->step == 0) { code = doTsdbNextDataBlock2(pReader->innerReader[0], hasNext); - if (code) { - (void) tsdbReleaseReader(pReader); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); pReader->step = EXTERNAL_ROWS_PREV; if (*hasNext) { @@ -5293,9 +6111,11 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { if (pStatus->composedDataBlock) { tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); code = tsdbReleaseReader(pReader); + acquired = false; + TSDB_CHECK_CODE(code, lino, _end); } - return code; + goto _end; } } @@ -5307,27 +6127,22 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { int32_t step = 1; resetAllDataBlockScanInfo(pReader->status.pTableMap, pReader->innerReader[0]->info.window.ekey, step); - - if (code != TSDB_CODE_SUCCESS) { - (void) tsdbReleaseReader(pReader); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); pReader->step = EXTERNAL_ROWS_MAIN; } code = doTsdbNextDataBlock2(pReader, hasNext); - if (code != TSDB_CODE_SUCCESS) { - (void) tsdbReleaseReader(pReader); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (*hasNext) { if (pStatus->composedDataBlock) { tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); code = tsdbReleaseReader(pReader); + acquired = false; + TSDB_CHECK_CODE(code, lino, _end); } - return code; + goto _end; } if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) { @@ -5338,16 +6153,10 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { int32_t step = -1; resetAllDataBlockScanInfo(pReader->innerReader[1]->status.pTableMap, pReader->info.window.ekey, step); - if (code != TSDB_CODE_SUCCESS) { - (void) tsdbReleaseReader(pReader); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = doTsdbNextDataBlock2(pReader->innerReader[1], hasNext); - if (code != TSDB_CODE_SUCCESS) { - (void) tsdbReleaseReader(pReader); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); pReader->step = EXTERNAL_ROWS_NEXT; if (*hasNext) { @@ -5355,25 +6164,43 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { if (pStatus->composedDataBlock) { tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); code = tsdbReleaseReader(pReader); + acquired = false; + TSDB_CHECK_CODE(code, lino, _end); } - return code; + goto _end; } } tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); code = tsdbReleaseReader(pReader); + acquired = false; + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (acquired) { + tsdbTrace("tsdb/read: %p, unlock read mutex", pReader); + (void)tsdbReleaseReader(pReader); + } return code; } -static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { +static int32_t doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, + SColumnDataAgg* pTsAgg) { // do fill all null column value SMA info + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t i = 0, j = 0; - int32_t size = (int32_t)TARRAY2_SIZE(&pSup->colAggArray); - int32_t code = TARRAY2_INSERT_PTR(&pSup->colAggArray, 0, pTsAgg); - if (code != TSDB_CODE_SUCCESS) { - return; - } + int32_t size = 0; + + TSDB_CHECK_NULL(pSup, code, lino, _end, TSDB_CODE_INVALID_PARA); + + size = (int32_t)TARRAY2_SIZE(&pSup->colAggArray); + code = TARRAY2_INSERT_PTR(&pSup->colAggArray, 0, pTsAgg); + TSDB_CHECK_CODE(code, lino, _end); size++; @@ -5388,9 +6215,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; code = TARRAY2_INSERT_PTR(&pSup->colAggArray, i, &nullColAgg); - if (code != TSDB_CODE_SUCCESS) { - return; - } + TSDB_CHECK_CODE(code, lino, _end); i += 1; size++; @@ -5403,42 +6228,51 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; code = TARRAY2_INSERT_PTR(&pSup->colAggArray, i, &nullColAgg); - if (code != TSDB_CODE_SUCCESS) { - return; - } + TSDB_CHECK_CODE(code, lino, _end); i += 1; } j++; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool* hasNullSMA) { - SColumnDataAgg** pBlockSMA = &pDataBlock->pBlockAgg; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SColumnDataAgg** pBlockSMA = NULL; SFileDataBlockInfo* pBlockInfo = NULL; - int32_t code = 0; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pDataBlock, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(allHave, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pBlockSMA = &pDataBlock->pBlockAgg; *allHave = false; *pBlockSMA = NULL; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { - return TSDB_CODE_SUCCESS; + goto _end; } // there is no statistics data for composed block if (pReader->status.composedDataBlock || (!pReader->suppInfo.smaValid)) { - return TSDB_CODE_SUCCESS; + goto _end; } code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); SBlockLoadSuppInfo* pSup = &pReader->suppInfo; SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; if (pResBlock->info.id.uid != pBlockInfo->uid) { - return TSDB_CODE_SUCCESS; + goto _end; } // int64_t st = taosGetTimestampUs(); @@ -5450,14 +6284,14 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, if (code != TSDB_CODE_SUCCESS) { tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pBlockInfo->uid, tstrerror(code), pReader->idStr); - return code; + TSDB_CHECK_CODE(code, lino, _end); } if (pSup->colAggArray.size > 0) { *allHave = true; } else { *pBlockSMA = NULL; - return TSDB_CODE_SUCCESS; + goto _end; } // always load the first primary timestamp column data @@ -5474,16 +6308,15 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, if (pResBlock->pBlockAgg == NULL) { size_t num = taosArrayGetSize(pResBlock->pDataBlock); pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg)); - if (pResBlock->pBlockAgg == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pResBlock->pBlockAgg, code, lino, _end, terrno); for (int i = 0; i < num; ++i) { pResBlock->pBlockAgg[i].colId = -1; } } // do fill all null column value SMA info - doFillNullColSMA(pSup, pBlockInfo->numRow, numOfCols, pTsAgg); + code = doFillNullColSMA(pSup, pBlockInfo->numRow, numOfCols, pTsAgg); + TSDB_CHECK_CODE(code, lino, _end); size_t size = pSup->colAggArray.size; @@ -5510,49 +6343,67 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, pReader->cost.smaLoadTime += 0; // elapsedTime; tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pBlockInfo->uid, pReader->idStr); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } static int32_t doRetrieveDataBlock(STsdbReader* pReader, SSDataBlock** pBlock) { - SReaderStatus* pStatus = &pReader->status; - int32_t code = TSDB_CODE_SUCCESS; - SFileDataBlockInfo* pBlockInfo = NULL; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + SFileDataBlockInfo* pBlockInfo = NULL; + STableBlockScanInfo* pBlockScanInfo = NULL; + bool reset = false; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pStatus = &pReader->status; *pBlock = NULL; code = getCurrentBlockInfo(&pStatus->blockIter, &pBlockInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); - if (pReader->code != TSDB_CODE_SUCCESS) { - return pReader->code; - } + code = pReader->code; + TSDB_CHECK_CODE(code, lino, _end); - STableBlockScanInfo* pBlockScanInfo = NULL; code = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); + reset = true; code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); - if (code != TSDB_CODE_SUCCESS) { - tBlockDataReset(&pStatus->fileBlockData); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = copyBlockDataToSDataBlock(pReader, &pBlockScanInfo->lastProcKey); - if (code != TSDB_CODE_SUCCESS) { - tBlockDataReset(&pStatus->fileBlockData); - } + TSDB_CHECK_CODE(code, lino, _end); *pBlock = pReader->resBlockInfo.pResBlock; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + if (reset) { + tBlockDataReset(&pStatus->fileBlockData); + } + } return code; } int32_t tsdbRetrieveDataBlock2(STsdbReader* pReader, SSDataBlock** pBlock, SArray* pIdList) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + STsdbReader* pTReader = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA); + *pBlock = NULL; - STsdbReader* pTReader = pReader; + pTReader = pReader; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pReader->step == EXTERNAL_ROWS_PREV) { pTReader = pReader->innerReader[0]; @@ -5566,47 +6417,55 @@ int32_t tsdbRetrieveDataBlock2(STsdbReader* pReader, SSDataBlock** pBlock, SArra // tsdbReaderSuspend2(pReader); // tsdbReaderResume2(pReader); *pBlock = pTReader->resBlockInfo.pResBlock; - return TSDB_CODE_SUCCESS; + goto _end; } - int32_t code = doRetrieveDataBlock(pTReader, pBlock); + code = doRetrieveDataBlock(pTReader, pBlock); tsdbTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); (void) tsdbReleaseReader(pReader); + TSDB_CHECK_CODE(code, lino, _end); // tsdbReaderSuspend2(pReader); // tsdbReaderResume2(pReader); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + bool acquired = false; tsdbTrace("tsdb/reader-reset: %p, take read mutex", pReader); code = tsdbAcquireReader(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); + acquired = true; if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); - if (code != TSDB_CODE_SUCCESS) { - (void) tsdbReleaseReader(pReader); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } if (isEmptyQueryTimeWindow(&pReader->info.window) || pReader->pReadSnap == NULL) { tsdbDebug("tsdb reader reset return %p, %s", pReader->pReadSnap, pReader->idStr); - return tsdbReleaseReader(pReader); + code = tsdbReleaseReader(pReader); + acquired = false; + TSDB_CHECK_CODE(code, lino, _end); + goto _end; } SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; + TSDB_CHECK_NULL(pCond, code, lino, _end, TSDB_CODE_INVALID_PARA); pReader->info.order = pCond->order; pReader->type = TIMEWINDOW_RANGE_CONTAINED; - pReader->info.window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); + code = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows, &pReader->info.window); + TSDB_CHECK_CODE(code, lino, _end); pStatus->loadFromFile = true; pStatus->pTableIter = NULL; @@ -5619,22 +6478,13 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { int32_t numOfTables = tSimpleHashGetSize(pStatus->pTableMap); code = initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); - if (code != TSDB_CODE_SUCCESS) { - (void) tsdbReleaseReader(pReader); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo), pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - (void) tsdbReleaseReader(pReader); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = resetTableListIndex(&pReader->status, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - (void) tsdbReleaseReader(pReader); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); bool asc = ASCENDING_TRAVERSE(pReader->info.order); int32_t step = asc ? 1 : -1; @@ -5650,18 +6500,13 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { if (pStatus->fileIter.numOfFiles == 0) { pStatus->loadFromFile = false; code = resetTableListIndex(pStatus, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - (void) tsdbReleaseReader(pReader); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } else { code = initForFirstBlockInFile(pReader, pBlockIter); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p reset reader failed, numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s", pReader, numOfTables, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); - - (void) tsdbReleaseReader(pReader); - return code; + TSDB_CHECK_CODE(code, lino, _end); } } @@ -5671,6 +6516,16 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { pReader->info.window.ekey, pReader->idStr); code = tsdbReleaseReader(pReader); + acquired = false; + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (acquired) { + (void)tsdbReleaseReader(pReader); + } return code; } @@ -5687,7 +6542,11 @@ static int32_t getBucketIndex(int32_t startRow, int32_t bucketRange, int32_t num int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pTableBlockInfo) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; const int32_t numOfBuckets = 20.0; + bool acquired = false; + + TSDB_CHECK_NULL(pTableBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); pTableBlockInfo->totalSize = 0; pTableBlockInfo->totalRows = 0; @@ -5695,15 +6554,12 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT // find the start data block in file code = tsdbAcquireReader(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); + acquired = true; if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); - if (code != TSDB_CODE_SUCCESS) { - return tsdbReleaseReader(pReader); - } + TSDB_CHECK_CODE(code, lino, _end); } SMergeTreeConf conf = { @@ -5744,9 +6600,7 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT if (hasNext) { SFileDataBlockInfo* pBlockInfo = NULL; code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - break; - } + TSDB_CHECK_CODE(code, lino, _end); int32_t numOfRows = pBlockInfo->numRow; @@ -5765,10 +6619,11 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT int32_t bucketIndex = getBucketIndex(pTableBlockInfo->defMinRows, bucketRange, numOfRows, numOfBuckets); pTableBlockInfo->blockRowsHisto[bucketIndex]++; - hasNext = blockIteratorNext(&pStatus->blockIter, pReader->idStr); + hasNext = blockIteratorNext(&pStatus->blockIter); } else { code = initForFirstBlockInFile(pReader, pBlockIter); - if ((code != TSDB_CODE_SUCCESS) || (pStatus->loadFromFile == false)) { + TSDB_CHECK_CODE(code, lino, _end); + if (pStatus->loadFromFile == false) { break; } @@ -5784,14 +6639,17 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT } // record the data in stt files - (void) tsdbReleaseReader(pReader); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (acquired) { + (void)tsdbReleaseReader(pReader); + } return code; } static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t* pMinKey) { - int32_t code = TSDB_CODE_SUCCESS; - int64_t rows = 0; - SReaderStatus* pStatus = &pReader->status; int32_t iter = 0; @@ -5836,22 +6694,24 @@ static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t *pMinKey = minKey; } -int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; - int64_t rows = 0; +int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader, uint32_t* rows) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SReaderStatus* pStatus = NULL; + bool acquired = false; - SReaderStatus* pStatus = &pReader->status; + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(rows, code, lino, _end, TSDB_CODE_INVALID_PARA); + + *rows = 0; + pStatus = &pReader->status; code = tsdbAcquireReader(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); + acquired = true; if (pReader->flag == READER_STATUS_SUSPEND) { code = tsdbReaderResume2(pReader); - if (code != TSDB_CODE_SUCCESS) { - (void) tsdbReleaseReader(pReader); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } int32_t iter = 0; @@ -5864,7 +6724,7 @@ int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) { if (pReader->pReadSnap->pMem != NULL) { d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->info.suid, pBlockScanInfo->uid); if (d != NULL) { - rows += tsdbGetNRowsInTbData(d); + *rows += tsdbGetNRowsInTbData(d); } } @@ -5872,7 +6732,7 @@ int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) { if (pReader->pReadSnap->pIMem != NULL) { di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->info.suid, pBlockScanInfo->uid); if (di != NULL) { - rows += tsdbGetNRowsInTbData(di); + *rows += tsdbGetNRowsInTbData(di); } } @@ -5880,9 +6740,14 @@ int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) { pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, pStatus->pTableIter, &iter); } - (void) tsdbReleaseReader(pReader); - - return rows; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (acquired) { + (void)tsdbReleaseReader(pReader); + } + return code; } int32_t tsdbGetTableSchema(SMeta* pMeta, int64_t uid, STSchema** pSchema, int64_t* suid) { @@ -6037,17 +6902,23 @@ void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proact // if failed, do nothing int32_t tsdbReaderSetId(void* p, const char* idstr) { - STsdbReader* pReader = (STsdbReader*) p; - taosMemoryFreeClear(pReader->idStr); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + STsdbReader* pReader = (STsdbReader*)p; + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + taosMemoryFreeClear(pReader->idStr); pReader->idStr = taosStrdup(idstr); - if (pReader->idStr == NULL) { - tsdbError("%s failed to build reader id, code:%s", idstr, tstrerror(terrno)); - return terrno; - } + TSDB_CHECK_NULL(pReader->idStr, code, lino, _end, terrno); pReader->status.fileIter.pSttBlockReader->mergeTree.idStr = pReader->idStr; - return 0; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index a65fc7f3ed..5f77d03efc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -25,47 +25,57 @@ static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, int32_t order); static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { - int32_t num = numOfTables / pBuf->numPerBucket; - int32_t remainder = numOfTables % pBuf->numPerBucket; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t num = 0; + int32_t remainder = 0; + STableBlockScanInfo* p = NULL; + const void* px = NULL; + + TSDB_CHECK_CONDITION(pBuf && pBuf->numPerBucket > 0, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_CONDITION(numOfTables >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA); + + num = numOfTables / pBuf->numPerBucket; + remainder = numOfTables % pBuf->numPerBucket; if (pBuf->pData == NULL) { pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES); - if (pBuf->pData == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pBuf->pData, code, lino, _end, terrno); } for (int32_t i = 0; i < num; ++i) { - char* p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo)); - if (p == NULL) { - return terrno; - } + p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo)); + TSDB_CHECK_NULL(p, code, lino, _end, terrno); - void* px = taosArrayPush(pBuf->pData, &p); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pBuf->pData, &p); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); + p = NULL; } if (remainder > 0) { - char* p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo)); - if (p == NULL) { - return terrno; - } - void* px = taosArrayPush(pBuf->pData, &p); - if (px == NULL) { - return terrno; - } + p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo)); + TSDB_CHECK_NULL(p, code, lino, _end, terrno); + + px = taosArrayPush(pBuf->pData, &p); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); + p = NULL; } pBuf->numOfTables = numOfTables; - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (p) { + taosMemoryFreeClear(p); + } + return code; } int32_t uidComparFunc(const void* p1, const void* p2) { - uint64_t pu1 = *(uint64_t*)p1; - uint64_t pu2 = *(uint64_t*)p2; + uint64_t pu1 = *(const uint64_t*)p1; + uint64_t pu2 = *(const uint64_t*)p2; if (pu1 == pu2) { return 0; } else { @@ -74,90 +84,133 @@ int32_t uidComparFunc(const void* p1, const void* p2) { } int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t num = 0; + int32_t remainder = 0; + STableBlockScanInfo* p = NULL; + const void* px = NULL; + + TSDB_CHECK_CONDITION(pBuf && pBuf->numPerBucket > 0 && pBuf->numOfTables >= 0, code, lino, _end, + TSDB_CODE_INVALID_PARA); + TSDB_CHECK_CONDITION(numOfTables >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA); + if (numOfTables <= pBuf->numOfTables) { - return TSDB_CODE_SUCCESS; + goto _end; } - if (pBuf->numOfTables > 0) { - STableBlockScanInfo** p = (STableBlockScanInfo**)taosArrayPop(pBuf->pData); - taosMemoryFree(*p); - pBuf->numOfTables /= pBuf->numPerBucket; + remainder = pBuf->numOfTables % pBuf->numPerBucket; + if (remainder > 0) { + TSDB_CHECK_CONDITION(taosArrayGetSize(pBuf->pData) > 0, code, lino, _end, TSDB_CODE_INVALID_PARA); + px = taosArrayPop(pBuf->pData); + TSDB_CHECK_NULL(px, code, lino, _end, TSDB_CODE_INVALID_PARA); + p = *(STableBlockScanInfo**)px; + taosMemoryFreeClear(p); + pBuf->numOfTables -= remainder; } - int32_t num = (numOfTables - pBuf->numOfTables) / pBuf->numPerBucket; - int32_t remainder = (numOfTables - pBuf->numOfTables) % pBuf->numPerBucket; + num = (numOfTables - pBuf->numOfTables) / pBuf->numPerBucket; + remainder = (numOfTables - pBuf->numOfTables) % pBuf->numPerBucket; + if (pBuf->pData == NULL) { pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES); - if (pBuf->pData == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pBuf->pData, code, lino, _end, terrno); } for (int32_t i = 0; i < num; ++i) { - char* p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo)); - if (p == NULL) { - return terrno; - } + p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo)); + TSDB_CHECK_NULL(p, code, lino, _end, terrno); - void* px = taosArrayPush(pBuf->pData, &p); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pBuf->pData, &p); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); + p = NULL; } if (remainder > 0) { - char* p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo)); - if (p == NULL) { - return terrno; - } - void* px = taosArrayPush(pBuf->pData, &p); - if (px == NULL) { - return terrno; - } + p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo)); + TSDB_CHECK_NULL(p, code, lino, _end, terrno); + + px = taosArrayPush(pBuf->pData, &p); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); + p = NULL; } pBuf->numOfTables = numOfTables; - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + if (p) { + taosMemoryFreeClear(p); + } + return code; } void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) { - size_t num = taosArrayGetSize(pBuf->pData); - for (int32_t i = 0; i < num; ++i) { - char** p = taosArrayGet(pBuf->pData, i); - if (p != NULL) { - taosMemoryFree(*p); - } + if (pBuf == NULL) return; + if (pBuf->pData != NULL) { + taosArrayDestroyP(pBuf->pData, (FDelete)taosMemoryFree); + pBuf->pData = NULL; } - - taosArrayDestroy(pBuf->pData); } int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t bucketIndex = 0; + STableBlockScanInfo** pBucket = NULL; + + TSDB_CHECK_CONDITION(pBuf && pBuf->numPerBucket > 0, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_CONDITION(index >= 0 && index < pBuf->numOfTables, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + *pInfo = NULL; - int32_t bucketIndex = index / pBuf->numPerBucket; - char** pBucket = taosArrayGet(pBuf->pData, bucketIndex); - if (pBucket == NULL) { - return TSDB_CODE_NOT_FOUND; - } + bucketIndex = index / pBuf->numPerBucket; + pBucket = taosArrayGet(pBuf->pData, bucketIndex); + TSDB_CHECK_NULL(pBucket, code, lino, _end, terrno); - *pInfo = (STableBlockScanInfo*)((*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo)); - return TSDB_CODE_SUCCESS; + *pInfo = (*pBucket) + (index % pBuf->numPerBucket); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id) { - *pInfo = *(STableBlockScanInfo**)tSimpleHashGet(pTableMap, &uid, sizeof(uid)); - if (pInfo == NULL) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + STableBlockScanInfo** pVal = NULL; + + TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(id, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pVal = (STableBlockScanInfo**)tSimpleHashGet(pTableMap, &uid, sizeof(uid)); + if (pVal == NULL) { int32_t size = tSimpleHashGetSize(pTableMap); tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id); - return TSDB_CODE_INVALID_PARA; + code = TSDB_CODE_INVALID_PARA; + TSDB_CHECK_CODE(code, lino, _end); } + *pInfo = *pVal; + TSDB_CHECK_NULL(*pInfo, code, lino, _end, TSDB_CODE_INTERNAL_ERROR); - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA); + pKey->numOfPKs = numOfPks; pKey->ts = ts; @@ -194,7 +247,8 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in break; } default: - return TSDB_CODE_INVALID_PARA; + code = TSDB_CODE_INVALID_PARA; + TSDB_CHECK_CODE(code, lino, _end); } } else { switch (type) { @@ -223,16 +277,14 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in pKey->pks[0].val = UINT8_MAX; break; default: - return TSDB_CODE_INVALID_PARA; + code = TSDB_CODE_INVALID_PARA; + TSDB_CHECK_CODE(code, lino, _end); } } } else { - pKey->pks[0].pData = taosMemoryCalloc(1, len); pKey->pks[0].nData = 0; - - if (pKey->pks[0].pData == NULL) { - return terrno; - } + pKey->pks[0].pData = taosMemoryCalloc(1, len); + TSDB_CHECK_NULL(pKey->pks[0].pData, code, lino, _end, terrno); if (!asc) { pKey->numOfPKs = 2; @@ -240,7 +292,11 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in } } - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void clearRowKey(SRowKey* pKey) { @@ -250,53 +306,65 @@ void clearRowKey(SRowKey* pKey) { taosMemoryFreeClear(pKey->pks[0].pData); } -static int32_t initLastProcKey(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { - int32_t code = 0; - int32_t numOfPks = pReader->suppInfo.numOfPks; - bool asc = ASCENDING_TRAVERSE(pReader->info.order); - int8_t type = pReader->suppInfo.pk.type; - int32_t bytes = pReader->suppInfo.pk.bytes; +static int32_t initLastProcKey(STableBlockScanInfo* pScanInfo, const STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t numOfPks = 0; + int32_t type = 0; + int32_t bytes = 0; + bool asc = false; + SRowKey* pRowKey = NULL; - SRowKey* pRowKey = &pScanInfo->lastProcKey; + TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + numOfPks = pReader->suppInfo.numOfPks; + asc = ASCENDING_TRAVERSE(pReader->info.order); + type = pReader->suppInfo.pk.type; + bytes = pReader->suppInfo.pk.bytes; + + pRowKey = &pScanInfo->lastProcKey; if (asc) { int64_t skey = pReader->info.window.skey; int64_t ts = (skey > INT64_MIN) ? (skey - 1) : skey; code = initRowKey(pRowKey, ts, numOfPks, type, bytes, asc); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, skey, numOfPks, type, bytes, asc); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } else { int64_t ekey = pReader->info.window.ekey; int64_t ts = (ekey < INT64_MAX) ? (ekey + 1) : ekey; code = initRowKey(pRowKey, ts, numOfPks, type, bytes, asc); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, ekey, numOfPks, type, bytes, asc); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } code = initRowKey(&pScanInfo->sttRange.skey, INT64_MAX, numOfPks, type, bytes, asc); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); code = initRowKey(&pScanInfo->sttRange.ekey, INT64_MIN, numOfPks, type, bytes, asc); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap, STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + pScanInfo->uid = uid; INIT_KEYRANGE(&pScanInfo->sttRange); INIT_TIMEWINDOW(&pScanInfo->filesetWindow); @@ -304,46 +372,52 @@ int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSH pScanInfo->cleanSttBlocks = false; pScanInfo->sttBlockReturned = false; - int32_t code = initLastProcKey(pScanInfo, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = initLastProcKey(pScanInfo, pReader); + TSDB_CHECK_CODE(code, lino, _end); pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pReader, pScanInfo->uid, pScanInfo->lastProcKey.ts, pReader->idStr); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj) { - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SSHashObj* pTableMap = NULL; + int64_t st = 0; + + TSDB_CHECK_NULL(pUidList, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pHashObj, code, lino, _end, TSDB_CODE_INVALID_PARA); + *pHashObj = NULL; // allocate buffer in order to load data blocks from file // todo use simple hash instead, optimize the memory consumption - SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); - if (pTableMap == NULL) { - return terrno; - } + pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); + TSDB_CHECK_NULL(pTableMap, code, lino, _end, terrno); - int64_t st = taosGetTimestampUs(); + st = taosGetTimestampUs(); code = initBlockScanInfoBuf(pBuf, numOfTables); if (code != TSDB_CODE_SUCCESS) { tSimpleHashCleanup(pTableMap); - return code; + TSDB_CHECK_CODE(code, lino, _end); } pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); if (pUidList->tableUidList == NULL) { tSimpleHashCleanup(pTableMap); - return terrno; + TSDB_CHECK_NULL(pUidList->tableUidList, code, lino, _end, terrno); } pUidList->currentIndex = 0; @@ -354,11 +428,13 @@ int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, c STableBlockScanInfo* pScanInfo = NULL; code = getPosInBlockInfoBuf(pBuf, j, &pScanInfo); if (code != TSDB_CODE_SUCCESS) { + lino = __LINE__; break; } code = initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader); if (code != TSDB_CODE_SUCCESS) { + lino = __LINE__; break; } } @@ -371,15 +447,23 @@ int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, c pTsdbReader->idStr); *pHashObj = pTableMap; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { - void* p = NULL; - int32_t iter = 0; + STableBlockScanInfo** p = NULL; + int32_t iter = 0; while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { - STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; + STableBlockScanInfo* pInfo = *p; + if (pInfo == NULL) { + continue; + } pInfo->iterInit = false; pInfo->iter.hasVal = false; @@ -402,6 +486,10 @@ void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { } void clearBlockScanInfo(STableBlockScanInfo* p) { + if (p == NULL) { + return; + } + p->iterInit = false; p->iter.hasVal = false; p->iiter.hasVal = false; @@ -432,18 +520,26 @@ void clearBlockScanInfo(STableBlockScanInfo* p) { clearRowKey(&p->sttKeyInfo.nextProcKey); } -void destroyAllBlockScanInfo(SSHashObj* pTableMap) { - void* p = NULL; - int32_t iter = 0; +void destroyAllBlockScanInfo(SSHashObj** pTableMap) { + STableBlockScanInfo** p = NULL; + int32_t iter = 0; - while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { - clearBlockScanInfo(*(STableBlockScanInfo**)p); + if (pTableMap == NULL || *pTableMap == NULL) { + return; } - tSimpleHashCleanup(pTableMap); + while ((p = tSimpleHashIterate(*pTableMap, p, &iter)) != NULL) { + clearBlockScanInfo(*p); + } + + tSimpleHashCleanup(*pTableMap); + *pTableMap = NULL; } static void doCleanupInfoForNextFileset(STableBlockScanInfo* pScanInfo) { + if (pScanInfo == NULL) { + return; + } // reset the index in last block when handing a new file taosArrayClear(pScanInfo->pBlockList); taosArrayClear(pScanInfo->pBlockIdxList); @@ -458,8 +554,8 @@ static void doCleanupInfoForNextFileset(STableBlockScanInfo* pScanInfo) { void cleanupInfoForNextFileset(SSHashObj* pTableMap) { STableBlockScanInfo** p = NULL; + int32_t iter = 0; - int32_t iter = 0; while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { doCleanupInfoForNextFileset(*p); } @@ -467,6 +563,10 @@ void cleanupInfoForNextFileset(SSHashObj* pTableMap) { // brin records iterator void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList) { + if (pIter == NULL) { + return; + } + (void)memset(&pIter->block, 0, sizeof(SBrinBlock)); (void)memset(&pIter->record, 0, sizeof(SBrinRecord)); pIter->blockIndex = -1; @@ -477,89 +577,133 @@ void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray } int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pRecord, code, lino, _end, TSDB_CODE_INVALID_PARA); + *pRecord = NULL; if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) { pIter->blockIndex += 1; if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) { - return TSDB_CODE_SUCCESS; + goto _end; } pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex); - if (pIter->pCurrentBlk == NULL) { - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_NULL(pIter->pCurrentBlk, code, lino, _end, terrno); tBrinBlockClear(&pIter->block); - int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block); + TSDB_CHECK_NULL(pIter->pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block); if (code != TSDB_CODE_SUCCESS) { tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code)); - return code; + TSDB_CHECK_CODE(code, lino, _end); } pIter->recordIndex = -1; } pIter->recordIndex += 1; - int32_t code = tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record); + code = tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record); *pRecord = &pIter->record; + TSDB_CHECK_CODE(code, lino, _end); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } -void clearBrinBlockIter(SBrinRecordIter* pIter) { tBrinBlockDestroy(&pIter->block); } +void clearBrinBlockIter(SBrinRecordIter* pIter) { + if (pIter != NULL) { + tBrinBlockDestroy(&pIter->block); + } +} // initialize the file block access order // sort the file blocks according to the offset of each data block in the files static void cleanupBlockOrderSupporter(SBlockOrderSupporter* pSup) { + if (pSup == NULL) { + return; + } + taosMemoryFreeClear(pSup->numOfBlocksPerTable); taosMemoryFreeClear(pSup->indexPerTable); - for (int32_t i = 0; i < pSup->numOfTables; ++i) { - SBlockOrderWrapper* pBlockInfo = pSup->pDataBlockInfo[i]; - taosMemoryFreeClear(pBlockInfo); - } + if (pSup->pDataBlockInfo != NULL) { + for (int32_t i = 0; i < pSup->numOfTables; ++i) { + SBlockOrderWrapper* pBlockInfo = pSup->pDataBlockInfo[i]; + taosMemoryFreeClear(pBlockInfo); + } - taosMemoryFreeClear(pSup->pDataBlockInfo); + taosMemoryFreeClear(pSup->pDataBlockInfo); + } } static int32_t initBlockOrderSupporter(SBlockOrderSupporter* pSup, int32_t numOfTables) { - pSup->pDataBlockInfo = taosMemoryCalloc(1, POINTER_BYTES * numOfTables); - pSup->indexPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); - pSup->numOfBlocksPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); - pSup->numOfTables = 0; - if (pSup->pDataBlockInfo == NULL || pSup->indexPerTable == NULL || pSup->numOfBlocksPerTable == NULL) { - cleanupBlockOrderSupporter(pSup); - return terrno; - } + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; - return TSDB_CODE_SUCCESS; + TSDB_CHECK_NULL(pSup, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_CONDITION(numOfTables >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pSup->pDataBlockInfo = taosMemoryCalloc(numOfTables, POINTER_BYTES); + TSDB_CHECK_NULL(pSup->pDataBlockInfo, code, lino, _end, terrno); + pSup->indexPerTable = taosMemoryCalloc(numOfTables, sizeof(int32_t)); + TSDB_CHECK_NULL(pSup->indexPerTable, code, lino, _end, terrno); + pSup->numOfBlocksPerTable = taosMemoryCalloc(numOfTables, sizeof(int32_t)); + TSDB_CHECK_NULL(pSup->numOfBlocksPerTable, code, lino, _end, terrno); + pSup->numOfTables = 0; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) { - int32_t leftIndex = *(int32_t*)pLeft; - int32_t rightIndex = *(int32_t*)pRight; + int32_t leftIndex = 0; + int32_t rightIndex = 0; + int32_t leftTableBlockIndex = 0; + int32_t rightTableBlockIndex = 0; + const SBlockOrderSupporter* pSupporter = NULL; + const SBlockOrderWrapper* pLeftBlock = NULL; + const SBlockOrderWrapper* pRightBlock = NULL; - SBlockOrderSupporter* pSupporter = (SBlockOrderSupporter*)param; + leftIndex = *(const int32_t*)pLeft; + rightIndex = *(const int32_t*)pRight; + pSupporter = (const SBlockOrderSupporter*)param; - int32_t leftTableBlockIndex = pSupporter->indexPerTable[leftIndex]; - int32_t rightTableBlockIndex = pSupporter->indexPerTable[rightIndex]; + leftTableBlockIndex = pSupporter->indexPerTable[leftIndex]; + rightTableBlockIndex = pSupporter->indexPerTable[rightIndex]; - if (leftTableBlockIndex > pSupporter->numOfBlocksPerTable[leftIndex]) { + if (leftTableBlockIndex >= pSupporter->numOfBlocksPerTable[leftIndex]) { /* left block is empty */ return 1; - } else if (rightTableBlockIndex > pSupporter->numOfBlocksPerTable[rightIndex]) { + } else if (rightTableBlockIndex >= pSupporter->numOfBlocksPerTable[rightIndex]) { /* right block is empty */ return -1; } - SBlockOrderWrapper* pLeftBlock = &pSupporter->pDataBlockInfo[leftIndex][leftTableBlockIndex]; - SBlockOrderWrapper* pRightBlock = &pSupporter->pDataBlockInfo[rightIndex][rightTableBlockIndex]; + pLeftBlock = &pSupporter->pDataBlockInfo[leftIndex][leftTableBlockIndex]; + pRightBlock = &pSupporter->pDataBlockInfo[rightIndex][rightTableBlockIndex]; return pLeftBlock->offset > pRightBlock->offset ? 1 : -1; } int32_t recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + const SRowKey* pFirstKey = NULL; + const SRowKey* pLastKey = NULL; + + TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(record, code, lino, _end, TSDB_CODE_INVALID_PARA); + pBlockInfo->uid = record->uid; pBlockInfo->firstKey = record->firstKey.key.ts; pBlockInfo->lastKey = record->lastKey.key.ts; @@ -573,40 +717,50 @@ int32_t recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record) { pBlockInfo->numRow = record->numRow; pBlockInfo->count = record->count; - SRowKey* pFirstKey = &record->firstKey.key; + pFirstKey = &record->firstKey.key; + pLastKey = &record->lastKey.key; + TSDB_CHECK_CONDITION((pFirstKey->numOfPKs == pLastKey->numOfPKs), code, lino, _end, TSDB_CODE_INVALID_PARA); if (pFirstKey->numOfPKs > 0) { if (IS_NUMERIC_TYPE(pFirstKey->pks[0].type)) { pBlockInfo->firstPk.val = pFirstKey->pks[0].val; - pBlockInfo->lastPk.val = record->lastKey.key.pks[0].val; + pBlockInfo->lastPk.val = pLastKey->pks[0].val; } else { - char* p = taosMemoryCalloc(1, pFirstKey->pks[0].nData + VARSTR_HEADER_SIZE); - if (p == NULL) { - return terrno; - } - memcpy(varDataVal(p), pFirstKey->pks[0].pData, pFirstKey->pks[0].nData); - varDataSetLen(p, pFirstKey->pks[0].nData); + int32_t keyLen = pFirstKey->pks[0].nData; + char* p = taosMemoryMalloc(keyLen + VARSTR_HEADER_SIZE); + TSDB_CHECK_NULL(p, code, lino, _end, terrno); + memcpy(varDataVal(p), pFirstKey->pks[0].pData, keyLen); + varDataSetLen(p, keyLen); pBlockInfo->firstPk.pData = (uint8_t*)p; - int32_t keyLen = record->lastKey.key.pks[0].nData; - p = taosMemoryCalloc(1, keyLen + VARSTR_HEADER_SIZE); - if (p == NULL) { - return terrno; - } - memcpy(varDataVal(p), record->lastKey.key.pks[0].pData, keyLen); + keyLen = pLastKey->pks[0].nData; + p = taosMemoryMalloc(keyLen + VARSTR_HEADER_SIZE); + TSDB_CHECK_NULL(p, code, lino, _end, terrno); + memcpy(varDataVal(p), pLastKey->pks[0].pData, keyLen); varDataSetLen(p, keyLen); pBlockInfo->lastPk.pData = (uint8_t*)p; } } - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void freePkItem(void* pItem) { SFileDataBlockInfo* p = pItem; - taosMemoryFreeClear(p->firstPk.pData); - taosMemoryFreeClear(p->lastPk.pData); + if (p != NULL) { + taosMemoryFreeClear(p->firstPk.pData); + taosMemoryFreeClear(p->lastPk.pData); + } } void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree) { + if (pIter == NULL) { + return; + } + pIter->index = -1; pIter->numOfBlocks = 0; @@ -618,6 +772,10 @@ void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree) { } void cleanupDataBlockIterator(SDataBlockIter* pIter, bool needFree) { + if (pIter == NULL) { + return; + } + pIter->index = -1; pIter->numOfBlocks = 0; if (needFree) { @@ -625,159 +783,149 @@ void cleanupDataBlockIterator(SDataBlockIter* pIter, bool needFree) { } else { taosArrayDestroy(pIter->blockList); } + pIter->blockList = NULL; } int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) { - bool asc = ASCENDING_TRAVERSE(pReader->info.order); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + bool asc = false; + int32_t numOfTables = 0; + int64_t st = 0; + int64_t et = 0; + int32_t cnt = 0; + SBlockOrderSupporter sup = {0}; + SMultiwayMergeTreeInfo* pTree = NULL; + STableBlockScanInfo* pTableScanInfo = NULL; + const SFileDataBlockInfo* pBlockInfo = NULL; + const void* px = NULL; - SBlockOrderSupporter sup = {0}; + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockIter, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_CONDITION(numOfBlocks >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA); + + asc = ASCENDING_TRAVERSE(pReader->info.order); clearDataBlockIterator(pBlockIter, shouldFreePkBuf(&pReader->suppInfo)); - pBlockIter->numOfBlocks = numOfBlocks; // access data blocks according to the offset of each block in asc/desc order. - int32_t numOfTables = taosArrayGetSize(pTableList); + numOfTables = taosArrayGetSize(pTableList); - int64_t st = taosGetTimestampUs(); - int32_t code = initBlockOrderSupporter(&sup, numOfTables); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - int32_t cnt = 0; + st = taosGetTimestampUs(); + code = initBlockOrderSupporter(&sup, numOfTables); + TSDB_CHECK_CODE(code, lino, _end); for (int32_t i = 0; i < numOfTables; ++i) { - STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, i); + pTableScanInfo = taosArrayGetP(pTableList, i); - size_t num = taosArrayGetSize(pTableScanInfo->pBlockList); + size_t num = taosArrayGetSize(pTableScanInfo->pBlockList); + SBlockOrderWrapper* buf = taosMemoryMalloc(sizeof(SBlockOrderWrapper) * num); + TSDB_CHECK_NULL(buf, code, lino, _end, terrno); sup.numOfBlocksPerTable[sup.numOfTables] = num; - - char* buf = taosMemoryMalloc(sizeof(SBlockOrderWrapper) * num); - if (buf == NULL) { - cleanupBlockOrderSupporter(&sup); - return terrno; - } - - sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf; + sup.pDataBlockInfo[sup.numOfTables] = buf; + sup.numOfTables++; for (int32_t k = 0; k < num; ++k) { - SFileDataBlockInfo* pBlockInfo = taosArrayGet(pTableScanInfo->pBlockList, k); - if (pBlockInfo == NULL) { - return TSDB_CODE_INVALID_PARA; - } + pBlockInfo = taosArrayGet(pTableScanInfo->pBlockList, k); + TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); - sup.pDataBlockInfo[sup.numOfTables][k] = + sup.pDataBlockInfo[i][k] = (SBlockOrderWrapper){.uid = pTableScanInfo->uid, .offset = pBlockInfo->blockOffset, .pInfo = pTableScanInfo}; cnt++; } - - sup.numOfTables += 1; } - if (numOfBlocks != cnt && sup.numOfTables != numOfTables) { - cleanupBlockOrderSupporter(&sup); - return TSDB_CODE_INVALID_PARA; - } + TSDB_CHECK_CONDITION(!(numOfBlocks != cnt && sup.numOfTables != numOfTables), code, lino, _end, + TSDB_CODE_INVALID_PARA); // since there is only one table qualified, blocks are not sorted if (sup.numOfTables == 1) { - STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, 0); + pTableScanInfo = taosArrayGetP(pTableList, 0); for (int32_t i = 0; i < numOfBlocks; ++i) { STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i}; - void* px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); } - void* p = taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList); - if (p == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } + px = taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); taosArrayDestroy(pTableScanInfo->pBlockList); pTableScanInfo->pBlockList = NULL; - int64_t et = taosGetTimestampUs(); + et = taosGetTimestampUs(); tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s", pReader, numOfBlocks, (et - st) / 1000.0, pReader->idStr); pBlockIter->index = asc ? 0 : (numOfBlocks - 1); - cleanupBlockOrderSupporter(&sup); - return TSDB_CODE_SUCCESS; + goto _end; } tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pReader, cnt, sup.numOfTables, pReader->idStr); - SMultiwayMergeTreeInfo* pTree = NULL; + code = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar); + TSDB_CHECK_CODE(code, lino, _end); - uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar); - if (ret != TSDB_CODE_SUCCESS) { - cleanupBlockOrderSupporter(&sup); - return TSDB_CODE_OUT_OF_MEMORY; - } - - int32_t numOfTotal = 0; - while (numOfTotal < cnt) { + for (int32_t i = 0; i < cnt; ++i) { int32_t pos = tMergeTreeGetChosenIndex(pTree); int32_t index = sup.indexPerTable[pos]++; + pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo; - SFileDataBlockInfo* pBlockInfo = taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); - if (pBlockInfo == NULL) { - return TSDB_CODE_INVALID_PARA; - } + pBlockInfo = taosArrayGet(pTableScanInfo->pBlockList, index); + TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, terrno); - void* px = taosArrayPush(pBlockIter->blockList, pBlockInfo); - if (px == NULL) { - return terrno; - } - - STableBlockScanInfo* pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo; - STableDataBlockIdx tableDataBlockIdx = {.globalIndex = numOfTotal}; + px = taosArrayPush(pBlockIter->blockList, pBlockInfo); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); + STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i}; px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx); - if (px == NULL) { - return terrno; - } + TSDB_CHECK_NULL(px, code, lino, _end, terrno); // set data block index overflow, in order to disable the offset comparator if (sup.indexPerTable[pos] >= sup.numOfBlocksPerTable[pos]) { sup.indexPerTable[pos] = sup.numOfBlocksPerTable[pos] + 1; } - numOfTotal += 1; code = tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree)); - if (TSDB_CODE_SUCCESS != code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } for (int32_t i = 0; i < numOfTables; ++i) { - STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, i); + pTableScanInfo = taosArrayGetP(pTableList, i); taosArrayDestroy(pTableScanInfo->pBlockList); pTableScanInfo->pBlockList = NULL; } - int64_t et = taosGetTimestampUs(); + et = taosGetTimestampUs(); tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks, (et - st) / 1000.0, pReader->idStr); - cleanupBlockOrderSupporter(&sup); - taosMemoryFree(pTree); pBlockIter->index = asc ? 0 : (numOfBlocks - 1); - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + cleanupBlockOrderSupporter(&sup); + if (pTree != NULL) { + tMergeTreeDestroy(&pTree); + } + return code; } -bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr) { - bool asc = ASCENDING_TRAVERSE(pBlockIter->order); +bool blockIteratorNext(SDataBlockIter* pBlockIter) { + bool asc = false; - int32_t step = asc ? 1 : -1; - if ((pBlockIter->index >= pBlockIter->numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) { + if (pBlockIter == NULL) { return false; } - pBlockIter->index += step; + asc = ASCENDING_TRAVERSE(pBlockIter->order); + if ((pBlockIter->index >= pBlockIter->numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) { + return false; + } + pBlockIter->index += asc ? 1 : -1; return true; } @@ -791,30 +939,29 @@ static int32_t loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatis int32_t* j); static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_t numOfTables, int32_t* j, ETombBlkCheckEnum* pRet) { - int32_t code = 0; - STombRecord record = {0}; - - uint64_t uid = pReader->status.uidList.tableUidList[*j]; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + STombRecord record = {0}; + uint64_t uid = 0; STableBlockScanInfo* pScanInfo = NULL; + TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pRet, code, lino, _end, TSDB_CODE_INVALID_PARA); + + *pRet = BLK_CHECK_QUIT; + uid = pReader->status.uidList.tableUidList[*j]; code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pScanInfo->pFileDelData == NULL) { pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); - if (pScanInfo->pFileDelData == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pScanInfo->pFileDelData, code, lino, _end, terrno); } for (int32_t k = 0; k < pBlock->numOfRecords; ++k) { code = tTombBlockGet(pBlock, k, &record); - if (code != TSDB_CODE_SUCCESS) { - *pRet = BLK_CHECK_QUIT; - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (record.suid < pReader->info.suid) { continue; @@ -822,7 +969,7 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_ if (record.suid > pReader->info.suid) { *pRet = BLK_CHECK_QUIT; - return TSDB_CODE_SUCCESS; + goto _end; } if (uid < record.uid) { @@ -832,20 +979,16 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_ if ((*j) >= numOfTables) { *pRet = BLK_CHECK_QUIT; - return TSDB_CODE_SUCCESS; + goto _end; } uid = pReader->status.uidList.tableUidList[*j]; code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); if (pScanInfo->pFileDelData == NULL) { pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); - if (pScanInfo->pFileDelData == NULL) { - return terrno; - } + TSDB_CHECK_NULL(pScanInfo->pFileDelData, code, lino, _end, terrno); } } @@ -853,30 +996,38 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_ continue; } - if (!(record.suid == pReader->info.suid && uid == record.uid)) { - tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_INTERNAL_ERROR; - } + TSDB_CHECK_CONDITION((record.suid == pReader->info.suid) && (uid == record.uid), code, lino, _end, + TSDB_CODE_INTERNAL_ERROR); if (record.version <= pReader->info.verRange.maxVer) { - SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey}; - void* px = taosArrayPush(pScanInfo->pFileDelData, &delData); - if (px == NULL) { - return terrno; - } + SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey}; + const void* px = taosArrayPush(pScanInfo->pFileDelData, &delData); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); } } *pRet = BLK_CHECK_CONTINUE; - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } // load tomb data API static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STsdbReader* pReader, void* pFileReader, bool isFile) { - int32_t code = 0; - STableUidList* pList = &pReader->status.uidList; - int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + const STableUidList* pList = NULL; + int32_t numOfTables = 0; + + TSDB_CHECK_NULL(pTombBlkArray, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + pList = &pReader->status.uidList; + numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); int32_t i = 0, j = 0; while (i < pTombBlkArray->size && j < numOfTables) { @@ -890,10 +1041,10 @@ static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STs break; } - if (!(pTombBlk->minTbid.suid <= pReader->info.suid && pTombBlk->maxTbid.suid >= pReader->info.suid)) { - tsdbError("tsdb reader failed at: %s:%d", __func__, __LINE__); - return TSDB_CODE_INTERNAL_ERROR; - } + TSDB_CHECK_CONDITION( + (pTombBlk->minTbid.suid <= pReader->info.suid) && (pTombBlk->maxTbid.suid >= pReader->info.suid), code, lino, + _end, TSDB_CODE_INTERNAL_ERROR); + if (pTombBlk->maxTbid.suid == pReader->info.suid && pTombBlk->maxTbid.uid < pList->tableUidList[0]) { i += 1; continue; @@ -906,79 +1057,98 @@ static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STs STombBlock block = {0}; code = isFile ? tsdbDataFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block) : tsdbSttFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // uint64_t uid = pReader->status.uidList.tableUidList[j]; - - // STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); - // if (pScanInfo->pFileDelData == NULL) { - // pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); - // } + TSDB_CHECK_CODE(code, lino, _end); ETombBlkCheckEnum ret = 0; code = doCheckTombBlock(&block, pReader, numOfTables, &j, &ret); tTombBlockDestroy(&block); - if (code != TSDB_CODE_SUCCESS || ret == BLK_CHECK_QUIT) { - return code; + TSDB_CHECK_CODE(code, lino, _end); + if (ret == BLK_CHECK_QUIT) { + break; } i += 1; } - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t loadDataFileTombDataForAll(STsdbReader* pReader) { - if (pReader->status.pCurrentFileset == NULL || pReader->status.pCurrentFileset->farr[3] == NULL) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + const TTombBlkArray* pBlkArray = NULL; + + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + if ((pReader->status.pCurrentFileset == NULL) || (pReader->status.pCurrentFileset->farr[TSDB_FTYPE_TOMB] == NULL)) { return TSDB_CODE_SUCCESS; } - const TTombBlkArray* pBlkArray = NULL; + code = tsdbDataFileReadTombBlk(pReader->pFileReader, &pBlkArray); + TSDB_CHECK_CODE(code, lino, _end); - int32_t code = tsdbDataFileReadTombBlk(pReader->pFileReader, &pBlkArray); + code = doLoadTombDataFromTombBlk(pBlkArray, pReader, pReader->pFileReader, true); + TSDB_CHECK_CODE(code, lino, _end); + +_end: if (code != TSDB_CODE_SUCCESS) { - return code; + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } - - return doLoadTombDataFromTombBlk(pBlkArray, pReader, pReader->pFileReader, true); + return code; } int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; const TTombBlkArray* pBlkArray = NULL; - int32_t code = tsdbSttFileReadTombBlk(pSttFileReader, &pBlkArray); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - return doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false); + TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + + code = tsdbSttFileReadTombBlk(pSttFileReader, &pBlkArray); + TSDB_CHECK_CODE(code, lino, _end); + + code = doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SArray* pMemDelData = NULL; + const SDelData* p = NULL; + const void* px = NULL; + + TSDB_CHECK_NULL(ppMemDelData, code, lino, _end, TSDB_CODE_INVALID_PARA); + if (*ppMemDelData == NULL) { *ppMemDelData = taosArrayInit(4, sizeof(SDelData)); - if (*ppMemDelData == NULL) { - return terrno; - } + TSDB_CHECK_NULL(*ppMemDelData, code, lino, _end, terrno); } - SArray* pMemDelData = *ppMemDelData; + pMemDelData = *ppMemDelData; - SDelData* p = NULL; if (pMemTbData != NULL) { taosRLockLatch(&pMemTbData->lock); p = pMemTbData->pHead; while (p) { if (p->version <= ver) { - void* px = taosArrayPush(pMemDelData, p); + px = taosArrayPush(pMemDelData, p); if (px == NULL) { taosRUnLockLatch(&pMemTbData->lock); - return terrno; + TSDB_CHECK_NULL(px, code, lino, _end, terrno); } } - p = p->pNext; } taosRUnLockLatch(&pMemTbData->lock); @@ -988,28 +1158,36 @@ int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piM p = piMemTbData->pHead; while (p) { if (p->version <= ver) { - void* px = taosArrayPush(pMemDelData, p); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pMemDelData, p); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); } p = p->pNext; } } - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo, TStatisBlkArray* pStatisBlkArray, uint64_t suid, const uint64_t* pUidList, int32_t numOfTables, int32_t* pNumOfRows) { - int32_t num = 0; - int32_t code = 0; - int32_t lino = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t num = 0; + int64_t st = 0; + const SStatisBlk* p = NULL; + STbStatisBlock* pStatisBlock = NULL; - if (pNumOfRows != 0) { - *pNumOfRows = 0; - } + TSDB_CHECK_NULL(pSttFileReader, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBlockLoadInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pStatisBlkArray, code, lino, _end, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pNumOfRows, code, lino, _end, TSDB_CODE_INVALID_PARA); + + *pNumOfRows = 0; if (TARRAY2_SIZE(pStatisBlkArray) <= 0) { return code; @@ -1021,21 +1199,21 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo } if (i >= TARRAY2_SIZE(pStatisBlkArray)) { - return code; + goto _end; } - SStatisBlk* p = &pStatisBlkArray->data[i]; - STbStatisBlock* pStatisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock)); - TSDB_CHECK_NULL(pStatisBlock, code, lino, _err, terrno); + p = &pStatisBlkArray->data[i]; + pStatisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock)); + TSDB_CHECK_NULL(pStatisBlock, code, lino, _end, terrno); code = tStatisBlockInit(pStatisBlock); - TSDB_CHECK_CODE(code, lino, _err); + TSDB_CHECK_CODE(code, lino, _end); - int64_t st = taosGetTimestampMs(); + st = taosGetTimestampUs(); code = tsdbSttFileReadStatisBlock(pSttFileReader, p, pStatisBlock); - TSDB_CHECK_CODE(code, lino, _err); + TSDB_CHECK_CODE(code, lino, _end); - double el = (taosGetTimestampMs() - st) / 1000.0; + double el = (taosGetTimestampUs() - st) / 1000.0; pBlockLoadInfo->cost.loadStatisBlocks += 1; pBlockLoadInfo->cost.statisElapsedTime += el; @@ -1045,10 +1223,8 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo } if (index >= pStatisBlock->numOfRecords) { - tStatisBlockDestroy(pStatisBlock); - taosMemoryFreeClear(pStatisBlock); *pNumOfRows = num; - return code; + goto _end; } int32_t j = index; @@ -1056,10 +1232,8 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo while (i < TARRAY2_SIZE(pStatisBlkArray) && uidIndex < numOfTables) { p = &pStatisBlkArray->data[i]; if (p->minTbid.suid > suid) { - tStatisBlockDestroy(pStatisBlock); - taosMemoryFreeClear(pStatisBlock); *pNumOfRows = num; - return code; + goto _end; } uint64_t uid = pUidList[uidIndex]; @@ -1069,24 +1243,26 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo uidIndex += 1; j += 1; code = loadNextStatisticsBlock(pSttFileReader, pStatisBlock, pStatisBlkArray, pStatisBlock->numOfRecords, &i, &j); - TSDB_CHECK_CODE(code, lino, _err); + TSDB_CHECK_CODE(code, lino, _end); } else if (((int64_t*)pStatisBlock->uids.data)[j] < uid) { j += 1; code = loadNextStatisticsBlock(pSttFileReader, pStatisBlock, pStatisBlkArray, pStatisBlock->numOfRecords, &i, &j); - TSDB_CHECK_CODE(code, lino, _err); + TSDB_CHECK_CODE(code, lino, _end); } else { uidIndex += 1; } } - tStatisBlockDestroy(pStatisBlock); - taosMemoryFreeClear(pStatisBlock); *pNumOfRows = num; - return code; -_err: - tsdbError("%p failed to get number of rows in stt block, %s at line:%d code:%s", pSttFileReader, __func__, lino, - tstrerror(code)); +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s with %p failed at line %d since %s", __func__, pSttFileReader, lino, tstrerror(code)); + } + if (pStatisBlock) { + tStatisBlockDestroy(pStatisBlock); + taosMemoryFreeClear(pStatisBlock); + } return code; } @@ -1094,93 +1270,118 @@ _err: static int32_t loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatisBlock* pStatisBlock, const TStatisBlkArray* pStatisBlkArray, int32_t numOfRows, int32_t* i, int32_t* j) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + if ((*j) >= numOfRows) { (*i) += 1; (*j) = 0; if ((*i) < TARRAY2_SIZE(pStatisBlkArray)) { - int32_t code = tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[(*i)], pStatisBlock); + code = tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[(*i)], pStatisBlock); if (code != 0) { tsdbError("%p failed to read statisBlock, code:%s", pSttFileReader, tstrerror(code)); - return code; + TSDB_CHECK_CODE(code, lino, _end); } } } - return 0; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) { - int32_t size = taosArrayGetSize(pLDIterList); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t size = 0; + int32_t inc = 0; + SLDataIter* pIter = NULL; + + size = taosArrayGetSize(pLDIterList); if (size < numOfFileObj) { - int32_t inc = numOfFileObj - size; + inc = numOfFileObj - size; for (int32_t k = 0; k < inc; ++k) { - SLDataIter* pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); - if (!pIter) { - return terrno; - } + pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); + TSDB_CHECK_NULL(pIter, code, lino, _end, terrno); void* px = taosArrayPush(pLDIterList, &pIter); if (px == NULL) { - taosMemoryFree(pIter); - return terrno; + TSDB_CHECK_NULL(px, code, lino, _end, terrno); + taosMemoryFreeClear(pIter); } } } else if (size > numOfFileObj) { // remove unused LDataIter - int32_t inc = size - numOfFileObj; + inc = size - numOfFileObj; - for (int i = 0; i < inc; ++i) { - SLDataIter* pIter = taosArrayPop(pLDIterList); + for (int32_t i = 0; i < inc; ++i) { + pIter = taosArrayPop(pLDIterList); destroyLDataIter(pIter); } } - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) { - int32_t numOfLevels = pFileSet->lvlArr->size; - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t numOfLevels = 0; + SSttLvl* pSttLevel = NULL; + SArray* pList = NULL; + + TSDB_CHECK_NULL(pFileSet, code, lino, _end, TSDB_CODE_INVALID_PARA); + + numOfLevels = pFileSet->lvlArr->size; // add the list/iter placeholder while (taosArrayGetSize(pSttFileBlockIterArray) < numOfLevels) { - SArray* pList = taosArrayInit(4, POINTER_BYTES); - if (pList == NULL) { - return terrno; - } + pList = taosArrayInit(4, POINTER_BYTES); + TSDB_CHECK_NULL(pList, code, lino, _end, terrno); void* px = taosArrayPush(pSttFileBlockIterArray, &pList); if (px == NULL) { - return terrno; + taosArrayDestroy(pList); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); } } for (int32_t j = 0; j < numOfLevels; ++j) { - SSttLvl* pSttLevel = pFileSet->lvlArr->data[j]; - SArray* pList = taosArrayGetP(pSttFileBlockIterArray, j); + pSttLevel = pFileSet->lvlArr->data[j]; + pList = taosArrayGetP(pSttFileBlockIterArray, j); code = doAdjustValidDataIters(pList, TARRAY2_SIZE(pSttLevel->fobjArr)); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); } - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf, const char* pstr) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t numOfRows = 0; - int32_t code = 0; + int32_t numOfLevels = 0; + + TSDB_CHECK_NULL(pFileSet, code, lino, _end, TSDB_CODE_INVALID_PARA); // no data exists, go to end - int32_t numOfLevels = pFileSet->lvlArr->size; + numOfLevels = pFileSet->lvlArr->size; if (numOfLevels == 0) { - return numOfRows; + goto _end; } // add the list/iter placeholder code = adjustSttDataIters(pSttFileBlockIterArray, pFileSet); - if (code != TSDB_CODE_SUCCESS) { - return numOfRows; - } + TSDB_CHECK_CODE(code, lino, _end); for (int32_t j = 0; j < numOfLevels; ++j) { SSttLvl* pSttLevel = pFileSet->lvlArr->data[j]; @@ -1233,26 +1434,43 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra } } +_end: + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return numOfRows; } static bool overlapHelper(const STimeWindow* pLeft, TSKEY minKey, TSKEY maxKey) { - return (pLeft->ekey >= minKey) && (pLeft->skey <= maxKey); + return (pLeft != NULL) && (pLeft->ekey >= minKey) && (pLeft->skey <= maxKey); } static bool overlapWithTimeWindow(STimeWindow* p1, STimeWindow* pQueryWindow, STableBlockScanInfo* pBlockScanInfo, int32_t order) { + SIterInfo* pMemIter = NULL; + SIterInfo* pIMemIter = NULL; + STbData* pTbData = NULL; + STimeWindow* pFileWin = NULL; + + if (p1 == NULL || pQueryWindow == NULL) { + return false; + } + // overlap with query window if (!(p1->skey >= pQueryWindow->skey && p1->ekey <= pQueryWindow->ekey)) { return true; } - SIterInfo* pMemIter = &pBlockScanInfo->iter; - SIterInfo* pIMemIter = &pBlockScanInfo->iiter; + if (pBlockScanInfo == NULL) { + return false; + } + + pMemIter = &pBlockScanInfo->iter; + pIMemIter = &pBlockScanInfo->iiter; // overlap with mem data if (pMemIter->hasVal) { - STbData* pTbData = pMemIter->iter->pTbData; + pTbData = pMemIter->iter->pTbData; if (overlapHelper(p1, pTbData->minKey, pTbData->maxKey)) { return true; } @@ -1260,14 +1478,14 @@ static bool overlapWithTimeWindow(STimeWindow* p1, STimeWindow* pQueryWindow, ST // overlap with imem data if (pIMemIter->hasVal) { - STbData* pITbData = pIMemIter->iter->pTbData; - if (overlapHelper(p1, pITbData->minKey, pITbData->maxKey)) { + pTbData = pIMemIter->iter->pTbData; + if (overlapHelper(p1, pTbData->minKey, pTbData->maxKey)) { return true; } } // overlap with data file block - STimeWindow* pFileWin = &pBlockScanInfo->filesetWindow; + pFileWin = &pBlockScanInfo->filesetWindow; if ((taosArrayGetSize(pBlockScanInfo->pBlockIdxList) > 0) && overlapHelper(p1, pFileWin->skey, pFileWin->ekey)) { return true; } @@ -1290,20 +1508,24 @@ static int32_t sortUidComparFn(const void* p1, const void* p2) { } bool isCleanSttBlock(SArray* pKeyRangeList, STimeWindow* pQueryWindow, STableBlockScanInfo* pScanInfo, int32_t order) { - // check if it overlap with del skyline - taosArraySort(pKeyRangeList, sortUidComparFn); + int32_t num = 0; + SSttKeyRange* pRange = NULL; + STimeWindow w; - int32_t num = taosArrayGetSize(pKeyRangeList); + num = taosArrayGetSize(pKeyRangeList); if (num == 0) { return false; } - SSttKeyRange* pRange = taosArrayGet(pKeyRangeList, 0); + // check if it overlap with del skyline + taosArraySort(pKeyRangeList, sortUidComparFn); + + pRange = taosArrayGet(pKeyRangeList, 0); if (pRange == NULL) { return false; } - STimeWindow w = {.skey = pRange->skey.ts, .ekey = pRange->ekey.ts}; + w = (STimeWindow){.skey = pRange->skey.ts, .ekey = pRange->ekey.ts}; if (overlapWithTimeWindow(&w, pQueryWindow, pScanInfo, order)) { return false; } @@ -1319,8 +1541,8 @@ bool isCleanSttBlock(SArray* pKeyRangeList, STimeWindow* pQueryWindow, STableBlo return false; } - STimeWindow w2 = {.skey = p2->skey.ts, .ekey = p2->ekey.ts}; - bool overlap = overlapWithTimeWindow(&w2, pQueryWindow, pScanInfo, order); + w = (STimeWindow){.skey = p2->skey.ts, .ekey = p2->ekey.ts}; + bool overlap = overlapWithTimeWindow(&w, pQueryWindow, pScanInfo, order); if (overlap) { return false; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 6ec1f99577..703c548aca 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -30,13 +30,13 @@ extern "C" { do { \ (_w)->skey = INT64_MAX; \ (_w)->ekey = INT64_MIN; \ - } while (0); + } while (0) #define INIT_KEYRANGE(_k) \ do { \ (_k)->skey.ts = INT64_MAX; \ (_k)->ekey.ts = INT64_MIN; \ - } while (0); + } while (0) #define tRowGetKeyEx(_pRow, _pKey) \ { \ @@ -72,7 +72,6 @@ typedef struct STsdbReaderInfo { } STsdbReaderInfo; typedef struct SBlockInfoBuf { - int32_t currentIndex; SArray* pData; int32_t numPerBucket; int32_t numOfTables; @@ -241,7 +240,6 @@ typedef struct SDataBlockIter { int32_t index; SArray* blockList; // SArray int32_t order; - SDataBlk block; // current SDataBlk data } SDataBlockIter; typedef struct SFileBlockDumpInfo { @@ -321,7 +319,7 @@ int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, c int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap, STsdbReader* pReader); void clearBlockScanInfo(STableBlockScanInfo* p); -void destroyAllBlockScanInfo(SSHashObj* pTableMap); +void destroyAllBlockScanInfo(SSHashObj** pTableMap); void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step); void cleanupInfoForNextFileset(SSHashObj* pTableMap); int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables); @@ -335,7 +333,7 @@ void clearBrinBlockIter(SBrinRecordIter* pIter); // initialize block iterator API int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList); -bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr); +bool blockIteratorNext(SDataBlockIter* pBlockIter); // load tomb data API (stt/mem only for one table each, tomb data from data files are load for all tables at one time) int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 8aad415f70..5f199c8023 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -2792,7 +2792,9 @@ static int32_t doBlockInfoScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes code = pAPI->tsdReader.tsdReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); QUERY_CHECK_CODE(code, lino, _end); - blockDistInfo.numOfInmemRows = (int32_t)pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pBlockScanInfo->pHandle); + blockDistInfo.numOfInmemRows = 0; + code = pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pBlockScanInfo->pHandle, &blockDistInfo.numOfInmemRows); + QUERY_CHECK_CODE(code, lino, _end); SSDataBlock* pBlock = pBlockScanInfo->pResBlock;