From 228cb6f974b0f02d8b5fb5817df52c6b103c3a17 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 11 Apr 2020 14:47:37 +0800 Subject: [PATCH] [td-98] fix compData buffer overflow --- src/common/inc/name.h | 4 +-- src/query/inc/queryExecutor.h | 4 +-- src/query/src/queryExecutor.c | 54 ++++++++++++++++------------------- src/vnode/tsdb/inc/tsdb.h | 2 +- src/vnode/tsdb/src/tsdbRead.c | 38 +++++++++++++++--------- 5 files changed, 54 insertions(+), 48 deletions(-) diff --git a/src/common/inc/name.h b/src/common/inc/name.h index bafa003609..a629128324 100644 --- a/src/common/inc/name.h +++ b/src/common/inc/name.h @@ -14,10 +14,10 @@ typedef struct SDataStatis { int16_t numOfNull; } SDataStatis; -typedef struct SColumnInfoEx { +typedef struct SColumnInfoData { SColumnInfo info; void* pData; // the corresponding block data in memory -} SColumnInfoEx; +} SColumnInfoData; void extractTableName(const char *tableId, char *name); diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index dc5f2fcb78..5aab3abf40 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -89,7 +89,7 @@ typedef struct SColumnFilterElem { } SColumnFilterElem; typedef struct SSingleColumnFilterInfo { - SColumnInfoEx info; + SColumnInfoData info; int32_t numOfFilters; SColumnFilterElem* pFilters; void* pData; @@ -130,7 +130,7 @@ typedef struct SQuery { int32_t rowSize; SSqlGroupbyExpr* pGroupbyExpr; SSqlFunctionExpr* pSelectExpr; - SColumnInfoEx* colList; + SColumnInfoData* colList; int32_t numOfFilterCols; int64_t* defaultVal; TSKEY lastKey; diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 1ba74af2be..6e540f6ae1 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -866,7 +866,7 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 int32_t numOfCols = taosArrayGetSize(pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoEx *p = taosArrayGet(pDataBlock, i); + SColumnInfoData *p = taosArrayGet(pDataBlock, i); if (pCol->colId == p->info.colId) { dataBlock = p->pData; break; @@ -894,7 +894,7 @@ static void blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStati SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; SQuery * pQuery = pRuntimeEnv->pQuery; - SColumnInfoEx *pColInfo = NULL; + SColumnInfoData *pColInfo = NULL; TSKEY * primaryKeyCol = NULL; if (pDataBlock != NULL) { @@ -2223,7 +2223,7 @@ char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWi pQuery->pSelectExpr[columnIndex].resBytes * realRowId; } -int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { +int32_t UNUSED_FUNC vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { dTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, @@ -2292,11 +2292,12 @@ int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, true); - STsdbQueryCond cond = {0}; - cond.twindow = (STimeWindow){.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; - cond.order = pQuery->order.order; - - cond.colList = *pQuery->colList; + STsdbQueryCond cond = { + .twindow = (STimeWindow) {.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}, + .order = pQuery->order.order, + .colList = pQuery->colList, + }; + SArray *sa = taosArrayInit(1, POINTER_BYTES); // for(int32_t i = 0; i < pQInfo->pSidSet->numOfTables; ++i) { @@ -2699,7 +2700,6 @@ void vnodeSetTagValueInParam(tSidSet *pSidSet, SQueryRuntimeEnv *pRuntimeEnv, vo } // set the join tag for first column - SSqlFuncExprMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; if (pFuncMsg->functionId == TSDB_FUNC_TS && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && pRuntimeEnv->pTSBuf != NULL) { assert(pFuncMsg->numOfParams == 1); @@ -3472,8 +3472,8 @@ void setQueryStatus(SQuery *pQuery, int8_t status) { bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - bool toContinue = false; - + + bool toContinue = false; if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { // for each group result, call the finalize function for each column SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; @@ -3567,8 +3567,7 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { doSingleMeterSupplementScan(pRuntimeEnv); - // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query during - // supplementary scan + // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query during reverse scan pQuery->lastKey = lkey; pQuery->window.ekey = ekey; @@ -3577,7 +3576,7 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { // tsdbNextDataBlock(pRuntimeEnv->pQueryHandle); } -void doFinalizeResult(SQueryRuntimeEnv *pRuntimeEnv) { +void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { @@ -4170,7 +4169,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) STsdbQueryCond cond = { .twindow = pQuery->window, .order = pQuery->order.order, - .colList = *pQuery->colList, + .colList = pQuery->colList, }; SArray *cols = taosArrayInit(pQuery->numOfCols, sizeof(pQuery->colList[0])); @@ -4450,7 +4449,7 @@ static UNUSED_FUNC int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, i scanAllDataBlocks(pRuntimeEnv); // first/last_row query, do not invoke the finalize for super table query - doFinalizeResult(pRuntimeEnv); + finalizeQueryResult(pRuntimeEnv); int64_t numOfRes = getNumOfResult(pRuntimeEnv); assert(numOfRes == 1 || numOfRes == 0); @@ -4690,7 +4689,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { * Only the ts-comp query requires the finalizer function to be executed here. */ if (isTSCompQuery(pQuery)) { - doFinalizeResult(pRuntimeEnv); + finalizeQueryResult(pRuntimeEnv); } if (pRuntimeEnv->pTSBuf != NULL) { @@ -4896,12 +4895,12 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { * select count(*)/top(field,k)/avg(field name) from table_name [where ts>now-1a]; * select count(*) from table_name group by status_column; */ -static void tableFixedOutputProcessor(SQInfo *pQInfo) { +static void tableFixedOutputProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; scanAllDataBlocks(pRuntimeEnv); - doFinalizeResult(pRuntimeEnv); + finalizeQueryResult(pRuntimeEnv); if (isQueryKilled(pQInfo)) { return; @@ -4917,8 +4916,6 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { doSkipResults(pRuntimeEnv); doRevisedResultsByLimit(pQInfo); - - pQuery->rec.rows = pQuery->rec.rows; } static void tableMultiOutputProcess(SQInfo *pQInfo) { @@ -4932,7 +4929,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo) { while (1) { scanAllDataBlocks(pRuntimeEnv); - doFinalizeResult(pRuntimeEnv); + finalizeQueryResult(pRuntimeEnv); if (isQueryKilled(pQInfo)) { return; @@ -4983,7 +4980,7 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { } assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_NOT_COMPLETED)); - doFinalizeResult(pRuntimeEnv); + finalizeQueryResult(pRuntimeEnv); // here we can ignore the records in case of no interpolation // todo handle offset, in case of top/bottom interval query @@ -5004,7 +5001,7 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { } // handle time interval query on table -static void tableIntervalProcessor(SQInfo *pQInfo) { +static void tableIntervalProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &(pQInfo->runtimeEnv); SQuery * pQuery = pRuntimeEnv->pQuery; @@ -5076,8 +5073,6 @@ static void tableQueryImpl(SQInfo* pQInfo) { doRevisedResultsByLimit(pQInfo); pQInfo->pointsInterpo += numOfInterpo; - pQuery->rec.rows += pQuery->rec.rows; - dTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); sem_post(&pQInfo->dataReady); return; @@ -5119,15 +5114,14 @@ static void tableQueryImpl(SQInfo* pQInfo) { // group by normal column, sliding window query, interval query are handled by interval query processor if (isIntervalQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) - tableIntervalProcessor(pQInfo); + tableIntervalProcess(pQInfo); } else if (isFixedOutputQuery(pQuery)) { - tableFixedOutputProcessor(pQInfo); + tableFixedOutputProcess(pQInfo); } else { // diff/add/multiply/subtract/division assert(pQuery->checkBuffer == 1); tableMultiOutputProcess(pQInfo); } - // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); assert(taosArrayGetSize(pQInfo->pTableIdList) == 1); @@ -5593,7 +5587,7 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { if (pQuery->colList[i].info.numOfFilters > 0) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[j]; - memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfoEx)); + memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfoData)); pFilterInfo->info.info.filters = NULL; pFilterInfo->numOfFilters = pQuery->colList[i].info.numOfFilters; diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h index ff3d4666fc..4f86066755 100644 --- a/src/vnode/tsdb/inc/tsdb.h +++ b/src/vnode/tsdb/inc/tsdb.h @@ -188,7 +188,7 @@ typedef void* tsdb_query_handle_t; // Use void to hide implementation details typedef struct STsdbQueryCond { STimeWindow twindow; int32_t order; // desc/asc order to iterate the data block - SColumnInfoEx colList; + SColumnInfoData* colList; } STsdbQueryCond; typedef struct SBlockInfo { diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 30181bed1d..e5f0aed05e 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -25,7 +25,7 @@ #include "tsdbMain.h" #define EXTRA_BYTES 2 -#define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoEx*)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) +#define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoData*)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) #define QUERY_IS_ASC_QUERY(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) (taosArrayGetSize((handle)->pColumns)) @@ -65,7 +65,10 @@ typedef struct STableCheckInfo { int64_t offsetInHeaderFile; int32_t start; bool checkFirstFileBlock; + SCompInfo* pCompInfo; + int32_t compSize; + int32_t numOfBlocks; // number of qualified data blocks not the original blocks SDataCols* pDataCols; @@ -106,7 +109,7 @@ typedef struct STsdbQueryHandle { SCompBlock* pBlock; int32_t numOfBlocks; SField** pFields; - SArray* pColumns; // column list, SColumnInfoEx array list + SArray* pColumns; // column list, SColumnInfoData array list bool locateStart; int32_t realNumOfRows; bool loadDataAfterSeek; // load data after seek. @@ -127,7 +130,7 @@ typedef struct STsdbQueryHandle { int32_t doAllocateBuf(STsdbQueryHandle* pQueryHandle, int32_t rowsPerFileBlock) { // record the maximum column width among columns of this meter/metric - SColumnInfoEx* pColumn = taosArrayGet(pQueryHandle->pColumns, 0); + SColumnInfoData* pColumn = taosArrayGet(pQueryHandle->pColumns, 0); int32_t maxColWidth = pColumn->info.bytes; for (int32_t i = 1; i < QH_GET_NUM_OF_COLS(pQueryHandle); ++i) { @@ -177,7 +180,7 @@ tsdb_query_handle_t* tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond* pCond .lastKey = pQueryHandle->window.skey, .tableId = id, .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid), // todo this may be failed - .pCompInfo = calloc(1, 1024), + .pCompInfo = NULL, }; assert(info.pTableObj != NULL); @@ -193,10 +196,10 @@ tsdb_query_handle_t* tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond* pCond int32_t numOfCols = taosArrayGetSize(pColumnInfo); size_t bufferCapacity = 4096; - pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoEx)); + pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoEx* pCol = taosArrayGet(pColumnInfo, i); - SColumnInfoEx pDest = {{0}, 0}; + SColumnInfoData* pCol = taosArrayGet(pColumnInfo, i); + SColumnInfoData pDest = {{0}, 0}; pDest.pData = calloc(1, EXTRA_BYTES + bufferCapacity * pCol->info.bytes); pDest.info = pCol->info; @@ -261,6 +264,15 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file } else { + if (pCheckInfo->compSize < compIndex->len) { + assert(compIndex->len > 0); + + char* t = realloc(pCheckInfo->pCompInfo, compIndex->len); + assert(t != NULL); + + pCheckInfo->pCompInfo = (SCompInfo*) t; + } + tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo); int32_t index = 0; @@ -589,7 +601,7 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf int16_t colId = *(int16_t*)taosArrayGet(sa, i); for (int32_t j = 0; j < numOfCols; ++j) { - SColumnInfoEx* pCol = taosArrayGet(pQueryHandle->pColumns, j); + SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, j); if (pCol->info.colId == colId) { SDataCol* pDataCol = &pCols->cols[i]; @@ -610,7 +622,7 @@ static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoEx* pCol = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); taosArrayPush(pIdList, &pCol->info.colId); } @@ -1067,7 +1079,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max int32_t offset = 0; for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoEx* pColInfo = taosArrayGet(pHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pHandle->pColumns, i); memcpy(pColInfo->pData + numOfRows * pColInfo->info.bytes, dataRowTuple(row) + offset, pColInfo->info.bytes); offset += pColInfo->info.bytes; } @@ -1102,7 +1114,7 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) { /* not a whole disk block, only the qualified rows, so this block is loaded in to buffer during the * block next function */ - SColumnInfoEx* pColInfoEx = taosArrayGet(pHandle->pColumns, 0); + SColumnInfoData* pColInfoEx = taosArrayGet(pHandle->pColumns, 0); rows = pHandle->realNumOfRows; skey = *(TSKEY*)pColInfoEx->pData; @@ -1141,7 +1153,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t* pQueryHandle, SData SArray* tsdbRetrieveDataBlock(tsdb_query_handle_t* pQueryHandle, SArray* pIdList) { /** - * In the following two cases, the data has been loaded to SColumnInfoEx. + * In the following two cases, the data has been loaded to SColumnInfoData. * 1. data is from cache, 2. data block is not completed qualified to query time range */ STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; @@ -1484,7 +1496,7 @@ void tsdbCleanupQueryHandle(tsdb_query_handle_t queryHandle) { size_t cols = taosArrayGetSize(pQueryHandle->pColumns); for (int32_t i = 0; i < cols; ++i) { - SColumnInfoEx* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); tfree(pColInfo->pData); }