From a935e645f12c142bb9924e50fdf7ae32718ee3d2 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 19 Dec 2019 13:23:14 +0800 Subject: [PATCH 1/4] fix bug in issue #946 [tbase-1363] --- src/system/detail/src/vnodeQueryImpl.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/system/detail/src/vnodeQueryImpl.c b/src/system/detail/src/vnodeQueryImpl.c index a18dfe4705..faf91c8117 100644 --- a/src/system/detail/src/vnodeQueryImpl.c +++ b/src/system/detail/src/vnodeQueryImpl.c @@ -1411,11 +1411,12 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t * * first filter the data block according to the value filter condition, then, if the top/bottom query applied, * invoke the filter function to decide if the data block need to be accessed or not. + * TODO handle the whole data block is NULL situation * @param pQuery * @param pField * @return */ -static bool needToLoadDataBlock(SQuery *pQuery, SField *pField, SQLFunctionCtx *pCtx) { +static bool needToLoadDataBlock(SQuery *pQuery, SField *pField, SQLFunctionCtx *pCtx, int32_t numOfTotalPoints) { if (pField == NULL) { return false; // no need to load data } @@ -1433,6 +1434,11 @@ static bool needToLoadDataBlock(SQuery *pQuery, SField *pField, SQLFunctionCtx * if (!vnodeSupportPrefilter(pFilterInfo->info.data.type)) { continue; } + + // all points in current column are NULL, no need to check its boundary value + if (pField[colIndex].numOfNullPoints == numOfTotalPoints) { + continue; + } if (pFilterInfo->info.data.type == TSDB_DATA_TYPE_FLOAT) { float minval = *(double *)(&pField[colIndex].min); @@ -1968,6 +1974,7 @@ int32_t getNextDataFileCompInfo(SQueryRuntimeEnv *pRuntimeEnv, SMeterObj *pMeter break; } + // failed to mmap header file into memory will cause the retrieval of compblock info failed if (vnodeGetCompBlockInfo(pMeterObj, pRuntimeEnv, fid) > 0) { break; @@ -6724,7 +6731,7 @@ int32_t LoadDatablockOnDemand(SCompBlock *pBlock, SField **pFields, uint8_t *blk * filter the data block according to the value filter condition. * no need to load the data block, continue for next block */ - if (!needToLoadDataBlock(pQuery, *pFields, pRuntimeEnv->pCtx)) { + if (!needToLoadDataBlock(pQuery, *pFields, pRuntimeEnv->pCtx, pBlock->numOfPoints)) { #if defined(_DEBUG_VIEW) dTrace("QInfo:%p fileId:%d, slot:%d, block discarded by per-filter, ", GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->slot); From 24d93f572982a88e411fc021bd8d5cfcdf6cd735 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 19 Dec 2019 13:31:31 +0800 Subject: [PATCH 2/4] opt performance by removing the mmap during read header file --- src/system/detail/src/vnodeQueryImpl.c | 39 ++++++++++++++++++-------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/system/detail/src/vnodeQueryImpl.c b/src/system/detail/src/vnodeQueryImpl.c index faf91c8117..c7e044eb6c 100644 --- a/src/system/detail/src/vnodeQueryImpl.c +++ b/src/system/detail/src/vnodeQueryImpl.c @@ -431,7 +431,7 @@ char *vnodeGetHeaderFileData(SQueryRuntimeEnv *pRuntimeEnv, int32_t vnodeId, int if (pVnodeFileInfo->current != fileIndex || pVnodeFileInfo->pHeaderFileData == NULL) { if (pVnodeFileInfo->current >= 0) { - assert(pVnodeFileInfo->pHeaderFileData != NULL); +// assert(pVnodeFileInfo->pHeaderFileData != NULL); } // do close the current memory mapped header file and corresponding fd @@ -487,16 +487,17 @@ static int vnodeGetCompBlockInfo(SMeterObj *pMeterObj, SQueryRuntimeEnv *pRuntim char *data = calloc(1, tmsize + TSDB_FILE_HEADER_LEN); read(fd, data, tmsize + TSDB_FILE_HEADER_LEN); #endif + + int64_t offset = TSDB_FILE_HEADER_LEN + sizeof(SCompHeader) * pMeterObj->sid; +#if 0 // check the offset value integrity if (validateHeaderOffsetSegment(pQInfo, pRuntimeEnv->vnodeFileInfo.headerFilePath, pMeterObj->vnode, data, getCompHeaderSegSize(pCfg)) < 0) { return -1; } - int64_t offset = TSDB_FILE_HEADER_LEN + sizeof(SCompHeader) * pMeterObj->sid; SCompHeader *compHeader = (SCompHeader *)(data + offset); - // no data in this file for specified meter, abort if (compHeader->compInfoOffset == 0) { return 0; @@ -507,14 +508,29 @@ static int vnodeGetCompBlockInfo(SMeterObj *pMeterObj, SQueryRuntimeEnv *pRuntim getCompHeaderStartPosition(pCfg)) < 0) { return -1; } +#else + char* buf = calloc(1, getCompHeaderSegSize(pCfg)); + SQueryFilesInfo *pVnodeFileInfo = &pRuntimeEnv->vnodeFileInfo; + + lseek(pVnodeFileInfo->headerFd, TSDB_FILE_HEADER_LEN, SEEK_SET); + read(pVnodeFileInfo->headerFd, buf, getCompHeaderSegSize(pCfg)); + + // check the offset value integrity + if (validateHeaderOffsetSegment(pQInfo, pRuntimeEnv->vnodeFileInfo.headerFilePath, pMeterObj->vnode, buf - TSDB_FILE_HEADER_LEN, + getCompHeaderSegSize(pCfg)) < 0) { + return -1; + } +#endif -#if 1 +#if 0 SCompInfo *compInfo = (SCompInfo *)(data + compHeader->compInfoOffset); #else - lseek(fd, compHeader->compInfoOffset, SEEK_SET); + SCompHeader *compHeader = (SCompHeader *)(buf + sizeof(SCompHeader) * pMeterObj->sid); + lseek(pVnodeFileInfo->headerFd, compHeader->compInfoOffset, SEEK_SET); + SCompInfo CompInfo = {0}; SCompInfo *compInfo = &CompInfo; - read(fd, compInfo, sizeof(SCompInfo)); + read(pVnodeFileInfo->headerFd, compInfo, sizeof(SCompInfo)); #endif // check compblock info integrity @@ -542,19 +558,19 @@ static int vnodeGetCompBlockInfo(SMeterObj *pMeterObj, SQueryRuntimeEnv *pRuntim memset(pQuery->pBlock, 0, (size_t)pQuery->blockBufferSize); -#if 1 +#if 0 memcpy(pQuery->pBlock, (char *)compInfo + sizeof(SCompInfo), (size_t)compBlockSize); TSCKSUM checksum = *(TSCKSUM *)((char *)compInfo + sizeof(SCompInfo) + compBlockSize); #else TSCKSUM checksum; - read(fd, pQuery->pBlock, compBlockSize); - read(fd, &checksum, sizeof(TSCKSUM)); + read(pVnodeFileInfo->headerFd, pQuery->pBlock, compBlockSize); + read(pVnodeFileInfo->headerFd, &checksum, sizeof(TSCKSUM)); #endif // check comp block integrity if (validateCompBlockSegment(pQInfo, pRuntimeEnv->vnodeFileInfo.headerFilePath, compInfo, (char *)pQuery->pBlock, pMeterObj->vnode, checksum) < 0) { - return -1; + return -1; //TODO free resource in error process } pQuery->pFields = (SField **)((char *)pQuery->pBlock + compBlockSize); @@ -567,7 +583,8 @@ static int vnodeGetCompBlockInfo(SMeterObj *pMeterObj, SQueryRuntimeEnv *pRuntim pSummary->totalCompInfoSize += compBlockSize; pSummary->loadCompInfoUs += (et - st); - + + free(buf); return pQuery->numOfBlocks; } From 897c23bd9a4abbf52a43c0d5009054f452056653 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 19 Dec 2019 13:33:12 +0800 Subject: [PATCH 3/4] opt performance by removing the mmap operation --- src/system/detail/src/vnodeQueryImpl.c | 38 +++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/system/detail/src/vnodeQueryImpl.c b/src/system/detail/src/vnodeQueryImpl.c index c7e044eb6c..a34a7bd06a 100644 --- a/src/system/detail/src/vnodeQueryImpl.c +++ b/src/system/detail/src/vnodeQueryImpl.c @@ -372,24 +372,24 @@ static int32_t doOpenQueryFileData(SQInfo *pQInfo, SQueryFilesInfo *pVnodeFileIn return -1; } - pVnodeFileInfo->pHeaderFileData = - mmap(NULL, pVnodeFileInfo->headFileSize, PROT_READ, MAP_SHARED, pVnodeFileInfo->headerFd, 0); - - if (pVnodeFileInfo->pHeaderFileData == MAP_FAILED) { - pVnodeFileInfo->pHeaderFileData = NULL; - - doCloseQueryFileInfoFD(pVnodeFileInfo); - doInitQueryFileInfoFD(pVnodeFileInfo); - - dError("QInfo:%p failed to mmap header file:%s, size:%lld, %s", pQInfo, pVnodeFileInfo->headerFilePath, - pVnodeFileInfo->headFileSize, strerror(errno)); - - return -1; - } else { - if (madvise(pVnodeFileInfo->pHeaderFileData, pVnodeFileInfo->headFileSize, MADV_SEQUENTIAL) == -1) { - dError("QInfo:%p failed to advise kernel the usage of header file, reason:%s", pQInfo, strerror(errno)); - } - } +// pVnodeFileInfo->pHeaderFileData = +// mmap(NULL, pVnodeFileInfo->headFileSize, PROT_READ, MAP_SHARED, pVnodeFileInfo->headerFd, 0); +// +// if (pVnodeFileInfo->pHeaderFileData == MAP_FAILED) { +// pVnodeFileInfo->pHeaderFileData = NULL; +// +// doCloseQueryFileInfoFD(pVnodeFileInfo); +// doInitQueryFileInfoFD(pVnodeFileInfo); +// +// dError("QInfo:%p failed to mmap header file:%s, size:%lld, %s", pQInfo, pVnodeFileInfo->headerFilePath, +// pVnodeFileInfo->headFileSize, strerror(errno)); +// +// return -1; +// } else { +// if (madvise(pVnodeFileInfo->pHeaderFileData, pVnodeFileInfo->headFileSize, MADV_SEQUENTIAL) == -1) { +// dError("QInfo:%p failed to advise kernel the usage of header file, reason:%s", pQInfo, strerror(errno)); +// } +// } return TSDB_CODE_SUCCESS; } @@ -450,7 +450,7 @@ char *vnodeGetHeaderFileData(SQueryRuntimeEnv *pRuntimeEnv, int32_t vnodeId, int } } - return pVnodeFileInfo->pHeaderFileData; + return 1;//pVnodeFileInfo->pHeaderFileData; } /* From 4c42be31b6288973874478dee99ca94444888ffd Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 19 Dec 2019 13:37:36 +0800 Subject: [PATCH 4/4] refactor code referred in issue #938 --- src/os/linux/src/tlinux.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/os/linux/src/tlinux.c b/src/os/linux/src/tlinux.c index 83782f4944..a23919e458 100644 --- a/src/os/linux/src/tlinux.c +++ b/src/os/linux/src/tlinux.c @@ -264,7 +264,6 @@ void *taosProcessAlarmSignal(void *tharg) { callback(0); } - assert(0); return NULL; }