From 4a5d765023f298325213cf931e481a708701dcf1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Feb 2022 17:13:39 +0800 Subject: [PATCH] [td-11818] Opt. --- include/common/tep.h | 10 +- source/common/src/tep.c | 108 +++++++++++++++----- source/libs/executor/test/executorTests.cpp | 52 +++++----- 3 files changed, 118 insertions(+), 52 deletions(-) diff --git a/include/common/tep.h b/include/common/tep.h index c7c4b15ac0..58ae6038e8 100644 --- a/include/common/tep.h +++ b/include/common/tep.h @@ -15,8 +15,9 @@ typedef struct SCorEpSet { } SCorEpSet; typedef struct SBlockOrderInfo { - int32_t order; - int32_t colIndex; + int32_t order; + int32_t colIndex; + SColumnInfoData *pColData; } SBlockOrderInfo; int taosGetFqdnPortFromEp(const char *ep, SEp *pEp); @@ -32,7 +33,10 @@ SEpSet getEpSet_s(SCorEpSet *pEpSet); #define BMCharPos(bm_, r_) ((bm_)[(r_) >> NBIT]) #define colDataIsNull_f(bm_, r_) ((BMCharPos(bm_, r_) & (1u << (7u - BitPos(r_)))) == (1u << (7u - BitPos(r_)))) -void colDataSetNull_f(char* bitmap, uint32_t row); +#define colDataSetNull_f(bm_, r_) \ + do { \ + BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ + } while (0) static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, SColumnDataAgg* pColAgg) { if (!pColumnInfoData->hasNull) { diff --git a/source/common/src/tep.c b/source/common/src/tep.c index 3522883528..bc0484a636 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tep.c @@ -62,31 +62,6 @@ SEpSet getEpSet_s(SCorEpSet *pEpSet) { #define BitmapLen(_n) (((_n) + ((1<> NBIT) -void colDataSetNull_f(char* bitmap, uint32_t row) { - bitmap[row>>3u] |= (1u << (7u - BitPos(row))); -} - -static int32_t ensureBitmapSize(SColumnInfoData* pColumnInfoData, uint32_t size) { -#if 0 - ASSERT(pColumnInfoData != NULL); - if (pColumnInfoData->bitmapLen * 8 < size) { - int32_t inc = pColumnInfoData->bitmapLen * 1.25; - if (inc < 8) { - inc = 8; - } - - char* tmp = realloc(pColumnInfoData->nullbitmap, inc + pColumnInfoData->bitmapLen); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pColumnInfoData->nullbitmap = tmp; - memset(pColumnInfoData->nullbitmap + pColumnInfoData->bitmapLen, 0, inc); - } -#endif - return TSDB_CODE_SUCCESS; -} - int32_t colDataGetSize(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { ASSERT(pColumnInfoData != NULL); if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { @@ -553,7 +528,7 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { for(int32_t i = 0; i < pInfo->size; ++i) { SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); - SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); + SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); if (pColInfoData->hasNull) { bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); @@ -623,13 +598,85 @@ static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, co } static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, int32_t* index) { +#if 0 for (int32_t i = 0; i < pDataBlock->info.rows; ++i) { int32_t code = doAssignOneTuple(pCols, i, pDataBlock, index[i]); if (code != TSDB_CODE_SUCCESS) { return code; } } +#else + for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* pDst = &pCols[i]; + SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); + if (IS_VAR_DATA_TYPE(pSrc->info.type)) { + memcpy(pDst->pData, pSrc->pData, pSrc->varmeta.length); + pDst->varmeta.length = pSrc->varmeta.length; + + for(int32_t j = 0; j < pDataBlock->info.rows; ++j) { + pDst->varmeta.offset[j] = pSrc->varmeta.offset[index[j]]; + } + } else { + switch (pSrc->info.type) { + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_INT: { + for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { + int32_t* p = (int32_t*)pDst->pData; + int32_t* srclist = (int32_t*)pSrc->pData; + + p[j] = srclist[index[j]]; + if (colDataIsNull_f(pSrc->nullbitmap, index[j])) { + colDataSetNull_f(pDst->nullbitmap, j); + } + } + break; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_TINYINT: { + for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { + int32_t* p = (int32_t*)pDst->pData; + int32_t* srclist = (int32_t*)pSrc->pData; + + p[j] = srclist[index[j]]; + if (colDataIsNull_f(pSrc->nullbitmap, index[j])) { + colDataSetNull_f(pDst->nullbitmap, j); + } + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_SMALLINT: { + for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { + int32_t* p = (int32_t*)pDst->pData; + int32_t* srclist = (int32_t*)pSrc->pData; + + p[j] = srclist[index[j]]; + if (colDataIsNull_f(pSrc->nullbitmap, index[j])) { + colDataSetNull_f(pDst->nullbitmap, j); + } + } + break; + } + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_BIGINT: { + for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { + int32_t* p = (int32_t*)pDst->pData; + int32_t* srclist = (int32_t*)pSrc->pData; + + p[j] = srclist[index[j]]; + if (colDataIsNull_f(pSrc->nullbitmap, index[j])) { + colDataSetNull_f(pDst->nullbitmap, j); + } + } + break; + } + default: + assert(0); + } + } + } +#endif return TSDB_CODE_SUCCESS; } @@ -648,6 +695,10 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { if (IS_VAR_DATA_TYPE(pCols[i].info.type)) { pCols[i].varmeta.offset = calloc(rows, sizeof(int32_t)); + pCols[i].pData = calloc(1, pColInfoData->varmeta.length); + + pCols[i].varmeta.length = pColInfoData->varmeta.length; + pCols[i].varmeta.allocLen = pCols[i].varmeta.length; } else { pCols[i].nullbitmap = calloc(1, BitmapLen(rows)); pCols[i].pData = calloc(rows, pCols[i].info.bytes); @@ -713,6 +764,11 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs int64_t p0 = taosGetTimestampUs(); SSDataBlockSortHelper helper = {.nullFirst = nullFirst, .pDataBlock = pDataBlock, .orderInfo = pOrderInfo}; + for(int32_t i = 0; i < taosArrayGetSize(helper.orderInfo); ++i) { + struct SBlockOrderInfo* pInfo = taosArrayGet(helper.orderInfo, i); + pInfo->pColData = taosArrayGet(pDataBlock->pDataBlock, pInfo->colIndex); + } + taosqsort(index, rows, sizeof(int32_t), &helper, dataBlockCompar); int64_t p1 = taosGetTimestampUs(); diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 18a073d825..c154bb0fbb 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -39,6 +39,7 @@ typedef struct SDummyInputInfo { int32_t max; int32_t current; int32_t startVal; + SSDataBlock* pBlock; } SDummyInputInfo; SSDataBlock* getDummyBlock(void* param, bool* newgroup) { @@ -48,32 +49,37 @@ SSDataBlock* getDummyBlock(void* param, bool* newgroup) { return NULL; } - SSDataBlock* pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); - assert(pBlock != NULL); - - pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); - int32_t numOfRows = 1000; - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfo.pData = static_cast(calloc(numOfRows, sizeof(int32_t))); - colInfo.nullbitmap = static_cast(calloc(1, (numOfRows + 7) / 8)); + if (pInfo->pBlock == NULL) { + pInfo->pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); - taosArrayPush(pBlock->pDataBlock, &colInfo); + pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); - SColumnInfoData colInfo1 = {0}; - colInfo1.info.type = TSDB_DATA_TYPE_BINARY; - colInfo1.info.bytes = 40; - colInfo1.info.colId = 2; + SColumnInfoData colInfo = {0}; + colInfo.info.type = TSDB_DATA_TYPE_INT; + colInfo.info.bytes = sizeof(int32_t); + colInfo.info.colId = 1; + colInfo.pData = static_cast(calloc(numOfRows, sizeof(int32_t))); + colInfo.nullbitmap = static_cast(calloc(1, (numOfRows + 7) / 8)); - colInfo1.varmeta.allocLen = 0;//numOfRows * sizeof(int32_t); - colInfo1.varmeta.length = 0; - colInfo1.varmeta.offset = static_cast(calloc(1, numOfRows * sizeof(int32_t))); + taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); - taosArrayPush(pBlock->pDataBlock, &colInfo1); + SColumnInfoData colInfo1 = {0}; + colInfo1.info.type = TSDB_DATA_TYPE_BINARY; + colInfo1.info.bytes = 40; + colInfo1.info.colId = 2; + + colInfo1.varmeta.allocLen = 0;//numOfRows * sizeof(int32_t); + colInfo1.varmeta.length = 0; + colInfo1.varmeta.offset = static_cast(calloc(1, numOfRows * sizeof(int32_t))); + + taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); + } else { + blockDataClearup(pInfo->pBlock, true); + } + + SSDataBlock* pBlock = pInfo->pBlock; char buf[128] = {0}; char b1[128] = {0}; @@ -104,7 +110,7 @@ SOperatorInfo* createDummyOperator(int32_t numOfBlocks) { SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo)); pInfo->max = numOfBlocks; - pInfo->startVal = 100000; + pInfo->startVal = 5000000; pOperator->info = pInfo; return pOperator; @@ -241,7 +247,8 @@ TEST(testCase, build_executor_tree_Test) { TEST(testCase, external_sort_Test) { SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); - SOrder o = {.order = TSDB_ORDER_ASC}; + SOrder o = {0}; + o.order = TSDB_ORDER_ASC; o.col.info.colId = 1; o.col.info.type = TSDB_DATA_TYPE_INT; taosArrayPush(pOrderVal, &o); @@ -289,7 +296,6 @@ TEST(testCase, external_sort_Test) { int64_t s2 = taosGetTimestampUs(); printf("total:%ld\n", s2 - s1); - pOperator->cleanupFn(pOperator->info, 2); tfree(exp); tfree(exp1);