From 658cb393a775207824a663fc87415442252620f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 12:11:20 +0800 Subject: [PATCH 1/6] enh(query): improve the performance and add some todo --- source/libs/executor/src/executil.c | 1 + source/libs/scalar/src/scalar.c | 5 +---- source/util/src/tpagedbuf.c | 12 ++++++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1241bdf5a4..b9681a4fde 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -129,6 +129,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in void* pData = NULL; pGroupResInfo->pRows = taosArrayInit(10, POINTER_BYTES); + // todo avoid repeated malloc memory size_t keyLen = 0; int32_t iter = 0; while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 55f33b7a3e..b60b651ebc 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -165,10 +165,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - colDataDestroy(out.columnData); - taosMemoryFreeClear(out.columnData); - out.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData)); - + colInfoDataCleanup(out.columnData, out.numOfRows); cell = cell->pNext; } diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 0f4c51060b..ad08b15c25 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -30,6 +30,7 @@ struct SDiskbasedBuf { TdFilePtr pFile; int32_t allocateId; // allocated page id char* path; // file path + char* prefix; // file name prefix int32_t pageSize; // current used page size int32_t inMemPages; // numOfPages that are allocated in memory SList* freePgList; // free page list @@ -48,6 +49,12 @@ struct SDiskbasedBuf { }; static int32_t createDiskFile(SDiskbasedBuf* pBuf) { + if (pBuf->path[0] == '\0') { // prepare the file name when needed it + char path[PATH_MAX] = {0}; + taosGetTmpfilePath(pBuf->prefix, "paged-buf", path); + pBuf->path = taosMemoryStrDup(path); + } + pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); if (pBuf->pFile == NULL) { @@ -356,10 +363,7 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem pPBuf->assistBuf = taosMemoryMalloc(pPBuf->pageSize + 2); // EXTRA BYTES pPBuf->all = taosHashInit(10, fn, true, false); - - char path[PATH_MAX] = {0}; - taosGetTmpfilePath(dir, "paged-buf", path); - pPBuf->path = strdup(path); + pPBuf->prefix = (char*) dir; pPBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); From b6199d8d99f9c527d027af8ffe1552e913e762ed Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 12:37:42 +0800 Subject: [PATCH 2/6] enh(query): update the aligned bits for colInfodata --- source/common/src/tdatablock.c | 2 +- source/os/src/osMemory.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c53e771f42..c253e842d6 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -19,7 +19,7 @@ #include "tlog.h" #include "tname.h" -#define MALLOC_ALIGN_BYTES 32 +#define MALLOC_ALIGN_BYTES 256 int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { ASSERT(pColumnInfoData != NULL); diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 2f30e8977a..230ca5b968 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -351,7 +351,8 @@ void* taosMemoryMallocAlign(uint32_t alignment, int64_t size) { ASSERT(0); #else #if defined(LINUX) - return memalign(alignment, size); + void* p = memalign(alignment, size); + return p; #else return taosMemoryMalloc(size); #endif From 85c1e3aabf9a8e6c04eff24ea4bcaa5a1410f0f9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 12:41:56 +0800 Subject: [PATCH 3/6] refactor: add an assert. --- source/os/src/osMemory.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 230ca5b968..b2d1dcf58e 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -352,6 +352,8 @@ void* taosMemoryMallocAlign(uint32_t alignment, int64_t size) { #else #if defined(LINUX) void* p = memalign(alignment, size); + ASSERT((((uint64_t)p) & (alignment - 1)) == 0x0); + return p; #else return taosMemoryMalloc(size); From fa43f86fe1af85ccc050c3ecb6232c548ef2a7fb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 12:43:03 +0800 Subject: [PATCH 4/6] refactor: add an assert. --- source/common/src/tdatablock.c | 3 ++- source/os/src/osMemory.c | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c253e842d6..2832e6d16e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1191,7 +1191,6 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* int32_t oldLen = BitmapLen(existedRows); pColumn->nullbitmap = tmp; memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen); - ASSERT(pColumn->info.bytes); // make sure the allocated memory is MALLOC_ALIGN_BYTES aligned @@ -1207,6 +1206,8 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* } pColumn->pData = tmp; + ASSERT((((uint64_t)pColumn->pData) & (MALLOC_ALIGN_BYTES - 1)) == 0x0); + if (clearPayload) { memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows)); } diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index b2d1dcf58e..230ca5b968 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -352,8 +352,6 @@ void* taosMemoryMallocAlign(uint32_t alignment, int64_t size) { #else #if defined(LINUX) void* p = memalign(alignment, size); - ASSERT((((uint64_t)p) & (alignment - 1)) == 0x0); - return p; #else return taosMemoryMalloc(size); From a3200b555cc992bcd06a7d45f93e89a0d0201d72 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 17:03:31 +0800 Subject: [PATCH 5/6] fix(query): fix null ptr access. --- source/util/src/tpagedbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index ad08b15c25..37874ae250 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -49,7 +49,7 @@ struct SDiskbasedBuf { }; static int32_t createDiskFile(SDiskbasedBuf* pBuf) { - if (pBuf->path[0] == '\0') { // prepare the file name when needed it + if (pBuf->path == NULL) { // prepare the file name when needed it char path[PATH_MAX] = {0}; taosGetTmpfilePath(pBuf->prefix, "paged-buf", path); pBuf->path = taosMemoryStrDup(path); From 392fe988a0f4de1726475be1568242d28864d8ba Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 20:16:56 +0800 Subject: [PATCH 6/6] fix(query): remove invalid assert for mac&win. --- source/common/src/tdatablock.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 2832e6d16e..dfd0b68039 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1206,7 +1206,11 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* } pColumn->pData = tmp; + + // todo remove it soon +#if defined LINUX ASSERT((((uint64_t)pColumn->pData) & (MALLOC_ALIGN_BYTES - 1)) == 0x0); +#endif if (clearPayload) { memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows));