From ee719d02b48d52a3d8b302eb41d884804ce5a17c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 30 Dec 2022 14:31:56 +0800 Subject: [PATCH 1/6] fix: insert into select with disorder column issue --- source/libs/executor/src/dataInserter.c | 12 +++++++++--- tests/script/tsim/insert/insert_select.sim | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 346fcc9729..376f4603f6 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -41,6 +41,7 @@ typedef struct SDataInserterHandle { SHashObj* pCols; int32_t status; bool queryEnd; + bool fullOrderColList; uint64_t useconds; uint64_t cachedSize; TdThreadMutex mutex; @@ -125,7 +126,6 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) { int64_t uid = pInserter->pNode->tableId; int64_t suid = pInserter->pNode->stableId; int32_t vgId = pInserter->pNode->vgId; - bool fullCol = (pInserter->pNode->pCols->length == pTSchema->numOfCols); SSubmitReq* ret = NULL; int32_t sz = taosArrayGetSize(pBlocks); @@ -176,7 +176,7 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) { const STColumn* pColumn = &pTSchema->columns[k]; SColumnInfoData* pColData = NULL; int16_t colIdx = k; - if (!fullCol) { + if (!pInserter->fullOrderColList) { int16_t* slotId = taosHashGet(pInserter->pCols, &pColumn->colId, sizeof(pColumn->colId)); if (NULL == slotId) { continue; @@ -212,7 +212,7 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); } } - if (!fullCol) { + if (!pInserter->fullOrderColList) { rb.hasNone = true; } tdSRowEnd(&rb); @@ -346,12 +346,18 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat return TSDB_CODE_OUT_OF_MEMORY; } + inserter->fullOrderColList = pInserterNode->pCols->length == inserter->pSchema->numOfCols; + inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); SNode* pNode = NULL; + int32_t i = 0; FOREACH(pNode, pInserterNode->pCols) { SColumnNode* pCol = (SColumnNode*)pNode; taosHashPut(inserter->pCols, &pCol->colId, sizeof(pCol->colId), &pCol->slotId, sizeof(pCol->slotId)); + if (inserter->fullOrderColList && pCol->colId != inserter->pSchema->columns[i].colId) { + inserter->fullOrderColList = false; + } } tsem_init(&inserter->ready, 0, 0); diff --git a/tests/script/tsim/insert/insert_select.sim b/tests/script/tsim/insert/insert_select.sim index e3374ee277..333964b1d6 100644 --- a/tests/script/tsim/insert/insert_select.sim +++ b/tests/script/tsim/insert/insert_select.sim @@ -42,4 +42,24 @@ if $rows != 2 then return -1 endi +print ======== step2 +sql drop database if exists db1; +sql create database db1 vgroups 1; +sql use db1; +sql create table t1(ts timestamp, a int, b int ); +sql create table t2(ts timestamp, a int, b int ); +sql insert into t1 values(1648791211000,1,2); +sql insert into t2 (ts, b, a) select ts, a, b from t1; +sql select * from t2; +if $rows != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != 1 then + return -1 +endi + + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 79227057e40044e589c32562f3f918f4d19dfcc7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Dec 2022 14:43:26 +0800 Subject: [PATCH 2/6] enh(tsdb): opt decompress int perf --- source/util/src/tcompression.c | 86 ++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 18305e594b..1a7002cfa1 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -273,6 +273,7 @@ int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, cha char bit = bit_per_integer[(int32_t)selector]; // bit = 3 int32_t elems = selector_to_elems[(int32_t)selector]; +#if 0 for (int32_t i = 0; i < elems; i++) { uint64_t zigzag_value; @@ -309,6 +310,91 @@ int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, cha count++; if (count == nelements) break; } +#endif + + int32_t v = 0; + uint64_t zigzag_value; + + switch (type) { + case TSDB_DATA_TYPE_BIGINT: { + for (int32_t i = 0; i < elems; i++) { + if (selector == 0 || selector == 1) { + zigzag_value = 0; + } else { + zigzag_value = ((w >> (4 + v)) & INT64MASK(bit)); + } + + int64_t diff = ZIGZAG_DECODE(int64_t, zigzag_value); + int64_t curr_value = diff + prev_value; + prev_value = curr_value; + + *((int64_t *)output + _pos) = (int64_t)curr_value; + _pos++; + + v += bit; + if ((++count) == nelements) break; + } + } break; + case TSDB_DATA_TYPE_INT: { + for (int32_t i = 0; i < elems; i++) { + if (selector == 0 || selector == 1) { + zigzag_value = 0; + } else { + zigzag_value = ((w >> (4 + v)) & INT64MASK(bit)); + } + + int64_t diff = ZIGZAG_DECODE(int64_t, zigzag_value); + int64_t curr_value = diff + prev_value; + prev_value = curr_value; + + *((int32_t *)output + _pos) = (int32_t)curr_value; + _pos++; + + v += bit; + if ((++count) == nelements) break; + } + } break; + case TSDB_DATA_TYPE_SMALLINT: { + for (int32_t i = 0; i < elems; i++) { + if (selector == 0 || selector == 1) { + zigzag_value = 0; + } else { + zigzag_value = ((w >> (4 + v)) & INT64MASK(bit)); + } + + int64_t diff = ZIGZAG_DECODE(int64_t, zigzag_value); + int64_t curr_value = diff + prev_value; + prev_value = curr_value; + + *((int16_t *)output + _pos) = (int16_t)curr_value; + _pos++; + + v += bit; + if ((++count) == nelements) break; + } + } break; + + case TSDB_DATA_TYPE_TINYINT: { + for (int32_t i = 0; i < elems; i++) { + if (selector == 0 || selector == 1) { + zigzag_value = 0; + } else { + zigzag_value = ((w >> (4 + v)) & INT64MASK(bit)); + } + + int64_t diff = ZIGZAG_DECODE(int64_t, zigzag_value); + int64_t curr_value = diff + prev_value; + prev_value = curr_value; + + *((int8_t *)output + _pos) = (int8_t)curr_value; + _pos++; + + v += bit; + if ((++count) == nelements) break; + } + } break; + } + ip += LONG_BYTES; } From dbc3dd97bb1aa19e3d1251ae9c10446082cb1505 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Dec 2022 17:42:20 +0800 Subject: [PATCH 3/6] rerfactor: remove unused code. --- source/util/src/tcompression.c | 42 ++-------------------------------- 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 1a7002cfa1..7cf4a7f510 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -273,45 +273,7 @@ int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, cha char bit = bit_per_integer[(int32_t)selector]; // bit = 3 int32_t elems = selector_to_elems[(int32_t)selector]; -#if 0 - for (int32_t i = 0; i < elems; i++) { - uint64_t zigzag_value; - - if (selector == 0 || selector == 1) { - zigzag_value = 0; - } else { - zigzag_value = ((w >> (4 + bit * i)) & INT64MASK(bit)); - } - int64_t diff = ZIGZAG_DECODE(int64_t, zigzag_value); - int64_t curr_value = diff + prev_value; - prev_value = curr_value; - - switch (type) { - case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)output + _pos) = (int64_t)curr_value; - _pos++; - break; - case TSDB_DATA_TYPE_INT: - *((int32_t *)output + _pos) = (int32_t)curr_value; - _pos++; - break; - case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)output + _pos) = (int16_t)curr_value; - _pos++; - break; - case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)output + _pos) = (int8_t)curr_value; - _pos++; - break; - default: - perror("Wrong integer types.\n"); - return -1; - } - count++; - if (count == nelements) break; - } -#endif - + // Optimize the performance, by remove the constantly switch operation. int32_t v = 0; uint64_t zigzag_value; @@ -844,7 +806,7 @@ int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, uint64_t prev_value = 0; for (int32_t i = 0; i < nelements; i++) { - if (i % 2 == 0) { + if ((i & 0x01) == 0) { flags = input[ipos++]; } From 98a2eb03f020db36b58b91589c95667321c9405b Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 30 Dec 2022 17:48:24 +0800 Subject: [PATCH 4/6] fix: index not updated issue --- source/libs/executor/src/dataInserter.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 376f4603f6..99d1968a5d 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -358,6 +358,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat if (inserter->fullOrderColList && pCol->colId != inserter->pSchema->columns[i].colId) { inserter->fullOrderColList = false; } + ++i; } tsem_init(&inserter->ready, 0, 0); From 28bccd21a90821420955b379148096175adae138 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 Jan 2023 09:47:01 +0800 Subject: [PATCH 5/6] refactor: disable all asserts. --- cmake/cmake.define | 10 ++++++++-- include/util/tlog.h | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index d32200bb91..fd7ef26e4d 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -117,12 +117,18 @@ ELSE () IF (${BUILD_SANITIZER}) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") - MESSAGE(STATUS "Will compile with Address Sanitizer!") + MESSAGE(STATUS "Compile with Address Sanitizer!") ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") ENDIF () + # disable all assert + IF (${DISABLE_ASSERT} OR ${DISABLE_ASSERTS}) + ADD_DEFINITIONS(-DDISABLE_ASSERT) + MESSAGE(STATUS "Disable all asserts") + ENDIF() + INCLUDE(CheckCCompilerFlag) IF (TD_ARM_64 OR TD_ARM_32) SET(COMPILER_SUPPORT_SSE42 false) @@ -155,7 +161,7 @@ ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2") ENDIF() - MESSAGE(STATUS "SIMD instructions (AVX/AVX2) is ACTIVATED") + MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED") ENDIF() ENDIF () diff --git a/include/util/tlog.h b/include/util/tlog.h index c7158def29..6e9b304e1d 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -85,12 +85,19 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons bool taosAssertDebug(bool condition, const char *file, int32_t line, const char *format, ...); bool taosAssertRelease(bool condition); + +// Disable all asserts that may compromise the performance. +#if defined DISABLE_ASSERT +#define ASSERT(condition) +#define ASSERTS(condition, ...) +#else #define ASSERTS(condition, ...) taosAssertDebug(condition, __FILE__, __LINE__, __VA_ARGS__) #ifdef NDEBUG #define ASSERT(condition) taosAssertRelease(condition) #else #define ASSERT(condition) taosAssertDebug(condition, __FILE__, __LINE__, "assert info not provided") #endif +#endif // clang-format off #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} From 879205eb9264d6151ba1a8b74486a43fb54314f5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 Jan 2023 09:53:37 +0800 Subject: [PATCH 6/6] fix(query): fix error in cmake. --- cmake/cmake.define | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index fd7ef26e4d..1b9a06d9e4 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -124,7 +124,7 @@ ELSE () ENDIF () # disable all assert - IF (${DISABLE_ASSERT} OR ${DISABLE_ASSERTS}) + IF ((${DISABLE_ASSERT} MATCHES "true") OR (${DISABLE_ASSERTS} MATCHES "true")) ADD_DEFINITIONS(-DDISABLE_ASSERT) MESSAGE(STATUS "Disable all asserts") ENDIF()