From 153221e1d607aa59733f15a561e8b466fa24e836 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Thu, 21 Dec 2023 19:46:33 +0800 Subject: [PATCH] fix: cache scan heap overflow and forbid DDD for to_char in windows --- include/common/ttime.h | 2 +- include/util/taoserror.h | 1 + source/common/src/ttime.c | 10 +++++++--- source/libs/executor/src/cachescanoperator.c | 21 ++++++++++++++++---- source/libs/scalar/src/sclfunc.c | 6 ++++-- source/util/src/terror.c | 1 + tests/system-test/2-query/last_cache_scan.py | 2 ++ 7 files changed, 33 insertions(+), 10 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index 1dfa609064..ed4d1a9290 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -105,7 +105,7 @@ int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision); /// formats array; If not NULL, [formats] will be used instead of [format] to skip parse formats again. /// @param out output buffer, should be initialized by memset /// @notes remember to free the generated formats -void taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen); +int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen); /// @brief convert a formatted timestamp string to a timestamp /// @param format must null terminated /// @param [in, out] formats, see taosTs2Char diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 06e6b8d041..b49eb916f0 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -760,6 +760,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR TAOS_DEF_ERROR_CODE(0, 0x2806) #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR TAOS_DEF_ERROR_CODE(0, 0x2807) #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2808) +#define TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2809) //udf #define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index ec00fce4dd..023a425df2 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -1297,7 +1297,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) { } } -static void tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) { +static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) { int32_t size = taosArrayGetSize(formats); const char* start = s; for (int32_t i = 0; i < size; ++i) { @@ -1332,6 +1332,9 @@ static void tm2char(const SArray* formats, const struct STm* tm, char* s, int32_ s += 4; break; case TSFKW_DDD: +#ifdef WINDOWS + return TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED; +#endif sprintf(s, "%03d", tm->tm.tm_yday + 1); s += strlen(s); break; @@ -1486,6 +1489,7 @@ static void tm2char(const SArray* formats, const struct STm* tm, char* s, int32_ break; } } + return TSDB_CODE_SUCCESS; } /// @brief find s in arr case insensitively @@ -1889,14 +1893,14 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec return ret; } -void taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen) { +int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen) { if (!*formats) { *formats = taosArrayInit(8, sizeof(TSFormatNode)); parseTsFormat(format, *formats); } struct STm tm; taosTs2Tm(ts, precision, &tm); - tm2char(*formats, &tm, out, outLen); + return tm2char(*formats, &tm, out, outLen); } int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg, diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 75fe3c51dd..9d4c20493a 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -54,10 +54,10 @@ static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColM #define SCAN_ROW_TYPE(_t) ((_t) ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW) -static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SNodeList* pTargets) { +static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SLastRowScanPhysiNode* pScan) { SNode* pNode; int32_t idx = 0; - FOREACH(pNode, pTargets) { + FOREACH(pNode, pScan->pTargets) { if (nodeType(pNode) == QUERY_NODE_COLUMN) { SColumnNode* pCol = (SColumnNode*)pNode; SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, idx); @@ -65,6 +65,19 @@ static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SNodeList* pTargets) } idx++; } + + for (; idx < pBlock->pDataBlock->size; ++idx) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, idx); + if (pScan->scan.pScanPseudoCols) { + FOREACH(pNode, pScan->scan.pScanPseudoCols) { + STargetNode* pTarget = (STargetNode*)pNode; + if (pColInfo->info.slotId == pTarget->slotId) { + pColInfo->info.colId = 0; + break; + } + } + } + } } SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, @@ -127,12 +140,12 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe capacity = TMIN(totalTables, 4096); pInfo->pBufferredRes = createOneDataBlock(pInfo->pRes, false); - setColIdForCacheReadBlock(pInfo->pBufferredRes, pScanNode->pTargets); + setColIdForCacheReadBlock(pInfo->pBufferredRes, pScanNode); blockDataEnsureCapacity(pInfo->pBufferredRes, capacity); } else { // by tags pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_SINGLE | SCAN_ROW_TYPE(pScanNode->ignoreNull); capacity = 1; // only one row output - setColIdForCacheReadBlock(pInfo->pRes, pScanNode->pTargets); + setColIdForCacheReadBlock(pInfo->pRes, pScanNode); } initResultSizeInfo(&pOperator->resultInfo, capacity); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 734d1e7d17..e713043b80 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1254,6 +1254,7 @@ int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOu char * out = taosMemoryCalloc(1, TS_FORMAT_MAX_LEN + VARSTR_HEADER_SIZE); int32_t len; SArray *formats = NULL; + int32_t code = 0; for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) { colDataSetNULL(pOutput->columnData, i); @@ -1272,14 +1273,15 @@ int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOu } } int32_t precision = pInput[0].columnData->info.precision; - taosTs2Char(format, &formats, *(int64_t *)ts, precision, varDataVal(out), TS_FORMAT_MAX_LEN); + code = taosTs2Char(format, &formats, *(int64_t *)ts, precision, varDataVal(out), TS_FORMAT_MAX_LEN); + if (code) break; varDataSetLen(out, strlen(varDataVal(out))); colDataSetVal(pOutput->columnData, i, out, false); } if (formats) taosArrayDestroy(formats); taosMemoryFree(format); taosMemoryFree(out); - return TSDB_CODE_SUCCESS; + return code; } /** Time functions **/ diff --git a/source/util/src/terror.c b/source/util/src/terror.c index dc5f44cf43..6a32fed147 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -622,6 +622,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP, "Duplicate timestamps TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR, "Func to_timestamp failed for format mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR, "Func to_timestamp failed for wrong timestamp") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED, "Func to_timestamp failed for unsupported timestamp format") +TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED, "Func to_char failed for unsupported format") //udf TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 01795f6eef..2ef2e50a88 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -284,6 +284,8 @@ class TDTestCase: tdSql.checkData(0, 3, 1001) tdSql.checkData(0, 4, "2018-11-25 19:30:00.000") + tdSql.query("select last(ts) from meters partition by tbname") + tdSql.query("select last(ts) from meters partition by t1") sql_template = 'select %s from meters partition by tbname' select_items = ["ts, last(c10), c10, ts", "ts, ts, last(c10), c10, tbname", "last(c10), c10, ts"] has_last_row_scan_res = [1,1,1]