xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #24181 from taosdata/fix/TD-27967_TD-28004 

fix: cache scan heap overflow and forbid DDD for to_char in windows
This commit is contained in:
dapan1121 2023-12-22 16:54:46 +08:00 committed by GitHub
parent 67edc29a65 153221e1d6
commit 709b7f4683
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 33 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/ttime.h
View File

@ -105,7 +105,7 @@ int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision);
/// formats array; If not NULL, [formats] will be used instead of [format] to skip parse formats again.
/// @param out output buffer, should be initialized by memset
/// @notes remember to free the generated formats
void taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen);
int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen);
/// @brief convert a formatted timestamp string to a timestamp
/// @param format must null terminated
/// @param [in, out] formats, see taosTs2Char

1
include/util/taoserror.h
View File

@ -760,6 +760,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR TAOS_DEF_ERROR_CODE(0, 0x2806)
#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR TAOS_DEF_ERROR_CODE(0, 0x2807)
#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2808)
#define TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2809)
//udf
#define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901)

10
source/common/src/ttime.c
View File

@ -1297,7 +1297,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) {
}
}
static void tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) {
static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) {
int32_t size = taosArrayGetSize(formats);
const char* start = s;
for (int32_t i = 0; i < size; ++i) {
@ -1332,6 +1332,9 @@ static void tm2char(const SArray* formats, const struct STm* tm, char* s, int32_
s += 4;
break;
case TSFKW_DDD:
#ifdef WINDOWS
return TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED;
#endif
sprintf(s, "%03d", tm->tm.tm_yday + 1);
s += strlen(s);
break;
@ -1486,6 +1489,7 @@ static void tm2char(const SArray* formats, const struct STm* tm, char* s, int32_
break;
}
}
return TSDB_CODE_SUCCESS;
}
/// @brief find s in arr case insensitively
@ -1889,14 +1893,14 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec
return ret;
}
void taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen) {
int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen) {
if (!*formats) {
*formats = taosArrayInit(8, sizeof(TSFormatNode));
parseTsFormat(format, *formats);
}
struct STm tm;
taosTs2Tm(ts, precision, &tm);
tm2char(*formats, &tm, out, outLen);
return tm2char(*formats, &tm, out, outLen);
}
int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg,

21
source/libs/executor/src/cachescanoperator.c
View File

@ -54,10 +54,10 @@ static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColM
#define SCAN_ROW_TYPE(_t) ((_t) ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW)
static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SNodeList* pTargets) {
static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SLastRowScanPhysiNode* pScan) {
SNode* pNode;
int32_t idx = 0;
FOREACH(pNode, pTargets) {
FOREACH(pNode, pScan->pTargets) {
if (nodeType(pNode) == QUERY_NODE_COLUMN) {
SColumnNode* pCol = (SColumnNode*)pNode;
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, idx);
@ -65,6 +65,19 @@ static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SNodeList* pTargets)
}
idx++;
}
for (; idx < pBlock->pDataBlock->size; ++idx) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, idx);
if (pScan->scan.pScanPseudoCols) {
FOREACH(pNode, pScan->scan.pScanPseudoCols) {
STargetNode* pTarget = (STargetNode*)pNode;
if (pColInfo->info.slotId == pTarget->slotId) {
pColInfo->info.colId = 0;
break;
}
}
}
}
}
SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle,
@ -127,12 +140,12 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
capacity = TMIN(totalTables, 4096);
pInfo->pBufferredRes = createOneDataBlock(pInfo->pRes, false);
setColIdForCacheReadBlock(pInfo->pBufferredRes, pScanNode->pTargets);
setColIdForCacheReadBlock(pInfo->pBufferredRes, pScanNode);
blockDataEnsureCapacity(pInfo->pBufferredRes, capacity);
} else { // by tags
pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_SINGLE | SCAN_ROW_TYPE(pScanNode->ignoreNull);
capacity = 1; // only one row output
setColIdForCacheReadBlock(pInfo->pRes, pScanNode->pTargets);
setColIdForCacheReadBlock(pInfo->pRes, pScanNode);
}
initResultSizeInfo(&pOperator->resultInfo, capacity);

6
source/libs/scalar/src/sclfunc.c
View File

@ -1254,6 +1254,7 @@ int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOu
char * out = taosMemoryCalloc(1, TS_FORMAT_MAX_LEN + VARSTR_HEADER_SIZE);
int32_t len;
SArray *formats = NULL;
int32_t code = 0;
for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) {
colDataSetNULL(pOutput->columnData, i);
@ -1272,14 +1273,15 @@ int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOu
}
}
int32_t precision = pInput[0].columnData->info.precision;
taosTs2Char(format, &formats, *(int64_t *)ts, precision, varDataVal(out), TS_FORMAT_MAX_LEN);
code = taosTs2Char(format, &formats, *(int64_t *)ts, precision, varDataVal(out), TS_FORMAT_MAX_LEN);
if (code) break;
varDataSetLen(out, strlen(varDataVal(out)));
colDataSetVal(pOutput->columnData, i, out, false);
}
if (formats) taosArrayDestroy(formats);
taosMemoryFree(format);
taosMemoryFree(out);
return TSDB_CODE_SUCCESS;
return code;
}
/** Time functions **/

1
source/util/src/terror.c
View File

@ -622,6 +622,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP, "Duplicate timestamps
TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR, "Func to_timestamp failed for format mismatch")
TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR, "Func to_timestamp failed for wrong timestamp")
TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED, "Func to_timestamp failed for unsupported timestamp format")
TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED, "Func to_char failed for unsupported format")
//udf
TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping")

2
tests/system-test/2-query/last_cache_scan.py
View File

@ -284,6 +284,8 @@ class TDTestCase:
tdSql.checkData(0, 3, 1001)
tdSql.checkData(0, 4, "2018-11-25 19:30:00.000")
tdSql.query("select last(ts) from meters partition by tbname")
tdSql.query("select last(ts) from meters partition by t1")
sql_template = 'select %s from meters partition by tbname'
select_items = ["ts, last(c10), c10, ts", "ts, ts, last(c10), c10, tbname", "last(c10), c10, ts"]
has_last_row_scan_res = [1,1,1]