Merge pull request #24181 from taosdata/fix/TD-27967_TD-28004

fix: cache scan heap overflow and forbid DDD for to_char in windows

include/common/ttime.h

@@ -105,7 +105,7 @@ int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision);
 /// formats array; If not NULL, [formats] will be used instead of [format] to skip parse formats again.
 /// @param out output buffer, should be initialized by memset
 /// @notes remember to free the generated formats
-void taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen);
+int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen);
 /// @brief convert a formatted timestamp string to a timestamp
 /// @param format must null terminated
 /// @param [in, out] formats, see taosTs2Char

include/util/taoserror.h

@@ -760,6 +760,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR TAOS_DEF_ERROR_CODE(0, 0x2806)
 #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR TAOS_DEF_ERROR_CODE(0, 0x2807)
 #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2808)
+#define TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED    TAOS_DEF_ERROR_CODE(0, 0x2809)
 
 //udf
 #define TSDB_CODE_UDF_STOPPING                  TAOS_DEF_ERROR_CODE(0, 0x2901)

source/common/src/ttime.c

@@ -1297,7 +1297,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) {
   }
 }
 
-static void tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) {
+static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) {
   int32_t size = taosArrayGetSize(formats);
   const char* start = s;
   for (int32_t i = 0; i < size; ++i) {
@@ -1332,6 +1332,9 @@ static void tm2char(const SArray* formats, const struct STm* tm, char* s, int32_
         s += 4;
         break;
       case TSFKW_DDD:
+#ifdef WINDOWS
+        return TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED;
+#endif
         sprintf(s, "%03d", tm->tm.tm_yday + 1);
         s += strlen(s);
         break;
@@ -1486,6 +1489,7 @@ static void tm2char(const SArray* formats, const struct STm* tm, char* s, int32_
         break;
     }
   }
+  return TSDB_CODE_SUCCESS;
 }
 
 /// @brief find s in arr case insensitively
@@ -1889,14 +1893,14 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec
   return ret;
 }
 
-void taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen) {
+int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen) {
   if (!*formats) {
     *formats = taosArrayInit(8, sizeof(TSFormatNode));
     parseTsFormat(format, *formats);
   }
   struct STm tm;
   taosTs2Tm(ts, precision, &tm);
-  tm2char(*formats, &tm, out, outLen);
+  return tm2char(*formats, &tm, out, outLen);
 }
 
 int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg,

source/libs/executor/src/cachescanoperator.c

@@ -54,10 +54,10 @@ static int32_t      removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColM
 
 #define SCAN_ROW_TYPE(_t) ((_t) ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW)
 
-static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SNodeList* pTargets) {
+static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SLastRowScanPhysiNode* pScan) {
   SNode*  pNode;
   int32_t idx = 0;
-  FOREACH(pNode, pTargets) {
+  FOREACH(pNode, pScan->pTargets) {
     if (nodeType(pNode) == QUERY_NODE_COLUMN) {
       SColumnNode* pCol = (SColumnNode*)pNode;
       SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, idx);
@@ -65,6 +65,19 @@ static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SNodeList* pTargets)
     }
     idx++;
   }
+
+  for (; idx < pBlock->pDataBlock->size; ++idx) {
+    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, idx);
+    if (pScan->scan.pScanPseudoCols) {
+      FOREACH(pNode, pScan->scan.pScanPseudoCols) {
+        STargetNode* pTarget = (STargetNode*)pNode;
+        if (pColInfo->info.slotId == pTarget->slotId) {
+          pColInfo->info.colId = 0;
+          break;
+        }
+      }
+    }
+  }
 }
 
 SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle,
@@ -127,12 +140,12 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
     capacity = TMIN(totalTables, 4096);
 
     pInfo->pBufferredRes = createOneDataBlock(pInfo->pRes, false);
-    setColIdForCacheReadBlock(pInfo->pBufferredRes, pScanNode->pTargets);
+    setColIdForCacheReadBlock(pInfo->pBufferredRes, pScanNode);
     blockDataEnsureCapacity(pInfo->pBufferredRes, capacity);
   } else {  // by tags
     pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_SINGLE | SCAN_ROW_TYPE(pScanNode->ignoreNull);
     capacity = 1;  // only one row output
-    setColIdForCacheReadBlock(pInfo->pRes, pScanNode->pTargets);
+    setColIdForCacheReadBlock(pInfo->pRes, pScanNode);
   }
 
   initResultSizeInfo(&pOperator->resultInfo, capacity);

source/libs/scalar/src/sclfunc.c

@@ -1254,6 +1254,7 @@ int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOu
   char *  out = taosMemoryCalloc(1, TS_FORMAT_MAX_LEN + VARSTR_HEADER_SIZE);
   int32_t len;
   SArray *formats = NULL;
+  int32_t code = 0;
   for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
     if (colDataIsNull_s(pInput[1].columnData, i) || colDataIsNull_s(pInput[0].columnData, i)) {
       colDataSetNULL(pOutput->columnData, i);
@@ -1272,14 +1273,15 @@ int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOu
       }
     }
     int32_t precision = pInput[0].columnData->info.precision;
-    taosTs2Char(format, &formats, *(int64_t *)ts, precision, varDataVal(out), TS_FORMAT_MAX_LEN);
+    code = taosTs2Char(format, &formats, *(int64_t *)ts, precision, varDataVal(out), TS_FORMAT_MAX_LEN);
+    if (code) break;
     varDataSetLen(out, strlen(varDataVal(out)));
     colDataSetVal(pOutput->columnData, i, out, false);
   }
   if (formats) taosArrayDestroy(formats);
   taosMemoryFree(format);
   taosMemoryFree(out);
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 /** Time functions **/

source/util/src/terror.c

@@ -622,6 +622,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP,            "Duplicate timestamps
 TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR, "Func to_timestamp failed for format mismatch")
 TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR, "Func to_timestamp failed for wrong timestamp")
 TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED, "Func to_timestamp failed for unsupported timestamp format")
+TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED,    "Func to_char failed for unsupported format")
 
 //udf
 TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING,                   "udf is stopping")

tests/system-test/2-query/last_cache_scan.py

@@ -284,6 +284,8 @@ class TDTestCase:
         tdSql.checkData(0, 3, 1001)
         tdSql.checkData(0, 4, "2018-11-25 19:30:00.000")
 
+        tdSql.query("select last(ts) from meters partition by tbname")
+        tdSql.query("select last(ts) from meters partition by t1")
         sql_template = 'select %s from meters partition by tbname'
         select_items = ["ts, last(c10), c10, ts", "ts, ts, last(c10), c10, tbname", "last(c10), c10, ts"]
         has_last_row_scan_res = [1,1,1]