Merge pull request #19032 from taosdata/fix/TD-21001

enh: optimize converting timestamp at client side

include/libs/nodes/querynodes.h

@@ -127,8 +127,7 @@ typedef struct SLogicConditionNode {
 } SLogicConditionNode;
 
 typedef struct SNodeListNode {
-  ENodeType  type;  // QUERY_NODE_NODE_LIST
+  SExprNode  node;  // QUERY_NODE_NODE_LIST
-  SDataType  dataType;
   SNodeList* pNodeList;
 } SNodeListNode;
 

source/libs/index/src/indexFilter.c

@@ -255,7 +255,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
         indexError("invalid length for node:%p, length: %d", node, LIST_LENGTH(nl->pNodeList));
         SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
       }
-      SIF_ERR_RET(scalarGenerateSetFromList((void **)&param->pFilter, node, nl->dataType.type));
+      SIF_ERR_RET(scalarGenerateSetFromList((void **)&param->pFilter, node, nl->node.resType.type));
       if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) {
         taosHashCleanup(param->pFilter);
         indexError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param));

source/libs/index/test/index_executor_tests.cpp

@@ -139,7 +139,7 @@ void sifMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *
 void sifMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) {
   SNode         *node = (SNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
   SNodeListNode *lnode = (SNodeListNode *)node;
-  lnode->dataType.type = resType;
+  lnode->node.resType.type = resType;
   lnode->pNodeList = list;
 
   *pNode = (SNode *)lnode;

source/libs/nodes/src/nodesCloneFuncs.c

@@ -311,7 +311,7 @@ static int32_t intervalWindowNodeCopy(const SIntervalWindowNode* pSrc, SInterval
 }
 
 static int32_t nodeListNodeCopy(const SNodeListNode* pSrc, SNodeListNode* pDst) {
-  COPY_OBJECT_FIELD(dataType, sizeof(SDataType));
+  COPY_OBJECT_FIELD(node.resType, sizeof(SDataType));
   CLONE_NODE_LIST_FIELD(pNodeList);
   return TSDB_CODE_SUCCESS;
 }

source/libs/nodes/src/nodesCodeFuncs.c

@@ -3712,7 +3712,7 @@ static const char* jkNodeListNodeList = "NodeList";
 static int32_t nodeListNodeToJson(const void* pObj, SJson* pJson) {
   const SNodeListNode* pNode = (const SNodeListNode*)pObj;
 
-  int32_t code = tjsonAddObject(pJson, jkNodeListDataType, dataTypeToJson, &pNode->dataType);
+  int32_t code = tjsonAddObject(pJson, jkNodeListDataType, dataTypeToJson, &pNode->node.resType);
   if (TSDB_CODE_SUCCESS == code) {
     code = nodeListToJson(pJson, jkNodeListNodeList, pNode->pNodeList);
   }
@@ -3723,7 +3723,7 @@ static int32_t nodeListNodeToJson(const void* pObj, SJson* pJson) {
 static int32_t jsonToNodeListNode(const SJson* pJson, void* pObj) {
   SNodeListNode* pNode = (SNodeListNode*)pObj;
 
-  int32_t code = tjsonToObject(pJson, jkNodeListDataType, jsonToDataType, &pNode->dataType);
+  int32_t code = tjsonToObject(pJson, jkNodeListDataType, jsonToDataType, &pNode->node.resType);
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeList(pJson, jkNodeListNodeList, &pNode->pNodeList);
   }

source/libs/nodes/src/nodesMsgFuncs.c

@@ -1298,7 +1298,7 @@ enum { NODE_LIST_CODE_DATA_TYPE = 1, NODE_LIST_CODE_NODE_LIST };
 static int32_t nodeListNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
   const SNodeListNode* pNode = (const SNodeListNode*)pObj;
 
-  int32_t code = tlvEncodeObj(pEncoder, NODE_LIST_CODE_DATA_TYPE, dataTypeInlineToMsg, &pNode->dataType);
+  int32_t code = tlvEncodeObj(pEncoder, NODE_LIST_CODE_DATA_TYPE, dataTypeInlineToMsg, &pNode->node.resType);
   if (TSDB_CODE_SUCCESS == code) {
     code = tlvEncodeObj(pEncoder, NODE_LIST_CODE_NODE_LIST, nodeListToMsg, pNode->pNodeList);
   }
@@ -1314,7 +1314,7 @@ static int32_t msgToNodeListNode(STlvDecoder* pDecoder, void* pObj) {
   tlvForEach(pDecoder, pTlv, code) {
     switch (pTlv->type) {
       case NODE_LIST_CODE_DATA_TYPE:
-        code = tlvDecodeObjFromTlv(pTlv, msgToDataTypeInline, &pNode->dataType);
+        code = tlvDecodeObjFromTlv(pTlv, msgToDataTypeInline, &pNode->node.resType);
         break;
       case NODE_LIST_CODE_NODE_LIST:
         code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pNodeList);

source/libs/scalar/inc/sclInt.h

@@ -40,8 +40,11 @@ typedef struct SScalarCtx {
 #define SCL_DATA_TYPE_DUMMY_HASH 9000
 #define SCL_DEFAULT_OP_NUM       10
 
+#define SCL_IS_NOTNULL_CONST_NODE(_node) ((QUERY_NODE_VALUE == (_node)->type) || (QUERY_NODE_NODE_LIST == (_node)->type))
 #define SCL_IS_CONST_NODE(_node) \
-  ((NULL == (_node)) || (QUERY_NODE_VALUE == (_node)->type) || (QUERY_NODE_NODE_LIST == (_node)->type))
+  ((NULL == (_node)) || SCL_IS_NOTNULL_CONST_NODE(_node))
+#define SCL_IS_VAR_VALUE_NODE(_node) ((QUERY_NODE_VALUE == (_node)->type) && IS_STR_DATA_TYPE(((SValueNode*)(_node))->node.resType.type))
+
 #define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList)
 //#define SCL_IS_NULL_VALUE_NODE(_node) ((QUERY_NODE_VALUE == nodeType(_node)) && (TSDB_DATA_TYPE_NULL == ((SValueNode
 //*)_node)->node.resType.type) && (((SValueNode *)_node)->placeholderNo <= 0))

source/libs/scalar/src/filter.c

@@ -3762,6 +3762,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) {
       return DEAL_RES_CONTINUE;
     }
 
+/*
     if (!FILTER_GET_FLAG(stat->info->options, FLT_OPTION_TIMESTAMP)) {
       return DEAL_RES_CONTINUE;
     }
@@ -3785,7 +3786,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) {
       stat->code = code;
       return DEAL_RES_ERROR;
     }
-
+*/
     return DEAL_RES_CONTINUE;
   }
 
@@ -3931,7 +3932,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) {
           stat->scalarMode = true;
           return DEAL_RES_CONTINUE;
         }
-        int32_t        type = vectorGetConvertType(refNode->node.resType.type, listNode->dataType.type);
+        int32_t        type = vectorGetConvertType(refNode->node.resType.type, listNode->node.resType.type);
         if (0 != type && type != refNode->node.resType.type) {
           stat->scalarMode = true;
           return DEAL_RES_CONTINUE;
@@ -3955,12 +3956,14 @@ int32_t fltReviseNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) {
 
   FLT_ERR_JRET(pStat->code);
 
+/*
   int32_t nodeNum = taosArrayGetSize(pStat->nodeList);
   for (int32_t i = 0; i < nodeNum; ++i) {
     SValueNode *valueNode = *(SValueNode **)taosArrayGet(pStat->nodeList, i);
 
     FLT_ERR_JRET(sclConvertToTsValueNode(pStat->precision, valueNode));
   }
+*/
 
 _return:
 

source/libs/scalar/src/scalar.c

@@ -349,7 +349,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t
 
       int32_t type = vectorGetConvertType(ctx->type.selfType, ctx->type.peerType);
       if (type == 0) {
-        type = nodeList->dataType.type;
+        type = nodeList->node.resType.type;
       }
 
       SCL_ERR_RET(scalarGenerateSetFromList((void **)&param->pHashFilter, node, type));
@@ -507,7 +507,7 @@ int32_t sclGetNodeType(SNode *pNode, SScalarCtx *ctx) {
     }
     case QUERY_NODE_NODE_LIST: {
       SNodeListNode *nodeList = (SNodeListNode *)pNode;
-      return nodeList->dataType.type;
+      return nodeList->node.resType.type;
     }
     case QUERY_NODE_COLUMN: {
       SColumnNode *colNode = (SColumnNode *)pNode;
@@ -1029,6 +1029,72 @@ bool sclContainsAggFuncNode(SNode *pNode) {
   return aggFunc;
 }
 
+
+int32_t sclConvertOpValueNodeTs(SOperatorNode *node, SScalarCtx *ctx) {
+  int32_t code = 0;
+  
+  if (node->pLeft && SCL_IS_VAR_VALUE_NODE(node->pLeft)) {
+    if (node->pRight && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pRight)->resType.type)) {
+      SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pRight)->resType.precision, (SValueNode*)node->pLeft));
+    }
+  } else if (node->pRight && SCL_IS_NOTNULL_CONST_NODE(node->pRight)) {
+    if (node->pLeft && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pLeft)->resType.type)) {
+      if (SCL_IS_VAR_VALUE_NODE(node->pRight)) {
+        SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pLeft)->resType.precision, (SValueNode*)node->pRight));
+      } else if (QUERY_NODE_NODE_LIST == node->pRight->type) {
+        SNode* pNode;
+        FOREACH(pNode, ((SNodeListNode*)node->pRight)->pNodeList) {
+          if (SCL_IS_VAR_VALUE_NODE(pNode)) {
+            SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pLeft)->resType.precision, (SValueNode*)pNode));
+          }
+        }
+      }
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  ctx->code = code;
+  return DEAL_RES_ERROR;
+}
+
+
+
+int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) {
+  int32_t code = 0;
+
+  if (NULL == node->pCase) {
+    return TSDB_CODE_SUCCESS;
+  }
+  
+  if (SCL_IS_VAR_VALUE_NODE(node->pCase)) {
+    SNode* pNode;
+    FOREACH(pNode, node->pWhenThenList) {
+      SExprNode *pExpr = (SExprNode *)((SWhenThenNode *)pNode)->pWhen;
+      if (TSDB_DATA_TYPE_TIMESTAMP == pExpr->resType.type) {
+        SCL_ERR_JRET(sclConvertToTsValueNode(pExpr->resType.precision, (SValueNode*)node->pCase));
+        break;
+      }
+    }
+  } else if (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pCase)->resType.type) {
+    SNode* pNode;
+    FOREACH(pNode, node->pWhenThenList) {
+      if (SCL_IS_VAR_VALUE_NODE(((SWhenThenNode *)pNode)->pWhen)) {
+        SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pCase)->resType.precision, (SValueNode*)((SWhenThenNode *)pNode)->pWhen));
+      }
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  ctx->code = code;
+  return DEAL_RES_ERROR;
+}
+
 EDealRes sclRewriteNonConstOperator(SNode **pNode, SScalarCtx *ctx) {
   SOperatorNode *node = (SOperatorNode *)*pNode;
   int32_t        code = 0;
@@ -1040,15 +1106,6 @@ EDealRes sclRewriteNonConstOperator(SNode **pNode, SScalarCtx *ctx) {
       return sclRewriteNullInOptr(pNode, ctx, node->opType);
     }
 
-    if (IS_STR_DATA_TYPE(valueNode->node.resType.type) && node->pRight && nodesIsExprNode(node->pRight) &&
-        ((SExprNode *)node->pRight)->resType.type == TSDB_DATA_TYPE_TIMESTAMP) {
-      code = sclConvertToTsValueNode(((SExprNode *)node->pRight)->resType.precision, valueNode);
-      if (code) {
-        ctx->code = code;
-        return DEAL_RES_ERROR;
-      }
-    }
-
     if (SCL_IS_COMPARISON_OPERATOR(node->opType) && SCL_DOWNGRADE_DATETYPE(valueNode->node.resType.type)) {
       sclDowngradeValueType(valueNode);
     }
@@ -1061,15 +1118,6 @@ EDealRes sclRewriteNonConstOperator(SNode **pNode, SScalarCtx *ctx) {
       return sclRewriteNullInOptr(pNode, ctx, node->opType);
     }
 
-    if (IS_STR_DATA_TYPE(valueNode->node.resType.type) && node->pLeft && nodesIsExprNode(node->pLeft) &&
-        ((SExprNode *)node->pLeft)->resType.type == TSDB_DATA_TYPE_TIMESTAMP) {
-      code = sclConvertToTsValueNode(((SExprNode *)node->pLeft)->resType.precision, valueNode);
-      if (code) {
-        ctx->code = code;
-        return DEAL_RES_ERROR;
-      }
-    }
-
     if (SCL_IS_COMPARISON_OPERATOR(node->opType) && SCL_DOWNGRADE_DATETYPE(valueNode->node.resType.type)) {
       sclDowngradeValueType(valueNode);
     }
@@ -1197,9 +1245,12 @@ EDealRes sclRewriteLogic(SNode **pNode, SScalarCtx *ctx) {
   return DEAL_RES_CONTINUE;
 }
 
+
 EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) {
   SOperatorNode *node = (SOperatorNode *)*pNode;
 
+  SCL_ERR_RET(sclConvertOpValueNodeTs(node, ctx));
+
   if ((!SCL_IS_CONST_NODE(node->pLeft)) || (!SCL_IS_CONST_NODE(node->pRight))) {
     return sclRewriteNonConstOperator(pNode, ctx);
   }
@@ -1245,6 +1296,8 @@ EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) {
 EDealRes sclRewriteCaseWhen(SNode **pNode, SScalarCtx *ctx) {
   SCaseWhenNode *node = (SCaseWhenNode *)*pNode;
 
+  SCL_ERR_RET(sclConvertCaseWhenValueNodeTs(node, ctx));
+
   if ((!SCL_IS_CONST_NODE(node->pCase)) || (!SCL_IS_CONST_NODE(node->pElse))) {
     return DEAL_RES_CONTINUE;
   }

source/libs/scalar/test/filter/filterTests.cpp

@@ -193,7 +193,7 @@ void flttMakeLogicNodeFromList(SNode **pNode, ELogicConditionType opType, SNodeL
 void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) {
   SNode         *node = (SNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
   SNodeListNode *lnode = (SNodeListNode *)node;
-  lnode->dataType.type = resType;
+  lnode->node.resType.type = resType;
   lnode->pNodeList = list;
 
   *pNode = (SNode *)lnode;

source/libs/scalar/test/scalar/scalarTests.cpp

@@ -233,7 +233,7 @@ void scltMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode
 void scltMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) {
   SNode         *node = (SNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
   SNodeListNode *lnode = (SNodeListNode *)node;
-  lnode->dataType.type = resType;
+  lnode->node.resType.type = resType;
   lnode->pNodeList = list;
 
   *pNode = (SNode *)lnode;

tests/parallel_test/cases.task

@@ -313,6 +313,7 @@
 ,,y,script,./test.sh -f tsim/scalar/scalar.sim
 ,,y,script,./test.sh -f tsim/scalar/filter.sim 
 ,,y,script,./test.sh -f tsim/scalar/caseWhen.sim 
+,,y,script,./test.sh -f tsim/scalar/tsConvert.sim 
 ,,y,script,./test.sh -f tsim/alter/cached_schema_after_alter.sim
 ,,y,script,./test.sh -f tsim/alter/dnode.sim
 ,,y,script,./test.sh -f tsim/alter/table.sim

tests/script/tsim/scalar/tsConvert.sim

@@ -0,0 +1,127 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+print ======== step1
+sql drop database if exists db1;
+sql create database db1;
+sql use db1;
+sql create stable st1 (ts timestamp, f1 int, f2 binary(30)) tags(t1 int, t2 binary(30));
+sql create table tb1 using st1 tags(1, '1');
+sql insert into tb1 values ('2022-07-10 16:31:00', 1, '1');
+sql insert into tb1 values ('2022-07-10 16:32:00', 2, '2');
+sql insert into tb1 values ('2022-07-10 16:33:00', 3, '3');
+sql insert into tb1 values ('2022-07-10 16:34:00', 4, '4');
+sql select * from (select ts,TIMETRUNCATE(ts,1d),TIMETRUNCATE(ts,1h),timediff(TIMETRUNCATE(ts,1d),TIMETRUNCATE(ts,1h),1h) as td from tb1 where ts >='2022-06-01 00:00:00' and ts <=' 2022-11-3 23:59:59' ) t where td >12;
+if $rows != 4 then
+  return -1
+endi
+if $data00 != @22-07-10 16:31:00.000@ then
+  return -1
+endi
+if $data10 != @22-07-10 16:32:00.000@ then
+  return -1
+endi
+if $data20 != @22-07-10 16:33:00.000@ then
+  return -1
+endi
+if $data30 != @22-07-10 16:34:00.000@ then
+  return -1
+endi
+
+sql select * from tb1 where ts > '2022-07-10 16:32:00';
+if $rows != 2 then
+  return -1
+endi
+if $data00 != @22-07-10 16:33:00.000@ then
+  return -1
+endi
+if $data10 != @22-07-10 16:34:00.000@ then
+  return -1
+endi
+
+sql select * from tb1 where ts + 1 > '2022-07-10 16:32:00';
+if $rows != 3 then
+  return -1
+endi
+if $data00 != @22-07-10 16:32:00.000@ then
+  return -1
+endi
+if $data10 != @22-07-10 16:33:00.000@ then
+  return -1
+endi
+if $data20 != @22-07-10 16:34:00.000@ then
+  return -1
+endi
+
+sql select * from tb1 where '2022-07-10 16:32:00' > timestamp '2022-07-10 16:31:59';
+if $rows != 4 then
+  return -1
+endi
+if $data00 != @22-07-10 16:31:00.000@ then
+  return -1
+endi
+if $data10 != @22-07-10 16:32:00.000@ then
+  return -1
+endi
+if $data20 != @22-07-10 16:33:00.000@ then
+  return -1
+endi
+if $data30 != @22-07-10 16:34:00.000@ then
+  return -1
+endi
+
+sql select case f1 when '1' then 1 when '2022-07-10 16:32:00' then '2' end from tb1;
+if $rows != 4 then
+  return -1
+endi
+if $data00 != 1 then
+  return -1
+endi
+if $data10 != NULL then
+  return -1
+endi
+if $data20 != NULL then
+  return -1
+endi
+if $data30 != NULL then
+  return -1
+endi
+
+sql select case ts when '2022-07-10 16:31:00' then 1 when '2022-07-10 16:32:00' then '2' end from tb1;
+if $rows != 4 then
+  return -1
+endi
+if $data00 != 1 then
+  return -1
+endi
+if $data10 != 2 then
+  return -1
+endi
+if $data20 != NULL then
+  return -1
+endi
+if $data30 != NULL then
+  return -1
+endi
+
+sql select case '2022-07-10 16:31:00' when ts then 1 when 2022 then '2' else 3 end from tb1;
+if $rows != 4 then
+  return -1
+endi
+if $data00 != 1 then
+  return -1
+endi
+if $data10 != 3 then
+  return -1
+endi
+if $data20 != 3 then
+  return -1
+endi
+if $data30 != 3 then
+  return -1
+endi
+
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT